sfantao created this revision.
sfantao added reviewers: echristo, tra, jlebar, hfinkel, ABataev.
sfantao added subscribers: caomhin, carlo.bertolli, arpith-jacob, Hahnfeld,
andreybokhanko, cfe-commits.
Herald added a subscriber: mehdi_amini.
This patch adds the capability to bundle object files in sections of the host
binary using a designated naming convention for these sections. This patch uses
the functionality of the object reader already in the LLVM library to read
bundled files, and invokes clang with the incremental linking options to create
bundle files.
Bundling files involves creating an IR file with the contents of the bundle
assigned as initializers of globals binded to the designated sections. This way
the bundling implementation is agnostic of the host object format.
The features added by this patch were requested in the RFC discussion in
http://lists.llvm.org/pipermail/cfe-dev/2016-February/047547.html.
http://reviews.llvm.org/D21851
Files:
test/Driver/clang-offload-bundler.c
tools/clang-offload-bundler/ClangOffloadBundler.cpp
Index: tools/clang-offload-bundler/ClangOffloadBundler.cpp
===================================================================
--- tools/clang-offload-bundler/ClangOffloadBundler.cpp
+++ tools/clang-offload-bundler/ClangOffloadBundler.cpp
@@ -75,14 +75,23 @@
/// \brief Magic string that marks the existence of offloading data.
#define OFFLOAD_BUNDLER_MAGIC_STR "__CLANG_OFFLOAD_BUNDLE__"
+/// \brief Path to the current binary.
+static std::string BundlerExecutable;
+
/// \brief Obtain the offload kind and real machine triple out of the target
/// information specified by the user.
static void getOffloadKindAndTriple(StringRef Target, StringRef &OffloadKind,
StringRef &Triple) {
auto KindTriplePair = Target.split('-');
OffloadKind = KindTriplePair.first;
Triple = KindTriplePair.second;
}
+static StringRef getTriple(StringRef Target) {
+ StringRef OffloadKind;
+ StringRef Triple;
+ getOffloadKindAndTriple(Target, OffloadKind, Triple);
+ return Triple;
+}
static bool hasHostKind(StringRef Target) {
StringRef OffloadKind;
StringRef Triple;
@@ -312,6 +321,211 @@
~BinaryFileHandler() {}
};
+// Handler for object files. The bundles are organized by sections with a
+// designated name.
+//
+// In order to bundle we create an IR file with the content of each section and
+// use incremental linking to produce the resulting object. We also add section
+// with a single byte to state the name of the component the main object file
+// (the one we are bundling into) refers to.
+//
+// To unbundle, we use just copy the contents of the designated section. If the
+// requested bundle refer to the main object file, we just copy it with no
+// changes.
+class ObjectFileHandler : public FileHandler {
+
+ /// \brief The object file we are currently dealing with.
+ ObjectFile &Obj;
+
+ /// \brief Return the input file contents.
+ StringRef getInputFileContents() const { return Obj.getData(); }
+
+ /// \brief Return true if the provided section is an offload section and
+ /// return the triple by reference.
+ bool isOffloadSection(SectionRef CurSection, StringRef &OffloadTriple) {
+ StringRef SectionName;
+ CurSection.getName(SectionName);
+
+ if (SectionName.empty())
+ return false;
+
+ // If it does not start with the reserved suffix, just skip this section.
+ if (!SectionName.startswith(OFFLOAD_BUNDLER_MAGIC_STR))
+ return false;
+
+ // Return the triple that is right after the reserved prefix.
+ OffloadTriple = SectionName.substr(sizeof(OFFLOAD_BUNDLER_MAGIC_STR) - 1);
+ return true;
+ }
+
+ /// \brief Total number of inputs.
+ unsigned NumberOfInputs = 0;
+
+ /// \brief Total number of processed inputs, i.e, inputs that were already
+ /// read from the buffers.
+ unsigned NumberOfProcessedInputs = 0;
+
+ /// \brief LLVM context used to to create the auxiliar modules.
+ LLVMContext VMContext;
+
+ /// \brief LLVM module used to create an object with all the bundle
+ /// components.
+ std::unique_ptr<Module> AuxModule;
+
+ /// \brief The current triple we are working with.
+ StringRef CurrentTriple;
+
+ /// \brief The name of the main input file.
+ StringRef MainInputFileName;
+
+ /// \brief Iterator of the current and next section.
+ section_iterator CurrentSection;
+ section_iterator NextSection;
+
+public:
+ void ReadHeader(MemoryBuffer &Input) {}
+ StringRef ReadBundleStart(MemoryBuffer &Input) {
+
+ while (NextSection != Obj.section_end()) {
+ CurrentSection = NextSection;
+ ++NextSection;
+
+ StringRef OffloadTriple;
+ // Check if the current section name starts with the reserved prefix. If
+ // so, return the triple.
+ if (isOffloadSection(*CurrentSection, OffloadTriple))
+ return OffloadTriple;
+ }
+ return StringRef();
+ }
+ void ReadBundleEnd(MemoryBuffer &Input) {}
+ void ReadBundle(raw_fd_ostream &OS, MemoryBuffer &Input) {
+ // If the current section has size one, that means that the content we are
+ // interested in is the file itself. Otherwise it is the content of the
+ // section.
+ //
+ // TODO: Instead of copying the input file as is, deactivate the section
+ // that are no longer needed.
+
+ StringRef Content;
+ CurrentSection->getContents(Content);
+
+ if (Content.size() < 2)
+ OS.write(Input.getBufferStart(), Input.getBufferSize());
+ else
+ OS.write(Content.data(), Content.size());
+
+ return;
+ }
+
+ void WriteHeader(raw_fd_ostream &OS,
+ ArrayRef<std::unique_ptr<MemoryBuffer>> Inputs) {
+ // Record number of inputs.
+ NumberOfInputs = Inputs.size();
+
+ // Create an LLVM module to have the content we need to bundle.
+ auto *M = new Module("clang-offload-bundle", VMContext);
+ M->setTargetTriple(getTriple(TargetNames.front()));
+ AuxModule.reset(M);
+ }
+ void WriteBundleStart(raw_fd_ostream &OS, StringRef TargetTriple) {
+ ++NumberOfProcessedInputs;
+
+ // Record the triple we are using, that will be used to name the section we
+ // will create.
+ CurrentTriple = TargetTriple;
+ }
+ void WriteBundleEnd(raw_fd_ostream &OS, StringRef TargetTriple) {
+ assert(NumberOfProcessedInputs <= NumberOfInputs &&
+ "Processing more inputs that actually exist!");
+
+ // If this is not the last output, we don't have to do anything.
+ if (NumberOfProcessedInputs != NumberOfInputs)
+ return;
+
+ // Create the bitcode file name to write the resulting code to.
+ SmallString<128> BitcodeFileName;
+ if (sys::fs::createTemporaryFile("clang-offload-bundler", "bc",
+ BitcodeFileName))
+ llvm_unreachable("Error trying to create temporary file!");
+
+ // Write the bitcode to the temporary file.
+ {
+ std::error_code EC;
+ raw_fd_ostream BitcodeFile(BitcodeFileName, EC, sys::fs::F_None);
+ if (EC)
+ llvm_unreachable("Error trying to open temporary file!");
+ WriteBitcodeToFile(AuxModule.get(), BitcodeFile);
+ }
+
+ // Find clang in order to create the bundle binary.
+ StringRef Dir = llvm::sys::path::parent_path(BundlerExecutable);
+
+ auto ClangBinary = sys::findProgramByName("clang", Dir);
+ if (ClangBinary.getError()) {
+ // Remove bitcode file.
+ sys::fs::remove(BitcodeFileName);
+ llvm_unreachable("Can't find clang in path!");
+ }
+
+ // Do the incremental linking. We write to the output file directly. So, we
+ // close it and use the name to pass down to clang.
+ OS.close();
+ SmallString<128> TargetName = getTriple(TargetNames.front());
+ const char *ClangArgs[] = {"clang",
+ "-r",
+ "-target",
+ TargetName.c_str(),
+ "-o",
+ OutputFileNames.front().c_str(),
+ InputFileNames.front().c_str(),
+ BitcodeFileName.c_str(),
+ "-nostdlib",
+ nullptr};
+
+ if (sys::ExecuteAndWait(ClangBinary.get(), ClangArgs)) {
+ // Remove bitcode file.
+ sys::fs::remove(BitcodeFileName);
+ llvm_unreachable("Bundle incremental link failed!");
+ }
+
+ // Remove bitcode file.
+ sys::fs::remove(BitcodeFileName);
+ }
+ void WriteBundle(raw_fd_ostream &OS, MemoryBuffer &Input) {
+ Module *M = AuxModule.get();
+
+ // Create the new section name, it will consist of the reserved prefix
+ // concatenated with the triple.
+ std::string SectionName = OFFLOAD_BUNDLER_MAGIC_STR;
+ SectionName += CurrentTriple;
+
+ // Create the constant with the content of the section. For the input we are
+ // bundling into (the first input), this is just a place-holder, so a single
+ // byte is sufficient.
+ Constant *Content;
+ if (NumberOfProcessedInputs == 1) {
+ uint8_t Byte[] = {0};
+ Content = ConstantDataArray::get(VMContext, Byte);
+ } else
+ Content = ConstantDataArray::get(
+ VMContext, ArrayRef<uint8_t>(reinterpret_cast<const uint8_t *>(
+ Input.getBufferStart()),
+ Input.getBufferSize()));
+
+ // Create the global in the desired section. We don't want these globals in
+ // the symbol table, so we mark them private.
+ auto *GV = new GlobalVariable(*M, Content->getType(), /*IsConstant=*/true,
+ GlobalVariable::PrivateLinkage, Content);
+ GV->setSection(SectionName);
+ }
+
+ ObjectFileHandler(ObjectFile &Obj)
+ : FileHandler(), Obj(Obj), CurrentSection(Obj.section_begin()),
+ NextSection(Obj.section_begin()) {}
+ ~ObjectFileHandler() {}
+};
+
// Handler for text files. The bundled file will have the following format.
//
// "Comment OFFLOAD_BUNDLER_MAGIC_STR__START__ triple"
@@ -409,6 +623,28 @@
}
};
+/// \brief Return an appropriate object file handler. We use the specific object
+/// handler if we know how to deal with that format, otherwise we use a default
+/// binary file handler.
+static FileHandler *CreateObjectFileHandler(MemoryBuffer &FirstInput) {
+ // Check if the input file format is one that we know how to deal with.
+ Expected<std::unique_ptr<Binary>> BinaryOrErr = createBinary(FirstInput);
+
+ // Failed to open the input as a known binary. Use the default binary handler.
+ if (!BinaryOrErr)
+ return new BinaryFileHandler();
+
+ // We only support regular object files. If this is not an object file,
+ // default to the binary handler. The handler will be owned by the client of
+ // this function.
+ ObjectFile *Obj = dyn_cast<ObjectFile>(BinaryOrErr.get().release());
+
+ if (!Obj)
+ return new BinaryFileHandler();
+
+ return new ObjectFileHandler(*Obj);
+}
+
/// \brief Return an appropriate handler given the input files and options.
static FileHandler *CreateFileHandler(MemoryBuffer &FirstInput) {
if (FilesType == "i")
@@ -422,7 +658,7 @@
if (FilesType == "s")
return new TextFileHandler(/*Comment=*/"#");
if (FilesType == "o")
- return new BinaryFileHandler();
+ return CreateObjectFileHandler(FirstInput);
if (FilesType == "gch")
return new BinaryFileHandler();
if (FilesType == "ast")
@@ -674,6 +910,11 @@
if (Error)
return 1;
+ // Save the current executable directory as it will be useful to find other
+ // tools.
+ BundlerExecutable =
+ llvm::sys::fs::getMainExecutable(argv[0], &BundlerExecutable);
+
if (Unbundle)
return UnbundleFiles();
else
Index: test/Driver/clang-offload-bundler.c
===================================================================
--- test/Driver/clang-offload-bundler.c
+++ test/Driver/clang-offload-bundler.c
@@ -6,6 +6,7 @@
// RUN: %clang -O0 -target powerpc64le-ibm-linux-gnu %s -S -emit-llvm -o %t.ll
// RUN: %clang -O0 -target powerpc64le-ibm-linux-gnu %s -c -emit-llvm -o %t.bc
// RUN: %clang -O0 -target powerpc64le-ibm-linux-gnu %s -S -o %t.s
+// RUN: %clang -O0 -target powerpc64le-ibm-linux-gnu %s -c -o %t.o
// RUN: %clang -O0 -target powerpc64le-ibm-linux-gnu %s -emit-ast -o %t.ast
//
@@ -181,6 +182,67 @@
// RUN: diff %t.empty %t.res.tgt1
// RUN: diff %t.empty %t.res.tgt2
+//
+// Check object bundle/unbundle. The content should be bundled into an ELF section (we are using a PowerPC little-endian host which uses ELF).
+//
+// RUN: clang-offload-bundler -type=o -targets=host-powerpc64le-ibm-linux-gnu,openmp-powerpc64le-ibm-linux-gnu,openmp-x86_64-pc-linux-gnu -inputs=%t.o,%t.tgt1,%t.tgt2 -outputs=%t.bundle3.o
+// RUN: llvm-readobj -sections %t.bundle3.o | FileCheck %s --check-prefix CK-OBJ
+// RUN: clang-offload-bundler -type=o -targets=host-powerpc64le-ibm-linux-gnu,openmp-powerpc64le-ibm-linux-gnu,openmp-x86_64-pc-linux-gnu -outputs=%t.res.o,%t.res.tgt1,%t.res.tgt2 -inputs=%t.bundle3.o -unbundle
+// RUN: diff %t.bundle3.o %t.res.o
+// RUN: diff %t.tgt1 %t.res.tgt1
+// RUN: diff %t.tgt2 %t.res.tgt2
+// CK-OBJ: Section {
+// CK-OBJ: Index:
+// CK-OBJ: Name: __CLANG_OFFLOAD_BUNDLE__host-powerpc64le-ibm-linux-gnu (
+// CK-OBJ: Type: SHT_PROGBITS (0x1)
+// CK-OBJ: Flags [ (0x2)
+// CK-OBJ: SHF_ALLOC (0x2)
+// CK-OBJ: ]
+// CK-OBJ: Address: 0x0
+// CK-OBJ: Offset: 0x
+// CK-OBJ: Size: 1
+// CK-OBJ: Link: 0
+// CK-OBJ: Info: 0
+// CK-OBJ: AddressAlignment:
+// CK-OBJ: EntrySize: 0
+// CK-OBJ: }
+// CK-OBJ: Section {
+// CK-OBJ: Index:
+// CK-OBJ: Name: __CLANG_OFFLOAD_BUNDLE__openmp-powerpc64le-ibm-linux-gnu (
+// CK-OBJ: Type: SHT_PROGBITS (0x1)
+// CK-OBJ: Flags [ (0x2)
+// CK-OBJ: SHF_ALLOC (0x2)
+// CK-OBJ: ]
+// CK-OBJ: Address: 0x0
+// CK-OBJ: Offset: 0x
+// CK-OBJ: Size: 25
+// CK-OBJ: Link: 0
+// CK-OBJ: Info: 0
+// CK-OBJ: AddressAlignment:
+// CK-OBJ: EntrySize: 0
+// CK-OBJ: }
+// CK-OBJ: Section {
+// CK-OBJ: Index:
+// CK-OBJ: Name: __CLANG_OFFLOAD_BUNDLE__openmp-x86_64-pc-linux-gnu (
+// CK-OBJ: Type: SHT_PROGBITS (0x1)
+// CK-OBJ: Flags [ (0x2)
+// CK-OBJ: SHF_ALLOC (0x2)
+// CK-OBJ: ]
+// CK-OBJ: Address: 0x0
+// CK-OBJ: Offset: 0x
+// CK-OBJ: Size: 25
+// CK-OBJ: Link: 0
+// CK-OBJ: Info: 0
+// CK-OBJ: AddressAlignment:
+// CK-OBJ: EntrySize: 0
+// CK-OBJ: }
+
+// Check if we can unbundle a file with no magic strings.
+// RUN: clang-offload-bundler -type=o -targets=host-powerpc64le-ibm-linux-gnu,openmp-powerpc64le-ibm-linux-gnu,openmp-x86_64-pc-linux-gnu -outputs=%t.res.o,%t.res.tgt1,%t.res.tgt2 -inputs=%t.o -unbundle
+// RUN: diff %t.o %t.res.o
+// RUN: diff %t.empty %t.res.tgt1
+// RUN: diff %t.empty %t.res.tgt2
+
// Some code so that we can create a binary out of this file.
int A = 0;
void test_func(void) {
_______________________________________________
cfe-commits mailing list
cfe-commits@lists.llvm.org
http://lists.llvm.org/cgi-bin/mailman/listinfo/cfe-commits
