[PATCH] D68213: [LTO] Support for embedding bitcode section during LTO

[Josef Eisl via Phabricator via cfe-commits]

zapster updated this revision to Diff 222401.
zapster added a comment.

re-ran clang format


CHANGES SINCE LAST ACTION
  https://reviews.llvm.org/D68213/new/

https://reviews.llvm.org/D68213

Files:
  clang/lib/CodeGen/BackendUtil.cpp
  clang/test/Frontend/x86-embed-bitcode.ll
  llvm/include/llvm/Bitcode/BitcodeWriter.h
  llvm/lib/Bitcode/Writer/BitcodeWriter.cpp
  llvm/lib/LTO/LTOBackend.cpp
  llvm/test/LTO/X86/Inputs/start-lib1.ll
  llvm/test/LTO/X86/Inputs/start-lib2.ll
  llvm/test/LTO/X86/embed-bitcode.ll

Index: llvm/test/LTO/X86/embed-bitcode.ll
===
--- /dev/null
+++ llvm/test/LTO/X86/embed-bitcode.ll
@@ -0,0 +1,33 @@
+; RUN: llvm-as %s -o %t1.o
+; RUN: llvm-as %p/Inputs/start-lib1.ll -o %t2.o
+; RUN: llvm-as %p/Inputs/start-lib2.ll -o %t3.o
+
+; RUN: llvm-lto2 run -r %t1.o,_start,px -r %t2.o,foo,px -r %t3.o,bar,px -r %t2.o,bar,px -lto-embed-bitcode=off -o %t3 %t1.o %t2.o %t3.o
+; RUN: llvm-readelf -S %t3.0 | FileCheck %s --implicit-check-not=.llvmbc
+
+; RUN: llvm-lto2 run -r %t1.o,_start,px -r %t2.o,foo,px -r %t3.o,bar,px -r %t2.o,bar,px -lto-embed-bitcode=marker -o %t3 %t1.o %t2.o %t3.o
+; RUN: llvm-readelf -S %t3.0 | FileCheck %s --check-prefix=CHECK-ELF
+
+; RUN: llvm-lto2 run -r %t1.o,_start,px -r %t2.o,foo,px -r %t3.o,bar,px -r %t2.o,bar,px -lto-embed-bitcode=bitcode -o %t3 %t1.o %t2.o %t3.o
+; RUN: llvm-readelf -S %t3.0 | FileCheck %s --check-prefix=CHECK-ELF
+; RUN: llvm-objcopy -O binary -j .llvmbc %t3.0 %t-embedded.bc
+; RUN: llvm-dis %t-embedded.bc -o - | FileCheck %s --check-prefix=CHECK-LL
+
+; RUN: llvm-lto2 run -r %t1.o,_start,px -r %t2.o,foo,px -r %t3.o,bar,px -r %t2.o,bar,px -lto-embed-bitcode=all -o %t3 %t1.o %t2.o %t3.o
+; RUN: llvm-readelf -S %t3.0 | FileCheck %s --check-prefix=CHECK-ELF
+; RUN: llvm-objcopy -O binary -j .llvmbc %t3.0 %t-embedded.bc
+; RUN: llvm-dis %t-embedded.bc -o - | FileCheck %s --check-prefix=CHECK-LL
+
+; CHECK-ELF: .text
+; CHECK-ELF: .llvmbc
+
+; CHECK-LL: @_start
+; CHECK-LL: @foo
+; CHECK-LL: @bar
+
+target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-unknown-linux-gnu"
+
+define void @_start() {
+  ret void
+}
Index: llvm/test/LTO/X86/Inputs/start-lib2.ll
===
--- /dev/null
+++ llvm/test/LTO/X86/Inputs/start-lib2.ll
@@ -0,0 +1,6 @@
+target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-unknown-linux-gnu"
+
+define void @bar() {
+  ret void
+}
Index: llvm/test/LTO/X86/Inputs/start-lib1.ll
===
--- /dev/null
+++ llvm/test/LTO/X86/Inputs/start-lib1.ll
@@ -0,0 +1,8 @@
+target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-unknown-linux-gnu"
+
+declare void @bar()
+
+define void @foo() {
+  ret void
+}
Index: llvm/lib/LTO/LTOBackend.cpp
===
--- llvm/lib/LTO/LTOBackend.cpp
+++ llvm/lib/LTO/LTOBackend.cpp
@@ -34,6 +34,7 @@
 #include "llvm/Support/MemoryBuffer.h"
 #include "llvm/Support/Path.h"
 #include "llvm/Support/Program.h"
+#include "llvm/Support/SmallVectorMemoryBuffer.h"
 #include "llvm/Support/TargetRegistry.h"
 #include "llvm/Support/ThreadPool.h"
 #include "llvm/Support/raw_ostream.h"
@@ -312,11 +313,56 @@
   return !Conf.PostOptModuleHook || Conf.PostOptModuleHook(Task, Mod);
 }
 
+cl::opt
+EmbedBitcode("lto-embed-bitcode", cl::desc("Embed LLVM bitcode"),
+ cl::value_desc("option: off, all, bitcode, marker"));
+
+enum EmbedBitcodeKind {
+  Embed_Off, // No embedded bitcode.
+  Embed_All, // Embed bitcode and commandline in the output.
+  Embed_Bitcode, // Embed only bitcode in the output.
+  Embed_Marker   // Embed a marker as a placeholder for bitcode.
+};
+
+static EmbedBitcodeKind getEmbedBitcode(Config &Conf) {
+  if (EmbedBitcode.empty())
+return EmbedBitcodeKind::Embed_Off;
+  StringRef value = EmbedBitcode;
+  unsigned Model = llvm::StringSwitch(value)
+   .Case("off", EmbedBitcodeKind::Embed_Off)
+   .Case("all", EmbedBitcodeKind::Embed_All)
+   .Case("bitcode", EmbedBitcodeKind::Embed_Bitcode)
+   .Case("marker", EmbedBitcodeKind::Embed_Marker)
+   .Default(~0U);
+  if (Model == ~0U) {
+report_fatal_error("invalid value '" + value + "' for 'lto-embed-bitcode'");
+  }
+  return static_cast(Model);
+}
+
+static void EmitBitcodeSection(Module &M, Config &Conf) {
+  const EmbedBitcodeKind embedBitcode = getEmbedBitcode(Conf);
+  if (embedBitcode == Embed_Off)
+return;
+  SmallVector Buffer;
+  raw_svector_ostream OS(Buffer);
+  WriteBitcodeToFile(M, OS);
+
+  std::unique_ptr Buf(
+  new SmallVectorMemoryBuffer(std::move(Buffer)));
+  std::vector CmdArgs;
+  llvm::EmbedBitcodeInModule(
+  M, Buf->getMemBufferRef(), embedBitcode != EmbedBitcodeKind::Embed_

[PATCH] D68213: [LTO] Support for embedding bitcode section during LTO

[Josef Eisl via Phabricator via cfe-commits]

zapster created this revision.
zapster added reviewers: LLVM, clang, rsmith, pcc.
Herald added subscribers: llvm-commits, cfe-commits, dang, dexonsmith, 
steven_wu, aheejin, hiraditya, inglorion, mehdi_amini.
Herald added a reviewer: alexshap.
Herald added projects: clang, LLVM.
zapster updated this revision to Diff 222401.
zapster added a comment.

re-ran clang format


This adds support for embedding bitcode in a binary during LTO. The libLTO 
gains supports the `-lto-embed-bitcode` option which accepts `off`, `all`, 
`bitcode`, and `marker`. The semantics are the same as for clangs 
`-fembed-bitcode`. The option allows users of the LTO library to embed a 
bitcode section. For example, LLD can pass the option via `ld.lld 
-mllvm=-lto-embed-bitcode=all`.

This feature allows doing something comparable to `clang -c -fembed-bitcode`, 
but on the (LTO) linker level. Having bitcode alongside native code has many 
use-cases. To give an example, the MacOS linker can create a `-bitcode_bundle` 
section containing bitcode. Also, having this feature built into LLVM is an 
alternative to 3rd party tools such as wllvm 
 or gllvm 
. As with these tools, this feature 
simplifies creating "whole-program" llvm bitcode files, but in contrast to 
wllvm/gllvm it does not rely on a specific llvm frontend/driver.

I originally proposed this feature as an addition to LLD 
. It turned 
out, however, that doing this purely on LLVM/LTO side is more general and might 
be useful to a broader audience.

The implementation is quite straight forward. The embedding logic moved from 
clang to llvm/lib/Bitcode and llvm/lib/LTO gained the `-lto-embed-bitcode` 
option. Most code just moved.


https://reviews.llvm.org/D68213

Files:
  clang/lib/CodeGen/BackendUtil.cpp
  clang/test/Frontend/x86-embed-bitcode.ll
  llvm/include/llvm/Bitcode/BitcodeWriter.h
  llvm/lib/Bitcode/Writer/BitcodeWriter.cpp
  llvm/lib/LTO/LTOBackend.cpp
  llvm/test/LTO/X86/Inputs/start-lib1.ll
  llvm/test/LTO/X86/Inputs/start-lib2.ll
  llvm/test/LTO/X86/embed-bitcode.ll

Index: llvm/test/LTO/X86/embed-bitcode.ll
===
--- /dev/null
+++ llvm/test/LTO/X86/embed-bitcode.ll
@@ -0,0 +1,33 @@
+; RUN: llvm-as %s -o %t1.o
+; RUN: llvm-as %p/Inputs/start-lib1.ll -o %t2.o
+; RUN: llvm-as %p/Inputs/start-lib2.ll -o %t3.o
+
+; RUN: llvm-lto2 run -r %t1.o,_start,px -r %t2.o,foo,px -r %t3.o,bar,px -r %t2.o,bar,px -lto-embed-bitcode=off -o %t3 %t1.o %t2.o %t3.o
+; RUN: llvm-readelf -S %t3.0 | FileCheck %s --implicit-check-not=.llvmbc
+
+; RUN: llvm-lto2 run -r %t1.o,_start,px -r %t2.o,foo,px -r %t3.o,bar,px -r %t2.o,bar,px -lto-embed-bitcode=marker -o %t3 %t1.o %t2.o %t3.o
+; RUN: llvm-readelf -S %t3.0 | FileCheck %s --check-prefix=CHECK-ELF
+
+; RUN: llvm-lto2 run -r %t1.o,_start,px -r %t2.o,foo,px -r %t3.o,bar,px -r %t2.o,bar,px -lto-embed-bitcode=bitcode -o %t3 %t1.o %t2.o %t3.o
+; RUN: llvm-readelf -S %t3.0 | FileCheck %s --check-prefix=CHECK-ELF
+; RUN: llvm-objcopy -O binary -j .llvmbc %t3.0 %t-embedded.bc
+; RUN: llvm-dis %t-embedded.bc -o - | FileCheck %s --check-prefix=CHECK-LL
+
+; RUN: llvm-lto2 run -r %t1.o,_start,px -r %t2.o,foo,px -r %t3.o,bar,px -r %t2.o,bar,px -lto-embed-bitcode=all -o %t3 %t1.o %t2.o %t3.o
+; RUN: llvm-readelf -S %t3.0 | FileCheck %s --check-prefix=CHECK-ELF
+; RUN: llvm-objcopy -O binary -j .llvmbc %t3.0 %t-embedded.bc
+; RUN: llvm-dis %t-embedded.bc -o - | FileCheck %s --check-prefix=CHECK-LL
+
+; CHECK-ELF: .text
+; CHECK-ELF: .llvmbc
+
+; CHECK-LL: @_start
+; CHECK-LL: @foo
+; CHECK-LL: @bar
+
+target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-unknown-linux-gnu"
+
+define void @_start() {
+  ret void
+}
Index: llvm/test/LTO/X86/Inputs/start-lib2.ll
===
--- /dev/null
+++ llvm/test/LTO/X86/Inputs/start-lib2.ll
@@ -0,0 +1,6 @@
+target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-unknown-linux-gnu"
+
+define void @bar() {
+  ret void
+}
Index: llvm/test/LTO/X86/Inputs/start-lib1.ll
===
--- /dev/null
+++ llvm/test/LTO/X86/Inputs/start-lib1.ll
@@ -0,0 +1,8 @@
+target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-unknown-linux-gnu"
+
+declare void @bar()
+
+define void @foo() {
+  ret void
+}
Index: llvm/lib/LTO/LTOBackend.cpp
===
--- llvm/lib/LTO/LTOBackend.cpp
+++ llvm/lib/LTO/LTOBackend.cpp
@@ -34,6 +34,7 @@
 #include "llvm/Support/MemoryBuffer.h"
 #include "llvm/Support/Path.h"
 #include "llvm/Support/Program.h"
+#include "llvm/Support/SmallVectorMemoryBuffer.h"
 #include "llvm/Support/TargetRegistry.h"
 #include "llvm/Support/ThreadPool.h"
 #include "ll

[PATCH] D68213: [LTO] Support for embedding bitcode section during LTO

[Josef Eisl via Phabricator via cfe-commits]

zapster marked 5 inline comments as done.
zapster added a comment.

Added inline remarks.




Comment at: clang/lib/CodeGen/BackendUtil.cpp:1547
 }
 
-static const char* getSectionNameForBitcode(const Triple &T) {

moved to `llvm/lib/Bitcode/Writer/BitcodeWriter.cpp`



Comment at: clang/test/Frontend/x86-embed-bitcode.ll:1
+; REQUIRES: x86-registered-target
+; check .ll input

This duplicates the `embed-bitcode.ll` test (which only runs on ARM) for x86.



Comment at: llvm/include/llvm/Bitcode/BitcodeWriter.h:158
+bool EmbedMarker,
+const std::vector *CmdArgs);
+

`BitcodeWriter.h` seems like a natural place for this functionality. However, 
suggestions for a better location are more than appreciated. 



Comment at: llvm/lib/Bitcode/Writer/BitcodeWriter.cpp:4670
 }
+
+static const char *getSectionNameForBitcode(const Triple &T) {

moved from `clang/lib/CodeGen/BackendUtil.cpp`



Comment at: llvm/lib/LTO/LTOBackend.cpp:327
+
+static EmbedBitcodeKind getEmbedBitcode(Config &Conf) {
+  if (EmbedBitcode.empty())

This options parsing logic is duplicated from clang. We might want move this to 
a shared place, but I failed to find a good candidate. 
`include/llvm/Support/CodeGen.h` came to mind, but it currently only contains 
types, no code. Any suggestions?


CHANGES SINCE LAST ACTION
  https://reviews.llvm.org/D68213/new/

https://reviews.llvm.org/D68213



___
cfe-commits mailing list
cfe-commits@lists.llvm.org
https://lists.llvm.org/cgi-bin/mailman/listinfo/cfe-commits

[PATCH] D68213: [LTO] Support for embedding bitcode section during LTO

[Josef Eisl via Phabricator via cfe-commits]

zapster added a comment.

(ping)


CHANGES SINCE LAST ACTION
  https://reviews.llvm.org/D68213/new/

https://reviews.llvm.org/D68213



___
cfe-commits mailing list
cfe-commits@lists.llvm.org
https://lists.llvm.org/cgi-bin/mailman/listinfo/cfe-commits

[PATCH] D68213: [LTO] Support for embedding bitcode section during LTO

[Josef Eisl via Phabricator via cfe-commits]

zapster added a comment.

(Ping)

Hi! I am still looking for reviews/reviewers for this change. Please let me 
know what I can do to make progress with this.
Many thanks in advance!


CHANGES SINCE LAST ACTION
  https://reviews.llvm.org/D68213/new/

https://reviews.llvm.org/D68213



___
cfe-commits mailing list
cfe-commits@lists.llvm.org
https://lists.llvm.org/cgi-bin/mailman/listinfo/cfe-commits

[PATCH] D68213: [LTO] Support for embedding bitcode section during LTO

[Josef Eisl via Phabricator via cfe-commits]

zapster added a comment.

Thanks for your comment. I replied inline.




Comment at: llvm/lib/LTO/LTOBackend.cpp:335
+   .Case("bitcode", EmbedBitcodeKind::Embed_Bitcode)
+   .Case("marker", EmbedBitcodeKind::Embed_Marker)
+   .Default(~0U);

tejohnson wrote:
> Does this value make any sense since CmdArgs is always empty below?
You are right. Currently, this option value is not utterly useful, but I kept 
it for compatibility with clangs `-fembed-bitcode`. It seems the marker section 
is needed (even if empty), otherwise certain tools will complain (notably the 
MacOS linker). Ideally, we would have something sensible to write into the 
section but I am not sure about reasonable values. I was not able to find any 
information about consumers of these commands. Pointers in that regard would be 
highly appreciated.

Anyhow, I lean toward keeping the options for compatibility with clang and for 
making it simple to properly implement the `marker` case. That said, I am 
perfectly fine with removing the option. In that case, I guess I should get rid 
of `EmbedBitcodeKind` altogether and always insert the bitcode and the marker, 
right?

On the other hand, we should maybe just consolidate the option (parsing) with 
the clang option. Some thoughts about that in my inline comment above.


CHANGES SINCE LAST ACTION
  https://reviews.llvm.org/D68213/new/

https://reviews.llvm.org/D68213



___
cfe-commits mailing list
cfe-commits@lists.llvm.org
https://lists.llvm.org/cgi-bin/mailman/listinfo/cfe-commits

[PATCH] D68213: [LTO] Support for embedding bitcode section during LTO

[Josef Eisl via Phabricator via cfe-commits]

zapster marked an inline comment as done.
zapster added inline comments.



Comment at: llvm/lib/LTO/LTOBackend.cpp:335
+   .Case("bitcode", EmbedBitcodeKind::Embed_Bitcode)
+   .Case("marker", EmbedBitcodeKind::Embed_Marker)
+   .Default(~0U);

steven_wu wrote:
> tejohnson wrote:
> > zapster wrote:
> > > tejohnson wrote:
> > > > Does this value make any sense since CmdArgs is always empty below?
> > > You are right. Currently, this option value is not utterly useful, but I 
> > > kept it for compatibility with clangs `-fembed-bitcode`. It seems the 
> > > marker section is needed (even if empty), otherwise certain tools will 
> > > complain (notably the MacOS linker). Ideally, we would have something 
> > > sensible to write into the section but I am not sure about reasonable 
> > > values. I was not able to find any information about consumers of these 
> > > commands. Pointers in that regard would be highly appreciated.
> > > 
> > > Anyhow, I lean toward keeping the options for compatibility with clang 
> > > and for making it simple to properly implement the `marker` case. That 
> > > said, I am perfectly fine with removing the option. In that case, I guess 
> > > I should get rid of `EmbedBitcodeKind` altogether and always insert the 
> > > bitcode and the marker, right?
> > > 
> > > On the other hand, we should maybe just consolidate the option (parsing) 
> > > with the clang option. Some thoughts about that in my inline comment 
> > > above.
> > It's unclear to me whether it is better to not provide the marker, and get 
> > a clear error, or put in an empty marker which may cause these tools to do 
> > the wrong thing. @steven_wu who might be able to provide some guidance 
> > about how this is used by the MacOS linker. Note that if someone is using 
> > MacOS to do their LTO linking, they won't even trigger this code, as that 
> > toolchain uses the old legacy LTO, which doesn't come here.
> > 
> > Presumably your tool doesn't use the marker section?
> I don't think marker is useful for this use case. I will suggest not to have 
> the `marker` LTO option unless we have the need. It is being tested by clang 
> and that is probably enough coverage for testing.
> 
> I don't think any users of the legacy interface express interests in using a 
> unified bitcode section so it is fine just to leave this out of the legacy C 
> interface.
> 
> 
> [...] if someone is using MacOS to do their LTO linking, they won't even 
> trigger this code, as that toolchain uses the old legacy LTO

Right. I mixed up the case where the embedding logic was called from clang and 
from LTO. And of course, the MacOS linker does (unfortunately) not use this 
code.

> Presumably your tool doesn't use the marker section?

Nope, I don't need the marker section.

> I don't think marker is useful for this use case.

Right, makes sense. I'll update the review and make `-lto-embed-bitcode` a 
boolean option.

Thanks to both of you for the clarifications.


CHANGES SINCE LAST ACTION
  https://reviews.llvm.org/D68213/new/

https://reviews.llvm.org/D68213



___
cfe-commits mailing list
cfe-commits@lists.llvm.org
https://lists.llvm.org/cgi-bin/mailman/listinfo/cfe-commits

[PATCH] D68213: [LTO] Support for embedding bitcode section during LTO

[Josef Eisl via Phabricator via cfe-commits]

zapster updated this revision to Diff 233282.
zapster edited the summary of this revision.
zapster added a comment.

Made `-lto-embed-bitcode` a boolean option.


CHANGES SINCE LAST ACTION
  https://reviews.llvm.org/D68213/new/

https://reviews.llvm.org/D68213

Files:
  clang/lib/CodeGen/BackendUtil.cpp
  clang/test/Frontend/x86-embed-bitcode.ll
  llvm/include/llvm/Bitcode/BitcodeWriter.h
  llvm/lib/Bitcode/Writer/BitcodeWriter.cpp
  llvm/lib/LTO/LTOBackend.cpp
  llvm/test/LTO/X86/Inputs/start-lib1.ll
  llvm/test/LTO/X86/Inputs/start-lib2.ll
  llvm/test/LTO/X86/embed-bitcode.ll

Index: llvm/test/LTO/X86/embed-bitcode.ll
===
--- /dev/null
+++ llvm/test/LTO/X86/embed-bitcode.ll
@@ -0,0 +1,41 @@
+; RUN: llvm-as %s -o %t1.o
+; RUN: llvm-as %p/Inputs/start-lib1.ll -o %t2.o
+; RUN: llvm-as %p/Inputs/start-lib2.ll -o %t3.o
+
+; RUN: llvm-lto2 run -r %t1.o,_start,px -r %t2.o,foo,px -r %t3.o,bar,px -r %t2.o,bar,px -o %t3 %t1.o %t2.o %t3.o
+; RUN: llvm-readelf -S %t3.0 | FileCheck %s --implicit-check-not=.llvmbc
+
+; RUN: llvm-lto2 run -r %t1.o,_start,px -r %t2.o,foo,px -r %t3.o,bar,px -r %t2.o,bar,px -lto-embed-bitcode=0 -o %t3 %t1.o %t2.o %t3.o
+; RUN: llvm-readelf -S %t3.0 | FileCheck %s --implicit-check-not=.llvmbc
+
+; RUN: llvm-lto2 run -r %t1.o,_start,px -r %t2.o,foo,px -r %t3.o,bar,px -r %t2.o,bar,px -lto-embed-bitcode=false -o %t3 %t1.o %t2.o %t3.o
+; RUN: llvm-readelf -S %t3.0 | FileCheck %s --implicit-check-not=.llvmbc
+
+; RUN: llvm-lto2 run -r %t1.o,_start,px -r %t2.o,foo,px -r %t3.o,bar,px -r %t2.o,bar,px -lto-embed-bitcode -o %t3 %t1.o %t2.o %t3.o
+; RUN: llvm-readelf -S %t3.0 | FileCheck %s --check-prefix=CHECK-ELF
+; RUN: llvm-objcopy -O binary -j .llvmbc %t3.0 %t-embedded.bc
+; RUN: llvm-dis %t-embedded.bc -o - | FileCheck %s --check-prefix=CHECK-LL
+
+; RUN: llvm-lto2 run -r %t1.o,_start,px -r %t2.o,foo,px -r %t3.o,bar,px -r %t2.o,bar,px -lto-embed-bitcode=1 -o %t3 %t1.o %t2.o %t3.o
+; RUN: llvm-readelf -S %t3.0 | FileCheck %s --check-prefix=CHECK-ELF
+; RUN: llvm-objcopy -O binary -j .llvmbc %t3.0 %t-embedded.bc
+; RUN: llvm-dis %t-embedded.bc -o - | FileCheck %s --check-prefix=CHECK-LL
+
+; RUN: llvm-lto2 run -r %t1.o,_start,px -r %t2.o,foo,px -r %t3.o,bar,px -r %t2.o,bar,px -lto-embed-bitcode=true -o %t3 %t1.o %t2.o %t3.o
+; RUN: llvm-readelf -S %t3.0 | FileCheck %s --check-prefix=CHECK-ELF
+; RUN: llvm-objcopy -O binary -j .llvmbc %t3.0 %t-embedded.bc
+; RUN: llvm-dis %t-embedded.bc -o - | FileCheck %s --check-prefix=CHECK-LL
+
+; CHECK-ELF: .text
+; CHECK-ELF: .llvmbc
+
+; CHECK-LL: @_start
+; CHECK-LL: @foo
+; CHECK-LL: @bar
+
+target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-unknown-linux-gnu"
+
+define void @_start() {
+  ret void
+}
Index: llvm/test/LTO/X86/Inputs/start-lib2.ll
===
--- /dev/null
+++ llvm/test/LTO/X86/Inputs/start-lib2.ll
@@ -0,0 +1,6 @@
+target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-unknown-linux-gnu"
+
+define void @bar() {
+  ret void
+}
Index: llvm/test/LTO/X86/Inputs/start-lib1.ll
===
--- /dev/null
+++ llvm/test/LTO/X86/Inputs/start-lib1.ll
@@ -0,0 +1,8 @@
+target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-unknown-linux-gnu"
+
+declare void @bar()
+
+define void @foo() {
+  ret void
+}
Index: llvm/lib/LTO/LTOBackend.cpp
===
--- llvm/lib/LTO/LTOBackend.cpp
+++ llvm/lib/LTO/LTOBackend.cpp
@@ -34,6 +34,7 @@
 #include "llvm/Support/MemoryBuffer.h"
 #include "llvm/Support/Path.h"
 #include "llvm/Support/Program.h"
+#include "llvm/Support/SmallVectorMemoryBuffer.h"
 #include "llvm/Support/TargetRegistry.h"
 #include "llvm/Support/ThreadPool.h"
 #include "llvm/Support/raw_ostream.h"
@@ -312,11 +313,29 @@
   return !Conf.PostOptModuleHook || Conf.PostOptModuleHook(Task, Mod);
 }
 
+static cl::opt EmbedBitcode("lto-embed-bitcode", cl::init(false),
+  cl::desc("Embed LLVM bitcode"));
+
+static void EmitBitcodeSection(Module &M, Config &Conf) {
+  if (!EmbedBitcode)
+return;
+  SmallVector Buffer;
+  raw_svector_ostream OS(Buffer);
+  WriteBitcodeToFile(M, OS);
+
+  std::unique_ptr Buf(
+  new SmallVectorMemoryBuffer(std::move(Buffer)));
+  llvm::EmbedBitcodeInModule(M, Buf->getMemBufferRef(), /*EmbedBitcode*/ true,
+ /*EmbedMarker*/ false, /*CmdArgs*/ nullptr);
+}
+
 void codegen(Config &Conf, TargetMachine *TM, AddStreamFn AddStream,
  unsigned Task, Module &Mod) {
   if (Conf.PreCodeGenModuleHook && !Conf.PreCodeGenModuleHook(Task, Mod))
 return;
 
+  EmitBitcodeSection(Mod, Conf);
+
   std::unique_ptr DwoOut;
   SmallString<1024> DwoFile(Conf.SplitDwarfOutput);
   if (!Conf.DwoDir.empty()) {
Index: llvm/lib/Bitcode/Writer/BitcodeWrit

[PATCH] D68213: [LTO] Support for embedding bitcode section during LTO

[Josef Eisl via Phabricator via cfe-commits]

zapster updated this revision to Diff 233548.
zapster added a comment.

Addressed suggestions from @tejohnson


CHANGES SINCE LAST ACTION
  https://reviews.llvm.org/D68213/new/

https://reviews.llvm.org/D68213

Files:
  clang/lib/CodeGen/BackendUtil.cpp
  clang/test/Frontend/x86-embed-bitcode.ll
  llvm/include/llvm/Bitcode/BitcodeWriter.h
  llvm/lib/Bitcode/Writer/BitcodeWriter.cpp
  llvm/lib/LTO/LTOBackend.cpp
  llvm/test/LTO/X86/Inputs/start-lib1.ll
  llvm/test/LTO/X86/Inputs/start-lib2.ll
  llvm/test/LTO/X86/embed-bitcode.ll

Index: llvm/test/LTO/X86/embed-bitcode.ll
===
--- /dev/null
+++ llvm/test/LTO/X86/embed-bitcode.ll
@@ -0,0 +1,28 @@
+; RUN: llvm-as %s -o %t1.o
+; RUN: llvm-as %p/Inputs/start-lib1.ll -o %t2.o
+; RUN: llvm-as %p/Inputs/start-lib2.ll -o %t3.o
+
+; RUN: llvm-lto2 run -r %t1.o,_start,px -r %t2.o,foo,px -r %t3.o,bar,px -r %t2.o,bar,px -o %t3 %t1.o %t2.o %t3.o
+; RUN: llvm-readelf -S %t3.0 | FileCheck %s --implicit-check-not=.llvmbc
+
+; RUN: llvm-lto2 run -r %t1.o,_start,px -r %t2.o,foo,px -r %t3.o,bar,px -r %t2.o,bar,px -lto-embed-bitcode=false -o %t3 %t1.o %t2.o %t3.o
+; RUN: llvm-readelf -S %t3.0 | FileCheck %s --implicit-check-not=.llvmbc
+
+; RUN: llvm-lto2 run -r %t1.o,_start,px -r %t2.o,foo,px -r %t3.o,bar,px -r %t2.o,bar,px -lto-embed-bitcode -o %t3 %t1.o %t2.o %t3.o
+; RUN: llvm-readelf -S %t3.0 | FileCheck %s --check-prefix=CHECK-ELF
+; RUN: llvm-objcopy -O binary -j .llvmbc %t3.0 %t-embedded.bc
+; RUN: llvm-dis %t-embedded.bc -o - | FileCheck %s --check-prefix=CHECK-LL
+
+; CHECK-ELF: .text
+; CHECK-ELF: .llvmbc
+
+; CHECK-LL: @_start
+; CHECK-LL: @foo
+; CHECK-LL: @bar
+
+target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-unknown-linux-gnu"
+
+define void @_start() {
+  ret void
+}
Index: llvm/test/LTO/X86/Inputs/start-lib2.ll
===
--- /dev/null
+++ llvm/test/LTO/X86/Inputs/start-lib2.ll
@@ -0,0 +1,6 @@
+target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-unknown-linux-gnu"
+
+define void @bar() {
+  ret void
+}
Index: llvm/test/LTO/X86/Inputs/start-lib1.ll
===
--- /dev/null
+++ llvm/test/LTO/X86/Inputs/start-lib1.ll
@@ -0,0 +1,8 @@
+target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-unknown-linux-gnu"
+
+declare void @bar()
+
+define void @foo() {
+  ret void
+}
Index: llvm/lib/LTO/LTOBackend.cpp
===
--- llvm/lib/LTO/LTOBackend.cpp
+++ llvm/lib/LTO/LTOBackend.cpp
@@ -34,6 +34,7 @@
 #include "llvm/Support/MemoryBuffer.h"
 #include "llvm/Support/Path.h"
 #include "llvm/Support/Program.h"
+#include "llvm/Support/SmallVectorMemoryBuffer.h"
 #include "llvm/Support/TargetRegistry.h"
 #include "llvm/Support/ThreadPool.h"
 #include "llvm/Support/raw_ostream.h"
@@ -312,11 +313,30 @@
   return !Conf.PostOptModuleHook || Conf.PostOptModuleHook(Task, Mod);
 }
 
+static cl::opt EmbedBitcode(
+"lto-embed-bitcode", cl::init(false),
+cl::desc("Embed LLVM bitcode in object files produced by LTO"));
+
+static void EmitBitcodeSection(Module &M, Config &Conf) {
+  if (!EmbedBitcode)
+return;
+  SmallVector Buffer;
+  raw_svector_ostream OS(Buffer);
+  WriteBitcodeToFile(M, OS);
+
+  std::unique_ptr Buf(
+  new SmallVectorMemoryBuffer(std::move(Buffer)));
+  llvm::EmbedBitcodeInModule(M, Buf->getMemBufferRef(), /*EmbedBitcode*/ true,
+ /*EmbedMarker*/ false, /*CmdArgs*/ nullptr);
+}
+
 void codegen(Config &Conf, TargetMachine *TM, AddStreamFn AddStream,
  unsigned Task, Module &Mod) {
   if (Conf.PreCodeGenModuleHook && !Conf.PreCodeGenModuleHook(Task, Mod))
 return;
 
+  EmitBitcodeSection(Mod, Conf);
+
   std::unique_ptr DwoOut;
   SmallString<1024> DwoFile(Conf.SplitDwarfOutput);
   if (!Conf.DwoDir.empty()) {
Index: llvm/lib/Bitcode/Writer/BitcodeWriter.cpp
===
--- llvm/lib/Bitcode/Writer/BitcodeWriter.cpp
+++ llvm/lib/Bitcode/Writer/BitcodeWriter.cpp
@@ -24,9 +24,10 @@
 #include "llvm/ADT/StringMap.h"
 #include "llvm/ADT/StringRef.h"
 #include "llvm/ADT/Triple.h"
+#include "llvm/Bitcode/BitcodeReader.h"
+#include "llvm/Bitcode/LLVMBitCodes.h"
 #include "llvm/Bitstream/BitCodes.h"
 #include "llvm/Bitstream/BitstreamWriter.h"
-#include "llvm/Bitcode/LLVMBitCodes.h"
 #include "llvm/Config/llvm-config.h"
 #include "llvm/IR/Attributes.h"
 #include "llvm/IR/BasicBlock.h"
@@ -4666,3 +4667,125 @@
 
   Out.write((char *)&Buffer.front(), Buffer.size());
 }
+
+static const char *getSectionNameForBitcode(const Triple &T) {
+  switch (T.getObjectFormat()) {
+  case Triple::MachO:
+return "__LLVM,__bitcode";
+  case Triple::COFF:
+  case Triple::ELF:
+  case Triple::Wasm:
+  case Triple::UnknownObjectFormat:
+return ".llvmb

[PATCH] D68213: [LTO] Support for embedding bitcode section during LTO

[Josef Eisl via Phabricator via cfe-commits]

zapster added a comment.

Thanks again for you reviews! Since I do not have commit rights, I would be 
grateful if someone could push it for me.


CHANGES SINCE LAST ACTION
  https://reviews.llvm.org/D68213/new/

https://reviews.llvm.org/D68213



___
cfe-commits mailing list
cfe-commits@lists.llvm.org
https://lists.llvm.org/cgi-bin/mailman/listinfo/cfe-commits

[PATCH] D68213: [LTO] Support for embedding bitcode section during LTO

[Josef Eisl via Phabricator via cfe-commits]

zapster updated this revision to Diff 233652.
zapster added a comment.

> The new LTO/X86/embed-bitcode.ll test is failing for me

Should be fixed now, thanks.


CHANGES SINCE LAST ACTION
  https://reviews.llvm.org/D68213/new/

https://reviews.llvm.org/D68213

Files:
  clang/lib/CodeGen/BackendUtil.cpp
  clang/test/Frontend/x86-embed-bitcode.ll
  llvm/include/llvm/Bitcode/BitcodeWriter.h
  llvm/lib/Bitcode/Writer/BitcodeWriter.cpp
  llvm/lib/LTO/LTOBackend.cpp
  llvm/test/LTO/X86/Inputs/start-lib1.ll
  llvm/test/LTO/X86/Inputs/start-lib2.ll
  llvm/test/LTO/X86/embed-bitcode.ll

Index: llvm/test/LTO/X86/embed-bitcode.ll
===
--- /dev/null
+++ llvm/test/LTO/X86/embed-bitcode.ll
@@ -0,0 +1,28 @@
+; RUN: llvm-as %s -o %t1.o
+; RUN: llvm-as %p/Inputs/start-lib1.ll -o %t2.o
+; RUN: llvm-as %p/Inputs/start-lib2.ll -o %t3.o
+
+; RUN: llvm-lto2 run -r %t1.o,_start,px -r %t2.o,foo,px -r %t3.o,bar,px -r %t2.o,bar,lx -o %t3 %t1.o %t2.o %t3.o
+; RUN: llvm-readelf -S %t3.0 | FileCheck %s --implicit-check-not=.llvmbc
+
+; RUN: llvm-lto2 run -r %t1.o,_start,px -r %t2.o,foo,px -r %t3.o,bar,px -r %t2.o,bar,lx -lto-embed-bitcode=false -o %t3 %t1.o %t2.o %t3.o
+; RUN: llvm-readelf -S %t3.0 | FileCheck %s --implicit-check-not=.llvmbc
+
+; RUN: llvm-lto2 run -r %t1.o,_start,px -r %t2.o,foo,px -r %t3.o,bar,px -r %t2.o,bar,lx -lto-embed-bitcode -o %t3 %t1.o %t2.o %t3.o
+; RUN: llvm-readelf -S %t3.0 | FileCheck %s --check-prefix=CHECK-ELF
+; RUN: llvm-objcopy -O binary -j .llvmbc %t3.0 %t-embedded.bc
+; RUN: llvm-dis %t-embedded.bc -o - | FileCheck %s --check-prefix=CHECK-LL
+
+; CHECK-ELF: .text
+; CHECK-ELF: .llvmbc
+
+; CHECK-LL: @_start
+; CHECK-LL: @foo
+; CHECK-LL: @bar
+
+target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-unknown-linux-gnu"
+
+define void @_start() {
+  ret void
+}
Index: llvm/test/LTO/X86/Inputs/start-lib2.ll
===
--- /dev/null
+++ llvm/test/LTO/X86/Inputs/start-lib2.ll
@@ -0,0 +1,6 @@
+target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-unknown-linux-gnu"
+
+define void @bar() {
+  ret void
+}
Index: llvm/test/LTO/X86/Inputs/start-lib1.ll
===
--- /dev/null
+++ llvm/test/LTO/X86/Inputs/start-lib1.ll
@@ -0,0 +1,8 @@
+target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-unknown-linux-gnu"
+
+declare void @bar()
+
+define void @foo() {
+  ret void
+}
Index: llvm/lib/LTO/LTOBackend.cpp
===
--- llvm/lib/LTO/LTOBackend.cpp
+++ llvm/lib/LTO/LTOBackend.cpp
@@ -34,6 +34,7 @@
 #include "llvm/Support/MemoryBuffer.h"
 #include "llvm/Support/Path.h"
 #include "llvm/Support/Program.h"
+#include "llvm/Support/SmallVectorMemoryBuffer.h"
 #include "llvm/Support/TargetRegistry.h"
 #include "llvm/Support/ThreadPool.h"
 #include "llvm/Support/raw_ostream.h"
@@ -312,11 +313,30 @@
   return !Conf.PostOptModuleHook || Conf.PostOptModuleHook(Task, Mod);
 }
 
+static cl::opt EmbedBitcode(
+"lto-embed-bitcode", cl::init(false),
+cl::desc("Embed LLVM bitcode in object files produced by LTO"));
+
+static void EmitBitcodeSection(Module &M, Config &Conf) {
+  if (!EmbedBitcode)
+return;
+  SmallVector Buffer;
+  raw_svector_ostream OS(Buffer);
+  WriteBitcodeToFile(M, OS);
+
+  std::unique_ptr Buf(
+  new SmallVectorMemoryBuffer(std::move(Buffer)));
+  llvm::EmbedBitcodeInModule(M, Buf->getMemBufferRef(), /*EmbedBitcode*/ true,
+ /*EmbedMarker*/ false, /*CmdArgs*/ nullptr);
+}
+
 void codegen(Config &Conf, TargetMachine *TM, AddStreamFn AddStream,
  unsigned Task, Module &Mod) {
   if (Conf.PreCodeGenModuleHook && !Conf.PreCodeGenModuleHook(Task, Mod))
 return;
 
+  EmitBitcodeSection(Mod, Conf);
+
   std::unique_ptr DwoOut;
   SmallString<1024> DwoFile(Conf.SplitDwarfOutput);
   if (!Conf.DwoDir.empty()) {
Index: llvm/lib/Bitcode/Writer/BitcodeWriter.cpp
===
--- llvm/lib/Bitcode/Writer/BitcodeWriter.cpp
+++ llvm/lib/Bitcode/Writer/BitcodeWriter.cpp
@@ -24,9 +24,10 @@
 #include "llvm/ADT/StringMap.h"
 #include "llvm/ADT/StringRef.h"
 #include "llvm/ADT/Triple.h"
+#include "llvm/Bitcode/BitcodeReader.h"
+#include "llvm/Bitcode/LLVMBitCodes.h"
 #include "llvm/Bitstream/BitCodes.h"
 #include "llvm/Bitstream/BitstreamWriter.h"
-#include "llvm/Bitcode/LLVMBitCodes.h"
 #include "llvm/Config/llvm-config.h"
 #include "llvm/IR/Attributes.h"
 #include "llvm/IR/BasicBlock.h"
@@ -4666,3 +4667,125 @@
 
   Out.write((char *)&Buffer.front(), Buffer.size());
 }
+
+static const char *getSectionNameForBitcode(const Triple &T) {
+  switch (T.getObjectFormat()) {
+  case Triple::MachO:
+return "__LLVM,__bitcode";
+  case Triple::COFF:
+  case Triple::ELF:
+  case Triple::Wasm:
+  cas