[Lldb-commits] [PATCH] D128465: [WIP] Zstandard as a second compression method to LLVM

[Cole Kissane via Phabricator via lldb-commits]

ckissane created this revision.
ckissane added reviewers: phosek, leonardchan.
ckissane added a project: LLVM.
Herald added subscribers: Enna1, abrachet, wenlei, usaxena95, kadircet, 
arphaman, hiraditya, arichardson, mgorny, emaste.
Herald added a reviewer: alexander-shaposhnikov.
Herald added a reviewer: rupprecht.
Herald added a reviewer: jhenderson.
Herald added a reviewer: MaskRay.
Herald added projects: Flang, All.
ckissane requested review of this revision.
Herald added subscribers: cfe-commits, llvm-commits, lldb-commits, Sanitizers, 
StephenFan, jdoerfert.
Herald added projects: clang, Sanitizers, LLDB, clang-tools-extra.

All work in progress POC implementation of the [RFC] Zstandard as a second 
compression method to LLVM.

Note:

- An interesting unresolved caveat crc32 implementation in the compression 
namespace requires ZLIB even if only enabling ZSTD




Repository:
  rG LLVM Github Monorepo

https://reviews.llvm.org/D128465

Files:
  clang-tools-extra/clangd/CMakeLists.txt
  clang-tools-extra/clangd/index/Serialization.cpp
  clang-tools-extra/clangd/test/lit.site.cfg.py.in
  clang-tools-extra/clangd/unittests/SerializationTests.cpp
  clang/cmake/caches/Apple-stage1.cmake
  clang/cmake/caches/Apple-stage2.cmake
  clang/cmake/caches/Fuchsia-stage2.cmake
  clang/cmake/caches/Fuchsia.cmake
  clang/lib/Driver/ToolChains/Clang.cpp
  clang/lib/Serialization/ASTReader.cpp
  clang/lib/Serialization/ASTWriter.cpp
  clang/test/CMakeLists.txt
  clang/test/lit.site.cfg.py.in
  compiler-rt/lib/sanitizer_common/symbolizer/scripts/build_symbolizer.sh
  compiler-rt/test/lit.common.configured.in
  flang/CMakeLists.txt
  lld/ELF/CMakeLists.txt
  lld/ELF/Driver.cpp
  lld/ELF/InputSection.cpp
  lldb/source/Plugins/Process/gdb-remote/CMakeLists.txt
  llvm/CMakeLists.txt
  llvm/cmake/config-ix.cmake
  llvm/cmake/modules/LLVMConfig.cmake.in
  llvm/include/llvm/Config/llvm-config.h.cmake
  llvm/include/llvm/Support/Compression.h
  llvm/lib/MC/ELFObjectWriter.cpp
  llvm/lib/ObjCopy/ELF/ELFObject.cpp
  llvm/lib/Object/Decompressor.cpp
  llvm/lib/ProfileData/Coverage/CoverageMappingReader.cpp
  llvm/lib/ProfileData/Coverage/CoverageMappingWriter.cpp
  llvm/lib/ProfileData/InstrProf.cpp
  llvm/lib/ProfileData/SampleProfReader.cpp
  llvm/lib/ProfileData/SampleProfWriter.cpp
  llvm/lib/Support/CMakeLists.txt
  llvm/lib/Support/Compression.cpp
  llvm/tools/llvm-mc/llvm-mc.cpp
  llvm/tools/llvm-objcopy/ObjcopyOptions.cpp
  llvm/unittests/ProfileData/InstrProfTest.cpp
  llvm/unittests/Support/CompressionTest.cpp
  utils/bazel/llvm_configs/llvm-config.h.cmake

Index: utils/bazel/llvm_configs/llvm-config.h.cmake
===
--- utils/bazel/llvm_configs/llvm-config.h.cmake
+++ utils/bazel/llvm_configs/llvm-config.h.cmake
@@ -92,6 +92,9 @@
 /* Define if zlib compression is available */
 #cmakedefine01 LLVM_ENABLE_ZLIB
 
+/* Define if zstd compression is available */
+#cmakedefine01 LLVM_ENABLE_ZSTD
+
 /* Define if LLVM was built with a dependency to the libtensorflow dynamic library */
 #cmakedefine LLVM_HAVE_TF_API
 
Index: llvm/unittests/Support/CompressionTest.cpp
===
--- llvm/unittests/Support/CompressionTest.cpp
+++ llvm/unittests/Support/CompressionTest.cpp
@@ -18,6 +18,7 @@
 #include "gtest/gtest.h"
 
 using namespace llvm;
+using namespace compression;
 
 namespace {
 
@@ -70,4 +71,53 @@
 
 #endif
 
+#if LLVM_ENABLE_ZSTD
+
+void TestZstdCompression(StringRef Input, int Level) {
+  SmallString<32> Compressed;
+  SmallString<32> Uncompressed;
+
+  zstd::compress(Input, Compressed, Level);
+
+  // Check that uncompressed buffer is the same as original.
+  Error E = zstd::uncompress(Compressed, Uncompressed, Input.size());
+  consumeError(std::move(E));
+
+  EXPECT_EQ(Input, Uncompressed);
+  if (Input.size() > 0) {
+// Uncompression fails if expected length is too short.
+E = zstd::uncompress(Compressed, Uncompressed, Input.size() - 1);
+EXPECT_EQ("zstd error: Z_BUF_ERROR", llvm::toString(std::move(E)));
+  }
+}
+
+TEST(CompressionTest, Zstd) {
+  TestZstdCompression("", zstd::DefaultCompression);
+
+  TestZstdCompression("hello, world!", zstd::NoCompression);
+  TestZstdCompression("hello, world!", zstd::BestSizeCompression);
+  TestZstdCompression("hello, world!", zstd::BestSpeedCompression);
+  TestZstdCompression("hello, world!", zstd::DefaultCompression);
+
+  const size_t kSize = 1024;
+  char BinaryData[kSize];
+  for (size_t i = 0; i < kSize; ++i) {
+BinaryData[i] = i & 255;
+  }
+  StringRef BinaryDataStr(BinaryData, kSize);
+
+  TestZstdCompression(BinaryDataStr, zstd::NoCompression);
+  TestZstdCompression(BinaryDataStr, zstd::BestSizeCompression);
+  TestZstdCompression(BinaryDataStr, zstd::BestSpeedCompression);
+  TestZstdCompression(BinaryDataStr, zstd::DefaultCompression);
+}
+
+TEST(CompressionTest, ZstdCRC32) {
+  EXPECT_EQ(
+  0x414FA339U,
+  zstd::crc32(StringR

[Lldb-commits] [PATCH] D128465: Zstandard as a second compression method to LLVM

[Cole Kissane via Phabricator via lldb-commits]

ckissane updated this revision to Diff 440288.
ckissane added a comment.

Fix missing cmake findZstd, remove unused crc32 from compression namespaces and 
simplify some code


Repository:
  rG LLVM Github Monorepo

CHANGES SINCE LAST ACTION
  https://reviews.llvm.org/D128465/new/

https://reviews.llvm.org/D128465

Files:
  clang-tools-extra/clangd/CMakeLists.txt
  clang-tools-extra/clangd/index/Serialization.cpp
  clang-tools-extra/clangd/test/lit.site.cfg.py.in
  clang-tools-extra/clangd/unittests/SerializationTests.cpp
  clang/cmake/caches/Apple-stage1.cmake
  clang/cmake/caches/Apple-stage2.cmake
  clang/cmake/caches/Fuchsia-stage2.cmake
  clang/cmake/caches/Fuchsia.cmake
  clang/lib/Driver/ToolChains/Clang.cpp
  clang/lib/Serialization/ASTReader.cpp
  clang/lib/Serialization/ASTWriter.cpp
  clang/test/CMakeLists.txt
  clang/test/lit.site.cfg.py.in
  compiler-rt/lib/sanitizer_common/symbolizer/scripts/build_symbolizer.sh
  compiler-rt/test/lit.common.configured.in
  flang/CMakeLists.txt
  lld/ELF/CMakeLists.txt
  lld/ELF/Driver.cpp
  lld/ELF/InputSection.cpp
  lldb/source/Plugins/Process/gdb-remote/CMakeLists.txt
  llvm/CMakeLists.txt
  llvm/cmake/config-ix.cmake
  llvm/cmake/modules/FindZSTD.cmake
  llvm/cmake/modules/LLVMConfig.cmake.in
  llvm/include/llvm/Config/llvm-config.h.cmake
  llvm/include/llvm/Support/Compression.h
  llvm/lib/MC/ELFObjectWriter.cpp
  llvm/lib/ObjCopy/ELF/ELFObject.cpp
  llvm/lib/Object/Decompressor.cpp
  llvm/lib/ProfileData/Coverage/CoverageMappingReader.cpp
  llvm/lib/ProfileData/Coverage/CoverageMappingWriter.cpp
  llvm/lib/ProfileData/InstrProf.cpp
  llvm/lib/ProfileData/SampleProfReader.cpp
  llvm/lib/ProfileData/SampleProfWriter.cpp
  llvm/lib/Support/CMakeLists.txt
  llvm/lib/Support/Compression.cpp
  llvm/tools/llvm-mc/llvm-mc.cpp
  llvm/tools/llvm-objcopy/ObjcopyOptions.cpp
  llvm/unittests/ProfileData/InstrProfTest.cpp
  llvm/unittests/Support/CompressionTest.cpp
  utils/bazel/llvm_configs/llvm-config.h.cmake

Index: utils/bazel/llvm_configs/llvm-config.h.cmake
===
--- utils/bazel/llvm_configs/llvm-config.h.cmake
+++ utils/bazel/llvm_configs/llvm-config.h.cmake
@@ -92,6 +92,9 @@
 /* Define if zlib compression is available */
 #cmakedefine01 LLVM_ENABLE_ZLIB
 
+/* Define if zstd compression is available */
+#cmakedefine01 LLVM_ENABLE_ZSTD
+
 /* Define if LLVM was built with a dependency to the libtensorflow dynamic library */
 #cmakedefine LLVM_HAVE_TF_API
 
Index: llvm/unittests/Support/CompressionTest.cpp
===
--- llvm/unittests/Support/CompressionTest.cpp
+++ llvm/unittests/Support/CompressionTest.cpp
@@ -11,13 +11,16 @@
 //===--===//
 
 #include "llvm/Support/Compression.h"
+#include "llvm/ADT/ArrayRef.h"
 #include "llvm/ADT/SmallString.h"
 #include "llvm/ADT/StringRef.h"
 #include "llvm/Config/config.h"
+#include "llvm/Support/CRC.h"
 #include "llvm/Support/Error.h"
 #include "gtest/gtest.h"
 
 using namespace llvm;
+using namespace compression;
 
 namespace {
 
@@ -63,9 +66,54 @@
 }
 
 TEST(CompressionTest, ZlibCRC32) {
+  const char* StringString = strdup("The quick brown fox jumps over the lazy dog");
+  const ArrayRef StringByteArray = ArrayRef(*StringString);
   EXPECT_EQ(
   0x414FA339U,
-  zlib::crc32(StringRef("The quick brown fox jumps over the lazy dog")));
+  llvm::crc32(StringByteArray));
+}
+
+#endif
+
+#if LLVM_ENABLE_ZSTD
+
+void TestZstdCompression(StringRef Input, int Level) {
+  SmallString<32> Compressed;
+  SmallString<32> Uncompressed;
+
+  zstd::compress(Input, Compressed, Level);
+
+  // Check that uncompressed buffer is the same as original.
+  Error E = zstd::uncompress(Compressed, Uncompressed, Input.size());
+  consumeError(std::move(E));
+
+  EXPECT_EQ(Input, Uncompressed);
+  if (Input.size() > 0) {
+// Uncompression fails if expected length is too short.
+E = zstd::uncompress(Compressed, Uncompressed, Input.size() - 1);
+EXPECT_EQ("zstd error: Z_BUF_ERROR", llvm::toString(std::move(E)));
+  }
+}
+
+TEST(CompressionTest, Zstd) {
+  TestZstdCompression("", zstd::DefaultCompression);
+
+  TestZstdCompression("hello, world!", zstd::NoCompression);
+  TestZstdCompression("hello, world!", zstd::BestSizeCompression);
+  TestZstdCompression("hello, world!", zstd::BestSpeedCompression);
+  TestZstdCompression("hello, world!", zstd::DefaultCompression);
+
+  const size_t kSize = 1024;
+  char BinaryData[kSize];
+  for (size_t i = 0; i < kSize; ++i) {
+BinaryData[i] = i & 255;
+  }
+  StringRef BinaryDataStr(BinaryData, kSize);
+
+  TestZstdCompression(BinaryDataStr, zstd::NoCompression);
+  TestZstdCompression(BinaryDataStr, zstd::BestSizeCompression);
+  TestZstdCompression(BinaryDataStr, zstd::BestSpeedCompression);
+  TestZstdCompression(BinaryDataStr, zstd::DefaultCompression);
 }
 
 #endif
Index: llvm/unittests/Profile

[Lldb-commits] [PATCH] D128465: Zstandard as a second compression method to LLVM

[Cole Kissane via Phabricator via lldb-commits]

ckissane updated this revision to Diff 440298.
ckissane added a comment.

fix type for FindZstd cmake usage


Repository:
  rG LLVM Github Monorepo

CHANGES SINCE LAST ACTION
  https://reviews.llvm.org/D128465/new/

https://reviews.llvm.org/D128465

Files:
  clang-tools-extra/clangd/CMakeLists.txt
  clang-tools-extra/clangd/index/Serialization.cpp
  clang-tools-extra/clangd/test/lit.site.cfg.py.in
  clang-tools-extra/clangd/unittests/SerializationTests.cpp
  clang/cmake/caches/Apple-stage1.cmake
  clang/cmake/caches/Apple-stage2.cmake
  clang/cmake/caches/Fuchsia-stage2.cmake
  clang/cmake/caches/Fuchsia.cmake
  clang/lib/Driver/ToolChains/Clang.cpp
  clang/lib/Serialization/ASTReader.cpp
  clang/lib/Serialization/ASTWriter.cpp
  clang/test/CMakeLists.txt
  clang/test/lit.site.cfg.py.in
  compiler-rt/lib/sanitizer_common/symbolizer/scripts/build_symbolizer.sh
  compiler-rt/test/lit.common.configured.in
  flang/CMakeLists.txt
  lld/ELF/CMakeLists.txt
  lld/ELF/Driver.cpp
  lld/ELF/InputSection.cpp
  lldb/source/Plugins/Process/gdb-remote/CMakeLists.txt
  llvm/CMakeLists.txt
  llvm/cmake/config-ix.cmake
  llvm/cmake/modules/FindZSTD.cmake
  llvm/cmake/modules/LLVMConfig.cmake.in
  llvm/include/llvm/Config/llvm-config.h.cmake
  llvm/include/llvm/Support/Compression.h
  llvm/lib/MC/ELFObjectWriter.cpp
  llvm/lib/ObjCopy/ELF/ELFObject.cpp
  llvm/lib/Object/Decompressor.cpp
  llvm/lib/ProfileData/Coverage/CoverageMappingReader.cpp
  llvm/lib/ProfileData/Coverage/CoverageMappingWriter.cpp
  llvm/lib/ProfileData/InstrProf.cpp
  llvm/lib/ProfileData/SampleProfReader.cpp
  llvm/lib/ProfileData/SampleProfWriter.cpp
  llvm/lib/Support/CMakeLists.txt
  llvm/lib/Support/Compression.cpp
  llvm/tools/llvm-mc/llvm-mc.cpp
  llvm/tools/llvm-objcopy/ObjcopyOptions.cpp
  llvm/unittests/ProfileData/InstrProfTest.cpp
  llvm/unittests/Support/CompressionTest.cpp
  utils/bazel/llvm_configs/llvm-config.h.cmake

Index: utils/bazel/llvm_configs/llvm-config.h.cmake
===
--- utils/bazel/llvm_configs/llvm-config.h.cmake
+++ utils/bazel/llvm_configs/llvm-config.h.cmake
@@ -92,6 +92,9 @@
 /* Define if zlib compression is available */
 #cmakedefine01 LLVM_ENABLE_ZLIB
 
+/* Define if zstd compression is available */
+#cmakedefine01 LLVM_ENABLE_ZSTD
+
 /* Define if LLVM was built with a dependency to the libtensorflow dynamic library */
 #cmakedefine LLVM_HAVE_TF_API
 
Index: llvm/unittests/Support/CompressionTest.cpp
===
--- llvm/unittests/Support/CompressionTest.cpp
+++ llvm/unittests/Support/CompressionTest.cpp
@@ -11,13 +11,16 @@
 //===--===//
 
 #include "llvm/Support/Compression.h"
+#include "llvm/ADT/ArrayRef.h"
 #include "llvm/ADT/SmallString.h"
 #include "llvm/ADT/StringRef.h"
 #include "llvm/Config/config.h"
+#include "llvm/Support/CRC.h"
 #include "llvm/Support/Error.h"
 #include "gtest/gtest.h"
 
 using namespace llvm;
+using namespace compression;
 
 namespace {
 
@@ -63,9 +66,54 @@
 }
 
 TEST(CompressionTest, ZlibCRC32) {
+  const char* StringString = strdup("The quick brown fox jumps over the lazy dog");
+  const ArrayRef StringByteArray = ArrayRef(*StringString);
   EXPECT_EQ(
   0x414FA339U,
-  zlib::crc32(StringRef("The quick brown fox jumps over the lazy dog")));
+  llvm::crc32(StringByteArray));
+}
+
+#endif
+
+#if LLVM_ENABLE_ZSTD
+
+void TestZstdCompression(StringRef Input, int Level) {
+  SmallString<32> Compressed;
+  SmallString<32> Uncompressed;
+
+  zstd::compress(Input, Compressed, Level);
+
+  // Check that uncompressed buffer is the same as original.
+  Error E = zstd::uncompress(Compressed, Uncompressed, Input.size());
+  consumeError(std::move(E));
+
+  EXPECT_EQ(Input, Uncompressed);
+  if (Input.size() > 0) {
+// Uncompression fails if expected length is too short.
+E = zstd::uncompress(Compressed, Uncompressed, Input.size() - 1);
+EXPECT_EQ("zstd error: Z_BUF_ERROR", llvm::toString(std::move(E)));
+  }
+}
+
+TEST(CompressionTest, Zstd) {
+  TestZstdCompression("", zstd::DefaultCompression);
+
+  TestZstdCompression("hello, world!", zstd::NoCompression);
+  TestZstdCompression("hello, world!", zstd::BestSizeCompression);
+  TestZstdCompression("hello, world!", zstd::BestSpeedCompression);
+  TestZstdCompression("hello, world!", zstd::DefaultCompression);
+
+  const size_t kSize = 1024;
+  char BinaryData[kSize];
+  for (size_t i = 0; i < kSize; ++i) {
+BinaryData[i] = i & 255;
+  }
+  StringRef BinaryDataStr(BinaryData, kSize);
+
+  TestZstdCompression(BinaryDataStr, zstd::NoCompression);
+  TestZstdCompression(BinaryDataStr, zstd::BestSizeCompression);
+  TestZstdCompression(BinaryDataStr, zstd::BestSpeedCompression);
+  TestZstdCompression(BinaryDataStr, zstd::DefaultCompression);
 }
 
 #endif
Index: llvm/unittests/ProfileData/InstrProfTest.cpp
===

[Lldb-commits] [PATCH] D128465: Zstandard as a second compression method to LLVM

[Cole Kissane via Phabricator via lldb-commits]

ckissane updated this revision to Diff 440325.
ckissane added a comment.

more context


CHANGES SINCE LAST ACTION
  https://reviews.llvm.org/D128465/new/

https://reviews.llvm.org/D128465

Files:
  clang-tools-extra/clangd/CMakeLists.txt
  clang-tools-extra/clangd/index/Serialization.cpp
  clang-tools-extra/clangd/test/lit.site.cfg.py.in
  clang-tools-extra/clangd/unittests/SerializationTests.cpp
  clang/cmake/caches/Apple-stage1.cmake
  clang/cmake/caches/Apple-stage2.cmake
  clang/cmake/caches/Fuchsia-stage2.cmake
  clang/cmake/caches/Fuchsia.cmake
  clang/lib/Driver/ToolChains/Clang.cpp
  clang/lib/Serialization/ASTReader.cpp
  clang/lib/Serialization/ASTWriter.cpp
  clang/test/CMakeLists.txt
  clang/test/lit.site.cfg.py.in
  compiler-rt/lib/sanitizer_common/symbolizer/scripts/build_symbolizer.sh
  compiler-rt/test/lit.common.configured.in
  flang/CMakeLists.txt
  lld/ELF/CMakeLists.txt
  lld/ELF/Driver.cpp
  lld/ELF/InputSection.cpp
  lldb/source/Plugins/Process/gdb-remote/CMakeLists.txt
  llvm/CMakeLists.txt
  llvm/cmake/config-ix.cmake
  llvm/cmake/modules/FindZSTD.cmake
  llvm/cmake/modules/LLVMConfig.cmake.in
  llvm/include/llvm/Config/llvm-config.h.cmake
  llvm/include/llvm/Support/Compression.h
  llvm/lib/MC/ELFObjectWriter.cpp
  llvm/lib/ObjCopy/ELF/ELFObject.cpp
  llvm/lib/Object/Decompressor.cpp
  llvm/lib/ProfileData/Coverage/CoverageMappingReader.cpp
  llvm/lib/ProfileData/Coverage/CoverageMappingWriter.cpp
  llvm/lib/ProfileData/InstrProf.cpp
  llvm/lib/ProfileData/SampleProfReader.cpp
  llvm/lib/ProfileData/SampleProfWriter.cpp
  llvm/lib/Support/CMakeLists.txt
  llvm/lib/Support/Compression.cpp
  llvm/tools/llvm-mc/llvm-mc.cpp
  llvm/tools/llvm-objcopy/ObjcopyOptions.cpp
  llvm/unittests/ProfileData/InstrProfTest.cpp
  llvm/unittests/Support/CompressionTest.cpp
  utils/bazel/llvm_configs/llvm-config.h.cmake

Index: utils/bazel/llvm_configs/llvm-config.h.cmake
===
--- utils/bazel/llvm_configs/llvm-config.h.cmake
+++ utils/bazel/llvm_configs/llvm-config.h.cmake
@@ -92,6 +92,9 @@
 /* Define if zlib compression is available */
 #cmakedefine01 LLVM_ENABLE_ZLIB
 
+/* Define if zstd compression is available */
+#cmakedefine01 LLVM_ENABLE_ZSTD
+
 /* Define if LLVM was built with a dependency to the libtensorflow dynamic library */
 #cmakedefine LLVM_HAVE_TF_API
 
Index: llvm/unittests/Support/CompressionTest.cpp
===
--- llvm/unittests/Support/CompressionTest.cpp
+++ llvm/unittests/Support/CompressionTest.cpp
@@ -11,13 +11,16 @@
 //===--===//
 
 #include "llvm/Support/Compression.h"
+#include "llvm/ADT/ArrayRef.h"
 #include "llvm/ADT/SmallString.h"
 #include "llvm/ADT/StringRef.h"
 #include "llvm/Config/config.h"
+#include "llvm/Support/CRC.h"
 #include "llvm/Support/Error.h"
 #include "gtest/gtest.h"
 
 using namespace llvm;
+using namespace compression;
 
 namespace {
 
@@ -63,9 +66,54 @@
 }
 
 TEST(CompressionTest, ZlibCRC32) {
+  const char* StringString = strdup("The quick brown fox jumps over the lazy dog");
+  const ArrayRef StringByteArray = ArrayRef(*StringString);
   EXPECT_EQ(
   0x414FA339U,
-  zlib::crc32(StringRef("The quick brown fox jumps over the lazy dog")));
+  llvm::crc32(StringByteArray));
+}
+
+#endif
+
+#if LLVM_ENABLE_ZSTD
+
+void TestZstdCompression(StringRef Input, int Level) {
+  SmallString<32> Compressed;
+  SmallString<32> Uncompressed;
+
+  zstd::compress(Input, Compressed, Level);
+
+  // Check that uncompressed buffer is the same as original.
+  Error E = zstd::uncompress(Compressed, Uncompressed, Input.size());
+  consumeError(std::move(E));
+
+  EXPECT_EQ(Input, Uncompressed);
+  if (Input.size() > 0) {
+// Uncompression fails if expected length is too short.
+E = zstd::uncompress(Compressed, Uncompressed, Input.size() - 1);
+EXPECT_EQ("zstd error: Z_BUF_ERROR", llvm::toString(std::move(E)));
+  }
+}
+
+TEST(CompressionTest, Zstd) {
+  TestZstdCompression("", zstd::DefaultCompression);
+
+  TestZstdCompression("hello, world!", zstd::NoCompression);
+  TestZstdCompression("hello, world!", zstd::BestSizeCompression);
+  TestZstdCompression("hello, world!", zstd::BestSpeedCompression);
+  TestZstdCompression("hello, world!", zstd::DefaultCompression);
+
+  const size_t kSize = 1024;
+  char BinaryData[kSize];
+  for (size_t i = 0; i < kSize; ++i) {
+BinaryData[i] = i & 255;
+  }
+  StringRef BinaryDataStr(BinaryData, kSize);
+
+  TestZstdCompression(BinaryDataStr, zstd::NoCompression);
+  TestZstdCompression(BinaryDataStr, zstd::BestSizeCompression);
+  TestZstdCompression(BinaryDataStr, zstd::BestSpeedCompression);
+  TestZstdCompression(BinaryDataStr, zstd::DefaultCompression);
 }
 
 #endif
Index: llvm/unittests/ProfileData/InstrProfTest.cpp
===
--- llvm/unittests/ProfileData/Inst

[Lldb-commits] [PATCH] D128465: Zstandard as a second compression method to LLVM

[Cole Kissane via Phabricator via lldb-commits]

ckissane updated this revision to Diff 440327.
ckissane added a comment.

format


CHANGES SINCE LAST ACTION
  https://reviews.llvm.org/D128465/new/

https://reviews.llvm.org/D128465

Files:
  clang-tools-extra/clangd/CMakeLists.txt
  clang-tools-extra/clangd/index/Serialization.cpp
  clang-tools-extra/clangd/test/lit.site.cfg.py.in
  clang-tools-extra/clangd/unittests/SerializationTests.cpp
  clang/cmake/caches/Apple-stage1.cmake
  clang/cmake/caches/Apple-stage2.cmake
  clang/cmake/caches/Fuchsia-stage2.cmake
  clang/cmake/caches/Fuchsia.cmake
  clang/lib/Driver/ToolChains/Clang.cpp
  clang/lib/Serialization/ASTReader.cpp
  clang/lib/Serialization/ASTWriter.cpp
  clang/test/CMakeLists.txt
  clang/test/lit.site.cfg.py.in
  compiler-rt/lib/sanitizer_common/symbolizer/scripts/build_symbolizer.sh
  compiler-rt/test/lit.common.configured.in
  flang/CMakeLists.txt
  lld/ELF/CMakeLists.txt
  lld/ELF/Driver.cpp
  lld/ELF/InputSection.cpp
  lldb/source/Plugins/Process/gdb-remote/CMakeLists.txt
  llvm/CMakeLists.txt
  llvm/cmake/config-ix.cmake
  llvm/cmake/modules/FindZSTD.cmake
  llvm/cmake/modules/LLVMConfig.cmake.in
  llvm/include/llvm/Config/llvm-config.h.cmake
  llvm/include/llvm/Support/Compression.h
  llvm/lib/MC/ELFObjectWriter.cpp
  llvm/lib/ObjCopy/ELF/ELFObject.cpp
  llvm/lib/Object/Decompressor.cpp
  llvm/lib/ProfileData/Coverage/CoverageMappingReader.cpp
  llvm/lib/ProfileData/Coverage/CoverageMappingWriter.cpp
  llvm/lib/ProfileData/InstrProf.cpp
  llvm/lib/ProfileData/SampleProfReader.cpp
  llvm/lib/ProfileData/SampleProfWriter.cpp
  llvm/lib/Support/CMakeLists.txt
  llvm/lib/Support/Compression.cpp
  llvm/tools/llvm-mc/llvm-mc.cpp
  llvm/tools/llvm-objcopy/ObjcopyOptions.cpp
  llvm/unittests/ProfileData/InstrProfTest.cpp
  llvm/unittests/Support/CompressionTest.cpp
  utils/bazel/llvm_configs/llvm-config.h.cmake

Index: utils/bazel/llvm_configs/llvm-config.h.cmake
===
--- utils/bazel/llvm_configs/llvm-config.h.cmake
+++ utils/bazel/llvm_configs/llvm-config.h.cmake
@@ -92,6 +92,9 @@
 /* Define if zlib compression is available */
 #cmakedefine01 LLVM_ENABLE_ZLIB
 
+/* Define if zstd compression is available */
+#cmakedefine01 LLVM_ENABLE_ZSTD
+
 /* Define if LLVM was built with a dependency to the libtensorflow dynamic library */
 #cmakedefine LLVM_HAVE_TF_API
 
Index: llvm/unittests/Support/CompressionTest.cpp
===
--- llvm/unittests/Support/CompressionTest.cpp
+++ llvm/unittests/Support/CompressionTest.cpp
@@ -11,13 +11,16 @@
 //===--===//
 
 #include "llvm/Support/Compression.h"
+#include "llvm/ADT/ArrayRef.h"
 #include "llvm/ADT/SmallString.h"
 #include "llvm/ADT/StringRef.h"
 #include "llvm/Config/config.h"
+#include "llvm/Support/CRC.h"
 #include "llvm/Support/Error.h"
 #include "gtest/gtest.h"
 
 using namespace llvm;
+using namespace compression;
 
 namespace {
 
@@ -63,9 +66,53 @@
 }
 
 TEST(CompressionTest, ZlibCRC32) {
-  EXPECT_EQ(
-  0x414FA339U,
-  zlib::crc32(StringRef("The quick brown fox jumps over the lazy dog")));
+  const char *StringString =
+  strdup("The quick brown fox jumps over the lazy dog");
+  const ArrayRef StringByteArray = ArrayRef(*StringString);
+  EXPECT_EQ(0x414FA339U, llvm::crc32(StringByteArray));
+}
+
+#endif
+
+#if LLVM_ENABLE_ZSTD
+
+void TestZstdCompression(StringRef Input, int Level) {
+  SmallString<32> Compressed;
+  SmallString<32> Uncompressed;
+
+  zstd::compress(Input, Compressed, Level);
+
+  // Check that uncompressed buffer is the same as original.
+  Error E = zstd::uncompress(Compressed, Uncompressed, Input.size());
+  consumeError(std::move(E));
+
+  EXPECT_EQ(Input, Uncompressed);
+  if (Input.size() > 0) {
+// Uncompression fails if expected length is too short.
+E = zstd::uncompress(Compressed, Uncompressed, Input.size() - 1);
+EXPECT_EQ("zstd error: Z_BUF_ERROR", llvm::toString(std::move(E)));
+  }
+}
+
+TEST(CompressionTest, Zstd) {
+  TestZstdCompression("", zstd::DefaultCompression);
+
+  TestZstdCompression("hello, world!", zstd::NoCompression);
+  TestZstdCompression("hello, world!", zstd::BestSizeCompression);
+  TestZstdCompression("hello, world!", zstd::BestSpeedCompression);
+  TestZstdCompression("hello, world!", zstd::DefaultCompression);
+
+  const size_t kSize = 1024;
+  char BinaryData[kSize];
+  for (size_t i = 0; i < kSize; ++i) {
+BinaryData[i] = i & 255;
+  }
+  StringRef BinaryDataStr(BinaryData, kSize);
+
+  TestZstdCompression(BinaryDataStr, zstd::NoCompression);
+  TestZstdCompression(BinaryDataStr, zstd::BestSizeCompression);
+  TestZstdCompression(BinaryDataStr, zstd::BestSpeedCompression);
+  TestZstdCompression(BinaryDataStr, zstd::DefaultCompression);
 }
 
 #endif
Index: llvm/unittests/ProfileData/InstrProfTest.cpp
===
--- llvm/unitte

[Lldb-commits] [PATCH] D128465: Zstandard as a second compression method to LLVM

[Cole Kissane via Phabricator via lldb-commits]

ckissane updated this revision to Diff 440747.
ckissane edited the summary of this revision.
ckissane added a comment.

make it the second stage


CHANGES SINCE LAST ACTION
  https://reviews.llvm.org/D128465/new/

https://reviews.llvm.org/D128465

Files:
  clang-tools-extra/clangd/CMakeLists.txt
  clang-tools-extra/clangd/test/lit.site.cfg.py.in
  clang/test/CMakeLists.txt
  clang/test/lit.site.cfg.py.in
  compiler-rt/lib/sanitizer_common/symbolizer/scripts/build_symbolizer.sh
  compiler-rt/test/lit.common.configured.in
  flang/CMakeLists.txt
  lld/ELF/CMakeLists.txt
  lldb/source/Plugins/Process/gdb-remote/CMakeLists.txt
  llvm/CMakeLists.txt
  llvm/cmake/config-ix.cmake
  llvm/cmake/modules/FindZSTD.cmake
  llvm/cmake/modules/LLVMConfig.cmake.in
  llvm/include/llvm/Config/llvm-config.h.cmake
  llvm/include/llvm/Support/Compression.h
  llvm/lib/Support/Compression.cpp
  utils/bazel/llvm_configs/llvm-config.h.cmake

Index: utils/bazel/llvm_configs/llvm-config.h.cmake
===
--- utils/bazel/llvm_configs/llvm-config.h.cmake
+++ utils/bazel/llvm_configs/llvm-config.h.cmake
@@ -92,6 +92,9 @@
 /* Define if zlib compression is available */
 #cmakedefine01 LLVM_ENABLE_ZLIB
 
+/* Define if zstd compression is available */
+#cmakedefine01 LLVM_ENABLE_ZSTD
+
 /* Define if LLVM was built with a dependency to the libtensorflow dynamic library */
 #cmakedefine LLVM_HAVE_TF_API
 
Index: llvm/lib/Support/Compression.cpp
===
--- llvm/lib/Support/Compression.cpp
+++ llvm/lib/Support/Compression.cpp
@@ -20,6 +20,9 @@
 #if LLVM_ENABLE_ZLIB
 #include 
 #endif
+#if LLVM_ENABLE_ZSTD
+#include 
+#endif
 
 using namespace llvm;
 
@@ -102,3 +105,63 @@
   llvm_unreachable("zlib::uncompress is unavailable");
 }
 #endif
+
+#if LLVM_ENABLE_ZSTD
+
+bool zstd::isAvailable() { return true; }
+
+void zstd::compress(StringRef InputBuffer,
+SmallVectorImpl &CompressedBuffer, int Level) {
+  unsigned long CompressedBufferSize = ::ZSTD_compressBound(InputBuffer.size());
+  CompressedBuffer.resize_for_overwrite(CompressedBufferSize);
+  unsigned long CompressedSize = ::ZSTD_compress(
+  (Bytef *)CompressedBuffer.data(), CompressedBufferSize,
+  (const Bytef *)InputBuffer.data(), InputBuffer.size(), Level);
+  if (ZSTD_isError(CompressedSize))
+report_bad_alloc_error("Allocation failed");
+  // Tell MemorySanitizer that zstd output buffer is fully initialized.
+  // This avoids a false report when running LLVM with uninstrumented ZLib.
+  __msan_unpoison(CompressedBuffer.data(), CompressedSize);
+  CompressedBuffer.truncate(CompressedSize);
+}
+
+Error zstd::uncompress(StringRef InputBuffer, char *UncompressedBuffer,
+   size_t &UncompressedSize) {
+  unsigned long long const rSize = ZSTD_getFrameContentSize(
+  (const Bytef *)InputBuffer.data(), InputBuffer.size());
+  size_t const Res =
+  ::ZSTD_decompress((Bytef *)UncompressedBuffer, rSize,
+(const Bytef *)InputBuffer.data(), InputBuffer.size());
+  UncompressedSize = Res;
+  // Tell MemorySanitizer that zstd output buffer is fully initialized.
+  // This avoids a false report when running LLVM with uninstrumented ZLib.
+  __msan_unpoison(UncompressedBuffer, UncompressedSize);
+  return Res != rSize ? createError(ZSTD_getErrorName(Res)) : Error::success();
+}
+
+Error zstd::uncompress(StringRef InputBuffer,
+   SmallVectorImpl &UncompressedBuffer,
+   size_t UncompressedSize) {
+  UncompressedBuffer.resize_for_overwrite(UncompressedSize);
+  Error E = zstd::uncompress(InputBuffer, UncompressedBuffer.data(),
+ UncompressedSize);
+  UncompressedBuffer.truncate(UncompressedSize);
+  return E;
+}
+
+#else
+bool zstd::isAvailable() { return false; }
+void zstd::compress(StringRef InputBuffer,
+SmallVectorImpl &CompressedBuffer, int Level) {
+  llvm_unreachable("zstd::compress is unavailable");
+}
+Error zstd::uncompress(StringRef InputBuffer, char *UncompressedBuffer,
+   size_t &UncompressedSize) {
+  llvm_unreachable("zstd::uncompress is unavailable");
+}
+Error zstd::uncompress(StringRef InputBuffer,
+   SmallVectorImpl &UncompressedBuffer,
+   size_t UncompressedSize) {
+  llvm_unreachable("zstd::uncompress is unavailable");
+}
+#endif
\ No newline at end of file
Index: llvm/include/llvm/Support/Compression.h
===
--- llvm/include/llvm/Support/Compression.h
+++ llvm/include/llvm/Support/Compression.h
@@ -43,7 +43,32 @@
 
 } // End of namespace zlib
 
+namespace zstd {
+
+static constexpr int NoCompression = -5;
+static constexpr int BestSpeedCompression = 1;
+static constexpr int DefaultCompression = 5;
+static constexpr int BestSizeCompression = 12;
+
+bool isAvailable();
+
+voi

[Lldb-commits] [PATCH] D128465: Zstandard as a second compression method to LLVM

[Cole Kissane via Phabricator via lldb-commits]

ckissane updated this revision to Diff 441102.
ckissane added a comment.

add AlgorithmName to compression namespaces


CHANGES SINCE LAST ACTION
  https://reviews.llvm.org/D128465/new/

https://reviews.llvm.org/D128465

Files:
  clang-tools-extra/clangd/CMakeLists.txt
  clang-tools-extra/clangd/test/lit.site.cfg.py.in
  clang/test/CMakeLists.txt
  clang/test/lit.site.cfg.py.in
  compiler-rt/lib/sanitizer_common/symbolizer/scripts/build_symbolizer.sh
  compiler-rt/test/lit.common.configured.in
  flang/CMakeLists.txt
  lld/ELF/CMakeLists.txt
  lldb/source/Plugins/Process/gdb-remote/CMakeLists.txt
  llvm/CMakeLists.txt
  llvm/cmake/config-ix.cmake
  llvm/cmake/modules/FindZSTD.cmake
  llvm/cmake/modules/LLVMConfig.cmake.in
  llvm/include/llvm/Config/llvm-config.h.cmake
  llvm/include/llvm/Support/Compression.h
  llvm/lib/Support/Compression.cpp
  utils/bazel/llvm_configs/llvm-config.h.cmake

Index: utils/bazel/llvm_configs/llvm-config.h.cmake
===
--- utils/bazel/llvm_configs/llvm-config.h.cmake
+++ utils/bazel/llvm_configs/llvm-config.h.cmake
@@ -92,6 +92,9 @@
 /* Define if zlib compression is available */
 #cmakedefine01 LLVM_ENABLE_ZLIB
 
+/* Define if zstd compression is available */
+#cmakedefine01 LLVM_ENABLE_ZSTD
+
 /* Define if LLVM was built with a dependency to the libtensorflow dynamic library */
 #cmakedefine LLVM_HAVE_TF_API
 
Index: llvm/lib/Support/Compression.cpp
===
--- llvm/lib/Support/Compression.cpp
+++ llvm/lib/Support/Compression.cpp
@@ -20,6 +20,9 @@
 #if LLVM_ENABLE_ZLIB
 #include 
 #endif
+#if LLVM_ENABLE_ZSTD
+#include 
+#endif
 
 using namespace llvm;
 
@@ -102,3 +105,63 @@
   llvm_unreachable("zlib::uncompress is unavailable");
 }
 #endif
+
+#if LLVM_ENABLE_ZSTD
+
+bool zstd::isAvailable() { return true; }
+
+void zstd::compress(StringRef InputBuffer,
+SmallVectorImpl &CompressedBuffer, int Level) {
+  unsigned long CompressedBufferSize = ::ZSTD_compressBound(InputBuffer.size());
+  CompressedBuffer.resize_for_overwrite(CompressedBufferSize);
+  unsigned long CompressedSize = ::ZSTD_compress(
+  (Bytef *)CompressedBuffer.data(), CompressedBufferSize,
+  (const Bytef *)InputBuffer.data(), InputBuffer.size(), Level);
+  if (ZSTD_isError(CompressedSize))
+report_bad_alloc_error("Allocation failed");
+  // Tell MemorySanitizer that zstd output buffer is fully initialized.
+  // This avoids a false report when running LLVM with uninstrumented ZLib.
+  __msan_unpoison(CompressedBuffer.data(), CompressedSize);
+  CompressedBuffer.truncate(CompressedSize);
+}
+
+Error zstd::uncompress(StringRef InputBuffer, char *UncompressedBuffer,
+   size_t &UncompressedSize) {
+  unsigned long long const rSize = ZSTD_getFrameContentSize(
+  (const Bytef *)InputBuffer.data(), InputBuffer.size());
+  size_t const Res =
+  ::ZSTD_decompress((Bytef *)UncompressedBuffer, rSize,
+(const Bytef *)InputBuffer.data(), InputBuffer.size());
+  UncompressedSize = Res;
+  // Tell MemorySanitizer that zstd output buffer is fully initialized.
+  // This avoids a false report when running LLVM with uninstrumented ZLib.
+  __msan_unpoison(UncompressedBuffer, UncompressedSize);
+  return Res != rSize ? createError(ZSTD_getErrorName(Res)) : Error::success();
+}
+
+Error zstd::uncompress(StringRef InputBuffer,
+   SmallVectorImpl &UncompressedBuffer,
+   size_t UncompressedSize) {
+  UncompressedBuffer.resize_for_overwrite(UncompressedSize);
+  Error E = zstd::uncompress(InputBuffer, UncompressedBuffer.data(),
+ UncompressedSize);
+  UncompressedBuffer.truncate(UncompressedSize);
+  return E;
+}
+
+#else
+bool zstd::isAvailable() { return false; }
+void zstd::compress(StringRef InputBuffer,
+SmallVectorImpl &CompressedBuffer, int Level) {
+  llvm_unreachable("zstd::compress is unavailable");
+}
+Error zstd::uncompress(StringRef InputBuffer, char *UncompressedBuffer,
+   size_t &UncompressedSize) {
+  llvm_unreachable("zstd::uncompress is unavailable");
+}
+Error zstd::uncompress(StringRef InputBuffer,
+   SmallVectorImpl &UncompressedBuffer,
+   size_t UncompressedSize) {
+  llvm_unreachable("zstd::uncompress is unavailable");
+}
+#endif
\ No newline at end of file
Index: llvm/include/llvm/Support/Compression.h
===
--- llvm/include/llvm/Support/Compression.h
+++ llvm/include/llvm/Support/Compression.h
@@ -44,7 +44,33 @@
 
 } // End of namespace zlib
 
+namespace zstd {
+
+static constexpr std::string AlgorithmName = "zstd";
+static constexpr int NoCompression = -5;
+static constexpr int BestSpeedCompression = 1;
+static constexpr int DefaultCompression = 5;
+static constexpr int BestSizeCompression = 12;
+

[Lldb-commits] [PATCH] D128465: Zstandard as a second compression method to LLVM

[Cole Kissane via Phabricator via lldb-commits]

ckissane updated this revision to Diff 441114.
ckissane edited the summary of this revision.
ckissane added a comment.

- feat: add zstd compression
- Merge branch 'ckissane.refactor-compression-namespace' into 
ckissane.add-zstd-compression
- Merge branch 'ckissane.refactor-compression-namespace' into 
ckissane.add-zstd-compression


Repository:
  rG LLVM Github Monorepo

CHANGES SINCE LAST ACTION
  https://reviews.llvm.org/D128465/new/

https://reviews.llvm.org/D128465

Files:
  clang-tools-extra/clangd/CMakeLists.txt
  clang-tools-extra/clangd/test/lit.site.cfg.py.in
  clang/test/CMakeLists.txt
  clang/test/lit.site.cfg.py.in
  compiler-rt/lib/sanitizer_common/symbolizer/scripts/build_symbolizer.sh
  compiler-rt/test/lit.common.configured.in
  flang/CMakeLists.txt
  lld/ELF/CMakeLists.txt
  lldb/source/Plugins/Process/gdb-remote/CMakeLists.txt
  llvm/CMakeLists.txt
  llvm/cmake/config-ix.cmake
  llvm/cmake/modules/FindZSTD.cmake
  llvm/cmake/modules/LLVMConfig.cmake.in
  llvm/include/llvm/Config/llvm-config.h.cmake
  llvm/include/llvm/Support/Compression.h
  llvm/lib/Support/Compression.cpp
  utils/bazel/llvm_configs/llvm-config.h.cmake

Index: utils/bazel/llvm_configs/llvm-config.h.cmake
===
--- utils/bazel/llvm_configs/llvm-config.h.cmake
+++ utils/bazel/llvm_configs/llvm-config.h.cmake
@@ -92,6 +92,9 @@
 /* Define if zlib compression is available */
 #cmakedefine01 LLVM_ENABLE_ZLIB
 
+/* Define if zstd compression is available */
+#cmakedefine01 LLVM_ENABLE_ZSTD
+
 /* Define if LLVM was built with a dependency to the libtensorflow dynamic library */
 #cmakedefine LLVM_HAVE_TF_API
 
Index: llvm/lib/Support/Compression.cpp
===
--- llvm/lib/Support/Compression.cpp
+++ llvm/lib/Support/Compression.cpp
@@ -20,6 +20,9 @@
 #if LLVM_ENABLE_ZLIB
 #include 
 #endif
+#if LLVM_ENABLE_ZSTD
+#include 
+#endif
 
 using namespace llvm;
 
@@ -102,3 +105,63 @@
   llvm_unreachable("zlib::uncompress is unavailable");
 }
 #endif
+
+#if LLVM_ENABLE_ZSTD
+
+bool zstd::isAvailable() { return true; }
+
+void zstd::compress(StringRef InputBuffer,
+SmallVectorImpl &CompressedBuffer, int Level) {
+  unsigned long CompressedBufferSize = ::ZSTD_compressBound(InputBuffer.size());
+  CompressedBuffer.resize_for_overwrite(CompressedBufferSize);
+  unsigned long CompressedSize = ::ZSTD_compress(
+  (Bytef *)CompressedBuffer.data(), CompressedBufferSize,
+  (const Bytef *)InputBuffer.data(), InputBuffer.size(), Level);
+  if (ZSTD_isError(CompressedSize))
+report_bad_alloc_error("Allocation failed");
+  // Tell MemorySanitizer that zstd output buffer is fully initialized.
+  // This avoids a false report when running LLVM with uninstrumented ZLib.
+  __msan_unpoison(CompressedBuffer.data(), CompressedSize);
+  CompressedBuffer.truncate(CompressedSize);
+}
+
+Error zstd::uncompress(StringRef InputBuffer, char *UncompressedBuffer,
+   size_t &UncompressedSize) {
+  unsigned long long const rSize = ZSTD_getFrameContentSize(
+  (const Bytef *)InputBuffer.data(), InputBuffer.size());
+  size_t const Res =
+  ::ZSTD_decompress((Bytef *)UncompressedBuffer, rSize,
+(const Bytef *)InputBuffer.data(), InputBuffer.size());
+  UncompressedSize = Res;
+  // Tell MemorySanitizer that zstd output buffer is fully initialized.
+  // This avoids a false report when running LLVM with uninstrumented ZLib.
+  __msan_unpoison(UncompressedBuffer, UncompressedSize);
+  return Res != rSize ? createError(ZSTD_getErrorName(Res)) : Error::success();
+}
+
+Error zstd::uncompress(StringRef InputBuffer,
+   SmallVectorImpl &UncompressedBuffer,
+   size_t UncompressedSize) {
+  UncompressedBuffer.resize_for_overwrite(UncompressedSize);
+  Error E = zstd::uncompress(InputBuffer, UncompressedBuffer.data(),
+ UncompressedSize);
+  UncompressedBuffer.truncate(UncompressedSize);
+  return E;
+}
+
+#else
+bool zstd::isAvailable() { return false; }
+void zstd::compress(StringRef InputBuffer,
+SmallVectorImpl &CompressedBuffer, int Level) {
+  llvm_unreachable("zstd::compress is unavailable");
+}
+Error zstd::uncompress(StringRef InputBuffer, char *UncompressedBuffer,
+   size_t &UncompressedSize) {
+  llvm_unreachable("zstd::uncompress is unavailable");
+}
+Error zstd::uncompress(StringRef InputBuffer,
+   SmallVectorImpl &UncompressedBuffer,
+   size_t UncompressedSize) {
+  llvm_unreachable("zstd::uncompress is unavailable");
+}
+#endif
\ No newline at end of file
Index: llvm/include/llvm/Support/Compression.h
===
--- llvm/include/llvm/Support/Compression.h
+++ llvm/include/llvm/Support/Compression.h
@@ -44,7 +44,33 @@
 
 } // End of namespace zlib
 
+nam

[Lldb-commits] [PATCH] D128465: Zstandard as a second compression method to LLVM

[Cole Kissane via Phabricator via lldb-commits]

ckissane updated this revision to Diff 443035.
ckissane added a comment.

start breaking this up further, this is now the add cmake config patch


Repository:
  rG LLVM Github Monorepo

CHANGES SINCE LAST ACTION
  https://reviews.llvm.org/D128465/new/

https://reviews.llvm.org/D128465

Files:
  clang-tools-extra/clangd/CMakeLists.txt
  clang-tools-extra/clangd/test/lit.site.cfg.py.in
  clang/test/CMakeLists.txt
  clang/test/lit.site.cfg.py.in
  compiler-rt/lib/sanitizer_common/symbolizer/scripts/build_symbolizer.sh
  compiler-rt/test/lit.common.configured.in
  flang/CMakeLists.txt
  lld/ELF/CMakeLists.txt
  lldb/source/Plugins/Process/gdb-remote/CMakeLists.txt
  llvm/CMakeLists.txt
  llvm/cmake/config-ix.cmake
  llvm/cmake/modules/FindZSTD.cmake
  llvm/cmake/modules/LLVMConfig.cmake.in
  llvm/include/llvm/Config/llvm-config.h.cmake
  utils/bazel/llvm_configs/llvm-config.h.cmake

Index: utils/bazel/llvm_configs/llvm-config.h.cmake
===
--- utils/bazel/llvm_configs/llvm-config.h.cmake
+++ utils/bazel/llvm_configs/llvm-config.h.cmake
@@ -95,6 +95,9 @@
 /* Define if zlib compression is available */
 #cmakedefine01 LLVM_ENABLE_ZLIB
 
+/* Define if zstd compression is available */
+#cmakedefine01 LLVM_ENABLE_ZSTD
+
 /* Define if LLVM was built with a dependency to the libtensorflow dynamic library */
 #cmakedefine LLVM_HAVE_TF_API
 
Index: llvm/include/llvm/Config/llvm-config.h.cmake
===
--- llvm/include/llvm/Config/llvm-config.h.cmake
+++ llvm/include/llvm/Config/llvm-config.h.cmake
@@ -95,6 +95,9 @@
 /* Define if zlib compression is available */
 #cmakedefine01 LLVM_ENABLE_ZLIB
 
+/* Define if zstd compression is available */
+#cmakedefine01 LLVM_ENABLE_ZSTD
+
 /* Define if LLVM was built with a dependency to the libtensorflow dynamic library */
 #cmakedefine LLVM_HAVE_TF_API
 
Index: llvm/cmake/modules/LLVMConfig.cmake.in
===
--- llvm/cmake/modules/LLVMConfig.cmake.in
+++ llvm/cmake/modules/LLVMConfig.cmake.in
@@ -73,6 +73,12 @@
   find_package(ZLIB)
 endif()
 
+set(LLVM_ENABLE_ZSTD @LLVM_ENABLE_ZSTD@)
+if(LLVM_ENABLE_ZSTD)
+  set(ZSTD_ROOT @ZSTD_ROOT@)
+  find_package(ZSTD)
+endif()
+
 set(LLVM_ENABLE_LIBXML2 @LLVM_ENABLE_LIBXML2@)
 if(LLVM_ENABLE_LIBXML2)
   find_package(LibXml2)
Index: llvm/cmake/modules/FindZSTD.cmake
===
--- /dev/null
+++ llvm/cmake/modules/FindZSTD.cmake
@@ -0,0 +1,31 @@
+#
+#//===--===//
+#//
+#// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+#// See https://llvm.org/LICENSE.txt for license information.
+#// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+#//
+#//===--===//
+#
+
+find_path(ZSTD_INCLUDE_DIR
+  NAMES zstd.h
+  HINTS ${ZSTD_ROOT_DIR}/include)
+
+find_library(ZSTD_LIBRARY
+  NAMES zstd
+  HINTS ${ZSTD_ROOT_DIR}/lib)
+
+include(FindPackageHandleStandardArgs)
+find_package_handle_standard_args(
+ZSTD DEFAULT_MSG
+ZSTD_LIBRARY ZSTD_INCLUDE_DIR)
+
+if(ZSTD_FOUND)
+set(ZSTD_LIBRARIES ${ZSTD_LIBRARY})
+set(ZSTD_INCLUDE_DIRS ${ZSTD_INCLUDE_DIR})
+endif()
+
+mark_as_advanced(
+  ZSTD_INCLUDE_DIR
+  ZSTD_LIBRARY)
Index: llvm/cmake/config-ix.cmake
===
--- llvm/cmake/config-ix.cmake
+++ llvm/cmake/config-ix.cmake
@@ -136,6 +136,27 @@
   set(LLVM_ENABLE_ZLIB "${HAVE_ZLIB}")
 endif()
 
+if(LLVM_ENABLE_ZSTD)
+  if(LLVM_ENABLE_ZSTD STREQUAL FORCE_ON)
+find_package(ZSTD REQUIRED)
+  elseif(NOT LLVM_USE_SANITIZER MATCHES "Memory.*")
+find_package(ZSTD)
+  endif()
+  if(ZSTD_FOUND)
+# Check if zstd we found is usable; for example, we may have found a 32-bit
+# library on a 64-bit system which would result in a link-time failure.
+cmake_push_check_state()
+list(APPEND CMAKE_REQUIRED_INCLUDES ${ZSTD_INCLUDE_DIR})
+list(APPEND CMAKE_REQUIRED_LIBRARIES ${ZSTD_LIBRARY})
+check_symbol_exists(ZSTD_compress zstd.h HAVE_ZSTD)
+cmake_pop_check_state()
+if(LLVM_ENABLE_ZSTD STREQUAL FORCE_ON AND NOT HAVE_ZSTD)
+  message(FATAL_ERROR "Failed to configure zstd")
+endif()
+  endif()
+  set(LLVM_ENABLE_ZSTD "${HAVE_ZSTD}")
+endif()
+
 if(LLVM_ENABLE_LIBXML2)
   if(LLVM_ENABLE_LIBXML2 STREQUAL FORCE_ON)
 find_package(LibXml2 REQUIRED)
Index: llvm/CMakeLists.txt
===
--- llvm/CMakeLists.txt
+++ llvm/CMakeLists.txt
@@ -438,6 +438,8 @@
 
 set(LLVM_ENABLE_ZLIB "ON" CACHE STRING "Use zlib for compression/decompression if available. Can be ON, OFF, or FORCE_ON")
 
+set(LLVM_ENABLE_ZSTD "ON" CACHE STRING "Use zstd for compression/decompression if available. Can be ON, OFF, or FORCE_ON")
+
 set(LLVM

[Lldb-commits] [PATCH] D128465: [llvm] cmake config groundwork to have ZSTD in LLVM

[Cole Kissane via Phabricator via lldb-commits]

ckissane marked 2 inline comments as done.
ckissane added a comment.

mark outdated comments done


Repository:
  rG LLVM Github Monorepo

CHANGES SINCE LAST ACTION
  https://reviews.llvm.org/D128465/new/

https://reviews.llvm.org/D128465

___
lldb-commits mailing list
lldb-commits@lists.llvm.org
https://lists.llvm.org/cgi-bin/mailman/listinfo/lldb-commits

[Lldb-commits] [PATCH] D128465: [llvm] cmake config groundwork to have ZSTD in LLVM

[Cole Kissane via Phabricator via lldb-commits]

ckissane updated this revision to Diff 443310.
ckissane added a comment.

- Merge remote-tracking branch 'origin/main' into ckissane.add-zstd.0-cmake


Repository:
  rG LLVM Github Monorepo

CHANGES SINCE LAST ACTION
  https://reviews.llvm.org/D128465/new/

https://reviews.llvm.org/D128465

Files:
  clang-tools-extra/clangd/CMakeLists.txt
  clang-tools-extra/clangd/test/lit.cfg.py
  clang-tools-extra/clangd/test/lit.site.cfg.py.in
  clang/test/CMakeLists.txt
  clang/test/lit.site.cfg.py.in
  compiler-rt/lib/sanitizer_common/symbolizer/scripts/build_symbolizer.sh
  compiler-rt/test/lit.common.cfg.py
  compiler-rt/test/lit.common.configured.in
  flang/CMakeLists.txt
  lld/ELF/CMakeLists.txt
  lld/test/lit.site.cfg.py.in
  lldb/source/Plugins/Process/gdb-remote/CMakeLists.txt
  lldb/test/Shell/lit.site.cfg.py.in
  llvm/CMakeLists.txt
  llvm/cmake/config-ix.cmake
  llvm/cmake/modules/FindZSTD.cmake
  llvm/cmake/modules/LLVMConfig.cmake.in
  llvm/include/llvm/Config/llvm-config.h.cmake
  llvm/test/lit.site.cfg.py.in
  utils/bazel/llvm_configs/llvm-config.h.cmake

Index: utils/bazel/llvm_configs/llvm-config.h.cmake
===
--- utils/bazel/llvm_configs/llvm-config.h.cmake
+++ utils/bazel/llvm_configs/llvm-config.h.cmake
@@ -95,6 +95,9 @@
 /* Define if zlib compression is available */
 #cmakedefine01 LLVM_ENABLE_ZLIB
 
+/* Define if zstd compression is available */
+#cmakedefine01 LLVM_ENABLE_ZSTD
+
 /* Define if LLVM was built with a dependency to the libtensorflow dynamic library */
 #cmakedefine LLVM_HAVE_TF_API
 
Index: llvm/test/lit.site.cfg.py.in
===
--- llvm/test/lit.site.cfg.py.in
+++ llvm/test/lit.site.cfg.py.in
@@ -37,6 +37,7 @@
 config.llvm_use_intel_jitevents = @LLVM_USE_INTEL_JITEVENTS@
 config.llvm_use_sanitizer = "@LLVM_USE_SANITIZER@"
 config.have_zlib = @LLVM_ENABLE_ZLIB@
+config.have_zstd = @LLVM_ENABLE_ZSTD@
 config.have_libxar = @LLVM_HAVE_LIBXAR@
 config.have_libxml2 = @LLVM_ENABLE_LIBXML2@
 config.have_curl = @LLVM_ENABLE_CURL@
Index: llvm/include/llvm/Config/llvm-config.h.cmake
===
--- llvm/include/llvm/Config/llvm-config.h.cmake
+++ llvm/include/llvm/Config/llvm-config.h.cmake
@@ -95,6 +95,9 @@
 /* Define if zlib compression is available */
 #cmakedefine01 LLVM_ENABLE_ZLIB
 
+/* Define if zstd compression is available */
+#cmakedefine01 LLVM_ENABLE_ZSTD
+
 /* Define if LLVM was built with a dependency to the libtensorflow dynamic library */
 #cmakedefine LLVM_HAVE_TF_API
 
Index: llvm/cmake/modules/LLVMConfig.cmake.in
===
--- llvm/cmake/modules/LLVMConfig.cmake.in
+++ llvm/cmake/modules/LLVMConfig.cmake.in
@@ -73,6 +73,12 @@
   find_package(ZLIB)
 endif()
 
+set(LLVM_ENABLE_ZSTD @LLVM_ENABLE_ZSTD@)
+if(LLVM_ENABLE_ZSTD)
+  set(ZSTD_ROOT @ZSTD_ROOT@)
+  find_package(ZSTD)
+endif()
+
 set(LLVM_ENABLE_LIBXML2 @LLVM_ENABLE_LIBXML2@)
 if(LLVM_ENABLE_LIBXML2)
   find_package(LibXml2)
Index: llvm/cmake/modules/FindZSTD.cmake
===
--- /dev/null
+++ llvm/cmake/modules/FindZSTD.cmake
@@ -0,0 +1,21 @@
+find_path(ZSTD_INCLUDE_DIR
+  NAMES zstd.h
+  HINTS ${ZSTD_ROOT_DIR}/include)
+
+find_library(ZSTD_LIBRARY
+  NAMES zstd
+  HINTS ${ZSTD_ROOT_DIR}/lib)
+
+include(FindPackageHandleStandardArgs)
+find_package_handle_standard_args(
+ZSTD DEFAULT_MSG
+ZSTD_LIBRARY ZSTD_INCLUDE_DIR)
+
+if(ZSTD_FOUND)
+set(ZSTD_LIBRARIES ${ZSTD_LIBRARY})
+set(ZSTD_INCLUDE_DIRS ${ZSTD_INCLUDE_DIR})
+endif()
+
+mark_as_advanced(
+  ZSTD_INCLUDE_DIR
+  ZSTD_LIBRARY)
Index: llvm/cmake/config-ix.cmake
===
--- llvm/cmake/config-ix.cmake
+++ llvm/cmake/config-ix.cmake
@@ -136,6 +136,27 @@
   set(LLVM_ENABLE_ZLIB "${HAVE_ZLIB}")
 endif()
 
+if(LLVM_ENABLE_ZSTD)
+  if(LLVM_ENABLE_ZSTD STREQUAL FORCE_ON)
+find_package(ZSTD REQUIRED)
+  elseif(NOT LLVM_USE_SANITIZER MATCHES "Memory.*")
+find_package(ZSTD)
+  endif()
+  if(ZSTD_FOUND)
+# Check if zstd we found is usable; for example, we may have found a 32-bit
+# library on a 64-bit system which would result in a link-time failure.
+cmake_push_check_state()
+list(APPEND CMAKE_REQUIRED_INCLUDES ${ZSTD_INCLUDE_DIR})
+list(APPEND CMAKE_REQUIRED_LIBRARIES ${ZSTD_LIBRARY})
+check_symbol_exists(ZSTD_compress zstd.h HAVE_ZSTD)
+cmake_pop_check_state()
+if(LLVM_ENABLE_ZSTD STREQUAL FORCE_ON AND NOT HAVE_ZSTD)
+  message(FATAL_ERROR "Failed to configure zstd")
+endif()
+  endif()
+  set(LLVM_ENABLE_ZSTD "${HAVE_ZSTD}")
+endif()
+
 if(LLVM_ENABLE_LIBXML2)
   if(LLVM_ENABLE_LIBXML2 STREQUAL FORCE_ON)
 find_package(LibXml2 REQUIRED)
Index: llvm/CMakeLists.txt
===
--- llvm/CMakeLists.txt
+++ llvm/C

[Lldb-commits] [PATCH] D128465: [llvm] cmake config groundwork to have ZSTD in LLVM

[Cole Kissane via Phabricator via lldb-commits]

ckissane marked 2 inline comments as done.
ckissane added a comment.

marked fixed comments as done


Repository:
  rG LLVM Github Monorepo

CHANGES SINCE LAST ACTION
  https://reviews.llvm.org/D128465/new/

https://reviews.llvm.org/D128465

___
lldb-commits mailing list
lldb-commits@lists.llvm.org
https://lists.llvm.org/cgi-bin/mailman/listinfo/lldb-commits

[Lldb-commits] [PATCH] D128465: [llvm] cmake config groundwork to have ZSTD in LLVM

[Cole Kissane via Phabricator via lldb-commits]

ckissane updated this revision to Diff 443308.
ckissane added a comment.

- Merge remote-tracking branch 'origin/main' into ckissane.add-zstd.0-cmake
- added have_zstd to compiler-rt/test/lit.common.cfg.py, 
clang-tools-extra/clangd/test/lit.cfg.py and several lit.site.cfg.py.in files


Repository:
  rG LLVM Github Monorepo

CHANGES SINCE LAST ACTION
  https://reviews.llvm.org/D128465/new/

https://reviews.llvm.org/D128465

Files:
  clang-tools-extra/clangd/CMakeLists.txt
  clang-tools-extra/clangd/index/Serialization.cpp
  clang-tools-extra/clangd/test/lit.cfg.py
  clang-tools-extra/clangd/test/lit.site.cfg.py.in
  clang-tools-extra/clangd/unittests/SerializationTests.cpp
  clang/lib/Driver/ToolChains/Clang.cpp
  clang/lib/Serialization/ASTReader.cpp
  clang/lib/Serialization/ASTWriter.cpp
  clang/test/CMakeLists.txt
  clang/test/lit.site.cfg.py.in
  compiler-rt/lib/sanitizer_common/symbolizer/scripts/build_symbolizer.sh
  compiler-rt/test/lit.common.cfg.py
  compiler-rt/test/lit.common.configured.in
  flang/CMakeLists.txt
  lld/ELF/CMakeLists.txt
  lld/ELF/Driver.cpp
  lld/ELF/InputSection.cpp
  lld/test/lit.site.cfg.py.in
  lldb/source/Plugins/Process/gdb-remote/CMakeLists.txt
  lldb/test/Shell/lit.site.cfg.py.in
  llvm/CMakeLists.txt
  llvm/cmake/config-ix.cmake
  llvm/cmake/modules/FindZSTD.cmake
  llvm/cmake/modules/LLVMConfig.cmake.in
  llvm/docs/ReleaseNotes.rst
  llvm/include/llvm/Config/llvm-config.h.cmake
  llvm/include/llvm/Support/Compression.h
  llvm/lib/MC/ELFObjectWriter.cpp
  llvm/lib/ObjCopy/ELF/ELFObject.cpp
  llvm/lib/Object/Decompressor.cpp
  llvm/lib/ProfileData/Coverage/CoverageMappingReader.cpp
  llvm/lib/ProfileData/Coverage/CoverageMappingWriter.cpp
  llvm/lib/ProfileData/InstrProf.cpp
  llvm/lib/ProfileData/SampleProfReader.cpp
  llvm/lib/ProfileData/SampleProfWriter.cpp
  llvm/lib/Support/Compression.cpp
  llvm/test/lit.site.cfg.py.in
  llvm/tools/llvm-mc/llvm-mc.cpp
  llvm/tools/llvm-objcopy/ObjcopyOptions.cpp
  llvm/unittests/ProfileData/InstrProfTest.cpp
  llvm/unittests/Support/CompressionTest.cpp
  utils/bazel/llvm_configs/llvm-config.h.cmake

Index: utils/bazel/llvm_configs/llvm-config.h.cmake
===
--- utils/bazel/llvm_configs/llvm-config.h.cmake
+++ utils/bazel/llvm_configs/llvm-config.h.cmake
@@ -95,6 +95,9 @@
 /* Define if zlib compression is available */
 #cmakedefine01 LLVM_ENABLE_ZLIB
 
+/* Define if zstd compression is available */
+#cmakedefine01 LLVM_ENABLE_ZSTD
+
 /* Define if LLVM was built with a dependency to the libtensorflow dynamic library */
 #cmakedefine LLVM_HAVE_TF_API
 
Index: llvm/unittests/Support/CompressionTest.cpp
===
--- llvm/unittests/Support/CompressionTest.cpp
+++ llvm/unittests/Support/CompressionTest.cpp
@@ -18,6 +18,7 @@
 #include "gtest/gtest.h"
 
 using namespace llvm;
+using namespace llvm::compression;
 
 namespace {
 
@@ -62,12 +63,6 @@
   TestZlibCompression(BinaryDataStr, zlib::DefaultCompression);
 }
 
-TEST(CompressionTest, ZlibCRC32) {
-  EXPECT_EQ(
-  0x414FA339U,
-  zlib::crc32(StringRef("The quick brown fox jumps over the lazy dog")));
-}
-
 #endif
 
 }
Index: llvm/unittests/ProfileData/InstrProfTest.cpp
===
--- llvm/unittests/ProfileData/InstrProfTest.cpp
+++ llvm/unittests/ProfileData/InstrProfTest.cpp
@@ -1147,14 +1147,16 @@
 // Compressing:
 std::string FuncNameStrings1;
 EXPECT_THAT_ERROR(collectPGOFuncNameStrings(
-  FuncNames1, (DoCompression && zlib::isAvailable()),
+  FuncNames1,
+  (DoCompression && compression::zlib::isAvailable()),
   FuncNameStrings1),
   Succeeded());
 
 // Compressing:
 std::string FuncNameStrings2;
 EXPECT_THAT_ERROR(collectPGOFuncNameStrings(
-  FuncNames2, (DoCompression && zlib::isAvailable()),
+  FuncNames2,
+  (DoCompression && compression::zlib::isAvailable()),
   FuncNameStrings2),
   Succeeded());
 
Index: llvm/tools/llvm-objcopy/ObjcopyOptions.cpp
===
--- llvm/tools/llvm-objcopy/ObjcopyOptions.cpp
+++ llvm/tools/llvm-objcopy/ObjcopyOptions.cpp
@@ -739,7 +739,7 @@
 .str()
 .c_str());
 }
-if (!zlib::isAvailable())
+if (!compression::zlib::isAvailable())
   return createStringError(
   errc::invalid_argument,
   "LLVM was not compiled with LLVM_ENABLE_ZLIB: can not compress");
@@ -998,7 +998,7 @@
 "--decompress-debug-sections");
   }
 
-  if (Config.DecompressDebugSections && !zlib::isAvailable())
+  if (Config.DecompressDebugSections && !compression::zlib::isAvailable())
 return createStringError(
   

[Lldb-commits] [PATCH] D128465: [llvm] cmake config groundwork to have ZSTD in LLVM

[Cole Kissane via Phabricator via lldb-commits]

This revision was landed with ongoing or failed builds.
This revision was automatically updated to reflect the committed changes.
Closed by commit rGf07caf20b9d3: [llvm] cmake config groundwork to have ZSTD in 
LLVM (authored by ckissane).

Repository:
  rG LLVM Github Monorepo

CHANGES SINCE LAST ACTION
  https://reviews.llvm.org/D128465/new/

https://reviews.llvm.org/D128465

Files:
  clang-tools-extra/clangd/CMakeLists.txt
  clang-tools-extra/clangd/test/lit.cfg.py
  clang-tools-extra/clangd/test/lit.site.cfg.py.in
  clang/test/CMakeLists.txt
  clang/test/lit.site.cfg.py.in
  compiler-rt/lib/sanitizer_common/symbolizer/scripts/build_symbolizer.sh
  compiler-rt/test/lit.common.cfg.py
  compiler-rt/test/lit.common.configured.in
  flang/CMakeLists.txt
  lld/ELF/CMakeLists.txt
  lld/test/lit.site.cfg.py.in
  lldb/source/Plugins/Process/gdb-remote/CMakeLists.txt
  lldb/test/Shell/lit.site.cfg.py.in
  llvm/CMakeLists.txt
  llvm/cmake/config-ix.cmake
  llvm/cmake/modules/FindZSTD.cmake
  llvm/cmake/modules/LLVMConfig.cmake.in
  llvm/include/llvm/Config/llvm-config.h.cmake
  llvm/test/lit.site.cfg.py.in
  utils/bazel/llvm_configs/llvm-config.h.cmake

Index: utils/bazel/llvm_configs/llvm-config.h.cmake
===
--- utils/bazel/llvm_configs/llvm-config.h.cmake
+++ utils/bazel/llvm_configs/llvm-config.h.cmake
@@ -95,6 +95,9 @@
 /* Define if zlib compression is available */
 #cmakedefine01 LLVM_ENABLE_ZLIB
 
+/* Define if zstd compression is available */
+#cmakedefine01 LLVM_ENABLE_ZSTD
+
 /* Define if LLVM was built with a dependency to the libtensorflow dynamic library */
 #cmakedefine LLVM_HAVE_TF_API
 
Index: llvm/test/lit.site.cfg.py.in
===
--- llvm/test/lit.site.cfg.py.in
+++ llvm/test/lit.site.cfg.py.in
@@ -37,6 +37,7 @@
 config.llvm_use_intel_jitevents = @LLVM_USE_INTEL_JITEVENTS@
 config.llvm_use_sanitizer = "@LLVM_USE_SANITIZER@"
 config.have_zlib = @LLVM_ENABLE_ZLIB@
+config.have_zstd = @LLVM_ENABLE_ZSTD@
 config.have_libxar = @LLVM_HAVE_LIBXAR@
 config.have_libxml2 = @LLVM_ENABLE_LIBXML2@
 config.have_curl = @LLVM_ENABLE_CURL@
Index: llvm/include/llvm/Config/llvm-config.h.cmake
===
--- llvm/include/llvm/Config/llvm-config.h.cmake
+++ llvm/include/llvm/Config/llvm-config.h.cmake
@@ -95,6 +95,9 @@
 /* Define if zlib compression is available */
 #cmakedefine01 LLVM_ENABLE_ZLIB
 
+/* Define if zstd compression is available */
+#cmakedefine01 LLVM_ENABLE_ZSTD
+
 /* Define if LLVM was built with a dependency to the libtensorflow dynamic library */
 #cmakedefine LLVM_HAVE_TF_API
 
Index: llvm/cmake/modules/LLVMConfig.cmake.in
===
--- llvm/cmake/modules/LLVMConfig.cmake.in
+++ llvm/cmake/modules/LLVMConfig.cmake.in
@@ -73,6 +73,12 @@
   find_package(ZLIB)
 endif()
 
+set(LLVM_ENABLE_ZSTD @LLVM_ENABLE_ZSTD@)
+if(LLVM_ENABLE_ZSTD)
+  set(ZSTD_ROOT @ZSTD_ROOT@)
+  find_package(ZSTD)
+endif()
+
 set(LLVM_ENABLE_LIBXML2 @LLVM_ENABLE_LIBXML2@)
 if(LLVM_ENABLE_LIBXML2)
   find_package(LibXml2)
Index: llvm/cmake/modules/FindZSTD.cmake
===
--- /dev/null
+++ llvm/cmake/modules/FindZSTD.cmake
@@ -0,0 +1,21 @@
+find_path(ZSTD_INCLUDE_DIR
+  NAMES zstd.h
+  HINTS ${ZSTD_ROOT_DIR}/include)
+
+find_library(ZSTD_LIBRARY
+  NAMES zstd
+  HINTS ${ZSTD_ROOT_DIR}/lib)
+
+include(FindPackageHandleStandardArgs)
+find_package_handle_standard_args(
+ZSTD DEFAULT_MSG
+ZSTD_LIBRARY ZSTD_INCLUDE_DIR)
+
+if(ZSTD_FOUND)
+set(ZSTD_LIBRARIES ${ZSTD_LIBRARY})
+set(ZSTD_INCLUDE_DIRS ${ZSTD_INCLUDE_DIR})
+endif()
+
+mark_as_advanced(
+  ZSTD_INCLUDE_DIR
+  ZSTD_LIBRARY)
Index: llvm/cmake/config-ix.cmake
===
--- llvm/cmake/config-ix.cmake
+++ llvm/cmake/config-ix.cmake
@@ -136,6 +136,27 @@
   set(LLVM_ENABLE_ZLIB "${HAVE_ZLIB}")
 endif()
 
+if(LLVM_ENABLE_ZSTD)
+  if(LLVM_ENABLE_ZSTD STREQUAL FORCE_ON)
+find_package(ZSTD REQUIRED)
+  elseif(NOT LLVM_USE_SANITIZER MATCHES "Memory.*")
+find_package(ZSTD)
+  endif()
+  if(ZSTD_FOUND)
+# Check if zstd we found is usable; for example, we may have found a 32-bit
+# library on a 64-bit system which would result in a link-time failure.
+cmake_push_check_state()
+list(APPEND CMAKE_REQUIRED_INCLUDES ${ZSTD_INCLUDE_DIR})
+list(APPEND CMAKE_REQUIRED_LIBRARIES ${ZSTD_LIBRARY})
+check_symbol_exists(ZSTD_compress zstd.h HAVE_ZSTD)
+cmake_pop_check_state()
+if(LLVM_ENABLE_ZSTD STREQUAL FORCE_ON AND NOT HAVE_ZSTD)
+  message(FATAL_ERROR "Failed to configure zstd")
+endif()
+  endif()
+  set(LLVM_ENABLE_ZSTD "${HAVE_ZSTD}")
+endif()
+
 if(LLVM_ENABLE_LIBXML2)
   if(LLVM_ENABLE_LIBXML2 STREQUAL FORCE_ON)
 find_package(LibXml2 REQUIRED)
Index: llvm/CMakeLists.txt
=

[Lldb-commits] [PATCH] D128465: [llvm] cmake config groundwork to have ZSTD in LLVM

[Cole Kissane via Phabricator via lldb-commits]

ckissane updated this revision to Diff 443687.
ckissane added a comment.

clean a space and make accurate


Repository:
  rG LLVM Github Monorepo

CHANGES SINCE LAST ACTION
  https://reviews.llvm.org/D128465/new/

https://reviews.llvm.org/D128465

Files:
  clang-tools-extra/clangd/CMakeLists.txt
  clang-tools-extra/clangd/test/lit.cfg.py
  clang-tools-extra/clangd/test/lit.site.cfg.py.in
  clang/test/CMakeLists.txt
  clang/test/lit.site.cfg.py.in
  compiler-rt/lib/sanitizer_common/symbolizer/scripts/build_symbolizer.sh
  compiler-rt/test/lit.common.cfg.py
  compiler-rt/test/lit.common.configured.in
  flang/CMakeLists.txt
  lld/ELF/CMakeLists.txt
  lld/test/lit.site.cfg.py.in
  lldb/source/Plugins/Process/gdb-remote/CMakeLists.txt
  lldb/test/Shell/lit.site.cfg.py.in
  llvm/CMakeLists.txt
  llvm/cmake/config-ix.cmake
  llvm/cmake/modules/FindZSTD.cmake
  llvm/cmake/modules/LLVMConfig.cmake.in
  llvm/include/llvm/Config/llvm-config.h.cmake
  llvm/lib/Support/CMakeLists.txt
  llvm/test/lit.site.cfg.py.in
  utils/bazel/llvm_configs/llvm-config.h.cmake

Index: utils/bazel/llvm_configs/llvm-config.h.cmake
===
--- utils/bazel/llvm_configs/llvm-config.h.cmake
+++ utils/bazel/llvm_configs/llvm-config.h.cmake
@@ -95,6 +95,9 @@
 /* Define if zlib compression is available */
 #cmakedefine01 LLVM_ENABLE_ZLIB
 
+/* Define if zstd compression is available */
+#cmakedefine01 LLVM_ENABLE_ZSTD
+
 /* Define if LLVM was built with a dependency to the libtensorflow dynamic library */
 #cmakedefine LLVM_HAVE_TF_API
 
Index: llvm/test/lit.site.cfg.py.in
===
--- llvm/test/lit.site.cfg.py.in
+++ llvm/test/lit.site.cfg.py.in
@@ -37,6 +37,7 @@
 config.llvm_use_intel_jitevents = @LLVM_USE_INTEL_JITEVENTS@
 config.llvm_use_sanitizer = "@LLVM_USE_SANITIZER@"
 config.have_zlib = @LLVM_ENABLE_ZLIB@
+config.have_zstd = @LLVM_ENABLE_ZSTD@
 config.have_libxar = @LLVM_HAVE_LIBXAR@
 config.have_libxml2 = @LLVM_ENABLE_LIBXML2@
 config.have_curl = @LLVM_ENABLE_CURL@
Index: llvm/lib/Support/CMakeLists.txt
===
--- llvm/lib/Support/CMakeLists.txt
+++ llvm/lib/Support/CMakeLists.txt
@@ -25,6 +25,10 @@
   set(imported_libs ZLIB::ZLIB)
 endif()
 
+if(LLVM_ENABLE_ZSTD)
+  list(APPEND imported_libs zstd)
+endif()
+
 if( MSVC OR MINGW )
   # libuuid required for FOLDERID_Profile usage in lib/Support/Windows/Path.inc.
   # advapi32 required for CryptAcquireContextW in lib/Support/Windows/Path.inc.
Index: llvm/include/llvm/Config/llvm-config.h.cmake
===
--- llvm/include/llvm/Config/llvm-config.h.cmake
+++ llvm/include/llvm/Config/llvm-config.h.cmake
@@ -95,6 +95,9 @@
 /* Define if zlib compression is available */
 #cmakedefine01 LLVM_ENABLE_ZLIB
 
+/* Define if zstd compression is available */
+#cmakedefine01 LLVM_ENABLE_ZSTD
+
 /* Define if LLVM was built with a dependency to the libtensorflow dynamic library */
 #cmakedefine LLVM_HAVE_TF_API
 
Index: llvm/cmake/modules/LLVMConfig.cmake.in
===
--- llvm/cmake/modules/LLVMConfig.cmake.in
+++ llvm/cmake/modules/LLVMConfig.cmake.in
@@ -73,6 +73,12 @@
   find_package(ZLIB)
 endif()
 
+set(LLVM_ENABLE_ZSTD @LLVM_ENABLE_ZSTD@)
+if(LLVM_ENABLE_ZSTD)
+  set(ZSTD_ROOT @ZSTD_ROOT@)
+  find_package(ZSTD)
+endif()
+
 set(LLVM_ENABLE_LIBXML2 @LLVM_ENABLE_LIBXML2@)
 if(LLVM_ENABLE_LIBXML2)
   find_package(LibXml2)
Index: llvm/cmake/modules/FindZSTD.cmake
===
--- /dev/null
+++ llvm/cmake/modules/FindZSTD.cmake
@@ -0,0 +1,21 @@
+find_path(ZSTD_INCLUDE_DIR
+  NAMES zstd.h
+  HINTS ${ZSTD_ROOT_DIR}/include)
+
+find_library(ZSTD_LIBRARY
+  NAMES zstd
+  HINTS ${ZSTD_ROOT_DIR}/lib)
+
+include(FindPackageHandleStandardArgs)
+find_package_handle_standard_args(
+ZSTD DEFAULT_MSG
+ZSTD_LIBRARY ZSTD_INCLUDE_DIR)
+
+if(ZSTD_FOUND)
+set(ZSTD_LIBRARIES ${ZSTD_LIBRARY})
+set(ZSTD_INCLUDE_DIRS ${ZSTD_INCLUDE_DIR})
+endif()
+
+mark_as_advanced(
+  ZSTD_INCLUDE_DIR
+  ZSTD_LIBRARY)
Index: llvm/cmake/config-ix.cmake
===
--- llvm/cmake/config-ix.cmake
+++ llvm/cmake/config-ix.cmake
@@ -136,6 +136,27 @@
   set(LLVM_ENABLE_ZLIB "${HAVE_ZLIB}")
 endif()
 
+if(LLVM_ENABLE_ZSTD)
+  if(LLVM_ENABLE_ZSTD STREQUAL FORCE_ON)
+find_package(ZSTD REQUIRED)
+  elseif(NOT LLVM_USE_SANITIZER MATCHES "Memory.*")
+find_package(ZSTD)
+  endif()
+  if(ZSTD_FOUND)
+# Check if zstd we found is usable; for example, we may have found a 32-bit
+# library on a 64-bit system which would result in a link-time failure.
+cmake_push_check_state()
+list(APPEND CMAKE_REQUIRED_INCLUDES ${ZSTD_INCLUDE_DIR})
+list(APPEND CMAKE_REQUIRED_LIBRARIES ${ZSTD_LIBRARY})
+check_symbol_exists(ZSTD

[Lldb-commits] [PATCH] D128465: [llvm] cmake config groundwork to have ZSTD in LLVM

[Cole Kissane via Phabricator via lldb-commits]

ckissane added inline comments.



Comment at: lld/test/lit.site.cfg.py.in:21
 config.have_zlib = @LLVM_ENABLE_ZLIB@
+config.have_zstd = @LLVM_ENABLE_ZSTD@
 config.have_libxar = @LLVM_HAVE_LIBXAR@

MaskRay wrote:
> MaskRay wrote:
> > This needs a change in lld/test/CMakeLists.txt
> > 
> > Looks like you haven't run `ninja check-lld` as I mentioned.
> Actually I am preparing a lld patch and you probably should drop have_zstd 
> from lld/ for this patch.
I get this when I try to run that:
`ninja: error: unknown target 'check-lld', did you mean 'check-lit'?`



Comment at: lld/test/lit.site.cfg.py.in:21
 config.have_zlib = @LLVM_ENABLE_ZLIB@
+config.have_zstd = @LLVM_ENABLE_ZSTD@
 config.have_libxar = @LLVM_HAVE_LIBXAR@

ckissane wrote:
> MaskRay wrote:
> > MaskRay wrote:
> > > This needs a change in lld/test/CMakeLists.txt
> > > 
> > > Looks like you haven't run `ninja check-lld` as I mentioned.
> > Actually I am preparing a lld patch and you probably should drop have_zstd 
> > from lld/ for this patch.
> I get this when I try to run that:
> `ninja: error: unknown target 'check-lld', did you mean 'check-lit'?`
Got it


Repository:
  rG LLVM Github Monorepo

CHANGES SINCE LAST ACTION
  https://reviews.llvm.org/D128465/new/

https://reviews.llvm.org/D128465

___
lldb-commits mailing list
lldb-commits@lists.llvm.org
https://lists.llvm.org/cgi-bin/mailman/listinfo/lldb-commits

[Lldb-commits] [PATCH] D128465: [llvm] cmake config groundwork to have ZSTD in LLVM

[Cole Kissane via Phabricator via lldb-commits]

ckissane updated this revision to Diff 444044.
ckissane added a comment.

- make sure LLVM_ENABLE_ZSTD OFF turns into 0
- add zstd compression namespace


Repository:
  rG LLVM Github Monorepo

CHANGES SINCE LAST ACTION
  https://reviews.llvm.org/D128465/new/

https://reviews.llvm.org/D128465

Files:
  clang-tools-extra/clangd/CMakeLists.txt
  clang-tools-extra/clangd/test/lit.cfg.py
  clang-tools-extra/clangd/test/lit.site.cfg.py.in
  clang/test/CMakeLists.txt
  clang/test/lit.site.cfg.py.in
  compiler-rt/lib/sanitizer_common/symbolizer/scripts/build_symbolizer.sh
  compiler-rt/test/lit.common.cfg.py
  compiler-rt/test/lit.common.configured.in
  flang/CMakeLists.txt
  lld/ELF/CMakeLists.txt
  lld/test/lit.site.cfg.py.in
  lldb/source/Plugins/Process/gdb-remote/CMakeLists.txt
  lldb/test/Shell/lit.site.cfg.py.in
  llvm/CMakeLists.txt
  llvm/cmake/config-ix.cmake
  llvm/cmake/modules/FindZSTD.cmake
  llvm/cmake/modules/LLVMConfig.cmake.in
  llvm/include/llvm/Config/llvm-config.h.cmake
  llvm/include/llvm/Support/Compression.h
  llvm/lib/Support/CMakeLists.txt
  llvm/lib/Support/Compression.cpp
  llvm/test/lit.site.cfg.py.in
  utils/bazel/llvm_configs/llvm-config.h.cmake

Index: utils/bazel/llvm_configs/llvm-config.h.cmake
===
--- utils/bazel/llvm_configs/llvm-config.h.cmake
+++ utils/bazel/llvm_configs/llvm-config.h.cmake
@@ -95,6 +95,9 @@
 /* Define if zlib compression is available */
 #cmakedefine01 LLVM_ENABLE_ZLIB
 
+/* Define if zstd compression is available */
+#cmakedefine01 LLVM_ENABLE_ZSTD
+
 /* Define if LLVM was built with a dependency to the libtensorflow dynamic library */
 #cmakedefine LLVM_HAVE_TF_API
 
Index: llvm/test/lit.site.cfg.py.in
===
--- llvm/test/lit.site.cfg.py.in
+++ llvm/test/lit.site.cfg.py.in
@@ -37,6 +37,7 @@
 config.llvm_use_intel_jitevents = @LLVM_USE_INTEL_JITEVENTS@
 config.llvm_use_sanitizer = "@LLVM_USE_SANITIZER@"
 config.have_zlib = @LLVM_ENABLE_ZLIB@
+config.have_zstd = @LLVM_ENABLE_ZSTD@
 config.have_libxar = @LLVM_HAVE_LIBXAR@
 config.have_libxml2 = @LLVM_ENABLE_LIBXML2@
 config.have_curl = @LLVM_ENABLE_CURL@
Index: llvm/lib/Support/Compression.cpp
===
--- llvm/lib/Support/Compression.cpp
+++ llvm/lib/Support/Compression.cpp
@@ -20,6 +20,9 @@
 #if LLVM_ENABLE_ZLIB
 #include 
 #endif
+#if LLVM_ENABLE_ZSTD
+#include 
+#endif
 
 using namespace llvm;
 using namespace llvm::compression;
@@ -99,3 +102,66 @@
   llvm_unreachable("zlib::uncompress is unavailable");
 }
 #endif
+
+#if LLVM_ENABLE_ZSTD
+
+bool zstd::isAvailable() { return true; }
+
+void zstd::compress(StringRef InputBuffer,
+SmallVectorImpl &CompressedBuffer, int Level) {
+  unsigned long CompressedBufferSize = ::ZSTD_compressBound(InputBuffer.size());
+  CompressedBuffer.resize_for_overwrite(CompressedBufferSize);
+  unsigned long CompressedSize = ::ZSTD_compress(
+  (char *)CompressedBuffer.data(), CompressedBufferSize,
+  (const char *)InputBuffer.data(), InputBuffer.size(), Level);
+  if (ZSTD_isError(CompressedSize))
+report_bad_alloc_error("Allocation failed");
+  // Tell MemorySanitizer that zstd output buffer is fully initialized.
+  // This avoids a false report when running LLVM with uninstrumented ZLib.
+  __msan_unpoison(CompressedBuffer.data(), CompressedSize);
+  CompressedBuffer.truncate(CompressedSize);
+}
+
+Error zstd::uncompress(StringRef InputBuffer, char *UncompressedBuffer,
+   size_t &UncompressedSize) {
+  unsigned long long const FrameContentSize = ZSTD_getFrameContentSize(
+  (const char *)InputBuffer.data(), InputBuffer.size());
+  size_t const Res =
+  ::ZSTD_decompress((char *)UncompressedBuffer, FrameContentSize,
+(const char *)InputBuffer.data(), InputBuffer.size());
+  UncompressedSize = Res;
+  // Tell MemorySanitizer that zstd output buffer is fully initialized.
+  // This avoids a false report when running LLVM with uninstrumented ZLib.
+  __msan_unpoison(UncompressedBuffer, UncompressedSize);
+  return Res != FrameContentSize
+ ? make_error(ZSTD_getErrorName(Res),
+   inconvertibleErrorCode())
+ : Error::success();
+}
+
+Error zstd::uncompress(StringRef InputBuffer,
+   SmallVectorImpl &UncompressedBuffer,
+   size_t UncompressedSize) {
+  UncompressedBuffer.resize_for_overwrite(UncompressedSize);
+  Error E = zstd::uncompress(InputBuffer, UncompressedBuffer.data(),
+ UncompressedSize);
+  UncompressedBuffer.truncate(UncompressedSize);
+  return E;
+}
+
+#else
+bool zstd::isAvailable() { return false; }
+void zstd::compress(StringRef InputBuffer,
+SmallVectorImpl &CompressedBuffer, int Level) {
+  llvm_unreachable("zstd::compress is unavailable");
+}
+Error 

[Lldb-commits] [PATCH] D128465: [llvm] cmake config groundwork to have ZSTD in LLVM and add ZSTD compression namespace

[Cole Kissane via Phabricator via lldb-commits]

ckissane added a comment.

In D128465#3642997 , @MaskRay wrote:

> As I mentioned, the proper approach is to add zstd functionality along with 
> the CMake change, instead of adding CMake to all llvm-project components 
> without a way to test them.

@MaskRay, I have now done this and ran the ldd tests as requested:

  With LLVM_ENABLE_ZSTD=ON
  $ ninja check-lld
  Testing Time: 8.98s
Unsupported  :   17
Passed   : 2638
Expectedly Failed:1
  With LLVM_ENABLE_ZSTD=OFF
  $ ninja check-lld
  Testing Time: 8.95s
Unsupported  :   17
Passed   : 2638
Expectedly Failed:1


Repository:
  rG LLVM Github Monorepo

CHANGES SINCE LAST ACTION
  https://reviews.llvm.org/D128465/new/

https://reviews.llvm.org/D128465

___
lldb-commits mailing list
lldb-commits@lists.llvm.org
https://lists.llvm.org/cgi-bin/mailman/listinfo/lldb-commits

[Lldb-commits] [PATCH] D128465: [llvm] cmake config groundwork to have ZSTD in LLVM and add ZSTD compression namespace

[Cole Kissane via Phabricator via lldb-commits]

ckissane updated this revision to Diff 444078.
ckissane added a comment.

- add zstd tests to CompressionTest.cpp


Repository:
  rG LLVM Github Monorepo

CHANGES SINCE LAST ACTION
  https://reviews.llvm.org/D128465/new/

https://reviews.llvm.org/D128465

Files:
  clang-tools-extra/clangd/CMakeLists.txt
  clang-tools-extra/clangd/test/lit.cfg.py
  clang-tools-extra/clangd/test/lit.site.cfg.py.in
  clang/test/CMakeLists.txt
  clang/test/lit.site.cfg.py.in
  compiler-rt/lib/sanitizer_common/symbolizer/scripts/build_symbolizer.sh
  compiler-rt/test/lit.common.cfg.py
  compiler-rt/test/lit.common.configured.in
  flang/CMakeLists.txt
  lld/ELF/CMakeLists.txt
  lld/test/lit.site.cfg.py.in
  lldb/source/Plugins/Process/gdb-remote/CMakeLists.txt
  lldb/test/Shell/lit.site.cfg.py.in
  llvm/CMakeLists.txt
  llvm/cmake/config-ix.cmake
  llvm/cmake/modules/FindZSTD.cmake
  llvm/cmake/modules/LLVMConfig.cmake.in
  llvm/include/llvm/Config/llvm-config.h.cmake
  llvm/include/llvm/Support/Compression.h
  llvm/lib/Support/CMakeLists.txt
  llvm/lib/Support/Compression.cpp
  llvm/test/lit.site.cfg.py.in
  llvm/unittests/Support/CompressionTest.cpp
  utils/bazel/llvm_configs/llvm-config.h.cmake

Index: utils/bazel/llvm_configs/llvm-config.h.cmake
===
--- utils/bazel/llvm_configs/llvm-config.h.cmake
+++ utils/bazel/llvm_configs/llvm-config.h.cmake
@@ -95,6 +95,9 @@
 /* Define if zlib compression is available */
 #cmakedefine01 LLVM_ENABLE_ZLIB
 
+/* Define if zstd compression is available */
+#cmakedefine01 LLVM_ENABLE_ZSTD
+
 /* Define if LLVM was built with a dependency to the libtensorflow dynamic library */
 #cmakedefine LLVM_HAVE_TF_API
 
Index: llvm/unittests/Support/CompressionTest.cpp
===
--- llvm/unittests/Support/CompressionTest.cpp
+++ llvm/unittests/Support/CompressionTest.cpp
@@ -65,4 +65,46 @@
 
 #endif
 
+#if LLVM_ENABLE_ZSTD
+
+void TestZstdCompression(StringRef Input, int Level) {
+  SmallString<32> Compressed;
+  SmallString<32> Uncompressed;
+
+  zstd::compress(Input, Compressed, Level);
+
+  // Check that uncompressed buffer is the same as original.
+  Error E = zstd::uncompress(Compressed, Uncompressed, Input.size());
+  consumeError(std::move(E));
+
+  EXPECT_EQ(Input, Uncompressed);
+  if (Input.size() > 0) {
+// Uncompression fails if expected length is too short.
+E = zstd::uncompress(Compressed, Uncompressed, Input.size() - 1);
+EXPECT_EQ("Destination buffer is too small", llvm::toString(std::move(E)));
+  }
+}
+
+TEST(CompressionTest, Zstd) {
+  TestZstdCompression("", zstd::DefaultCompression);
+
+  TestZstdCompression("hello, world!", zstd::NoCompression);
+  TestZstdCompression("hello, world!", zstd::BestSizeCompression);
+  TestZstdCompression("hello, world!", zstd::BestSpeedCompression);
+  TestZstdCompression("hello, world!", zstd::DefaultCompression);
+
+  const size_t kSize = 1024;
+  char BinaryData[kSize];
+  for (size_t i = 0; i < kSize; ++i) {
+BinaryData[i] = i & 255;
+  }
+  StringRef BinaryDataStr(BinaryData, kSize);
+
+  TestZstdCompression(BinaryDataStr, zstd::NoCompression);
+  TestZstdCompression(BinaryDataStr, zstd::BestSizeCompression);
+  TestZstdCompression(BinaryDataStr, zstd::BestSpeedCompression);
+  TestZstdCompression(BinaryDataStr, zstd::DefaultCompression);
+}
+
+#endif
 }
Index: llvm/test/lit.site.cfg.py.in
===
--- llvm/test/lit.site.cfg.py.in
+++ llvm/test/lit.site.cfg.py.in
@@ -37,6 +37,7 @@
 config.llvm_use_intel_jitevents = @LLVM_USE_INTEL_JITEVENTS@
 config.llvm_use_sanitizer = "@LLVM_USE_SANITIZER@"
 config.have_zlib = @LLVM_ENABLE_ZLIB@
+config.have_zstd = @LLVM_ENABLE_ZSTD@
 config.have_libxar = @LLVM_HAVE_LIBXAR@
 config.have_libxml2 = @LLVM_ENABLE_LIBXML2@
 config.have_curl = @LLVM_ENABLE_CURL@
Index: llvm/lib/Support/Compression.cpp
===
--- llvm/lib/Support/Compression.cpp
+++ llvm/lib/Support/Compression.cpp
@@ -20,6 +20,9 @@
 #if LLVM_ENABLE_ZLIB
 #include 
 #endif
+#if LLVM_ENABLE_ZSTD
+#include 
+#endif
 
 using namespace llvm;
 using namespace llvm::compression;
@@ -57,7 +60,9 @@
   // Tell MemorySanitizer that zlib output buffer is fully initialized.
   // This avoids a false report when running LLVM with uninstrumented ZLib.
   __msan_unpoison(CompressedBuffer.data(), CompressedSize);
-  CompressedBuffer.truncate(CompressedSize);
+  if (CompressedSize < CompressedBuffer.size()) {
+CompressedBuffer.truncate(CompressedSize);
+  }
 }
 
 Error zlib::uncompress(StringRef InputBuffer, char *UncompressedBuffer,
@@ -79,7 +84,9 @@
   UncompressedBuffer.resize_for_overwrite(UncompressedSize);
   Error E = zlib::uncompress(InputBuffer, UncompressedBuffer.data(),
  UncompressedSize);
-  UncompressedBuffer.truncate(UncompressedSize);
+  if (UncompressedSize < Uncompr

[Lldb-commits] [PATCH] D128465: [llvm] cmake config groundwork to have ZSTD in LLVM and add ZSTD compression namespace

[Cole Kissane via Phabricator via lldb-commits]

ckissane added a comment.

In D128465#3646258 , @MaskRay wrote:

> In D128465#3646203 , @ckissane 
> wrote:
>
>> In D128465#3642997 , @MaskRay 
>> wrote:
>>
>>> As I mentioned, the proper approach is to add zstd functionality along with 
>>> the CMake change, instead of adding CMake to all llvm-project components 
>>> without a way to test them.
>>
>> @MaskRay, I have now done this and ran the ldd tests as requested:
>>
>>   With LLVM_ENABLE_ZSTD=ON
>>   $ ninja check-lld
>>   Testing Time: 8.98s
>> Unsupported  :   17
>> Passed   : 2638
>> Expectedly Failed:1
>>   With LLVM_ENABLE_ZSTD=OFF
>>   $ ninja check-lld
>>   Testing Time: 8.95s
>> Unsupported  :   17
>> Passed   : 2638
>> Expectedly Failed:1
>
> I request that you abandon this patch and incorporate the llvm cmake part 
> into the llvm patch which you actually use zstd.
> It is not appropriate to add zstd cmake to llvm-project components which 
> don't use zstd.

Let me see if I understand this correctly:
Are you saying that I should abandon this patch, and create a new patch that:

- adds findZSTD.cmake
- adds zstd to compression namespace
- adds tests to CompressionTest.cpp
- and does the **minimal** amount of cmake work to have the flags and test work
  - thus differing per component cmake config to patches like how you are doing 
in https://reviews.llvm.org/D129406

Is this correct?

I realize though that then https://reviews.llvm.org/D129406 or similar would be 
"the [first] llvm patch which actually use[s] zstd"


Repository:
  rG LLVM Github Monorepo

CHANGES SINCE LAST ACTION
  https://reviews.llvm.org/D128465/new/

https://reviews.llvm.org/D128465

___
lldb-commits mailing list
lldb-commits@lists.llvm.org
https://lists.llvm.org/cgi-bin/mailman/listinfo/lldb-commits

[Lldb-commits] [PATCH] D128465: [llvm] cmake config groundwork to have ZSTD in LLVM and add ZSTD compression namespace

[Cole Kissane via Phabricator via lldb-commits]

ckissane added a comment.

In D128465#3646601 , @MaskRay wrote:

> In D128465#3646561 , @ckissane 
> wrote:
>
>> In D128465#3646258 , @MaskRay 
>> wrote:
>>
>>> In D128465#3646203 , @ckissane 
>>> wrote:
>>>
 In D128465#3642997 , @MaskRay 
 wrote:

> As I mentioned, the proper approach is to add zstd functionality along 
> with the CMake change, instead of adding CMake to all llvm-project 
> components without a way to test them.

 @MaskRay, I have now done this and ran the ldd tests as requested:

   With LLVM_ENABLE_ZSTD=ON
   $ ninja check-lld
   Testing Time: 8.98s
 Unsupported  :   17
 Passed   : 2638
 Expectedly Failed:1
   With LLVM_ENABLE_ZSTD=OFF
   $ ninja check-lld
   Testing Time: 8.95s
 Unsupported  :   17
 Passed   : 2638
 Expectedly Failed:1
>>>
>>> I request that you abandon this patch and incorporate the llvm cmake part 
>>> into the llvm patch which you actually use zstd.
>>> It is not appropriate to add zstd cmake to llvm-project components which 
>>> don't use zstd.
>>
>> Let me see if I understand this correctly:
>> Are you saying that I should abandon this patch, and create a new patch that:
>>
>> - adds findZSTD.cmake
>> - adds zstd to compression namespace
>> - adds tests to CompressionTest.cpp
>> - and does the **minimal** amount of cmake work to have the flags and test 
>> work
>>   - thus differing per component cmake config to patches like how you are 
>> doing in https://reviews.llvm.org/D129406
>>
>> Is this correct?
>>
>> I realize though that then https://reviews.llvm.org/D129406 or similar would 
>> be "the [first] llvm patch which actually use[s] zstd"
>
> You can find a component in llvm which uses zstd (e.g. lib/Support). Add the 
> functionality with tests (e.g. a CompressionTest.cpp unittest) along with the 
> CMake/(optional Bazel, gn) changes.
> Keep clang, clang-tools-extra, lld, etc unchanged. These components don't yet 
> use zstd and the CMake changes in this patch will be untestable.

Got it!, this patch already has such tests (CompressionTest.cpp unittest) in 
lib/Support, so I should just remove the untestable cmake config.


Repository:
  rG LLVM Github Monorepo

CHANGES SINCE LAST ACTION
  https://reviews.llvm.org/D128465/new/

https://reviews.llvm.org/D128465

___
lldb-commits mailing list
lldb-commits@lists.llvm.org
https://lists.llvm.org/cgi-bin/mailman/listinfo/lldb-commits

[Lldb-commits] [PATCH] D128465: [llvm] cmake config groundwork to have ZSTD in LLVM and add ZSTD compression namespace

[Cole Kissane via Phabricator via lldb-commits]

ckissane updated this revision to Diff 444374.
ckissane added a comment.

- Trim CMAKE modifications to minimum needed


Repository:
  rG LLVM Github Monorepo

CHANGES SINCE LAST ACTION
  https://reviews.llvm.org/D128465/new/

https://reviews.llvm.org/D128465

Files:
  llvm/CMakeLists.txt
  llvm/cmake/config-ix.cmake
  llvm/cmake/modules/FindZSTD.cmake
  llvm/cmake/modules/LLVMConfig.cmake.in
  llvm/include/llvm/Config/llvm-config.h.cmake
  llvm/include/llvm/Support/Compression.h
  llvm/lib/Support/CMakeLists.txt
  llvm/lib/Support/Compression.cpp
  llvm/test/lit.site.cfg.py.in
  llvm/unittests/Support/CompressionTest.cpp
  utils/bazel/llvm_configs/llvm-config.h.cmake

Index: utils/bazel/llvm_configs/llvm-config.h.cmake
===
--- utils/bazel/llvm_configs/llvm-config.h.cmake
+++ utils/bazel/llvm_configs/llvm-config.h.cmake
@@ -95,6 +95,9 @@
 /* Define if zlib compression is available */
 #cmakedefine01 LLVM_ENABLE_ZLIB
 
+/* Define if zstd compression is available */
+#cmakedefine01 LLVM_ENABLE_ZSTD
+
 /* Define if LLVM was built with a dependency to the libtensorflow dynamic library */
 #cmakedefine LLVM_HAVE_TF_API
 
Index: llvm/unittests/Support/CompressionTest.cpp
===
--- llvm/unittests/Support/CompressionTest.cpp
+++ llvm/unittests/Support/CompressionTest.cpp
@@ -65,4 +65,46 @@
 
 #endif
 
+#if LLVM_ENABLE_ZSTD
+
+void TestZstdCompression(StringRef Input, int Level) {
+  SmallString<32> Compressed;
+  SmallString<32> Uncompressed;
+
+  zstd::compress(Input, Compressed, Level);
+
+  // Check that uncompressed buffer is the same as original.
+  Error E = zstd::uncompress(Compressed, Uncompressed, Input.size());
+  consumeError(std::move(E));
+
+  EXPECT_EQ(Input, Uncompressed);
+  if (Input.size() > 0) {
+// Uncompression fails if expected length is too short.
+E = zstd::uncompress(Compressed, Uncompressed, Input.size() - 1);
+EXPECT_EQ("Destination buffer is too small", llvm::toString(std::move(E)));
+  }
+}
+
+TEST(CompressionTest, Zstd) {
+  TestZstdCompression("", zstd::DefaultCompression);
+
+  TestZstdCompression("hello, world!", zstd::NoCompression);
+  TestZstdCompression("hello, world!", zstd::BestSizeCompression);
+  TestZstdCompression("hello, world!", zstd::BestSpeedCompression);
+  TestZstdCompression("hello, world!", zstd::DefaultCompression);
+
+  const size_t kSize = 1024;
+  char BinaryData[kSize];
+  for (size_t i = 0; i < kSize; ++i) {
+BinaryData[i] = i & 255;
+  }
+  StringRef BinaryDataStr(BinaryData, kSize);
+
+  TestZstdCompression(BinaryDataStr, zstd::NoCompression);
+  TestZstdCompression(BinaryDataStr, zstd::BestSizeCompression);
+  TestZstdCompression(BinaryDataStr, zstd::BestSpeedCompression);
+  TestZstdCompression(BinaryDataStr, zstd::DefaultCompression);
+}
+
+#endif
 }
Index: llvm/test/lit.site.cfg.py.in
===
--- llvm/test/lit.site.cfg.py.in
+++ llvm/test/lit.site.cfg.py.in
@@ -37,6 +37,7 @@
 config.llvm_use_intel_jitevents = @LLVM_USE_INTEL_JITEVENTS@
 config.llvm_use_sanitizer = "@LLVM_USE_SANITIZER@"
 config.have_zlib = @LLVM_ENABLE_ZLIB@
+config.have_zstd = @LLVM_ENABLE_ZSTD@
 config.have_libxar = @LLVM_HAVE_LIBXAR@
 config.have_libxml2 = @LLVM_ENABLE_LIBXML2@
 config.have_curl = @LLVM_ENABLE_CURL@
Index: llvm/lib/Support/Compression.cpp
===
--- llvm/lib/Support/Compression.cpp
+++ llvm/lib/Support/Compression.cpp
@@ -20,6 +20,9 @@
 #if LLVM_ENABLE_ZLIB
 #include 
 #endif
+#if LLVM_ENABLE_ZSTD
+#include 
+#endif
 
 using namespace llvm;
 using namespace llvm::compression;
@@ -57,7 +60,9 @@
   // Tell MemorySanitizer that zlib output buffer is fully initialized.
   // This avoids a false report when running LLVM with uninstrumented ZLib.
   __msan_unpoison(CompressedBuffer.data(), CompressedSize);
-  CompressedBuffer.truncate(CompressedSize);
+  if (CompressedSize < CompressedBuffer.size()) {
+CompressedBuffer.truncate(CompressedSize);
+  }
 }
 
 Error zlib::uncompress(StringRef InputBuffer, char *UncompressedBuffer,
@@ -79,7 +84,9 @@
   UncompressedBuffer.resize_for_overwrite(UncompressedSize);
   Error E = zlib::uncompress(InputBuffer, UncompressedBuffer.data(),
  UncompressedSize);
-  UncompressedBuffer.truncate(UncompressedSize);
+  if (UncompressedSize < UncompressedBuffer.size()) {
+UncompressedBuffer.truncate(UncompressedSize);
+  }
   return E;
 }
 
@@ -99,3 +106,67 @@
   llvm_unreachable("zlib::uncompress is unavailable");
 }
 #endif
+
+#if LLVM_ENABLE_ZSTD
+
+bool zstd::isAvailable() { return true; }
+
+void zstd::compress(StringRef InputBuffer,
+SmallVectorImpl &CompressedBuffer, int Level) {
+  unsigned long CompressedBufferSize = ::ZSTD_compressBound(InputBuffer.size());
+  CompressedBuffer.resize_for_overwrite(CompressedBufferSize);
+  

[Lldb-commits] [PATCH] D128465: [llvm] cmake config groundwork to have ZSTD in LLVM and add ZSTD compression namespace

[Cole Kissane via Phabricator via lldb-commits]

ckissane updated this revision to Diff 00.
ckissane added a comment.

tidy some code


Repository:
  rG LLVM Github Monorepo

CHANGES SINCE LAST ACTION
  https://reviews.llvm.org/D128465/new/

https://reviews.llvm.org/D128465

Files:
  llvm/CMakeLists.txt
  llvm/cmake/config-ix.cmake
  llvm/cmake/modules/FindZSTD.cmake
  llvm/cmake/modules/LLVMConfig.cmake.in
  llvm/include/llvm/Config/llvm-config.h.cmake
  llvm/include/llvm/Support/Compression.h
  llvm/lib/Support/CMakeLists.txt
  llvm/lib/Support/Compression.cpp
  llvm/test/lit.site.cfg.py.in
  llvm/unittests/Support/CompressionTest.cpp
  utils/bazel/llvm_configs/llvm-config.h.cmake

Index: utils/bazel/llvm_configs/llvm-config.h.cmake
===
--- utils/bazel/llvm_configs/llvm-config.h.cmake
+++ utils/bazel/llvm_configs/llvm-config.h.cmake
@@ -95,6 +95,9 @@
 /* Define if zlib compression is available */
 #cmakedefine01 LLVM_ENABLE_ZLIB
 
+/* Define if zstd compression is available */
+#cmakedefine01 LLVM_ENABLE_ZSTD
+
 /* Define if LLVM was built with a dependency to the libtensorflow dynamic library */
 #cmakedefine LLVM_HAVE_TF_API
 
Index: llvm/unittests/Support/CompressionTest.cpp
===
--- llvm/unittests/Support/CompressionTest.cpp
+++ llvm/unittests/Support/CompressionTest.cpp
@@ -23,11 +23,9 @@
 namespace {
 
 #if LLVM_ENABLE_ZLIB
-
-void TestZlibCompression(StringRef Input, int Level) {
+static void testZlibCompression(StringRef Input, int Level) {
   SmallString<32> Compressed;
   SmallString<32> Uncompressed;
-
   zlib::compress(Input, Compressed, Level);
 
   // Check that uncompressed buffer is the same as original.
@@ -43,26 +41,63 @@
 }
 
 TEST(CompressionTest, Zlib) {
-  TestZlibCompression("", zlib::DefaultCompression);
+  testZlibCompression("", zlib::DefaultCompression);
 
-  TestZlibCompression("hello, world!", zlib::NoCompression);
-  TestZlibCompression("hello, world!", zlib::BestSizeCompression);
-  TestZlibCompression("hello, world!", zlib::BestSpeedCompression);
-  TestZlibCompression("hello, world!", zlib::DefaultCompression);
+  testZlibCompression("hello, world!", zlib::NoCompression);
+  testZlibCompression("hello, world!", zlib::BestSizeCompression);
+  testZlibCompression("hello, world!", zlib::BestSpeedCompression);
+  testZlibCompression("hello, world!", zlib::DefaultCompression);
 
   const size_t kSize = 1024;
   char BinaryData[kSize];
-  for (size_t i = 0; i < kSize; ++i) {
+  for (size_t i = 0; i < kSize; ++i)
 BinaryData[i] = i & 255;
-  }
   StringRef BinaryDataStr(BinaryData, kSize);
 
-  TestZlibCompression(BinaryDataStr, zlib::NoCompression);
-  TestZlibCompression(BinaryDataStr, zlib::BestSizeCompression);
-  TestZlibCompression(BinaryDataStr, zlib::BestSpeedCompression);
-  TestZlibCompression(BinaryDataStr, zlib::DefaultCompression);
+  testZlibCompression(BinaryDataStr, zlib::NoCompression);
+  testZlibCompression(BinaryDataStr, zlib::BestSizeCompression);
+  testZlibCompression(BinaryDataStr, zlib::BestSpeedCompression);
+  testZlibCompression(BinaryDataStr, zlib::DefaultCompression);
+}
+#endif
+
+#if LLVM_ENABLE_ZSTD
+static void testZstdCompression(StringRef Input, int Level) {
+  SmallString<32> Compressed;
+  SmallString<32> Uncompressed;
+  zstd::compress(Input, Compressed, Level);
+
+  // Check that uncompressed buffer is the same as original.
+  Error E = zstd::uncompress(Compressed, Uncompressed, Input.size());
+  consumeError(std::move(E));
+
+  EXPECT_EQ(Input, Uncompressed);
+  if (Input.size() > 0) {
+// Uncompression fails if expected length is too short.
+E = zstd::uncompress(Compressed, Uncompressed, Input.size() - 1);
+EXPECT_EQ("Destination buffer is too small", llvm::toString(std::move(E)));
+  }
 }
 
+TEST(CompressionTest, Zstd) {
+  testZstdCompression("", zstd::DefaultCompression);
+
+  testZstdCompression("hello, world!", zstd::NoCompression);
+  testZstdCompression("hello, world!", zstd::BestSizeCompression);
+  testZstdCompression("hello, world!", zstd::BestSpeedCompression);
+  testZstdCompression("hello, world!", zstd::DefaultCompression);
+
+  const size_t kSize = 1024;
+  char BinaryData[kSize];
+  for (size_t i = 0; i < kSize; ++i)
+BinaryData[i] = i & 255;
+  StringRef BinaryDataStr(BinaryData, kSize);
+
+  testZstdCompression(BinaryDataStr, zstd::NoCompression);
+  testZstdCompression(BinaryDataStr, zstd::BestSizeCompression);
+  testZstdCompression(BinaryDataStr, zstd::BestSpeedCompression);
+  testZstdCompression(BinaryDataStr, zstd::DefaultCompression);
+}
 #endif
 
 }
Index: llvm/test/lit.site.cfg.py.in
===
--- llvm/test/lit.site.cfg.py.in
+++ llvm/test/lit.site.cfg.py.in
@@ -37,6 +37,7 @@
 config.llvm_use_intel_jitevents = @LLVM_USE_INTEL_JITEVENTS@
 config.llvm_use_sanitizer = "@LLVM_USE_SANITIZER@"
 config.have_zlib = @LLVM_ENABLE_ZLIB@
+config.have_zstd = @LLVM_EN

[Lldb-commits] [PATCH] D128465: [llvm] add zstd to `llvm::compression` namespace

[Cole Kissane via Phabricator via lldb-commits]

ckissane marked 6 inline comments as done.
ckissane added a comment.

Mark handled comments from @MaskRay as done


Repository:
  rG LLVM Github Monorepo

CHANGES SINCE LAST ACTION
  https://reviews.llvm.org/D128465/new/

https://reviews.llvm.org/D128465

___
lldb-commits mailing list
lldb-commits@lists.llvm.org
https://lists.llvm.org/cgi-bin/mailman/listinfo/lldb-commits

[Lldb-commits] [PATCH] D128465: [llvm] add zstd to `llvm::compression` namespace

[Cole Kissane via Phabricator via lldb-commits]

ckissane added inline comments.



Comment at: llvm/lib/Support/Compression.cpp:63
   __msan_unpoison(CompressedBuffer.data(), CompressedSize);
-  CompressedBuffer.truncate(CompressedSize);
+  if (CompressedSize < CompressedBuffer.size())
+CompressedBuffer.truncate(CompressedSize);

phosek wrote:
> I'd make changes to zlib separately.
Here https://reviews.llvm.org/D129698


Repository:
  rG LLVM Github Monorepo

CHANGES SINCE LAST ACTION
  https://reviews.llvm.org/D128465/new/

https://reviews.llvm.org/D128465

___
lldb-commits mailing list
lldb-commits@lists.llvm.org
https://lists.llvm.org/cgi-bin/mailman/listinfo/lldb-commits

[Lldb-commits] [PATCH] D128465: [llvm] add zstd to `llvm::compression` namespace

[Cole Kissane via Phabricator via lldb-commits]

ckissane updated this revision to Diff 32.
ckissane added a comment.

- Merge branch main


Repository:
  rG LLVM Github Monorepo

CHANGES SINCE LAST ACTION
  https://reviews.llvm.org/D128465/new/

https://reviews.llvm.org/D128465

Files:
  llvm/CMakeLists.txt
  llvm/cmake/config-ix.cmake
  llvm/cmake/modules/FindZSTD.cmake
  llvm/cmake/modules/LLVMConfig.cmake.in
  llvm/include/llvm/Config/llvm-config.h.cmake
  llvm/include/llvm/Support/Compression.h
  llvm/lib/Support/CMakeLists.txt
  llvm/lib/Support/Compression.cpp
  llvm/test/lit.site.cfg.py.in
  llvm/unittests/Support/CompressionTest.cpp
  utils/bazel/llvm_configs/llvm-config.h.cmake

Index: utils/bazel/llvm_configs/llvm-config.h.cmake
===
--- utils/bazel/llvm_configs/llvm-config.h.cmake
+++ utils/bazel/llvm_configs/llvm-config.h.cmake
@@ -95,6 +95,9 @@
 /* Define if zlib compression is available */
 #cmakedefine01 LLVM_ENABLE_ZLIB
 
+/* Define if zstd compression is available */
+#cmakedefine01 LLVM_ENABLE_ZSTD
+
 /* Define if LLVM was built with a dependency to the libtensorflow dynamic library */
 #cmakedefine LLVM_HAVE_TF_API
 
Index: llvm/unittests/Support/CompressionTest.cpp
===
--- llvm/unittests/Support/CompressionTest.cpp
+++ llvm/unittests/Support/CompressionTest.cpp
@@ -61,4 +61,43 @@
 }
 #endif
 
+#if LLVM_ENABLE_ZSTD
+static void testZstdCompression(StringRef Input, int Level) {
+  SmallString<32> Compressed;
+  SmallString<32> Uncompressed;
+  zstd::compress(Input, Compressed, Level);
+
+  // Check that uncompressed buffer is the same as original.
+  Error E = zstd::uncompress(Compressed, Uncompressed, Input.size());
+  consumeError(std::move(E));
+
+  EXPECT_EQ(Input, Uncompressed);
+  if (Input.size() > 0) {
+// Uncompression fails if expected length is too short.
+E = zstd::uncompress(Compressed, Uncompressed, Input.size() - 1);
+EXPECT_EQ("Destination buffer is too small", llvm::toString(std::move(E)));
+  }
+}
+
+TEST(CompressionTest, Zstd) {
+  testZstdCompression("", zstd::DefaultCompression);
+
+  testZstdCompression("hello, world!", zstd::NoCompression);
+  testZstdCompression("hello, world!", zstd::BestSizeCompression);
+  testZstdCompression("hello, world!", zstd::BestSpeedCompression);
+  testZstdCompression("hello, world!", zstd::DefaultCompression);
+
+  const size_t kSize = 1024;
+  char BinaryData[kSize];
+  for (size_t i = 0; i < kSize; ++i)
+BinaryData[i] = i & 255;
+  StringRef BinaryDataStr(BinaryData, kSize);
+
+  testZstdCompression(BinaryDataStr, zstd::NoCompression);
+  testZstdCompression(BinaryDataStr, zstd::BestSizeCompression);
+  testZstdCompression(BinaryDataStr, zstd::BestSpeedCompression);
+  testZstdCompression(BinaryDataStr, zstd::DefaultCompression);
+}
+#endif
+
 }
Index: llvm/test/lit.site.cfg.py.in
===
--- llvm/test/lit.site.cfg.py.in
+++ llvm/test/lit.site.cfg.py.in
@@ -37,6 +37,7 @@
 config.llvm_use_intel_jitevents = @LLVM_USE_INTEL_JITEVENTS@
 config.llvm_use_sanitizer = "@LLVM_USE_SANITIZER@"
 config.have_zlib = @LLVM_ENABLE_ZLIB@
+config.have_zstd = @LLVM_ENABLE_ZSTD@
 config.have_libxar = @LLVM_HAVE_LIBXAR@
 config.have_libxml2 = @LLVM_ENABLE_LIBXML2@
 config.have_curl = @LLVM_ENABLE_CURL@
Index: llvm/lib/Support/Compression.cpp
===
--- llvm/lib/Support/Compression.cpp
+++ llvm/lib/Support/Compression.cpp
@@ -20,6 +20,9 @@
 #if LLVM_ENABLE_ZLIB
 #include 
 #endif
+#if LLVM_ENABLE_ZSTD
+#include 
+#endif
 
 using namespace llvm;
 using namespace llvm::compression;
@@ -101,3 +104,65 @@
   llvm_unreachable("zlib::uncompress is unavailable");
 }
 #endif
+
+#if LLVM_ENABLE_ZSTD
+
+bool zstd::isAvailable() { return true; }
+
+void zstd::compress(StringRef InputBuffer,
+SmallVectorImpl &CompressedBuffer, int Level) {
+  unsigned long CompressedBufferSize = ::ZSTD_compressBound(InputBuffer.size());
+  CompressedBuffer.resize_for_overwrite(CompressedBufferSize);
+  unsigned long CompressedSize = ::ZSTD_compress(
+  (char *)CompressedBuffer.data(), CompressedBufferSize,
+  (const char *)InputBuffer.data(), InputBuffer.size(), Level);
+  if (ZSTD_isError(CompressedSize))
+report_bad_alloc_error("Allocation failed");
+  // Tell MemorySanitizer that zstd output buffer is fully initialized.
+  // This avoids a false report when running LLVM with uninstrumented ZLib.
+  __msan_unpoison(CompressedBuffer.data(), CompressedSize);
+  if (CompressedSize < CompressedBuffer.size())
+CompressedBuffer.truncate(CompressedSize);
+}
+
+Error zstd::uncompress(StringRef InputBuffer, char *UncompressedBuffer,
+   size_t &UncompressedSize) {
+  const size_t Res =
+  ::ZSTD_decompress((char *)UncompressedBuffer, UncompressedSize,
+(const char *)InputBuffer.data(), InputBu

[Lldb-commits] [PATCH] D128465: [llvm] add zstd to `llvm::compression` namespace

[Cole Kissane via Phabricator via lldb-commits]

ckissane updated this revision to Diff 89.
ckissane added a comment.

- Merge remote-tracking branch 'origin/main' into ckissane.add-zstd
- [Support] update zstd interface to use uint8_t * and ArrayRef


Repository:
  rG LLVM Github Monorepo

CHANGES SINCE LAST ACTION
  https://reviews.llvm.org/D128465/new/

https://reviews.llvm.org/D128465

Files:
  llvm/CMakeLists.txt
  llvm/cmake/config-ix.cmake
  llvm/cmake/modules/FindZSTD.cmake
  llvm/cmake/modules/LLVMConfig.cmake.in
  llvm/include/llvm/Config/llvm-config.h.cmake
  llvm/include/llvm/Support/Compression.h
  llvm/lib/Support/CMakeLists.txt
  llvm/lib/Support/Compression.cpp
  llvm/test/lit.site.cfg.py.in
  llvm/unittests/Support/CompressionTest.cpp
  utils/bazel/llvm_configs/llvm-config.h.cmake

Index: utils/bazel/llvm_configs/llvm-config.h.cmake
===
--- utils/bazel/llvm_configs/llvm-config.h.cmake
+++ utils/bazel/llvm_configs/llvm-config.h.cmake
@@ -95,6 +95,9 @@
 /* Define if zlib compression is available */
 #cmakedefine01 LLVM_ENABLE_ZLIB
 
+/* Define if zstd compression is available */
+#cmakedefine01 LLVM_ENABLE_ZSTD
+
 /* Define if LLVM was built with a dependency to the libtensorflow dynamic library */
 #cmakedefine LLVM_HAVE_TF_API
 
Index: llvm/unittests/Support/CompressionTest.cpp
===
--- llvm/unittests/Support/CompressionTest.cpp
+++ llvm/unittests/Support/CompressionTest.cpp
@@ -61,4 +61,42 @@
 }
 #endif
 
+#if LLVM_ENABLE_ZSTD
+static void testZstdCompression(StringRef Input, int Level) {
+  SmallVector Compressed;
+  SmallVector Uncompressed;
+  zstd::compress(arrayRefFromStringRef(Input), Compressed, Level);
+
+  // Check that uncompressed buffer is the same as original.
+  Error E = zstd::uncompress(Compressed, Uncompressed, Input.size());
+  consumeError(std::move(E));
+
+  EXPECT_EQ(Input, toStringRef(Uncompressed));
+  if (Input.size() > 0) {
+// Uncompression fails if expected length is too short.
+E = zstd::uncompress(Compressed, Uncompressed, Input.size() - 1);
+EXPECT_EQ("Destination buffer is too small", llvm::toString(std::move(E)));
+  }
+}
+
+TEST(CompressionTest, Zstd) {
+  testZstdCompression("", zstd::DefaultCompression);
+
+  testZstdCompression("hello, world!", zstd::NoCompression);
+  testZstdCompression("hello, world!", zstd::BestSizeCompression);
+  testZstdCompression("hello, world!", zstd::BestSpeedCompression);
+  testZstdCompression("hello, world!", zstd::DefaultCompression);
+
+  const size_t kSize = 1024;
+  char BinaryData[kSize];
+  for (size_t i = 0; i < kSize; ++i)
+BinaryData[i] = i & 255;
+  StringRef BinaryDataStr(BinaryData, kSize);
+
+  testZstdCompression(BinaryDataStr, zstd::NoCompression);
+  testZstdCompression(BinaryDataStr, zstd::BestSizeCompression);
+  testZstdCompression(BinaryDataStr, zstd::BestSpeedCompression);
+  testZstdCompression(BinaryDataStr, zstd::DefaultCompression);
+}
+#endif
 }
Index: llvm/test/lit.site.cfg.py.in
===
--- llvm/test/lit.site.cfg.py.in
+++ llvm/test/lit.site.cfg.py.in
@@ -37,6 +37,7 @@
 config.llvm_use_intel_jitevents = @LLVM_USE_INTEL_JITEVENTS@
 config.llvm_use_sanitizer = "@LLVM_USE_SANITIZER@"
 config.have_zlib = @LLVM_ENABLE_ZLIB@
+config.have_zstd = @LLVM_ENABLE_ZSTD@
 config.have_libxar = @LLVM_HAVE_LIBXAR@
 config.have_libxml2 = @LLVM_ENABLE_LIBXML2@
 config.have_curl = @LLVM_ENABLE_CURL@
Index: llvm/lib/Support/Compression.cpp
===
--- llvm/lib/Support/Compression.cpp
+++ llvm/lib/Support/Compression.cpp
@@ -20,6 +20,9 @@
 #if LLVM_ENABLE_ZLIB
 #include 
 #endif
+#if LLVM_ENABLE_ZSTD
+#include 
+#endif
 
 using namespace llvm;
 using namespace llvm::compression;
@@ -100,3 +103,65 @@
   llvm_unreachable("zlib::uncompress is unavailable");
 }
 #endif
+
+#if LLVM_ENABLE_ZSTD
+
+bool zstd::isAvailable() { return true; }
+
+void zstd::compress(ArrayRef Input,
+SmallVectorImpl &CompressedBuffer, int Level) {
+  unsigned long CompressedBufferSize = ::ZSTD_compressBound(Input.size());
+  CompressedBuffer.resize_for_overwrite(CompressedBufferSize);
+  unsigned long CompressedSize =
+  ::ZSTD_compress((char *)CompressedBuffer.data(), CompressedBufferSize,
+  (const char *)Input.data(), Input.size(), Level);
+  if (ZSTD_isError(CompressedSize))
+report_bad_alloc_error("Allocation failed");
+  // Tell MemorySanitizer that zstd output buffer is fully initialized.
+  // This avoids a false report when running LLVM with uninstrumented ZLib.
+  __msan_unpoison(CompressedBuffer.data(), CompressedSize);
+  if (CompressedSize < CompressedBuffer.size())
+CompressedBuffer.truncate(CompressedSize);
+}
+
+Error zstd::uncompress(ArrayRef Input, uint8_t *UncompressedBuffer,
+   size_t &UncompressedSize) {
+  const size_t Res =
+  ::ZSTD

[Lldb-commits] [PATCH] D128465: [llvm] add zstd to `llvm::compression` namespace

[Cole Kissane via Phabricator via lldb-commits]

This revision was landed with ongoing or failed builds.
This revision was automatically updated to reflect the committed changes.
Closed by commit rGcef07169ec9f: [llvm] add zstd to `llvm::compression` 
namespace (authored by ckissane).

Repository:
  rG LLVM Github Monorepo

CHANGES SINCE LAST ACTION
  https://reviews.llvm.org/D128465/new/

https://reviews.llvm.org/D128465

Files:
  llvm/CMakeLists.txt
  llvm/cmake/config-ix.cmake
  llvm/cmake/modules/FindZSTD.cmake
  llvm/cmake/modules/LLVMConfig.cmake.in
  llvm/include/llvm/Config/llvm-config.h.cmake
  llvm/include/llvm/Support/Compression.h
  llvm/lib/Support/CMakeLists.txt
  llvm/lib/Support/Compression.cpp
  llvm/test/lit.site.cfg.py.in
  llvm/unittests/Support/CompressionTest.cpp
  utils/bazel/llvm_configs/llvm-config.h.cmake

Index: utils/bazel/llvm_configs/llvm-config.h.cmake
===
--- utils/bazel/llvm_configs/llvm-config.h.cmake
+++ utils/bazel/llvm_configs/llvm-config.h.cmake
@@ -95,6 +95,9 @@
 /* Define if zlib compression is available */
 #cmakedefine01 LLVM_ENABLE_ZLIB
 
+/* Define if zstd compression is available */
+#cmakedefine01 LLVM_ENABLE_ZSTD
+
 /* Define if LLVM was built with a dependency to the libtensorflow dynamic library */
 #cmakedefine LLVM_HAVE_TF_API
 
Index: llvm/unittests/Support/CompressionTest.cpp
===
--- llvm/unittests/Support/CompressionTest.cpp
+++ llvm/unittests/Support/CompressionTest.cpp
@@ -61,4 +61,42 @@
 }
 #endif
 
+#if LLVM_ENABLE_ZSTD
+static void testZstdCompression(StringRef Input, int Level) {
+  SmallVector Compressed;
+  SmallVector Uncompressed;
+  zstd::compress(arrayRefFromStringRef(Input), Compressed, Level);
+
+  // Check that uncompressed buffer is the same as original.
+  Error E = zstd::uncompress(Compressed, Uncompressed, Input.size());
+  consumeError(std::move(E));
+
+  EXPECT_EQ(Input, toStringRef(Uncompressed));
+  if (Input.size() > 0) {
+// Uncompression fails if expected length is too short.
+E = zstd::uncompress(Compressed, Uncompressed, Input.size() - 1);
+EXPECT_EQ("Destination buffer is too small", llvm::toString(std::move(E)));
+  }
+}
+
+TEST(CompressionTest, Zstd) {
+  testZstdCompression("", zstd::DefaultCompression);
+
+  testZstdCompression("hello, world!", zstd::NoCompression);
+  testZstdCompression("hello, world!", zstd::BestSizeCompression);
+  testZstdCompression("hello, world!", zstd::BestSpeedCompression);
+  testZstdCompression("hello, world!", zstd::DefaultCompression);
+
+  const size_t kSize = 1024;
+  char BinaryData[kSize];
+  for (size_t i = 0; i < kSize; ++i)
+BinaryData[i] = i & 255;
+  StringRef BinaryDataStr(BinaryData, kSize);
+
+  testZstdCompression(BinaryDataStr, zstd::NoCompression);
+  testZstdCompression(BinaryDataStr, zstd::BestSizeCompression);
+  testZstdCompression(BinaryDataStr, zstd::BestSpeedCompression);
+  testZstdCompression(BinaryDataStr, zstd::DefaultCompression);
+}
+#endif
 }
Index: llvm/test/lit.site.cfg.py.in
===
--- llvm/test/lit.site.cfg.py.in
+++ llvm/test/lit.site.cfg.py.in
@@ -37,6 +37,7 @@
 config.llvm_use_intel_jitevents = @LLVM_USE_INTEL_JITEVENTS@
 config.llvm_use_sanitizer = "@LLVM_USE_SANITIZER@"
 config.have_zlib = @LLVM_ENABLE_ZLIB@
+config.have_zstd = @LLVM_ENABLE_ZSTD@
 config.have_libxar = @LLVM_HAVE_LIBXAR@
 config.have_libxml2 = @LLVM_ENABLE_LIBXML2@
 config.have_curl = @LLVM_ENABLE_CURL@
Index: llvm/lib/Support/Compression.cpp
===
--- llvm/lib/Support/Compression.cpp
+++ llvm/lib/Support/Compression.cpp
@@ -20,6 +20,9 @@
 #if LLVM_ENABLE_ZLIB
 #include 
 #endif
+#if LLVM_ENABLE_ZSTD
+#include 
+#endif
 
 using namespace llvm;
 using namespace llvm::compression;
@@ -100,3 +103,65 @@
   llvm_unreachable("zlib::uncompress is unavailable");
 }
 #endif
+
+#if LLVM_ENABLE_ZSTD
+
+bool zstd::isAvailable() { return true; }
+
+void zstd::compress(ArrayRef Input,
+SmallVectorImpl &CompressedBuffer, int Level) {
+  unsigned long CompressedBufferSize = ::ZSTD_compressBound(Input.size());
+  CompressedBuffer.resize_for_overwrite(CompressedBufferSize);
+  unsigned long CompressedSize =
+  ::ZSTD_compress((char *)CompressedBuffer.data(), CompressedBufferSize,
+  (const char *)Input.data(), Input.size(), Level);
+  if (ZSTD_isError(CompressedSize))
+report_bad_alloc_error("Allocation failed");
+  // Tell MemorySanitizer that zstd output buffer is fully initialized.
+  // This avoids a false report when running LLVM with uninstrumented ZLib.
+  __msan_unpoison(CompressedBuffer.data(), CompressedSize);
+  if (CompressedSize < CompressedBuffer.size())
+CompressedBuffer.truncate(CompressedSize);
+}
+
+Error zstd::uncompress(ArrayRef Input, uint8_t *UncompressedBuffer,
+   size_t &UncompressedSize) {
+  co

[Lldb-commits] [PATCH] D128465: [llvm] add zstd to `llvm::compression` namespace

[Cole Kissane via Phabricator via lldb-commits]

ckissane added a comment.

In D128465#3651025 , @aemerson wrote:

> I just reverted this in 6e6be5f9504d 
>  because 
> it seems to have broken macOS builds:
>
>   llvm/lib/Support/Compression.cpp:24:10: fatal error: 'zstd.h' file not found
>   #include 
>^~~~

@aemerson Could you provide the output of your cmake command?
(I can't easily reproduce because I don't have a mac on hand)


Repository:
  rG LLVM Github Monorepo

CHANGES SINCE LAST ACTION
  https://reviews.llvm.org/D128465/new/

https://reviews.llvm.org/D128465

___
lldb-commits mailing list
lldb-commits@lists.llvm.org
https://lists.llvm.org/cgi-bin/mailman/listinfo/lldb-commits

[Lldb-commits] [PATCH] D128465: [llvm] add zstd to `llvm::compression` namespace

[Cole Kissane via Phabricator via lldb-commits]

ckissane added a comment.

@aemerson can you let me know if https://reviews.llvm.org/D129786 works on 
MacOS?
I am continuing work on that revision because this one has a lot of outdated 
unrelated history.


Repository:
  rG LLVM Github Monorepo

CHANGES SINCE LAST ACTION
  https://reviews.llvm.org/D128465/new/

https://reviews.llvm.org/D128465

___
lldb-commits mailing list
lldb-commits@lists.llvm.org
https://lists.llvm.org/cgi-bin/mailman/listinfo/lldb-commits

[Lldb-commits] [PATCH] D128465: [llvm] add zstd to `llvm::compression` namespace

[Cole Kissane via Phabricator via lldb-commits]

ckissane added a comment.

In D128465#3653051 , @jasonmolenda 
wrote:

> In D128465#3652525 , @ckissane 
> wrote:
>
>> In D128465#3651025 , @aemerson 
>> wrote:
>>
>>> I just reverted this in 6e6be5f9504d 
>>>  
>>> because it seems to have broken macOS builds:
>>>
>>>   llvm/lib/Support/Compression.cpp:24:10: fatal error: 'zstd.h' file not 
>>> found
>>>   #include 
>>>^~~~
>>
>> @aemerson Could you provide the output of your cmake command?
>> (I can't easily reproduce because I don't have a mac on hand)
>
> FWIW I hit this last night on my mac desktop - it looked like I had zstd 
> installed by homebrew (as a dependency on something I installed), and cmake 
> was able to find it in /opt/homebrew (somehow) but when Compression.cpp was 
> built, nothing had added -I/opt/homebrew/include so the header wasn't found.  
> I hacked the FindZSTD.cmake to not find it, to finish what I was working on 
> before bedtime.  This might not be related to what @aemerson saw though.

@jasonmolenda Thank you for the info, I believe it is similar, could you try: 
https://reviews.llvm.org/D129786 for me?


Repository:
  rG LLVM Github Monorepo

CHANGES SINCE LAST ACTION
  https://reviews.llvm.org/D128465/new/

https://reviews.llvm.org/D128465

___
lldb-commits mailing list
lldb-commits@lists.llvm.org
https://lists.llvm.org/cgi-bin/mailman/listinfo/lldb-commits

[Lldb-commits] [PATCH] D128465: [OLD] [llvm] add zstd to `llvm::compression` namespace

[Cole Kissane via Phabricator via lldb-commits]

ckissane added a comment.

In D128465#3660714 , @MaskRay wrote:

> Instead of marking a differential `[OLD] ` (which may confuse readers whether 
> this is to be abandoned), you can mark a differential `Request 
> Reviews`/`Request Changes`. Then it will not be in the green state.
>
> I clicked "Reopen" to make it clear the patch hasn't relanded.

Oh, sorry about that!
I had actually created https://reviews.llvm.org/D129786 instead of reopening 
this one because at the time I tried, reopening didn't seem to work.


Repository:
  rG LLVM Github Monorepo

CHANGES SINCE LAST ACTION
  https://reviews.llvm.org/D128465/new/

https://reviews.llvm.org/D128465

___
lldb-commits mailing list
lldb-commits@lists.llvm.org
https://lists.llvm.org/cgi-bin/mailman/listinfo/lldb-commits

[Lldb-commits] [PATCH] D128465: [llvm] add zstd to `llvm::compression` namespace

[Cole Kissane via Phabricator via lldb-commits]

ckissane added a comment.

In D128465#3661049 , @MaskRay wrote:

> Does this address the macOS build failure?

I believe so!


Repository:
  rG LLVM Github Monorepo

CHANGES SINCE LAST ACTION
  https://reviews.llvm.org/D128465/new/

https://reviews.llvm.org/D128465

___
lldb-commits mailing list
lldb-commits@lists.llvm.org
https://lists.llvm.org/cgi-bin/mailman/listinfo/lldb-commits

[Lldb-commits] [PATCH] D128465: [llvm] add zstd to `llvm::compression` namespace

[Cole Kissane via Phabricator via lldb-commits]

ckissane updated this revision to Diff 445623.
ckissane added a comment.

- remove extra cmake info


Repository:
  rG LLVM Github Monorepo

CHANGES SINCE LAST ACTION
  https://reviews.llvm.org/D128465/new/

https://reviews.llvm.org/D128465

Files:
  llvm/CMakeLists.txt
  llvm/cmake/config-ix.cmake
  llvm/cmake/modules/LLVMConfig.cmake.in
  llvm/include/llvm/Config/llvm-config.h.cmake
  llvm/include/llvm/Support/Compression.h
  llvm/lib/Support/CMakeLists.txt
  llvm/lib/Support/Compression.cpp
  llvm/test/lit.site.cfg.py.in
  llvm/unittests/Support/CompressionTest.cpp
  utils/bazel/llvm_configs/llvm-config.h.cmake

Index: utils/bazel/llvm_configs/llvm-config.h.cmake
===
--- utils/bazel/llvm_configs/llvm-config.h.cmake
+++ utils/bazel/llvm_configs/llvm-config.h.cmake
@@ -95,6 +95,9 @@
 /* Define if zlib compression is available */
 #cmakedefine01 LLVM_ENABLE_ZLIB
 
+/* Define if zstd compression is available */
+#cmakedefine01 LLVM_ENABLE_ZSTD
+
 /* Define if LLVM was built with a dependency to the libtensorflow dynamic library */
 #cmakedefine LLVM_HAVE_TF_API
 
Index: llvm/unittests/Support/CompressionTest.cpp
===
--- llvm/unittests/Support/CompressionTest.cpp
+++ llvm/unittests/Support/CompressionTest.cpp
@@ -61,4 +61,42 @@
 }
 #endif
 
+#if LLVM_ENABLE_ZSTD
+static void testZstdCompression(StringRef Input, int Level) {
+  SmallVector Compressed;
+  SmallVector Uncompressed;
+  zstd::compress(arrayRefFromStringRef(Input), Compressed, Level);
+
+  // Check that uncompressed buffer is the same as original.
+  Error E = zstd::uncompress(Compressed, Uncompressed, Input.size());
+  consumeError(std::move(E));
+
+  EXPECT_EQ(Input, toStringRef(Uncompressed));
+  if (Input.size() > 0) {
+// Uncompression fails if expected length is too short.
+E = zstd::uncompress(Compressed, Uncompressed, Input.size() - 1);
+EXPECT_EQ("Destination buffer is too small", llvm::toString(std::move(E)));
+  }
+}
+
+TEST(CompressionTest, Zstd) {
+  testZstdCompression("", zstd::DefaultCompression);
+
+  testZstdCompression("hello, world!", zstd::NoCompression);
+  testZstdCompression("hello, world!", zstd::BestSizeCompression);
+  testZstdCompression("hello, world!", zstd::BestSpeedCompression);
+  testZstdCompression("hello, world!", zstd::DefaultCompression);
+
+  const size_t kSize = 1024;
+  char BinaryData[kSize];
+  for (size_t i = 0; i < kSize; ++i)
+BinaryData[i] = i & 255;
+  StringRef BinaryDataStr(BinaryData, kSize);
+
+  testZstdCompression(BinaryDataStr, zstd::NoCompression);
+  testZstdCompression(BinaryDataStr, zstd::BestSizeCompression);
+  testZstdCompression(BinaryDataStr, zstd::BestSpeedCompression);
+  testZstdCompression(BinaryDataStr, zstd::DefaultCompression);
+}
+#endif
 }
Index: llvm/test/lit.site.cfg.py.in
===
--- llvm/test/lit.site.cfg.py.in
+++ llvm/test/lit.site.cfg.py.in
@@ -37,6 +37,7 @@
 config.llvm_use_intel_jitevents = @LLVM_USE_INTEL_JITEVENTS@
 config.llvm_use_sanitizer = "@LLVM_USE_SANITIZER@"
 config.have_zlib = @LLVM_ENABLE_ZLIB@
+config.have_zstd = @LLVM_ENABLE_ZSTD@
 config.have_libxar = @LLVM_HAVE_LIBXAR@
 config.have_libxml2 = @LLVM_ENABLE_LIBXML2@
 config.have_curl = @LLVM_ENABLE_CURL@
Index: llvm/lib/Support/Compression.cpp
===
--- llvm/lib/Support/Compression.cpp
+++ llvm/lib/Support/Compression.cpp
@@ -20,6 +20,9 @@
 #if LLVM_ENABLE_ZLIB
 #include 
 #endif
+#if LLVM_ENABLE_ZSTD
+#include 
+#endif
 
 using namespace llvm;
 using namespace llvm::compression;
@@ -100,3 +103,65 @@
   llvm_unreachable("zlib::uncompress is unavailable");
 }
 #endif
+
+#if LLVM_ENABLE_ZSTD
+
+bool zstd::isAvailable() { return true; }
+
+void zstd::compress(ArrayRef Input,
+SmallVectorImpl &CompressedBuffer, int Level) {
+  unsigned long CompressedBufferSize = ::ZSTD_compressBound(Input.size());
+  CompressedBuffer.resize_for_overwrite(CompressedBufferSize);
+  unsigned long CompressedSize =
+  ::ZSTD_compress((char *)CompressedBuffer.data(), CompressedBufferSize,
+  (const char *)Input.data(), Input.size(), Level);
+  if (ZSTD_isError(CompressedSize))
+report_bad_alloc_error("Allocation failed");
+  // Tell MemorySanitizer that zstd output buffer is fully initialized.
+  // This avoids a false report when running LLVM with uninstrumented ZLib.
+  __msan_unpoison(CompressedBuffer.data(), CompressedSize);
+  if (CompressedSize < CompressedBuffer.size())
+CompressedBuffer.truncate(CompressedSize);
+}
+
+Error zstd::uncompress(ArrayRef Input, uint8_t *UncompressedBuffer,
+   size_t &UncompressedSize) {
+  const size_t Res =
+  ::ZSTD_decompress(UncompressedBuffer, UncompressedSize,
+(const uint8_t *)Input.data(), Input.size());
+  UncompressedSize =

[Lldb-commits] [PATCH] D128465: [llvm] add zstd to `llvm::compression` namespace

[Cole Kissane via Phabricator via lldb-commits]

This revision was landed with ongoing or failed builds.
This revision was automatically updated to reflect the committed changes.
Closed by commit rGe939bf67e340: [llvm] add zstd to `llvm::compression` 
namespace (authored by ckissane).

Repository:
  rG LLVM Github Monorepo

CHANGES SINCE LAST ACTION
  https://reviews.llvm.org/D128465/new/

https://reviews.llvm.org/D128465

Files:
  llvm/CMakeLists.txt
  llvm/cmake/config-ix.cmake
  llvm/cmake/modules/LLVMConfig.cmake.in
  llvm/include/llvm/Config/llvm-config.h.cmake
  llvm/include/llvm/Support/Compression.h
  llvm/lib/Support/CMakeLists.txt
  llvm/lib/Support/Compression.cpp
  llvm/test/lit.site.cfg.py.in
  llvm/unittests/Support/CompressionTest.cpp
  utils/bazel/llvm_configs/llvm-config.h.cmake

Index: utils/bazel/llvm_configs/llvm-config.h.cmake
===
--- utils/bazel/llvm_configs/llvm-config.h.cmake
+++ utils/bazel/llvm_configs/llvm-config.h.cmake
@@ -95,6 +95,9 @@
 /* Define if zlib compression is available */
 #cmakedefine01 LLVM_ENABLE_ZLIB
 
+/* Define if zstd compression is available */
+#cmakedefine01 LLVM_ENABLE_ZSTD
+
 /* Define if LLVM was built with a dependency to the libtensorflow dynamic library */
 #cmakedefine LLVM_HAVE_TF_API
 
Index: llvm/unittests/Support/CompressionTest.cpp
===
--- llvm/unittests/Support/CompressionTest.cpp
+++ llvm/unittests/Support/CompressionTest.cpp
@@ -61,4 +61,42 @@
 }
 #endif
 
+#if LLVM_ENABLE_ZSTD
+static void testZstdCompression(StringRef Input, int Level) {
+  SmallVector Compressed;
+  SmallVector Uncompressed;
+  zstd::compress(arrayRefFromStringRef(Input), Compressed, Level);
+
+  // Check that uncompressed buffer is the same as original.
+  Error E = zstd::uncompress(Compressed, Uncompressed, Input.size());
+  consumeError(std::move(E));
+
+  EXPECT_EQ(Input, toStringRef(Uncompressed));
+  if (Input.size() > 0) {
+// Uncompression fails if expected length is too short.
+E = zstd::uncompress(Compressed, Uncompressed, Input.size() - 1);
+EXPECT_EQ("Destination buffer is too small", llvm::toString(std::move(E)));
+  }
+}
+
+TEST(CompressionTest, Zstd) {
+  testZstdCompression("", zstd::DefaultCompression);
+
+  testZstdCompression("hello, world!", zstd::NoCompression);
+  testZstdCompression("hello, world!", zstd::BestSizeCompression);
+  testZstdCompression("hello, world!", zstd::BestSpeedCompression);
+  testZstdCompression("hello, world!", zstd::DefaultCompression);
+
+  const size_t kSize = 1024;
+  char BinaryData[kSize];
+  for (size_t i = 0; i < kSize; ++i)
+BinaryData[i] = i & 255;
+  StringRef BinaryDataStr(BinaryData, kSize);
+
+  testZstdCompression(BinaryDataStr, zstd::NoCompression);
+  testZstdCompression(BinaryDataStr, zstd::BestSizeCompression);
+  testZstdCompression(BinaryDataStr, zstd::BestSpeedCompression);
+  testZstdCompression(BinaryDataStr, zstd::DefaultCompression);
+}
+#endif
 }
Index: llvm/test/lit.site.cfg.py.in
===
--- llvm/test/lit.site.cfg.py.in
+++ llvm/test/lit.site.cfg.py.in
@@ -37,6 +37,7 @@
 config.llvm_use_intel_jitevents = @LLVM_USE_INTEL_JITEVENTS@
 config.llvm_use_sanitizer = "@LLVM_USE_SANITIZER@"
 config.have_zlib = @LLVM_ENABLE_ZLIB@
+config.have_zstd = @LLVM_ENABLE_ZSTD@
 config.have_libxar = @LLVM_HAVE_LIBXAR@
 config.have_libxml2 = @LLVM_ENABLE_LIBXML2@
 config.have_curl = @LLVM_ENABLE_CURL@
Index: llvm/lib/Support/Compression.cpp
===
--- llvm/lib/Support/Compression.cpp
+++ llvm/lib/Support/Compression.cpp
@@ -20,6 +20,9 @@
 #if LLVM_ENABLE_ZLIB
 #include 
 #endif
+#if LLVM_ENABLE_ZSTD
+#include 
+#endif
 
 using namespace llvm;
 using namespace llvm::compression;
@@ -100,3 +103,65 @@
   llvm_unreachable("zlib::uncompress is unavailable");
 }
 #endif
+
+#if LLVM_ENABLE_ZSTD
+
+bool zstd::isAvailable() { return true; }
+
+void zstd::compress(ArrayRef Input,
+SmallVectorImpl &CompressedBuffer, int Level) {
+  unsigned long CompressedBufferSize = ::ZSTD_compressBound(Input.size());
+  CompressedBuffer.resize_for_overwrite(CompressedBufferSize);
+  unsigned long CompressedSize =
+  ::ZSTD_compress((char *)CompressedBuffer.data(), CompressedBufferSize,
+  (const char *)Input.data(), Input.size(), Level);
+  if (ZSTD_isError(CompressedSize))
+report_bad_alloc_error("Allocation failed");
+  // Tell MemorySanitizer that zstd output buffer is fully initialized.
+  // This avoids a false report when running LLVM with uninstrumented ZLib.
+  __msan_unpoison(CompressedBuffer.data(), CompressedSize);
+  if (CompressedSize < CompressedBuffer.size())
+CompressedBuffer.truncate(CompressedSize);
+}
+
+Error zstd::uncompress(ArrayRef Input, uint8_t *UncompressedBuffer,
+   size_t &UncompressedSize) {
+  const size_t Res =
+  ::ZSTD_decom

[Lldb-commits] [PATCH] D128465: [llvm] add zstd to `llvm::compression` namespace

[Cole Kissane via Phabricator via lldb-commits]

ckissane updated this revision to Diff 445615.
ckissane added a comment.

- move try_find_dependency into it's own file
- add newline to end of TryFindDependencyMacro.cmake
- clarify comment in TryFindDependencyMacro.cmake


Repository:
  rG LLVM Github Monorepo

CHANGES SINCE LAST ACTION
  https://reviews.llvm.org/D128465/new/

https://reviews.llvm.org/D128465

Files:
  llvm/CMakeLists.txt
  llvm/cmake/config-ix.cmake
  llvm/cmake/modules/LLVMConfig.cmake.in
  llvm/cmake/modules/TryFindDependencyMacro.cmake
  llvm/include/llvm/Config/llvm-config.h.cmake
  llvm/include/llvm/Support/Compression.h
  llvm/lib/Support/CMakeLists.txt
  llvm/lib/Support/Compression.cpp
  llvm/test/lit.site.cfg.py.in
  llvm/unittests/Support/CompressionTest.cpp
  utils/bazel/llvm_configs/llvm-config.h.cmake

Index: utils/bazel/llvm_configs/llvm-config.h.cmake
===
--- utils/bazel/llvm_configs/llvm-config.h.cmake
+++ utils/bazel/llvm_configs/llvm-config.h.cmake
@@ -95,6 +95,9 @@
 /* Define if zlib compression is available */
 #cmakedefine01 LLVM_ENABLE_ZLIB
 
+/* Define if zstd compression is available */
+#cmakedefine01 LLVM_ENABLE_ZSTD
+
 /* Define if LLVM was built with a dependency to the libtensorflow dynamic library */
 #cmakedefine LLVM_HAVE_TF_API
 
Index: llvm/unittests/Support/CompressionTest.cpp
===
--- llvm/unittests/Support/CompressionTest.cpp
+++ llvm/unittests/Support/CompressionTest.cpp
@@ -61,4 +61,42 @@
 }
 #endif
 
+#if LLVM_ENABLE_ZSTD
+static void testZstdCompression(StringRef Input, int Level) {
+  SmallVector Compressed;
+  SmallVector Uncompressed;
+  zstd::compress(arrayRefFromStringRef(Input), Compressed, Level);
+
+  // Check that uncompressed buffer is the same as original.
+  Error E = zstd::uncompress(Compressed, Uncompressed, Input.size());
+  consumeError(std::move(E));
+
+  EXPECT_EQ(Input, toStringRef(Uncompressed));
+  if (Input.size() > 0) {
+// Uncompression fails if expected length is too short.
+E = zstd::uncompress(Compressed, Uncompressed, Input.size() - 1);
+EXPECT_EQ("Destination buffer is too small", llvm::toString(std::move(E)));
+  }
+}
+
+TEST(CompressionTest, Zstd) {
+  testZstdCompression("", zstd::DefaultCompression);
+
+  testZstdCompression("hello, world!", zstd::NoCompression);
+  testZstdCompression("hello, world!", zstd::BestSizeCompression);
+  testZstdCompression("hello, world!", zstd::BestSpeedCompression);
+  testZstdCompression("hello, world!", zstd::DefaultCompression);
+
+  const size_t kSize = 1024;
+  char BinaryData[kSize];
+  for (size_t i = 0; i < kSize; ++i)
+BinaryData[i] = i & 255;
+  StringRef BinaryDataStr(BinaryData, kSize);
+
+  testZstdCompression(BinaryDataStr, zstd::NoCompression);
+  testZstdCompression(BinaryDataStr, zstd::BestSizeCompression);
+  testZstdCompression(BinaryDataStr, zstd::BestSpeedCompression);
+  testZstdCompression(BinaryDataStr, zstd::DefaultCompression);
+}
+#endif
 }
Index: llvm/test/lit.site.cfg.py.in
===
--- llvm/test/lit.site.cfg.py.in
+++ llvm/test/lit.site.cfg.py.in
@@ -37,6 +37,7 @@
 config.llvm_use_intel_jitevents = @LLVM_USE_INTEL_JITEVENTS@
 config.llvm_use_sanitizer = "@LLVM_USE_SANITIZER@"
 config.have_zlib = @LLVM_ENABLE_ZLIB@
+config.have_zstd = @LLVM_ENABLE_ZSTD@
 config.have_libxar = @LLVM_HAVE_LIBXAR@
 config.have_libxml2 = @LLVM_ENABLE_LIBXML2@
 config.have_curl = @LLVM_ENABLE_CURL@
Index: llvm/lib/Support/Compression.cpp
===
--- llvm/lib/Support/Compression.cpp
+++ llvm/lib/Support/Compression.cpp
@@ -20,6 +20,9 @@
 #if LLVM_ENABLE_ZLIB
 #include 
 #endif
+#if LLVM_ENABLE_ZSTD
+#include 
+#endif
 
 using namespace llvm;
 using namespace llvm::compression;
@@ -100,3 +103,65 @@
   llvm_unreachable("zlib::uncompress is unavailable");
 }
 #endif
+
+#if LLVM_ENABLE_ZSTD
+
+bool zstd::isAvailable() { return true; }
+
+void zstd::compress(ArrayRef Input,
+SmallVectorImpl &CompressedBuffer, int Level) {
+  unsigned long CompressedBufferSize = ::ZSTD_compressBound(Input.size());
+  CompressedBuffer.resize_for_overwrite(CompressedBufferSize);
+  unsigned long CompressedSize =
+  ::ZSTD_compress((char *)CompressedBuffer.data(), CompressedBufferSize,
+  (const char *)Input.data(), Input.size(), Level);
+  if (ZSTD_isError(CompressedSize))
+report_bad_alloc_error("Allocation failed");
+  // Tell MemorySanitizer that zstd output buffer is fully initialized.
+  // This avoids a false report when running LLVM with uninstrumented ZLib.
+  __msan_unpoison(CompressedBuffer.data(), CompressedSize);
+  if (CompressedSize < CompressedBuffer.size())
+CompressedBuffer.truncate(CompressedSize);
+}
+
+Error zstd::uncompress(ArrayRef Input, uint8_t *UncompressedBuffer,
+   size_t &UncompressedSize) {
+  c

[Lldb-commits] [PATCH] D133525: fix extra bytes when compressing for 32bit objcopy

[Cole Kissane via Phabricator via lldb-commits]

ckissane created this revision.
ckissane added a reviewer: MaskRay.
Herald added subscribers: StephenFan, abrachet, hiraditya, mgorny, emaste.
Herald added a reviewer: jhenderson.
Herald added a project: All.
ckissane requested review of this revision.
Herald added projects: LLDB, LLVM.
Herald added subscribers: llvm-commits, lldb-commits.

fixes extra bytes that zlib is "ok" with but zstd detects and errors on when 
decompressing


Repository:
  rG LLVM Github Monorepo

https://reviews.llvm.org/D133525

Files:
  lldb/test/CMakeLists.txt
  lldb/test/Shell/ObjectFile/ELF/compressed-sections-zstd.yaml
  lldb/test/Shell/lit.site.cfg.py.in
  llvm/include/llvm/Object/Decompressor.h
  llvm/include/llvm/Support/Compression.h
  llvm/lib/ObjCopy/ELF/ELFObjcopy.cpp
  llvm/lib/ObjCopy/ELF/ELFObject.cpp
  llvm/lib/ObjCopy/ELF/ELFObject.h
  llvm/lib/Object/Decompressor.cpp
  llvm/lib/Support/Compression.cpp

Index: llvm/lib/Support/Compression.cpp
===
--- llvm/lib/Support/Compression.cpp
+++ llvm/lib/Support/Compression.cpp
@@ -73,6 +73,22 @@
   return decompress(formatFor(T), Input, Output, UncompressedSize);
 }
 
+Error compression::decompress(compression::Format F, ArrayRef Input,
+  uint8_t *Output, size_t &UncompressedSize) {
+  switch (F) {
+  case compression::Format::Zlib:
+return zlib::uncompress(Input, Output, UncompressedSize);
+  case compression::Format::Zstd:
+return zstd::uncompress(Input, Output, UncompressedSize);
+  }
+  llvm_unreachable("");
+}
+
+Error compression::decompress(DebugCompressionType T, ArrayRef Input,
+  uint8_t *Output, size_t &UncompressedSize) {
+  return decompress(formatFor(T), Input, Output, UncompressedSize);
+}
+
 #if LLVM_ENABLE_ZLIB
 
 static StringRef convertZlibCodeToString(int Code) {
Index: llvm/lib/Object/Decompressor.cpp
===
--- llvm/lib/Object/Decompressor.cpp
+++ llvm/lib/Object/Decompressor.cpp
@@ -14,16 +14,15 @@
 #include "llvm/Support/Endian.h"
 
 using namespace llvm;
+using namespace llvm::compression;
 using namespace llvm::support::endian;
 using namespace object;
 
 Expected Decompressor::create(StringRef Name, StringRef Data,
 bool IsLE, bool Is64Bit) {
-  if (!compression::zlib::isAvailable())
-return createError("zlib is not available");
 
   Decompressor D(Data);
-  if (Error Err = D.consumeCompressedZLibHeader(Is64Bit, IsLE))
+  if (Error Err = D.consumeCompressedSectionHeader(Is64Bit, IsLE))
 return std::move(Err);
   return D;
 }
@@ -31,8 +30,8 @@
 Decompressor::Decompressor(StringRef Data)
 : SectionData(Data), DecompressedSize(0) {}
 
-Error Decompressor::consumeCompressedZLibHeader(bool Is64Bit,
-bool IsLittleEndian) {
+Error Decompressor::consumeCompressedSectionHeader(bool Is64Bit,
+   bool IsLittleEndian) {
   using namespace ELF;
   uint64_t HdrSize = Is64Bit ? sizeof(Elf64_Chdr) : sizeof(Elf32_Chdr);
   if (SectionData.size() < HdrSize)
@@ -40,10 +39,25 @@
 
   DataExtractor Extractor(SectionData, IsLittleEndian, 0);
   uint64_t Offset = 0;
-  if (Extractor.getUnsigned(&Offset, Is64Bit ? sizeof(Elf64_Word)
- : sizeof(Elf32_Word)) !=
-  ELFCOMPRESS_ZLIB)
+  uint64_t ELFCompressionSchemeId = Extractor.getUnsigned(
+  &Offset, Is64Bit ? sizeof(Elf64_Word) : sizeof(Elf32_Word));
+
+  if (ELFCompressionSchemeId == ELFCOMPRESS_ZLIB)
+CompressionType = llvm::DebugCompressionType::Z;
+  if (ELFCompressionSchemeId == ELFCOMPRESS_ZSTD)
+CompressionType = llvm::DebugCompressionType::Zstd;
+
+  if (CompressionType == llvm::DebugCompressionType::None)
 return createError("unsupported compression type");
+  if (llvm::compression::getReasonIfUnsupported(
+  compression::formatFor(CompressionType)) != nullptr) {
+switch (compression::formatFor(CompressionType)) {
+case llvm::compression::Format::Zstd:
+  return createError("zstd is not available");
+default:
+  return createError("zlib is not available");
+}
+  }
 
   // Skip Elf64_Chdr::ch_reserved field.
   if (Is64Bit)
@@ -57,6 +71,7 @@
 
 Error Decompressor::decompress(MutableArrayRef Buffer) {
   size_t Size = Buffer.size();
-  return compression::zlib::uncompress(arrayRefFromStringRef(SectionData),
-   Buffer.data(), Size);
+  return compression::decompress(compression::formatFor(CompressionType),
+ arrayRefFromStringRef(SectionData),
+ Buffer.data(), Size);
 }
Index: llvm/lib/ObjCopy/ELF/ELFObject.h
===
--- llvm/lib/ObjCopy/ELF/ELFObject.h
+++ llvm/lib/ObjCopy/ELF/ELFObject.h
@@ -537,6 +537,7 @@
   MAKE_SEC_WRI

[Lldb-commits] [PATCH] D133525: fix extra bytes when compressing for 32bit objcopy, and add zstd support to Decompressor

[Cole Kissane via Phabricator via lldb-commits]

ckissane updated this revision to Diff 458871.
ckissane added a comment.

- fix extra bytes when compressing for 32bit objcopy
- fix lldb compressed section zstd test File Size


Repository:
  rG LLVM Github Monorepo

CHANGES SINCE LAST ACTION
  https://reviews.llvm.org/D133525/new/

https://reviews.llvm.org/D133525

Files:
  lldb/test/CMakeLists.txt
  lldb/test/Shell/ObjectFile/ELF/compressed-sections-zstd.yaml
  lldb/test/Shell/lit.site.cfg.py.in
  llvm/include/llvm/Object/Decompressor.h
  llvm/include/llvm/Support/Compression.h
  llvm/lib/ObjCopy/ELF/ELFObjcopy.cpp
  llvm/lib/ObjCopy/ELF/ELFObject.cpp
  llvm/lib/ObjCopy/ELF/ELFObject.h
  llvm/lib/Object/Decompressor.cpp
  llvm/lib/Support/Compression.cpp

Index: llvm/lib/Support/Compression.cpp
===
--- llvm/lib/Support/Compression.cpp
+++ llvm/lib/Support/Compression.cpp
@@ -73,6 +73,22 @@
   return decompress(formatFor(T), Input, Output, UncompressedSize);
 }
 
+Error compression::decompress(compression::Format F, ArrayRef Input,
+  uint8_t *Output, size_t &UncompressedSize) {
+  switch (F) {
+  case compression::Format::Zlib:
+return zlib::uncompress(Input, Output, UncompressedSize);
+  case compression::Format::Zstd:
+return zstd::uncompress(Input, Output, UncompressedSize);
+  }
+  llvm_unreachable("");
+}
+
+Error compression::decompress(DebugCompressionType T, ArrayRef Input,
+  uint8_t *Output, size_t &UncompressedSize) {
+  return decompress(formatFor(T), Input, Output, UncompressedSize);
+}
+
 #if LLVM_ENABLE_ZLIB
 
 static StringRef convertZlibCodeToString(int Code) {
Index: llvm/lib/Object/Decompressor.cpp
===
--- llvm/lib/Object/Decompressor.cpp
+++ llvm/lib/Object/Decompressor.cpp
@@ -14,16 +14,15 @@
 #include "llvm/Support/Endian.h"
 
 using namespace llvm;
+using namespace llvm::compression;
 using namespace llvm::support::endian;
 using namespace object;
 
 Expected Decompressor::create(StringRef Name, StringRef Data,
 bool IsLE, bool Is64Bit) {
-  if (!compression::zlib::isAvailable())
-return createError("zlib is not available");
 
   Decompressor D(Data);
-  if (Error Err = D.consumeCompressedZLibHeader(Is64Bit, IsLE))
+  if (Error Err = D.consumeCompressedSectionHeader(Is64Bit, IsLE))
 return std::move(Err);
   return D;
 }
@@ -31,8 +30,8 @@
 Decompressor::Decompressor(StringRef Data)
 : SectionData(Data), DecompressedSize(0) {}
 
-Error Decompressor::consumeCompressedZLibHeader(bool Is64Bit,
-bool IsLittleEndian) {
+Error Decompressor::consumeCompressedSectionHeader(bool Is64Bit,
+   bool IsLittleEndian) {
   using namespace ELF;
   uint64_t HdrSize = Is64Bit ? sizeof(Elf64_Chdr) : sizeof(Elf32_Chdr);
   if (SectionData.size() < HdrSize)
@@ -40,10 +39,25 @@
 
   DataExtractor Extractor(SectionData, IsLittleEndian, 0);
   uint64_t Offset = 0;
-  if (Extractor.getUnsigned(&Offset, Is64Bit ? sizeof(Elf64_Word)
- : sizeof(Elf32_Word)) !=
-  ELFCOMPRESS_ZLIB)
+  uint64_t ELFCompressionSchemeId = Extractor.getUnsigned(
+  &Offset, Is64Bit ? sizeof(Elf64_Word) : sizeof(Elf32_Word));
+
+  if (ELFCompressionSchemeId == ELFCOMPRESS_ZLIB)
+CompressionType = llvm::DebugCompressionType::Z;
+  if (ELFCompressionSchemeId == ELFCOMPRESS_ZSTD)
+CompressionType = llvm::DebugCompressionType::Zstd;
+
+  if (CompressionType == llvm::DebugCompressionType::None)
 return createError("unsupported compression type");
+  if (llvm::compression::getReasonIfUnsupported(
+  compression::formatFor(CompressionType)) != nullptr) {
+switch (compression::formatFor(CompressionType)) {
+case llvm::compression::Format::Zstd:
+  return createError("zstd is not available");
+default:
+  return createError("zlib is not available");
+}
+  }
 
   // Skip Elf64_Chdr::ch_reserved field.
   if (Is64Bit)
@@ -57,6 +71,7 @@
 
 Error Decompressor::decompress(MutableArrayRef Buffer) {
   size_t Size = Buffer.size();
-  return compression::zlib::uncompress(arrayRefFromStringRef(SectionData),
-   Buffer.data(), Size);
+  return compression::decompress(compression::formatFor(CompressionType),
+ arrayRefFromStringRef(SectionData),
+ Buffer.data(), Size);
 }
Index: llvm/lib/ObjCopy/ELF/ELFObject.h
===
--- llvm/lib/ObjCopy/ELF/ELFObject.h
+++ llvm/lib/ObjCopy/ELF/ELFObject.h
@@ -537,6 +537,7 @@
   MAKE_SEC_WRITER_FRIEND
 
   uint32_t ChType = 0;
+  bool Is64Bits;
   DebugCompressionType CompressionType;
   uint64_t DecompressedSize;
   uint64_t DecompressedAlign;
@@ -544,9 +545,10 @@
 
 public:
  

[Lldb-commits] [PATCH] D133525: fix extra bytes when compressing for 32bit objcopy, and add zstd support to Decompressor

[Cole Kissane via Phabricator via lldb-commits]

ckissane updated this revision to Diff 458875.
ckissane added a comment.

- remove lldb changes


Repository:
  rG LLVM Github Monorepo

CHANGES SINCE LAST ACTION
  https://reviews.llvm.org/D133525/new/

https://reviews.llvm.org/D133525

Files:
  llvm/lib/ObjCopy/ELF/ELFObjcopy.cpp
  llvm/lib/ObjCopy/ELF/ELFObject.cpp
  llvm/lib/ObjCopy/ELF/ELFObject.h

Index: llvm/lib/ObjCopy/ELF/ELFObject.h
===
--- llvm/lib/ObjCopy/ELF/ELFObject.h
+++ llvm/lib/ObjCopy/ELF/ELFObject.h
@@ -537,6 +537,7 @@
   MAKE_SEC_WRITER_FRIEND
 
   uint32_t ChType = 0;
+  bool Is64Bits;
   DebugCompressionType CompressionType;
   uint64_t DecompressedSize;
   uint64_t DecompressedAlign;
@@ -544,9 +545,10 @@
 
 public:
   CompressedSection(const SectionBase &Sec,
-DebugCompressionType CompressionType);
+DebugCompressionType CompressionType, bool Is64Bits);
   CompressedSection(ArrayRef CompressedData, uint32_t ChType,
-uint64_t DecompressedSize, uint64_t DecompressedAlign);
+uint64_t DecompressedSize, uint64_t DecompressedAlign,
+bool Is64Bits);
 
   uint64_t getDecompressedSize() const { return DecompressedSize; }
   uint64_t getDecompressedAlign() const { return DecompressedAlign; }
Index: llvm/lib/ObjCopy/ELF/ELFObject.cpp
===
--- llvm/lib/ObjCopy/ELF/ELFObject.cpp
+++ llvm/lib/ObjCopy/ELF/ELFObject.cpp
@@ -527,26 +527,28 @@
 }
 
 CompressedSection::CompressedSection(const SectionBase &Sec,
- DebugCompressionType CompressionType)
-: SectionBase(Sec), CompressionType(CompressionType),
+ DebugCompressionType CompressionType,
+ bool Is64Bits)
+: SectionBase(Sec), Is64Bits(Is64Bits), CompressionType(CompressionType),
   DecompressedSize(Sec.OriginalData.size()), DecompressedAlign(Sec.Align) {
   compression::compress(compression::Params(CompressionType), OriginalData,
 CompressedData);
 
   Flags |= ELF::SHF_COMPRESSED;
   size_t ChdrSize =
-  std::max(std::max(sizeof(object::Elf_Chdr_Impl),
-sizeof(object::Elf_Chdr_Impl)),
-   std::max(sizeof(object::Elf_Chdr_Impl),
-sizeof(object::Elf_Chdr_Impl)));
+  Is64Bits ? std::max(sizeof(object::Elf_Chdr_Impl),
+  sizeof(object::Elf_Chdr_Impl))
+   : std::max(sizeof(object::Elf_Chdr_Impl),
+  sizeof(object::Elf_Chdr_Impl));
   Size = ChdrSize + CompressedData.size();
   Align = 8;
 }
 
 CompressedSection::CompressedSection(ArrayRef CompressedData,
  uint32_t ChType, uint64_t DecompressedSize,
- uint64_t DecompressedAlign)
-: ChType(ChType), CompressionType(DebugCompressionType::None),
+ uint64_t DecompressedAlign, bool Is64Bits)
+: ChType(ChType), Is64Bits(Is64Bits),
+  CompressionType(DebugCompressionType::None),
   DecompressedSize(DecompressedSize), DecompressedAlign(DecompressedAlign) {
   OriginalData = CompressedData;
 }
@@ -1724,8 +1726,9 @@
 if (!(Shdr.sh_flags & ELF::SHF_COMPRESSED))
   return Obj.addSection(*Data);
 auto *Chdr = reinterpret_cast *>(Data->data());
-return Obj.addSection(CompressedSection(
-*Data, Chdr->ch_type, Chdr->ch_size, Chdr->ch_addralign));
+return Obj.addSection(
+CompressedSection(*Data, Chdr->ch_type, Chdr->ch_size,
+  Chdr->ch_addralign, ELFT::Is64Bits));
   }
   }
 }
Index: llvm/lib/ObjCopy/ELF/ELFObjcopy.cpp
===
--- llvm/lib/ObjCopy/ELF/ELFObjcopy.cpp
+++ llvm/lib/ObjCopy/ELF/ELFObjcopy.cpp
@@ -505,12 +505,17 @@
   if (Error E = Obj.removeSections(ELFConfig.AllowBrokenLinks, RemovePred))
 return E;
 
+  const ElfType OutputElfType =
+  getOutputElfType(Config.OutputArch.value_or(MachineInfo()));
+  const bool Is64Bit =
+  OutputElfType == ELFT_ELF64LE || OutputElfType == ELFT_ELF64BE;
   if (Config.CompressionType != DebugCompressionType::None) {
 if (Error Err = replaceDebugSections(
 Obj, isCompressable,
-[&Config, &Obj](const SectionBase *S) -> Expected {
+[&Config, &Obj,
+ Is64Bit](const SectionBase *S) -> Expected {
   return &Obj.addSection(
-  CompressedSection(*S, Config.CompressionType));
+  CompressedSection(*S, Config.CompressionType, Is64Bit));
 }))
   return Err;
   } else if (Config.DecompressDebugSections) {
___
lldb-commits mailing list
lldb-commits@lists.llvm.org
https://lists.llvm.org/cgi-bin/mailman/listinfo/lldb-commits

[Lldb-commits] [PATCH] D133530: add zstd support for lldb

[Cole Kissane via Phabricator via lldb-commits]

ckissane created this revision.
ckissane added reviewers: MaskRay, phosek.
Herald added subscribers: StephenFan, hiraditya, mgorny, emaste.
Herald added a project: All.
ckissane requested review of this revision.
Herald added projects: LLDB, LLVM.
Herald added subscribers: llvm-commits, lldb-commits.

adds zstd support to Decompressor so lldb can work with zstd


Repository:
  rG LLVM Github Monorepo

https://reviews.llvm.org/D133530

Files:
  lldb/test/CMakeLists.txt
  lldb/test/Shell/ObjectFile/ELF/compressed-sections-zstd.yaml
  lldb/test/Shell/lit.site.cfg.py.in
  llvm/include/llvm/Object/Decompressor.h
  llvm/include/llvm/Support/Compression.h
  llvm/lib/Object/Decompressor.cpp
  llvm/lib/Support/Compression.cpp

Index: llvm/lib/Support/Compression.cpp
===
--- llvm/lib/Support/Compression.cpp
+++ llvm/lib/Support/Compression.cpp
@@ -73,6 +73,22 @@
   return decompress(formatFor(T), Input, Output, UncompressedSize);
 }
 
+Error compression::decompress(compression::Format F, ArrayRef Input,
+  uint8_t *Output, size_t &UncompressedSize) {
+  switch (F) {
+  case compression::Format::Zlib:
+return zlib::uncompress(Input, Output, UncompressedSize);
+  case compression::Format::Zstd:
+return zstd::uncompress(Input, Output, UncompressedSize);
+  }
+  llvm_unreachable("");
+}
+
+Error compression::decompress(DebugCompressionType T, ArrayRef Input,
+  uint8_t *Output, size_t &UncompressedSize) {
+  return decompress(formatFor(T), Input, Output, UncompressedSize);
+}
+
 #if LLVM_ENABLE_ZLIB
 
 static StringRef convertZlibCodeToString(int Code) {
Index: llvm/lib/Object/Decompressor.cpp
===
--- llvm/lib/Object/Decompressor.cpp
+++ llvm/lib/Object/Decompressor.cpp
@@ -14,16 +14,15 @@
 #include "llvm/Support/Endian.h"
 
 using namespace llvm;
+using namespace llvm::compression;
 using namespace llvm::support::endian;
 using namespace object;
 
 Expected Decompressor::create(StringRef Name, StringRef Data,
 bool IsLE, bool Is64Bit) {
-  if (!compression::zlib::isAvailable())
-return createError("zlib is not available");
 
   Decompressor D(Data);
-  if (Error Err = D.consumeCompressedZLibHeader(Is64Bit, IsLE))
+  if (Error Err = D.consumeCompressedSectionHeader(Is64Bit, IsLE))
 return std::move(Err);
   return D;
 }
@@ -31,8 +30,8 @@
 Decompressor::Decompressor(StringRef Data)
 : SectionData(Data), DecompressedSize(0) {}
 
-Error Decompressor::consumeCompressedZLibHeader(bool Is64Bit,
-bool IsLittleEndian) {
+Error Decompressor::consumeCompressedSectionHeader(bool Is64Bit,
+   bool IsLittleEndian) {
   using namespace ELF;
   uint64_t HdrSize = Is64Bit ? sizeof(Elf64_Chdr) : sizeof(Elf32_Chdr);
   if (SectionData.size() < HdrSize)
@@ -40,10 +39,25 @@
 
   DataExtractor Extractor(SectionData, IsLittleEndian, 0);
   uint64_t Offset = 0;
-  if (Extractor.getUnsigned(&Offset, Is64Bit ? sizeof(Elf64_Word)
- : sizeof(Elf32_Word)) !=
-  ELFCOMPRESS_ZLIB)
+  uint64_t ELFCompressionSchemeId = Extractor.getUnsigned(
+  &Offset, Is64Bit ? sizeof(Elf64_Word) : sizeof(Elf32_Word));
+
+  if (ELFCompressionSchemeId == ELFCOMPRESS_ZLIB)
+CompressionType = llvm::DebugCompressionType::Z;
+  if (ELFCompressionSchemeId == ELFCOMPRESS_ZSTD)
+CompressionType = llvm::DebugCompressionType::Zstd;
+
+  if (CompressionType == llvm::DebugCompressionType::None)
 return createError("unsupported compression type");
+  if (llvm::compression::getReasonIfUnsupported(
+  compression::formatFor(CompressionType)) != nullptr) {
+switch (compression::formatFor(CompressionType)) {
+case llvm::compression::Format::Zstd:
+  return createError("zstd is not available");
+default:
+  return createError("zlib is not available");
+}
+  }
 
   // Skip Elf64_Chdr::ch_reserved field.
   if (Is64Bit)
@@ -57,6 +71,7 @@
 
 Error Decompressor::decompress(MutableArrayRef Buffer) {
   size_t Size = Buffer.size();
-  return compression::zlib::uncompress(arrayRefFromStringRef(SectionData),
-   Buffer.data(), Size);
+  return compression::decompress(compression::formatFor(CompressionType),
+ arrayRefFromStringRef(SectionData),
+ Buffer.data(), Size);
 }
Index: llvm/include/llvm/Support/Compression.h
===
--- llvm/include/llvm/Support/Compression.h
+++ llvm/include/llvm/Support/Compression.h
@@ -121,6 +121,11 @@
 Error decompress(DebugCompressionType T, ArrayRef Input,
  SmallVectorImpl &Output, size_t UncompressedSize);
 
+Error decompress(Format F, ArrayRef Input, u