https://github.com/jhuber6 updated https://github.com/llvm/llvm-project/pull/222773
>From 965a7714b47ebae18cf8c11429b41fa0f5f32cd4 Mon Sep 17 00:00:00 2001 From: Joseph Huber <[email protected]> Date: Fri, 18 Sep 2026 11:51:07 -0500 Subject: [PATCH] [Support] Infer compression format from zlib and zstd headers Summary: Identify compressed streams from their bitstream headers so callers can decompress without naming the format. Recognize every valid RFC 1950 zlib header and the zstd frame magic, and reuse that for getReasonIfUnsupported and decompress. Collapse the AST reader onto the new helpers. --- clang/lib/Serialization/ASTReader.cpp | 12 ++---- llvm/include/llvm/Support/Compression.h | 7 ++++ llvm/lib/Support/Compression.cpp | 48 ++++++++++++++++++++++ llvm/unittests/Support/CompressionTest.cpp | 42 +++++++++++++++++++ 4 files changed, 101 insertions(+), 8 deletions(-) diff --git a/clang/lib/Serialization/ASTReader.cpp b/clang/lib/Serialization/ASTReader.cpp index a9c230d767c509..f327ddb1b84348 100644 --- a/clang/lib/Serialization/ASTReader.cpp +++ b/clang/lib/Serialization/ASTReader.cpp @@ -1927,19 +1927,15 @@ bool ASTReader::ReadSLocEntry(int ID) { unsigned RecCode = MaybeRecCode.get(); if (RecCode == SM_SLOC_BUFFER_BLOB_COMPRESSED) { - // Inspect the first byte to differentiate zlib (\x78) and zstd - // (little-endian 0xFD2FB528). - const llvm::compression::Format F = - Blob.size() > 0 && Blob.data()[0] == 0x78 - ? llvm::compression::Format::Zlib - : llvm::compression::Format::Zstd; - if (const char *Reason = llvm::compression::getReasonIfUnsupported(F)) { + ArrayRef<uint8_t> Compressed = llvm::arrayRefFromStringRef(Blob); + if (const char *Reason = + llvm::compression::getReasonIfUnsupported(Compressed)) { Error(Reason); return nullptr; } SmallVector<uint8_t, 0> Decompressed; if (llvm::Error E = llvm::compression::decompress( - F, llvm::arrayRefFromStringRef(Blob), Decompressed, Record[0])) { + Compressed, Decompressed, Record[0])) { Error("could not decompress embedded file contents: " + llvm::toString(std::move(E))); return nullptr; diff --git a/llvm/include/llvm/Support/Compression.h b/llvm/include/llvm/Support/Compression.h index 246ccbd6f6dcfe..2ce1c0ebd0395b 100644 --- a/llvm/include/llvm/Support/Compression.h +++ b/llvm/include/llvm/Support/Compression.h @@ -114,6 +114,9 @@ struct Params { // return a string literal describing the reason. LLVM_ABI const char *getReasonIfUnsupported(Format F); +// Return nullptr if LLVM can decompress Input, otherwise a string literal. +LLVM_ABI const char *getReasonIfUnsupported(ArrayRef<uint8_t> Input); + // Compress Input with the specified format P.Format. If Level is -1, use // *::DefaultCompression for the format. LLVM_ABI void compress(Params P, ArrayRef<uint8_t> Input, @@ -128,6 +131,10 @@ LLVM_ABI Error decompress(Format F, ArrayRef<uint8_t> Input, LLVM_ABI Error decompress(DebugCompressionType T, ArrayRef<uint8_t> Input, SmallVectorImpl<uint8_t> &Output, size_t UncompressedSize); +// Infer the compression format from the input. +LLVM_ABI Error decompress(ArrayRef<uint8_t> Input, + SmallVectorImpl<uint8_t> &Output, + size_t UncompressedSize); } // End of namespace compression diff --git a/llvm/lib/Support/Compression.cpp b/llvm/lib/Support/Compression.cpp index 3979ca6acaf74e..01734213c0359d 100644 --- a/llvm/lib/Support/Compression.cpp +++ b/llvm/lib/Support/Compression.cpp @@ -17,6 +17,7 @@ #include "llvm/Support/Compiler.h" #include "llvm/Support/Error.h" #include "llvm/Support/ErrorHandling.h" +#include <optional> #if LLVM_ENABLE_ZLIB #include <zlib.h> #endif @@ -27,6 +28,37 @@ using namespace llvm; using namespace llvm::compression; +// RFC 1950 section 2.2 zlib wrapper. Two-byte header CMF then FLG: +// CMF: CM (bits 0-3) must be 8 (deflate). CINFO (bits 4-7) is +// log2(windowSize)-8 and must be <= 7. +// FLG: FCHECK (bits 0-4) is chosen so CMF*256+FLG is a multiple of 31; +// FDICT (bit 5) marks a preset dictionary; FLEVEL (bits 6-7) is a +// compressor hint. This only identifies the wrapper. +static bool isZlibHeader(ArrayRef<uint8_t> Input) { + if (Input.size() < 2) + return false; + unsigned CMF = Input[0]; + unsigned FLG = Input[1]; + if ((CMF & 0x0f) != 8 || (CMF >> 4) > 7) + return false; + return (CMF * 256 + FLG) % 31 == 0; +} + +// RFC 8878 section 3.1.1: Zstandard frame magic 0xFD2FB528, little-endian. +static bool isZstdMagic(ArrayRef<uint8_t> Input) { + static constexpr uint8_t Magic[] = {0x28, 0xb5, 0x2f, 0xfd}; + return Input.take_front(4) == ArrayRef(Magic); +} + +// Check zstd first: 0x28 is a valid zlib CMF (CINFO=2, 1KiB window). +static std::optional<Format> identifyFormat(ArrayRef<uint8_t> Input) { + if (isZstdMagic(Input)) + return Format::Zstd; + if (isZlibHeader(Input)) + return Format::Zlib; + return std::nullopt; +} + const char *compression::getReasonIfUnsupported(compression::Format F) { switch (F) { case compression::Format::Zlib: @@ -43,6 +75,12 @@ const char *compression::getReasonIfUnsupported(compression::Format F) { llvm_unreachable(""); } +const char *compression::getReasonIfUnsupported(ArrayRef<uint8_t> Input) { + if (std::optional<Format> F = identifyFormat(Input)) + return getReasonIfUnsupported(*F); + return "unknown compression format"; +} + void compression::compress(Params P, ArrayRef<uint8_t> Input, SmallVectorImpl<uint8_t> &Output) { switch (P.format) { @@ -84,6 +122,16 @@ Error compression::decompress(DebugCompressionType T, ArrayRef<uint8_t> Input, return decompress(formatFor(T), Input, Output, UncompressedSize); } +Error compression::decompress(ArrayRef<uint8_t> Input, + SmallVectorImpl<uint8_t> &Output, + size_t UncompressedSize) { + std::optional<Format> F = identifyFormat(Input); + if (const char *Reason = + F ? getReasonIfUnsupported(*F) : "unknown compression format") + return createStringError(Reason); + return decompress(*F, Input, Output, UncompressedSize); +} + #if LLVM_ENABLE_ZLIB static StringRef convertZlibCodeToString(int Code) { diff --git a/llvm/unittests/Support/CompressionTest.cpp b/llvm/unittests/Support/CompressionTest.cpp index 5d326cafbe3a1c..40fdadb1dee9ed 100644 --- a/llvm/unittests/Support/CompressionTest.cpp +++ b/llvm/unittests/Support/CompressionTest.cpp @@ -39,6 +39,11 @@ static void testZlibCompression(StringRef Input, int Level) { EXPECT_FALSE(std::move(E)); EXPECT_EQ(Input, toStringRef(Uncompressed)); + // decompress infers zlib from the RFC 1950 header. + E = compression::decompress(Compressed, Uncompressed, Input.size()); + EXPECT_FALSE(std::move(E)); + EXPECT_EQ(Input, toStringRef(Uncompressed)); + if (Input.size() > 0) { // Decompression fails if expected length is too short. E = zlib::decompress(Compressed, Uncompressed, Input.size() - 1); @@ -84,6 +89,11 @@ static void testZstdCompression(StringRef Input, int Level) { EXPECT_FALSE(std::move(E)); EXPECT_EQ(Input, toStringRef(Uncompressed)); + // decompress infers Zstd from the frame magic. + E = compression::decompress(Compressed, Uncompressed, Input.size()); + EXPECT_FALSE(std::move(E)); + EXPECT_EQ(Input, toStringRef(Uncompressed)); + if (Input.size() > 0) { // Decompression fails if expected length is too short. E = zstd::decompress(Compressed, Uncompressed, Input.size() - 1); @@ -111,4 +121,36 @@ TEST(CompressionTest, Zstd) { testZstdCompression(BinaryDataStr, zstd::DefaultCompression); } #endif + +TEST(CompressionTest, IdentifyHeaders) { + EXPECT_STREQ("unknown compression format", + getReasonIfUnsupported(ArrayRef<uint8_t>())); + uint8_t Truncated[] = {0x78}; + EXPECT_STREQ("unknown compression format", getReasonIfUnsupported(Truncated)); + + // RFC 1950 headers LLVM's compress2 does not emit. + uint8_t SmallWindow[] = {0x28, 0x15}; // CINFO=2, FCHECK valid + EXPECT_EQ(getReasonIfUnsupported(Format::Zlib), + getReasonIfUnsupported(ArrayRef<uint8_t>(SmallWindow))); + uint8_t WithDict[] = {0x78, 0x20}; // FDICT set, FCHECK valid + EXPECT_EQ(getReasonIfUnsupported(Format::Zlib), + getReasonIfUnsupported(ArrayRef<uint8_t>(WithDict))); + + uint8_t BadFCheck[] = {0x78, 0x00}; + EXPECT_STREQ("unknown compression format", getReasonIfUnsupported(BadFCheck)); + uint8_t BadCINFO[] = {0x88, 0x01}; + EXPECT_STREQ("unknown compression format", getReasonIfUnsupported(BadCINFO)); + uint8_t BadCM[] = {0x79, 0x9c}; + EXPECT_STREQ("unknown compression format", getReasonIfUnsupported(BadCM)); + + uint8_t ZstdMagic[] = {0x28, 0xb5, 0x2f, 0xfd}; + EXPECT_EQ(getReasonIfUnsupported(Format::Zstd), + getReasonIfUnsupported(ArrayRef<uint8_t>(ZstdMagic))); + + uint8_t Unknown[] = {0x00, 0x01, 0x02, 0x03}; + EXPECT_STREQ("unknown compression format", getReasonIfUnsupported(Unknown)); + SmallVector<uint8_t, 0> Out; + Error E = compression::decompress(ArrayRef<uint8_t>(Unknown), Out, 0); + EXPECT_EQ("unknown compression format", toString(std::move(E))); } +} // namespace _______________________________________________ cfe-commits mailing list [email protected] https://lists.llvm.org/cgi-bin/mailman/listinfo/cfe-commits
