This is an automated email from the ASF dual-hosted git repository.
wgtmac pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/iceberg-cpp.git
The following commit(s) were added to refs/heads/main by this push:
new 0bdf7ce9 feat: add standard encryption key metadata (#991)
0bdf7ce9 is described below
commit 0bdf7ce9961d7ef8e1c2b8670ae412e86526c53b
Author: Xinli Shang <[email protected]>
AuthorDate: Fri Oct 16 15:37:16 2026 +0900
feat: add standard encryption key metadata (#991)
StandardKeyMetadata is the per-file key metadata of the standard Iceberg
encryption scheme: the data encryption key, the AAD prefix (file ID) and,
for AES GCM streams, the encrypted file length. It is stored in the
key_metadata field of data, delete and manifest files, and (encrypted) in
the table metadata encryption-keys for manifest lists.
The serialized form is Java's: a schema version byte (1) followed by an
Avro binary record. Tests check byte-for-byte compatibility with vectors
written by Java (checked in under test/resources/encryption/vectors,
excluded from the text-fixing pre-commit hooks).
Co-authored-by: Claude Opus 5.5 <[email protected]>
---
src/iceberg/CMakeLists.txt | 1 +
src/iceberg/cpp20_compatibility_internal.h | 1 +
src/iceberg/encryption/standard_key_metadata.cc | 168 ++++++++++++++++++++++++
src/iceberg/encryption/standard_key_metadata.h | 68 ++++++++++
src/iceberg/test/CMakeLists.txt | 2 +
src/iceberg/test/standard_key_metadata_test.cc | 139 ++++++++++++++++++++
6 files changed, 379 insertions(+)
diff --git a/src/iceberg/CMakeLists.txt b/src/iceberg/CMakeLists.txt
index 717f1da4..c283b8af 100644
--- a/src/iceberg/CMakeLists.txt
+++ b/src/iceberg/CMakeLists.txt
@@ -33,6 +33,7 @@ set(ICEBERG_SOURCES
deletes/position_delete_index.cc
deletes/position_delete_range_consumer.cc
deletes/roaring_position_bitmap.cc
+ encryption/standard_key_metadata.cc
expression/aggregate.cc
expression/binder.cc
expression/evaluator.cc
diff --git a/src/iceberg/cpp20_compatibility_internal.h
b/src/iceberg/cpp20_compatibility_internal.h
index 1b719f1b..359e5ddf 100644
--- a/src/iceberg/cpp20_compatibility_internal.h
+++ b/src/iceberg/cpp20_compatibility_internal.h
@@ -49,6 +49,7 @@
#include "iceberg/deletes/position_delete_range_consumer.h"
#include "iceberg/deletes/roaring_position_bitmap.h"
#include "iceberg/encryption/encrypted_key.h"
+#include "iceberg/encryption/standard_key_metadata.h"
#include "iceberg/exception.h"
#include "iceberg/expression/aggregate.h"
#include "iceberg/expression/binder.h"
diff --git a/src/iceberg/encryption/standard_key_metadata.cc
b/src/iceberg/encryption/standard_key_metadata.cc
new file mode 100644
index 00000000..18f2d92e
--- /dev/null
+++ b/src/iceberg/encryption/standard_key_metadata.cc
@@ -0,0 +1,168 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#include "iceberg/encryption/standard_key_metadata.h"
+
+#include <utility>
+
+#include "iceberg/util/macros.h"
+
+namespace iceberg {
+
+namespace {
+
+void WriteLong(int64_t value, std::vector<uint8_t>& out) {
+ // Avro zig-zag varint encoding
+ auto n = (static_cast<uint64_t>(value) << 1) ^ static_cast<uint64_t>(value
>> 63);
+ while (n & ~0x7FULL) {
+ out.push_back(static_cast<uint8_t>((n & 0x7F) | 0x80));
+ n >>= 7;
+ }
+ out.push_back(static_cast<uint8_t>(n));
+}
+
+void WriteBytes(std::span<const uint8_t> bytes, std::vector<uint8_t>& out) {
+ WriteLong(static_cast<int64_t>(bytes.size()), out);
+ out.insert(out.end(), bytes.begin(), bytes.end());
+}
+
+class Decoder {
+ public:
+ explicit Decoder(std::span<const uint8_t> buffer) : buffer_(buffer) {}
+
+ Result<int64_t> ReadLong() {
+ uint64_t n = 0;
+ int shift = 0;
+ while (true) {
+ if (pos_ >= buffer_.size()) {
+ return Invalid("Truncated key metadata");
+ }
+ if (shift > 63) {
+ return Invalid("Invalid varint in key metadata");
+ }
+ uint8_t b = buffer_[pos_++];
+ n |= static_cast<uint64_t>(b & 0x7F) << shift;
+ if ((b & 0x80) == 0) {
+ break;
+ }
+ shift += 7;
+ }
+ return static_cast<int64_t>((n >> 1) ^ -(n & 1));
+ }
+
+ /// \brief Read a zig-zag varint, limited to the five bytes of an Avro int.
+ Result<int32_t> ReadInt() {
+ uint32_t n = 0;
+ int shift = 0;
+ while (true) {
+ if (pos_ >= buffer_.size()) {
+ return Invalid("Truncated key metadata");
+ }
+ if (shift > 28) {
+ return Invalid("Invalid int varint in key metadata");
+ }
+ uint8_t b = buffer_[pos_++];
+ n |= static_cast<uint32_t>(b & 0x7F) << shift;
+ if ((b & 0x80) == 0) {
+ break;
+ }
+ shift += 7;
+ }
+ return static_cast<int32_t>((n >> 1) ^ -(n & 1));
+ }
+
+ Result<std::vector<uint8_t>> ReadBytes() {
+ ICEBERG_ASSIGN_OR_RAISE(auto length, ReadLong());
+ if (length < 0 || std::cmp_greater(length, buffer_.size() - pos_)) {
+ return Invalid("Invalid bytes length in key metadata: {}", length);
+ }
+ std::vector<uint8_t> bytes(buffer_.begin() + pos_, buffer_.begin() + pos_
+ length);
+ pos_ += length;
+ return bytes;
+ }
+
+ /// \brief Read the branch index of a ["null", T] union; true if T is
present.
+ ///
+ /// Avro encodes the branch index as an int, so this rejects the longer
+ /// encodings that ReadLong accepts, as Java's readIndex does.
+ Result<bool> ReadOptionalBranch() {
+ ICEBERG_ASSIGN_OR_RAISE(auto branch, ReadInt());
+ if (branch != 0 && branch != 1) {
+ return Invalid("Invalid union branch in key metadata: {}", branch);
+ }
+ return branch == 1;
+ }
+
+ private:
+ std::span<const uint8_t> buffer_;
+ size_t pos_ = 0;
+};
+
+} // namespace
+
+Result<StandardKeyMetadata> StandardKeyMetadata::Parse(std::span<const
uint8_t> buffer) {
+ if (buffer.empty()) {
+ return Invalid("Empty key metadata");
+ }
+ if (buffer[0] != kV1) {
+ return NotSupported("Unsupported key metadata schema version: {}",
+ static_cast<int>(buffer[0]));
+ }
+
+ Decoder decoder(buffer.subspan(1));
+ StandardKeyMetadata metadata;
+ ICEBERG_ASSIGN_OR_RAISE(metadata.encryption_key, decoder.ReadBytes());
+ ICEBERG_ASSIGN_OR_RAISE(auto has_aad_prefix, decoder.ReadOptionalBranch());
+ if (has_aad_prefix) {
+ ICEBERG_ASSIGN_OR_RAISE(metadata.aad_prefix, decoder.ReadBytes());
+ }
+ ICEBERG_ASSIGN_OR_RAISE(auto has_file_length, decoder.ReadOptionalBranch());
+ if (has_file_length) {
+ ICEBERG_ASSIGN_OR_RAISE(metadata.file_length, decoder.ReadLong());
+ }
+ // Trailing bytes are ignored, matching Java, which decodes a single record.
+ return metadata;
+}
+
+std::vector<uint8_t> StandardKeyMetadata::Serialize() const {
+ std::vector<uint8_t> out;
+ out.push_back(kV1);
+ WriteBytes(encryption_key, out);
+ if (aad_prefix.has_value()) {
+ WriteLong(1, out);
+ WriteBytes(*aad_prefix, out);
+ } else {
+ WriteLong(0, out);
+ }
+ if (file_length.has_value()) {
+ WriteLong(1, out);
+ WriteLong(*file_length, out);
+ } else {
+ WriteLong(0, out);
+ }
+ return out;
+}
+
+StandardKeyMetadata StandardKeyMetadata::WithFileLength(int64_t length) const {
+ StandardKeyMetadata copy = *this;
+ copy.file_length = length;
+ return copy;
+}
+
+} // namespace iceberg
diff --git a/src/iceberg/encryption/standard_key_metadata.h
b/src/iceberg/encryption/standard_key_metadata.h
new file mode 100644
index 00000000..ebad2d80
--- /dev/null
+++ b/src/iceberg/encryption/standard_key_metadata.h
@@ -0,0 +1,68 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#pragma once
+
+/// \file iceberg/encryption/standard_key_metadata.h
+/// Key metadata of the standard Iceberg encryption scheme.
+
+#include <cstdint>
+#include <optional>
+#include <span>
+#include <vector>
+
+#include "iceberg/iceberg_export.h"
+#include "iceberg/result.h"
+
+namespace iceberg {
+
+/// \brief Per-file key metadata of the standard encryption scheme.
+///
+/// Stored in the `key_metadata` field of data files, delete files and
manifest files,
+/// and (encrypted by a key encryption key) in the table metadata
`encryption-keys` for
+/// manifest lists.
+///
+/// The serialized form is compatible with Java's StandardKeyMetadata: a
one-byte schema
+/// version (1) followed by an Avro binary encoded record with the fields
+/// `encryption_key: bytes`, `aad_prefix: union{null, bytes}` and
+/// `file_length: union{null, long}`.
+struct ICEBERG_EXPORT StandardKeyMetadata {
+ static constexpr uint8_t kV1 = 1;
+
+ /// \brief The data encryption key (16, 24 or 32 bytes).
+ std::vector<uint8_t> encryption_key;
+ /// \brief The AAD prefix (file ID) used to bind ciphertext to this file.
+ std::optional<std::vector<uint8_t>> aad_prefix;
+ /// \brief The encrypted (stored) file length, required to read AES GCM
streams.
+ std::optional<int64_t> file_length;
+
+ /// \brief Parse serialized key metadata.
+ static Result<StandardKeyMetadata> Parse(std::span<const uint8_t> buffer);
+
+ /// \brief Serialize to the version-prefixed Avro binary form.
+ std::vector<uint8_t> Serialize() const;
+
+ /// \brief Return a copy with the given encrypted file length.
+ StandardKeyMetadata WithFileLength(int64_t length) const;
+
+ friend bool operator==(const StandardKeyMetadata&,
+ const StandardKeyMetadata&) = default;
+};
+
+} // namespace iceberg
diff --git a/src/iceberg/test/CMakeLists.txt b/src/iceberg/test/CMakeLists.txt
index 292132bb..a86bc908 100644
--- a/src/iceberg/test/CMakeLists.txt
+++ b/src/iceberg/test/CMakeLists.txt
@@ -116,6 +116,8 @@ add_iceberg_test(expression_test
residual_evaluator_test.cc
strict_metrics_evaluator_test.cc)
+add_iceberg_test(encryption_test SOURCES standard_key_metadata_test.cc)
+
add_iceberg_test(json_serde_test
SOURCES
json_serde_test.cc
diff --git a/src/iceberg/test/standard_key_metadata_test.cc
b/src/iceberg/test/standard_key_metadata_test.cc
new file mode 100644
index 00000000..dd83c1ce
--- /dev/null
+++ b/src/iceberg/test/standard_key_metadata_test.cc
@@ -0,0 +1,139 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#include "iceberg/encryption/standard_key_metadata.h"
+
+#include <limits>
+#include <vector>
+
+#include <gtest/gtest.h>
+
+#include "iceberg/test/matchers.h"
+
+namespace iceberg {
+
+namespace {
+
+/// \brief Deterministic test bytes: byte i is (i * 31 + seed).
+std::vector<uint8_t> TestBytes(size_t length, int seed) {
+ std::vector<uint8_t> bytes(length);
+ for (size_t i = 0; i < length; ++i) {
+ bytes[i] = static_cast<uint8_t>(static_cast<int>(i) * 31 + seed);
+ }
+ return bytes;
+}
+
+// Serialized by Iceberg Java's StandardKeyMetadata, with encryption key
+// TestBytes(16, 1), AAD prefix TestBytes(16, 2) and no file length.
+const std::vector<uint8_t> kJavaKeyMetadata = {
+ 0x01, 0x20, 0x01, 0x20, 0x3f, 0x5e, 0x7d, 0x9c, 0xbb, 0xda, 0xf9, 0x18,
0x37,
+ 0x56, 0x75, 0x94, 0xb3, 0xd2, 0x02, 0x20, 0x02, 0x21, 0x40, 0x5f, 0x7e,
0x9d,
+ 0xbc, 0xdb, 0xfa, 0x19, 0x38, 0x57, 0x76, 0x95, 0xb4, 0xd3, 0x00};
+
+// The same, with file length 1234567.
+const std::vector<uint8_t> kJavaKeyMetadataWithLength = {
+ 0x01, 0x20, 0x01, 0x20, 0x3f, 0x5e, 0x7d, 0x9c, 0xbb, 0xda, 0xf9, 0x18,
0x37, 0x56,
+ 0x75, 0x94, 0xb3, 0xd2, 0x02, 0x20, 0x02, 0x21, 0x40, 0x5f, 0x7e, 0x9d,
0xbc, 0xdb,
+ 0xfa, 0x19, 0x38, 0x57, 0x76, 0x95, 0xb4, 0xd3, 0x02, 0x8e, 0xda, 0x96,
0x01};
+
+} // namespace
+
+TEST(StandardKeyMetadataTest, ParseJavaVectors) {
+ auto key = TestBytes(16, 1);
+ auto aad_prefix = TestBytes(16, 2);
+
+ const auto& with_length = kJavaKeyMetadataWithLength;
+ ICEBERG_UNWRAP_OR_FAIL(auto parsed, StandardKeyMetadata::Parse(with_length));
+ EXPECT_EQ(parsed.encryption_key, key);
+ EXPECT_EQ(parsed.aad_prefix, aad_prefix);
+ EXPECT_EQ(parsed.file_length, 1234567);
+ // Serialization is byte-for-byte identical to Java
+ EXPECT_EQ(parsed.Serialize(), with_length);
+
+ const auto& without_length = kJavaKeyMetadata;
+ ICEBERG_UNWRAP_OR_FAIL(auto parsed2,
StandardKeyMetadata::Parse(without_length));
+ EXPECT_EQ(parsed2.encryption_key, key);
+ EXPECT_FALSE(parsed2.file_length.has_value());
+ EXPECT_EQ(parsed2.Serialize(), without_length);
+ EXPECT_EQ(parsed2.WithFileLength(1234567).Serialize(), with_length);
+}
+
+TEST(StandardKeyMetadataTest, RoundTrip) {
+ StandardKeyMetadata metadata{.encryption_key = TestBytes(32, 7)};
+ ICEBERG_UNWRAP_OR_FAIL(auto parsed,
StandardKeyMetadata::Parse(metadata.Serialize()));
+ EXPECT_EQ(parsed, metadata);
+
+ // Zig-zag varint boundaries, including negative values and ten-byte
encodings
+ for (int64_t length :
+ {int64_t{0}, int64_t{63}, int64_t{64}, int64_t{1} << 40, int64_t{-1},
+ std::numeric_limits<int64_t>::min(),
std::numeric_limits<int64_t>::max()}) {
+ auto with_length = metadata.WithFileLength(length);
+ ICEBERG_UNWRAP_OR_FAIL(auto round_trip,
+
StandardKeyMetadata::Parse(with_length.Serialize()));
+ EXPECT_EQ(round_trip, with_length);
+ }
+
+ // A present but empty AAD prefix is distinct from an absent one
+ StandardKeyMetadata empty_aad{.encryption_key = TestBytes(16, 7),
+ .aad_prefix = std::vector<uint8_t>{}};
+ ICEBERG_UNWRAP_OR_FAIL(auto parsed_empty,
+ StandardKeyMetadata::Parse(empty_aad.Serialize()));
+ ASSERT_TRUE(parsed_empty.aad_prefix.has_value());
+ EXPECT_TRUE(parsed_empty.aad_prefix->empty());
+ ICEBERG_UNWRAP_OR_FAIL(auto parsed_null,
+ StandardKeyMetadata::Parse(metadata.Serialize()));
+ EXPECT_FALSE(parsed_null.aad_prefix.has_value());
+}
+
+TEST(StandardKeyMetadataTest, InvalidInput) {
+ EXPECT_THAT(StandardKeyMetadata::Parse(std::vector<uint8_t>{}),
+ IsError(ErrorKind::kInvalid));
+ // Unknown schema version
+ EXPECT_THAT(StandardKeyMetadata::Parse(std::vector<uint8_t>{2, 0, 0, 0}),
+ IsError(ErrorKind::kNotSupported));
+ // Key length beyond the buffer
+ EXPECT_THAT(StandardKeyMetadata::Parse(std::vector<uint8_t>{1, 0x20, 1, 2}),
+ IsError(ErrorKind::kInvalid));
+ // Invalid union branch
+ EXPECT_THAT(StandardKeyMetadata::Parse(std::vector<uint8_t>{1, 0x02, 9,
0x04}),
+ IsError(ErrorKind::kInvalid));
+ // Truncated optional fields
+ EXPECT_THAT(StandardKeyMetadata::Parse(std::vector<uint8_t>{1, 0x02, 9}),
+ IsError(ErrorKind::kInvalid));
+ // Negative key length (zig-zag 0x01 is -1)
+ EXPECT_THAT(StandardKeyMetadata::Parse(std::vector<uint8_t>{1, 0x01, 9}),
+ IsError(ErrorKind::kInvalid));
+ // Truncated varint
+ EXPECT_THAT(StandardKeyMetadata::Parse(std::vector<uint8_t>{1, 0x80}),
+ IsError(ErrorKind::kInvalid));
+ // Varint longer than ten bytes, rejected as by Java's Avro decoder
+ std::vector<uint8_t> long_varint{1};
+ long_varint.insert(long_varint.end(), 10, 0x80);
+ long_varint.push_back(0x01);
+ EXPECT_THAT(StandardKeyMetadata::Parse(long_varint),
IsError(ErrorKind::kInvalid));
+ // Union index encoded in six bytes: a valid long but not a valid int, so
+ // Java's readIndex rejects it even though it decodes to branch zero. The
+ // trailing 0x00 is a well-formed file_length branch, so the record is
+ // otherwise complete and only the index encoding makes it invalid.
+ EXPECT_THAT(StandardKeyMetadata::Parse(std::vector<uint8_t>{
+ 1, 0x02, 9, 0x80, 0x80, 0x80, 0x80, 0x80, 0x00, 0x00}),
+ IsError(ErrorKind::kInvalid));
+}
+
+} // namespace iceberg
