This is an automated email from the ASF dual-hosted git repository.
wgtmac pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/iceberg-cpp.git
The following commit(s) were added to refs/heads/main by this push:
new 4f195d0b feat(encryption): add AES GCM cipher and AES GCM Stream
(AGS1) files (#1030)
4f195d0b is described below
commit 4f195d0be65dde346802b3a6555bd46eaf344fa2
Author: Xinli Shang <[email protected]>
AuthorDate: Wed Oct 21 01:38:27 2026 -0700
feat(encryption): add AES GCM cipher and AES GCM Stream (AGS1) files (#1030)
---
ci/scripts/generate_public_headers.py | 56 ++--
cmake_modules/IcebergBuildUtils.cmake | 3 +-
src/iceberg/CMakeLists.txt | 4 +
src/iceberg/cpp20_compatibility_internal.h | 4 +
src/iceberg/encryption/CMakeLists.txt | 10 +-
src/iceberg/encryption/aes_gcm.cc | 177 ++++++++++++
src/iceberg/encryption/aes_gcm_internal.h | 71 +++++
src/iceberg/encryption/aes_gcm_stream.cc | 369 ++++++++++++++++++++++++
src/iceberg/encryption/aes_gcm_stream.h | 99 +++++++
src/iceberg/encryption/standard_key_metadata.h | 4 +
src/iceberg/test/CMakeLists.txt | 8 +
src/iceberg/test/aes_gcm_test.cc | 376 +++++++++++++++++++++++++
12 files changed, 1162 insertions(+), 19 deletions(-)
diff --git a/ci/scripts/generate_public_headers.py
b/ci/scripts/generate_public_headers.py
index 56f196d8..d4b80be8 100755
--- a/ci/scripts/generate_public_headers.py
+++ b/ci/scripts/generate_public_headers.py
@@ -19,20 +19,21 @@
"""Regenerate src/iceberg/cpp20_compatibility_internal.h.
-The generated header includes every public header, i.e. every header that
-iceberg_install_all_headers() installs: *.h files in a directory whose
-CMakeLists.txt calls it, except those with "internal" in their name. This
script
-applies the same rule, so the list never has to be maintained by hand. The
-generate-public-headers pre-commit hook runs it and fails when the checked-in
-header is out of date.
-
-Headers of optional components are wrapped in a macro that
-cpp20_compatibility_test defines only when that component is built.
+The generated header covers all possible public headers installed by
+iceberg_install_all_headers(): *.h and *.hpp files matching its PATTERN
arguments,
+except those with "internal" in their name. The generate-public-headers
pre-commit
+hook runs this script and fails when the checked-in header is out of date.
+
+Headers of optional components are grouped by directory or an explicit
+HEADER_GROUP in the installation call. cpp20_compatibility_test defines each
+group's macro only when the corresponding component is built.
"""
from __future__ import annotations
+import fnmatch
import re
+import shlex
import sys
from pathlib import Path
@@ -51,7 +52,7 @@ OPTIONAL_COMPONENTS = [
("iceberg/parquet", "ICEBERG_PUBLIC_HEADERS_BUNDLE"),
]
-INSTALL_CALL = re.compile(r"iceberg_install_all_headers\(\s*([^\s)]+)")
+INSTALL_CALL = re.compile(r"iceberg_install_all_headers\((.*?)\)", re.DOTALL)
LICENSE = """\
/*
@@ -75,7 +76,9 @@ LICENSE = """\
"""
-def component_of(install_path: str) -> str | None:
+def component_of(install_path: str, header_group: str | None = None) -> str |
None:
+ if header_group:
+ return header_group
for prefix, macro in OPTIONAL_COMPONENTS:
if install_path == prefix or install_path.startswith(prefix + "/"):
return macro
@@ -85,13 +88,29 @@ def component_of(install_path: str) -> str | None:
def collect_public_headers() -> dict[str | None, list[str]]:
headers: dict[str | None, list[str]] = {}
for cmake_file in sorted(SRC.rglob("CMakeLists.txt")):
- for install_path in INSTALL_CALL.findall(cmake_file.read_text()):
+ for call in INSTALL_CALL.findall(cmake_file.read_text()):
+ tokens = shlex.split(call, comments=True)
+ if not tokens:
+ continue
+ install_path = tokens[0]
+ try:
+ pattern_start = tokens.index("PATTERN") + 1
+ except ValueError:
+ pattern_start = len(tokens)
+ try:
+ group_start = tokens.index("HEADER_GROUP")
+ header_group = tokens[group_start + 1]
+ except ValueError:
+ group_start = len(tokens)
+ header_group = None
+ patterns = tokens[pattern_start:group_start] or ["*.h", "*.hpp"]
+ group = component_of(install_path, header_group)
for header in cmake_file.parent.iterdir():
- if header.suffix != ".h" or "internal" in header.name:
+ if not any(fnmatch.fnmatch(header.name, pattern) for pattern
in patterns):
continue
- headers.setdefault(component_of(install_path), []).append(
- f"{install_path}/{header.name}"
- )
+ if header.suffix not in {".h", ".hpp"} or "internal" in
header.name:
+ continue
+ headers.setdefault(group,
[]).append(f"{install_path}/{header.name}")
return headers
@@ -107,7 +126,10 @@ def render(headers: dict[str | None, list[str]]) -> str:
"",
]
lines += [f'#include "{h}"' for h in sorted(headers.get(None, []))]
- for macro in sorted({m for _, m in OPTIONAL_COMPONENTS}):
+ optional_macros = {m for _, m in OPTIONAL_COMPONENTS} | {
+ group for group in headers if group is not None
+ }
+ for macro in sorted(optional_macros):
if macro not in headers:
continue
lines += ["", f"#ifdef {macro}"]
diff --git a/cmake_modules/IcebergBuildUtils.cmake
b/cmake_modules/IcebergBuildUtils.cmake
index 035c312d..fa1accd2 100644
--- a/cmake_modules/IcebergBuildUtils.cmake
+++ b/cmake_modules/IcebergBuildUtils.cmake
@@ -275,7 +275,8 @@ endfunction()
function(iceberg_install_all_headers PATH)
set(options)
- set(one_value_args)
+ # HEADER_GROUP labels optional headers for
ci/scripts/generate_public_headers.py.
+ set(one_value_args HEADER_GROUP)
set(multi_value_args PATTERN)
cmake_parse_arguments(ARG
"${options}"
diff --git a/src/iceberg/CMakeLists.txt b/src/iceberg/CMakeLists.txt
index 091acd3e..c751cd7a 100644
--- a/src/iceberg/CMakeLists.txt
+++ b/src/iceberg/CMakeLists.txt
@@ -326,6 +326,10 @@ if(ICEBERG_BUILD_BUNDLE)
parquet/parquet_writer.cc
parquet/row_number_reader.cc)
+ if(ICEBERG_ENCRYPTION)
+ list(APPEND ICEBERG_BUNDLE_SOURCES encryption/aes_gcm.cc
encryption/aes_gcm_stream.cc)
+ endif()
+
# Libraries to link with exported libiceberg_bundle.{so,a}.
set(ICEBERG_BUNDLE_STATIC_BUILD_INTERFACE_LIBS)
set(ICEBERG_BUNDLE_SHARED_BUILD_INTERFACE_LIBS)
diff --git a/src/iceberg/cpp20_compatibility_internal.h
b/src/iceberg/cpp20_compatibility_internal.h
index 359e5ddf..bd9a9475 100644
--- a/src/iceberg/cpp20_compatibility_internal.h
+++ b/src/iceberg/cpp20_compatibility_internal.h
@@ -216,6 +216,10 @@
# include "iceberg/parquet/parquet_writer.h"
#endif // ICEBERG_PUBLIC_HEADERS_BUNDLE
+#ifdef ICEBERG_PUBLIC_HEADERS_ENCRYPTION
+# include "iceberg/encryption/aes_gcm_stream.h"
+#endif // ICEBERG_PUBLIC_HEADERS_ENCRYPTION
+
#ifdef ICEBERG_PUBLIC_HEADERS_HIVE
# include "iceberg/catalog/hive/hive_catalog.h"
# include "iceberg/catalog/hive/hive_catalog_properties.h"
diff --git a/src/iceberg/encryption/CMakeLists.txt
b/src/iceberg/encryption/CMakeLists.txt
index c493ee25..73072dbc 100644
--- a/src/iceberg/encryption/CMakeLists.txt
+++ b/src/iceberg/encryption/CMakeLists.txt
@@ -15,4 +15,12 @@
# specific language governing permissions and limitations
# under the License.
-iceberg_install_all_headers(iceberg/encryption)
+iceberg_install_all_headers(iceberg/encryption PATTERN encrypted_key.h
+ standard_key_metadata.h)
+if(ICEBERG_BUILD_BUNDLE AND ICEBERG_ENCRYPTION)
+ iceberg_install_all_headers(iceberg/encryption
+ PATTERN
+ aes_gcm_stream.h
+ HEADER_GROUP
+ ICEBERG_PUBLIC_HEADERS_ENCRYPTION)
+endif()
diff --git a/src/iceberg/encryption/aes_gcm.cc
b/src/iceberg/encryption/aes_gcm.cc
new file mode 100644
index 00000000..eefb7654
--- /dev/null
+++ b/src/iceberg/encryption/aes_gcm.cc
@@ -0,0 +1,177 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#include <limits>
+
+#include <openssl/evp.h>
+#include <openssl/rand.h>
+
+#include "iceberg/encryption/aes_gcm_internal.h"
+#include "iceberg/util/macros.h"
+
+namespace iceberg::encryption {
+
+namespace {
+
+const EVP_CIPHER* CipherForKey(size_t key_length) {
+ switch (key_length) {
+ case 16:
+ return EVP_aes_128_gcm();
+ case 24:
+ return EVP_aes_192_gcm();
+ case 32:
+ return EVP_aes_256_gcm();
+ default:
+ return nullptr;
+ }
+}
+
+EVP_CIPHER_CTX* Ctx(void* ctx) { return static_cast<EVP_CIPHER_CTX*>(ctx); }
+
+} // namespace
+
+Status SecureRandomBytes(std::span<uint8_t> out) {
+ if (out.size() > static_cast<size_t>(std::numeric_limits<int>::max()) ||
+ RAND_bytes(out.data(), static_cast<int>(out.size())) != 1) {
+ return IOError("Failed to generate secure random bytes");
+ }
+ return {};
+}
+
+Result<std::unique_ptr<AesGcmCipher>> AesGcmCipher::Make(std::span<const
uint8_t> key) {
+ if (CipherForKey(key.size()) == nullptr) {
+ return InvalidArgument("Invalid key length: {} (must be 16, 24, or 32
bytes)",
+ key.size());
+ }
+ EVP_CIPHER_CTX* ctx = EVP_CIPHER_CTX_new();
+ if (ctx == nullptr) {
+ return IOError("Failed to create cipher context");
+ }
+ return std::unique_ptr<AesGcmCipher>(
+ new AesGcmCipher(std::vector<uint8_t>(key.begin(), key.end()), ctx));
+}
+
+AesGcmCipher::AesGcmCipher(std::vector<uint8_t> key, void* ctx)
+ : key_(std::move(key)), ctx_(ctx) {}
+
+AesGcmCipher::~AesGcmCipher() { EVP_CIPHER_CTX_free(Ctx(ctx_)); }
+
+Status AesGcmCipher::Encrypt(std::span<const uint8_t> plaintext,
+ std::span<const uint8_t> aad, std::span<uint8_t>
out) {
+ if (out.size() != plaintext.size() + kNonceLength + kGcmTagLength) {
+ return InvalidArgument("Invalid ciphertext buffer size: {}", out.size());
+ }
+ if (plaintext.size() > static_cast<size_t>(std::numeric_limits<int>::max()))
{
+ return InvalidArgument("Plaintext too large: {}", plaintext.size());
+ }
+
+ auto nonce = out.first(kNonceLength);
+ auto ciphertext = out.subspan(kNonceLength, plaintext.size());
+ auto tag = out.last(kGcmTagLength);
+ ICEBERG_RETURN_UNEXPECTED(SecureRandomBytes(nonce));
+
+ auto* ctx = Ctx(ctx_);
+ int len = 0;
+ if (EVP_EncryptInit_ex(ctx, CipherForKey(key_.size()), /*impl=*/nullptr,
+ /*key=*/nullptr, /*iv=*/nullptr) != 1 ||
+ EVP_CIPHER_CTX_ctrl(ctx, EVP_CTRL_GCM_SET_IVLEN, kNonceLength,
/*ptr=*/nullptr) !=
+ 1 ||
+ EVP_EncryptInit_ex(ctx, /*cipher=*/nullptr, /*impl=*/nullptr,
key_.data(),
+ nonce.data()) != 1) {
+ return IOError("Failed to initialize AES GCM encryption");
+ }
+ if (!aad.empty() && EVP_EncryptUpdate(ctx, /*out=*/nullptr, &len, aad.data(),
+ static_cast<int>(aad.size())) != 1) {
+ return IOError("Failed to set AES GCM AAD");
+ }
+ if (!plaintext.empty() &&
+ EVP_EncryptUpdate(ctx, ciphertext.data(), &len, plaintext.data(),
+ static_cast<int>(plaintext.size())) != 1) {
+ return IOError("Failed to encrypt");
+ }
+ if (EVP_EncryptFinal_ex(ctx, ciphertext.data() + plaintext.size(), &len) !=
1 ||
+ EVP_CIPHER_CTX_ctrl(ctx, EVP_CTRL_GCM_GET_TAG, kGcmTagLength,
tag.data()) != 1) {
+ return IOError("Failed to finalize AES GCM encryption");
+ }
+ return {};
+}
+
+Status AesGcmCipher::Decrypt(std::span<const uint8_t> ciphertext,
+ std::span<const uint8_t> aad, std::span<uint8_t>
out) {
+ if (ciphertext.size() < static_cast<size_t>(kNonceLength + kGcmTagLength)) {
+ return Invalid(
+ "Cannot decrypt cipher text of length {}: shorter than nonce and tag,
text may "
+ "not be encrypted with AES GCM",
+ ciphertext.size());
+ }
+ size_t plaintext_length = ciphertext.size() - kNonceLength - kGcmTagLength;
+ if (out.size() != plaintext_length) {
+ return InvalidArgument("Invalid plaintext buffer size: {}", out.size());
+ }
+
+ auto nonce = ciphertext.first(kNonceLength);
+ auto data = ciphertext.subspan(kNonceLength, plaintext_length);
+ auto tag = ciphertext.last(kGcmTagLength);
+
+ auto* ctx = Ctx(ctx_);
+ int len = 0;
+ if (EVP_DecryptInit_ex(ctx, CipherForKey(key_.size()), /*impl=*/nullptr,
+ /*key=*/nullptr, /*iv=*/nullptr) != 1 ||
+ EVP_CIPHER_CTX_ctrl(ctx, EVP_CTRL_GCM_SET_IVLEN, kNonceLength,
/*ptr=*/nullptr) !=
+ 1 ||
+ EVP_DecryptInit_ex(ctx, /*cipher=*/nullptr, /*impl=*/nullptr,
key_.data(),
+ nonce.data()) != 1) {
+ return IOError("Failed to initialize AES GCM decryption");
+ }
+ if (!aad.empty() && EVP_DecryptUpdate(ctx, /*out=*/nullptr, &len, aad.data(),
+ static_cast<int>(aad.size())) != 1) {
+ return IOError("Failed to set AES GCM AAD");
+ }
+ if (!data.empty() && EVP_DecryptUpdate(ctx, out.data(), &len, data.data(),
+ static_cast<int>(data.size())) != 1) {
+ return IOError("Failed to decrypt");
+ }
+ if (EVP_CIPHER_CTX_ctrl(ctx, EVP_CTRL_GCM_SET_TAG, kGcmTagLength,
+ const_cast<uint8_t*>(tag.data())) != 1 ||
+ EVP_DecryptFinal_ex(ctx, out.data() + plaintext_length, &len) != 1) {
+ return Invalid(
+ "GCM tag check failed. Possible reasons: wrong decryption key; or "
+ "corrupt/tampered data. AES GCM doesn't differentiate between these
two.");
+ }
+ return {};
+}
+
+Result<std::vector<uint8_t>> AesGcmCipher::Encrypt(std::span<const uint8_t>
plaintext,
+ std::span<const uint8_t>
aad) {
+ std::vector<uint8_t> out(plaintext.size() + kNonceLength + kGcmTagLength);
+ ICEBERG_RETURN_UNEXPECTED(Encrypt(plaintext, aad, out));
+ return out;
+}
+
+Result<std::vector<uint8_t>> AesGcmCipher::Decrypt(std::span<const uint8_t>
ciphertext,
+ std::span<const uint8_t>
aad) {
+ if (ciphertext.size() < static_cast<size_t>(kNonceLength + kGcmTagLength)) {
+ return Invalid("Cannot decrypt cipher text of length {}",
ciphertext.size());
+ }
+ std::vector<uint8_t> out(ciphertext.size() - kNonceLength - kGcmTagLength);
+ ICEBERG_RETURN_UNEXPECTED(Decrypt(ciphertext, aad, out));
+ return out;
+}
+
+} // namespace iceberg::encryption
diff --git a/src/iceberg/encryption/aes_gcm_internal.h
b/src/iceberg/encryption/aes_gcm_internal.h
new file mode 100644
index 00000000..e09a7802
--- /dev/null
+++ b/src/iceberg/encryption/aes_gcm_internal.h
@@ -0,0 +1,71 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#pragma once
+
+/// \file iceberg/encryption/aes_gcm_internal.h
+/// AES GCM cipher compatible with Java's
org.apache.iceberg.encryption.Ciphers.
+
+#include <cstdint>
+#include <memory>
+#include <span>
+#include <vector>
+
+#include "iceberg/result.h"
+
+namespace iceberg::encryption {
+
+inline constexpr int32_t kNonceLength = 12;
+inline constexpr int32_t kGcmTagLength = 16;
+
+/// \brief Fill `out` with cryptographically secure random bytes.
+Status SecureRandomBytes(std::span<uint8_t> out);
+
+/// \brief AES GCM with a random 12-byte nonce.
+///
+/// Ciphertext layout: nonce (12) | encrypted data | tag (16).
+class AesGcmCipher {
+ public:
+ /// \brief Create a cipher for a 16, 24 or 32 byte key.
+ static Result<std::unique_ptr<AesGcmCipher>> Make(std::span<const uint8_t>
key);
+
+ ~AesGcmCipher();
+
+ /// \brief Encrypt `plaintext` into `out`, which must hold plaintext.size()
+ 28 bytes.
+ Status Encrypt(std::span<const uint8_t> plaintext, std::span<const uint8_t>
aad,
+ std::span<uint8_t> out);
+
+ /// \brief Decrypt and verify `ciphertext` into `out`, which must hold
+ /// ciphertext.size() - 28 bytes.
+ Status Decrypt(std::span<const uint8_t> ciphertext, std::span<const uint8_t>
aad,
+ std::span<uint8_t> out);
+
+ Result<std::vector<uint8_t>> Encrypt(std::span<const uint8_t> plaintext,
+ std::span<const uint8_t> aad);
+ Result<std::vector<uint8_t>> Decrypt(std::span<const uint8_t> ciphertext,
+ std::span<const uint8_t> aad);
+
+ private:
+ AesGcmCipher(std::vector<uint8_t> key, void* ctx);
+
+ std::vector<uint8_t> key_;
+ void* ctx_; // EVP_CIPHER_CTX
+};
+
+} // namespace iceberg::encryption
diff --git a/src/iceberg/encryption/aes_gcm_stream.cc
b/src/iceberg/encryption/aes_gcm_stream.cc
new file mode 100644
index 00000000..7fb545db
--- /dev/null
+++ b/src/iceberg/encryption/aes_gcm_stream.cc
@@ -0,0 +1,369 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#include "iceberg/encryption/aes_gcm_stream.h"
+
+#include <algorithm>
+#include <array>
+#include <cstring>
+#include <limits>
+#include <utility>
+
+#include "iceberg/encryption/aes_gcm_internal.h"
+#include "iceberg/util/macros.h"
+
+namespace iceberg {
+
+namespace {
+
+using encryption::AesGcmCipher;
+
+constexpr int64_t kPlainBlock = AesGcmStream::kPlainBlockSize;
+constexpr int64_t kCipherBlock = AesGcmStream::kCipherBlockSize;
+constexpr int64_t kBlockOverhead =
+ AesGcmStream::kNonceLength + AesGcmStream::kGcmTagLength;
+
+/// \brief Block AAD: file AAD prefix followed by the block index as a 4-byte
LE int.
+std::vector<uint8_t> BlockAad(const std::vector<uint8_t>& aad_prefix,
+ int32_t block_index) {
+ std::vector<uint8_t> aad(aad_prefix);
+ for (int i = 0; i < 4; ++i) {
+ aad.push_back(
+ static_cast<uint8_t>((static_cast<uint32_t>(block_index) >> (8 * i)) &
0xFF));
+ }
+ return aad;
+}
+
+std::array<std::byte, AesGcmStream::kHeaderLength> HeaderBytes() {
+ std::array<std::byte, AesGcmStream::kHeaderLength> header{};
+ std::memcpy(header.data(), AesGcmStream::kMagic.data(),
AesGcmStream::kMagic.size());
+ for (int i = 0; i < 4; ++i) {
+ header[4 + i] = static_cast<std::byte>(
+ (static_cast<uint32_t>(AesGcmStream::kPlainBlockSize) >> (8 * i)) &
0xFF);
+ }
+ return header;
+}
+
+class AesGcmInputStream : public SeekableInputStream {
+ public:
+ /// \brief Construct over an encrypted length already accepted by
+ /// AesGcmStream::PlaintextLength, which bounds the block count.
+ AesGcmInputStream(std::unique_ptr<SeekableInputStream> source,
+ std::unique_ptr<AesGcmCipher> cipher, std::vector<uint8_t>
aad_prefix,
+ int64_t encrypted_length, int64_t plain_size)
+ : source_(std::move(source)),
+ cipher_(std::move(cipher)),
+ aad_prefix_(std::move(aad_prefix)),
+ cipher_block_(kCipherBlock),
+ plain_block_(kPlainBlock),
+ plain_size_(plain_size) {
+ int64_t stream_length = encrypted_length - AesGcmStream::kHeaderLength;
+ int64_t num_full_blocks = stream_length / kCipherBlock;
+ int64_t last_bytes = stream_length - num_full_blocks * kCipherBlock;
+ bool full_blocks_only = last_bytes == 0;
+ num_blocks_ = full_blocks_only ? num_full_blocks : num_full_blocks + 1;
+ last_cipher_block_size_ = full_blocks_only ? kCipherBlock : last_bytes;
+ }
+
+ Status ValidateHeader() {
+ std::array<std::byte, AesGcmStream::kHeaderLength> header{};
+ ICEBERG_RETURN_UNEXPECTED(source_->ReadFully(0, header));
+ if (std::memcmp(header.data(), AesGcmStream::kMagic.data(),
+ AesGcmStream::kMagic.size()) != 0) {
+ return Invalid("Invalid GCM stream: magic does not match AGS1");
+ }
+ uint32_t plain_block_size = 0;
+ for (int i = 0; i < 4; ++i) {
+ plain_block_size |= static_cast<uint32_t>(header[4 + i]) << (8 * i);
+ }
+ if (plain_block_size !=
static_cast<uint32_t>(AesGcmStream::kPlainBlockSize)) {
+ return Invalid("Invalid GCM stream: block size {} != {}",
plain_block_size,
+ AesGcmStream::kPlainBlockSize);
+ }
+ return {};
+ }
+
+ /// \brief Authenticate the sole block of an empty stream.
+ ///
+ /// An empty AGS1 file still carries one empty block with a nonce and a tag,
so a
+ /// reader that stops at plain_size_ == 0 would accept a wrong key, a wrong
AAD
+ /// prefix, a corrupt tag or a file truncated behind its trusted length. Java
+ /// authenticates the block on the first read; an empty stream is never read
from,
+ /// so it is authenticated when the stream is opened.
+ Status ValidateEmptyStream() {
+ if (plain_size_ != 0) {
+ return {};
+ }
+ return DecryptBlock(0);
+ }
+
+ Result<int64_t> Position() const override { return position_; }
+
+ Status Seek(int64_t position) override {
+ if (position < 0 || position > plain_size_) {
+ return IOError("Invalid position {} for stream of length {}", position,
+ plain_size_);
+ }
+ position_ = position;
+ return {};
+ }
+
+ Result<int64_t> Read(std::span<std::byte> out) override {
+ int64_t total = 0;
+ while (std::cmp_less(total, out.size()) && position_ < plain_size_) {
+ int64_t block_index = position_ / kPlainBlock;
+ ICEBERG_RETURN_UNEXPECTED(DecryptBlock(block_index));
+ int64_t offset = position_ % kPlainBlock;
+ int64_t n = std::min<int64_t>(current_block_size_ - offset,
+ static_cast<int64_t>(out.size()) - total);
+ std::memcpy(out.data() + total, plain_block_.data() + offset, n);
+ total += n;
+ position_ += n;
+ }
+ return total;
+ }
+
+ Status ReadFully(int64_t position, std::span<std::byte> out) override {
+ // The position is bounded before the length is added to it, so that a
position
+ // near int64 max does not overflow the range check.
+ if (position < 0 || position > plain_size_ ||
+ std::cmp_greater(out.size(), plain_size_ - position)) {
+ return IOError("Cannot read {} bytes at {} from stream of length {}",
out.size(),
+ position, plain_size_);
+ }
+ ICEBERG_RETURN_UNEXPECTED(Seek(position));
+ ICEBERG_ASSIGN_OR_RAISE(auto n, Read(out));
+ if (std::cmp_not_equal(n, out.size())) {
+ return IOError("Short read from GCM stream");
+ }
+ return {};
+ }
+
+ Status Close() override { return source_->Close(); }
+
+ private:
+ Status DecryptBlock(int64_t block_index) {
+ if (block_index == current_block_index_) {
+ return {};
+ }
+ // num_blocks_ is bounded by int32 max at construction, so the cast below
is safe.
+ int64_t size =
+ block_index == num_blocks_ - 1 ? last_cipher_block_size_ :
kCipherBlock;
+ auto cipher_block = std::span(cipher_block_).first(size);
+ ICEBERG_RETURN_UNEXPECTED(
+ source_->ReadFully(AesGcmStream::kHeaderLength + block_index *
kCipherBlock,
+ std::as_writable_bytes(cipher_block)));
+ auto aad = BlockAad(aad_prefix_, static_cast<int32_t>(block_index));
+ int64_t plain_size = size - kBlockOverhead;
+ // Decryption writes plaintext before the tag is checked, so the cached
block is
+ // invalid until the whole block authenticates.
+ current_block_index_ = -1;
+ current_block_size_ = 0;
+ ICEBERG_RETURN_UNEXPECTED(
+ cipher_->Decrypt(cipher_block, aad,
std::span(plain_block_).first(plain_size)));
+ current_block_size_ = plain_size;
+ current_block_index_ = block_index;
+ return {};
+ }
+
+ std::unique_ptr<SeekableInputStream> source_;
+ std::unique_ptr<AesGcmCipher> cipher_;
+ std::vector<uint8_t> aad_prefix_;
+ std::vector<uint8_t> cipher_block_;
+ std::vector<uint8_t> plain_block_;
+ int64_t num_blocks_ = 0;
+ int64_t last_cipher_block_size_ = 0;
+ int64_t plain_size_ = 0;
+ int64_t position_ = 0;
+ int64_t current_block_index_ = -1;
+ int64_t current_block_size_ = 0;
+};
+
+class AesGcmOutputStream : public PositionOutputStream {
+ public:
+ AesGcmOutputStream(std::unique_ptr<PositionOutputStream> target,
+ std::unique_ptr<AesGcmCipher> cipher,
+ std::vector<uint8_t> aad_prefix)
+ : target_(std::move(target)),
+ cipher_(std::move(cipher)),
+ aad_prefix_(std::move(aad_prefix)),
+ plain_block_(kPlainBlock),
+ cipher_block_(kCipherBlock) {}
+
+ Result<int64_t> Position() const override {
+ // Closing flushes the last block, which resets the in-block position, so
the
+ // plaintext length is captured before that happens.
+ if (closed_) {
+ return final_position_;
+ }
+ return block_index_ * kPlainBlock + position_in_block_;
+ }
+
+ Result<int64_t> StoredLength() const override { return
target_->StoredLength(); }
+
+ Status Write(std::span<const std::byte> data) override {
+ if (closed_) {
+ return IOError("Writing to closed stream");
+ }
+ ICEBERG_RETURN_UNEXPECTED(WriteHeaderIfNeeded());
+ while (!data.empty()) {
+ int64_t n = std::min<int64_t>(kPlainBlock - position_in_block_,
data.size());
+ std::memcpy(plain_block_.data() + position_in_block_, data.data(), n);
+ position_in_block_ += n;
+ data = data.subspan(n);
+ if (position_in_block_ == kPlainBlock) {
+ ICEBERG_RETURN_UNEXPECTED(EncryptAndWriteBlock());
+ }
+ }
+ return {};
+ }
+
+ Status Flush() override { return target_->Flush(); }
+
+ Status Close() override {
+ if (closed_) {
+ return {};
+ }
+ ICEBERG_RETURN_UNEXPECTED(WriteHeaderIfNeeded());
+ ICEBERG_ASSIGN_OR_RAISE(final_position_, Position());
+ // Like Java, an empty stream gets one empty block; otherwise only a
non-empty
+ // partial block is written.
+ if (position_in_block_ > 0 || block_index_ == 0) {
+ ICEBERG_RETURN_UNEXPECTED(EncryptAndWriteBlock());
+ }
+ closed_ = true;
+ return target_->Close();
+ }
+
+ private:
+ Status WriteHeaderIfNeeded() {
+ if (!header_written_) {
+ auto header = HeaderBytes();
+ ICEBERG_RETURN_UNEXPECTED(target_->Write(header));
+ header_written_ = true;
+ }
+ return {};
+ }
+
+ Status EncryptAndWriteBlock() {
+ if (last_block_written_) {
+ return Invalid("Cannot encrypt block: a partial block has already been
written");
+ }
+ if (block_index_ == std::numeric_limits<int32_t>::max()) {
+ return IOError("Cannot write block: exceeded Integer.MAX_VALUE blocks");
+ }
+ if (position_in_block_ != kPlainBlock) {
+ last_block_written_ = true;
+ }
+ auto aad = BlockAad(aad_prefix_, static_cast<int32_t>(block_index_));
+ auto out = std::span(cipher_block_).first(position_in_block_ +
kBlockOverhead);
+ ICEBERG_RETURN_UNEXPECTED(
+ cipher_->Encrypt(std::span(plain_block_).first(position_in_block_),
aad, out));
+ ICEBERG_RETURN_UNEXPECTED(target_->Write(std::as_bytes(out)));
+ position_in_block_ = 0;
+ ++block_index_;
+ return {};
+ }
+
+ std::unique_ptr<PositionOutputStream> target_;
+ std::unique_ptr<AesGcmCipher> cipher_;
+ std::vector<uint8_t> aad_prefix_;
+ std::vector<uint8_t> plain_block_;
+ std::vector<uint8_t> cipher_block_;
+ int64_t position_in_block_ = 0;
+ int64_t block_index_ = 0;
+ int64_t final_position_ = 0;
+ bool header_written_ = false;
+ bool last_block_written_ = false;
+ bool closed_ = false;
+};
+
+} // namespace
+
+Result<int64_t> AesGcmStream::PlaintextLength(int64_t encrypted_length) {
+ if (encrypted_length < kMinStreamLength) {
+ return Invalid(
+ "Invalid encrypted stream: {} is shorter than the minimum possible
stream "
+ "length {}",
+ encrypted_length, kMinStreamLength);
+ }
+ int64_t stream_length = encrypted_length - kHeaderLength;
+ int64_t num_full_blocks = stream_length / kCipherBlockSize;
+ int64_t last_bytes = stream_length - num_full_blocks * kCipherBlockSize;
+ int64_t num_blocks = last_bytes == 0 ? num_full_blocks : num_full_blocks + 1;
+ // Block AAD carries the block index as an int32, so Java bounds the block
count
+ // with Math.toIntExact. Reject here rather than when the block is decrypted.
+ if (num_blocks > std::numeric_limits<int32_t>::max()) {
+ return Invalid("Invalid encrypted stream: {} bytes exceeds the maximum of
{} blocks",
+ encrypted_length, std::numeric_limits<int32_t>::max());
+ }
+ if (last_bytes != 0 && last_bytes < kNonceLength + kGcmTagLength) {
+ return Invalid("Invalid GCM stream: truncated last block");
+ }
+ int64_t last_plain = last_bytes == 0 ? 0 : last_bytes - kNonceLength -
kGcmTagLength;
+ return num_full_blocks * kPlainBlockSize + last_plain;
+}
+
+AesGcmInputFile::AesGcmInputFile(std::unique_ptr<InputFile> source,
+ std::vector<uint8_t> key,
+ std::vector<uint8_t> aad_prefix,
+ int64_t encrypted_length)
+ : source_(std::move(source)),
+ key_(std::move(key)),
+ aad_prefix_(std::move(aad_prefix)),
+ encrypted_length_(encrypted_length) {}
+
+Result<int64_t> AesGcmInputFile::Size() const {
+ return AesGcmStream::PlaintextLength(encrypted_length_);
+}
+
+Result<std::unique_ptr<SeekableInputStream>> AesGcmInputFile::Open() {
+ ICEBERG_ASSIGN_OR_RAISE(auto plain_size,
+ AesGcmStream::PlaintextLength(encrypted_length_));
+ ICEBERG_ASSIGN_OR_RAISE(auto cipher, AesGcmCipher::Make(key_));
+ ICEBERG_ASSIGN_OR_RAISE(auto source, source_->Open());
+ auto stream = std::make_unique<AesGcmInputStream>(
+ std::move(source), std::move(cipher), aad_prefix_, encrypted_length_,
plain_size);
+ ICEBERG_RETURN_UNEXPECTED(stream->ValidateHeader());
+ ICEBERG_RETURN_UNEXPECTED(stream->ValidateEmptyStream());
+ return stream;
+}
+
+AesGcmOutputFile::AesGcmOutputFile(std::unique_ptr<OutputFile> target,
+ std::vector<uint8_t> key,
+ std::vector<uint8_t> aad_prefix)
+ : target_(std::move(target)),
+ key_(std::move(key)),
+ aad_prefix_(std::move(aad_prefix)) {}
+
+Result<std::unique_ptr<PositionOutputStream>> AesGcmOutputFile::Create() {
+ ICEBERG_ASSIGN_OR_RAISE(auto cipher, encryption::AesGcmCipher::Make(key_));
+ ICEBERG_ASSIGN_OR_RAISE(auto target, target_->Create());
+ return std::make_unique<AesGcmOutputStream>(std::move(target),
std::move(cipher),
+ aad_prefix_);
+}
+
+Result<std::unique_ptr<PositionOutputStream>>
AesGcmOutputFile::CreateOrOverwrite() {
+ ICEBERG_ASSIGN_OR_RAISE(auto cipher, encryption::AesGcmCipher::Make(key_));
+ ICEBERG_ASSIGN_OR_RAISE(auto target, target_->CreateOrOverwrite());
+ return std::make_unique<AesGcmOutputStream>(std::move(target),
std::move(cipher),
+ aad_prefix_);
+}
+
+} // namespace iceberg
diff --git a/src/iceberg/encryption/aes_gcm_stream.h
b/src/iceberg/encryption/aes_gcm_stream.h
new file mode 100644
index 00000000..766eb85e
--- /dev/null
+++ b/src/iceberg/encryption/aes_gcm_stream.h
@@ -0,0 +1,99 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#pragma once
+
+/// \file iceberg/encryption/aes_gcm_stream.h
+/// AES GCM Stream (AGS1) files, see format/gcm-stream-spec.md.
+
+#include <array>
+#include <cstdint>
+#include <memory>
+#include <span>
+#include <vector>
+
+#include "iceberg/file_io.h"
+#include "iceberg/iceberg_bundle_export.h"
+#include "iceberg/result.h"
+
+namespace iceberg {
+
+/// \brief AES GCM Stream format constants.
+///
+/// AGS1 stores the block length in its header. This implementation only reads
and
+/// writes 1 MiB blocks, matching Java, rather than supporting other block
lengths.
+///
+/// For Java compatibility, an empty file contains one zero-length plaintext
block
+/// with a nonce and tag. The AGS1 spec describes a non-empty last block and
does not
+/// explicitly define this empty-file representation.
+struct AesGcmStream {
+ static constexpr int32_t kPlainBlockSize = 1024 * 1024;
+ static constexpr int32_t kNonceLength = 12;
+ static constexpr int32_t kGcmTagLength = 16;
+ static constexpr int32_t kCipherBlockSize =
+ kPlainBlockSize + kNonceLength + kGcmTagLength;
+ static constexpr std::array<char, 4> kMagic = {'A', 'G', 'S', '1'};
+ static constexpr int32_t kHeaderLength = 8; // magic + plain block size (LE
int32)
+ static constexpr int64_t kMinStreamLength =
+ kHeaderLength + kNonceLength + kGcmTagLength;
+
+ /// \brief Plaintext length of a stream with the given encrypted length.
+ ///
+ /// Returns an error if the length is shorter than an empty stream, implies
more
+ /// than int32 max blocks, or ends in a block too short to hold a nonce and
tag.
+ ICEBERG_BUNDLE_EXPORT static Result<int64_t> PlaintextLength(int64_t
encrypted_length);
+};
+
+/// \brief An InputFile that decrypts an AGS1 file.
+///
+/// The encrypted length must come from a trusted source (the key metadata),
so that
+/// truncation of the file is detected.
+class ICEBERG_BUNDLE_EXPORT AesGcmInputFile : public InputFile {
+ public:
+ AesGcmInputFile(std::unique_ptr<InputFile> source, std::vector<uint8_t> key,
+ std::vector<uint8_t> aad_prefix, int64_t encrypted_length);
+
+ std::string_view location() const override { return source_->location(); }
+ Result<int64_t> Size() const override;
+ Result<std::unique_ptr<SeekableInputStream>> Open() override;
+
+ private:
+ std::unique_ptr<InputFile> source_;
+ std::vector<uint8_t> key_;
+ std::vector<uint8_t> aad_prefix_;
+ int64_t encrypted_length_;
+};
+
+/// \brief An OutputFile that writes an AGS1 file.
+class ICEBERG_BUNDLE_EXPORT AesGcmOutputFile : public OutputFile {
+ public:
+ AesGcmOutputFile(std::unique_ptr<OutputFile> target, std::vector<uint8_t>
key,
+ std::vector<uint8_t> aad_prefix);
+
+ std::string_view location() const override { return target_->location(); }
+ Result<std::unique_ptr<PositionOutputStream>> Create() override;
+ Result<std::unique_ptr<PositionOutputStream>> CreateOrOverwrite() override;
+
+ private:
+ std::unique_ptr<OutputFile> target_;
+ std::vector<uint8_t> key_;
+ std::vector<uint8_t> aad_prefix_;
+};
+
+} // namespace iceberg
diff --git a/src/iceberg/encryption/standard_key_metadata.h
b/src/iceberg/encryption/standard_key_metadata.h
index ebad2d80..7e788eae 100644
--- a/src/iceberg/encryption/standard_key_metadata.h
+++ b/src/iceberg/encryption/standard_key_metadata.h
@@ -38,6 +38,10 @@ namespace iceberg {
/// and (encrypted by a key encryption key) in the table metadata
`encryption-keys` for
/// manifest lists.
///
+/// The Iceberg table spec leaves the key metadata representation to the
encryption
+/// scheme; it mandates neither this encoding nor AES-GCM. The schema below
follows
+/// Java's standard scheme.
+///
/// The serialized form is compatible with Java's StandardKeyMetadata: a
one-byte schema
/// version (1) followed by an Avro binary encoded record with the fields
/// `encryption_key: bytes`, `aad_prefix: union{null, bytes}` and
diff --git a/src/iceberg/test/CMakeLists.txt b/src/iceberg/test/CMakeLists.txt
index 8b9dd2e9..8622a1a6 100644
--- a/src/iceberg/test/CMakeLists.txt
+++ b/src/iceberg/test/CMakeLists.txt
@@ -229,6 +229,10 @@ if(ICEBERG_BUILD_BUNDLE)
target_sources(parquet_test PRIVATE parquet_encryption_test.cc)
endif()
+ if(ICEBERG_ENCRYPTION)
+ add_iceberg_test(crypto_test USE_BUNDLE SOURCES aes_gcm_test.cc)
+ endif()
+
add_iceberg_test(scan_test
USE_BUNDLE
SOURCES
@@ -410,6 +414,10 @@ if(ICEBERG_CXX20_COMPAT)
)
target_compile_definitions(cpp20_compatibility_test
PRIVATE ICEBERG_PUBLIC_HEADERS_BUNDLE)
+ if(ICEBERG_ENCRYPTION)
+ target_compile_definitions(cpp20_compatibility_test
+ PRIVATE ICEBERG_PUBLIC_HEADERS_ENCRYPTION)
+ endif()
endif()
if(ICEBERG_BUILD_REST)
target_link_libraries(cpp20_compatibility_test
diff --git a/src/iceberg/test/aes_gcm_test.cc b/src/iceberg/test/aes_gcm_test.cc
new file mode 100644
index 00000000..9b21a2c4
--- /dev/null
+++ b/src/iceberg/test/aes_gcm_test.cc
@@ -0,0 +1,376 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#include <cstring>
+#include <filesystem>
+#include <fstream>
+#include <limits>
+#include <span>
+#include <string>
+#include <string_view>
+#include <vector>
+
+#include <gtest/gtest.h>
+
+#include "iceberg/arrow/arrow_io_util.h"
+#include "iceberg/encryption/aes_gcm_internal.h"
+#include "iceberg/encryption/aes_gcm_stream.h"
+#include "iceberg/encryption/standard_key_metadata.h"
+#include "iceberg/test/matchers.h"
+#include "iceberg/test/temp_file_test_base.h"
+
+namespace iceberg {
+
+namespace {
+
+using encryption::AesGcmCipher;
+
+/// \brief Deterministic test bytes: byte i is (i * 31 + seed).
+std::vector<uint8_t> TestBytes(size_t length, int seed) {
+ std::vector<uint8_t> bytes(length);
+ for (size_t i = 0; i < length; ++i) {
+ bytes[i] = static_cast<uint8_t>(static_cast<int>(i) * 31 + seed);
+ }
+ return bytes;
+}
+
+std::span<const uint8_t> AsBytes(std::string_view s) {
+ return {reinterpret_cast<const uint8_t*>(s.data()), s.size()};
+}
+
+// The fixtures below were produced by Iceberg Java. It used TestBytes(16, 1)
as the
+// data key, TestBytes(16, 2) as the file AAD prefix and TestBytes(16, 3) as
the key
+// encryption key.
+const std::vector<uint8_t> kDataKey = TestBytes(16, 1);
+const std::vector<uint8_t> kAadPrefix = TestBytes(16, 2);
+const std::vector<uint8_t> kKek = TestBytes(16, 3);
+
+// StandardEncryptionManager encrypts manifest list key metadata under the key
+// encryption key, with the KEK timestamp as AAD.
+constexpr std::string_view kKekTimestamp = "1791455219054";
+
+// Manifest list key metadata encrypted by Java under kKek with kKekTimestamp
+// as AAD: encryption key TestBytes(16, 1), AAD prefix TestBytes(16, 2),
+// file length 4242.
+const std::vector<uint8_t> kJavaManifestListKeyMetadata = {
+ 0xf2, 0x80, 0x57, 0xac, 0x14, 0xdc, 0x8d, 0xa7, 0xb2, 0x25, 0x58, 0x24,
0xa3, 0xe1,
+ 0x88, 0xb0, 0xde, 0x2e, 0x94, 0x09, 0x9b, 0x7d, 0xe4, 0x7d, 0x0e, 0xd6,
0xb0, 0x7a,
+ 0xb1, 0x86, 0x7b, 0xca, 0x61, 0x11, 0x2a, 0x11, 0x4e, 0xc1, 0xa8, 0x23,
0xd8, 0xa5,
+ 0xca, 0x43, 0x9c, 0x33, 0x6a, 0xe9, 0x6b, 0xf2, 0xc6, 0xdb, 0xa8, 0xe0,
0x2e, 0x7f,
+ 0xc0, 0xcc, 0x0e, 0xf0, 0xb7, 0xec, 0xb9, 0x1f, 0x06, 0x3a, 0xe7};
+// AES GCM stream of TestBytes(0, 5) written by Java's AesGcmOutputStream
+// with kDataKey and kAadPrefix.
+const std::vector<uint8_t> kJavaAgs1Empty = {
+ 0x41, 0x47, 0x53, 0x31, 0x00, 0x00, 0x10, 0x00, 0x92, 0xd6, 0x53, 0x8c,
+ 0x2b, 0x71, 0x45, 0x37, 0x37, 0x86, 0x76, 0x8f, 0xef, 0xd4, 0xdc, 0x98,
+ 0x15, 0xf1, 0x93, 0x9d, 0x6c, 0x56, 0x4b, 0xae, 0x24, 0xff, 0xdc, 0xb4};
+// AES GCM stream of TestBytes(1, 5) written by Java's AesGcmOutputStream
+// with kDataKey and kAadPrefix.
+const std::vector<uint8_t> kJavaAgs1OneByte = {
+ 0x41, 0x47, 0x53, 0x31, 0x00, 0x00, 0x10, 0x00, 0xf8, 0xde, 0xe0, 0x44,
0x00,
+ 0x8d, 0xbb, 0x2b, 0x6b, 0x29, 0x58, 0xb2, 0x70, 0x62, 0x46, 0xbb, 0xb1,
0xf8,
+ 0x76, 0x29, 0x69, 0x86, 0x14, 0x6c, 0x08, 0x98, 0x84, 0xdc, 0x3b};
+// AES GCM stream of TestBytes(100, 5) written by Java's AesGcmOutputStream
+// with kDataKey and kAadPrefix.
+const std::vector<uint8_t> kJavaAgs1Hundred = {
+ 0x41, 0x47, 0x53, 0x31, 0x00, 0x00, 0x10, 0x00, 0x02, 0xb0, 0x1c, 0x1f,
0x7e, 0x37,
+ 0xeb, 0x85, 0xd3, 0x4b, 0x14, 0x75, 0x91, 0x83, 0xcf, 0x68, 0xde, 0xdd,
0xe0, 0xff,
+ 0xd3, 0xe0, 0xeb, 0x31, 0x10, 0x4b, 0x95, 0x86, 0x5c, 0x67, 0x1f, 0x68,
0xdf, 0x24,
+ 0xbd, 0x0a, 0x07, 0xf1, 0x06, 0xc7, 0x13, 0x1e, 0x42, 0xd4, 0x7a, 0xff,
0x87, 0x32,
+ 0xc3, 0xe5, 0xea, 0x75, 0x6a, 0x53, 0xaa, 0xab, 0xab, 0xe0, 0x81, 0x2c,
0x71, 0x4c,
+ 0xb4, 0x89, 0x32, 0x89, 0xc6, 0xbb, 0x84, 0x5d, 0x58, 0x93, 0x6b, 0x80,
0xbd, 0xa3,
+ 0xe6, 0xc3, 0xef, 0x53, 0x83, 0xa4, 0x00, 0x9c, 0x30, 0x02, 0xd4, 0xca,
0x7e, 0x0e,
+ 0xb3, 0xa3, 0xff, 0x4c, 0x96, 0xe3, 0x31, 0xa1, 0x6a, 0xc1, 0xf2, 0xb7,
0xa0, 0x40,
+ 0xc4, 0x0b, 0x5c, 0x45, 0x09, 0xb7, 0x35, 0xda, 0x6c, 0x03, 0xce, 0xd5,
0x59, 0xd0,
+ 0x47, 0xf9, 0x0c, 0x9d, 0x1f, 0x89, 0x2f, 0x1b, 0xcb, 0x44};
+
+Result<std::vector<uint8_t>> ReadAll(InputFile& file) {
+ ICEBERG_ASSIGN_OR_RAISE(auto size, file.Size());
+ ICEBERG_ASSIGN_OR_RAISE(auto stream, file.Open());
+ std::vector<uint8_t> out(size);
+ if (size > 0) {
+ ICEBERG_RETURN_UNEXPECTED(
+ stream->ReadFully(0, std::as_writable_bytes(std::span(out))));
+ }
+ return out;
+}
+
+} // namespace
+
+TEST(AesGcmCipherTest, RoundTrip) {
+ for (size_t key_length : {16, 24, 32}) {
+ ICEBERG_UNWRAP_OR_FAIL(auto cipher,
AesGcmCipher::Make(TestBytes(key_length, 1)));
+ auto plaintext = TestBytes(100, 2);
+ ICEBERG_UNWRAP_OR_FAIL(auto ciphertext, cipher->Encrypt(plaintext,
AsBytes("aad")));
+ EXPECT_EQ(ciphertext.size(), plaintext.size() + 28);
+ ICEBERG_UNWRAP_OR_FAIL(auto decrypted, cipher->Decrypt(ciphertext,
AsBytes("aad")));
+ EXPECT_EQ(decrypted, plaintext);
+
+ // Random nonces: encrypting twice gives different ciphertexts
+ ICEBERG_UNWRAP_OR_FAIL(auto again, cipher->Encrypt(plaintext,
AsBytes("aad")));
+ EXPECT_NE(again, ciphertext);
+
+ // Wrong AAD, tampered ciphertext and truncated ciphertext fail
authentication
+ EXPECT_THAT(cipher->Decrypt(ciphertext, AsBytes("other")),
+ IsError(ErrorKind::kInvalid));
+ ciphertext[20] ^= 1;
+ EXPECT_THAT(cipher->Decrypt(ciphertext, AsBytes("aad")),
+ IsError(ErrorKind::kInvalid));
+ EXPECT_THAT(cipher->Decrypt(std::span(ciphertext).first(27),
AsBytes("aad")),
+ IsError(ErrorKind::kInvalid));
+ }
+ EXPECT_THAT(AesGcmCipher::Make(TestBytes(15, 1)),
IsError(ErrorKind::kInvalidArgument));
+}
+
+TEST(AesGcmCipherTest, DecryptsJavaManifestListKeyMetadata) {
+ ICEBERG_UNWRAP_OR_FAIL(auto cipher, AesGcmCipher::Make(kKek));
+ ICEBERG_UNWRAP_OR_FAIL(auto decrypted,
cipher->Decrypt(kJavaManifestListKeyMetadata,
+
AsBytes(kKekTimestamp)));
+ ICEBERG_UNWRAP_OR_FAIL(auto metadata, StandardKeyMetadata::Parse(decrypted));
+ EXPECT_EQ(metadata.encryption_key, TestBytes(16, 1));
+ EXPECT_EQ(metadata.aad_prefix, TestBytes(16, 2));
+ EXPECT_EQ(metadata.file_length, 4242);
+}
+
+class AesGcmStreamTest : public TempFileTestBase {
+ protected:
+ void SetUp() override {
+ TempFileTestBase::SetUp();
+ io_ = std::shared_ptr<FileIO>(arrow::MakeLocalFileIO());
+ }
+
+ // Write `plain` as an AGS1 stream and return the stored length.
+ int64_t Write(const std::string& path, std::span<const uint8_t> plain,
+ const std::vector<uint8_t>& key, const std::vector<uint8_t>&
aad,
+ size_t chunk = 1 << 20) {
+ auto out = io_->NewOutputFile(path);
+ EXPECT_THAT(out, IsOk());
+ AesGcmOutputFile file(std::move(out.value()), key, aad);
+ auto stream = file.CreateOrOverwrite();
+ EXPECT_THAT(stream, IsOk());
+ for (size_t pos = 0; pos < plain.size(); pos += chunk) {
+ auto part = plain.subspan(pos, std::min(chunk, plain.size() - pos));
+ EXPECT_THAT((*stream)->Write(std::as_bytes(part)), IsOk());
+ }
+ EXPECT_EQ((*stream)->Position().value(),
static_cast<int64_t>(plain.size()));
+ EXPECT_THAT((*stream)->Close(), IsOk());
+ // Java's AesGcmOutputStream keeps reporting the plaintext length once
closed
+ EXPECT_EQ((*stream)->Position().value(),
static_cast<int64_t>(plain.size()));
+ auto stored = (*stream)->StoredLength().value();
+ EXPECT_EQ(stored, static_cast<int64_t>(std::filesystem::file_size(path)));
+ return stored;
+ }
+
+ std::unique_ptr<AesGcmInputFile> Open(const std::string& path,
+ const std::vector<uint8_t>& key,
+ const std::vector<uint8_t>& aad,
int64_t length) {
+ auto in = io_->NewInputFile(path);
+ EXPECT_THAT(in, IsOk());
+ return std::make_unique<AesGcmInputFile>(std::move(in.value()), key, aad,
length);
+ }
+
+ // Materialize an inlined Java fixture as a file the reader can open.
+ std::string WriteRaw(const std::vector<uint8_t>& bytes) {
+ auto path = CreateNewTempFilePath();
+ std::ofstream out(path, std::ios::binary);
+ out.write(reinterpret_cast<const char*>(bytes.data()),
+ static_cast<std::streamsize>(bytes.size()));
+ out.close();
+ return path;
+ }
+
+ std::shared_ptr<FileIO> io_;
+};
+
+TEST_F(AesGcmStreamTest, ReadsJavaVectors) {
+ // Streams of TestBytes(0, 5), TestBytes(1, 5) and TestBytes(100, 5) written
by Java
+ const std::vector<std::pair<size_t, const std::vector<uint8_t>*>> vectors = {
+ {0, &kJavaAgs1Empty}, {1, &kJavaAgs1OneByte}, {100, &kJavaAgs1Hundred}};
+ for (const auto& [length, encrypted] : vectors) {
+ auto path = WriteRaw(*encrypted);
+ auto file = Open(path, kDataKey, kAadPrefix,
static_cast<int64_t>(encrypted->size()));
+ ICEBERG_UNWRAP_OR_FAIL(auto plain, ReadAll(*file));
+ EXPECT_EQ(plain, TestBytes(length, 5)) << "plaintext length " << length;
+ }
+}
+
+TEST_F(AesGcmStreamTest, WritesJavaCompatibleLayout) {
+ // Same plaintext, key and AAD prefix as Java's 100-byte vector: same header
and
+ // length (the ciphertext differs only by the random nonce).
+ auto path = CreateNewTempFilePath();
+ auto stored = Write(path, TestBytes(100, 5), kDataKey, kAadPrefix);
+ ASSERT_EQ(stored, static_cast<int64_t>(kJavaAgs1Hundred.size()));
+ std::ifstream in(path, std::ios::binary);
+ std::vector<uint8_t> header(8);
+ in.read(reinterpret_cast<char*>(header.data()), 8);
+ EXPECT_EQ(header,
+ std::vector<uint8_t>(kJavaAgs1Hundred.begin(),
kJavaAgs1Hundred.begin() + 8));
+}
+
+TEST_F(AesGcmStreamTest, RoundTripAcrossBlocks) {
+ auto key = TestBytes(32, 7);
+ auto aad = TestBytes(16, 9);
+ constexpr size_t kBlock = AesGcmStream::kPlainBlockSize;
+ for (size_t size :
+ {size_t{0}, size_t{1}, kBlock - 1, kBlock, kBlock + 1, 2 * kBlock +
17}) {
+ auto path = CreateNewTempFilePath();
+ auto plain = TestBytes(size, 5);
+ // Odd write sizes exercise writes that straddle blocks
+ int64_t stored = Write(path, plain, key, aad, /*chunk=*/333331);
+ size_t blocks = size == 0 ? 1 : (size + kBlock - 1) / kBlock;
+ EXPECT_EQ(stored, static_cast<int64_t>(8 + size + blocks * 28));
+ ICEBERG_UNWRAP_OR_FAIL(auto plain_size,
AesGcmStream::PlaintextLength(stored));
+ EXPECT_EQ(plain_size, static_cast<int64_t>(size));
+
+ auto file = Open(path, key, aad, stored);
+ ICEBERG_UNWRAP_OR_FAIL(auto read, ReadAll(*file));
+ EXPECT_EQ(read, plain);
+
+ if (size > kBlock + 10) {
+ // Random access across a block boundary
+ ICEBERG_UNWRAP_OR_FAIL(auto stream, file->Open());
+ std::vector<std::byte> window(10);
+ ASSERT_THAT(stream->ReadFully(kBlock - 5, window), IsOk());
+ EXPECT_EQ(std::memcmp(window.data(), plain.data() + kBlock - 5, 10), 0);
+ }
+ }
+}
+
+TEST_F(AesGcmStreamTest, DetectsTampering) {
+ auto key = TestBytes(16, 7);
+ auto aad = TestBytes(16, 9);
+ auto path = CreateNewTempFilePath();
+ auto plain = TestBytes(AesGcmStream::kPlainBlockSize + 100, 5);
+ int64_t stored = Write(path, plain, key, aad);
+
+ // Wrong key or AAD prefix (a file swapped in from another manifest entry)
+ EXPECT_THAT(ReadAll(*Open(path, TestBytes(16, 8), aad, stored)),
+ IsError(ErrorKind::kInvalid));
+ EXPECT_THAT(ReadAll(*Open(path, key, TestBytes(16, 10), stored)),
+ IsError(ErrorKind::kInvalid));
+
+ // Truncation: the trusted length from the key metadata no longer matches
the file
+ std::filesystem::resize_file(path, 8 + AesGcmStream::kCipherBlockSize);
+ EXPECT_THAT(ReadAll(*Open(path, key, aad, stored)),
IsError(ErrorKind::kIOError));
+ // AGS1 blocks do not mark the end of the stream: a truncated file read with
its own
+ // (untrusted) length still decrypts. This is why readers must take the
length from
+ // the key metadata.
+ auto truncated = Open(path, key, aad, 8 + AesGcmStream::kCipherBlockSize);
+ ICEBERG_UNWRAP_OR_FAIL(auto first_block, ReadAll(*truncated));
+ EXPECT_EQ(first_block.size(), AesGcmStream::kPlainBlockSize);
+
+ // Bad magic
+ std::filesystem::resize_file(path, 0);
+ std::ofstream(path, std::ios::binary) << "XXXX0000" << std::string(28, '\0');
+ EXPECT_THAT(Open(path, key, aad, 36)->Open(), IsError(ErrorKind::kInvalid));
+ // Shorter than an empty stream
+ EXPECT_THAT(Open(path, key, aad, 35)->Open(), IsError(ErrorKind::kInvalid));
+}
+
+TEST_F(AesGcmStreamTest, AuthenticatesEmptyStream) {
+ auto key = TestBytes(16, 7);
+ auto aad = TestBytes(16, 9);
+ auto path = CreateNewTempFilePath();
+ int64_t stored = Write(path, {}, key, aad);
+ ASSERT_EQ(stored, AesGcmStream::kMinStreamLength);
+
+ // An empty stream still carries one block with a nonce and a tag, so
opening it
+ // authenticates that block even though there is no plaintext to read.
+ ICEBERG_UNWRAP_OR_FAIL(auto plain, ReadAll(*Open(path, key, aad, stored)));
+ EXPECT_TRUE(plain.empty());
+
+ // Wrong key or AAD prefix (a file swapped in from another manifest entry)
+ EXPECT_THAT(Open(path, TestBytes(16, 8), aad, stored)->Open(),
+ IsError(ErrorKind::kInvalid));
+ EXPECT_THAT(Open(path, key, TestBytes(16, 10), stored)->Open(),
+ IsError(ErrorKind::kInvalid));
+
+ // Truncated to its header behind a trusted length that says otherwise
+ std::filesystem::resize_file(path, AesGcmStream::kHeaderLength);
+ EXPECT_THAT(Open(path, key, aad, stored)->Open(),
IsError(ErrorKind::kIOError));
+
+ // A corrupt tag
+ auto corrupt = WriteRaw(kJavaAgs1Empty);
+ {
+ std::fstream out(corrupt, std::ios::binary | std::ios::in | std::ios::out);
+ out.seekp(static_cast<std::streamoff>(kJavaAgs1Empty.size() - 1));
+ out.put(static_cast<char>(kJavaAgs1Empty.back() ^ 0xFF));
+ }
+ EXPECT_THAT(
+ Open(corrupt, kDataKey, kAadPrefix,
static_cast<int64_t>(kJavaAgs1Empty.size()))
+ ->Open(),
+ IsError(ErrorKind::kInvalid));
+}
+
+TEST_F(AesGcmStreamTest, RejectsOutOfRangeReadFully) {
+ auto key = TestBytes(16, 7);
+ auto aad = TestBytes(16, 9);
+ auto path = CreateNewTempFilePath();
+ int64_t stored = Write(path, TestBytes(100, 5), key, aad);
+ ICEBERG_UNWRAP_OR_FAIL(auto stream, Open(path, key, aad, stored)->Open());
+
+ // A position near int64 max must be rejected without overflowing the range
check
+ std::vector<std::byte> one(1);
+ EXPECT_THAT(stream->ReadFully(std::numeric_limits<int64_t>::max(), one),
+ IsError(ErrorKind::kIOError));
+ EXPECT_THAT(stream->ReadFully(-1, one), IsError(ErrorKind::kIOError));
+
+ // Reading up to the last byte is allowed; one byte past it is not
+ EXPECT_THAT(stream->ReadFully(99, one), IsOk());
+ EXPECT_THAT(stream->ReadFully(100, one), IsError(ErrorKind::kIOError));
+ EXPECT_THAT(stream->ReadFully(100, std::span<std::byte>{}), IsOk());
+}
+
+TEST_F(AesGcmStreamTest, RejectsInvalidEncryptedLength) {
+ constexpr int64_t kHeader = AesGcmStream::kHeaderLength;
+ constexpr int64_t kCipherBlock = AesGcmStream::kCipherBlockSize;
+ constexpr int64_t kOverhead = AesGcmStream::kNonceLength +
AesGcmStream::kGcmTagLength;
+
+ // Shorter than an empty stream
+ EXPECT_THAT(AesGcmStream::PlaintextLength(-1), IsError(ErrorKind::kInvalid));
+ EXPECT_THAT(AesGcmStream::PlaintextLength(0), IsError(ErrorKind::kInvalid));
+ EXPECT_THAT(AesGcmStream::PlaintextLength(AesGcmStream::kMinStreamLength -
1),
+ IsError(ErrorKind::kInvalid));
+ EXPECT_THAT(AesGcmStream::PlaintextLength(AesGcmStream::kMinStreamLength),
+ HasValue(::testing::Eq(0)));
+
+ // A last block too short to hold a nonce and a tag
+ EXPECT_THAT(AesGcmStream::PlaintextLength(kHeader + kCipherBlock + kOverhead
- 1),
+ IsError(ErrorKind::kInvalid));
+ EXPECT_THAT(AesGcmStream::PlaintextLength(kHeader + kCipherBlock +
kOverhead),
+ HasValue(::testing::Eq(AesGcmStream::kPlainBlockSize)));
+
+ // More blocks than the int32 block index in the block AAD can address. The
check
+ // must happen here rather than when a block is decrypted, so that a caller
asking
+ // only for the size never sees a length it could not read back.
+ constexpr int64_t kMaxBlocks = std::numeric_limits<int32_t>::max();
+ EXPECT_THAT(AesGcmStream::PlaintextLength(kHeader + kMaxBlocks *
kCipherBlock),
+ HasValue(::testing::Eq(kMaxBlocks *
AesGcmStream::kPlainBlockSize)));
+ EXPECT_THAT(AesGcmStream::PlaintextLength(kHeader + kMaxBlocks *
kCipherBlock + 1),
+ IsError(ErrorKind::kInvalid));
+
+ // Both Size() and Open() reject it, rather than Open() alone
+ auto path = CreateNewTempFilePath();
+ Write(path, TestBytes(10, 5), kDataKey, kAadPrefix);
+ auto file = Open(path, kDataKey, kAadPrefix, kHeader + kMaxBlocks *
kCipherBlock + 1);
+ EXPECT_THAT(file->Size(), IsError(ErrorKind::kInvalid));
+ EXPECT_THAT(file->Open(), IsError(ErrorKind::kInvalid));
+}
+
+} // namespace iceberg
