mbutrovich commented on code in PR #3969: URL: https://github.com/apache/iceberg-python/pull/3969#discussion_r4085185107
########## pyiceberg/encryption/stream.py: ########## @@ -0,0 +1,159 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +"""Format primitives for the AGS1 stream, used to encrypt manifests and manifest lists. + +An AGS1 stream is an 8 byte header followed by a sequence of AES-GCM blocks:: + + "AGS1" || plain_block_size (4 bytes, little endian) + nonce || ciphertext || tag (block 0, up to PLAIN_BLOCK_SIZE of plaintext) + nonce || ciphertext || tag (block 1..n, the last of which may be shorter) + +Each block authenticates `aad_prefix || block_index` as additional data, so blocks cannot +be reordered or moved between files. A stream holds at least one block, so an empty file is +a header followed by a single empty block rather than a bare header. +""" + +from __future__ import annotations + +from dataclasses import dataclass + +from pyiceberg.encryption.ciphers import AesGcmCipher + +GCM_STREAM_MAGIC = b"AGS1" +PLAIN_BLOCK_SIZE = 1024 * 1024 +GCM_STREAM_HEADER_LENGTH = len(GCM_STREAM_MAGIC) + 4 +BLOCK_OVERHEAD = AesGcmCipher.NONCE_LENGTH + AesGcmCipher.TAG_LENGTH +CIPHER_BLOCK_SIZE = PLAIN_BLOCK_SIZE + BLOCK_OVERHEAD +BLOCK_INDEX_LENGTH = 4 +MAX_BLOCKS = 2 ** (8 * BLOCK_INDEX_LENGTH) - 1 +MIN_STREAM_LENGTH = GCM_STREAM_HEADER_LENGTH + BLOCK_OVERHEAD + + +def stream_block_aad(aad_prefix: bytes | None, block_index: int) -> bytes: + """Return the additional authenticated data for the block at `block_index`. + + Args: + aad_prefix (bytes | None): The file's AAD prefix, from its key metadata. + block_index (int): The zero-based index of the block within the stream. + """ + return (aad_prefix or b"") + block_index.to_bytes(BLOCK_INDEX_LENGTH, "little") + + +def encode_stream_header() -> bytes: + """Encode the AGS1 header that precedes the first block.""" + return GCM_STREAM_MAGIC + PLAIN_BLOCK_SIZE.to_bytes(4, "little") + + +def decode_stream_header(header: bytes) -> int: + """Decode an AGS1 header, returning the plaintext block size it declares. + + Args: + header (bytes): At least `GCM_STREAM_HEADER_LENGTH` bytes from the start of the stream. + """ + if len(header) < GCM_STREAM_HEADER_LENGTH: + raise ValueError(f"Invalid AGS1 header: expected {GCM_STREAM_HEADER_LENGTH} bytes, got {len(header)}") + + if (magic := header[: len(GCM_STREAM_MAGIC)]) != GCM_STREAM_MAGIC: + raise ValueError(f"Invalid AGS1 header: magic {magic!r} does not match {GCM_STREAM_MAGIC!r}") + + plain_block_size = int.from_bytes(header[len(GCM_STREAM_MAGIC) : GCM_STREAM_HEADER_LENGTH], "little") + if plain_block_size != PLAIN_BLOCK_SIZE: + raise ValueError(f"Unsupported AGS1 block size: {plain_block_size} (expected {PLAIN_BLOCK_SIZE})") + + return plain_block_size + + +def calculate_plaintext_length(encrypted_length: int) -> int: + """Return the plaintext length of an AGS1 stream that occupies `encrypted_length` bytes. + + Args: + encrypted_length (int): The stream's length, which must be the trusted `file_length` from the file's + `StandardKeyMetadata`, never a file system stat. The spec requires the trusted length because a + stat lets an attacker drop trailing blocks while every remaining block still authenticates. + """ + if encrypted_length < MIN_STREAM_LENGTH: + raise ValueError(f"Invalid AGS1 stream: expected at least {MIN_STREAM_LENGTH} bytes, got {encrypted_length}") + + stream_length = encrypted_length - GCM_STREAM_HEADER_LENGTH + full_blocks, cipher_bytes_in_last_block = divmod(stream_length, CIPHER_BLOCK_SIZE) + if cipher_bytes_in_last_block == 0: + return full_blocks * PLAIN_BLOCK_SIZE + + if cipher_bytes_in_last_block < BLOCK_OVERHEAD: + raise ValueError( + f"Truncated AGS1 stream: last block is {cipher_bytes_in_last_block} bytes, expected at least {BLOCK_OVERHEAD}" + ) + + return full_blocks * PLAIN_BLOCK_SIZE + cipher_bytes_in_last_block - BLOCK_OVERHEAD + + +@dataclass(frozen=True) +class Ags1Layout: + """Where each block of an AGS1 stream sits, derived from the trusted encrypted file length. + + Only the final block may hold less than `PLAIN_BLOCK_SIZE` of plaintext, so the layout + follows from the encrypted length alone, without reading the stream. + """ + + plaintext_length: int + num_blocks: int + last_cipher_block_size: int + + @classmethod + def from_encrypted_length(cls, encrypted_length: int) -> Ags1Layout: + """Derive the layout of an AGS1 stream that occupies `encrypted_length` bytes. + + Args: + encrypted_length (int): The stream's length, which must be the trusted `file_length` from the file's + `StandardKeyMetadata`, never a file system stat. See `calculate_plaintext_length`. + """ + plaintext_length = calculate_plaintext_length(encrypted_length) + stream_length = encrypted_length - GCM_STREAM_HEADER_LENGTH + full_blocks, cipher_bytes_in_last_block = divmod(stream_length, CIPHER_BLOCK_SIZE) + if cipher_bytes_in_last_block == 0: + num_blocks, last_cipher_block_size = full_blocks, CIPHER_BLOCK_SIZE + else: + num_blocks, last_cipher_block_size = full_blocks + 1, cipher_bytes_in_last_block + + if num_blocks > MAX_BLOCKS: + raise ValueError(f"AGS1 streams hold at most {MAX_BLOCKS} blocks, but {encrypted_length} bytes needs {num_blocks}") + + return cls(plaintext_length=plaintext_length, num_blocks=num_blocks, last_cipher_block_size=last_cipher_block_size) Review Comment: `from_encrypted_length` repeats the `stream_length` and `divmod` work that `calculate_plaintext_length` just did (lines 91-92), so the block layout rule lives in two places. Could `from_encrypted_length` do the validation and the `divmod` once, and derive `plaintext_length` as `(num_blocks - 1) * PLAIN_BLOCK_SIZE + last_cipher_block_size - BLOCK_OVERHEAD`? `calculate_plaintext_length` can then return `Ags1Layout.from_encrypted_length(encrypted_length).plaintext_length`, or go away if the public API thread ends with only `Ags1Layout`. It also means the `MAX_BLOCKS` check applies to both entry points, where today `calculate_plaintext_length` accepts lengths that `from_encrypted_length` rejects. ########## tests/encryption/ags1/README.md: ########## @@ -0,0 +1,74 @@ +<!-- + ~ Licensed to the Apache Software Foundation (ASF) under one + ~ or more contributor license agreements. See the NOTICE file + ~ distributed with this work for additional information + ~ regarding copyright ownership. The ASF licenses this file + ~ to you under the Apache License, Version 2.0 (the + ~ "License"); you may not use this file except in compliance + ~ with the License. You may obtain a copy of the License at + ~ + ~ http://www.apache.org/licenses/LICENSE-2.0 + ~ + ~ Unless required by applicable law or agreed to in writing, + ~ software distributed under the License is distributed on an + ~ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + ~ KIND, either express or implied. See the License for the + ~ specific language governing permissions and limitations + ~ under the License. +--> + +# AGS1 cross-client test fixtures + +These files were written by Java's `AesGcmOutputStream` (Apache Iceberg 1.11.0), not +by PyIceberg. They exist so PyIceberg's AGS1 support is checked against another +implementation's bytes rather than only against its own round trip. All four also +decrypt with iceberg-rust 0.10.1, through its `EncryptedInputFile`. + +## Fixtures + +| File | Encrypted size | Plaintext | Pins | +| --- | --- | --- | --- | +| `empty.ags1` | 36 B | 0 B | Java writes an 8 byte header **plus one empty block** for an empty file, not a bare header | +| `partial-block.ags1` | 136 B | 100 B | Header, nonce/tag layout, and a single short block | +| `partial-block-no-aad.ags1` | 136 B | 100 B | The same stream with a null AAD prefix, so the block index alone is the AAD | +| `aligned-multi-block.ags1` | 2097216 B | 2 MiB | Two full blocks: the little-endian block index in each block's AAD, and that a block-aligned write appends **no** trailing empty block | Review Comment: `MANIFEST.in` has `recursive-include tests *`, so these fixtures ship in every sdist, and they add about 2 MiB to a `tests/` directory that is 4.8 MB today. They also stay in git history after #4010 moves them to iceberg-verification. Is that the trade-off the maintainers want, or should the fixtures land in iceberg-verification first and be fetched from there? @kevinjqliu, what do you think? ########## tests/encryption/ags1/README.md: ########## @@ -0,0 +1,74 @@ +<!-- + ~ Licensed to the Apache Software Foundation (ASF) under one + ~ or more contributor license agreements. See the NOTICE file + ~ distributed with this work for additional information + ~ regarding copyright ownership. The ASF licenses this file + ~ to you under the Apache License, Version 2.0 (the + ~ "License"); you may not use this file except in compliance + ~ with the License. You may obtain a copy of the License at + ~ + ~ http://www.apache.org/licenses/LICENSE-2.0 + ~ + ~ Unless required by applicable law or agreed to in writing, + ~ software distributed under the License is distributed on an + ~ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + ~ KIND, either express or implied. See the License for the + ~ specific language governing permissions and limitations + ~ under the License. +--> + +# AGS1 cross-client test fixtures + +These files were written by Java's `AesGcmOutputStream` (Apache Iceberg 1.11.0), not +by PyIceberg. They exist so PyIceberg's AGS1 support is checked against another +implementation's bytes rather than only against its own round trip. All four also +decrypt with iceberg-rust 0.10.1, through its `EncryptedInputFile`. + +## Fixtures + +| File | Encrypted size | Plaintext | Pins | +| --- | --- | --- | --- | +| `empty.ags1` | 36 B | 0 B | Java writes an 8 byte header **plus one empty block** for an empty file, not a bare header | +| `partial-block.ags1` | 136 B | 100 B | Header, nonce/tag layout, and a single short block | +| `partial-block-no-aad.ags1` | 136 B | 100 B | The same stream with a null AAD prefix, so the block index alone is the AAD | +| `aligned-multi-block.ags1` | 2097216 B | 2 MiB | Two full blocks: the little-endian block index in each block's AAD, and that a block-aligned write appends **no** trailing empty block | + +`empty.ags1` is worth calling out. The spec says the last block has a non-zero +length, which makes a bare 8 byte header the natural encoding of an empty file, but +Java writes 36 bytes and its `AesGcmInputFile` rejects anything shorter, as does +iceberg-rust. PyIceberg matches them: `MIN_STREAM_LENGTH` is 36, so a bare header is +rejected rather than read as an empty stream. + +Block-aligned and partial *single* block variants are deliberately not checked in. +The 1 MiB block size is hard-coded, so each would add another 1 MiB of +incompressible ciphertext without covering a case the four files above miss. + +## Parameters + +Every fixture uses: + +- **Key**: 16 bytes, `0x00` through `0x0f` +- **AAD prefix**: ASCII `pyiceberg-ags1`, except `partial-block-no-aad.ags1`, which has none +- **Plaintext**: byte `i` is `i % 251`. The period is prime and therefore coprime with the + 1 MiB block size, so the pattern shifts phase at every block boundary and a + misordered or misindexed block is detectable from the plaintext alone + +## Regenerating + +Each block uses a fresh random nonce, so regenerating produces different bytes. +The file lengths, and the plaintext each file decrypts to, are deterministic. Tests +decrypt these fixtures rather than comparing them byte for byte, so a regeneration +is safe as long as the parameters above are unchanged. + +From this directory, with a JDK 17 or later: + +<!-- markdown-link-check-disable-next-line --> +```bash +V=1.11.0 +for a in iceberg-core iceberg-api iceberg-bundled-guava; do + curl -sfLO "https://repo1.maven.org/maven2/org/apache/iceberg/$a/$V/$a-$V.jar"; Review Comment: This README fails the `markdown-link-check` job on the head commit: the templated URL `https://repo1.maven.org/maven2/org/apache/iceberg/$a/$V/$a-$V.jar` returns 404 ([run](https://github.com/apache/iceberg-python/actions/runs/35843353745/job/107123467607)). `disable-next-line` only covers line 66, the opening fence, and the URL is on line 69. Wrapping the block with `<!-- markdown-link-check-disable -->` and `<!-- markdown-link-check-enable -->` should fix it. -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: [email protected] For queries about this service, please contact Infrastructure at: [email protected] --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
