kevinjqliu commented on code in PR #3963: URL: https://github.com/apache/iceberg-python/pull/3963#discussion_r4008894994
########## pyiceberg/encryption/ciphers.py: ########## @@ -0,0 +1,120 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +"""AES-GCM primitives for table encryption.""" + +from __future__ import annotations + +import os +from dataclasses import dataclass, field +from enum import IntEnum +from typing import TYPE_CHECKING + +from pyiceberg.utils.lazy_import import not_installed + +if TYPE_CHECKING: + from cryptography.exceptions import InvalidTag + from cryptography.hazmat.primitives.ciphers.aead import AESGCM + + +class AesKeySize(IntEnum): + """The supported AES key sizes, in bits.""" + + BITS_128 = 128 + BITS_192 = 192 + BITS_256 = 256 + + @property + def key_length(self) -> int: + """Return the key length in bytes.""" + return self.value // 8 + + @classmethod + def from_key_length(cls, key_length: int) -> AesKeySize: + """Return the key size for a key of `key_length` bytes.""" + try: + return cls(key_length * 8) + except ValueError as e: + raise ValueError(f"Unsupported key length: {key_length} (must be 16, 24 or 32)") from e + + +@dataclass(frozen=True) +class SecureKey: + """An AES key of a length the spec allows, kept out of reprs and tracebacks.""" + + key: bytes = field(repr=False) + + def __post_init__(self) -> None: + """Reject keys that are not a supported AES key length.""" + AesKeySize.from_key_length(len(self.key)) + + @property + def key_size(self) -> AesKeySize: + """Return the size of this key.""" + return AesKeySize.from_key_length(len(self.key)) + + @classmethod + def generate(cls, key_size: AesKeySize = AesKeySize.BITS_128) -> SecureKey: + """Generate a new key of `key_size`.""" + return cls(os.urandom(key_size.key_length)) + + +class AesGcmCipher: + """Encrypts and decrypts using AES-GCM. + + Ciphertext is laid out as `nonce || ciphertext || tag`, matching Java and iceberg-rust. + """ + + NONCE_LENGTH = 12 + TAG_LENGTH = 16 + + def __init__(self, key: SecureKey) -> None: + try: + from cryptography.exceptions import InvalidTag + from cryptography.hazmat.primitives.ciphers.aead import AESGCM + except ImportError: + raise not_installed("cryptography", extras_name="encryption") from None + + self._aes_gcm: AESGCM = AESGCM(key.key) + self._invalid_tag: type[InvalidTag] = InvalidTag Review Comment: nit: codex suggested this strange logic instead of `not_installed` ``` self._aes_gcm: AESGCM = try_import("cryptography.hazmat.primitives.ciphers.aead", extras_name="encryption").AESGCM( key.key ) self._invalid_tag: type[InvalidTag] = try_import("cryptography.exceptions", extras_name="encryption").InvalidTag ``` ########## tests/encryption/test_ciphers.py: ########## @@ -0,0 +1,137 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +import pytest +from pytest_mock import MockFixture + +from pyiceberg.encryption.ciphers import AesGcmCipher, AesKeySize, SecureKey +from pyiceberg.exceptions import NotInstalledError + +AES128_KEY = b"0123456789012345" +PLAINTEXT = b"the quick brown fox" + + [email protected]( + "key_length, key_size", + [(16, AesKeySize.BITS_128), (24, AesKeySize.BITS_192), (32, AesKeySize.BITS_256)], +) +def test_key_size_from_key_length(key_length: int, key_size: AesKeySize) -> None: + assert AesKeySize.from_key_length(key_length) == key_size + assert key_size.key_length == key_length + + [email protected]("key_length", [0, 4, 15, 20, 33]) +def test_key_size_rejects_invalid_key_length(key_length: int) -> None: + with pytest.raises(ValueError, match=f"Unsupported key length: {key_length}"): + AesKeySize.from_key_length(key_length) + + [email protected]("key_length", [0, 4, 15, 20, 33]) +def test_secure_key_rejects_invalid_key_length(key_length: int) -> None: + with pytest.raises(ValueError, match="Unsupported key length"): + SecureKey(bytes(key_length)) + + [email protected]("key_size", list(AesKeySize)) +def test_secure_key_generate(key_size: AesKeySize) -> None: + key = SecureKey.generate(key_size) + + assert len(key.key) == key_size.key_length + assert key.key_size == key_size + assert SecureKey.generate(key_size) != key + + +def test_secure_key_repr_redacts_key() -> None: + key = SecureKey(AES128_KEY) + + assert repr(key) == "SecureKey()" + assert repr(AES128_KEY) not in repr(key) + + [email protected]("key_size", list(AesKeySize)) [email protected]("aad", [None, b"", b"aad"]) +def test_encrypt_decrypt_round_trip(key_size: AesKeySize, aad: bytes | None) -> None: + cipher = AesGcmCipher(SecureKey.generate(key_size)) + + ciphertext = cipher.encrypt(PLAINTEXT, aad) + + assert ciphertext != PLAINTEXT + assert cipher.decrypt(ciphertext, aad) == PLAINTEXT + + Review Comment: something like this would be good. just as a regression test ``` def test_aes128_gcm_known_answer(mocker: MockFixture) -> None: # NIST CAVS gcmEncryptExtIV128.rsp vector from RustCrypto's aes-gcm tests. # https://github.com/RustCrypto/AEADs/blob/aes-gcm-v0.10.3/aes-gcm/tests/aes128gcm.rs#L737-L744 key = bytes.fromhex("c939cc13397c1d37de6ae0e1cb7c423c") nonce = bytes.fromhex("b3d8cc017cbb89b39e0f67e2") plaintext = bytes.fromhex("c3b3c41f113a31b73d9a5cd432103069") aad = bytes.fromhex("24825602bd12a984e0092d3e448eda5f") ciphertext = bytes.fromhex("93fe7d9e9bfd10348a5606e5cafa7354") tag = bytes.fromhex("0032a1dc85f1c9786925a2e71d8272dd") expected = nonce + ciphertext + tag cipher = AesGcmCipher(SecureKey(key)) mocker.patch("pyiceberg.encryption.ciphers.os.urandom", return_value=nonce) assert cipher.encrypt(plaintext, aad) == expected assert cipher.decrypt(expected, aad) == plaintext ``` ########## pyiceberg/encryption/ciphers.py: ########## @@ -0,0 +1,120 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +"""AES-GCM primitives for table encryption.""" + +from __future__ import annotations + +import os +from dataclasses import dataclass, field +from enum import IntEnum +from typing import TYPE_CHECKING + +from pyiceberg.utils.lazy_import import not_installed + +if TYPE_CHECKING: + from cryptography.exceptions import InvalidTag + from cryptography.hazmat.primitives.ciphers.aead import AESGCM + + +class AesKeySize(IntEnum): + """The supported AES key sizes, in bits.""" + + BITS_128 = 128 + BITS_192 = 192 + BITS_256 = 256 + + @property + def key_length(self) -> int: + """Return the key length in bytes.""" + return self.value // 8 + + @classmethod + def from_key_length(cls, key_length: int) -> AesKeySize: + """Return the key size for a key of `key_length` bytes.""" + try: + return cls(key_length * 8) + except ValueError as e: + raise ValueError(f"Unsupported key length: {key_length} (must be 16, 24 or 32)") from e + + +@dataclass(frozen=True) +class SecureKey: + """An AES key of a length the spec allows, kept out of reprs and tracebacks.""" + + key: bytes = field(repr=False) + + def __post_init__(self) -> None: + """Reject keys that are not a supported AES key length.""" + AesKeySize.from_key_length(len(self.key)) + + @property + def key_size(self) -> AesKeySize: + """Return the size of this key.""" + return AesKeySize.from_key_length(len(self.key)) + + @classmethod + def generate(cls, key_size: AesKeySize = AesKeySize.BITS_128) -> SecureKey: + """Generate a new key of `key_size`.""" + return cls(os.urandom(key_size.key_length)) + + +class AesGcmCipher: + """Encrypts and decrypts using AES-GCM. + + Ciphertext is laid out as `nonce || ciphertext || tag`, matching Java and iceberg-rust. + """ + + NONCE_LENGTH = 12 + TAG_LENGTH = 16 + + def __init__(self, key: SecureKey) -> None: + try: + from cryptography.exceptions import InvalidTag + from cryptography.hazmat.primitives.ciphers.aead import AESGCM + except ImportError: + raise not_installed("cryptography", extras_name="encryption") from None + + self._aes_gcm: AESGCM = AESGCM(key.key) + self._invalid_tag: type[InvalidTag] = InvalidTag + + def encrypt(self, plaintext: bytes, aad: bytes | None = None) -> bytes: + """Encrypt `plaintext`, authenticating `aad` alongside it. + + Args: + plaintext (bytes): The data to encrypt. + aad (bytes | None): Additional data to authenticate but not encrypt. + """ + nonce = os.urandom(self.NONCE_LENGTH) + return nonce + self._aes_gcm.encrypt(nonce, plaintext, aad) + + def decrypt(self, ciphertext: bytes, aad: bytes | None = None) -> bytes: + """Decrypt `ciphertext`, verifying `aad` alongside it. + + Args: + ciphertext (bytes): The data to decrypt, as returned by `encrypt`. + aad (bytes | None): The additional data that was authenticated on encryption. + """ + if len(ciphertext) < self.NONCE_LENGTH + self.TAG_LENGTH: + raise ValueError( + f"Ciphertext too short: expected at least {self.NONCE_LENGTH + self.TAG_LENGTH} bytes, got {len(ciphertext)}" + ) + + nonce, encrypted = ciphertext[: self.NONCE_LENGTH], ciphertext[self.NONCE_LENGTH :] + try: + return self._aes_gcm.decrypt(nonce, encrypted, aad) + except self._invalid_tag as e: + raise ValueError("AES-GCM decryption failed") from e Review Comment: I think that'll be a nice touch -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: [email protected] For queries about this service, please contact Infrastructure at: [email protected] --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
