This is an automated email from the ASF dual-hosted git repository.
amogh-jahagirdar pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/iceberg.git
The following commit(s) were added to refs/heads/main by this push:
new 6951360ec8 Parquet: Read file metrics from natively encrypted files
(#18535)
6951360ec8 is described below
commit 6951360ec8a01644d3c24824ab635e06a82f8a7c
Author: Rahul Shivu Mahadev <[email protected]>
AuthorDate: Mon Nov 2 09:12:00 2026 -0800
Parquet: Read file metrics from natively encrypted files (#18535)
ParquetUtil.fileMetrics opened the footer without decryption properties,
so it failed on files written with Parquet native encryption. When the
input is a NativeEncryptionInputFile, open the footer with the file's
encryption key and AAD prefix.
---
.../org/apache/iceberg/parquet/ParquetUtil.java | 24 ++++-
.../apache/iceberg/parquet/TestParquetUtil.java | 104 +++++++++++++++++++++
2 files changed, 127 insertions(+), 1 deletion(-)
diff --git a/parquet/src/main/java/org/apache/iceberg/parquet/ParquetUtil.java
b/parquet/src/main/java/org/apache/iceberg/parquet/ParquetUtil.java
index 9f5e1caa24..b1e5348c32 100644
--- a/parquet/src/main/java/org/apache/iceberg/parquet/ParquetUtil.java
+++ b/parquet/src/main/java/org/apache/iceberg/parquet/ParquetUtil.java
@@ -29,18 +29,23 @@ import org.apache.iceberg.FieldMetrics;
import org.apache.iceberg.Metrics;
import org.apache.iceberg.MetricsConfig;
import org.apache.iceberg.Schema;
+import org.apache.iceberg.encryption.NativeEncryptionInputFile;
import org.apache.iceberg.exceptions.RuntimeIOException;
import org.apache.iceberg.io.InputFile;
import org.apache.iceberg.mapping.NameMapping;
import org.apache.iceberg.relocated.com.google.common.base.Preconditions;
import org.apache.iceberg.relocated.com.google.common.collect.Lists;
import org.apache.iceberg.relocated.com.google.common.collect.Sets;
+import org.apache.iceberg.util.ByteBuffers;
+import org.apache.parquet.ParquetReadOptions;
import org.apache.parquet.column.ColumnDescriptor;
import org.apache.parquet.column.Dictionary;
import org.apache.parquet.column.Encoding;
import org.apache.parquet.column.EncodingStats;
import org.apache.parquet.column.page.DictionaryPage;
import org.apache.parquet.column.page.PageReader;
+import org.apache.parquet.conf.PlainParquetConfiguration;
+import org.apache.parquet.crypto.FileDecryptionProperties;
import org.apache.parquet.hadoop.ParquetFileReader;
import org.apache.parquet.hadoop.metadata.BlockMetaData;
import org.apache.parquet.hadoop.metadata.ColumnChunkMetaData;
@@ -61,13 +66,30 @@ public class ParquetUtil {
public static Metrics fileMetrics(
InputFile file, MetricsConfig metricsConfig, NameMapping nameMapping) {
- try (ParquetFileReader reader =
ParquetFileReader.open(ParquetIO.file(file))) {
+ try (ParquetFileReader reader = openFileReader(file)) {
return footerMetrics(reader.getFooter(), Stream.empty(), metricsConfig,
nameMapping);
} catch (IOException e) {
throw new RuntimeIOException(e, "Failed to read footer of file: %s",
file.location());
}
}
+ private static ParquetFileReader openFileReader(InputFile file) throws
IOException {
+ if (file instanceof NativeEncryptionInputFile nativeFile) {
+ FileDecryptionProperties decryptionProperties =
+ FileDecryptionProperties.builder()
+
.withFooterKey(ByteBuffers.toByteArray(nativeFile.keyMetadata().encryptionKey()))
+
.withAADPrefix(ByteBuffers.toByteArray(nativeFile.keyMetadata().aadPrefix()))
+ .build();
+ ParquetReadOptions options =
+ ParquetReadOptions.builder(new PlainParquetConfiguration())
+ .withDecryption(decryptionProperties)
+ .build();
+ return
ParquetFileReader.open(ParquetIO.file(nativeFile.encryptedInputFile()),
options);
+ }
+
+ return ParquetFileReader.open(ParquetIO.file(file));
+ }
+
public static Metrics footerMetrics(
ParquetMetadata metadata, Stream<FieldMetrics<?>> fieldMetrics,
MetricsConfig metricsConfig) {
return footerMetrics(metadata, fieldMetrics, metricsConfig, null);
diff --git
a/parquet/src/test/java/org/apache/iceberg/parquet/TestParquetUtil.java
b/parquet/src/test/java/org/apache/iceberg/parquet/TestParquetUtil.java
new file mode 100644
index 0000000000..c8fd4cdcee
--- /dev/null
+++ b/parquet/src/test/java/org/apache/iceberg/parquet/TestParquetUtil.java
@@ -0,0 +1,104 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.iceberg.parquet;
+
+import static org.apache.iceberg.Files.localInput;
+import static org.apache.iceberg.Files.localOutput;
+import static
org.apache.iceberg.parquet.ParquetWritingTestUtils.createTempFile;
+import static org.apache.iceberg.types.Types.NestedField.optional;
+import static org.assertj.core.api.Assertions.assertThat;
+import static org.assertj.core.api.Assertions.assertThatThrownBy;
+
+import java.io.File;
+import java.io.IOException;
+import java.nio.file.Path;
+import java.util.List;
+import org.apache.avro.generic.GenericData;
+import org.apache.iceberg.Metrics;
+import org.apache.iceberg.MetricsConfig;
+import org.apache.iceberg.Schema;
+import org.apache.iceberg.avro.AvroSchemaUtil;
+import org.apache.iceberg.encryption.EncryptedFiles;
+import org.apache.iceberg.encryption.EncryptedOutputFile;
+import org.apache.iceberg.encryption.EncryptionManager;
+import org.apache.iceberg.encryption.EncryptionTestHelpers;
+import org.apache.iceberg.encryption.NativeEncryptionInputFile;
+import org.apache.iceberg.io.FileAppender;
+import org.apache.iceberg.io.InputFile;
+import org.apache.iceberg.relocated.com.google.common.collect.Lists;
+import org.apache.iceberg.types.Conversions;
+import org.apache.iceberg.types.Types;
+import org.apache.parquet.crypto.ParquetCryptoRuntimeException;
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.io.TempDir;
+
+class TestParquetUtil {
+ private static final Schema SCHEMA = new Schema(optional(1, "id",
Types.IntegerType.get()));
+ private static final List<Integer> VALUES = Lists.newArrayList(3, null, 1,
2);
+
+ @TempDir private Path temp;
+
+ @Test
+ void fileMetricsOfPlainFile() throws IOException {
+ File file = createTempFile(temp);
+ write(Parquet.write(localOutput(file)));
+
+ assertMetrics(ParquetUtil.fileMetrics(localInput(file),
MetricsConfig.getDefault()));
+ }
+
+ @Test
+ void fileMetricsOfNativelyEncryptedFile() throws IOException {
+ File file = createTempFile(temp);
+ EncryptionManager encryption =
EncryptionTestHelpers.createEncryptionManager();
+ EncryptedOutputFile output = encryption.encrypt(localOutput(file));
+ write(Parquet.write(output));
+
+ assertThatThrownBy(() -> ParquetUtil.fileMetrics(localInput(file),
MetricsConfig.getDefault()))
+ .isInstanceOf(ParquetCryptoRuntimeException.class)
+ .hasMessage("Trying to read file with encrypted footer. No keys
available");
+
+ InputFile input =
+ encryption.decrypt(EncryptedFiles.encryptedInput(localInput(file),
output.keyMetadata()));
+ assertThat(input).isInstanceOf(NativeEncryptionInputFile.class);
+ assertMetrics(ParquetUtil.fileMetrics(input, MetricsConfig.getDefault()));
+ }
+
+ private static void write(Parquet.WriteBuilder builder) throws IOException {
+ org.apache.avro.Schema avroSchema =
AvroSchemaUtil.convert(SCHEMA.asStruct());
+ try (FileAppender<GenericData.Record> writer =
builder.schema(SCHEMA).build()) {
+ for (Integer value : VALUES) {
+ GenericData.Record record = new GenericData.Record(avroSchema);
+ record.put("id", value);
+ writer.add(record);
+ }
+ }
+ }
+
+ private static void assertMetrics(Metrics metrics) {
+ assertThat(metrics.recordCount()).isEqualTo(4L);
+ assertThat(metrics.valueCounts()).containsEntry(1, 4L);
+ assertThat(metrics.nullValueCounts()).containsEntry(1, 1L);
+ Integer lower =
+ Conversions.fromByteBuffer(Types.IntegerType.get(),
metrics.lowerBounds().get(1));
+ Integer upper =
+ Conversions.fromByteBuffer(Types.IntegerType.get(),
metrics.upperBounds().get(1));
+ assertThat(lower).isEqualTo(1);
+ assertThat(upper).isEqualTo(3);
+ }
+}
