stevenzwu commented on code in PR #16936: URL: https://github.com/apache/iceberg/pull/16936#discussion_r3462545831
########## core/src/main/java/org/apache/iceberg/ContentEntryAdapters.java: ########## @@ -0,0 +1,400 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.iceberg; + +import java.nio.ByteBuffer; +import java.util.Collections; +import java.util.Locale; +import java.util.Map; +import java.util.WeakHashMap; +import org.apache.iceberg.relocated.com.google.common.base.Preconditions; +import org.apache.iceberg.relocated.com.google.common.collect.Maps; +import org.apache.iceberg.types.Type; +import org.apache.iceberg.types.Types; +import org.apache.iceberg.util.ContentFileUtil; + +/** + * Builds {@link TrackedFile} instances for v4 content_entry rows from legacy {@link ManifestEntry} + * and {@link ManifestFile} inputs. + */ +class ContentEntryAdapters { + /** + * writer_format_version for content_entry rows produced by a v4 writer. Matches the table format + * version (4). + */ + static final int V4_WRITER_FORMAT_VERSION = 4; + + /** + * writer_format_version for content_entry rows that reference a leaf manifest written by a pre-v4 + * writer (v1, v2, or v3). Used at the root manifest level when a v4 root carries over legacy leaf + * manifests during a v3-to-v4 upgrade. Matches a pre-v4 table format version (0 is the sentinel; + * v1/v2/v3 leaves are not re-encoded as content_entry, so the root just tags the reference as + * legacy). + */ + static final int LEGACY_WRITER_FORMAT_VERSION = 0; + + // Cache of primitive-field types keyed by table schema. The originalTypes map a Metrics + // instance carries is read-only inside MetricsUtil.fromMetrics, so a single per-schema map + // can be shared across every adapter call for a writer's lifetime instead of being rebuilt + // per row. WeakHashMap keys release when callers stop referencing the schema. + private static final Map<Schema, Map<Integer, Type>> PRIMITIVE_TYPES_BY_SCHEMA = + Collections.synchronizedMap(new WeakHashMap<>()); + + private ContentEntryAdapters() {} + + static TrackedFile fromDataFile( + ManifestEntry<DataFile> entry, Schema tableSchema, EntryStatus statusOverride) { + Preconditions.checkArgument(entry != null, "Invalid manifest entry: null"); + DataFile file = entry.file(); + Preconditions.checkArgument(file != null, "Invalid data file: null"); + Preconditions.checkArgument( + file.content() == FileContent.DATA, "Invalid content for data file: %s", file.content()); + return buildContentFileEntry( + file, + statusOverride, + snapshotIdOrZero(entry), + entry.dataSequenceNumber(), + entry.fileSequenceNumber(), + tableSchema); + } + + static TrackedFile fromDeleteFile( + ManifestEntry<DeleteFile> entry, Schema tableSchema, EntryStatus statusOverride) { + Preconditions.checkArgument(entry != null, "Invalid manifest entry: null"); + DeleteFile file = entry.file(); + Preconditions.checkArgument(file != null, "Invalid delete file: null"); + // v4 leaf delete manifests must only contain content_type=EQUALITY_DELETES (per spec PR + // #16025). Reject POSITION_DELETES with a hint that distinguishes the two legacy shapes by + // file format (the canonical DV check per ContentFileUtil.isDV — both shapes can carry a + // referencedDataFile, so that field is not a reliable distinguisher): + // - v3 delete vector (POSITION_DELETES stored as a Puffin blob) must be colocated on the + // data file's content_entry via TrackedFileBuilder.deletionVector(...) — see + // MergingSnapshotProducer's row-delta path. + // - v2 standalone position delete file (POSITION_DELETES stored in Parquet/Avro/ORC) has no + // v4 representation; it can only live in pre-v4 legacy manifests carried over via a + // writer_format_version=0 manifest reference. + if (file.content() == FileContent.POSITION_DELETES) { + throw new IllegalArgumentException( + ContentFileUtil.isDV(file) + ? String.format( + Locale.ROOT, + "v3 delete vectors must be colocated on the data file's content_entry, not " + + "written as a delete manifest entry: %s referencing %s", + file.location(), + file.referencedDataFile()) + : String.format( + Locale.ROOT, + "v2 position delete files have no v4 representation; carry them over via a " + + "legacy v3 manifest with writer_format_version=0: %s", + file.location())); + } + + Preconditions.checkArgument( + file.content() == FileContent.EQUALITY_DELETES, + "Invalid content for delete file: %s", + file.content()); + return buildContentFileEntry( + file, + statusOverride, + snapshotIdOrZero(entry), + entry.dataSequenceNumber(), + entry.fileSequenceNumber(), + tableSchema); + } + + /** + * Builds a manifest reference content_entry for the v4 root manifest. + * + * @param manifest the leaf manifest being referenced + * @param writerFormatVersion {@link #V4_WRITER_FORMAT_VERSION} (4) for a v4 leaf manifest, or + * {@link #LEGACY_WRITER_FORMAT_VERSION} (0) for a pre-v4 (v1, v2, or v3) leaf manifest + * carried over during a v3-to-v4 upgrade. Callers are responsible for resolving the value; + * this avoids adding a v4-only accessor to the public ManifestFile interface for a value with + * no production consumer at this layer. + * @param status entry status (typically ADDED for a newly written leaf, EXISTING for a + * carried-over reference) + * @param firstRowId the resolved first-row-id to write for this reference, or null for delete + * manifests. Callers are responsible for resolving the value (either carrying over {@link + * ManifestFile#firstRowId()} or assigning from a writer-side counter); the adapter does not + * decide between the two. + */ + static TrackedFile fromManifestFile( + ManifestFile manifest, int writerFormatVersion, EntryStatus status, Long firstRowId) { + Preconditions.checkArgument(manifest != null, "Invalid manifest file: null"); + Preconditions.checkArgument( Review Comment: the `writerFormatVersion` argument will be removed when it is added to the `ManifestFile` interface in later phases. so probably good to keep it here for now. https://github.com/apache/iceberg/pull/16936#discussion_r3461486313 ########## core/src/main/java/org/apache/iceberg/ContentEntryAdapters.java: ########## @@ -0,0 +1,400 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.iceberg; + +import java.nio.ByteBuffer; +import java.util.Collections; +import java.util.Locale; +import java.util.Map; +import java.util.WeakHashMap; +import org.apache.iceberg.relocated.com.google.common.base.Preconditions; +import org.apache.iceberg.relocated.com.google.common.collect.Maps; +import org.apache.iceberg.types.Type; +import org.apache.iceberg.types.Types; +import org.apache.iceberg.util.ContentFileUtil; + +/** + * Builds {@link TrackedFile} instances for v4 content_entry rows from legacy {@link ManifestEntry} + * and {@link ManifestFile} inputs. + */ +class ContentEntryAdapters { + /** + * writer_format_version for content_entry rows produced by a v4 writer. Matches the table format + * version (4). + */ + static final int V4_WRITER_FORMAT_VERSION = 4; + + /** + * writer_format_version for content_entry rows that reference a leaf manifest written by a pre-v4 + * writer (v1, v2, or v3). Used at the root manifest level when a v4 root carries over legacy leaf + * manifests during a v3-to-v4 upgrade. Matches a pre-v4 table format version (0 is the sentinel; + * v1/v2/v3 leaves are not re-encoded as content_entry, so the root just tags the reference as + * legacy). + */ + static final int LEGACY_WRITER_FORMAT_VERSION = 0; + + // Cache of primitive-field types keyed by table schema. The originalTypes map a Metrics + // instance carries is read-only inside MetricsUtil.fromMetrics, so a single per-schema map + // can be shared across every adapter call for a writer's lifetime instead of being rebuilt + // per row. WeakHashMap keys release when callers stop referencing the schema. + private static final Map<Schema, Map<Integer, Type>> PRIMITIVE_TYPES_BY_SCHEMA = + Collections.synchronizedMap(new WeakHashMap<>()); + + private ContentEntryAdapters() {} + + static TrackedFile fromDataFile( + ManifestEntry<DataFile> entry, Schema tableSchema, EntryStatus statusOverride) { + Preconditions.checkArgument(entry != null, "Invalid manifest entry: null"); + DataFile file = entry.file(); + Preconditions.checkArgument(file != null, "Invalid data file: null"); + Preconditions.checkArgument( + file.content() == FileContent.DATA, "Invalid content for data file: %s", file.content()); + return buildContentFileEntry( + file, + statusOverride, + snapshotIdOrZero(entry), + entry.dataSequenceNumber(), + entry.fileSequenceNumber(), + tableSchema); + } + + static TrackedFile fromDeleteFile( + ManifestEntry<DeleteFile> entry, Schema tableSchema, EntryStatus statusOverride) { + Preconditions.checkArgument(entry != null, "Invalid manifest entry: null"); + DeleteFile file = entry.file(); + Preconditions.checkArgument(file != null, "Invalid delete file: null"); + // v4 leaf delete manifests must only contain content_type=EQUALITY_DELETES (per spec PR + // #16025). Reject POSITION_DELETES with a hint that distinguishes the two legacy shapes by + // file format (the canonical DV check per ContentFileUtil.isDV — both shapes can carry a + // referencedDataFile, so that field is not a reliable distinguisher): + // - v3 delete vector (POSITION_DELETES stored as a Puffin blob) must be colocated on the + // data file's content_entry via TrackedFileBuilder.deletionVector(...) — see + // MergingSnapshotProducer's row-delta path. + // - v2 standalone position delete file (POSITION_DELETES stored in Parquet/Avro/ORC) has no + // v4 representation; it can only live in pre-v4 legacy manifests carried over via a + // writer_format_version=0 manifest reference. + if (file.content() == FileContent.POSITION_DELETES) { Review Comment: yeah, the purpose is to provide a more specific error msg. I can remove it if people think it is unnecessary -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: [email protected] For queries about this service, please contact Infrastructure at: [email protected] --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
