gaborkaszab commented on code in PR #16769:
URL: https://github.com/apache/iceberg/pull/16769#discussion_r3454657439
##########
core/src/main/java/org/apache/iceberg/DeletionVectorStruct.java:
##########
@@ -128,6 +129,26 @@ static Builder builder() {
return new Builder();
}
+ @Override
+ public boolean equals(Object other) {
Review Comment:
Yes, added some test coverage. Also, I think for equality it's enough to
compare location. Reduced the check here.
##########
core/src/main/java/org/apache/iceberg/TrackedFileBuilder.java:
##########
@@ -0,0 +1,369 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.iceberg;
+
+import java.nio.ByteBuffer;
+import java.util.List;
+import org.apache.iceberg.relocated.com.google.common.base.Preconditions;
+
+class TrackedFileBuilder {
+ private final long snapshotId;
+ private final FileContent contentType;
+
+ // Required fields
+ private Integer writerFormatVersion = null;
+ private String location = null;
+ private FileFormat fileFormat = null;
+ private Long recordCount = null;
+ private Long fileSizeInBytes = null;
+ private PartitionData partitionData = null;
+
+ // optional fields
+ private Integer specId = null;
+ private ContentStats contentStats = null;
+ private Integer sortOrderId = null;
+ private DeletionVector deletionVector = null;
+ private ManifestInfo manifestInfo = null;
+ private ByteBuffer keyMetadata = null;
+ private List<Long> splitOffsets = null;
+ private List<Integer> equalityIds = null;
+
+ // tracking-related fields
+ private Tracking sourceTracking = null;
+ private boolean dvUpdated = false;
+ private ByteBuffer deletedPositions = null;
+ private ByteBuffer replacedPositions = null;
+
+ /**
+ * Creates a builder for a newly added data file entry.
+ *
+ * @param newSnapshotId the snapshot ID in which the new tracked file will
be committed
+ */
+ static TrackedFileBuilder data(long newSnapshotId) {
+ return new TrackedFileBuilder(FileContent.DATA, newSnapshotId);
+ }
+
+ /**
+ * Creates a builder for a newly added equality delete file entry.
+ *
+ * @param newSnapshotId the snapshot ID in which the new tracked file will
be committed
+ */
+ static TrackedFileBuilder equalityDelete(long newSnapshotId) {
+ return new TrackedFileBuilder(FileContent.EQUALITY_DELETES, newSnapshotId);
+ }
+
+ /**
+ * Creates a builder for a newly added data manifest entry.
+ *
+ * @param newSnapshotId the snapshot ID in which the new tracked file will
be committed
+ */
+ static TrackedFileBuilder dataManifest(long newSnapshotId) {
+ return new TrackedFileBuilder(FileContent.DATA_MANIFEST, newSnapshotId);
+ }
+
+ /**
+ * Creates a builder for a newly added delete manifest entry.
+ *
+ * @param newSnapshotId the snapshot ID in which the new tracked file will
be committed
+ */
+ static TrackedFileBuilder deleteManifest(long newSnapshotId) {
+ return new TrackedFileBuilder(FileContent.DELETE_MANIFEST, newSnapshotId);
+ }
+
+ /**
+ * Creates a builder for a tracked file derived from {@code newSource}.
+ *
+ * @param source source tracked file to copy fields from
+ * @param newSnapshotId the snapshot ID in which the new tracked file will
be committed
+ */
+ static TrackedFileBuilder from(TrackedFile source, long newSnapshotId) {
+ Preconditions.checkArgument(source != null, "Invalid source: null");
+ return new TrackedFileBuilder(source, newSnapshotId);
+ }
+
+ /**
+ * Returns a DELETED tracked file derived from {@code source}.
+ *
+ * @param source source tracked file
+ * @param newSnapshotId the snapshot ID in which the new tracked file will
be committed
+ */
+ static TrackedFile deleted(TrackedFile source, long newSnapshotId) {
+ Preconditions.checkArgument(source != null, "Invalid source: null");
+ return terminal(source, TrackingBuilder.deleted(source.tracking(),
newSnapshotId));
+ }
+
+ /**
+ * Returns a REPLACED tracked file derived from {@code source}.
+ *
+ * <p>Manifest entries cannot transition to REPLACED.
+ *
+ * @param source source tracked file
+ * @param newSnapshotId the snapshot ID in which the new tracked file will
be committed
+ */
+ static TrackedFile replaced(TrackedFile source, long newSnapshotId) {
+ Preconditions.checkArgument(source != null, "Invalid source: null");
+ Preconditions.checkArgument(
+ !isLeafManifest(source.contentType()),
+ "Manifest entries cannot transition to REPLACED, but entry type is:
%s",
+ source.contentType());
+ return terminal(source, TrackingBuilder.replaced(source.tracking(),
newSnapshotId));
+ }
+
+ private static TrackedFile terminal(TrackedFile source, Tracking tracking) {
+ return new TrackedFileStruct(
+ tracking,
+ source.contentType(),
+ source.writerFormatVersion(),
+ source.location(),
+ source.fileFormat(),
+ (PartitionData) source.partition(),
+ source.recordCount(),
+ source.fileSizeInBytes(),
+ source.specId(),
+ source.contentStats(),
+ source.sortOrderId(),
+ source.deletionVector(),
+ source.manifestInfo(),
+ source.keyMetadata(),
+ source.splitOffsets(),
+ source.equalityIds());
+ }
+
+ private TrackedFileBuilder(FileContent contentType, long snapshotId) {
+ this.contentType = contentType;
+ this.snapshotId = snapshotId;
+ }
+
+ private TrackedFileBuilder(TrackedFile source, long snapshotId) {
+ this.contentType = source.contentType();
+ this.snapshotId = snapshotId;
+ this.writerFormatVersion = source.writerFormatVersion();
+ this.location = source.location();
+ this.fileFormat = source.fileFormat();
+ this.recordCount = source.recordCount();
+ this.fileSizeInBytes = source.fileSizeInBytes();
+ this.partitionData = (PartitionData) source.partition();
+ this.specId = source.specId();
+ this.contentStats = source.contentStats();
+ this.sortOrderId = source.sortOrderId();
+ this.deletionVector = source.deletionVector();
+ this.manifestInfo = source.manifestInfo();
+ this.keyMetadata = source.keyMetadata();
+ this.splitOffsets = source.splitOffsets();
+ this.equalityIds = source.equalityIds();
+ this.sourceTracking = source.tracking();
+ }
+
+ TrackedFileBuilder writerFormatVersion(int newWriterFormatVersion) {
+ Preconditions.checkArgument(
+ newWriterFormatVersion >= 0,
+ "Invalid writer format version: %s (must be >= 0)",
+ newWriterFormatVersion);
+ this.writerFormatVersion = newWriterFormatVersion;
+ return this;
+ }
+
+ TrackedFileBuilder location(String newLocation) {
+ Preconditions.checkArgument(newLocation != null, "Invalid location: null");
+ this.location = newLocation;
+ return this;
+ }
+
+ TrackedFileBuilder fileFormat(FileFormat newFileFormat) {
+ Preconditions.checkArgument(newFileFormat != null, "Invalid file format:
null");
+ this.fileFormat = newFileFormat;
+ return this;
+ }
+
+ TrackedFileBuilder recordCount(long newRecordCount) {
+ Preconditions.checkArgument(
+ newRecordCount >= 0, "Invalid record count: %s (must be >= 0)",
newRecordCount);
+ this.recordCount = newRecordCount;
+ return this;
+ }
+
+ TrackedFileBuilder fileSizeInBytes(long newFileSizeInBytes) {
+ Preconditions.checkArgument(
+ newFileSizeInBytes >= 0,
+ "Invalid file size in bytes: %s (must be >= 0)",
+ newFileSizeInBytes);
+ this.fileSizeInBytes = newFileSizeInBytes;
+ return this;
+ }
+
+ TrackedFileBuilder specId(int newSpecId) {
+ Preconditions.checkArgument(newSpecId >= 0, "Invalid spec ID: %s (must be
>= 0)", newSpecId);
+ this.specId = newSpecId;
+ return this;
+ }
+
+ TrackedFileBuilder partition(PartitionData newPartitionData) {
+ Preconditions.checkArgument(newPartitionData != null, "Invalid partition:
null");
+ this.partitionData = newPartitionData;
+ return this;
+ }
+
+ TrackedFileBuilder contentStats(ContentStats newContentStats) {
+ Preconditions.checkArgument(newContentStats != null, "Invalid content
stats: null");
+ this.contentStats = newContentStats;
+ return this;
+ }
+
+ TrackedFileBuilder sortOrderId(int newSortOrderId) {
+ Preconditions.checkArgument(
+ !isLeafManifest(contentType),
+ "Sort order ID can not be added to manifest entries, but entry type
is: %s",
Review Comment:
done
##########
core/src/main/java/org/apache/iceberg/TrackedFileBuilder.java:
##########
@@ -0,0 +1,369 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.iceberg;
+
+import java.nio.ByteBuffer;
+import java.util.List;
+import org.apache.iceberg.relocated.com.google.common.base.Preconditions;
+
+class TrackedFileBuilder {
+ private final long snapshotId;
+ private final FileContent contentType;
+
+ // Required fields
+ private Integer writerFormatVersion = null;
+ private String location = null;
+ private FileFormat fileFormat = null;
+ private Long recordCount = null;
+ private Long fileSizeInBytes = null;
+ private PartitionData partitionData = null;
+
+ // optional fields
+ private Integer specId = null;
+ private ContentStats contentStats = null;
+ private Integer sortOrderId = null;
+ private DeletionVector deletionVector = null;
+ private ManifestInfo manifestInfo = null;
+ private ByteBuffer keyMetadata = null;
+ private List<Long> splitOffsets = null;
+ private List<Integer> equalityIds = null;
+
+ // tracking-related fields
+ private Tracking sourceTracking = null;
+ private boolean dvUpdated = false;
+ private ByteBuffer deletedPositions = null;
+ private ByteBuffer replacedPositions = null;
+
+ /**
+ * Creates a builder for a newly added data file entry.
+ *
+ * @param newSnapshotId the snapshot ID in which the new tracked file will
be committed
+ */
+ static TrackedFileBuilder data(long newSnapshotId) {
+ return new TrackedFileBuilder(FileContent.DATA, newSnapshotId);
+ }
+
+ /**
+ * Creates a builder for a newly added equality delete file entry.
+ *
+ * @param newSnapshotId the snapshot ID in which the new tracked file will
be committed
+ */
+ static TrackedFileBuilder equalityDelete(long newSnapshotId) {
+ return new TrackedFileBuilder(FileContent.EQUALITY_DELETES, newSnapshotId);
+ }
+
+ /**
+ * Creates a builder for a newly added data manifest entry.
+ *
+ * @param newSnapshotId the snapshot ID in which the new tracked file will
be committed
+ */
+ static TrackedFileBuilder dataManifest(long newSnapshotId) {
+ return new TrackedFileBuilder(FileContent.DATA_MANIFEST, newSnapshotId);
+ }
+
+ /**
+ * Creates a builder for a newly added delete manifest entry.
+ *
+ * @param newSnapshotId the snapshot ID in which the new tracked file will
be committed
+ */
+ static TrackedFileBuilder deleteManifest(long newSnapshotId) {
+ return new TrackedFileBuilder(FileContent.DELETE_MANIFEST, newSnapshotId);
+ }
+
+ /**
+ * Creates a builder for a tracked file derived from {@code newSource}.
+ *
+ * @param source source tracked file to copy fields from
+ * @param newSnapshotId the snapshot ID in which the new tracked file will
be committed
+ */
+ static TrackedFileBuilder from(TrackedFile source, long newSnapshotId) {
+ Preconditions.checkArgument(source != null, "Invalid source: null");
+ return new TrackedFileBuilder(source, newSnapshotId);
+ }
+
+ /**
+ * Returns a DELETED tracked file derived from {@code source}.
+ *
+ * @param source source tracked file
+ * @param newSnapshotId the snapshot ID in which the new tracked file will
be committed
+ */
+ static TrackedFile deleted(TrackedFile source, long newSnapshotId) {
+ Preconditions.checkArgument(source != null, "Invalid source: null");
+ return terminal(source, TrackingBuilder.deleted(source.tracking(),
newSnapshotId));
+ }
+
+ /**
+ * Returns a REPLACED tracked file derived from {@code source}.
+ *
+ * <p>Manifest entries cannot transition to REPLACED.
+ *
+ * @param source source tracked file
+ * @param newSnapshotId the snapshot ID in which the new tracked file will
be committed
+ */
+ static TrackedFile replaced(TrackedFile source, long newSnapshotId) {
+ Preconditions.checkArgument(source != null, "Invalid source: null");
+ Preconditions.checkArgument(
+ !isLeafManifest(source.contentType()),
+ "Manifest entries cannot transition to REPLACED, but entry type is:
%s",
+ source.contentType());
+ return terminal(source, TrackingBuilder.replaced(source.tracking(),
newSnapshotId));
+ }
+
+ private static TrackedFile terminal(TrackedFile source, Tracking tracking) {
+ return new TrackedFileStruct(
+ tracking,
+ source.contentType(),
+ source.writerFormatVersion(),
+ source.location(),
+ source.fileFormat(),
+ (PartitionData) source.partition(),
+ source.recordCount(),
+ source.fileSizeInBytes(),
+ source.specId(),
+ source.contentStats(),
+ source.sortOrderId(),
+ source.deletionVector(),
+ source.manifestInfo(),
+ source.keyMetadata(),
+ source.splitOffsets(),
+ source.equalityIds());
+ }
+
+ private TrackedFileBuilder(FileContent contentType, long snapshotId) {
+ this.contentType = contentType;
+ this.snapshotId = snapshotId;
+ }
+
+ private TrackedFileBuilder(TrackedFile source, long snapshotId) {
+ this.contentType = source.contentType();
+ this.snapshotId = snapshotId;
+ this.writerFormatVersion = source.writerFormatVersion();
+ this.location = source.location();
+ this.fileFormat = source.fileFormat();
+ this.recordCount = source.recordCount();
+ this.fileSizeInBytes = source.fileSizeInBytes();
+ this.partitionData = (PartitionData) source.partition();
+ this.specId = source.specId();
+ this.contentStats = source.contentStats();
+ this.sortOrderId = source.sortOrderId();
+ this.deletionVector = source.deletionVector();
+ this.manifestInfo = source.manifestInfo();
+ this.keyMetadata = source.keyMetadata();
+ this.splitOffsets = source.splitOffsets();
+ this.equalityIds = source.equalityIds();
+ this.sourceTracking = source.tracking();
+ }
+
+ TrackedFileBuilder writerFormatVersion(int newWriterFormatVersion) {
+ Preconditions.checkArgument(
+ newWriterFormatVersion >= 0,
+ "Invalid writer format version: %s (must be >= 0)",
+ newWriterFormatVersion);
+ this.writerFormatVersion = newWriterFormatVersion;
+ return this;
+ }
+
+ TrackedFileBuilder location(String newLocation) {
+ Preconditions.checkArgument(newLocation != null, "Invalid location: null");
+ this.location = newLocation;
+ return this;
+ }
+
+ TrackedFileBuilder fileFormat(FileFormat newFileFormat) {
+ Preconditions.checkArgument(newFileFormat != null, "Invalid file format:
null");
+ this.fileFormat = newFileFormat;
+ return this;
+ }
+
+ TrackedFileBuilder recordCount(long newRecordCount) {
+ Preconditions.checkArgument(
+ newRecordCount >= 0, "Invalid record count: %s (must be >= 0)",
newRecordCount);
+ this.recordCount = newRecordCount;
+ return this;
+ }
+
+ TrackedFileBuilder fileSizeInBytes(long newFileSizeInBytes) {
+ Preconditions.checkArgument(
+ newFileSizeInBytes >= 0,
+ "Invalid file size in bytes: %s (must be >= 0)",
+ newFileSizeInBytes);
+ this.fileSizeInBytes = newFileSizeInBytes;
+ return this;
+ }
+
+ TrackedFileBuilder specId(int newSpecId) {
+ Preconditions.checkArgument(newSpecId >= 0, "Invalid spec ID: %s (must be
>= 0)", newSpecId);
+ this.specId = newSpecId;
+ return this;
+ }
+
+ TrackedFileBuilder partition(PartitionData newPartitionData) {
+ Preconditions.checkArgument(newPartitionData != null, "Invalid partition:
null");
+ this.partitionData = newPartitionData;
+ return this;
+ }
+
+ TrackedFileBuilder contentStats(ContentStats newContentStats) {
+ Preconditions.checkArgument(newContentStats != null, "Invalid content
stats: null");
+ this.contentStats = newContentStats;
+ return this;
+ }
+
+ TrackedFileBuilder sortOrderId(int newSortOrderId) {
+ Preconditions.checkArgument(
+ !isLeafManifest(contentType),
+ "Sort order ID can not be added to manifest entries, but entry type
is: %s",
+ contentType);
+ Preconditions.checkArgument(
+ newSortOrderId >= 0, "Invalid sort order ID: %s (must be >= 0)",
newSortOrderId);
+ this.sortOrderId = newSortOrderId;
+ return this;
+ }
+
+ /**
+ * This method is for signalling that a new deletion vector is added. It is
forbidden to add the
Review Comment:
The suggestion for the doc comment came from @stevenzwu
[here](https://github.com/apache/iceberg/pull/16769/changes#r3422893663). I
think this function is easy to read and interpret as is now, we can drop the
comment. Let me know if you think otherwise @stevenzwu !
##########
core/src/main/java/org/apache/iceberg/TrackedFileStruct.java:
##########
@@ -123,6 +130,15 @@ public PartitionData copy() {
if (partition != null) {
this.partitionData = partition;
}
+
+ this.specId = specId;
+ this.contentStats = contentStats;
+ this.sortOrderId = sortOrderId;
+ this.deletionVector = deletionVector;
+ this.manifestInfo = manifestInfo;
+ this.keyMetadata = keyMetadata != null ?
ByteBuffers.toByteArray(keyMetadata) : null;
Review Comment:
Thanks, fixed!
##########
core/src/main/java/org/apache/iceberg/TrackedFileBuilder.java:
##########
@@ -0,0 +1,369 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.iceberg;
+
+import java.nio.ByteBuffer;
+import java.util.List;
+import org.apache.iceberg.relocated.com.google.common.base.Preconditions;
+
+class TrackedFileBuilder {
+ private final long snapshotId;
+ private final FileContent contentType;
+
+ // Required fields
+ private Integer writerFormatVersion = null;
+ private String location = null;
+ private FileFormat fileFormat = null;
+ private Long recordCount = null;
+ private Long fileSizeInBytes = null;
+ private PartitionData partitionData = null;
+
+ // optional fields
+ private Integer specId = null;
+ private ContentStats contentStats = null;
+ private Integer sortOrderId = null;
+ private DeletionVector deletionVector = null;
+ private ManifestInfo manifestInfo = null;
+ private ByteBuffer keyMetadata = null;
+ private List<Long> splitOffsets = null;
+ private List<Integer> equalityIds = null;
+
+ // tracking-related fields
+ private Tracking sourceTracking = null;
+ private boolean dvUpdated = false;
+ private ByteBuffer deletedPositions = null;
+ private ByteBuffer replacedPositions = null;
+
+ /**
+ * Creates a builder for a newly added data file entry.
+ *
+ * @param newSnapshotId the snapshot ID in which the new tracked file will
be committed
+ */
+ static TrackedFileBuilder data(long newSnapshotId) {
+ return new TrackedFileBuilder(FileContent.DATA, newSnapshotId);
+ }
+
+ /**
+ * Creates a builder for a newly added equality delete file entry.
+ *
+ * @param newSnapshotId the snapshot ID in which the new tracked file will
be committed
+ */
+ static TrackedFileBuilder equalityDelete(long newSnapshotId) {
+ return new TrackedFileBuilder(FileContent.EQUALITY_DELETES, newSnapshotId);
+ }
+
+ /**
+ * Creates a builder for a newly added data manifest entry.
+ *
+ * @param newSnapshotId the snapshot ID in which the new tracked file will
be committed
+ */
+ static TrackedFileBuilder dataManifest(long newSnapshotId) {
+ return new TrackedFileBuilder(FileContent.DATA_MANIFEST, newSnapshotId);
+ }
+
+ /**
+ * Creates a builder for a newly added delete manifest entry.
+ *
+ * @param newSnapshotId the snapshot ID in which the new tracked file will
be committed
+ */
+ static TrackedFileBuilder deleteManifest(long newSnapshotId) {
+ return new TrackedFileBuilder(FileContent.DELETE_MANIFEST, newSnapshotId);
+ }
+
+ /**
+ * Creates a builder for a tracked file derived from {@code newSource}.
Review Comment:
Thanks for spotting. Fixed
##########
core/src/main/java/org/apache/iceberg/TrackedFileBuilder.java:
##########
@@ -0,0 +1,369 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.iceberg;
+
+import java.nio.ByteBuffer;
+import java.util.List;
+import org.apache.iceberg.relocated.com.google.common.base.Preconditions;
+
+class TrackedFileBuilder {
+ private final long snapshotId;
+ private final FileContent contentType;
+
+ // Required fields
+ private Integer writerFormatVersion = null;
+ private String location = null;
+ private FileFormat fileFormat = null;
+ private Long recordCount = null;
+ private Long fileSizeInBytes = null;
+ private PartitionData partitionData = null;
+
+ // optional fields
+ private Integer specId = null;
+ private ContentStats contentStats = null;
+ private Integer sortOrderId = null;
+ private DeletionVector deletionVector = null;
+ private ManifestInfo manifestInfo = null;
+ private ByteBuffer keyMetadata = null;
+ private List<Long> splitOffsets = null;
+ private List<Integer> equalityIds = null;
+
+ // tracking-related fields
+ private Tracking sourceTracking = null;
+ private boolean dvUpdated = false;
+ private ByteBuffer deletedPositions = null;
+ private ByteBuffer replacedPositions = null;
+
+ /**
+ * Creates a builder for a newly added data file entry.
+ *
+ * @param newSnapshotId the snapshot ID in which the new tracked file will
be committed
+ */
+ static TrackedFileBuilder data(long newSnapshotId) {
+ return new TrackedFileBuilder(FileContent.DATA, newSnapshotId);
+ }
+
+ /**
+ * Creates a builder for a newly added equality delete file entry.
+ *
+ * @param newSnapshotId the snapshot ID in which the new tracked file will
be committed
+ */
+ static TrackedFileBuilder equalityDelete(long newSnapshotId) {
+ return new TrackedFileBuilder(FileContent.EQUALITY_DELETES, newSnapshotId);
+ }
+
+ /**
+ * Creates a builder for a newly added data manifest entry.
+ *
+ * @param newSnapshotId the snapshot ID in which the new tracked file will
be committed
+ */
+ static TrackedFileBuilder dataManifest(long newSnapshotId) {
+ return new TrackedFileBuilder(FileContent.DATA_MANIFEST, newSnapshotId);
+ }
+
+ /**
+ * Creates a builder for a newly added delete manifest entry.
+ *
+ * @param newSnapshotId the snapshot ID in which the new tracked file will
be committed
+ */
+ static TrackedFileBuilder deleteManifest(long newSnapshotId) {
+ return new TrackedFileBuilder(FileContent.DELETE_MANIFEST, newSnapshotId);
+ }
+
+ /**
+ * Creates a builder for a tracked file derived from {@code newSource}.
+ *
+ * @param source source tracked file to copy fields from
+ * @param newSnapshotId the snapshot ID in which the new tracked file will
be committed
+ */
+ static TrackedFileBuilder from(TrackedFile source, long newSnapshotId) {
+ Preconditions.checkArgument(source != null, "Invalid source: null");
+ return new TrackedFileBuilder(source, newSnapshotId);
+ }
+
+ /**
+ * Returns a DELETED tracked file derived from {@code source}.
+ *
+ * @param source source tracked file
+ * @param newSnapshotId the snapshot ID in which the new tracked file will
be committed
+ */
+ static TrackedFile deleted(TrackedFile source, long newSnapshotId) {
+ Preconditions.checkArgument(source != null, "Invalid source: null");
+ return terminal(source, TrackingBuilder.deleted(source.tracking(),
newSnapshotId));
+ }
+
+ /**
+ * Returns a REPLACED tracked file derived from {@code source}.
+ *
+ * <p>Manifest entries cannot transition to REPLACED.
+ *
+ * @param source source tracked file
+ * @param newSnapshotId the snapshot ID in which the new tracked file will
be committed
+ */
+ static TrackedFile replaced(TrackedFile source, long newSnapshotId) {
+ Preconditions.checkArgument(source != null, "Invalid source: null");
+ Preconditions.checkArgument(
+ !isLeafManifest(source.contentType()),
+ "Manifest entries cannot transition to REPLACED, but entry type is:
%s",
+ source.contentType());
+ return terminal(source, TrackingBuilder.replaced(source.tracking(),
newSnapshotId));
+ }
+
+ private static TrackedFile terminal(TrackedFile source, Tracking tracking) {
+ return new TrackedFileStruct(
+ tracking,
+ source.contentType(),
+ source.writerFormatVersion(),
+ source.location(),
+ source.fileFormat(),
+ (PartitionData) source.partition(),
+ source.recordCount(),
+ source.fileSizeInBytes(),
+ source.specId(),
+ source.contentStats(),
+ source.sortOrderId(),
+ source.deletionVector(),
+ source.manifestInfo(),
+ source.keyMetadata(),
+ source.splitOffsets(),
+ source.equalityIds());
+ }
+
+ private TrackedFileBuilder(FileContent contentType, long snapshotId) {
+ this.contentType = contentType;
+ this.snapshotId = snapshotId;
+ }
+
+ private TrackedFileBuilder(TrackedFile source, long snapshotId) {
+ this.contentType = source.contentType();
+ this.snapshotId = snapshotId;
+ this.writerFormatVersion = source.writerFormatVersion();
+ this.location = source.location();
+ this.fileFormat = source.fileFormat();
+ this.recordCount = source.recordCount();
+ this.fileSizeInBytes = source.fileSizeInBytes();
+ this.partitionData = (PartitionData) source.partition();
+ this.specId = source.specId();
+ this.contentStats = source.contentStats();
+ this.sortOrderId = source.sortOrderId();
+ this.deletionVector = source.deletionVector();
+ this.manifestInfo = source.manifestInfo();
+ this.keyMetadata = source.keyMetadata();
+ this.splitOffsets = source.splitOffsets();
+ this.equalityIds = source.equalityIds();
+ this.sourceTracking = source.tracking();
+ }
+
+ TrackedFileBuilder writerFormatVersion(int newWriterFormatVersion) {
+ Preconditions.checkArgument(
+ newWriterFormatVersion >= 0,
+ "Invalid writer format version: %s (must be >= 0)",
+ newWriterFormatVersion);
+ this.writerFormatVersion = newWriterFormatVersion;
+ return this;
+ }
+
+ TrackedFileBuilder location(String newLocation) {
+ Preconditions.checkArgument(newLocation != null, "Invalid location: null");
+ this.location = newLocation;
+ return this;
+ }
+
+ TrackedFileBuilder fileFormat(FileFormat newFileFormat) {
+ Preconditions.checkArgument(newFileFormat != null, "Invalid file format:
null");
+ this.fileFormat = newFileFormat;
+ return this;
+ }
+
+ TrackedFileBuilder recordCount(long newRecordCount) {
+ Preconditions.checkArgument(
+ newRecordCount >= 0, "Invalid record count: %s (must be >= 0)",
newRecordCount);
+ this.recordCount = newRecordCount;
+ return this;
+ }
+
+ TrackedFileBuilder fileSizeInBytes(long newFileSizeInBytes) {
+ Preconditions.checkArgument(
+ newFileSizeInBytes >= 0,
+ "Invalid file size in bytes: %s (must be >= 0)",
+ newFileSizeInBytes);
+ this.fileSizeInBytes = newFileSizeInBytes;
+ return this;
+ }
+
+ TrackedFileBuilder specId(int newSpecId) {
+ Preconditions.checkArgument(newSpecId >= 0, "Invalid spec ID: %s (must be
>= 0)", newSpecId);
+ this.specId = newSpecId;
+ return this;
+ }
+
+ TrackedFileBuilder partition(PartitionData newPartitionData) {
+ Preconditions.checkArgument(newPartitionData != null, "Invalid partition:
null");
+ this.partitionData = newPartitionData;
+ return this;
+ }
+
+ TrackedFileBuilder contentStats(ContentStats newContentStats) {
+ Preconditions.checkArgument(newContentStats != null, "Invalid content
stats: null");
+ this.contentStats = newContentStats;
+ return this;
+ }
+
+ TrackedFileBuilder sortOrderId(int newSortOrderId) {
+ Preconditions.checkArgument(
+ !isLeafManifest(contentType),
+ "Sort order ID can not be added to manifest entries, but entry type
is: %s",
+ contentType);
+ Preconditions.checkArgument(
+ newSortOrderId >= 0, "Invalid sort order ID: %s (must be >= 0)",
newSortOrderId);
+ this.sortOrderId = newSortOrderId;
+ return this;
+ }
+
+ /**
+ * This method is for signalling that a new deletion vector is added. It is
forbidden to add the
+ * same deletion vector that already exists for this {@link TrackedFile}.
+ */
+ TrackedFileBuilder deletionVector(DeletionVector newDeletionVector) {
+ Preconditions.checkArgument(newDeletionVector != null, "Invalid deletion
vector: null");
+ Preconditions.checkArgument(
+ contentType == FileContent.DATA,
+ "Deletion vector can only be added to DATA entries, but entry type is:
%s",
+ contentType);
+ Preconditions.checkArgument(
+ this.deletionVector == null ||
!this.deletionVector.equals(newDeletionVector),
+ "The same deletion vector already added");
+ this.deletionVector = newDeletionVector;
+ this.dvUpdated = true;
+ return this;
+ }
+
+ TrackedFileBuilder manifestInfo(ManifestInfo newManifestInfo) {
+ Preconditions.checkArgument(newManifestInfo != null, "Invalid manifest
info: null");
+ Preconditions.checkArgument(
+ isLeafManifest(contentType),
+ "Manifest info can only be added to manifests, but entry type is: %s",
+ contentType);
+ this.manifestInfo = newManifestInfo;
+ return this;
+ }
+
+ TrackedFileBuilder keyMetadata(ByteBuffer newKeyMetadata) {
+ Preconditions.checkArgument(newKeyMetadata != null, "Invalid key metadata:
null");
+ this.keyMetadata = newKeyMetadata;
+ return this;
+ }
+
+ TrackedFileBuilder splitOffsets(List<Long> newSplitOffsets) {
+ Preconditions.checkArgument(newSplitOffsets != null, "Invalid split
offsets: null");
+ Preconditions.checkArgument(
+ !isLeafManifest(contentType),
+ "Split offsets can not be added to manifest entries, but entry type
is: %s",
+ contentType);
+ this.splitOffsets = newSplitOffsets;
+ return this;
+ }
+
+ TrackedFileBuilder equalityIds(List<Integer> newEqualityIds) {
+ Preconditions.checkArgument(newEqualityIds != null, "Invalid equality IDs:
null");
+ Preconditions.checkArgument(
+ contentType == FileContent.EQUALITY_DELETES,
+ "Equality IDs can only be added to EQUALITY_DELETES entries, but entry
type is: %s",
+ contentType);
+ this.equalityIds = newEqualityIds;
+ return this;
+ }
+
+ TrackedFileBuilder deletedPositions(ByteBuffer newDeletedPositions) {
+ Preconditions.checkArgument(newDeletedPositions != null, "Invalid deleted
positions: null");
+ Preconditions.checkArgument(
+ isLeafManifest(contentType),
+ "Deleted positions can only be added to manifest entries, but entry
type is: %s",
+ contentType);
+ this.deletedPositions = newDeletedPositions;
+ return this;
+ }
+
+ TrackedFileBuilder replacedPositions(ByteBuffer newReplacedPositions) {
+ Preconditions.checkArgument(newReplacedPositions != null, "Invalid
replaced positions: null");
+ Preconditions.checkArgument(
+ isLeafManifest(contentType),
+ "Replaced positions can only be added to manifest entries, but entry
type is: %s",
+ contentType);
+ this.replacedPositions = newReplacedPositions;
+ return this;
+ }
+
+ private static boolean isLeafManifest(FileContent contentType) {
+ return contentType == FileContent.DATA_MANIFEST || contentType ==
FileContent.DELETE_MANIFEST;
+ }
+
+ TrackedFile build() {
+ Preconditions.checkArgument(
+ writerFormatVersion != null, "Missing required field: writer format
version");
+ Preconditions.checkArgument(location != null, "Missing required field:
location");
+ Preconditions.checkArgument(fileFormat != null, "Missing required field:
file format");
+ Preconditions.checkArgument(recordCount != null, "Missing required field:
record count");
+ Preconditions.checkArgument(
+ fileSizeInBytes != null, "Missing required field: file size in bytes");
+ Preconditions.checkArgument(partitionData != null, "Missing required
field: partition data");
+ Preconditions.checkArgument(
+ (contentType != FileContent.DATA_MANIFEST && contentType !=
FileContent.DELETE_MANIFEST)
Review Comment:
Indeed, thanks for spotting! Fixed
##########
core/src/main/java/org/apache/iceberg/TrackedFileBuilder.java:
##########
@@ -0,0 +1,369 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.iceberg;
+
+import java.nio.ByteBuffer;
+import java.util.List;
+import org.apache.iceberg.relocated.com.google.common.base.Preconditions;
+
+class TrackedFileBuilder {
+ private final long snapshotId;
+ private final FileContent contentType;
+
+ // Required fields
+ private Integer writerFormatVersion = null;
+ private String location = null;
+ private FileFormat fileFormat = null;
+ private Long recordCount = null;
+ private Long fileSizeInBytes = null;
+ private PartitionData partitionData = null;
+
+ // optional fields
+ private Integer specId = null;
+ private ContentStats contentStats = null;
+ private Integer sortOrderId = null;
+ private DeletionVector deletionVector = null;
+ private ManifestInfo manifestInfo = null;
+ private ByteBuffer keyMetadata = null;
+ private List<Long> splitOffsets = null;
+ private List<Integer> equalityIds = null;
+
+ // tracking-related fields
+ private Tracking sourceTracking = null;
+ private boolean dvUpdated = false;
+ private ByteBuffer deletedPositions = null;
+ private ByteBuffer replacedPositions = null;
+
+ /**
+ * Creates a builder for a newly added data file entry.
+ *
+ * @param newSnapshotId the snapshot ID in which the new tracked file will
be committed
+ */
+ static TrackedFileBuilder data(long newSnapshotId) {
+ return new TrackedFileBuilder(FileContent.DATA, newSnapshotId);
+ }
+
+ /**
+ * Creates a builder for a newly added equality delete file entry.
+ *
+ * @param newSnapshotId the snapshot ID in which the new tracked file will
be committed
+ */
+ static TrackedFileBuilder equalityDelete(long newSnapshotId) {
+ return new TrackedFileBuilder(FileContent.EQUALITY_DELETES, newSnapshotId);
+ }
+
+ /**
+ * Creates a builder for a newly added data manifest entry.
+ *
+ * @param newSnapshotId the snapshot ID in which the new tracked file will
be committed
+ */
+ static TrackedFileBuilder dataManifest(long newSnapshotId) {
+ return new TrackedFileBuilder(FileContent.DATA_MANIFEST, newSnapshotId);
+ }
+
+ /**
+ * Creates a builder for a newly added delete manifest entry.
+ *
+ * @param newSnapshotId the snapshot ID in which the new tracked file will
be committed
+ */
+ static TrackedFileBuilder deleteManifest(long newSnapshotId) {
+ return new TrackedFileBuilder(FileContent.DELETE_MANIFEST, newSnapshotId);
+ }
+
+ /**
+ * Creates a builder for a tracked file derived from {@code newSource}.
+ *
+ * @param source source tracked file to copy fields from
+ * @param newSnapshotId the snapshot ID in which the new tracked file will
be committed
+ */
+ static TrackedFileBuilder from(TrackedFile source, long newSnapshotId) {
+ Preconditions.checkArgument(source != null, "Invalid source: null");
+ return new TrackedFileBuilder(source, newSnapshotId);
+ }
+
+ /**
+ * Returns a DELETED tracked file derived from {@code source}.
+ *
+ * @param source source tracked file
+ * @param newSnapshotId the snapshot ID in which the new tracked file will
be committed
+ */
+ static TrackedFile deleted(TrackedFile source, long newSnapshotId) {
+ Preconditions.checkArgument(source != null, "Invalid source: null");
+ return terminal(source, TrackingBuilder.deleted(source.tracking(),
newSnapshotId));
+ }
+
+ /**
+ * Returns a REPLACED tracked file derived from {@code source}.
+ *
+ * <p>Manifest entries cannot transition to REPLACED.
+ *
+ * @param source source tracked file
+ * @param newSnapshotId the snapshot ID in which the new tracked file will
be committed
+ */
+ static TrackedFile replaced(TrackedFile source, long newSnapshotId) {
+ Preconditions.checkArgument(source != null, "Invalid source: null");
+ Preconditions.checkArgument(
+ !isLeafManifest(source.contentType()),
+ "Manifest entries cannot transition to REPLACED, but entry type is:
%s",
+ source.contentType());
+ return terminal(source, TrackingBuilder.replaced(source.tracking(),
newSnapshotId));
+ }
+
+ private static TrackedFile terminal(TrackedFile source, Tracking tracking) {
+ return new TrackedFileStruct(
+ tracking,
+ source.contentType(),
+ source.writerFormatVersion(),
+ source.location(),
+ source.fileFormat(),
+ (PartitionData) source.partition(),
+ source.recordCount(),
+ source.fileSizeInBytes(),
+ source.specId(),
+ source.contentStats(),
+ source.sortOrderId(),
+ source.deletionVector(),
+ source.manifestInfo(),
+ source.keyMetadata(),
+ source.splitOffsets(),
+ source.equalityIds());
+ }
+
+ private TrackedFileBuilder(FileContent contentType, long snapshotId) {
+ this.contentType = contentType;
+ this.snapshotId = snapshotId;
+ }
+
+ private TrackedFileBuilder(TrackedFile source, long snapshotId) {
+ this.contentType = source.contentType();
+ this.snapshotId = snapshotId;
+ this.writerFormatVersion = source.writerFormatVersion();
+ this.location = source.location();
+ this.fileFormat = source.fileFormat();
+ this.recordCount = source.recordCount();
+ this.fileSizeInBytes = source.fileSizeInBytes();
+ this.partitionData = (PartitionData) source.partition();
+ this.specId = source.specId();
+ this.contentStats = source.contentStats();
+ this.sortOrderId = source.sortOrderId();
+ this.deletionVector = source.deletionVector();
+ this.manifestInfo = source.manifestInfo();
+ this.keyMetadata = source.keyMetadata();
+ this.splitOffsets = source.splitOffsets();
+ this.equalityIds = source.equalityIds();
+ this.sourceTracking = source.tracking();
+ }
+
+ TrackedFileBuilder writerFormatVersion(int newWriterFormatVersion) {
+ Preconditions.checkArgument(
+ newWriterFormatVersion >= 0,
+ "Invalid writer format version: %s (must be >= 0)",
+ newWriterFormatVersion);
+ this.writerFormatVersion = newWriterFormatVersion;
+ return this;
+ }
+
+ TrackedFileBuilder location(String newLocation) {
+ Preconditions.checkArgument(newLocation != null, "Invalid location: null");
+ this.location = newLocation;
+ return this;
+ }
+
+ TrackedFileBuilder fileFormat(FileFormat newFileFormat) {
+ Preconditions.checkArgument(newFileFormat != null, "Invalid file format:
null");
+ this.fileFormat = newFileFormat;
+ return this;
+ }
+
+ TrackedFileBuilder recordCount(long newRecordCount) {
+ Preconditions.checkArgument(
+ newRecordCount >= 0, "Invalid record count: %s (must be >= 0)",
newRecordCount);
+ this.recordCount = newRecordCount;
+ return this;
+ }
+
+ TrackedFileBuilder fileSizeInBytes(long newFileSizeInBytes) {
+ Preconditions.checkArgument(
+ newFileSizeInBytes >= 0,
+ "Invalid file size in bytes: %s (must be >= 0)",
+ newFileSizeInBytes);
+ this.fileSizeInBytes = newFileSizeInBytes;
+ return this;
+ }
+
+ TrackedFileBuilder specId(int newSpecId) {
+ Preconditions.checkArgument(newSpecId >= 0, "Invalid spec ID: %s (must be
>= 0)", newSpecId);
+ this.specId = newSpecId;
+ return this;
+ }
+
+ TrackedFileBuilder partition(PartitionData newPartitionData) {
+ Preconditions.checkArgument(newPartitionData != null, "Invalid partition:
null");
+ this.partitionData = newPartitionData;
+ return this;
+ }
+
+ TrackedFileBuilder contentStats(ContentStats newContentStats) {
+ Preconditions.checkArgument(newContentStats != null, "Invalid content
stats: null");
+ this.contentStats = newContentStats;
+ return this;
+ }
+
+ TrackedFileBuilder sortOrderId(int newSortOrderId) {
+ Preconditions.checkArgument(
+ !isLeafManifest(contentType),
+ "Sort order ID can not be added to manifest entries, but entry type
is: %s",
+ contentType);
+ Preconditions.checkArgument(
+ newSortOrderId >= 0, "Invalid sort order ID: %s (must be >= 0)",
newSortOrderId);
+ this.sortOrderId = newSortOrderId;
+ return this;
+ }
+
+ /**
+ * This method is for signalling that a new deletion vector is added. It is
forbidden to add the
+ * same deletion vector that already exists for this {@link TrackedFile}.
+ */
+ TrackedFileBuilder deletionVector(DeletionVector newDeletionVector) {
+ Preconditions.checkArgument(newDeletionVector != null, "Invalid deletion
vector: null");
+ Preconditions.checkArgument(
+ contentType == FileContent.DATA,
+ "Deletion vector can only be added to DATA entries, but entry type is:
%s",
+ contentType);
+ Preconditions.checkArgument(
+ this.deletionVector == null ||
!this.deletionVector.equals(newDeletionVector),
+ "The same deletion vector already added");
+ this.deletionVector = newDeletionVector;
+ this.dvUpdated = true;
+ return this;
+ }
+
+ TrackedFileBuilder manifestInfo(ManifestInfo newManifestInfo) {
+ Preconditions.checkArgument(newManifestInfo != null, "Invalid manifest
info: null");
+ Preconditions.checkArgument(
+ isLeafManifest(contentType),
+ "Manifest info can only be added to manifests, but entry type is: %s",
+ contentType);
+ this.manifestInfo = newManifestInfo;
+ return this;
+ }
+
+ TrackedFileBuilder keyMetadata(ByteBuffer newKeyMetadata) {
+ Preconditions.checkArgument(newKeyMetadata != null, "Invalid key metadata:
null");
+ this.keyMetadata = newKeyMetadata;
+ return this;
+ }
+
+ TrackedFileBuilder splitOffsets(List<Long> newSplitOffsets) {
+ Preconditions.checkArgument(newSplitOffsets != null, "Invalid split
offsets: null");
+ Preconditions.checkArgument(
+ !isLeafManifest(contentType),
+ "Split offsets can not be added to manifest entries, but entry type
is: %s",
Review Comment:
done
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
