badalprasadsingh commented on code in PR #2434: URL: https://github.com/apache/iceberg-go/pull/2434#discussion_r4521359266
########## puffin/dv_golden_test.go: ########## @@ -37,14 +38,9 @@ import ( // https://github.com/apache/iceberg/blob/main/core/src/test/resources/org/apache/iceberg/deletes/small-alternating-values-position-index.bin const dvFixturePayloadName = "deletion-vector-v1-payload.bin" -// dvFixturePuffinName is the complete puffin file wrapping the Java-produced -// DV payload. The envelope is what puffin.Writer emits today; this is a -// Go-writer wire-format pin, not a strong Java cross-impl pin. (The basic -// puffin envelope shape is cross-checked by TestWriterBitIdenticalWithJava, -// but that test does not exercise empty Fields arrays or multi-key blob -// Properties, both of which this fixture relies on. JSON key ordering of -// blob Properties is also encoder-defined.) Regenerate via -// `go generate ./puffin/...` (driven by gen_dv_fixture.go). +// dvFixturePuffinName is the Go-written Puffin envelope around a Java-produced +// DV payload. It pins the Go writer's bytes and checks the Java row-position +// field metadata. Regenerate via `go generate ./puffin/...`. Review Comment: nit: `puffin/testdata/README.md` still says this fixture relies on empty Fields arrays please update it to match ########## table/dv/dv_writer.go: ########## @@ -220,7 +220,7 @@ func (w *DVWriter) Flush(ctx context.Context, location string) ([]iceberg.DataFi Type: puffin.BlobTypeDeletionVector, SnapshotID: -1, SequenceNumber: -1, - Fields: []int32{}, + Fields: []int32{iceberg.RowPositionFieldID}, Review Comment: this changes what every go dv write stores and replaces a binary test fixture: a fine Java-parity fix, but unrelated to this featur can we move it, with the fixture and the new constant, into its own fix(dv) PR, so it gets its own review and release note? ########## metadata_columns.go: ########## @@ -25,6 +25,10 @@ import "math" // _partition, ...). Java: TypeUtil.MAX_STRUCT_FIELD_ID (Integer.MAX_VALUE - 200). const MaxStructFieldID = math.MaxInt32 - 200 +// RowPositionFieldID is the field ID for _pos, the ordinal position of a row +// in its source data file. It is reserved as Integer.MAX_VALUE - 2. +const RowPositionFieldID = math.MaxInt32 - 2 Review Comment: nit: the other reserved IDs here are literals in grouped blocks, like `RowIDFieldID=2147483540` please write 2147483645 and move this next to them ########## table/convert_equality_deletes_to_dv.go: ########## @@ -0,0 +1,390 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package table + +import ( + "context" + "fmt" + "slices" + "sync" + + "github.com/apache/iceberg-go" + "github.com/apache/iceberg-go/table/dv" + "github.com/google/uuid" +) + +// equalityDeletePositionBuffer accumulates matched positions for one data file. +// Each buffer has its own mutex so parallel scans of different files do not +// contend on a table-wide lock. The scan finishes before bitmap is read. +type equalityDeletePositionBuffer struct { + mu sync.Mutex + bitmap *dv.RoaringPositionBitmap + invalidPosition int64 +} + +func (b *equalityDeletePositionBuffer) addPositions(positions []int64) { + if len(positions) == 0 { + return + } + + b.mu.Lock() + defer b.mu.Unlock() + + if b.invalidPosition < 0 { + return + } + for _, position := range positions { + if position < 0 { + b.invalidPosition = position + + return + } + } + + if b.bitmap == nil { + b.bitmap = dv.NewRoaringPositionBitmap() + } + for _, position := range positions { + b.bitmap.Set(uint64(position)) + } +} + +// EqualityDeleteToDVResult summarizes an equality-delete conversion. +type EqualityDeleteToDVResult struct { + // Table is the table returned by the table-level action. It is nil for + // transaction callers, including after the transaction commits. + Table *Table + + // ConvertedEqualityDeleteFiles is the number of live equality delete files + // removed by the rewrite. + ConvertedEqualityDeleteFiles int + // AddedDeletionVectors is the number of file-specific deletion vectors added. + AddedDeletionVectors int + // RemovedPositionDeleteFiles is the number of legacy position delete files + // scoped to one data file, either by referenced_data_file or equal + // file_path bounds, removed with the conversion. + RemovedPositionDeleteFiles int + // ReplacedDeletionVectors is the number of existing vectors merged into new + // vectors and removed by the rewrite. + ReplacedDeletionVectors int +} + +// ConvertEqualityDeletesToDeletionVectors converts live equality deletes on +// the main branch of a v3 table to file-specific deletion vectors. The rewrite +// commits as one replace snapshot so changelog readers do not see a second +// logical deletion event. See [Transaction.ConvertEqualityDeletesToDeletionVectors] +// for its limitations. +func (t Table) ConvertEqualityDeletesToDeletionVectors(ctx context.Context) (*EqualityDeleteToDVResult, error) { + txn, err := t.NewTransactionOnBranch(MainBranch) + if err != nil { + return nil, err + } + + result, err := txn.ConvertEqualityDeletesToDeletionVectors(ctx) + if err != nil { + return result, err + } + if result.ConvertedEqualityDeleteFiles == 0 { + result.Table = &t + + return result, nil + } + + committed, err := txn.Commit(ctx) + if err == nil { + result.Table = committed + } + + return result, err +} + +// ConvertEqualityDeletesToDeletionVectors stages an equality-delete to +// deletion-vector conversion on the transaction's branch. Commit the +// transaction to publish the replace snapshot. +// +// The conversion reads every data file targeted by an applicable equality +// delete. It merges file-scoped position deletes and existing vectors, but is +// not supported when an applicable unscoped or partition-scoped position +// delete would remain live. A concurrent commit is not replayed against a +// refreshed conversion plan and may make Commit return [ErrCommitFailed]. If a +// transaction is abandoned after staging, its Puffin file may remain for +// orphan cleanup. +func (t *Transaction) ConvertEqualityDeletesToDeletionVectors(ctx context.Context) (*EqualityDeleteToDVResult, error) { + meta, err := t.txnMeta() + if err != nil { + return nil, err + } + if meta.formatVersion != 3 { + return nil, fmt.Errorf("%w: equality-delete to deletion-vector conversion requires a v3 table, got v%d", + iceberg.ErrNotImplemented, meta.formatVersion) + } + + snapshot := t.planningSnapshot(meta) + if snapshot == nil { + return &EqualityDeleteToDVResult{}, nil + } + + fs, err := t.tbl.FS(ctx) + if err != nil { + return nil, err + } + manifests, err := snapshot.Manifests(ctx, fs) + if err != nil { + return nil, err + } + equalityDeleteFiles := make(map[string]iceberg.DataFile) + for _, manifest := range manifests { + if err := ctx.Err(); err != nil { + return nil, err + } + if manifest.ManifestContent() != iceberg.ManifestContentDeletes { + continue + } + for entry, err := range manifest.Entries(ctx, fs, true) { + if err != nil { + return nil, err + } + file := entry.DataFile() + if file.ContentType() != iceberg.EntryContentEqDeletes { + continue + } + if _, exists := equalityDeleteFiles[file.FilePath()]; exists { + return nil, fmt.Errorf("%w: duplicate live equality delete file %s", + ErrInvalidMetadata, file.FilePath()) + } + equalityDeleteFiles[file.FilePath()] = file + } + } + if len(equalityDeleteFiles) == 0 { + return &EqualityDeleteToDVResult{}, nil + } + // Only projected equality keys are needed to resolve their matching row + // positions. Missing (dropped) keys are handled by the delete loader. + keyNames := make(map[string]struct{}) + for _, file := range equalityDeleteFiles { + for _, id := range file.EqualityFieldIDs() { + if name, ok := meta.CurrentSchema().FindColumnName(id); ok { + keyNames[name] = struct{}{} + } + } + } + selectedFields := make([]string, 0, len(keyNames)) + for name := range keyNames { + selectedFields = append(selectedFields, name) + } + slices.Sort(selectedFields) + scanOpts := []ScanOption{WithScanPlanningMode(ScanPlanningLocal)} Review Comment: nit: txn.Scan always plans locally, so `WithScanPlanningMode(ScanPlanningLocal)` isn't needed ########## table/convert_equality_deletes_to_dv_test.go: ########## @@ -0,0 +1,597 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package table_test + +import ( + "fmt" + "os" + "path/filepath" + "strings" + "testing" + + "github.com/apache/arrow-go/v18/arrow/array" + "github.com/apache/arrow-go/v18/arrow/memory" + "github.com/apache/iceberg-go" + "github.com/apache/iceberg-go/table" + "github.com/stretchr/testify/require" +) + +func TestConvertEqualityDeletesToDeletionVectors(t *testing.T) { + ctx := t.Context() + tbl := appendRows(t, newMergeOnReadTestTableVersion(t, "3"), + `[{"id":1,"data":"a"},{"id":2,"data":"b"},{"id":3,"data":"c"},{"id":4,"data":"d"},{"id":5,"data":"e"}]`) + beforeDelete := tbl.CurrentSnapshot().SnapshotID + tbl = appendEqualityDelete(t, tbl, []int{1}, `[{"id":2},{"id":4}]`) + deleteSnapshot := tbl.CurrentSnapshot().SnapshotID + + result, err := tbl.ConvertEqualityDeletesToDeletionVectors(ctx) + require.NoError(t, err) + require.NotNil(t, result.Table) + require.Equal(t, 1, result.ConvertedEqualityDeleteFiles) + require.Equal(t, 1, result.AddedDeletionVectors) + require.Equal(t, []int64{1, 3, 5}, idsInTable(t, result.Table)) + require.Equal(t, 1, liveDVCount(t, result.Table)) + require.Equal(t, 0, liveDeleteFileCount(t, result.Table, iceberg.EntryContentEqDeletes)) + require.Equal(t, table.OpReplace, result.Table.CurrentSnapshot().Summary.Operation) + + // The equality-delete snapshot emits the deletes once. The conversion + // snapshot is maintenance and must not emit them again. + deleteRows := readChangelog(t, ctx, + tbl.NewIncrementalChangelogScan().FromSnapshotExclusive(beforeDelete).ToSnapshot(deleteSnapshot)) + require.Equal(t, []int64{2, 4}, changelogIDs(deleteRows, table.ChangelogOpDelete)) + conversionRows := readChangelog(t, ctx, + result.Table.NewIncrementalChangelogScan().FromSnapshotExclusive(deleteSnapshot)) + require.Empty(t, conversionRows) +} + +// Equality deletes may use different key sets. Each must resolve against the +// same data file before their positions are merged into a single DV. +func TestConvertEqualityDeletesWithDifferentEqualityFields(t *testing.T) { + ctx := t.Context() + tbl := appendRows(t, newMergeOnReadTestTableVersion(t, "3"), + `[{"id":1,"data":"a"},{"id":2,"data":"b"},{"id":3,"data":"c"},{"id":4,"data":"d"}]`) + tbl = appendEqualityDelete(t, tbl, []int{1}, `[{"id":2}]`) + tbl = appendEqualityDelete(t, tbl, []int{2}, `[{"data":"c"}]`) + + require.Equal(t, []int64{1, 4}, idsInTable(t, tbl)) + result, err := tbl.ConvertEqualityDeletesToDeletionVectors(ctx) + require.NoError(t, err) + require.Equal(t, 2, result.ConvertedEqualityDeleteFiles) + require.Equal(t, 1, result.AddedDeletionVectors) + require.Equal(t, 1, liveDVCount(t, result.Table)) + require.Zero(t, liveDeleteFileCount(t, result.Table, iceberg.EntryContentEqDeletes)) + require.Equal(t, []int64{1, 4}, idsInTable(t, result.Table)) +} + +// A conversion staged after write.data.path changes must use the transaction's +// updated output location, not the base table's data directory. +func TestConvertEqualityDeletesUsesStagedDataPath(t *testing.T) { + ctx := t.Context() + tbl := appendRows(t, newMergeOnReadTestTableVersion(t, "3"), + `[{"id":1,"data":"a"},{"id":2,"data":"b"}]`) + tbl = appendEqualityDelete(t, tbl, []int{1}, `[{"id":2}]`) + + dataPath := filepath.ToSlash(filepath.Join(t.TempDir(), "custom-dv-data")) + tx := tbl.NewTransaction() + require.NoError(t, tx.SetProperties(iceberg.Properties{table.WriteDataPathKey: dataPath})) + + result, err := tx.ConvertEqualityDeletesToDeletionVectors(ctx) + require.NoError(t, err) + require.Equal(t, 1, result.AddedDeletionVectors) + committed, err := tx.Commit(ctx) + require.NoError(t, err) + require.Equal(t, []int64{1}, idsInTable(t, committed)) + + fs, err := committed.FS(ctx) + require.NoError(t, err) + manifests, err := committed.CurrentSnapshot().Manifests(ctx, fs) + require.NoError(t, err) + + var dvPaths []string + for _, manifest := range manifests { + if manifest.ManifestContent() != iceberg.ManifestContentDeletes { + continue + } + for entry, err := range manifest.Entries(ctx, fs, false) { + require.NoError(t, err) + if entry.Status() != iceberg.EntryStatusDELETED && table.IsDeletionVector(entry.DataFile()) { + dvPaths = append(dvPaths, entry.DataFile().FilePath()) + } + } + } + require.Len(t, dvPaths, 1) + require.True(t, strings.HasPrefix(dvPaths[0], dataPath+"/"), dvPaths[0]) +} + +func TestConvertEqualityDeletesMergesExistingPositionDeletes(t *testing.T) { + ctx := t.Context() + tbl := appendRows(t, newMergeOnReadTestTableVersion(t, "2"), + `[{"id":1,"data":"a"},{"id":2,"data":"b"},{"id":3,"data":"c"},{"id":4,"data":"d"},{"id":5,"data":"e"}]`) + var err error + tbl, err = tbl.Delete(ctx, iceberg.EqualTo(iceberg.Reference("id"), int64(2)), nil) + require.NoError(t, err) + require.Equal(t, 1, liveDeleteFileCount(t, tbl, iceberg.EntryContentPosDeletes)) + + tx := tbl.NewTransaction() + require.NoError(t, tx.UpgradeFormatVersion(3)) + tbl, err = tx.Commit(ctx) + require.NoError(t, err) + tbl = appendEqualityDelete(t, tbl, []int{1}, `[{"id":4}]`) + + result, err := tbl.ConvertEqualityDeletesToDeletionVectors(ctx) + require.NoError(t, err) + require.Equal(t, 1, result.RemovedPositionDeleteFiles) + require.Zero(t, liveDeleteFileCount(t, result.Table, iceberg.EntryContentPosDeletes)) + require.Equal(t, 1, liveDVCount(t, result.Table)) + require.Equal(t, []int64{1, 3, 5}, idsInTable(t, result.Table), + "the old position delete and the new equality delete must both remain applied") +} + +// A legacy position-delete file can cover multiple data files. Replacing it +// with a DV for only one of those files must not leave the shared file live. +func TestConvertEqualityDeletesRejectsSharedPositionDeletes(t *testing.T) { + ctx := t.Context() + tbl := newMergeOnReadTestTableVersion(t, "2") + tbl = appendRows(t, tbl, + `[{"id":1,"data":"a"},{"id":2,"data":"b"}]`) + tbl = appendRows(t, tbl, + `[{"id":3,"data":"c"},{"id":4,"data":"d"}]`) + + scan := tbl.Scan() + defer scan.Close() + tasks, err := scan.PlanFiles(ctx) + require.NoError(t, err) + require.Len(t, tasks, 2) + + positionPath := tbl.Location() + "/data/shared-position-delete.parquet" + writeParquetFile(t, positionPath, table.PositionalDeleteArrowSchema, fmt.Sprintf( + `[{"file_path":%q,"pos":0},{"file_path":%q,"pos":0}]`, + tasks[0].File.FilePath(), tasks[1].File.FilePath())) + info, err := os.Stat(positionPath) + require.NoError(t, err) + builder, err := iceberg.NewDataFileBuilder(*iceberg.UnpartitionedSpec, + iceberg.EntryContentPosDeletes, positionPath, iceberg.ParquetFile, nil, nil, nil, 2, info.Size()) + require.NoError(t, err) + // Deliberately omit ReferencedDataFile: both files share this delete file. + tx := tbl.NewTransaction() + require.NoError(t, tx.NewRowDelta(nil).AddDeletes(builder.Build()).Commit(ctx)) + tbl, err = tx.Commit(ctx) + require.NoError(t, err) + + tx = tbl.NewTransaction() + require.NoError(t, tx.UpgradeFormatVersion(3)) + tbl, err = tx.Commit(ctx) + require.NoError(t, err) + tbl = appendEqualityDelete(t, tbl, []int{1}, `[{"id":2}]`) + + // Enable retries so the final commit verifies replay after the peer update, + // rather than failing immediately when main changes. + retryTx := tbl.NewTransaction() + require.NoError(t, retryTx.SetProperties(iceberg.Properties{table.CommitNumRetriesKey: "2"})) + tbl, err = retryTx.Commit(ctx) + require.NoError(t, err) + + beforeSnapshot := tbl.CurrentSnapshot().SnapshotID + require.Equal(t, []int64{4}, idsInTable(t, tbl)) + puffinFiles := func() []string { + var paths []string + err := filepath.WalkDir(tbl.Location(), func(path string, entry os.DirEntry, walkErr error) error { + if walkErr != nil { + return walkErr + } + if !entry.IsDir() && filepath.Ext(path) == ".puffin" { + paths = append(paths, path) + } + + return nil + }) + require.NoError(t, err) + + return paths + } + puffinFilesBefore := puffinFiles() + + _, err = tbl.ConvertEqualityDeletesToDeletionVectors(ctx) + require.ErrorIs(t, err, table.ErrInvalidOperation) + require.Equal(t, beforeSnapshot, tbl.CurrentSnapshot().SnapshotID) + require.Equal(t, 1, liveDeleteFileCount(t, tbl, iceberg.EntryContentEqDeletes)) + require.Equal(t, 1, liveDeleteFileCount(t, tbl, iceberg.EntryContentPosDeletes)) + require.Zero(t, liveDVCount(t, tbl)) + require.Equal(t, []int64{4}, idsInTable(t, tbl)) + require.Equal(t, puffinFilesBefore, puffinFiles(), "a rejected conversion must remove its staged Puffin file") + + // A failed conversion must not leave the transaction pinned: callers + // can still stage unrelated metadata changes and retry on a newer head. + tx = tbl.NewTransaction() + _, err = tx.ConvertEqualityDeletesToDeletionVectors(ctx) + require.ErrorIs(t, err, table.ErrInvalidOperation) + peer := appendRows(t, tbl, `[{"id":5,"data":"e"}]`) + require.NoError(t, tx.SetProperties(iceberg.Properties{"conversion-retry": "allowed"})) + committed, err := tx.Commit(ctx) + require.NoError(t, err) + require.Equal(t, []int64{4, 5}, idsInTable(t, committed)) + require.Equal(t, []int64{4, 5}, idsInTable(t, peer)) +} + +// A legacy v2 position-delete file can be file-scoped through equal file_path +// bounds without an explicit referenced_data_file value. +func TestConvertEqualityDeletesRemovesBoundsScopedPositionDeletes(t *testing.T) { + ctx := t.Context() + tbl := appendRows(t, newMergeOnReadTestTableVersion(t, "2"), + `[{"id":1,"data":"a"},{"id":2,"data":"b"},{"id":3,"data":"c"},{"id":4,"data":"d"}]`) + tasks, err := tbl.Scan().PlanFiles(ctx) + require.NoError(t, err) + require.Len(t, tasks, 1) + dataPath := tasks[0].File.FilePath() + + deletePath := filepath.ToSlash(filepath.Join(tbl.Location(), "data", "bounds-position-delete.parquet")) + writeParquetFile(t, deletePath, table.PositionalDeleteArrowSchema, + fmt.Sprintf(`[{"file_path":%q,"pos":1}]`, dataPath)) + info, err := os.Stat(deletePath) + require.NoError(t, err) + builder, err := iceberg.NewDataFileBuilder(*iceberg.UnpartitionedSpec, + iceberg.EntryContentPosDeletes, deletePath, iceberg.ParquetFile, nil, nil, nil, 1, info.Size()) + require.NoError(t, err) + bounds := map[int][]byte{iceberg.DeleteFilePathFieldID: []byte(dataPath)} + builder.LowerBoundValues(bounds).UpperBoundValues(bounds) + + tx := tbl.NewTransaction() + require.NoError(t, tx.NewRowDelta(nil).AddDeletes(builder.Build()).Commit(ctx)) + tbl, err = tx.Commit(ctx) + require.NoError(t, err) + require.Equal(t, []int64{1, 3, 4}, idsInTable(t, tbl)) + + tx = tbl.NewTransaction() + require.NoError(t, tx.UpgradeFormatVersion(3)) + tbl, err = tx.Commit(ctx) + require.NoError(t, err) + tbl = appendEqualityDelete(t, tbl, []int{1}, `[{"id":3}]`) + + result, err := tbl.ConvertEqualityDeletesToDeletionVectors(ctx) + require.NoError(t, err) + require.Equal(t, 1, result.RemovedPositionDeleteFiles) + require.Zero(t, liveDeleteFileCount(t, result.Table, iceberg.EntryContentPosDeletes)) + require.Equal(t, 1, liveDVCount(t, result.Table)) + require.Equal(t, []int64{1, 4}, idsInTable(t, result.Table)) +} + +func TestConvertEqualityDeletesMergesExistingDeletionVector(t *testing.T) { + ctx := t.Context() + tbl := appendRows(t, newMergeOnReadTestTableVersion(t, "3"), + `[{"id":1,"data":"a"},{"id":2,"data":"b"},{"id":3,"data":"c"},{"id":4,"data":"d"},{"id":5,"data":"e"}]`) + var err error + tbl, err = tbl.Delete(ctx, iceberg.EqualTo(iceberg.Reference("id"), int64(2)), nil) + require.NoError(t, err) + tbl = appendEqualityDelete(t, tbl, []int{1}, `[{"id":4}]`) + + result, err := tbl.ConvertEqualityDeletesToDeletionVectors(ctx) + require.NoError(t, err) + require.Equal(t, 1, result.ReplacedDeletionVectors) + require.Equal(t, 1, liveDVCount(t, result.Table)) + require.Equal(t, []int64{1, 3, 5}, idsInTable(t, result.Table), + "merging a new equality delete must not resurrect rows from the old vector") +} + +func TestConvertEqualityDeletesWritesOneVectorPerPartitionFile(t *testing.T) { + ctx := t.Context() + tbl := newPartitionedMergeOnReadTestTable(t) + arrowSchema, err := table.SchemaToArrowSchema(tbl.Metadata().CurrentSchema(), nil, false, false) + require.NoError(t, err) + data, err := array.TableFromJSON(memory.DefaultAllocator, arrowSchema, []string{ + `[{"id":1,"category":"x"},{"id":2,"category":"x"},{"id":1,"category":"y"},{"id":3,"category":"y"}]`, + }) + require.NoError(t, err) + defer data.Release() + tbl, err = tbl.Append(ctx, array.NewTableReader(data, -1), nil) + require.NoError(t, err) + tbl = appendEqualityDelete(t, tbl, []int{1, 2}, + `[{"id":1,"category":"x"},{"id":1,"category":"y"}]`) + + result, err := tbl.ConvertEqualityDeletesToDeletionVectors(ctx) + require.NoError(t, err) + require.Equal(t, 2, result.ConvertedEqualityDeleteFiles) + require.Equal(t, 2, result.AddedDeletionVectors) + require.Equal(t, 2, liveDVCount(t, result.Table)) + require.Equal(t, []int64{2, 3}, idsInTable(t, result.Table)) +} + +func TestConvertEqualityDeletesAcrossDataFileSplits(t *testing.T) { + ctx := t.Context() + tbl := appendRows(t, newMergeOnReadTestTableVersion(t, "3"), + `[{"id":1,"data":"a"},{"id":2,"data":"b"},{"id":3,"data":"c"},{"id":4,"data":"d"},{"id":5,"data":"e"},{"id":6,"data":"f"},{"id":7,"data":"g"},{"id":8,"data":"h"},{"id":9,"data":"i"},{"id":10,"data":"j"}]`) + tx := tbl.NewTransaction() + require.NoError(t, tx.SetProperties(iceberg.Properties{table.ReadSplitTargetSizeKey: "1"})) + tbl, err := tx.Commit(ctx) + require.NoError(t, err) + tbl = appendEqualityDelete(t, tbl, []int{1}, `[{"id":2},{"id":7}]`) + + tasks, err := tbl.Scan().PlanFiles(ctx) + require.NoError(t, err) + require.Len(t, tasks, 2, "the data file has two row groups and each must be planned as a split") + require.Equal(t, tasks[0].File.FilePath(), tasks[1].File.FilePath()) + + result, err := tbl.ConvertEqualityDeletesToDeletionVectors(ctx) + require.NoError(t, err) + require.Equal(t, 1, result.AddedDeletionVectors, "all split positions must form one vector for the data file") + require.Equal(t, 1, liveDVCount(t, result.Table)) + require.Equal(t, []int64{1, 3, 4, 5, 6, 8, 9, 10}, idsInTable(t, result.Table)) +} + +func TestConvertEqualityDeletesRemovesDeletesWithNoMatches(t *testing.T) { + tbl := appendRows(t, newMergeOnReadTestTableVersion(t, "3"), + `[{"id":1,"data":"a"},{"id":2,"data":"b"}]`) + tbl = appendEqualityDelete(t, tbl, []int{1}, `[{"id":99}]`) + + result, err := tbl.ConvertEqualityDeletesToDeletionVectors(t.Context()) + require.NoError(t, err) + require.Equal(t, 1, result.ConvertedEqualityDeleteFiles) + require.Zero(t, result.AddedDeletionVectors) + require.Zero(t, liveDeleteFileCount(t, result.Table, iceberg.EntryContentEqDeletes)) + require.Empty(t, liveDVCount(t, result.Table)) + require.Equal(t, []int64{1, 2}, idsInTable(t, result.Table)) +} + +func TestConvertEqualityDeletesPreservesSameSnapshotSequenceSemantics(t *testing.T) { + ctx := t.Context() + tbl := newMergeOnReadTestTableVersion(t, "3") + dataPath := filepath.ToSlash(tbl.Location()) + "/data/same-snapshot-data.parquet" + equalityPath := filepath.ToSlash(tbl.Location()) + "/data/same-snapshot-equality.parquet" + dataSchema, err := table.SchemaToArrowSchema(tbl.Metadata().CurrentSchema(), nil, true, false) + require.NoError(t, err) + writeParquetFile(t, dataPath, dataSchema, `[{"id":7,"data":"same snapshot"}]`) + equalitySchema, err := table.SchemaToArrowSchema( + iceberg.NewSchema(0, iceberg.NestedField{ID: 1, Name: "id", Type: iceberg.PrimitiveTypes.Int64, Required: true}), + nil, true, false) + require.NoError(t, err) + writeParquetFile(t, equalityPath, equalitySchema, `[{"id":7}]`) + info, err := os.Stat(equalityPath) + require.NoError(t, err) + equalityBuilder, err := iceberg.NewDataFileBuilder( + *iceberg.UnpartitionedSpec, iceberg.EntryContentEqDeletes, + equalityPath, iceberg.ParquetFile, nil, nil, nil, 1, info.Size()) + require.NoError(t, err) + equalityBuilder.EqualityFieldIDs([]int{1}) + + tx := tbl.NewTransaction() + require.NoError(t, tx.NewRowDelta(nil). + AddRows(changelogDataFile(t, dataPath, 1)). + AddDeletes(equalityBuilder.Build()). + Commit(ctx)) + tbl, err = tx.Commit(ctx) + require.NoError(t, err) + require.Equal(t, []int64{7}, idsInTable(t, tbl), "same-snapshot equality deletes do not apply to same-snapshot data") + + result, err := tbl.ConvertEqualityDeletesToDeletionVectors(ctx) + require.NoError(t, err) + require.Equal(t, 1, result.ConvertedEqualityDeleteFiles) + require.Zero(t, result.AddedDeletionVectors) + require.Equal(t, []int64{7}, idsInTable(t, result.Table)) +} + +// Replacing equality deletes with a new DV must not change the applicability +// of older delete files to newer data files that contain the same key. +func TestConvertEqualityDeletesPreservesMultipleDeleteSequences(t *testing.T) { + ctx := t.Context() + tbl := appendRows(t, newMergeOnReadTestTableVersion(t, "3"), + `[{"id":1,"data":"old"},{"id":2,"data":"b"},{"id":3,"data":"c"}]`) + tbl = appendEqualityDelete(t, tbl, []int{1}, `[{"id":1}]`) + tbl = appendRows(t, tbl, + `[{"id":1,"data":"new"},{"id":4,"data":"d"}]`) + tbl = appendEqualityDelete(t, tbl, []int{1}, `[{"id":2}]`) + require.Equal(t, []int64{1, 3, 4}, idsInTable(t, tbl)) + + result, err := tbl.ConvertEqualityDeletesToDeletionVectors(ctx) + require.NoError(t, err) + require.Equal(t, 2, result.ConvertedEqualityDeleteFiles) + require.Equal(t, []int64{1, 3, 4}, idsInTable(t, result.Table), + "an older equality delete must not delete a newer row after conversion") + require.Zero(t, liveDeleteFileCount(t, result.Table, iceberg.EntryContentEqDeletes)) + + // Re-running the maintenance action must not create another snapshot. + convertedSnapshotID := result.Table.CurrentSnapshot().SnapshotID + again, err := result.Table.ConvertEqualityDeletesToDeletionVectors(ctx) + require.NoError(t, err) + require.Zero(t, again.ConvertedEqualityDeleteFiles) + require.Zero(t, again.AddedDeletionVectors) + require.Equal(t, convertedSnapshotID, again.Table.CurrentSnapshot().SnapshotID) +} + +// A global equality delete uses an unpartitioned delete spec even when the +// existing data files were written with a partitioned spec. +func TestConvertEqualityDeletesGlobalDeleteAcrossPartitions(t *testing.T) { + ctx := t.Context() + tbl := appendRows(t, newPartitionedMergeOnReadTestTable(t), + `[{"id":1,"category":"x"},{"id":2,"category":"x"},{"id":1,"category":"y"},{"id":3,"category":"y"}]`) + + // Leave the data files in their original category partitions, but write + // the next equality delete using a new unpartitioned spec. + tx := tbl.NewTransaction() + require.NoError(t, tx.UpdateSpec(true).RemoveField("category").Commit()) + var err error + tbl, err = tx.Commit(ctx) + require.NoError(t, err) + spec := tbl.Spec() + require.Zero(t, spec.NumFields()) + + tbl = appendEqualityDelete(t, tbl, []int{1}, `[{"id":1}]`) + require.Equal(t, []int64{2, 3}, idsInTable(t, tbl)) + + result, err := tbl.ConvertEqualityDeletesToDeletionVectors(ctx) + require.NoError(t, err) + require.Equal(t, 1, result.ConvertedEqualityDeleteFiles) + require.Equal(t, 2, result.AddedDeletionVectors) + require.Equal(t, 2, liveDVCount(t, result.Table)) + require.Zero(t, liveDeleteFileCount(t, result.Table, iceberg.EntryContentEqDeletes)) + require.Equal(t, []int64{2, 3}, idsInTable(t, result.Table)) +} + +func TestConvertEqualityDeletesUnpartitionedSpec(t *testing.T) { + tbl := appendRows(t, newMergeOnReadTestTableVersion(t, "3"), + `[{"id":1,"data":"a"},{"id":2,"data":"b"},{"id":3,"data":"c"}]`) + spec := tbl.Spec() + require.Zero(t, spec.NumFields()) + tbl = appendEqualityDelete(t, tbl, []int{1}, `[{"id":2}]`) + + result, err := tbl.ConvertEqualityDeletesToDeletionVectors(t.Context()) + require.NoError(t, err) + require.Equal(t, 1, result.AddedDeletionVectors) + require.Equal(t, 1, liveDVCount(t, result.Table)) + require.Equal(t, []int64{1, 3}, idsInTable(t, result.Table)) +} + +func TestConvertEqualityDeletesOnBranchLeavesMainUntouched(t *testing.T) { + ctx := t.Context() + tbl := newBranchPlanningTable(t, iceberg.Properties{table.PropertyFormatVersion: "3"}) + tbl = appendRowsOnRef(t, tbl, table.MainBranch, 1, 2, 3, 4) + tbl = appendRowsOnRef(t, tbl, branchPlanningRef, 5) + + // Write an equality delete onto the branch only. + deleteSchema := iceberg.NewSchema(0, + iceberg.NestedField{ID: 1, Name: "id", Type: iceberg.PrimitiveTypes.Int64, Required: true}) + arrowSchema, err := table.SchemaToArrowSchema(deleteSchema, nil, true, false) + require.NoError(t, err) + deletePath := filepath.ToSlash(filepath.Join(tbl.Location(), "data", "branch-equality-delete.parquet")) + writeParquetFile(t, deletePath, arrowSchema, `[{"id":2},{"id":5}]`) + info, err := os.Stat(deletePath) + require.NoError(t, err) + builder, err := iceberg.NewDataFileBuilder(*iceberg.UnpartitionedSpec, + iceberg.EntryContentEqDeletes, deletePath, iceberg.ParquetFile, nil, nil, nil, 2, info.Size()) + require.NoError(t, err) + builder.EqualityFieldIDs([]int{1}) + + tx, err := tbl.NewTransactionOnBranch(branchPlanningRef) + require.NoError(t, err) + require.NoError(t, tx.NewRowDelta(nil).AddDeletes(builder.Build()).Commit(ctx)) + tbl, err = tx.Commit(ctx) + require.NoError(t, err) + require.Equal(t, []int64{1, 3, 4}, idsOnRef(t, tbl, branchPlanningRef)) + require.Equal(t, []int64{1, 2, 3, 4}, idsOnRef(t, tbl, table.MainBranch)) + + tx, err = tbl.NewTransactionOnBranch(branchPlanningRef) + require.NoError(t, err) + result, err := tx.ConvertEqualityDeletesToDeletionVectors(ctx) + require.NoError(t, err) + require.Equal(t, 1, result.ConvertedEqualityDeleteFiles) + require.Equal(t, 2, result.AddedDeletionVectors) + tbl, err = tx.Commit(ctx) + require.NoError(t, err) + + require.Equal(t, []int64{1, 3, 4}, idsOnRef(t, tbl, branchPlanningRef)) + require.Equal(t, 2, liveDVCountOnRef(t, tbl, branchPlanningRef)) + require.Equal(t, []int64{1, 2, 3, 4}, idsOnRef(t, tbl, table.MainBranch)) + require.Zero(t, liveDVCountOnRef(t, tbl, table.MainBranch)) +} + +func TestConvertEqualityDeletesRequiresV3(t *testing.T) { + tbl := appendRows(t, newMergeOnReadTestTableVersion(t, "2"), + `[{"id":1,"data":"a"}]`) + tbl = appendEqualityDelete(t, tbl, []int{1}, `[{"id":1}]`) + + _, err := tbl.ConvertEqualityDeletesToDeletionVectors(t.Context()) + require.ErrorIs(t, err, iceberg.ErrNotImplemented) +} + +func liveDeleteFileCount(t *testing.T, tbl *table.Table, content iceberg.ManifestEntryContent) int { + t.Helper() + + ctx := t.Context() + fs, err := tbl.FS(ctx) + require.NoError(t, err) + snapshot := tbl.CurrentSnapshot() + manifests, err := snapshot.Manifests(ctx, fs) + require.NoError(t, err) + + count := 0 + for _, manifest := range manifests { + if manifest.ManifestContent() != iceberg.ManifestContentDeletes { + continue + } + for entry, err := range manifest.Entries(ctx, fs, false) { + require.NoError(t, err) + if entry.Status() == iceberg.EntryStatusDELETED { + continue + } + file := entry.DataFile() + if file.ContentType() != content { + continue + } + if content == iceberg.EntryContentPosDeletes && table.IsDeletionVector(file) { + continue + } + count++ + } + } + + return count +} + +func TestConvertEqualityDeletesTransactionStagesOneReplace(t *testing.T) { + tbl := appendRows(t, newMergeOnReadTestTableVersion(t, "3"), + `[{"id":1,"data":"a"},{"id":2,"data":"b"}]`) + tbl = appendEqualityDelete(t, tbl, []int{1}, `[{"id":2}]`) + + tx := tbl.NewTransaction() + result, err := tx.ConvertEqualityDeletesToDeletionVectors(t.Context()) + require.NoError(t, err) + require.Nil(t, result.Table) + require.Equal(t, 1, result.AddedDeletionVectors) + + committed, err := tx.Commit(t.Context()) + require.NoError(t, err) + require.Equal(t, table.OpReplace, committed.CurrentSnapshot().Summary.Operation) + require.Equal(t, []int64{1}, idsInTable(t, committed)) +} + +func TestConvertEqualityDeletesMergesTransactionStagedDeletionVector(t *testing.T) { + ctx := t.Context() + tbl := appendRows(t, newMergeOnReadTestTableVersion(t, "3"), + `[{"id":1,"data":"a"},{"id":2,"data":"b"},{"id":3,"data":"c"},{"id":4,"data":"d"}]`) + tbl = appendEqualityDelete(t, tbl, []int{1}, `[{"id":2}]`) + + tx := tbl.NewTransaction() + require.NoError(t, tx.Delete(ctx, iceberg.EqualTo(iceberg.Reference("id"), int64(3)), nil)) + + result, err := tx.ConvertEqualityDeletesToDeletionVectors(ctx) + require.NoError(t, err) + require.Equal(t, 1, result.ReplacedDeletionVectors) + require.Equal(t, 1, result.AddedDeletionVectors) + + committed, err := tx.Commit(ctx) + require.NoError(t, err) + require.Equal(t, 1, liveDVCount(t, committed), "the staged and converted positions must share one live DV") + require.Zero(t, liveDeleteFileCount(t, committed, iceberg.EntryContentEqDeletes)) + require.Equal(t, []int64{1, 4}, idsInTable(t, committed), + "conversion must preserve both the staged delete and equality-delete matches") +} + +func TestConvertEqualityDeletesRejectsStaleTransactionAtCommit(t *testing.T) { + ctx := t.Context() + tbl := appendRows(t, newMergeOnReadTestTableVersion(t, "3"), + `[{"id":1,"data":"a"},{"id":2,"data":"b"}]`) + tbl = appendEqualityDelete(t, tbl, []int{1}, `[{"id":2}]`) + + tx := tbl.NewTransaction() + _, err := tx.ConvertEqualityDeletesToDeletionVectors(ctx) + require.NoError(t, err) + + concurrent := appendRows(t, tbl, `[{"id":3,"data":"concurrent"}]`) + _, err = tx.Commit(ctx) + require.ErrorIs(t, err, table.ErrCommitFailed) Review Comment: retries default to 0 on this table: so the first conflict ends the commit and no replay is tried please set `commit.retry.num-retries` here so the test proves a retried commit still fails ########## table/dv/dv_writer.go: ########## @@ -220,7 +220,7 @@ func (w *DVWriter) Flush(ctx context.Context, location string) ([]iceberg.DataFi Type: puffin.BlobTypeDeletionVector, SnapshotID: -1, SequenceNumber: -1, - Fields: []int32{}, + Fields: []int32{iceberg.RowPositionFieldID}, Review Comment: also, no test reads back what Flush writes here the golden tests build their own blobs now, wherever you decide to keep this change: please add a test that flushes a dv and checks the blob's fields is [RowPositionFieldID] ########## website/src/releases.md: ########## @@ -99,6 +99,7 @@ Per-release notes (highlights, breaking changes, contributors) are published on - `maintenance.PurgeFiles` (previously `Table.PurgeFiles`) now calls `IO.Remove` concurrently, with up to 32 calls in flight, when the filesystem does not implement `BulkRemovableIO`. Custom IO implementations, including those registered with `io.Register`, must make `Remove` safe for concurrent use and synchronize any shared mutable state. The bulk deletion path is unchanged. - **Snapshot summaries report deletion vectors separately.** On v3 tables, DVs are counted under `added-dvs` and `removed-dvs` instead of `added-position-delete-files` and `removed-position-delete-files`. `added-files-size` and `removed-files-size` use each DV's `content_size_in_bytes` instead of its whole Puffin file, so they go down for commits with DVs. `total-files-size` is updated from these values. Partition summaries, commit reports and OTel metrics change the same way. Existing snapshots are not rewritten. +- **Deletion-vector Puffin blobs now identify the `_pos` field.** Newly written deletion vectors list the reserved row-position field ID `2147483546` in Puffin blob `fields` metadata instead of an empty list, matching Iceberg Java. This affects all deletion-vector writes, not just equality-delete conversion; existing Puffin files are unchanged. Review Comment: the ID here is 2147483546, but Integer.MAX_VALUE - 2 is 2147483645: please fix the digits ########## table/convert_equality_deletes_to_dv.go: ########## @@ -0,0 +1,390 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package table + +import ( + "context" + "fmt" + "slices" + "sync" + + "github.com/apache/iceberg-go" + "github.com/apache/iceberg-go/table/dv" + "github.com/google/uuid" +) + +// equalityDeletePositionBuffer accumulates matched positions for one data file. +// Each buffer has its own mutex so parallel scans of different files do not +// contend on a table-wide lock. The scan finishes before bitmap is read. +type equalityDeletePositionBuffer struct { + mu sync.Mutex + bitmap *dv.RoaringPositionBitmap + invalidPosition int64 +} + +func (b *equalityDeletePositionBuffer) addPositions(positions []int64) { + if len(positions) == 0 { + return + } + + b.mu.Lock() + defer b.mu.Unlock() + + if b.invalidPosition < 0 { + return + } + for _, position := range positions { + if position < 0 { + b.invalidPosition = position + + return + } + } + + if b.bitmap == nil { + b.bitmap = dv.NewRoaringPositionBitmap() + } + for _, position := range positions { + b.bitmap.Set(uint64(position)) + } +} + +// EqualityDeleteToDVResult summarizes an equality-delete conversion. +type EqualityDeleteToDVResult struct { + // Table is the table returned by the table-level action. It is nil for + // transaction callers, including after the transaction commits. + Table *Table + + // ConvertedEqualityDeleteFiles is the number of live equality delete files + // removed by the rewrite. + ConvertedEqualityDeleteFiles int + // AddedDeletionVectors is the number of file-specific deletion vectors added. + AddedDeletionVectors int + // RemovedPositionDeleteFiles is the number of legacy position delete files + // scoped to one data file, either by referenced_data_file or equal + // file_path bounds, removed with the conversion. + RemovedPositionDeleteFiles int + // ReplacedDeletionVectors is the number of existing vectors merged into new + // vectors and removed by the rewrite. + ReplacedDeletionVectors int +} + +// ConvertEqualityDeletesToDeletionVectors converts live equality deletes on +// the main branch of a v3 table to file-specific deletion vectors. The rewrite +// commits as one replace snapshot so changelog readers do not see a second +// logical deletion event. See [Transaction.ConvertEqualityDeletesToDeletionVectors] +// for its limitations. +func (t Table) ConvertEqualityDeletesToDeletionVectors(ctx context.Context) (*EqualityDeleteToDVResult, error) { + txn, err := t.NewTransactionOnBranch(MainBranch) + if err != nil { + return nil, err + } + + result, err := txn.ConvertEqualityDeletesToDeletionVectors(ctx) + if err != nil { + return result, err + } + if result.ConvertedEqualityDeleteFiles == 0 { + result.Table = &t + + return result, nil + } + + committed, err := txn.Commit(ctx) + if err == nil { + result.Table = committed + } + + return result, err +} + +// ConvertEqualityDeletesToDeletionVectors stages an equality-delete to +// deletion-vector conversion on the transaction's branch. Commit the +// transaction to publish the replace snapshot. +// +// The conversion reads every data file targeted by an applicable equality +// delete. It merges file-scoped position deletes and existing vectors, but is +// not supported when an applicable unscoped or partition-scoped position +// delete would remain live. A concurrent commit is not replayed against a +// refreshed conversion plan and may make Commit return [ErrCommitFailed]. If a +// transaction is abandoned after staging, its Puffin file may remain for +// orphan cleanup. +func (t *Transaction) ConvertEqualityDeletesToDeletionVectors(ctx context.Context) (*EqualityDeleteToDVResult, error) { + meta, err := t.txnMeta() + if err != nil { + return nil, err + } + if meta.formatVersion != 3 { + return nil, fmt.Errorf("%w: equality-delete to deletion-vector conversion requires a v3 table, got v%d", + iceberg.ErrNotImplemented, meta.formatVersion) + } + + snapshot := t.planningSnapshot(meta) + if snapshot == nil { + return &EqualityDeleteToDVResult{}, nil + } + + fs, err := t.tbl.FS(ctx) + if err != nil { + return nil, err + } + manifests, err := snapshot.Manifests(ctx, fs) + if err != nil { + return nil, err + } + equalityDeleteFiles := make(map[string]iceberg.DataFile) + for _, manifest := range manifests { + if err := ctx.Err(); err != nil { + return nil, err + } + if manifest.ManifestContent() != iceberg.ManifestContentDeletes { + continue + } + for entry, err := range manifest.Entries(ctx, fs, true) { + if err != nil { + return nil, err + } + file := entry.DataFile() + if file.ContentType() != iceberg.EntryContentEqDeletes { + continue + } + if _, exists := equalityDeleteFiles[file.FilePath()]; exists { + return nil, fmt.Errorf("%w: duplicate live equality delete file %s", + ErrInvalidMetadata, file.FilePath()) + } + equalityDeleteFiles[file.FilePath()] = file + } + } + if len(equalityDeleteFiles) == 0 { + return &EqualityDeleteToDVResult{}, nil + } + // Only projected equality keys are needed to resolve their matching row + // positions. Missing (dropped) keys are handled by the delete loader. + keyNames := make(map[string]struct{}) + for _, file := range equalityDeleteFiles { + for _, id := range file.EqualityFieldIDs() { + if name, ok := meta.CurrentSchema().FindColumnName(id); ok { + keyNames[name] = struct{}{} + } + } + } + selectedFields := make([]string, 0, len(keyNames)) + for name := range keyNames { + selectedFields = append(selectedFields, name) + } + slices.Sort(selectedFields) + scanOpts := []ScanOption{WithScanPlanningMode(ScanPlanningLocal)} + // WithSelectedFields with no names selects every column. Keep that + // fallback when all equality keys were dropped from the current schema. + if len(selectedFields) > 0 { + scanOpts = append(scanOpts, WithSelectedFields(selectedFields...)) + } + + scan, err := t.Scan(scanOpts...) + if err != nil { + return nil, err + } + if t.branch != "" && t.branch != MainBranch && scan.metadata.SnapshotByName(t.branch) != nil { + branchScan, err := scan.UseRef(t.branch) + _ = scan.Close() + if err != nil { + return nil, err + } + scan = branchScan + } + defer scan.Close() + + plannedTasks, err := scan.PlanFiles(ctx) + if err != nil { + return nil, err + } + baseTasks := make([]FileScanTask, 0, len(plannedTasks)) + addedDeleteTasks := make([]FileScanTask, 0, len(plannedTasks)) + positionsByPath := make(map[string]*equalityDeletePositionBuffer) + positionSinks := make(map[string]func([]int64)) + addedTaskByPath := make(map[string]int) + for _, task := range plannedTasks { + if len(task.EqualityDeleteFiles) == 0 { + continue + } + if task.File == nil { + return nil, fmt.Errorf("%w: scan task with equality deletes has no data file", ErrInvalidMetadata) + } + path := task.File.FilePath() + if _, exists := positionsByPath[path]; !exists { + positions := &equalityDeletePositionBuffer{} + positionsByPath[path] = positions + positionSinks[path] = positions.addPositions + } + + base := task + base.EqualityDeleteFiles = nil + baseTasks = append(baseTasks, base) + + if _, exists := addedTaskByPath[path]; !exists { + added := task + added.DeleteFiles = nil + added.DeletionVectorFiles = nil + addedTaskByPath[path] = len(addedDeleteTasks) + addedDeleteTasks = append(addedDeleteTasks, added) + } + } + + if len(baseTasks) > 0 { + _, records, err := scan.readFileTasksWithAddedDeletes(ctx, baseTasks, addedDeleteTasks, positionSinks) + if err != nil { + return nil, err + } + for record, err := range records { + if err != nil { + return nil, err + } + record.Release() + } + } + + // A row-position cursor should never produce negative positions. Unlike + // DVWriter.Add, DVWriter.Load accepts an unsigned bitmap, so reject + // invalid positions before they can wrap to huge uint64 values. + for path, positions := range positionsByPath { + if positions.invalidPosition < 0 { + return nil, fmt.Errorf("%w: negative equality-delete row position %d for data file %s", + iceberg.ErrInvalidArgument, positions.invalidPosition, path) + } + } + + writeFS, err := requireWriteFileIO(fs) + if err != nil { + return nil, err + } Review Comment: a read-only fileio only fails here, after every affected data file was read please move this above the scan: tables with no equality deletes still return early before it -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: [email protected] For queries about this service, please contact Infrastructure at: [email protected] --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
