This is an automated email from the ASF dual-hosted git repository.
laskoviymishka pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/iceberg-go.git
The following commit(s) were added to refs/heads/main by this push:
new 5f5afa0c6 perf(rest): reuse delete reference markers during scan task
decode (#2136)
5f5afa0c6 is described below
commit 5f5afa0c67e030bbb5788915ab6ab7ee7fd4afcb
Author: Minh Vu <[email protected]>
AuthorDate: Wed Oct 7 13:53:30 2026 +0200
perf(rest): reuse delete reference markers during scan task decode (#2136)
perf(rest): reuse delete reference markers
---
catalog/rest/scan_task_decoder.go | 14 +++++++-------
catalog/rest/scan_task_decoder_test.go | 21 +++++++++++++++++++++
2 files changed, 28 insertions(+), 7 deletions(-)
diff --git a/catalog/rest/scan_task_decoder.go
b/catalog/rest/scan_task_decoder.go
index 36289c43e..1ed3900a5 100644
--- a/catalog/rest/scan_task_decoder.go
+++ b/catalog/rest/scan_task_decoder.go
@@ -160,7 +160,7 @@ func decodeScanTasks(
out := make([]table.FileScanTask, 0, len(wire.FileScanTasks))
dvOwners := make(map[int]string)
- referencedDeletes := make([]bool, len(deletes))
+ deleteRefTaskIDs := make([]int, len(deletes))
for i := range wire.FileScanTasks {
wireTask := wire.FileScanTasks[i]
if wireTask.DataFile == nil {
@@ -187,20 +187,20 @@ func decodeScanTasks(
// The REST FileScanTask schema does not carry manifest data
sequence
// numbers, so DataSequenceNumber intentionally remains nil.
- seenRefs := make(map[int]struct{},
len(wireTask.DeleteFileReferences))
+ // Use one-based task IDs so zero remains the unreferenced
marker.
+ taskID := i + 1
for j, ref := range wireTask.DeleteFileReferences {
if ref < 0 || ref >= len(deletes) {
return nil, fmt.Errorf(
"%w: decoding scan tasks:
file-scan-tasks[%d].delete-file-references[%d] is %d, want 0 <= index < %d",
ErrRESTError, i, j, ref, len(deletes))
}
- if _, ok := seenRefs[ref]; ok {
+ if deleteRefTaskIDs[ref] == taskID {
return nil, fmt.Errorf(
"%w: decoding scan tasks:
file-scan-tasks[%d] repeats delete-file reference %d",
ErrRESTError, i, ref)
}
- seenRefs[ref] = struct{}{}
- referencedDeletes[ref] = true
+ deleteRefTaskIDs[ref] = taskID
wireDelete := wire.DeleteFiles[ref]
if wireDelete.ReferencedDataFile != nil &&
*wireDelete.ReferencedDataFile != dataFile.FilePath() {
@@ -244,8 +244,8 @@ func decodeScanTasks(
out = append(out, task)
}
- for i, referenced := range referencedDeletes {
- if !referenced {
+ for i, taskID := range deleteRefTaskIDs {
+ if taskID == 0 {
return nil, fmt.Errorf(
"%w: decoding scan tasks: delete-files[%d] is
not referenced by any file scan task",
ErrRESTError, i)
diff --git a/catalog/rest/scan_task_decoder_test.go
b/catalog/rest/scan_task_decoder_test.go
index 75abd6852..7d0022505 100644
--- a/catalog/rest/scan_task_decoder_test.go
+++ b/catalog/rest/scan_task_decoder_test.go
@@ -237,6 +237,27 @@ func
TestDecodeScanTasksKeepsDeleteReferencesEnvelopeLocal(t *testing.T) {
assert.Equal(t, "s3://bucket/table/second-delete.parquet",
secondTasks[0].DeleteFiles[0].FilePath())
}
+func TestDecodeScanTasksAllowsDeleteReferenceAcrossTasks(t *testing.T) {
+ t.Parallel()
+
+ metadata := newScanTaskDecoderMetadata()
+ wire := validScanTasksWire()
+ secondDataFile := *wire.FileScanTasks[0].DataFile
+ secondDataFile.FilePath = "s3://bucket/table/second-data.parquet"
+ wire.FileScanTasks = append(wire.FileScanTasks, RESTFileScanTask{
+ DataFile: &secondDataFile,
+ DeleteFileReferences: []int{0},
+ })
+
+ tasks, err := DecodeScanTasks(wire, metadata, metadata.schema, nil)
+ require.NoError(t, err)
+ require.Len(t, tasks, 2)
+ require.Len(t, tasks[0].DeleteFiles, 1)
+ require.Len(t, tasks[1].DeleteFiles, 1)
+ assert.Equal(t, "s3://bucket/table/delete.parquet",
tasks[0].DeleteFiles[0].FilePath())
+ assert.Equal(t, "s3://bucket/table/delete.parquet",
tasks[1].DeleteFiles[0].FilePath())
+}
+
func TestDecodeScanTasksReusesPartitionDecodePlan(t *testing.T) {
t.Parallel()
