This is an automated email from the ASF dual-hosted git repository.
laskoviymishka pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/iceberg-go.git
The following commit(s) were added to refs/heads/main by this push:
new b5bb5b547 perf(table): avoid redundant filter cache schema traversal
(#2028)
b5bb5b547 is described below
commit b5bb5b547d48a09c564be649807b6cde4bc711a2
Author: Minh Vu <[email protected]>
AuthorDate: Wed Sep 23 19:48:23 2026 +0200
perf(table): avoid redundant filter cache schema traversal (#2028)
* perf(table): avoid redundant filter cache schema traversal
Signed-off-by: Minh Vu <[email protected]>
* test(table): benchmark physical filter schema keys
Signed-off-by: Minh Vu <[email protected]>
* perf(table): compact physical schema key types
Signed-off-by: Minh Vu <[email protected]>
* fix(table): preserve unexpected schema key panics
Recover only explicitly marked invalid inputs, retain typed-nil rejection,
and cover compact type-tag uniqueness and panic propagation.
Signed-off-by: Minh Vu <[email protected]>
---------
Signed-off-by: Minh Vu <[email protected]>
---
table/arrow_filter_plan.go | 125 +++++++++++++++++++++------------
table/arrow_filter_plan_key_test.go | 91 ++++++++++++++++++++++++
table/arrow_filter_plan_test.go | 135 ++++++++++++++++++++++++++++++++++++
table/arrow_scanner_bench_test.go | 91 ++++++++++++++++++++++++
4 files changed, 399 insertions(+), 43 deletions(-)
diff --git a/table/arrow_filter_plan.go b/table/arrow_filter_plan.go
index e839ce820..0c4227c82 100644
--- a/table/arrow_filter_plan.go
+++ b/table/arrow_filter_plan.go
@@ -20,6 +20,7 @@ package table
import (
"context"
"fmt"
+ "reflect"
"strconv"
"strings"
"sync"
@@ -27,6 +28,7 @@ import (
"github.com/apache/arrow-go/v18/arrow/compute/exprs"
"github.com/apache/arrow-go/v18/parquet/metadata"
"github.com/apache/iceberg-go"
+ iceinternal "github.com/apache/iceberg-go/internal"
tblutils "github.com/apache/iceberg-go/table/internal"
"github.com/apache/iceberg-go/table/substrait"
"github.com/substrait-io/substrait-go/v8/expr"
@@ -160,51 +162,33 @@ func (as *arrowScan) cachedFileFilterPlans(fileSchema
*iceberg.Schema, includePr
return plans, nil
}
-func physicalSchemaKey(fileSchema *iceberg.Schema) (string, error) {
- // The key includes field names, IDs, requiredness, and type details.
Schema
- // IDs, docs, and defaults do not affect filter translation or
compilation.
- visitor := &physicalSchemaKeyVisitor{}
- _, err := iceberg.Visit(fileSchema, visitor)
- if err != nil {
- return "", fmt.Errorf("%w: cannot encode physical schema key:
%v", iceberg.ErrInvalidSchema, err)
- }
-
- return visitor.builder.String(), nil
-}
+// physicalSchemaKeyPanic marks invalid input, not unexpected implementation
panics.
+type physicalSchemaKeyPanic string
-type physicalSchemaKeyVisitor struct {
- builder strings.Builder
- depth int
-}
-
-func (v *physicalSchemaKeyVisitor) Schema(_ *iceberg.Schema, _ struct{})
struct{} { return struct{}{} }
-func (v *physicalSchemaKeyVisitor) Struct(_ iceberg.StructType, _ []struct{})
struct{} {
- return struct{}{}
-}
+func physicalSchemaKey(fileSchema *iceberg.Schema) (key string, err error) {
+ defer func() {
+ if r := recover(); r != nil {
+ invalid, ok := r.(physicalSchemaKeyPanic)
+ if !ok {
+ panic(r)
+ }
+ err = fmt.Errorf("%w: cannot encode physical schema
key: %s", iceberg.ErrInvalidSchema, invalid)
+ }
+ }()
-func (v *physicalSchemaKeyVisitor) Field(_ iceberg.NestedField, _ struct{})
struct{} {
- return struct{}{}
-}
-func (v *physicalSchemaKeyVisitor) List(_ iceberg.ListType, _ struct{})
struct{} { return struct{}{} }
-func (v *physicalSchemaKeyVisitor) Map(_ iceberg.MapType, _, _ struct{})
struct{} { return struct{}{} }
-func (v *physicalSchemaKeyVisitor) Primitive(_ iceberg.PrimitiveType) struct{}
{ return struct{}{} }
-func (v *physicalSchemaKeyVisitor) Variant(_ iceberg.VariantType) struct{}
{ return struct{}{} }
-
-func (v *physicalSchemaKeyVisitor) BeforeField(field iceberg.NestedField) {
- if v.depth == 0 {
- writePhysicalFieldKey(&v.builder, field)
+ if fileSchema == nil {
+ panic(physicalSchemaKeyPanic("nil schema"))
}
- v.depth++
-}
-func (v *physicalSchemaKeyVisitor) AfterField(_ iceberg.NestedField) {
v.depth-- }
+ // The key includes field names, IDs, requiredness, and type details.
Schema
+ // IDs, docs, and defaults do not affect filter translation or
compilation.
+ var builder strings.Builder
+ for _, field := range fileSchema.FieldsRef(iceinternal.SchemaRef{}) {
+ writePhysicalFieldKey(&builder, field)
+ }
-func (v *physicalSchemaKeyVisitor) BeforeListElement(_ iceberg.NestedField) {
v.depth++ }
-func (v *physicalSchemaKeyVisitor) AfterListElement(_ iceberg.NestedField) {
v.depth-- }
-func (v *physicalSchemaKeyVisitor) BeforeMapKey(_ iceberg.NestedField) {
v.depth++ }
-func (v *physicalSchemaKeyVisitor) AfterMapKey(_ iceberg.NestedField) {
v.depth-- }
-func (v *physicalSchemaKeyVisitor) BeforeMapValue(_ iceberg.NestedField) {
v.depth++ }
-func (v *physicalSchemaKeyVisitor) AfterMapValue(_ iceberg.NestedField) {
v.depth-- }
+ return builder.String(), nil
+}
func writePhysicalFieldKey(builder *strings.Builder, field
iceberg.NestedField) {
builder.WriteByte('f')
@@ -219,8 +203,13 @@ func writePhysicalFieldKey(builder *strings.Builder, field
iceberg.NestedField)
}
func writePhysicalTypeKey(builder *strings.Builder, typ iceberg.Type) {
+ // Type tags are internal to the cache key. Parameterized types append
their
+ // parameters so equal physical layouts still produce equal keys.
switch t := typ.(type) {
case *iceberg.StructType:
+ if t == nil {
+ panic(physicalSchemaKeyPanic("nil struct type"))
+ }
builder.WriteByte('s')
builder.WriteByte('{')
for _, field := range t.FieldList {
@@ -228,6 +217,9 @@ func writePhysicalTypeKey(builder *strings.Builder, typ
iceberg.Type) {
}
builder.WriteByte('}')
case *iceberg.ListType:
+ if t == nil {
+ panic(physicalSchemaKeyPanic("nil list type"))
+ }
builder.WriteByte('l')
writePhysicalInt(builder, t.ElementID)
if t.ElementRequired {
@@ -237,6 +229,9 @@ func writePhysicalTypeKey(builder *strings.Builder, typ
iceberg.Type) {
}
writePhysicalTypeKey(builder, t.Element)
case *iceberg.MapType:
+ if t == nil {
+ panic(physicalSchemaKeyPanic("nil map type"))
+ }
builder.WriteByte('m')
writePhysicalInt(builder, t.KeyID)
writePhysicalTypeKey(builder, t.KeyType)
@@ -247,11 +242,55 @@ func writePhysicalTypeKey(builder *strings.Builder, typ
iceberg.Type) {
builder.WriteByte('0')
}
writePhysicalTypeKey(builder, t.ValueType)
- case nil:
- builder.WriteString("n")
- default:
+ case iceberg.BooleanType:
+ builder.WriteByte('b')
+ case iceberg.Int32Type:
+ builder.WriteByte('i')
+ case iceberg.Int64Type:
+ builder.WriteByte('j')
+ case iceberg.Float32Type:
+ builder.WriteByte('k')
+ case iceberg.Float64Type:
+ builder.WriteByte('d')
+ case iceberg.DateType:
+ builder.WriteByte('D')
+ case iceberg.TimeType:
+ builder.WriteByte('T')
+ case iceberg.TimestampType:
+ builder.WriteByte('t')
+ case iceberg.TimestampTzType:
+ builder.WriteByte('z')
+ case iceberg.StringType:
+ builder.WriteByte('S')
+ case iceberg.UUIDType:
+ builder.WriteByte('u')
+ case iceberg.BinaryType:
+ builder.WriteByte('B')
+ case iceberg.TimestampNsType:
+ builder.WriteByte('n')
+ case iceberg.TimestampTzNsType:
+ builder.WriteByte('N')
+ case iceberg.UnknownType:
+ builder.WriteByte('U')
+ case iceberg.VariantType:
+ builder.WriteByte('v')
+ case iceberg.FixedType:
+ builder.WriteByte('F')
+ writePhysicalInt(builder, t.Len())
+ case iceberg.DecimalType:
+ builder.WriteByte('q')
+ writePhysicalInt(builder, t.Precision())
+ writePhysicalInt(builder, t.Scale())
+ case iceberg.PrimitiveType:
+ if value := reflect.ValueOf(t); value.Kind() == reflect.Pointer
&& value.IsNil() {
+ panic(physicalSchemaKeyPanic("nil primitive type"))
+ }
+ // Keep custom and parameterized primitive types (for example
geometry)
+ // compatible with the previous structural key encoding.
builder.WriteByte('p')
writePhysicalString(builder, typ.String())
+ default:
+ panic(physicalSchemaKeyPanic(fmt.Sprintf("unsupported physical
type: %T", typ)))
}
}
diff --git a/table/arrow_filter_plan_key_test.go
b/table/arrow_filter_plan_key_test.go
new file mode 100644
index 000000000..838cc51cb
--- /dev/null
+++ b/table/arrow_filter_plan_key_test.go
@@ -0,0 +1,91 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package table
+
+import (
+ "testing"
+
+ "github.com/apache/iceberg-go"
+ "github.com/stretchr/testify/require"
+)
+
+func TestPhysicalSchemaKeyDistinctTypeTags(t *testing.T) {
+ types := []iceberg.Type{
+ iceberg.BooleanType{},
+ iceberg.Int32Type{},
+ iceberg.Int64Type{},
+ iceberg.Float32Type{},
+ iceberg.Float64Type{},
+ iceberg.DateType{},
+ iceberg.TimeType{},
+ iceberg.TimestampType{},
+ iceberg.TimestampTzType{},
+ iceberg.TimestampNsType{},
+ iceberg.TimestampTzNsType{},
+ iceberg.StringType{},
+ iceberg.UUIDType{},
+ iceberg.BinaryType{},
+ iceberg.UnknownType{},
+ iceberg.VariantType{},
+ iceberg.FixedTypeOf(8),
+ iceberg.FixedTypeOf(16),
+ iceberg.DecimalTypeOf(10, 2),
+ iceberg.DecimalTypeOf(11, 2),
+ iceberg.DecimalTypeOf(10, 3),
+ }
+ keys := make(map[string]string, len(types))
+ for _, typ := range types {
+ t.Run(typ.String(), func(t *testing.T) {
+ schema := iceberg.NewSchema(1, iceberg.NestedField{ID:
1, Name: "value", Type: typ})
+ key, err := physicalSchemaKey(schema)
+ require.NoError(t, err)
+ require.NotContains(t, keys, key, "type %s collides
with %s", typ, keys[key])
+ keys[key] = typ.String()
+ })
+ }
+}
+
+func TestPhysicalSchemaKeyRejectsNilPrimitives(t *testing.T) {
+ for _, typ := range []iceberg.Type{
+ (*iceberg.Int32Type)(nil),
+ (*iceberg.FixedType)(nil),
+ (*iceberg.DecimalType)(nil),
+ } {
+ key, err := physicalSchemaKey(iceberg.NewSchema(1,
+ iceberg.NestedField{ID: 1, Name: "invalid", Type: typ}))
+ require.ErrorIs(t, err, iceberg.ErrInvalidSchema)
+ require.Empty(t, key)
+ }
+}
+
+type panickingPhysicalPrimitive struct {
+ iceberg.StringType
+}
+
+func (panickingPhysicalPrimitive) String() string {
+ panic("unexpected primitive panic")
+}
+
+func TestPhysicalSchemaKeyPreservesUnexpectedPanics(t *testing.T) {
+ schema := iceberg.NewSchema(1, iceberg.NestedField{
+ ID: 1, Name: "value", Type: panickingPhysicalPrimitive{},
+ })
+ require.PanicsWithValue(t, "unexpected primitive panic", func() {
+ _, _ = physicalSchemaKey(schema)
+ })
+}
diff --git a/table/arrow_filter_plan_test.go b/table/arrow_filter_plan_test.go
index 76f3bf87f..c96e726de 100644
--- a/table/arrow_filter_plan_test.go
+++ b/table/arrow_filter_plan_test.go
@@ -201,3 +201,138 @@ func
TestCompiledFileFilterPlansDisablePruningForMissingInitialDefault(t *testin
assert.Empty(t, plans.pruning.bloomPreds)
assert.Empty(t, plans.pruning.dictionaryPreds)
}
+
+func TestPhysicalSchemaKeyIgnoresMetadata(t *testing.T) {
+ fields := []iceberg.NestedField{{
+ ID: 1, Name: "root", Type: &iceberg.StructType{FieldList:
[]iceberg.NestedField{
+ {ID: 2, Name: "value", Type:
iceberg.PrimitiveTypes.Int64, Required: true},
+ }},
+ }}
+ original := iceberg.NewSchema(1, fields...)
+ key, err := physicalSchemaKey(original)
+ require.NoError(t, err)
+
+ fields = original.Fields()
+ fields[0].Doc = "root documentation"
+ child := &fields[0].Type.(*iceberg.StructType).FieldList[0]
+ child.Doc = "value documentation"
+ child.InitialDefault = int64(7)
+ child.WriteDefault = int64(9)
+ withMetadata := iceberg.NewSchemaWithIdentifiers(2, []int{2}, fields...)
+ metadataKey, err := physicalSchemaKey(withMetadata)
+ require.NoError(t, err)
+ assert.Equal(t, key, metadataKey)
+
+ emptyKey, err := physicalSchemaKey(iceberg.NewSchema(1))
+ require.NoError(t, err)
+ assert.Empty(t, emptyKey)
+}
+
+func TestPhysicalSchemaKeyDistinguishesNestedLayouts(t *testing.T) {
+ schema := iceberg.NewSchema(1, iceberg.NestedField{
+ ID: 1, Name: "root", Type: &iceberg.StructType{FieldList:
[]iceberg.NestedField{
+ {ID: 2, Name: "items", Type: &iceberg.ListType{
+ ElementID: 3, Element:
iceberg.PrimitiveTypes.Int64,
+ }},
+ {ID: 4, Name: "lookup", Type: &iceberg.MapType{
+ KeyID: 5, KeyType:
iceberg.PrimitiveTypes.String,
+ ValueID: 6, ValueType:
iceberg.DecimalTypeOf(10, 2),
+ }},
+ {ID: 7, Name: "fixed", Type: iceberg.FixedTypeOf(8)},
+ {ID: 8, Name: "variant", Type: iceberg.VariantType{}},
+ }},
+ })
+ original, err := physicalSchemaKey(schema)
+ require.NoError(t, err)
+
+ for _, tc := range []struct {
+ name string
+ change func(*iceberg.StructType)
+ }{
+ {"field_id", func(s *iceberg.StructType) { s.FieldList[0].ID =
20 }},
+ {"field_name", func(s *iceberg.StructType) {
s.FieldList[0].Name = "items:;{}" }},
+ {"field_required", func(s *iceberg.StructType) {
s.FieldList[0].Required = true }},
+ {"field_order", func(s *iceberg.StructType) { s.FieldList[0],
s.FieldList[1] = s.FieldList[1], s.FieldList[0] }},
+ {"list_id", func(s *iceberg.StructType) {
s.FieldList[0].Type.(*iceberg.ListType).ElementID = 30 }},
+ {"list_required", func(s *iceberg.StructType) {
s.FieldList[0].Type.(*iceberg.ListType).ElementRequired = true }},
+ {"list_type", func(s *iceberg.StructType) {
+ s.FieldList[0].Type.(*iceberg.ListType).Element =
iceberg.PrimitiveTypes.Int32
+ }},
+ {"map_key_id", func(s *iceberg.StructType) {
s.FieldList[1].Type.(*iceberg.MapType).KeyID = 50 }},
+ {"map_key_type", func(s *iceberg.StructType) {
+ s.FieldList[1].Type.(*iceberg.MapType).KeyType =
iceberg.PrimitiveTypes.Int32
+ }},
+ {"map_value_id", func(s *iceberg.StructType) {
s.FieldList[1].Type.(*iceberg.MapType).ValueID = 60 }},
+ {"map_value_required", func(s *iceberg.StructType) {
s.FieldList[1].Type.(*iceberg.MapType).ValueRequired = true }},
+ {"decimal_precision", func(s *iceberg.StructType) {
+ s.FieldList[1].Type.(*iceberg.MapType).ValueType =
iceberg.DecimalTypeOf(11, 2)
+ }},
+ {"decimal_scale", func(s *iceberg.StructType) {
+ s.FieldList[1].Type.(*iceberg.MapType).ValueType =
iceberg.DecimalTypeOf(10, 3)
+ }},
+ {"fixed_size", func(s *iceberg.StructType) {
s.FieldList[2].Type = iceberg.FixedTypeOf(16) }},
+ {"variant_type", func(s *iceberg.StructType) {
s.FieldList[3].Type = iceberg.PrimitiveTypes.Binary }},
+ {"empty_struct", func(s *iceberg.StructType) { s.FieldList =
nil }},
+ } {
+ t.Run(tc.name, func(t *testing.T) {
+ fields := schema.Fields()
+ tc.change(fields[0].Type.(*iceberg.StructType))
+ key, err := physicalSchemaKey(iceberg.NewSchema(1,
fields...))
+ require.NoError(t, err)
+ assert.NotEqual(t, original, key)
+ })
+ }
+}
+
+func TestPhysicalSchemaKeyInvalidSchema(t *testing.T) {
+ for _, tc := range []struct {
+ name string
+ typ iceberg.Type
+ }{
+ {"nil_type", nil},
+ {"nil_struct", (*iceberg.StructType)(nil)},
+ {"nil_list", (*iceberg.ListType)(nil)},
+ {"nil_map", (*iceberg.MapType)(nil)},
+ {"nil_struct_field", &iceberg.StructType{FieldList:
[]iceberg.NestedField{{ID: 2, Name: "child"}}}},
+ {"nil_list_element", &iceberg.ListType{ElementID: 2}},
+ {"nil_map_key", &iceberg.MapType{KeyID: 2, ValueID: 3,
ValueType: iceberg.PrimitiveTypes.String}},
+ {"nil_map_value", &iceberg.MapType{KeyID: 2, KeyType:
iceberg.PrimitiveTypes.String, ValueID: 3}},
+ } {
+ t.Run(tc.name, func(t *testing.T) {
+ key, err := physicalSchemaKey(iceberg.NewSchema(1,
iceberg.NestedField{ID: 1, Name: "invalid", Type: tc.typ}))
+ require.ErrorIs(t, err, iceberg.ErrInvalidSchema)
+ assert.Empty(t, key)
+ })
+ }
+
+ key, err := physicalSchemaKey(nil)
+ require.ErrorIs(t, err, iceberg.ErrInvalidSchema)
+ assert.Empty(t, key)
+
+ _, err = (&arrowScan{}).cachedFileFilterPlans(nil, true)
+ require.ErrorIs(t, err, iceberg.ErrInvalidArgument)
+}
+
+func TestPhysicalSchemaKeySupportsParameterizedPrimitiveTypes(t *testing.T) {
+ geometry, err := iceberg.GeometryTypeOf("srid:4326")
+ require.NoError(t, err)
+ geography, err := iceberg.GeographyTypeOf("srid:4326", "spherical")
+ require.NoError(t, err)
+
+ schema := iceberg.NewSchema(1,
+ iceberg.NestedField{ID: 1, Name: "geometry", Type: geometry},
+ iceberg.NestedField{ID: 2, Name: "geography", Type: geography},
+ )
+ original, err := physicalSchemaKey(schema)
+ require.NoError(t, err)
+
+ otherGeometry, err := iceberg.GeometryTypeOf("srid:3857")
+ require.NoError(t, err)
+ other := iceberg.NewSchema(1,
+ iceberg.NestedField{ID: 1, Name: "geometry", Type:
otherGeometry},
+ iceberg.NestedField{ID: 2, Name: "geography", Type: geography},
+ )
+ changed, err := physicalSchemaKey(other)
+ require.NoError(t, err)
+ assert.NotEqual(t, original, changed)
+}
diff --git a/table/arrow_scanner_bench_test.go
b/table/arrow_scanner_bench_test.go
index 7706938ca..f3a1e5aa3 100644
--- a/table/arrow_scanner_bench_test.go
+++ b/table/arrow_scanner_bench_test.go
@@ -405,6 +405,97 @@ func BenchmarkArrowScanFilterPlanSetup(b *testing.B) {
})
}
+func BenchmarkArrowScanFilterPlanCacheHit(b *testing.B) {
+ for _, tc := range benchmarkPhysicalSchemaCases() {
+ b.Run(tc.name, func(b *testing.B) {
+ scan := &arrowScan{boundRowFilter:
iceberg.AlwaysTrue{}, caseSensitive: true}
+ if _, err := scan.cachedFileFilterPlans(tc.schema,
true); err != nil {
+ b.Fatal(err)
+ }
+
+ b.ReportAllocs()
+ for b.Loop() {
+ plans, err :=
scan.cachedFileFilterPlans(tc.schema, true)
+ if err != nil {
+ b.Fatal(err)
+ }
+ benchmarkCompiledFilterPlansSink = plans
+ }
+ })
+ }
+}
+
+var benchmarkPhysicalSchemaKeySink string
+
+func BenchmarkPhysicalSchemaKey(b *testing.B) {
+ for _, tc := range benchmarkPhysicalSchemaCases() {
+ b.Run(tc.name, func(b *testing.B) {
+ b.ReportAllocs()
+ b.ResetTimer()
+ for b.Loop() {
+ key, err := physicalSchemaKey(tc.schema)
+ if err != nil {
+ b.Fatal(err)
+ }
+ benchmarkPhysicalSchemaKeySink = key
+ }
+ })
+ }
+}
+
+func benchmarkPhysicalSchemaCases() []struct {
+ name string
+ schema *iceberg.Schema
+} {
+ flatFields := make([]iceberg.NestedField, 10)
+ for i := range flatFields {
+ flatFields[i] = iceberg.NestedField{
+ ID: i + 1, Name: fmt.Sprintf("field_%d", i), Type:
iceberg.PrimitiveTypes.Int64,
+ }
+ }
+ wideFields := make([]iceberg.NestedField, 100)
+ for i := range wideFields {
+ wideFields[i] = iceberg.NestedField{
+ ID: i + 2, Name: fmt.Sprintf("field_%d", i), Type:
iceberg.PrimitiveTypes.Int64,
+ }
+ }
+ mixedFields := []iceberg.NestedField{
+ {ID: 1, Name: "flag", Type: iceberg.PrimitiveTypes.Bool},
+ {ID: 2, Name: "count", Type: iceberg.PrimitiveTypes.Int32},
+ {ID: 3, Name: "ratio", Type: iceberg.PrimitiveTypes.Float64},
+ {ID: 4, Name: "fixed", Type: iceberg.FixedTypeOf(16)},
+ {ID: 5, Name: "price", Type: iceberg.DecimalTypeOf(20, 4)},
+ {ID: 6, Name: "created", Type:
iceberg.PrimitiveTypes.TimestampTzNs},
+ {ID: 7, Name: "payload", Type: iceberg.VariantType{}},
+ {ID: 8, Name: "tags", Type: &iceberg.ListType{
+ ElementID: 9, Element: iceberg.PrimitiveTypes.String,
+ }},
+ }
+ nestedSchema := func(depth int) *iceberg.Schema {
+ var typ iceberg.Type = iceberg.PrimitiveTypes.Int64
+ for id := depth + 1; id > 1; id-- {
+ typ = &iceberg.StructType{FieldList:
[]iceberg.NestedField{
+ {ID: id, Name: "child", Type: typ},
+ }}
+ }
+
+ return iceberg.NewSchema(1, iceberg.NestedField{ID: 1, Name:
"root", Type: typ})
+ }
+
+ return []struct {
+ name string
+ schema *iceberg.Schema
+ }{
+ {"flat_10", iceberg.NewSchema(1, flatFields...)},
+ {"nested_depth_3", nestedSchema(3)},
+ {"nested_depth_8", nestedSchema(8)},
+ {"wide_struct_100", iceberg.NewSchema(1, iceberg.NestedField{
+ ID: 1, Name: "root", Type:
&iceberg.StructType{FieldList: wideFields},
+ })},
+ {"mixed_types", iceberg.NewSchema(1, mixedFields...)},
+ }
+}
+
func BenchmarkArrowScanAddTaskProjectedFieldIDs(b *testing.B) {
schema := benchmarkScanSchema(8)
filter := benchmarkComplexBoundFilter(b, schema)
