This is an automated email from the ASF dual-hosted git repository.
zeroshade pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/iceberg-go.git
The following commit(s) were added to refs/heads/main by this push:
new 65da9a886 fix!: store binary and fixed defaults as hex strings in
schemas (#2372)
65da9a886 is described below
commit 65da9a886fec8126fd5ebe9a506d78a8949374fe
Author: Revanth <[email protected]>
AuthorDate: Thu Nov 5 11:11:59 2026 -0600
fix!: store binary and fixed defaults as hex strings in schemas (#2372)
* refactor!: store binary and fixed defaults as hex strings in schemas
* docs: scope the hex default conversion to binary and fixed fields
---
defaults.go | 113 ++++++++++++++++++++++++++++++++++++++++-
defaults_test.go | 98 +++++++++++++++++++++++++++++++++++
schema.go | 7 +++
schema_test.go | 3 +-
table/metadata_getters_test.go | 2 -
table/update_schema_test.go | 44 ++++++++++++++++
types.go | 12 +----
view/metadata_test.go | 6 +--
website/src/releases.md | 1 +
9 files changed, 267 insertions(+), 19 deletions(-)
diff --git a/defaults.go b/defaults.go
index db18843ed..1c655e50a 100644
--- a/defaults.go
+++ b/defaults.go
@@ -17,7 +17,11 @@
package iceberg
-import "slices"
+import (
+ "slices"
+
+ "github.com/apache/iceberg-go/internal"
+)
// CloneDefaultValue returns a deep copy of an Iceberg initial or write
default value.
func CloneDefaultValue(value any) any {
@@ -49,3 +53,110 @@ func CloneDefaultValue(value any) any {
return value
}
}
+
+// hexDefault returns the hex single-value form of a binary or fixed default
+// given as bytes. It reports false for any other type or value, including a
+// default that is already a string.
+func hexDefault(typ Type, value any) (string, bool) {
+ switch typ.(type) {
+ case BinaryType, FixedType:
+ switch value := value.(type) {
+ case []byte:
+ return internal.EncodeDefaultBytes(value), true
+ case BinaryLiteral:
+ return internal.EncodeDefaultBytes(value), true
+ case FixedLiteral:
+ return internal.EncodeDefaultBytes(value), true
+ }
+ }
+
+ return "", false
+}
+
+// normalizeDefaults stores binary and fixed defaults in their hex single-value
+// form, the same form metadata JSON decodes to, so the default of a binary or
+// fixed field no longer shares a byte slice with the caller. Byte slices
+// inside other defaults, such as a []byte in a []any or map[string]any
+// default, are left as they are. When something is rewritten it
+// copies the fields and nested types on the path to that default and reports
+// true; otherwise it returns fields itself. The input is never modified.
+func normalizeDefaults(fields []NestedField) ([]NestedField, bool) {
+ var out []NestedField
+ for i, field := range fields {
+ normalized, ok := normalizeFieldDefaults(field)
+ if !ok {
+ continue
+ }
+ if out == nil {
+ out = slices.Clone(fields)
+ }
+ out[i] = normalized
+ }
+ if out == nil {
+ return fields, false
+ }
+
+ return out, true
+}
+
+func normalizeFieldDefaults(field NestedField) (NestedField, bool) {
+ changed := false
+ if value, ok := hexDefault(field.Type, field.InitialDefault); ok {
+ field.InitialDefault, changed = value, true
+ }
+ if value, ok := hexDefault(field.Type, field.WriteDefault); ok {
+ field.WriteDefault, changed = value, true
+ }
+ if typ, ok := normalizeTypeDefaults(field.Type); ok {
+ field.Type, changed = typ, true
+ }
+
+ return field, changed
+}
+
+func normalizeTypeDefaults(typ Type) (Type, bool) {
+ switch typ := typ.(type) {
+ case *StructType:
+ if typ == nil {
+ return nil, false
+ }
+ fields, ok := normalizeDefaults(typ.FieldList)
+ if !ok {
+ return nil, false
+ }
+
+ return &StructType{FieldList: fields}, true
+ case *ListType:
+ if typ == nil {
+ return nil, false
+ }
+ elem, ok := normalizeTypeDefaults(typ.Element)
+ if !ok {
+ return nil, false
+ }
+ cloned := *typ
+ cloned.Element = elem
+
+ return &cloned, true
+ case *MapType:
+ if typ == nil {
+ return nil, false
+ }
+ key, keyOK := normalizeTypeDefaults(typ.KeyType)
+ value, valueOK := normalizeTypeDefaults(typ.ValueType)
+ if !keyOK && !valueOK {
+ return nil, false
+ }
+ cloned := *typ
+ if keyOK {
+ cloned.KeyType = key
+ }
+ if valueOK {
+ cloned.ValueType = value
+ }
+
+ return &cloned, true
+ }
+
+ return nil, false
+}
diff --git a/defaults_test.go b/defaults_test.go
index 50f91b023..b4298995b 100644
--- a/defaults_test.go
+++ b/defaults_test.go
@@ -18,6 +18,7 @@
package iceberg_test
import (
+ "encoding/json"
"testing"
iceberg "github.com/apache/iceberg-go"
@@ -51,3 +52,100 @@ func TestCloneDefaultValue(t *testing.T) {
assert.NotContains(t, original, "added")
assert.NotContains(t, original["nested"].([]any)[0], "added")
}
+
+func TestNewSchemaStoresByteDefaultsAsHex(t *testing.T) {
+ tests := []struct {
+ name string
+ typ iceberg.Type
+ value any
+ }{
+ {"binary bytes", iceberg.PrimitiveTypes.Binary, []byte{0x0a,
0xff}},
+ {"binary literal", iceberg.PrimitiveTypes.Binary,
iceberg.BinaryLiteral{0x0a, 0xff}},
+ {"fixed bytes", iceberg.MustFixedTypeOf(2), []byte{0x0a, 0xff}},
+ {"fixed literal", iceberg.MustFixedTypeOf(2),
iceberg.FixedLiteral{0x0a, 0xff}},
+ {"hex string", iceberg.PrimitiveTypes.Binary, "0aff"},
+ }
+
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ sc := iceberg.NewSchema(0, iceberg.NestedField{
+ ID: 1, Name: "b", Type: tt.typ, InitialDefault:
tt.value, WriteDefault: tt.value,
+ })
+
+ field := sc.Field(0)
+ assert.Equal(t, "0aff", field.InitialDefault)
+ assert.Equal(t, "0aff", field.WriteDefault)
+ })
+ }
+}
+
+func TestNewSchemaLeavesOtherDefaultsAlone(t *testing.T) {
+ sc := iceberg.NewSchema(0,
+ iceberg.NestedField{ID: 1, Name: "s", Type:
iceberg.PrimitiveTypes.String, InitialDefault: "0aff"},
+ iceberg.NestedField{ID: 2, Name: "i", Type:
iceberg.PrimitiveTypes.Int32, InitialDefault: int32(7)},
+ iceberg.NestedField{ID: 3, Name: "n", Type:
iceberg.PrimitiveTypes.Binary},
+ )
+
+ assert.Equal(t, "0aff", sc.Field(0).InitialDefault)
+ assert.Equal(t, int32(7), sc.Field(1).InitialDefault)
+ assert.Nil(t, sc.Field(2).InitialDefault)
+}
+
+// A schema built in process must equal the same schema read back from
+// metadata, where binary and fixed defaults are hex strings.
+func TestNewSchemaByteDefaultsSurviveJSONRoundTrip(t *testing.T) {
+ sc := iceberg.NewSchema(0,
+ iceberg.NestedField{ID: 1, Name: "b", Type:
iceberg.PrimitiveTypes.Binary, InitialDefault: []byte{1, 2}},
+ iceberg.NestedField{ID: 2, Name: "f", Type:
iceberg.MustFixedTypeOf(2), WriteDefault: iceberg.FixedLiteral{3, 4}},
+ )
+
+ data, err := json.Marshal(sc)
+ require.NoError(t, err)
+
+ var decoded iceberg.Schema
+ require.NoError(t, json.Unmarshal(data, &decoded))
+ assert.True(t, sc.Equals(&decoded), "built: %v, decoded: %v",
sc.Fields(), decoded.Fields())
+}
+
+func TestNewSchemaByteDefaultIsIsolatedFromCaller(t *testing.T) {
+ value := []byte{1, 2}
+ sc := iceberg.NewSchema(0, iceberg.NestedField{
+ ID: 1, Name: "b", Type: iceberg.PrimitiveTypes.Binary,
InitialDefault: value,
+ })
+
+ value[0] = 9
+ assert.Equal(t, "0102", sc.Field(0).InitialDefault)
+}
+
+func TestNewSchemaNormalizesNestedByteDefaultsWithoutModifyingCaller(t
*testing.T) {
+ inner := &iceberg.StructType{FieldList: []iceberg.NestedField{
+ {ID: 5, Name: "payload", Type: iceberg.PrimitiveTypes.Binary,
InitialDefault: []byte{1, 2}},
+ {ID: 6, Name: "label", Type: iceberg.PrimitiveTypes.String,
InitialDefault: "x"},
+ }}
+ list := &iceberg.ListType{ElementID: 4, Element: inner,
ElementRequired: true}
+ mapType := &iceberg.MapType{
+ KeyID: 2, KeyType: iceberg.PrimitiveTypes.String,
+ ValueID: 3, ValueType: list, ValueRequired: true,
+ }
+ fields := []iceberg.NestedField{
+ {ID: 1, Name: "m", Type: mapType},
+ {ID: 7, Name: "top", Type: iceberg.PrimitiveTypes.Binary,
WriteDefault: []byte{3}},
+ }
+
+ sc := iceberg.NewSchema(0, fields...)
+
+ payload, ok := sc.FindFieldByID(5)
+ require.True(t, ok)
+ assert.Equal(t, "0102", payload.InitialDefault)
+ label, ok := sc.FindFieldByID(6)
+ require.True(t, ok)
+ assert.Equal(t, "x", label.InitialDefault)
+ top, ok := sc.FindFieldByID(7)
+ require.True(t, ok)
+ assert.Equal(t, "03", top.WriteDefault)
+
+ assert.Equal(t, []byte{1, 2}, inner.FieldList[0].InitialDefault)
+ assert.Same(t, list, mapType.ValueType)
+ assert.Same(t, inner, list.Element)
+ assert.Equal(t, []byte{3}, fields[1].WriteDefault)
+}
diff --git a/schema.go b/schema.go
index e518e3a5e..5b3ef89c3 100644
--- a/schema.go
+++ b/schema.go
@@ -77,7 +77,14 @@ func NewSchema(id int, fields ...NestedField) *Schema {
// NewSchemaWithIdentifiers constructs a new schema with the provided ID
// and fields, along with a slice of field IDs to be listed as identifier
// fields.
+//
+// The defaults of binary and fixed fields, including fields nested in structs,
+// lists and maps, are stored as hex strings when given as []byte,
+// [BinaryLiteral] or [FixedLiteral], the form they take in metadata JSON.
+// Byte slices inside other defaults are left as they are. The provided fields
+// and types are not modified.
func NewSchemaWithIdentifiers(id int, identifierIDs []int, fields
...NestedField) *Schema {
+ fields, _ = normalizeDefaults(fields)
s := &Schema{ID: id, fields: fields, IdentifierFieldIDs: identifierIDs}
s.init()
diff --git a/schema_test.go b/schema_test.go
index 573df769d..7a895586d 100644
--- a/schema_test.go
+++ b/schema_test.go
@@ -346,7 +346,6 @@ func TestSchemaFieldGettersReturnDefensiveCopies(t
*testing.T) {
person, ok := personField.Type.(*iceberg.StructType)
require.True(t, ok)
person.FieldList[0].Name = "hijacked"
- payloadField.InitialDefault.(iceberg.BinaryLiteral)[0] = 9
payloadField.WriteDefault.(map[string]any)["nested"].([]any)[0].([]byte)[0] = 9
actual, ok := schema.FindFieldByID(2)
@@ -354,7 +353,7 @@ func TestSchemaFieldGettersReturnDefensiveCopies(t
*testing.T) {
assert.Equal(t, "name", actual.Name)
payload, ok := schema.FindFieldByID(3)
require.True(t, ok)
- assert.Equal(t, iceberg.BinaryLiteral{1, 2},
payload.InitialDefault)
+ assert.Equal(t, "0102", payload.InitialDefault)
assert.Equal(t, map[string]any{"nested": []any{[]byte{3, 4}}},
payload.WriteDefault)
}
diff --git a/table/metadata_getters_test.go b/table/metadata_getters_test.go
index b463b22cd..f427845b0 100644
--- a/table/metadata_getters_test.go
+++ b/table/metadata_getters_test.go
@@ -191,8 +191,6 @@ func TestMetadataSchemaGetterCopiesNestedValues(t
*testing.T) {
cloned.IdentifierFieldIDs[0] = 99
fields := cloned.FieldsRef(iceinternal.SchemaRef{})
payload := fields[0].Type.(*iceberg.StructType)
- payload.FieldList[0].InitialDefault.([]byte)[0] = 99
- payload.FieldList[0].WriteDefault.(iceberg.BinaryLiteral)[0] = 99
payload.FieldList[1].Name = "changed"
payload.FieldList[1].Type.(*iceberg.ListType).Element.(*iceberg.StructType).FieldList[0].InitialDefault.([]any)[0].(map[string]any)["bytes"].([]byte)[0]
= 99
payload.FieldList[2].Type.(*iceberg.MapType).ValueType.(*iceberg.StructType).FieldList[0].WriteDefault.(map[string]any)["values"].([]any)[0]
= "changed"
diff --git a/table/update_schema_test.go b/table/update_schema_test.go
index 6ad7ca53b..6a1d795a6 100644
--- a/table/update_schema_test.go
+++ b/table/update_schema_test.go
@@ -18,6 +18,7 @@
package table
import (
+ "encoding/json"
"testing"
"github.com/apache/iceberg-go"
@@ -2221,6 +2222,49 @@ func TestUnionByNameAddedColumnPreservesInitialDefault(t
*testing.T) {
assert.Equal(t, int32(42), count.WriteDefault)
}
+func TestAddColumnStoresBinaryDefaultAsHex(t *testing.T) {
+ current := iceberg.NewSchema(1,
+ iceberg.NestedField{ID: 1, Name: "id", Type:
iceberg.PrimitiveTypes.Int32, Required: true},
+ )
+
+ applied, err := NewUpdateSchema(unionTxn(t, current), true, false).
+ AddColumn([]string{"payload"}, iceberg.PrimitiveTypes.Binary,
"", false, iceberg.NewLiteral([]byte{0x0a, 0xff})).
+ Apply()
+ require.NoError(t, err)
+
+ payload, ok := applied.FindFieldByName("payload")
+ require.True(t, ok)
+ assert.Equal(t, "0aff", payload.InitialDefault)
+ assert.Equal(t, "0aff", payload.WriteDefault)
+
+ data, err := json.Marshal(applied)
+ require.NoError(t, err)
+ var decoded iceberg.Schema
+ require.NoError(t, json.Unmarshal(data, &decoded))
+ assert.True(t, applied.Equals(&decoded))
+}
+
+// A binary write default read from metadata and the same default built in
+// process must compare equal, so union-by-name does not rewrite it.
+func TestUnionByNameKeepsUnchangedBinaryWriteDefault(t *testing.T) {
+ var current iceberg.Schema
+ require.NoError(t,
json.Unmarshal([]byte(`{"type":"struct","schema-id":1,"fields":[`+
+
`{"id":1,"name":"payload","type":"binary","required":false,"initial-default":"0aff","write-default":"0aff"}]}`),
¤t))
+ incoming := iceberg.NewSchema(1, iceberg.NestedField{
+ ID: 1, Name: "payload", Type: iceberg.PrimitiveTypes.Binary,
+ InitialDefault: []byte{0x0a, 0xff}, WriteDefault: []byte{0x0a,
0xff},
+ })
+
+ meta, err := NewMetadata(¤t, iceberg.UnpartitionedSpec,
UnsortedSortOrder, "",
+ iceberg.Properties{PropertyFormatVersion: "3"})
+ require.NoError(t, err)
+ txn := New([]string{"tbl"}, meta, "", nil, nil).NewTransaction()
+
+ applied, err := NewUpdateSchema(txn, true,
false).UnionByNameWith(incoming).Apply()
+ require.NoError(t, err)
+ assert.Equal(t, current.AsStruct(), applied.AsStruct())
+}
+
func TestUnionByNameRejectsInvalidAddedColumnDefault(t *testing.T) {
current := iceberg.NewSchema(1,
iceberg.NestedField{ID: 1, Name: "id", Type:
iceberg.PrimitiveTypes.Int32, Required: true},
diff --git a/types.go b/types.go
index 0e780fbdc..50bbf01dd 100644
--- a/types.go
+++ b/types.go
@@ -358,16 +358,8 @@ func (n NestedField) MarshalJSON() ([]byte, error) {
}
func defaultValueToJSON(typ Type, value any) any {
- switch typ.(type) {
- case BinaryType, FixedType:
- switch value := value.(type) {
- case []byte:
- return internal.EncodeDefaultBytes(value)
- case BinaryLiteral:
- return internal.EncodeDefaultBytes(value)
- case FixedLiteral:
- return internal.EncodeDefaultBytes(value)
- }
+ if encoded, ok := hexDefault(typ, value); ok {
+ return encoded
}
return value
diff --git a/view/metadata_test.go b/view/metadata_test.go
index 957cb7d28..953fb4c93 100644
--- a/view/metadata_test.go
+++ b/view/metadata_test.go
@@ -761,8 +761,6 @@ func TestCloneSchemaCopiesNestedValues(t *testing.T) {
fields := cloned.FieldsRef(iceinternal.SchemaRef{})
structField := fields[0].Type.(*iceberg.StructType)
structField.FieldList[0].Name = "changed"
- structField.FieldList[0].InitialDefault.([]byte)[0] = 9
- structField.FieldList[0].WriteDefault.(iceberg.BinaryLiteral)[0] = 9
listField := fields[1].Type.(*iceberg.ListType)
listField.Element.(*iceberg.StructType).FieldList[0].Name = "changed"
mapField := fields[2].Type.(*iceberg.MapType)
@@ -771,8 +769,8 @@ func TestCloneSchemaCopiesNestedValues(t *testing.T) {
assert.Equal(t, []int{1}, schema.IdentifierFieldIDs)
originalStruct := schema.Field(0).Type.(*iceberg.StructType)
assert.Equal(t, "nested", originalStruct.FieldList[0].Name)
- assert.Equal(t, []byte{1}, originalStruct.FieldList[0].InitialDefault)
- assert.Equal(t, iceberg.BinaryLiteral{2},
originalStruct.FieldList[0].WriteDefault)
+ assert.Equal(t, "01", originalStruct.FieldList[0].InitialDefault)
+ assert.Equal(t, "02", originalStruct.FieldList[0].WriteDefault)
assert.Equal(t, "element",
schema.Field(1).Type.(*iceberg.ListType).Element.(*iceberg.StructType).FieldList[0].Name)
assert.Equal(t, "value",
schema.Field(2).Type.(*iceberg.MapType).ValueType.(*iceberg.StructType).FieldList[0].Name)
}
diff --git a/website/src/releases.md b/website/src/releases.md
index 5e1518b31..d748cb181 100644
--- a/website/src/releases.md
+++ b/website/src/releases.md
@@ -42,6 +42,7 @@ Per-release notes (highlights, breaking changes,
contributors) are published on
### Unreleased compatibility notes
+- **Schemas store binary and fixed defaults as hex strings.** `NewSchema`,
`NewSchemaWithIdentifiers` and `UpdateSchema.AddColumn` (through `Apply()`)
convert `[]byte`, `BinaryLiteral` and `FixedLiteral` defaults of binary and
fixed fields, including fields nested in structs, lists and maps, to the
lowercase hex string iceberg-go writes to metadata JSON, so a field read back
from a schema holds a `string` such as `"0aff"`. Code that type-asserts these
defaults to `[]byte` must decode the [...]
- **Predicates reject NaN literals when bound.** `EqualTo`, `NotEqualTo`,
`LessThan`, `LessThanEqual`, `GreaterThan`, `GreaterThanEqual`, `IsIn`, `NotIn`
and `LiteralPredicate`/`SetPredicate` with a NaN float literal now fail in
`Bind` with `ErrInvalidArgument`, as Java does. That includes the string
`"NaN"` on a float column and a NaN on a decimal column, which used to bind as
zero. Scans, `ExpressionEvaluator` and the metrics evaluators all bind first,
so they return the error too. Use [...]
- **SQL view APIs align with `catalog.ViewCatalog`.** `CreateView` now takes a
`*view.Version`, schema and options, and returns `*view.View`. `LoadView` now
returns `*view.View`; use `Metadata()` to access its metadata. If the caller
leaves `DefaultCatalog` empty, SQL still sets it to the catalog name.
- **`LocationProvider.NewDataLocation` takes the partition path.** The method
is now `NewDataLocation(dataFileName, partitionPath string)`, matching Iceberg
Java and PyIceberg, so custom `LocationProvider` implementations must add the
argument. Callers pass the partition path there, or `""` for an unpartitioned
file, instead of prefixing the file name.
