This is an automated email from the ASF dual-hosted git repository.
zeroshade pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/iceberg-go.git
The following commit(s) were added to refs/heads/main by this push:
new 100025739 fix(table): write statistics blob metadata Java can read
(#2358)
100025739 is described below
commit 10002573905efc80336d4cce8fb49d51f2bf117a
Author: Andrei Tserakhau <[email protected]>
AuthorDate: Thu Nov 5 18:36:55 2026 +0100
fix(table): write statistics blob metadata Java can read (#2358)
* fix(table): write statistics blob metadata Java can read
A statistics blob with no properties was written as "properties": null,
and one with no fields as "fields": null, in table metadata and
set-statistics updates. Iceberg Java 1.12 rejects both in
TableMetadataParser and MetadataUpdateParser, so the whole metadata load
fails. Metadata Java wrote hit it too: Java omits empty properties, Go
decoded that to a nil map, and the next Go write emitted null.
Omit empty properties and write nil fields as [], which is what Java
writes for the same blobs.
* fix(table): write empty statistics blob metadata as []
A StatisticsFile with no blobs was written as "blob-metadata": null,
which Java's StatisticsFileParser rejects.
---
table/statistics.go | 28 ++++++++++++++++++++++++++--
table/statistics_test.go | 28 ++++++++++++++++++++++++++++
2 files changed, 54 insertions(+), 2 deletions(-)
diff --git a/table/statistics.go b/table/statistics.go
index 46a429c4d..9368c7b12 100644
--- a/table/statistics.go
+++ b/table/statistics.go
@@ -17,7 +17,11 @@
package table
-import "github.com/apache/iceberg-go/puffin"
+import (
+ "encoding/json"
+
+ "github.com/apache/iceberg-go/puffin"
+)
// StatisticsFile represents a statistics file in the Puffin format, that can
be used to read table data more
// efficiently.
@@ -33,6 +37,16 @@ type StatisticsFile struct {
BlobMetadata []BlobMetadata `json:"blob-metadata"`
}
+// MarshalJSON writes nil BlobMetadata as [], since the spec requires the
field.
+func (s StatisticsFile) MarshalJSON() ([]byte, error) {
+ type statisticsFile StatisticsFile
+ if s.BlobMetadata == nil {
+ s.BlobMetadata = []BlobMetadata{}
+ }
+
+ return json.Marshal(statisticsFile(s))
+}
+
// BlobMetadata is the metadata of a statistics or indices blob.
// Its type uses the same Puffin blob identifiers as the file reader and
writer.
type BlobMetadata struct {
@@ -40,7 +54,17 @@ type BlobMetadata struct {
SnapshotID int64 `json:"snapshot-id"`
SequenceNumber int64 `json:"sequence-number"`
Fields []int32 `json:"fields"`
- Properties map[string]string `json:"properties"`
+ Properties map[string]string `json:"properties,omitempty"`
+}
+
+// MarshalJSON writes nil Fields as [], since the spec requires the field.
+func (b BlobMetadata) MarshalJSON() ([]byte, error) {
+ type blobMetadata BlobMetadata
+ if b.Fields == nil {
+ b.Fields = []int32{}
+ }
+
+ return json.Marshal(blobMetadata(b))
}
// PartitionStatisticsFile represents a partition statistics file that can be
used to read table data more efficiently.
diff --git a/table/statistics_test.go b/table/statistics_test.go
index d9796bde6..aa8abe40d 100644
--- a/table/statistics_test.go
+++ b/table/statistics_test.go
@@ -66,3 +66,31 @@ func TestBlobMetadataRejectsNullAndEmptyType(t *testing.T) {
})
}
}
+
+func TestStatisticsJSONMatchesJava(t *testing.T) {
+ for _, tt := range []struct {
+ value any
+ want string
+ }{
+ {
+ value: BlobMetadata{Type:
puffin.BlobTypeDataSketchesTheta, SnapshotID: 1, SequenceNumber: 2, Fields:
[]int32{1}},
+ want:
`{"type":"apache-datasketches-theta-v1","snapshot-id":1,"sequence-number":2,"fields":[1]}`,
+ },
+ {
+ value: BlobMetadata{Type:
puffin.BlobTypeDataSketchesTheta, SnapshotID: 1, SequenceNumber: 2, Fields:
[]int32{1}, Properties: map[string]string{}},
+ want:
`{"type":"apache-datasketches-theta-v1","snapshot-id":1,"sequence-number":2,"fields":[1]}`,
+ },
+ {
+ value: BlobMetadata{Type: "vendor-sketch-v1",
SnapshotID: 1, SequenceNumber: 2, Properties: map[string]string{"k": "v"}},
+ want:
`{"type":"vendor-sketch-v1","snapshot-id":1,"sequence-number":2,"fields":[],"properties":{"k":"v"}}`,
+ },
+ {
+ value: StatisticsFile{SnapshotID: 1, StatisticsPath:
"s3://bucket/stats.puffin", FileSizeInBytes: 10, FileFooterSizeInBytes: 2},
+ want:
`{"snapshot-id":1,"statistics-path":"s3://bucket/stats.puffin","file-size-in-bytes":10,"file-footer-size-in-bytes":2,"blob-metadata":[]}`,
+ },
+ } {
+ data, err := json.Marshal(tt.value)
+ require.NoError(t, err)
+ assert.JSONEq(t, tt.want, string(data))
+ }
+}
