laskoviymishka commented on code in PR #1214: URL: https://github.com/apache/iceberg-go/pull/1214#discussion_r3449057492
########## table/data_file_meta.go: ########## @@ -0,0 +1,178 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package table + +import ( + "errors" + "fmt" + + "github.com/apache/iceberg-go" + tblutils "github.com/apache/iceberg-go/table/internal" +) + +// DataFileArgs collects everything needed to compute a fully populated +// [iceberg.DataFile] (data, equality-delete, or positional-delete) from +// a file's in-memory, format-specific metadata object without performing +// any filesystem reads. +type DataFileArgs struct { + // Schema is the Iceberg schema the file's contents conform to. The + // file's columns MUST carry per-leaf field-ID metadata whose values + // match this schema's field IDs (for Parquet this is the + // "PARQUET:field_id" key on each leaf). + Schema *iceberg.Schema + + // Spec is the partition spec the resulting DataFile is bound to — + // typically tbl.Spec(). For an unpartitioned table pass + // *iceberg.UnpartitionedSpec. + Spec iceberg.PartitionSpec + + // Format identifies the on-disk file format of the file being + // described and selects the internal FileFormat implementation used + // to extract per-column statistics from Metadata. + Format iceberg.FileFormat + + // Metadata is the format-specific, in-memory file metadata object + // produced by the external writer (for example + // *parquet/metadata.FileMetaData when Format is + // [iceberg.ParquetFile]). + Metadata any + + // FilePath is the fully qualified location the file bytes were + // written to (the same path that will be recorded in the manifest entry). + FilePath string + + // FileSize is the byte length of the object at FilePath. Must be + // greater than zero. + FileSize int64 + + // Content selects which manifest-entry kind to produce — one of + // [iceberg.EntryContentData], [iceberg.EntryContentEqDeletes] or + // [iceberg.EntryContentPosDeletes]. + Content iceberg.ManifestEntryContent + + // PartitionValues maps spec field ID → partition value. Required when Spec + // is partitioned; leave nil/empty for unpartitioned tables. + PartitionValues map[int]any + + // SortOrderID is the sort-order ID recorded on the manifest entry, + // typically tbl.SortOrder().OrderID(). Zero means "unsorted". + SortOrderID int + + // Properties are consulted only for write-side metrics-mode keys + // (write.metadata.metrics.default and per-column + // write.metadata.metrics.column.<name> overrides) that decide + // which columns get full vs truncated vs no statistics. Pass + // tbl.Properties() for table-default behavior; nil is safe. + Properties iceberg.Properties + + // EqualityFieldIDs lists the schema field IDs that an equality + // delete file matches on. Required when + // Content == [iceberg.EntryContentEqDeletes] and MUST be empty + // for every other Content value. + EqualityFieldIDs []int +} + +// DataFileFromMetadata builds a fully populated [iceberg.DataFile] from a +// file's in-memory, format-specific metadata object, extracting per-column +// statistics (column_sizes, value_counts, null_value_counts, lower_bounds, +// upper_bounds), split offsets, and the manifest-entry shape required for +// the requested content type. The format-specific extraction is dispatched +// through the internal FileFormat interface; no filesystem reads are +// performed. +// +// The returned DataFile is intended to be handed to +// [Transaction.AddDataFiles] (data files) or [Transaction.NewRowDelta] +// (delete files); it is NOT committed automatically. The file bytes +// described by args.Metadata must already have been uploaded to +// args.FilePath. +func DataFileFromMetadata(args DataFileArgs) (iceberg.DataFile, error) { + if args.Schema == nil { + return nil, errors.New("schema is required") + } + if args.Metadata == nil { + return nil, errors.New("file metadata is required") + } + if args.FilePath == "" { + return nil, errors.New("file path is required") + } + if args.FileSize <= 0 { + return nil, fmt.Errorf("file size must be greater than 0, got %d", args.FileSize) + } + + switch args.Content { + case iceberg.EntryContentData, + iceberg.EntryContentEqDeletes, + iceberg.EntryContentPosDeletes: + default: + return nil, fmt.Errorf("invalid content: %v", args.Content) + } + + if args.Content == iceberg.EntryContentEqDeletes && len(args.EqualityFieldIDs) == 0 { + return nil, errors.New("equalityFieldIDs is required for equality-delete files") + } + if args.Content != iceberg.EntryContentEqDeletes && len(args.EqualityFieldIDs) > 0 { + return nil, fmt.Errorf("equalityFieldIDs must be empty for content %v", args.Content) + } + + format := tblutils.GetFileFormat(args.Format) + if format == nil { + return nil, fmt.Errorf("unsupported file format: %s", args.Format) + } + + statsPlan, err := computeStatsPlan(args.Schema, args.Properties) + if err != nil { + return nil, fmt.Errorf("failed to build stats plan: %w", err) + } + + pathMapping, err := format.PathToIDMapping(args.Schema) + if err != nil { + return nil, fmt.Errorf("failed to build path mapping: %w", err) + } + + stats := format.DataFileStatsFromMeta( + args.Metadata, + statsPlan, + pathMapping, + tblutils.VariantFieldIDsFromSchema(args.Schema), + ) + if stats == nil { + return nil, errors.New("failed to build data file stats from metadata") + } + + if len(args.EqualityFieldIDs) > 0 { + stats.EqualityFieldIDs = append(stats.EqualityFieldIDs, args.EqualityFieldIDs...) + } + + partitionValues := args.PartitionValues + if partitionValues == nil { + partitionValues = make(map[int]any) + } + + df := stats.ToDataFile(tblutils.DataFileOpts{ + Schema: args.Schema, + Spec: args.Spec, + Path: args.FilePath, + Format: args.Format, + Content: args.Content, + FileSize: args.FileSize, + PartitionValues: partitionValues, + SortOrderID: args.SortOrderID, Review Comment: Positional deletes need a null sort order id, and we pass `args.SortOrderID` unconditionally here. The spec requires pos-delete files to set sort order id to null, and our own writer enforces this at `writer.go:134` ("position delete file claims sort order id %d; the spec requires a null sort order id"). This helper happily accepts a nonzero `SortOrderID` for `EntryContentPosDeletes` and produces a file the writer would have rejected. I'd return an error when `Content == EntryContentPosDeletes && SortOrderID != UnsortedSortOrderID` so the two paths agree. ########## table/data_file_meta.go: ########## @@ -0,0 +1,178 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package table + +import ( + "errors" + "fmt" + + "github.com/apache/iceberg-go" + tblutils "github.com/apache/iceberg-go/table/internal" +) + +// DataFileArgs collects everything needed to compute a fully populated +// [iceberg.DataFile] (data, equality-delete, or positional-delete) from +// a file's in-memory, format-specific metadata object without performing +// any filesystem reads. +type DataFileArgs struct { + // Schema is the Iceberg schema the file's contents conform to. The + // file's columns MUST carry per-leaf field-ID metadata whose values + // match this schema's field IDs (for Parquet this is the + // "PARQUET:field_id" key on each leaf). + Schema *iceberg.Schema + + // Spec is the partition spec the resulting DataFile is bound to — + // typically tbl.Spec(). For an unpartitioned table pass + // *iceberg.UnpartitionedSpec. + Spec iceberg.PartitionSpec + + // Format identifies the on-disk file format of the file being + // described and selects the internal FileFormat implementation used + // to extract per-column statistics from Metadata. + Format iceberg.FileFormat + + // Metadata is the format-specific, in-memory file metadata object + // produced by the external writer (for example + // *parquet/metadata.FileMetaData when Format is + // [iceberg.ParquetFile]). + Metadata any + + // FilePath is the fully qualified location the file bytes were + // written to (the same path that will be recorded in the manifest entry). + FilePath string + + // FileSize is the byte length of the object at FilePath. Must be + // greater than zero. + FileSize int64 + + // Content selects which manifest-entry kind to produce — one of + // [iceberg.EntryContentData], [iceberg.EntryContentEqDeletes] or + // [iceberg.EntryContentPosDeletes]. + Content iceberg.ManifestEntryContent + + // PartitionValues maps spec field ID → partition value. Required when Spec + // is partitioned; leave nil/empty for unpartitioned tables. + PartitionValues map[int]any + + // SortOrderID is the sort-order ID recorded on the manifest entry, + // typically tbl.SortOrder().OrderID(). Zero means "unsorted". + SortOrderID int + + // Properties are consulted only for write-side metrics-mode keys + // (write.metadata.metrics.default and per-column + // write.metadata.metrics.column.<name> overrides) that decide + // which columns get full vs truncated vs no statistics. Pass + // tbl.Properties() for table-default behavior; nil is safe. + Properties iceberg.Properties + + // EqualityFieldIDs lists the schema field IDs that an equality + // delete file matches on. Required when + // Content == [iceberg.EntryContentEqDeletes] and MUST be empty + // for every other Content value. + EqualityFieldIDs []int +} + +// DataFileFromMetadata builds a fully populated [iceberg.DataFile] from a +// file's in-memory, format-specific metadata object, extracting per-column +// statistics (column_sizes, value_counts, null_value_counts, lower_bounds, +// upper_bounds), split offsets, and the manifest-entry shape required for +// the requested content type. The format-specific extraction is dispatched +// through the internal FileFormat interface; no filesystem reads are +// performed. +// +// The returned DataFile is intended to be handed to +// [Transaction.AddDataFiles] (data files) or [Transaction.NewRowDelta] +// (delete files); it is NOT committed automatically. The file bytes +// described by args.Metadata must already have been uploaded to +// args.FilePath. +func DataFileFromMetadata(args DataFileArgs) (iceberg.DataFile, error) { + if args.Schema == nil { + return nil, errors.New("schema is required") + } + if args.Metadata == nil { + return nil, errors.New("file metadata is required") + } + if args.FilePath == "" { + return nil, errors.New("file path is required") + } + if args.FileSize <= 0 { + return nil, fmt.Errorf("file size must be greater than 0, got %d", args.FileSize) + } + + switch args.Content { + case iceberg.EntryContentData, + iceberg.EntryContentEqDeletes, + iceberg.EntryContentPosDeletes: + default: + return nil, fmt.Errorf("invalid content: %v", args.Content) + } + + if args.Content == iceberg.EntryContentEqDeletes && len(args.EqualityFieldIDs) == 0 { + return nil, errors.New("equalityFieldIDs is required for equality-delete files") + } + if args.Content != iceberg.EntryContentEqDeletes && len(args.EqualityFieldIDs) > 0 { + return nil, fmt.Errorf("equalityFieldIDs must be empty for content %v", args.Content) + } + + format := tblutils.GetFileFormat(args.Format) + if format == nil { + return nil, fmt.Errorf("unsupported file format: %s", args.Format) + } + + statsPlan, err := computeStatsPlan(args.Schema, args.Properties) + if err != nil { + return nil, fmt.Errorf("failed to build stats plan: %w", err) + } + + pathMapping, err := format.PathToIDMapping(args.Schema) + if err != nil { + return nil, fmt.Errorf("failed to build path mapping: %w", err) + } + + stats := format.DataFileStatsFromMeta( Review Comment: This is a public function that can panic instead of returning an error, and the `stats == nil` guard doesn't catch the real risk. `DataFileStatsFromMeta` does a hard `meta.(*metadata.FileMetaData)` assertion — pass any other concrete type with `Format: ParquetFile` and it panics, never reaching the nil check. The column-mapping path inside it and the multi-value `PartitionValue()` path inside `ToDataFile` panic too. Internally this is fine because `filesToDataFiles` wraps `fileToDataFile` in `recover()`, but here there's no net, so a bad arg crashes the caller's goroutine. I'd at minimum comma-ok the metadata type up front and return something like `fmt.Errorf("unsupported metadata type for %s: %T", args.Format, args.Metadata)`, and wrap the body in a deferred `recover()` that converts the deeper panics into error returns. wdyt? ########## table/data_file_meta.go: ########## @@ -0,0 +1,178 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package table + +import ( + "errors" + "fmt" + + "github.com/apache/iceberg-go" + tblutils "github.com/apache/iceberg-go/table/internal" +) + +// DataFileArgs collects everything needed to compute a fully populated +// [iceberg.DataFile] (data, equality-delete, or positional-delete) from +// a file's in-memory, format-specific metadata object without performing +// any filesystem reads. +type DataFileArgs struct { + // Schema is the Iceberg schema the file's contents conform to. The + // file's columns MUST carry per-leaf field-ID metadata whose values + // match this schema's field IDs (for Parquet this is the + // "PARQUET:field_id" key on each leaf). + Schema *iceberg.Schema + + // Spec is the partition spec the resulting DataFile is bound to — + // typically tbl.Spec(). For an unpartitioned table pass + // *iceberg.UnpartitionedSpec. + Spec iceberg.PartitionSpec + + // Format identifies the on-disk file format of the file being + // described and selects the internal FileFormat implementation used + // to extract per-column statistics from Metadata. + Format iceberg.FileFormat + + // Metadata is the format-specific, in-memory file metadata object + // produced by the external writer (for example + // *parquet/metadata.FileMetaData when Format is + // [iceberg.ParquetFile]). + Metadata any + + // FilePath is the fully qualified location the file bytes were + // written to (the same path that will be recorded in the manifest entry). + FilePath string + + // FileSize is the byte length of the object at FilePath. Must be + // greater than zero. + FileSize int64 + + // Content selects which manifest-entry kind to produce — one of + // [iceberg.EntryContentData], [iceberg.EntryContentEqDeletes] or + // [iceberg.EntryContentPosDeletes]. + Content iceberg.ManifestEntryContent + + // PartitionValues maps spec field ID → partition value. Required when Spec + // is partitioned; leave nil/empty for unpartitioned tables. + PartitionValues map[int]any + + // SortOrderID is the sort-order ID recorded on the manifest entry, + // typically tbl.SortOrder().OrderID(). Zero means "unsorted". + SortOrderID int + + // Properties are consulted only for write-side metrics-mode keys + // (write.metadata.metrics.default and per-column + // write.metadata.metrics.column.<name> overrides) that decide + // which columns get full vs truncated vs no statistics. Pass + // tbl.Properties() for table-default behavior; nil is safe. + Properties iceberg.Properties + + // EqualityFieldIDs lists the schema field IDs that an equality + // delete file matches on. Required when + // Content == [iceberg.EntryContentEqDeletes] and MUST be empty + // for every other Content value. + EqualityFieldIDs []int +} + +// DataFileFromMetadata builds a fully populated [iceberg.DataFile] from a +// file's in-memory, format-specific metadata object, extracting per-column +// statistics (column_sizes, value_counts, null_value_counts, lower_bounds, +// upper_bounds), split offsets, and the manifest-entry shape required for +// the requested content type. The format-specific extraction is dispatched +// through the internal FileFormat interface; no filesystem reads are +// performed. +// +// The returned DataFile is intended to be handed to +// [Transaction.AddDataFiles] (data files) or [Transaction.NewRowDelta] +// (delete files); it is NOT committed automatically. The file bytes +// described by args.Metadata must already have been uploaded to +// args.FilePath. +func DataFileFromMetadata(args DataFileArgs) (iceberg.DataFile, error) { + if args.Schema == nil { + return nil, errors.New("schema is required") + } + if args.Metadata == nil { + return nil, errors.New("file metadata is required") + } + if args.FilePath == "" { + return nil, errors.New("file path is required") + } + if args.FileSize <= 0 { + return nil, fmt.Errorf("file size must be greater than 0, got %d", args.FileSize) + } + + switch args.Content { + case iceberg.EntryContentData, + iceberg.EntryContentEqDeletes, + iceberg.EntryContentPosDeletes: + default: + return nil, fmt.Errorf("invalid content: %v", args.Content) + } + + if args.Content == iceberg.EntryContentEqDeletes && len(args.EqualityFieldIDs) == 0 { + return nil, errors.New("equalityFieldIDs is required for equality-delete files") + } + if args.Content != iceberg.EntryContentEqDeletes && len(args.EqualityFieldIDs) > 0 { + return nil, fmt.Errorf("equalityFieldIDs must be empty for content %v", args.Content) + } + + format := tblutils.GetFileFormat(args.Format) + if format == nil { + return nil, fmt.Errorf("unsupported file format: %s", args.Format) + } + + statsPlan, err := computeStatsPlan(args.Schema, args.Properties) + if err != nil { + return nil, fmt.Errorf("failed to build stats plan: %w", err) + } + + pathMapping, err := format.PathToIDMapping(args.Schema) + if err != nil { + return nil, fmt.Errorf("failed to build path mapping: %w", err) + } + + stats := format.DataFileStatsFromMeta( + args.Metadata, + statsPlan, + pathMapping, + tblutils.VariantFieldIDsFromSchema(args.Schema), + ) + if stats == nil { + return nil, errors.New("failed to build data file stats from metadata") + } + + if len(args.EqualityFieldIDs) > 0 { + stats.EqualityFieldIDs = append(stats.EqualityFieldIDs, args.EqualityFieldIDs...) Review Comment: `stats.EqualityFieldIDs` is always nil coming out of `DataFileStatsFromMeta`, so the `append` is just a misleading way to write an assignment — it implies accumulation that isn't happening, and if `DataFileStatsFromMeta` ever does populate it we'd silently duplicate. The internal write path uses a direct assignment; I'd match it: ```go stats.EqualityFieldIDs = args.EqualityFieldIDs ``` ########## table/data_file_meta.go: ########## @@ -0,0 +1,178 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package table + +import ( + "errors" + "fmt" + + "github.com/apache/iceberg-go" + tblutils "github.com/apache/iceberg-go/table/internal" +) + +// DataFileArgs collects everything needed to compute a fully populated +// [iceberg.DataFile] (data, equality-delete, or positional-delete) from +// a file's in-memory, format-specific metadata object without performing +// any filesystem reads. +type DataFileArgs struct { + // Schema is the Iceberg schema the file's contents conform to. The + // file's columns MUST carry per-leaf field-ID metadata whose values + // match this schema's field IDs (for Parquet this is the + // "PARQUET:field_id" key on each leaf). + Schema *iceberg.Schema + + // Spec is the partition spec the resulting DataFile is bound to — + // typically tbl.Spec(). For an unpartitioned table pass + // *iceberg.UnpartitionedSpec. + Spec iceberg.PartitionSpec + + // Format identifies the on-disk file format of the file being + // described and selects the internal FileFormat implementation used + // to extract per-column statistics from Metadata. + Format iceberg.FileFormat + + // Metadata is the format-specific, in-memory file metadata object + // produced by the external writer (for example + // *parquet/metadata.FileMetaData when Format is + // [iceberg.ParquetFile]). + Metadata any + + // FilePath is the fully qualified location the file bytes were + // written to (the same path that will be recorded in the manifest entry). + FilePath string + + // FileSize is the byte length of the object at FilePath. Must be + // greater than zero. + FileSize int64 + + // Content selects which manifest-entry kind to produce — one of + // [iceberg.EntryContentData], [iceberg.EntryContentEqDeletes] or + // [iceberg.EntryContentPosDeletes]. + Content iceberg.ManifestEntryContent + + // PartitionValues maps spec field ID → partition value. Required when Spec + // is partitioned; leave nil/empty for unpartitioned tables. + PartitionValues map[int]any + + // SortOrderID is the sort-order ID recorded on the manifest entry, + // typically tbl.SortOrder().OrderID(). Zero means "unsorted". + SortOrderID int + + // Properties are consulted only for write-side metrics-mode keys + // (write.metadata.metrics.default and per-column + // write.metadata.metrics.column.<name> overrides) that decide + // which columns get full vs truncated vs no statistics. Pass + // tbl.Properties() for table-default behavior; nil is safe. + Properties iceberg.Properties + + // EqualityFieldIDs lists the schema field IDs that an equality + // delete file matches on. Required when + // Content == [iceberg.EntryContentEqDeletes] and MUST be empty + // for every other Content value. + EqualityFieldIDs []int +} + +// DataFileFromMetadata builds a fully populated [iceberg.DataFile] from a +// file's in-memory, format-specific metadata object, extracting per-column +// statistics (column_sizes, value_counts, null_value_counts, lower_bounds, +// upper_bounds), split offsets, and the manifest-entry shape required for +// the requested content type. The format-specific extraction is dispatched +// through the internal FileFormat interface; no filesystem reads are +// performed. +// +// The returned DataFile is intended to be handed to +// [Transaction.AddDataFiles] (data files) or [Transaction.NewRowDelta] +// (delete files); it is NOT committed automatically. The file bytes +// described by args.Metadata must already have been uploaded to +// args.FilePath. +func DataFileFromMetadata(args DataFileArgs) (iceberg.DataFile, error) { + if args.Schema == nil { + return nil, errors.New("schema is required") + } + if args.Metadata == nil { + return nil, errors.New("file metadata is required") + } + if args.FilePath == "" { + return nil, errors.New("file path is required") + } + if args.FileSize <= 0 { + return nil, fmt.Errorf("file size must be greater than 0, got %d", args.FileSize) + } + + switch args.Content { + case iceberg.EntryContentData, + iceberg.EntryContentEqDeletes, + iceberg.EntryContentPosDeletes: + default: + return nil, fmt.Errorf("invalid content: %v", args.Content) + } + + if args.Content == iceberg.EntryContentEqDeletes && len(args.EqualityFieldIDs) == 0 { + return nil, errors.New("equalityFieldIDs is required for equality-delete files") + } + if args.Content != iceberg.EntryContentEqDeletes && len(args.EqualityFieldIDs) > 0 { + return nil, fmt.Errorf("equalityFieldIDs must be empty for content %v", args.Content) + } + + format := tblutils.GetFileFormat(args.Format) + if format == nil { + return nil, fmt.Errorf("unsupported file format: %s", args.Format) + } + + statsPlan, err := computeStatsPlan(args.Schema, args.Properties) + if err != nil { + return nil, fmt.Errorf("failed to build stats plan: %w", err) + } + + pathMapping, err := format.PathToIDMapping(args.Schema) + if err != nil { + return nil, fmt.Errorf("failed to build path mapping: %w", err) + } + + stats := format.DataFileStatsFromMeta( + args.Metadata, + statsPlan, + pathMapping, + tblutils.VariantFieldIDsFromSchema(args.Schema), + ) + if stats == nil { + return nil, errors.New("failed to build data file stats from metadata") + } + + if len(args.EqualityFieldIDs) > 0 { + stats.EqualityFieldIDs = append(stats.EqualityFieldIDs, args.EqualityFieldIDs...) + } + + partitionValues := args.PartitionValues + if partitionValues == nil { + partitionValues = make(map[int]any) + } + + df := stats.ToDataFile(tblutils.DataFileOpts{ Review Comment: `FirstRowID` never gets set, which means the result is dead on arrival for v3 tables. `Transaction.AddDataFiles` enforces that every `DataFile` carries a `FirstRowID` on format-version-3 tables, and this helper is documented as the entry point into exactly that. So a v3 caller following the godoc gets an immediate rejection at commit. I'd add a `FirstRowID *int64` to `DataFileArgs` and plumb it through to the builder, with a doc note that it's required for v3. Thoughts? ########## table/data_file_meta.go: ########## @@ -0,0 +1,178 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package table + +import ( + "errors" + "fmt" + + "github.com/apache/iceberg-go" + tblutils "github.com/apache/iceberg-go/table/internal" +) + +// DataFileArgs collects everything needed to compute a fully populated +// [iceberg.DataFile] (data, equality-delete, or positional-delete) from +// a file's in-memory, format-specific metadata object without performing +// any filesystem reads. +type DataFileArgs struct { + // Schema is the Iceberg schema the file's contents conform to. The + // file's columns MUST carry per-leaf field-ID metadata whose values + // match this schema's field IDs (for Parquet this is the + // "PARQUET:field_id" key on each leaf). + Schema *iceberg.Schema + + // Spec is the partition spec the resulting DataFile is bound to — + // typically tbl.Spec(). For an unpartitioned table pass + // *iceberg.UnpartitionedSpec. + Spec iceberg.PartitionSpec Review Comment: There's no validation that partition values line up with the spec, and the failure modes here are bad. A zero-value `PartitionSpec{}` equals the unpartitioned spec, so a caller who forgets `Spec` on a partitioned table silently gets a `DataFile` with no partition info — which breaks partition pruning on other clients and only surfaces as a `validateDataFilePartitionData` failure at commit. Worse, if they do supply `PartitionValues` but the file spans more than one value for a field, `ToDataFile → PartitionValue()` panics (see the recover comment above). I'd validate that when `Spec` is partitioned, `PartitionValues` has an entry per field, and return an error rather than deferring to a commit-time failure or a crash. Also a small doc nit: the comment says pass `*iceberg.UnpartitionedSpec` but the field is a value `iceberg.PartitionSpec` — the zero value is already equivalent, so I'd just say that. ########## table/data_file_meta.go: ########## @@ -0,0 +1,178 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package table + +import ( + "errors" + "fmt" + + "github.com/apache/iceberg-go" + tblutils "github.com/apache/iceberg-go/table/internal" +) + +// DataFileArgs collects everything needed to compute a fully populated +// [iceberg.DataFile] (data, equality-delete, or positional-delete) from +// a file's in-memory, format-specific metadata object without performing +// any filesystem reads. +type DataFileArgs struct { + // Schema is the Iceberg schema the file's contents conform to. The + // file's columns MUST carry per-leaf field-ID metadata whose values + // match this schema's field IDs (for Parquet this is the + // "PARQUET:field_id" key on each leaf). + Schema *iceberg.Schema + + // Spec is the partition spec the resulting DataFile is bound to — + // typically tbl.Spec(). For an unpartitioned table pass + // *iceberg.UnpartitionedSpec. + Spec iceberg.PartitionSpec + + // Format identifies the on-disk file format of the file being + // described and selects the internal FileFormat implementation used + // to extract per-column statistics from Metadata. + Format iceberg.FileFormat + + // Metadata is the format-specific, in-memory file metadata object + // produced by the external writer (for example + // *parquet/metadata.FileMetaData when Format is + // [iceberg.ParquetFile]). + Metadata any + + // FilePath is the fully qualified location the file bytes were + // written to (the same path that will be recorded in the manifest entry). + FilePath string + + // FileSize is the byte length of the object at FilePath. Must be + // greater than zero. + FileSize int64 + + // Content selects which manifest-entry kind to produce — one of + // [iceberg.EntryContentData], [iceberg.EntryContentEqDeletes] or + // [iceberg.EntryContentPosDeletes]. + Content iceberg.ManifestEntryContent + + // PartitionValues maps spec field ID → partition value. Required when Spec + // is partitioned; leave nil/empty for unpartitioned tables. + PartitionValues map[int]any + + // SortOrderID is the sort-order ID recorded on the manifest entry, + // typically tbl.SortOrder().OrderID(). Zero means "unsorted". + SortOrderID int + + // Properties are consulted only for write-side metrics-mode keys + // (write.metadata.metrics.default and per-column + // write.metadata.metrics.column.<name> overrides) that decide + // which columns get full vs truncated vs no statistics. Pass + // tbl.Properties() for table-default behavior; nil is safe. + Properties iceberg.Properties + + // EqualityFieldIDs lists the schema field IDs that an equality + // delete file matches on. Required when + // Content == [iceberg.EntryContentEqDeletes] and MUST be empty + // for every other Content value. + EqualityFieldIDs []int +} + +// DataFileFromMetadata builds a fully populated [iceberg.DataFile] from a +// file's in-memory, format-specific metadata object, extracting per-column +// statistics (column_sizes, value_counts, null_value_counts, lower_bounds, +// upper_bounds), split offsets, and the manifest-entry shape required for +// the requested content type. The format-specific extraction is dispatched +// through the internal FileFormat interface; no filesystem reads are +// performed. +// +// The returned DataFile is intended to be handed to +// [Transaction.AddDataFiles] (data files) or [Transaction.NewRowDelta] +// (delete files); it is NOT committed automatically. The file bytes +// described by args.Metadata must already have been uploaded to +// args.FilePath. +func DataFileFromMetadata(args DataFileArgs) (iceberg.DataFile, error) { + if args.Schema == nil { + return nil, errors.New("schema is required") + } + if args.Metadata == nil { + return nil, errors.New("file metadata is required") + } + if args.FilePath == "" { + return nil, errors.New("file path is required") + } + if args.FileSize <= 0 { + return nil, fmt.Errorf("file size must be greater than 0, got %d", args.FileSize) + } + + switch args.Content { Review Comment: We validate `Content` shape but never validate the field IDs against the schema, which is the check the real writers do. For eq-deletes the spec says every id in `equality_ids` must be present in the delete file, and `equality_delete_writer.go:44` validates exactly that at write time; here we accept any ints. For pos-deletes the spec mandates the reserved field IDs (`2147483546`/`2147483545`) and we never check the schema matches `iceberg.PositionalDeleteSchema`. Either way an invalid file gets built and is only caught at `RowDelta.Commit`, after the bytes are already uploaded. I'd loop the `EqualityFieldIDs` through `args.Schema.FindFieldByID` for the eq case, and verify schema compatibility with `PositionalDeleteSchema` for the pos case. wdyt? ########## table/data_file_meta_test.go: ########## @@ -0,0 +1,166 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package table_test + +import ( + "bytes" + "strings" + "testing" + + "github.com/apache/arrow-go/v18/arrow/array" + "github.com/apache/arrow-go/v18/arrow/memory" + "github.com/apache/arrow-go/v18/parquet" + "github.com/apache/arrow-go/v18/parquet/file" + "github.com/apache/arrow-go/v18/parquet/metadata" + "github.com/apache/arrow-go/v18/parquet/pqarrow" + "github.com/apache/iceberg-go" + "github.com/apache/iceberg-go/table" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func parquetMetaFromSchema(t *testing.T, sch *iceberg.Schema, jsonRows string) ([]byte, *metadata.FileMetaData) { + t.Helper() + + arrowSch, err := table.SchemaToArrowSchema(sch, nil, true, false) + require.NoError(t, err) + + rec, _, err := array.RecordFromJSON(memory.DefaultAllocator, arrowSch, strings.NewReader(jsonRows)) + require.NoError(t, err) + defer rec.Release() + + var buf bytes.Buffer + wr, err := pqarrow.NewFileWriter(arrowSch, &buf, + parquet.NewWriterProperties(parquet.WithStats(true)), + pqarrow.DefaultWriterProps()) + require.NoError(t, err) + require.NoError(t, wr.Write(rec)) + require.NoError(t, wr.Close()) + + rdr, err := file.NewParquetReader(bytes.NewReader(buf.Bytes())) + require.NoError(t, err) + defer rdr.Close() + + return buf.Bytes(), rdr.MetaData() +} + +func TestDataFileFromMetadata_DataFile(t *testing.T) { + sch := iceberg.NewSchema(0, + iceberg.NestedField{ID: 1, Name: "id", Type: iceberg.PrimitiveTypes.Int32, Required: false}, + iceberg.NestedField{ID: 2, Name: "name", Type: iceberg.PrimitiveTypes.String, Required: false}, + ) + spec := iceberg.NewPartitionSpec() + + const jsonRows = `[ + {"id": 1, "name": "alpha"}, + {"id": 2, "name": "beta"}, + {"id": 3, "name": "gamma"} + ]` + + pqBytes, pqMeta := parquetMetaFromSchema(t, sch, jsonRows) + + const filePath = "s3://bucket/data/test-data-001.parquet" + + df, err := table.DataFileFromMetadata(table.DataFileArgs{ + Schema: sch, + Spec: spec, + Format: iceberg.ParquetFile, + Metadata: pqMeta, + FilePath: filePath, + FileSize: int64(len(pqBytes)), + Content: iceberg.EntryContentData, + }) + require.NoError(t, err) + require.NotNil(t, df) + + assert.Equal(t, iceberg.EntryContentData, df.ContentType()) + assert.Equal(t, iceberg.ParquetFile, df.FileFormat()) + assert.Equal(t, filePath, df.FilePath()) + assert.Equal(t, int64(len(pqBytes)), df.FileSizeBytes()) + assert.EqualValues(t, 3, df.Count()) + + assert.Len(t, df.ValueCounts(), 2, "one value-count entry per column") Review Comment: These assertions only check that the bounds maps have two entries — they'd pass even if every bound were zeroed or the min/max were swapped, so the stats path isn't really being exercised. For `id` 1-3 and `name` alpha/gamma the bounds are computable, so I'd decode at least one field with `iceberg.LiteralFromBytes` and assert the actual lower/upper values. While we're here, all three tests are happy-path and near-identical skeletons — I'd fold them into one table-driven `TestDataFileFromMetadata` with subtests and add the error cases (nil schema, nil/wrong-type metadata, empty path, non-positive size, invalid content, eq-delete with empty `EqualityFieldIDs`). Several of those paths panic today rather than error, so the table is how a fix here actually gets caught. ########## table/data_file_meta.go: ########## @@ -0,0 +1,178 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package table + +import ( + "errors" + "fmt" + + "github.com/apache/iceberg-go" + tblutils "github.com/apache/iceberg-go/table/internal" +) + +// DataFileArgs collects everything needed to compute a fully populated +// [iceberg.DataFile] (data, equality-delete, or positional-delete) from +// a file's in-memory, format-specific metadata object without performing +// any filesystem reads. +type DataFileArgs struct { + // Schema is the Iceberg schema the file's contents conform to. The + // file's columns MUST carry per-leaf field-ID metadata whose values + // match this schema's field IDs (for Parquet this is the + // "PARQUET:field_id" key on each leaf). + Schema *iceberg.Schema + + // Spec is the partition spec the resulting DataFile is bound to — + // typically tbl.Spec(). For an unpartitioned table pass + // *iceberg.UnpartitionedSpec. + Spec iceberg.PartitionSpec + + // Format identifies the on-disk file format of the file being + // described and selects the internal FileFormat implementation used + // to extract per-column statistics from Metadata. + Format iceberg.FileFormat + + // Metadata is the format-specific, in-memory file metadata object + // produced by the external writer (for example + // *parquet/metadata.FileMetaData when Format is + // [iceberg.ParquetFile]). + Metadata any + + // FilePath is the fully qualified location the file bytes were + // written to (the same path that will be recorded in the manifest entry). + FilePath string + + // FileSize is the byte length of the object at FilePath. Must be + // greater than zero. + FileSize int64 + + // Content selects which manifest-entry kind to produce — one of + // [iceberg.EntryContentData], [iceberg.EntryContentEqDeletes] or + // [iceberg.EntryContentPosDeletes]. + Content iceberg.ManifestEntryContent + + // PartitionValues maps spec field ID → partition value. Required when Spec + // is partitioned; leave nil/empty for unpartitioned tables. + PartitionValues map[int]any + + // SortOrderID is the sort-order ID recorded on the manifest entry, + // typically tbl.SortOrder().OrderID(). Zero means "unsorted". + SortOrderID int + + // Properties are consulted only for write-side metrics-mode keys + // (write.metadata.metrics.default and per-column + // write.metadata.metrics.column.<name> overrides) that decide + // which columns get full vs truncated vs no statistics. Pass + // tbl.Properties() for table-default behavior; nil is safe. + Properties iceberg.Properties + + // EqualityFieldIDs lists the schema field IDs that an equality + // delete file matches on. Required when + // Content == [iceberg.EntryContentEqDeletes] and MUST be empty + // for every other Content value. + EqualityFieldIDs []int +} + +// DataFileFromMetadata builds a fully populated [iceberg.DataFile] from a +// file's in-memory, format-specific metadata object, extracting per-column +// statistics (column_sizes, value_counts, null_value_counts, lower_bounds, +// upper_bounds), split offsets, and the manifest-entry shape required for +// the requested content type. The format-specific extraction is dispatched +// through the internal FileFormat interface; no filesystem reads are +// performed. +// +// The returned DataFile is intended to be handed to +// [Transaction.AddDataFiles] (data files) or [Transaction.NewRowDelta] +// (delete files); it is NOT committed automatically. The file bytes +// described by args.Metadata must already have been uploaded to +// args.FilePath. +func DataFileFromMetadata(args DataFileArgs) (iceberg.DataFile, error) { + if args.Schema == nil { + return nil, errors.New("schema is required") + } + if args.Metadata == nil { + return nil, errors.New("file metadata is required") + } + if args.FilePath == "" { + return nil, errors.New("file path is required") + } + if args.FileSize <= 0 { + return nil, fmt.Errorf("file size must be greater than 0, got %d", args.FileSize) + } + + switch args.Content { + case iceberg.EntryContentData, + iceberg.EntryContentEqDeletes, + iceberg.EntryContentPosDeletes: + default: + return nil, fmt.Errorf("invalid content: %v", args.Content) + } + + if args.Content == iceberg.EntryContentEqDeletes && len(args.EqualityFieldIDs) == 0 { + return nil, errors.New("equalityFieldIDs is required for equality-delete files") Review Comment: Small one: these messages say `equalityFieldIDs` but the exported field is `EqualityFieldIDs`, and the rest of the codebase uses the exported casing (e.g. `row_delta.go:134` "equality delete file must have non-empty EqualityFieldIDs"). Lowercase doesn't grep back to the symbol — I'd capitalize both. -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: [email protected] For queries about this service, please contact Infrastructure at: [email protected] --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
