zeroshade commented on code in PR #177:
URL: https://github.com/apache/iceberg-go/pull/177#discussion_r1826113696
##########
manifest.go:
##########
@@ -567,6 +584,87 @@ func ReadManifestList(in io.Reader) ([]ManifestFile,
error) {
return out, dec.Error()
}
+// WriteManifestList writes a list of v2 manifest files to an avro file.
+func WriteManifestList(out io.Writer, files []ManifestFile) error {
+ var version int
+ if len(files) > 0 {
+ version = files[0].Version()
+ } else {
+ // No files to write
+ return nil
+ }
+
+ for _, file := range files {
+
+ if file.Version() != version {
+ return fmt.Errorf(
+ "%w: ManifestFile '%s' has non-matching version
%d instead of %d",
+ ErrInvalidArgument, file.FilePath(),
file.Version(), version,
+ )
+ }
+ }
+
+ var key string
+ switch version {
+ case 1:
+ key = internal.ManifestListV1Key
+ case 2:
+ key = internal.ManifestListV2Key
+ default:
+ return fmt.Errorf("%w: non-recognized version %d",
ErrInvalidArgument, version)
+ }
+
+ enc, err := ocf.NewEncoderWithSchema(
+ internal.AvroSchemaCache.Get(key),
+ out, ocf.WithMetadata(map[string][]byte{
+ "format-version": []byte(strconv.Itoa(version)),
+ }),
+ ocf.WithCodec(ocf.Deflate),
+ )
+ if err != nil {
+ return err
+ }
+
+ for _, file := range files {
+ if err := enc.Encode(file); err != nil {
+ return err
+ }
+ }
+
+ return enc.Close()
Review Comment:
It looks like we could potentially generalize this with some generics,
something like:
```go
func avroEncode[T ManifestFile | ManifestEntry](key string, version int,
vals []T, out io.Writer) error {
enc, err := ocf.NewEncoderWithSchema(
internal.AvroSchemaCache.Get(key),
out,
ocf.WithMetadata(map[string][]byte{"format-version":
[]byte(strconv.Itoa(version))}),
ocf.WithCodec(ocf.Deflate))
if err != nil {
return err
}
.....
```
then both `writeManfiestList` and `writeManifestEntries` can just do
```go
return avroEncode(key, version, entries, out)
```
etc.
##########
manifest.go:
##########
@@ -567,6 +584,87 @@ func ReadManifestList(in io.Reader) ([]ManifestFile,
error) {
return out, dec.Error()
}
+// WriteManifestList writes a list of v2 manifest files to an avro file.
+func WriteManifestList(out io.Writer, files []ManifestFile) error {
+ var version int
+ if len(files) > 0 {
+ version = files[0].Version()
+ } else {
+ // No files to write
+ return nil
+ }
+
+ for _, file := range files {
Review Comment:
`range files[1:]`?
##########
manifest.go:
##########
@@ -567,6 +584,87 @@ func ReadManifestList(in io.Reader) ([]ManifestFile,
error) {
return out, dec.Error()
}
+// WriteManifestList writes a list of v2 manifest files to an avro file.
+func WriteManifestList(out io.Writer, files []ManifestFile) error {
+ var version int
+ if len(files) > 0 {
+ version = files[0].Version()
+ } else {
+ // No files to write
+ return nil
+ }
Review Comment:
let's simplify this a little
```suggestion
if len(files) == 0 {
return nil
}
version := files[0].Version()
```
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: issues-unsubscr...@iceberg.apache.org
For queries about this service, please contact Infrastructure at:
us...@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscr...@iceberg.apache.org
For additional commands, e-mail: issues-h...@iceberg.apache.org
