zeroshade commented on code in PR #917:
URL: https://github.com/apache/iceberg-go/pull/917#discussion_r3102101733
##########
io/local.go:
##########
@@ -47,3 +48,12 @@ func (LocalFS) WriteFile(name string, content []byte) error {
func (LocalFS) Remove(name string) error {
return os.Remove(strings.TrimPrefix(name, "file://"))
}
+
+func (LocalFS) WalkDir(root string, fn fs.WalkDirFunc) error {
+ cleanRoot := strings.TrimPrefix(root, "file://")
+ if cleanRoot == "" {
+ cleanRoot = "."
+ }
Review Comment:
is this actually needed?
##########
table/orphan_cleanup.go:
##########
@@ -308,62 +307,28 @@ func (t Table) scanFiles(fs iceio.IO, location string,
cfg *orphanCleanupConfig)
return allFiles, totalSize, nil
}
-// getBucket gets the Bucket field from blob storage - absolute minimal
approach
-func getBucketName(fsys iceio.IO) stdfs.FS {
- v := reflect.ValueOf(fsys).Elem() // We know it's a pointer to struct
-
- return v.FieldByName("Bucket").Interface().(stdfs.FS)
-}
+func walkDirectory(fsys iceio.IO, root string, fn func(path string, info
stdfs.FileInfo) error) error {
+ listable, ok := fsys.(iceio.ListableIO)
+ if !ok {
+ return fmt.Errorf("IO implementation %T does not support
directory listing for %s (does not implement ListableIO)", fsys, root)
+ }
Review Comment:
can we fallback to the original impl? Does that make sense to do so? Or is
there some other way to implement a fallback?
##########
io/gocloud/blob.go:
##########
@@ -190,6 +192,64 @@ func createBlobFS(ctx context.Context, bucket
*blob.Bucket, keyExtractor KeyExtr
return &blobFileIO{Bucket: bucket, keyExtractor: keyExtractor, ctx: ctx}
}
+func (bfs *blobFileIO) WalkDir(root string, fn fs.WalkDirFunc) error {
+ parsed, err := url.Parse(root)
+ if err != nil {
+ return fmt.Errorf("invalid URL %s: %w", root, err)
+ }
+
+ walkPath := strings.TrimPrefix(parsed.Path, "/")
+ if walkPath == "" {
+ walkPath = "."
+ }
+
+ prefix := parsed.Scheme + "://"
+ if parsed.User != nil {
+ prefix += parsed.User.String() + "@"
+ }
+ prefix += parsed.Host + "/"
+
+ return fs.WalkDir(bfs.Bucket, walkPath, func(path string, d
fs.DirEntry, err error) error {
+ return fn(prefix+path, d, err)
Review Comment:
instead of `prefix+path` should we use `url.Join` or an equivalent?
##########
io/gocloud/blob.go:
##########
@@ -190,6 +192,64 @@ func createBlobFS(ctx context.Context, bucket
*blob.Bucket, keyExtractor KeyExtr
return &blobFileIO{Bucket: bucket, keyExtractor: keyExtractor, ctx: ctx}
}
+func (bfs *blobFileIO) WalkDir(root string, fn fs.WalkDirFunc) error {
+ parsed, err := url.Parse(root)
+ if err != nil {
+ return fmt.Errorf("invalid URL %s: %w", root, err)
+ }
+
+ walkPath := strings.TrimPrefix(parsed.Path, "/")
+ if walkPath == "" {
+ walkPath = "."
+ }
+
+ prefix := parsed.Scheme + "://"
+ if parsed.User != nil {
+ prefix += parsed.User.String() + "@"
+ }
+ prefix += parsed.Host + "/"
Review Comment:
Isn't this pretty much just equivalent to something like:
```go
walkPath := strings.TrimPrefix(parsed.Path, "/")
if walkPath == "" {
walkPath = "."
}
parsed.Path = ""
return fs.WalkDir(bfs.Bucket, walkPath, func(path string, d fs.DirEntry,
err error) error {
return fn(parsed.JoinPath(path).String(), d, err)
})
```
Or am I missing something?
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
