kevinjqliu commented on code in PR #2291:
URL: https://github.com/apache/iceberg-python/pull/2291#discussion_r2302404340
##########
tests/io/test_pyarrow.py:
##########
@@ -2638,3 +2638,34 @@ def test_retry_strategy_not_found() -> None:
io = PyArrowFileIO(properties={S3_RETRY_STRATEGY_IMPL:
"pyiceberg.DoesNotExist"})
with pytest.warns(UserWarning, match="Could not initialize S3 retry
strategy: pyiceberg.DoesNotExist"):
io.new_input("s3://bucket/path/to/file")
+
+
+def test_parse_location_environment_defaults() -> None:
+ """Test that parse_location uses environment variables for defaults."""
+
+ from pyiceberg.io.pyarrow import PyArrowFileIO
+
+ # Test with default environment (no env vars set)
+ scheme, netloc, path = PyArrowFileIO.parse_location("/foo/bar")
+ assert scheme == "file"
+ assert netloc == ""
+ assert path == "/foo/bar"
+
+ # Test with properties set
+ properties = {}
+ properties["DEFAULT_SCHEME"] = "scheme"
+ properties["DEFAULT_NETLOC"] = "netloc:8000"
+
+ scheme, netloc, path = PyArrowFileIO.parse_location("/foo/bar",
properties=properties)
Review Comment:
nit: can we inline this for simplicity?
```suggestion
scheme, netloc, path = PyArrowFileIO.parse_location("/foo/bar",
properties={"DEFAULT_SCHEME": "scheme", "DEFAULT_NETLOC": "netloc:8000"})
```
##########
pyiceberg/io/pyarrow.py:
##########
@@ -387,15 +387,29 @@ def __init__(self, properties: Properties = EMPTY_DICT):
super().__init__(properties=properties)
@staticmethod
- def parse_location(location: str) -> Tuple[str, str, str]:
- """Return the path without the scheme."""
+ def parse_location(location: str, properties: Properties = EMPTY_DICT) ->
Tuple[str, str, str]:
+ """Return (scheme, netloc, path) for the given location.
+
+ Uses DEFAULT_SCHEME and DEFAULT_NETLOC if scheme/netloc are missing.
+ """
uri = urlparse(location)
+
+ default_scheme = properties.get("DEFAULT_SCHEME", "file")
+ default_netloc = properties.get("DEFAULT_NETLOC", "")
+
if not uri.scheme:
- return "file", uri.netloc, os.path.abspath(location)
- elif uri.scheme in ("hdfs", "viewfs"):
- return uri.scheme, uri.netloc, uri.path
+ scheme = default_scheme
+ if not uri.netloc:
+ netloc = default_netloc
+
+ if scheme in ("hdfs", "viewfs"):
+ return scheme, netloc, uri.path
else:
- return uri.scheme, uri.netloc, f"{uri.netloc}{uri.path}"
+ # For non-HDFS URIs, include netloc in the path if present
+ path = uri.path if uri.scheme else os.path.abspath(location)
+ if netloc and not path.startswith(netloc):
+ path = f"{netloc}{path}"
+ return scheme, netloc, path
Review Comment:
i dont remember the purpose of changing the logic inside the else block. i
thought we only need to change when `uri.scheme` is empty
what do you think about this?
```
if not uri.scheme:
# If no scheme is provided, default to file:// (local
filesystem).
# If properties contains DEFAULT_SCHEME, use that scheme instead
of "file".
default_scheme = properties.get("DEFAULT_SCHEME", "file")
default_netloc = properties.get("DEFAULT_NETLOC", uri.netloc)
return default_scheme, default_netloc, os.path.abspath(location)
elif uri.scheme in ("hdfs", "viewfs"):
return uri.scheme, uri.netloc, uri.path
else:
return uri.scheme, uri.netloc, f"{uri.netloc}{uri.path}"
```
##########
tests/io/test_pyarrow.py:
##########
@@ -2638,3 +2638,34 @@ def test_retry_strategy_not_found() -> None:
io = PyArrowFileIO(properties={S3_RETRY_STRATEGY_IMPL:
"pyiceberg.DoesNotExist"})
with pytest.warns(UserWarning, match="Could not initialize S3 retry
strategy: pyiceberg.DoesNotExist"):
io.new_input("s3://bucket/path/to/file")
+
+
+def test_parse_location_environment_defaults() -> None:
+ """Test that parse_location uses environment variables for defaults."""
+
+ from pyiceberg.io.pyarrow import PyArrowFileIO
+
+ # Test with default environment (no env vars set)
Review Comment:
maybe add a comment here about by default, path without scheme is parsed as
local filesystem
##########
tests/io/test_pyarrow.py:
##########
@@ -2638,3 +2638,34 @@ def test_retry_strategy_not_found() -> None:
io = PyArrowFileIO(properties={S3_RETRY_STRATEGY_IMPL:
"pyiceberg.DoesNotExist"})
with pytest.warns(UserWarning, match="Could not initialize S3 retry
strategy: pyiceberg.DoesNotExist"):
io.new_input("s3://bucket/path/to/file")
+
+
+def test_parse_location_environment_defaults() -> None:
+ """Test that parse_location uses environment variables for defaults."""
Review Comment:
nit: change references to env vars here too
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]