kevinjqliu commented on code in PR #2291:
URL: https://github.com/apache/iceberg-python/pull/2291#discussion_r2302404340


##########
tests/io/test_pyarrow.py:
##########
@@ -2638,3 +2638,34 @@ def test_retry_strategy_not_found() -> None:
     io = PyArrowFileIO(properties={S3_RETRY_STRATEGY_IMPL: 
"pyiceberg.DoesNotExist"})
     with pytest.warns(UserWarning, match="Could not initialize S3 retry 
strategy: pyiceberg.DoesNotExist"):
         io.new_input("s3://bucket/path/to/file")
+
+
+def test_parse_location_environment_defaults() -> None:
+    """Test that parse_location uses environment variables for defaults."""
+
+    from pyiceberg.io.pyarrow import PyArrowFileIO
+
+    # Test with default environment (no env vars set)
+    scheme, netloc, path = PyArrowFileIO.parse_location("/foo/bar")
+    assert scheme == "file"
+    assert netloc == ""
+    assert path == "/foo/bar"
+
+    # Test with properties set
+    properties = {}
+    properties["DEFAULT_SCHEME"] = "scheme"
+    properties["DEFAULT_NETLOC"] = "netloc:8000"
+
+    scheme, netloc, path = PyArrowFileIO.parse_location("/foo/bar", 
properties=properties)

Review Comment:
   nit: can we inline this for simplicity?
   ```suggestion
       scheme, netloc, path = PyArrowFileIO.parse_location("/foo/bar", 
properties={"DEFAULT_SCHEME": "scheme", "DEFAULT_NETLOC": "netloc:8000"})
   ```



##########
pyiceberg/io/pyarrow.py:
##########
@@ -387,15 +387,29 @@ def __init__(self, properties: Properties = EMPTY_DICT):
         super().__init__(properties=properties)
 
     @staticmethod
-    def parse_location(location: str) -> Tuple[str, str, str]:
-        """Return the path without the scheme."""
+    def parse_location(location: str, properties: Properties = EMPTY_DICT) -> 
Tuple[str, str, str]:
+        """Return (scheme, netloc, path) for the given location.
+
+        Uses DEFAULT_SCHEME and DEFAULT_NETLOC if scheme/netloc are missing.
+        """
         uri = urlparse(location)
+
+        default_scheme = properties.get("DEFAULT_SCHEME", "file")
+        default_netloc = properties.get("DEFAULT_NETLOC", "")
+
         if not uri.scheme:
-            return "file", uri.netloc, os.path.abspath(location)
-        elif uri.scheme in ("hdfs", "viewfs"):
-            return uri.scheme, uri.netloc, uri.path
+            scheme = default_scheme
+        if not uri.netloc:
+            netloc = default_netloc
+
+        if scheme in ("hdfs", "viewfs"):
+            return scheme, netloc, uri.path
         else:
-            return uri.scheme, uri.netloc, f"{uri.netloc}{uri.path}"
+            # For non-HDFS URIs, include netloc in the path if present
+            path = uri.path if uri.scheme else os.path.abspath(location)
+            if netloc and not path.startswith(netloc):
+                path = f"{netloc}{path}"
+            return scheme, netloc, path

Review Comment:
   i dont remember the purpose of changing the logic inside the else block. i 
thought we only need to change when `uri.scheme` is empty
   
   what do you think about this? 
   ```
           if not uri.scheme:
               # If no scheme is provided, default to file:// (local 
filesystem).
               # If properties contains DEFAULT_SCHEME, use that scheme instead 
of "file".
               default_scheme = properties.get("DEFAULT_SCHEME", "file")
               default_netloc = properties.get("DEFAULT_NETLOC", uri.netloc)
               return default_scheme, default_netloc, os.path.abspath(location)
           elif uri.scheme in ("hdfs", "viewfs"):
               return uri.scheme, uri.netloc, uri.path
           else:
               return uri.scheme, uri.netloc, f"{uri.netloc}{uri.path}"
   ```



##########
tests/io/test_pyarrow.py:
##########
@@ -2638,3 +2638,34 @@ def test_retry_strategy_not_found() -> None:
     io = PyArrowFileIO(properties={S3_RETRY_STRATEGY_IMPL: 
"pyiceberg.DoesNotExist"})
     with pytest.warns(UserWarning, match="Could not initialize S3 retry 
strategy: pyiceberg.DoesNotExist"):
         io.new_input("s3://bucket/path/to/file")
+
+
+def test_parse_location_environment_defaults() -> None:
+    """Test that parse_location uses environment variables for defaults."""
+
+    from pyiceberg.io.pyarrow import PyArrowFileIO
+
+    # Test with default environment (no env vars set)

Review Comment:
   maybe add a comment here about by default, path without scheme is parsed as 
local filesystem 



##########
tests/io/test_pyarrow.py:
##########
@@ -2638,3 +2638,34 @@ def test_retry_strategy_not_found() -> None:
     io = PyArrowFileIO(properties={S3_RETRY_STRATEGY_IMPL: 
"pyiceberg.DoesNotExist"})
     with pytest.warns(UserWarning, match="Could not initialize S3 retry 
strategy: pyiceberg.DoesNotExist"):
         io.new_input("s3://bucket/path/to/file")
+
+
+def test_parse_location_environment_defaults() -> None:
+    """Test that parse_location uses environment variables for defaults."""

Review Comment:
   nit: change references to env vars here too



-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: [email protected]

For queries about this service, please contact Infrastructure at:
[email protected]


---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]

Reply via email to