This is an automated email from the ASF dual-hosted git repository. github-merge-queue[bot] pushed a commit to branch gh-readonly-queue/main/pr-3936-068aae50402285657b41f5357acb67672b0a053d in repository https://gitbox.apache.org/repos/asf/iceberg-python.git
commit 0a536e40259ecf85adee264d19d2080500a07b82 Author: Kaanchan Krishna <[email protected]> AuthorDate: Sat Oct 10 12:37:22 2026 +0000 PyArrow: Derive the ADLS account from the location when it is not configured (#3936) * PyArrow: Derive the ADLS account from the location when it is not configured _initialize_azure_fs took no netloc, unlike the S3 and HDFS initialisers next to it, so the account could only come from adls.account-name and the account in an abfss location was dropped. Pass the netloc through and fall back to the account in its host part. An explicit adls.account-name still wins, same precedence as FsspecFileIO. The fallback is gated on the userinfo part being present, since a container only netloc carries no account. * Remove unnecessary comment --- pyiceberg/io/pyarrow.py | 10 +++++++--- tests/io/test_pyarrow.py | 42 +++++++++++++++++++++++++++++++++++++++++- 2 files changed, 48 insertions(+), 4 deletions(-) diff --git a/pyiceberg/io/pyarrow.py b/pyiceberg/io/pyarrow.py index 2dcb8a579..e85973408 100644 --- a/pyiceberg/io/pyarrow.py +++ b/pyiceberg/io/pyarrow.py @@ -447,7 +447,7 @@ class PyArrowFileIO(FileIO): return self._initialize_gcs_fs() elif scheme in ADLS_SCHEMES: - return self._initialize_azure_fs() + return self._initialize_azure_fs(netloc) elif scheme in {"file"}: return self._initialize_local_fs() @@ -555,7 +555,7 @@ class PyArrowFileIO(FileIO): return S3FileSystem(**client_kwargs) - def _initialize_azure_fs(self) -> FileSystem: + def _initialize_azure_fs(self, netloc: str | None = None) -> FileSystem: # https://arrow.apache.org/docs/python/generated/pyarrow.fs.AzureFileSystem.html from packaging import version @@ -570,7 +570,11 @@ class PyArrowFileIO(FileIO): client_kwargs: dict[str, str] = {} - if account_name := self.properties.get(ADLS_ACCOUNT_NAME): + account_name = self.properties.get(ADLS_ACCOUNT_NAME) + if account_name is None and netloc and "@" in netloc: + account_name = netloc.rpartition("@")[2].split(".")[0] or None + + if account_name: client_kwargs["account_name"] = account_name if account_key := self.properties.get(ADLS_ACCOUNT_KEY): diff --git a/tests/io/test_pyarrow.py b/tests/io/test_pyarrow.py index 4d5d4431c..6e66ff43e 100644 --- a/tests/io/test_pyarrow.py +++ b/tests/io/test_pyarrow.py @@ -63,7 +63,7 @@ from pyiceberg.expressions import ( Or, ) from pyiceberg.expressions.literals import literal -from pyiceberg.io import S3_RETRY_STRATEGY_IMPL, S3_SSL_CA_CERT, InputStream, OutputStream, load_file_io +from pyiceberg.io import ADLS_ACCOUNT_NAME, S3_RETRY_STRATEGY_IMPL, S3_SSL_CA_CERT, InputStream, OutputStream, load_file_io from pyiceberg.io.pyarrow import ( ICEBERG_SCHEMA, PYARROW_PARQUET_FIELD_ID_KEY, @@ -3272,6 +3272,46 @@ def test__to_requested_schema_float_promotion( assert result.column(0).to_pylist() == [1.5, 2.25, 3.0, None] +@skip_if_pyarrow_too_old [email protected]("scheme", ["abfs", "abfss", "wasb", "wasbs"]) +def test_adls_account_name_from_location(scheme: str) -> None: + """The account in an account qualified location is used when the property is not set.""" + with patch("pyarrow.fs.AzureFileSystem") as mock_azure_fs: + PyArrowFileIO().fs_by_scheme(scheme, "[email protected]") + + assert mock_azure_fs.call_args.kwargs["account_name"] == "myaccount" + + +@skip_if_pyarrow_too_old +def test_adls_account_name_property_wins_over_location() -> None: + """An explicit adls.account-name is not overridden by the account in the location.""" + with patch("pyarrow.fs.AzureFileSystem") as mock_azure_fs: + PyArrowFileIO({ADLS_ACCOUNT_NAME: "configured"}).fs_by_scheme("abfss", "[email protected]") + + assert mock_azure_fs.call_args.kwargs["account_name"] == "configured" + + +@skip_if_pyarrow_too_old +def test_adls_no_account_name_from_container_only_location() -> None: + """A container only netloc carries no account, so nothing should be inferred from it.""" + with patch("pyarrow.fs.AzureFileSystem") as mock_azure_fs: + PyArrowFileIO().fs_by_scheme("abfss", "warehouse") + + assert "account_name" not in mock_azure_fs.call_args.kwargs + + +@skip_if_pyarrow_too_old +def test_adls_account_name_per_location() -> None: + """Two accounts served by one FileIO must each get their own filesystem.""" + file_io = PyArrowFileIO() + + with patch("pyarrow.fs.AzureFileSystem") as mock_azure_fs: + file_io.fs_by_scheme("abfss", "[email protected]") + file_io.fs_by_scheme("abfss", "[email protected]") + + assert [call.kwargs["account_name"] for call in mock_azure_fs.call_args_list] == ["accountone", "accounttwo"] + + def test_pyarrow_file_io_fs_by_scheme_cache() -> None: # It's better to set up multi-region minio servers for an integration test once `endpoint_url` argument # becomes available for `resolve_s3_region`
