This is an automated email from the ASF dual-hosted git repository.
laskoviymishka pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/iceberg-go.git
The following commit(s) were added to refs/heads/main by this push:
new f5e2d7496 feat(io/azure): authenticate with static adls.token
credentials (#2163)
f5e2d7496 is described below
commit f5e2d7496f27c70c25481080724d3b586b0a94ab
Author: Revanth <[email protected]>
AuthorDate: Sun Oct 11 08:53:32 2026 -0500
feat(io/azure): authenticate with static adls.token credentials (#2163)
---
catalog/rest/refresh_table_credentials_test.go | 185 +++++++++++++
catalog/rest/rest.go | 8 +-
catalog/rest/vended_creds.go | 51 +++-
catalog/rest/vended_creds_test.go | 190 ++++++++++++++
internal/adlsauth/adlsauth.go | 83 ++++++
io/config.go | 16 +-
io/gocloud/azure/azure.go | 53 ++--
io/gocloud/azure/azure_test.go | 4 +-
io/gocloud/azure/token_test.go | 346 +++++++++++++++++++++++++
website/src/configuration.md | 18 +-
10 files changed, 920 insertions(+), 34 deletions(-)
diff --git a/catalog/rest/refresh_table_credentials_test.go
b/catalog/rest/refresh_table_credentials_test.go
index 81cb40e60..3170efd66 100644
--- a/catalog/rest/refresh_table_credentials_test.go
+++ b/catalog/rest/refresh_table_credentials_test.go
@@ -85,6 +85,8 @@ type credsCatalogOpts struct {
// loadTable handles GET /v1/namespaces/db/tables/tbl. When nil, the
// endpoint is left unrouted so a request to it would 404.
loadTable http.HandlerFunc
+ // routes are further handlers, keyed by request path.
+ routes map[string]http.HandlerFunc
}
func newCredsTestCatalog(t *testing.T, opts credsCatalogOpts) *rest.Catalog {
@@ -112,6 +114,9 @@ func newCredsTestCatalog(t *testing.T, opts
credsCatalogOpts) *rest.Catalog {
if opts.loadTable != nil {
mux.HandleFunc("/v1/namespaces/db/tables/tbl", opts.loadTable)
}
+ for path, handler := range opts.routes {
+ mux.HandleFunc(path, handler)
+ }
srv := httptest.NewServer(mux)
t.Cleanup(srv.Close)
@@ -466,3 +471,183 @@ func TestRefreshTableCredentialsNotFound(t *testing.T) {
assert.Nil(t, refreshed)
assert.ErrorIs(t, err, catalog.ErrNoSuchTable)
}
+
+// TestRefreshTableCredentialsClearsADLSAuthShadowingVendedToken checks that a
+// refreshed adls.token is not shadowed by shared-key, SAS-token, or
+// connection-string credentials left in the table's saved config for the same
+// storage account, which the Azure FileIO would otherwise select ahead of it.
+// Another account's credentials are kept. The recording scheme's host,
+// "warehouse", stands in for the storage account.
+func TestRefreshTableCredentialsClearsADLSAuthShadowingVendedToken(t
*testing.T) {
+ const (
+ scheme = "restcreds-stale-adls"
+ otherHost = "other.dfs.core.windows.net"
+ )
+
+ rec := &ioPropsRecorder{}
+ rec.register(t, scheme)
+
+ cat := newCredsTestCatalog(t, credsCatalogOpts{
+ creds: func(w http.ResponseWriter, _ *http.Request) {
+ assert.NoError(t,
json.NewEncoder(w).Encode(storageCredentialsBody(
+ scheme+"://warehouse/database/table",
+ map[string]string{iceio.ADLSToken:
"vended-token"})))
+ },
+ })
+ tbl, _ := newExternalTable(t, cat, scheme,
table.WithSavedConfig(iceberg.Properties{
+ iceio.ADLSSharedKeyAccountName: "warehouse",
+ iceio.ADLSSharedKeyAccountKey: "stale-key",
+ iceio.ADLSSasTokenPrefix + "warehouse": "stale-sas",
+ iceio.ADLSConnectionStringPrefix + "warehouse": "stale-conn",
+ iceio.ADLSSasTokenPrefix + otherHost: "other-sas",
+ }))
+
+ refreshed, err := cat.RefreshTableCredentials(context.Background(), tbl)
+ require.NoError(t, err)
+ _, err = refreshed.FS(context.Background())
+ require.NoError(t, err)
+
+ props := rec.lastLoad(t)
+ assert.Equal(t, "vended-token", props[iceio.ADLSToken])
+ for _, k := range []string{
+ iceio.ADLSSharedKeyAccountName,
+ iceio.ADLSSharedKeyAccountKey,
+ iceio.ADLSSasTokenPrefix + "warehouse",
+ iceio.ADLSConnectionStringPrefix + "warehouse",
+ } {
+ assert.NotContains(t, props, k, "stale credential %q must not
survive the refresh", k)
+ }
+ assert.Equal(t, "other-sas", props[iceio.ADLSSasTokenPrefix+otherHost])
+}
+
+// tableResponseCalls are the catalog methods that build a table from a
+// load-table response, each of which merges the response's storage credentials
+// over the table-wide config.
+var tableResponseCalls = []struct {
+ name string
+ call func(context.Context, *rest.Catalog) (*table.Table, error)
+}{
+ {"CreateTable", func(ctx context.Context, cat *rest.Catalog)
(*table.Table, error) {
+ return cat.CreateTable(ctx, catalog.ToIdentifier("db", "tbl"),
tableSchemaSimple)
+ }},
+ {"RegisterTable", func(ctx context.Context, cat *rest.Catalog)
(*table.Table, error) {
+ return cat.RegisterTable(ctx, catalog.ToIdentifier("db",
"tbl"), "unused-metadata-location")
+ }},
+ {"LoadTable", func(ctx context.Context, cat *rest.Catalog)
(*table.Table, error) {
+ return cat.LoadTable(ctx, catalog.ToIdentifier("db", "tbl"))
+ }},
+}
+
+// tableResponseFSProps calls each of tableResponseCalls against a server that
+// answers with a load-table response carrying tableConfig and one storage
+// credential, vended, for metadataLoc's prefix. It returns the properties each
+// call's table FileIO was built from, keyed by call name.
+func tableResponseFSProps(t *testing.T, scheme, metadataLoc string,
tableConfig, vended map[string]string) map[string]map[string]string {
+ t.Helper()
+
+ rec := &ioPropsRecorder{}
+ rec.register(t, scheme)
+
+ respond := func(w http.ResponseWriter, _ *http.Request) {
+ body :=
storageCredentialsBody(metadataLoc[:strings.LastIndex(metadataLoc,
"/metadata/")], vended)
+ body["metadata-location"] = metadataLoc
+ body["metadata"] = json.RawMessage(strings.ReplaceAll(
+ exampleTableMetadataNoSnapshotV1, "s3://",
scheme+"://"))
+ body["config"] = tableConfig
+ assert.NoError(t, json.NewEncoder(w).Encode(body))
+ }
+ cat := newCredsTestCatalog(t, credsCatalogOpts{
+ loadTable: respond,
+ routes: map[string]http.HandlerFunc{
+ "/v1/namespaces/db/tables": respond,
+ "/v1/namespaces/db/register": respond,
+ },
+ })
+
+ got := make(map[string]map[string]string, len(tableResponseCalls))
+ for _, tc := range tableResponseCalls {
+ tbl, err := tc.call(context.Background(), cat)
+ require.NoError(t, err, tc.name)
+ _, err = tbl.FS(context.Background())
+ require.NoError(t, err, tc.name)
+ got[tc.name] = rec.lastLoad(t)
+ }
+
+ return got
+}
+
+// TestTableResponseVendedADLSTokenClearsShadowingCredentials covers the
+// table-wide merge sites, where config is the whole table config rather than a
+// per-location copy. The table's FileIO serves only the metadata location's
+// account, so a vended adls.token drops what would apply to that account ahead
+// of it or set its lifetime: that account's SAS token and connection string,
+// any shared key, and every inherited credential expiry. Another account's
SAS token and
+// connection string never apply to it and are kept.
+func TestTableResponseVendedADLSTokenClearsShadowingCredentials(t *testing.T) {
+ const (
+ scheme = "restcreds-table-adls"
+ host = "acct.dfs.core.windows.net"
+ otherHost = "other.dfs.core.windows.net"
+ )
+
+ got := tableResponseFSProps(t, scheme,
+
scheme+"://container@"+host+"/table/metadata/00000-a.metadata.json",
+ map[string]string{
+ // The factory would sign this account's requests with
another
+ // account's shared key.
+ iceio.ADLSSharedKeyAccountName: "other",
+ iceio.ADLSSharedKeyAccountKey:
"other-key",
+ iceio.ADLSSasTokenPrefix + host:
"stale-sas",
+ "adls.sas-token-expires-at-ms." + host: "1000",
+ iceio.ADLSConnectionStringPrefix + "acct":
"stale-conn",
+ iceio.ADLSSasTokenPrefix + otherHost:
"other-sas",
+ "adls.sas-token-expires-at-ms." + otherHost: "1000",
+ iceio.ADLSConnectionStringPrefix + "other":
"other-conn",
+ "expiration-time": "1000",
+ },
+ map[string]string{iceio.ADLSToken: "vended-token"})
+
+ for name, props := range got {
+ assert.Equal(t, "vended-token", props[iceio.ADLSToken], name)
+ for _, k := range []string{
+ iceio.ADLSSharedKeyAccountName,
+ iceio.ADLSSharedKeyAccountKey,
+ iceio.ADLSSasTokenPrefix + host,
+ iceio.ADLSConnectionStringPrefix + "acct",
+ } {
+ assert.NotContains(t, props, k, "%s: credential %q
would be selected ahead of the token", name, k)
+ }
+ for _, k := range []string{
+ "adls.sas-token-expires-at-ms." + host,
+ "adls.sas-token-expires-at-ms." + otherHost,
+ "expiration-time",
+ } {
+ assert.NotContains(t, props, k, "%s: inherited expiry
%q would set the token's lifetime", name, k)
+ }
+ assert.Equal(t, "other-sas",
props[iceio.ADLSSasTokenPrefix+otherHost], name)
+ assert.Equal(t, "other-conn",
props[iceio.ADLSConnectionStringPrefix+"other"], name)
+ }
+}
+
+// TestTableResponseLayersPartialS3Credential pins the existing behavior at the
+// table-wide merge sites: a partial vended S3 credential is layered over the
+// table config rather than replacing the key pair there. Whether a partial
+// credential should replace instead is a separate decision from adls.token
+// support.
+func TestTableResponseLayersPartialS3Credential(t *testing.T) {
+ const scheme = "restcreds-table-s3"
+
+ got := tableResponseFSProps(t, scheme,
+
scheme+"://warehouse/database/table/metadata/00000-a.metadata.json",
+ map[string]string{
+ iceio.S3AccessKeyID: "static-key",
+ iceio.S3SecretAccessKey: "static-secret",
+ },
+ map[string]string{iceio.S3SessionToken: "vended-token"})
+
+ for name, props := range got {
+ assert.Equal(t, "static-key", props[iceio.S3AccessKeyID], name)
+ assert.Equal(t, "static-secret",
props[iceio.S3SecretAccessKey], name)
+ assert.Equal(t, "vended-token", props[iceio.S3SessionToken],
name)
+ }
+}
diff --git a/catalog/rest/rest.go b/catalog/rest/rest.go
index 2ad477634..06bd3eaae 100644
--- a/catalog/rest/rest.go
+++ b/catalog/rest/rest.go
@@ -1334,7 +1334,7 @@ func (r *Catalog) RefreshTableCredentials(ctx
context.Context, tbl *table.Table)
config := maps.Clone(r.props)
maps.Copy(config, tbl.SavedConfig())
scanCfg := maps.Clone(config)
- maps.Copy(config, resp)
+ mergeVendedCredentials(config, resp, metadataLoc)
// Keep a reporter the caller set on the table rather than reverting it
to
// the catalog default, as Refresh does.
@@ -1566,7 +1566,7 @@ func (r *Catalog) CreateTable(ctx context.Context,
identifier table.Identifier,
maps.Copy(saved, ret.Config)
scanPlanningConfig := maps.Clone(config)
credsVended := len(ret.StorageCredentials) > 0
- maps.Copy(config, resolveStorageCredentials(ret.StorageCredentials,
ret.MetadataLoc))
+ mergeVendedCredentials(config,
resolveStorageCredentials(ret.StorageCredentials, ret.MetadataLoc),
ret.MetadataLoc)
return r.tableFromResponse(ctx, identifier, ret.Metadata,
ret.MetadataLoc, config, scanPlanningConfig, credsVended, ret.Labels,
table.WithSavedConfig(saved))
}
@@ -1796,7 +1796,7 @@ func (r *Catalog) RegisterTable(ctx context.Context,
identifier table.Identifier
maps.Copy(saved, ret.Config)
scanPlanningConfig := maps.Clone(config)
credsVended := len(ret.StorageCredentials) > 0
- maps.Copy(config, resolveStorageCredentials(ret.StorageCredentials,
ret.MetadataLoc))
+ mergeVendedCredentials(config,
resolveStorageCredentials(ret.StorageCredentials, ret.MetadataLoc),
ret.MetadataLoc)
return r.tableFromResponse(ctx, identifier, ret.Metadata,
ret.MetadataLoc, config, scanPlanningConfig, credsVended, ret.Labels,
table.WithSavedConfig(saved))
}
@@ -1844,7 +1844,7 @@ func (r *Catalog) loadTableWithMode(ctx context.Context,
identifier table.Identi
maps.Copy(saved, ret.Config)
scanPlanningConfig := maps.Clone(config)
credsVended := len(ret.StorageCredentials) > 0
- maps.Copy(config, resolveStorageCredentials(ret.StorageCredentials,
ret.MetadataLoc))
+ mergeVendedCredentials(config,
resolveStorageCredentials(ret.StorageCredentials, ret.MetadataLoc),
ret.MetadataLoc)
return r.tableFromResponse(ctx, identifier, ret.Metadata,
ret.MetadataLoc, config, scanPlanningConfig, credsVended, ret.Labels,
table.WithSavedConfig(saved))
}
diff --git a/catalog/rest/vended_creds.go b/catalog/rest/vended_creds.go
index 0c8f3359e..e42c1c53f 100644
--- a/catalog/rest/vended_creds.go
+++ b/catalog/rest/vended_creds.go
@@ -30,6 +30,7 @@ import (
"time"
"github.com/apache/iceberg-go"
+ "github.com/apache/iceberg-go/internal/adlsauth"
iceio "github.com/apache/iceberg-go/io"
"golang.org/x/sync/semaphore"
)
@@ -80,13 +81,18 @@ var credentialExpiryKeys = []string{
keyExpirationTime,
}
+// isCredentialExpiryKey reports whether key is one parseCredentialExpiry
reads.
+func isCredentialExpiryKey(key string) bool {
+ return slices.Contains(credentialExpiryKeys, key) ||
+ key == keyAdlsSasExpiresAtMs ||
+ strings.HasPrefix(key, keyAdlsSasExpiresAtMs+".")
+}
+
func parseCredentialExpiry(config iceberg.Properties) (time.Time, bool) {
var earliest time.Time
found := false
for key, value := range config {
- if !slices.Contains(credentialExpiryKeys, key) &&
- key != keyAdlsSasExpiresAtMs &&
- !strings.HasPrefix(key, keyAdlsSasExpiresAtMs+".") {
+ if !isCredentialExpiryKey(key) {
continue
}
@@ -173,7 +179,7 @@ func (v *vendedCredentialRefresher) loadFS(ctx
context.Context) (iceio.IO, error
}
config = maps.Clone(v.props)
- maps.Copy(config, freshCreds)
+ mergeVendedCredentials(config, freshCreds, v.location)
}
// The IO is cached and shared by every later caller, so it must not
@@ -434,12 +440,47 @@ func (p *prefixScopedIO) propertiesForLocation(name
string) iceberg.Properties {
if credentialIndex >= 0 {
credentialConfig := p.credentials[credentialIndex].Config
clearOverriddenCredentialProperties(props, credentialConfig)
- maps.Copy(props, credentialConfig)
+ mergeVendedCredentials(props, credentialConfig, name)
}
return props
}
+// mergeVendedCredentials layers vended storage credential properties over
+// config, the properties an IO for location is built from. Every site that
+// merges storage credentials goes through here.
+//
+// A nonempty adls.token is the one case that also removes properties. config
is
+// always a copy built for one IO, and the Azure FileIO serves only location's
+// storage account, so the removals cannot reach another account's IO:
+// - Credentials the Azure FileIO selects ahead of the token would shadow it
and
+// the read would 403; see adlsauth.ClearTokenShadowingCredentials.
+// - Every inherited credential expiry is dropped, whatever credential or
+// account it was for. The IO's renewal takes the earliest expiry in
config,
+// so any of them would set the token's lifetime in place of the expiry
+// vended with the token, or the default TTL without one. A server that
+// vends adls.token for a location outside Azure is misconfigured; its IO
+// loses inherited expiries too.
+//
+// Unlike the S3 and GCS clearing in clearOverriddenCredentialProperties, which
+// keys on property presence, this keys on a nonempty value: the Azure FileIO
+// skips an empty adls.token, so clearing for one would discard working
+// credentials.
+func mergeVendedCredentials(config, credentialConfig iceberg.Properties,
location string) {
+ if credentialConfig[iceio.ADLSToken] != "" {
+ if acct, ok := adlsauth.AccountFor(location); ok {
+ adlsauth.ClearTokenShadowingCredentials(config, acct)
+ for key := range config {
+ if isCredentialExpiryKey(key) {
+ delete(config, key)
+ }
+ }
+ }
+ }
+
+ maps.Copy(config, credentialConfig)
+}
+
// clearOverriddenCredentialProperties prevents a matched credential from being
// combined with optional fields from another credential. In particular, an S3
// access/secret pair without a session token must not inherit a stale token
diff --git a/catalog/rest/vended_creds_test.go
b/catalog/rest/vended_creds_test.go
index 6a4ac6a1b..615f1eaa5 100644
--- a/catalog/rest/vended_creds_test.go
+++ b/catalog/rest/vended_creds_test.go
@@ -692,6 +692,196 @@ func TestPrefixScopedIOReplacesS3CredentialAtomically(t
*testing.T) {
assert.NotContains(t, props, keyS3TokenExpiresAtMs)
}
+func TestPrefixScopedIOClearsStaleADLSAuthForVendedToken(t *testing.T) {
+ t.Parallel()
+
+ const (
+ host = "acct.dfs.core.windows.net"
+ otherHost = "other.dfs.core.windows.net"
+ )
+
+ p := newPrefixScopedIO(context.Background(), iceberg.Properties{
+ iceio.ADLSEndpoint:
"dfs.core.windows.net",
+ iceio.ADLSSharedKeyAccountName: "acct",
+ iceio.ADLSSharedKeyAccountKey: "stale-key",
+ iceio.ADLSSasTokenPrefix + host: "stale-sas",
+ keyAdlsSasExpiresAtMs + "." + host: "1000",
+ keyAdlsSasExpiresAtMs: "1000",
+ iceio.ADLSConnectionStringPrefix + "acct":
"stale-connection-string",
+ iceio.ADLSSasTokenPrefix + otherHost: "other-sas",
+ keyAdlsSasExpiresAtMs + "." + otherHost: "1000",
+ iceio.ADLSConnectionStringPrefix + "other":
"other-connection-string",
+ }, []StorageCredential{{
+ Prefix: "abfss://container@" + host + "/",
+ Config: iceberg.Properties{iceio.ADLSToken: "vended-token"},
+ }})
+
+ props := p.propertiesForLocation("abfss://container@" + host +
"/table/metadata.json")
+
+ // The vended token and unrelated tuning properties survive.
+ assert.Equal(t, "vended-token", props[iceio.ADLSToken])
+ assert.Equal(t, "dfs.core.windows.net", props[iceio.ADLSEndpoint])
+
+ // Everything that outranks adls.token in the Azure auth chain is gone,
so the
+ // vended token is what actually authenticates the read.
+ assert.NotContains(t, props, iceio.ADLSSharedKeyAccountName)
+ assert.NotContains(t, props, iceio.ADLSSharedKeyAccountKey)
+ assert.NotContains(t, props, iceio.ADLSSasTokenPrefix+host)
+ assert.NotContains(t, props, keyAdlsSasExpiresAtMs+"."+host)
+ assert.NotContains(t, props, keyAdlsSasExpiresAtMs)
+ assert.NotContains(t, props, iceio.ADLSConnectionStringPrefix+"acct")
+
+ // Another account's SAS expiry would still set this IO's expiry, so it
goes;
+ // its SAS token and connection string never apply here and are kept.
+ assert.NotContains(t, props, keyAdlsSasExpiresAtMs+"."+otherHost)
+ assert.Equal(t, "other-sas", props[iceio.ADLSSasTokenPrefix+otherHost])
+ assert.Equal(t, "other-connection-string",
props[iceio.ADLSConnectionStringPrefix+"other"])
+}
+
+func TestPrefixScopedIOKeepsCredentialsWhenVendedADLSTokenIsEmpty(t
*testing.T) {
+ t.Parallel()
+
+ const host = "acct.dfs.core.windows.net"
+
+ p := newPrefixScopedIO(context.Background(), iceberg.Properties{
+ iceio.ADLSSharedKeyAccountName: "acct",
+ iceio.ADLSSharedKeyAccountKey: "working-key",
+ iceio.ADLSSasTokenPrefix + host: "working-sas",
+ }, []StorageCredential{{
+ Prefix: "abfss://container@" + host + "/",
+ Config: iceberg.Properties{iceio.ADLSToken: ""},
+ }})
+
+ props := p.propertiesForLocation("abfss://container@" + host +
"/table/metadata.json")
+
+ // The Azure bucket factory skips an empty adls.token, so clearing on
key
+ // presence alone would discard these working credentials and fall
through to
+ // a weaker auth method.
+ assert.Equal(t, "acct", props[iceio.ADLSSharedKeyAccountName])
+ assert.Equal(t, "working-key", props[iceio.ADLSSharedKeyAccountKey])
+ assert.Equal(t, "working-sas", props[iceio.ADLSSasTokenPrefix+host])
+}
+
+func TestVendedCredsRefreshClearsStaleADLSAuth(t *testing.T) {
+ const (
+ scheme = "vended-adls-refresh-test"
+ host = "acct.dfs.core.windows.net"
+ otherHost = "other.dfs.core.windows.net"
+ )
+
+ // loadFS calls the factory synchronously, so captured needs no lock.
+ var captured iceberg.Properties
+ iceio.Register(scheme, func(_ context.Context, _ *url.URL, props
map[string]string) (iceio.IO, error) {
+ captured = iceberg.Properties(props)
+
+ return iceio.LocalFS{}, nil
+ })
+ t.Cleanup(func() { iceio.Unregister(scheme) })
+
+ r := newTestRefresher(func(context.Context, []string)
(iceberg.Properties, error) {
+ return iceberg.Properties{iceio.ADLSToken: "fresh-token"}, nil
+ })
+ r.location = scheme + "://container@" + host + "/table/metadata.json"
+ r.props = iceberg.Properties{
+ iceio.ADLSSharedKeyAccountName: "acct",
+ iceio.ADLSSharedKeyAccountKey: "stale-key",
+ iceio.ADLSSasTokenPrefix + host: "stale-sas",
+ iceio.ADLSSasTokenPrefix + otherHost: "other-sas",
+ }
+ // Force the refresh branch: a cached IO whose credentials have expired.
+ r.cachedIO = iceio.LocalFS{}
+ r.expiresAt = time.Now().Add(-time.Hour)
+
+ _, err := r.loadFS(context.Background())
+ require.NoError(t, err)
+
+ // The refresher merges through mergeVendedCredentials, so the freshly
vended
+ // token is not shadowed by credentials inherited from the catalog
config.
+ assert.Equal(t, "fresh-token", captured[iceio.ADLSToken])
+ assert.NotContains(t, captured, iceio.ADLSSharedKeyAccountName)
+ assert.NotContains(t, captured, iceio.ADLSSharedKeyAccountKey)
+ assert.NotContains(t, captured, iceio.ADLSSasTokenPrefix+host)
+ assert.Equal(t, "other-sas",
captured[iceio.ADLSSasTokenPrefix+otherHost])
+}
+
+// TestVendedCredsRefreshedADLSTokenExpiry checks that a vended adls.token gets
+// its expiry from expiration-time, or the default TTL without one, and never
+// from an inherited expiry, whether a SAS expiry for its own account or
another
+// or an expiry for another kind of credential, which would otherwise win as
the
+// earliest expiry. An already-expired one would make every
+// later filesystem load fetch credentials and rebuild the IO again.
+func TestVendedCredsRefreshedADLSTokenExpiry(t *testing.T) {
+ const (
+ scheme = "vended-adls-expiry-test"
+ host = "acct.dfs.core.windows.net"
+ otherHost = "other.dfs.core.windows.net"
+ )
+
+ iceio.Register(scheme, func(context.Context, *url.URL,
map[string]string) (iceio.IO, error) {
+ return iceio.LocalFS{}, nil
+ })
+ t.Cleanup(func() { iceio.Unregister(scheme) })
+
+ now := time.Now()
+ tokenExpiry := now.Add(45 * time.Minute)
+ staleExpiry := strconv.FormatInt(now.Add(-time.Hour).UnixMilli(), 10)
+
+ for _, tc := range []struct {
+ name string
+ vended iceberg.Properties
+ want time.Time
+ }{
+ {
+ name: "expiration-time is honored",
+ vended: iceberg.Properties{
+ iceio.ADLSToken: "fresh-token",
+ keyExpirationTime:
strconv.FormatInt(tokenExpiry.UnixMilli(), 10),
+ },
+ want: tokenExpiry,
+ },
+ {
+ name: "no expiry falls back to the default TTL",
+ vended: iceberg.Properties{iceio.ADLSToken:
"fresh-token"},
+ want: now.Add(defaultVendedCredentialsTTL),
+ },
+ } {
+ t.Run(tc.name, func(t *testing.T) {
+ var fetches atomic.Int32
+ r := newTestRefresher(func(context.Context, []string)
(iceberg.Properties, error) {
+ fetches.Add(1)
+
+ return tc.vended, nil
+ })
+ r.nowFunc = func() time.Time { return now }
+ r.location = scheme + "://container@" + host +
"/table/metadata.json"
+ r.props = iceberg.Properties{
+ iceio.ADLSSasTokenPrefix + host: "stale-sas",
+ keyAdlsSasExpiresAtMs + "." + host: staleExpiry,
+ keyAdlsSasExpiresAtMs: staleExpiry,
+ keyS3TokenExpiresAtMs: staleExpiry,
+ keyGcsOAuthExpiresAt: staleExpiry,
+ // Overwritten when the token is vended with
its own expiration-time.
+ keyExpirationTime:
staleExpiry,
+ iceio.ADLSSasTokenPrefix + otherHost:
"other-sas",
+ keyAdlsSasExpiresAtMs + "." + otherHost:
staleExpiry,
+ }
+ // Force the refresh branch: a cached IO whose
credentials have expired.
+ r.cachedIO = iceio.LocalFS{}
+ r.expiresAt = now.Add(-time.Minute)
+
+ _, err := r.loadFS(context.Background())
+ require.NoError(t, err)
+ assert.Equal(t, tc.want.UnixMilli(),
r.expiresAt.UnixMilli())
+
+ // The token is still valid, so the next load reuses
the IO instead of
+ // fetching credentials again.
+ _, err = r.loadFS(context.Background())
+ require.NoError(t, err)
+ assert.Equal(t, int32(1), fetches.Load())
+ })
+ }
+}
+
func TestPrefixScopedIODoesNotHoldLockDuringFilesystemLoad(t *testing.T) {
loadStarted := make(chan struct{})
releaseLoad := make(chan struct{})
diff --git a/internal/adlsauth/adlsauth.go b/internal/adlsauth/adlsauth.go
new file mode 100644
index 000000000..ec14e7c23
--- /dev/null
+++ b/internal/adlsauth/adlsauth.go
@@ -0,0 +1,83 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+// Package adlsauth holds the part of the Azure FileIO's authentication order
+// that code outside io/gocloud/azure has to agree with. The REST catalog uses
it
+// to keep a vended adls.token from being shadowed by inherited credentials,
and
+// the io/gocloud/azure tests pin it against the bucket factory's actual
+// selection, so a change to the factory's order fails a test instead of
+// silently shadowing the token again.
+package adlsauth
+
+import (
+ "net/url"
+ "strings"
+
+ iceio "github.com/apache/iceberg-go/io"
+)
+
+// Account identifies the storage account an ADLS location authenticates
+// against, in the two forms the Azure bucket factory keys per-account
+// credentials by.
+type Account struct {
+ // Host keys SAS tokens, for example myaccount.dfs.core.windows.net.
+ Host string
+ // Name keys connection strings, for example myaccount.
+ Name string
+}
+
+// AccountFor returns the storage account for location, or false if location
+// has no account.
+func AccountFor(location string) (Account, bool) {
+ parsed, err := url.Parse(location)
+ if err != nil {
+ return Account{}, false
+ }
+
+ return AccountForURL(parsed)
+}
+
+// AccountForURL returns the storage account for a parsed location, or false if
+// it has no account. The Azure bucket factory derives its account through this
+// too, so credentials are cleared under the same keys they are looked up by.
+func AccountForURL(parsed *url.URL) (Account, bool) {
+ host := parsed.Hostname()
+ name, _, _ := strings.Cut(host, ".")
+ if name == "" {
+ return Account{}, false
+ }
+
+ return Account{Host: host, Name: name}, true
+}
+
+// ClearTokenShadowingCredentials deletes from props, the properties of an IO
+// for acct, the credentials the Azure bucket factory selects ahead of a
+// nonempty adls.token: any shared key, the SAS token for acct's host and the
+// connection string for acct's name. SAS tokens and connection strings for
other
+// accounts are keyed by account, never apply to acct, and are kept.
+//
+// The shared key is cleared whichever account it names, because the factory
+// does not scope it: a configured shared key signs requests to every account,
+// so one for another account would sign acct's requests with the wrong key.
+// Callers pass a copy, so the shared key stays in the configuration other
+// accounts' IO is built from.
+func ClearTokenShadowingCredentials(props map[string]string, acct Account) {
+ delete(props, iceio.ADLSSharedKeyAccountName)
+ delete(props, iceio.ADLSSharedKeyAccountKey)
+ delete(props, iceio.ADLSSasTokenPrefix+acct.Host)
+ delete(props, iceio.ADLSConnectionStringPrefix+acct.Name)
+}
diff --git a/io/config.go b/io/config.go
index 39a6602d3..bae44883c 100644
--- a/io/config.go
+++ b/io/config.go
@@ -64,9 +64,9 @@ const (
// managed identity.
//
// It is evaluated only after shared-key credentials, a SAS-token
property for
- // the target hostname, and a connection-string property for the target
account
- // name, all of which take precedence over it; if any of them is
configured,
- // this flag has no effect.
+ // the target hostname, a connection-string property for the target
account
+ // name, and a nonempty ADLSToken, all of which take precedence over
it; if any
+ // of them is configured, this flag has no effect.
//
// This property is specific to this implementation. Neither the Java
nor the
// PyIceberg implementation defines an equivalent property, so catalog
properties
@@ -85,6 +85,16 @@ const (
// "adls.client-id" at all.
ADLSClientID = "adls.client-id"
+ // ADLSToken is a static OAuth2 access token for Azure Storage, without
the
+ // "Bearer " prefix. A nonempty token takes precedence over managed
identity
+ // and the default credential chain, but not shared-key credentials, a
SAS-token
+ // property for the target hostname, or a connection-string property
for the
+ // target account name. An empty token is ignored.
+ //
+ // The token is not refreshed. Azure enforces its expiration; callers
must
+ // recreate the FileIO with a new token when it expires.
+ ADLSToken = "adls.token"
+
// Not in use yet
// ADLSReadBlockSize = "adls.read.block-size-bytes"
// ADLSWriteBlockSize = "adls.write.block-size-bytes"
diff --git a/io/gocloud/azure/azure.go b/io/gocloud/azure/azure.go
index c8af8b990..8a8b7eab1 100644
--- a/io/gocloud/azure/azure.go
+++ b/io/gocloud/azure/azure.go
@@ -24,11 +24,14 @@ import (
"net/url"
"regexp"
"strings"
+ "time"
"github.com/Azure/azure-sdk-for-go/sdk/azcore"
+ "github.com/Azure/azure-sdk-for-go/sdk/azcore/policy"
"github.com/Azure/azure-sdk-for-go/sdk/azidentity"
"github.com/Azure/azure-sdk-for-go/sdk/storage/azblob"
"github.com/Azure/azure-sdk-for-go/sdk/storage/azblob/container"
+ "github.com/apache/iceberg-go/internal/adlsauth"
"github.com/apache/iceberg-go/io"
"github.com/apache/iceberg-go/io/gocloud/blobfs"
"gocloud.dev/blob"
@@ -52,6 +55,17 @@ type azureCredentialFactories struct {
newManagedIdentity func(*azidentity.ManagedIdentityCredentialOptions)
(azcore.TokenCredential, error)
}
+type staticTokenCredential struct {
+ token string
+}
+
+func (c staticTokenCredential) GetToken(context.Context,
policy.TokenRequestOptions) (azcore.AccessToken, error) {
+ // adls.token carries no expiration metadata. Like Java and PyIceberg,
use a
+ // one-hour SDK cache window. This does not refresh the token or extend
its
+ // actual validity, which Azure enforces.
+ return azcore.AccessToken{Token: c.token, ExpiresOn:
time.Now().Add(time.Hour)}, nil
+}
+
func productionAzureCredentialFactories() azureCredentialFactories {
return azureCredentialFactories{
newDefaultCredential: func(opts
*azidentity.DefaultAzureCredentialOptions) (azcore.TokenCredential, error) {
@@ -102,38 +116,37 @@ func newAdlsLocation(adlsURI *url.URL) (*adlsLocation,
error) {
return nil, errors.New("container name is required for azure
bucket")
}
- hostname := adlsURI.Hostname()
- if hostname == "" {
+ if adlsURI.Hostname() == "" {
return nil, errors.New("hostname is required for azure bucket")
}
- // Extract account name from hostname (ignore the storage domain)
- parts := strings.Split(hostname, ".")
- if len(parts) == 0 || parts[0] == "" {
+ // The account name is the hostname's first label (ignore the storage
domain).
+ acct, ok := adlsauth.AccountForURL(adlsURI)
+ if !ok {
return nil, errors.New("account name is required for azure
bucket")
}
- accountName := parts[0]
path := adlsURI.Path
return &adlsLocation{
- accountName: accountName,
+ accountName: acct.Name,
containerName: containerName,
- hostname: hostname,
+ hostname: acct.Host,
path: path,
}, nil
}
// Construct a Azure bucket from a URL
func createAzureBucket(ctx context.Context, parsed *url.URL, props
map[string]string) (*blob.Bucket, error) {
- return createAzureBucketWithCredentialFactories(ctx, parsed, props,
productionAzureCredentialFactories())
+ return createAzureBucketWithOptions(ctx, parsed, props,
productionAzureCredentialFactories(), nil)
}
-func createAzureBucketWithCredentialFactories(
+func createAzureBucketWithOptions(
ctx context.Context,
parsed *url.URL,
props map[string]string,
credentialFactories azureCredentialFactories,
+ clientOptions *container.ClientOptions,
) (*blob.Bucket, error) {
adlsSasTokens := propertiesWithPrefix(props, io.ADLSSasTokenPrefix)
adlsConnectionStrings := propertiesWithPrefix(props,
io.ADLSConnectionStringPrefix)
@@ -166,7 +179,7 @@ func createAzureBucketWithCredentialFactories(
return nil, fmt.Errorf("failed
azblob.NewSharedKeyCredential: %w", err)
}
- client, err =
container.NewClientWithSharedKeyCredential(containerURL, sharedKeyCred, nil)
+ client, err =
container.NewClientWithSharedKeyCredential(containerURL, sharedKeyCred,
clientOptions)
if err != nil {
return nil, fmt.Errorf("failed
container.NewClientWithSharedKeyCredential: %w", err)
}
@@ -176,16 +189,26 @@ func createAzureBucketWithCredentialFactories(
return nil, err
}
- client, err = container.NewClientWithNoCredential(containerURL,
nil)
+ client, err = container.NewClientWithNoCredential(containerURL,
clientOptions)
if err != nil {
return nil, fmt.Errorf("failed
container.NewClientWithNoCredential: %w", err)
}
} else if connectionString, ok :=
adlsConnectionStrings[location.accountName]; ok {
var err error
- client, err =
container.NewClientFromConnectionString(connectionString,
location.containerName, nil)
+ client, err =
container.NewClientFromConnectionString(connectionString,
location.containerName, clientOptions)
if err != nil {
return nil, fmt.Errorf("failed
container.NewClientFromConnectionString: %w", err)
}
+ } else if token := props[io.ADLSToken]; token != "" {
+ containerURL, err := createContainerURL(location.accountName,
protocol, endpoint, "", location.containerName)
+ if err != nil {
+ return nil, err
+ }
+
+ client, err = container.NewClient(containerURL,
staticTokenCredential{token: token}, clientOptions)
+ if err != nil {
+ return nil, fmt.Errorf("failed container.NewClient:
%w", err)
+ }
} else if props[io.ADLSManagedIdentityEnabled] == "true" {
containerURL, err := createContainerURL(location.accountName,
protocol, endpoint, "", location.containerName)
if err != nil {
@@ -204,7 +227,7 @@ func createAzureBucketWithCredentialFactories(
return nil, fmt.Errorf("failed
azidentity.NewManagedIdentityCredential: %w", err)
}
- client, err = container.NewClient(containerURL, cred, nil)
+ client, err = container.NewClient(containerURL, cred,
clientOptions)
if err != nil {
return nil, fmt.Errorf("failed container.NewClient:
%w", err)
}
@@ -219,7 +242,7 @@ func createAzureBucketWithCredentialFactories(
return nil, fmt.Errorf("failed
azidentity.NewDefaultAzureCredential: %w", err)
}
- client, err = container.NewClient(containerURL, cred, nil)
+ client, err = container.NewClient(containerURL, cred,
clientOptions)
if err != nil {
return nil, fmt.Errorf("failed container.NewClient:
%w", err)
}
diff --git a/io/gocloud/azure/azure_test.go b/io/gocloud/azure/azure_test.go
index 496480df3..a51c945ab 100644
--- a/io/gocloud/azure/azure_test.go
+++ b/io/gocloud/azure/azure_test.go
@@ -80,7 +80,7 @@ func TestCreateAzureBucketDefaultCredentialCalled(t
*testing.T) {
},
}
- bucket, err := createAzureBucketWithCredentialFactories(ctx, parsedURL,
nil, credentialFactories)
+ bucket, err := createAzureBucketWithOptions(ctx, parsedURL, nil,
credentialFactories, nil)
require.NoError(t, err)
assert.NotNil(t, bucket)
@@ -160,7 +160,7 @@ func TestCreateAzureBucketManagedIdentityCredentialCalled(t
*testing.T) {
},
}
- bucket, err :=
createAzureBucketWithCredentialFactories(ctx, parsedURL, test.props,
credentialFactories)
+ bucket, err := createAzureBucketWithOptions(ctx,
parsedURL, test.props, credentialFactories, nil)
require.NoError(t, err)
assert.NotNil(t, bucket)
diff --git a/io/gocloud/azure/token_test.go b/io/gocloud/azure/token_test.go
new file mode 100644
index 000000000..2fde1f744
--- /dev/null
+++ b/io/gocloud/azure/token_test.go
@@ -0,0 +1,346 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package azure
+
+import (
+ "context"
+ "io"
+ "maps"
+ "net/http"
+ "net/url"
+ "strings"
+ "testing"
+ "time"
+
+ "github.com/Azure/azure-sdk-for-go/sdk/azcore"
+ "github.com/Azure/azure-sdk-for-go/sdk/azcore/fake"
+ "github.com/Azure/azure-sdk-for-go/sdk/azcore/policy"
+ "github.com/Azure/azure-sdk-for-go/sdk/azidentity"
+ "github.com/Azure/azure-sdk-for-go/sdk/storage/azblob/container"
+ "github.com/apache/iceberg-go/internal/adlsauth"
+ icebergio "github.com/apache/iceberg-go/io"
+ "github.com/stretchr/testify/assert"
+ "github.com/stretchr/testify/require"
+)
+
+type tokenTestTransport func(*http.Request) (*http.Response, error)
+
+func (f tokenTestTransport) Do(req *http.Request) (*http.Response, error) {
+ return f(req)
+}
+
+func TestStaticTokenCredential(t *testing.T) {
+ // Treat access tokens as opaque: this need not be a parseable JWT.
+ credential := staticTokenCredential{token: "opaque-access-token"}
+ for range 2 {
+ before := time.Now().Add(time.Hour)
+ token, err := credential.GetToken(t.Context(),
policy.TokenRequestOptions{
+ Scopes: []string{"https://storage.azure.com/.default"},
+ })
+ after := time.Now().Add(time.Hour)
+ require.NoError(t, err)
+ assert.Equal(t, "opaque-access-token", token.Token)
+ assert.False(t, token.ExpiresOn.Before(before))
+ assert.False(t, token.ExpiresOn.After(after))
+ }
+}
+
+func TestAzureTokenAuthentication(t *testing.T) {
+ const token = "opaque-access-token"
+ tests := []struct {
+ name string
+ props map[string]string
+ wantAuthPrefix string
+ wantSAS string
+ wantDefault bool
+ wantManaged bool
+ wantCreateError string
+ rejectToken bool
+ }{
+ {
+ name: "static token",
+ props: map[string]string{icebergio.ADLSToken:
token},
+ wantAuthPrefix: "Bearer " + token,
+ },
+ {
+ name: "token precedes managed identity",
+ props: map[string]string{
+ icebergio.ADLSToken: token,
+ icebergio.ADLSManagedIdentityEnabled: "true",
+ icebergio.ADLSClientID:
"managed-identity-client-id",
+ },
+ wantAuthPrefix: "Bearer " + token,
+ },
+ {
+ name: "unrelated SAS and connection string do not
override token",
+ props: map[string]string{
+ icebergio.ADLSToken: token,
+ icebergio.ADLSSasTokenPrefix +
"other.dfs.core.windows.net": "sig=other",
+ icebergio.ADLSConnectionStringPrefix + "other":
"invalid-connection-string",
+ },
+ wantAuthPrefix: "Bearer " + token,
+ },
+ {
+ name: "shared key precedes token",
+ props: map[string]string{
+ icebergio.ADLSToken: token,
+ icebergio.ADLSSharedKeyAccountName:
"testaccount",
+ icebergio.ADLSSharedKeyAccountKey: "YQ==",
+ },
+ wantAuthPrefix: "SharedKey testaccount:",
+ },
+ {
+ name: "SAS precedes token",
+ props: map[string]string{
+ icebergio.ADLSToken: token,
+ icebergio.ADLSSasTokenPrefix +
"testaccount.dfs.core.windows.net": "sig=sas-signature",
+ },
+ wantSAS: "sas-signature",
+ },
+ {
+ name: "connection string precedes token",
+ props: map[string]string{
+ icebergio.ADLSToken: token,
+ icebergio.ADLSConnectionStringPrefix +
"testaccount":
"DefaultEndpointsProtocol=https;AccountName=connectionaccount;AccountKey=YQ==;EndpointSuffix=core.windows.net",
+ },
+ wantAuthPrefix: "SharedKey connectionaccount:",
+ },
+ {
+ name: "empty token uses default chain",
+ props: map[string]string{icebergio.ADLSToken:
""},
+ wantAuthPrefix: "Bearer fake_token",
+ wantDefault: true,
+ },
+ {
+ name: "empty token uses managed identity when enabled",
+ props: map[string]string{
+ icebergio.ADLSToken: "",
+ icebergio.ADLSManagedIdentityEnabled: "true",
+ },
+ wantAuthPrefix: "Bearer fake_token",
+ wantManaged: true,
+ },
+ {
+ name: "incomplete shared key does not fall back to
token",
+ props: map[string]string{
+ icebergio.ADLSToken: token,
+ icebergio.ADLSSharedKeyAccountName:
"testaccount",
+ },
+ wantCreateError: "shared-key requires both",
+ },
+ {
+ name: "invalid connection string does not fall back to
token",
+ props: map[string]string{
+ icebergio.ADLSToken: token,
+ icebergio.ADLSConnectionStringPrefix +
"testaccount": "invalid-connection-string",
+ },
+ wantCreateError: "failed
container.NewClientFromConnectionString",
+ },
+ {
+ name: "rejected token does not fall back to managed
identity",
+ props: map[string]string{
+ icebergio.ADLSToken: token,
+ icebergio.ADLSManagedIdentityEnabled: "true",
+ },
+ wantAuthPrefix: "Bearer " + token,
+ rejectToken: true,
+ },
+ }
+
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ parsed, err :=
url.Parse("abfss://[email protected]/path")
+ require.NoError(t, err)
+ defaultCalled, managedCalled := false, false
+ factories := azureCredentialFactories{
+ newDefaultCredential:
func(*azidentity.DefaultAzureCredentialOptions) (azcore.TokenCredential, error)
{
+ defaultCalled = true
+
+ return &fake.TokenCredential{}, nil
+ },
+ newManagedIdentity:
func(*azidentity.ManagedIdentityCredentialOptions) (azcore.TokenCredential,
error) {
+ managedCalled = true
+
+ return &fake.TokenCredential{}, nil
+ },
+ }
+ requests := 0
+ options := &container.ClientOptions{ClientOptions:
azcore.ClientOptions{
+ Retry: policy.RetryOptions{MaxRetries: -1},
+ Transport: tokenTestTransport(func(req
*http.Request) (*http.Response, error) {
+ requests++
+ auth := req.Header.Get("Authorization")
+ if strings.HasPrefix(tt.wantAuthPrefix,
"SharedKey ") {
+ assert.True(t,
strings.HasPrefix(auth, tt.wantAuthPrefix))
+ } else {
+ assert.Equal(t,
tt.wantAuthPrefix, auth)
+ }
+ assert.Equal(t, tt.wantSAS,
req.URL.Query().Get("sig"))
+ assert.NotContains(t, req.URL.String(),
token)
+ status := http.StatusOK
+ header := http.Header{}
+ header.Set("Last-Modified", "Mon, 21
Sep 2026 00:00:00 GMT")
+ header.Set("x-ms-creation-time", "Mon,
21 Sep 2026 00:00:00 GMT")
+ if tt.rejectToken {
+ status = http.StatusUnauthorized
+ header.Set("x-ms-error-code",
"AuthenticationFailed")
+ }
+
+ return &http.Response{
+ StatusCode: status,
+ Header: header,
+ Body:
io.NopCloser(strings.NewReader("")),
+ Request: req,
+ }, nil
+ }),
+ }}
+ bucket, err :=
createAzureBucketWithOptions(t.Context(), parsed, tt.props, factories, options)
+ if tt.wantCreateError != "" {
+ require.ErrorContains(t, err,
tt.wantCreateError)
+ assert.NotContains(t, err.Error(), token)
+ assert.Zero(t, requests)
+ } else {
+ require.NoError(t, err)
+ defer bucket.Close()
+ exists, err :=
bucket.Exists(context.Background(), "file.parquet")
+ if tt.rejectToken {
+ require.Error(t, err)
+ var responseErr *azcore.ResponseError
+ require.ErrorAs(t, err, &responseErr)
+ assert.Equal(t,
http.StatusUnauthorized, responseErr.StatusCode)
+ assert.NotContains(t, err.Error(),
token)
+ } else {
+ require.NoError(t, err)
+ assert.True(t, exists)
+ }
+ assert.Equal(t, 1, requests)
+ }
+ assert.Equal(t, tt.wantDefault, defaultCalled)
+ assert.Equal(t, tt.wantManaged, managedCalled)
+ })
+ }
+}
+
+// TestClearTokenShadowingCredentialsSelectsToken pins adlsauth, which the REST
+// catalog uses to keep a vended adls.token from being shadowed, to this
+// package's precedence order: once it has cleared a location's account, the
+// factory must authenticate with the token. If the factory gains or reorders
an
+// auth method ahead of the token, this fails instead of the token silently
+// losing to a stale credential.
+func TestClearTokenShadowingCredentialsSelectsToken(t *testing.T) {
+ const (
+ token = "opaque-access-token"
+ location =
"abfss://[email protected]/path"
+ )
+ const wantAuth = "Bearer " + token
+ tests := []struct {
+ name string
+ props map[string]string
+ }{
+ {
+ name: "shared key for the account",
+ props: map[string]string{
+ icebergio.ADLSSharedKeyAccountName:
"testaccount",
+ icebergio.ADLSSharedKeyAccountKey: "YQ==",
+ },
+ },
+ {
+ name: "SAS for the account",
+ props: map[string]string{
+ icebergio.ADLSSasTokenPrefix +
"testaccount.dfs.core.windows.net": "sig=sas-signature",
+ },
+ },
+ {
+ name: "connection string for the account",
+ props: map[string]string{
+ icebergio.ADLSConnectionStringPrefix +
"testaccount":
"DefaultEndpointsProtocol=https;AccountName=testaccount;AccountKey=YQ==;EndpointSuffix=core.windows.net",
+ },
+ },
+ {
+ name: "every method for the account, plus another
account's",
+ props: map[string]string{
+ icebergio.ADLSSharedKeyAccountName:
"testaccount",
+ icebergio.ADLSSharedKeyAccountKey:
"YQ==",
+ icebergio.ADLSSasTokenPrefix +
"testaccount.dfs.core.windows.net": "sig=sas-signature",
+ icebergio.ADLSConnectionStringPrefix +
"testaccount": "invalid-connection-string",
+ icebergio.ADLSSasTokenPrefix +
"other.dfs.core.windows.net": "sig=other",
+ icebergio.ADLSConnectionStringPrefix + "other":
"invalid-connection-string",
+ },
+ },
+ {
+ // The factory applies a shared key to every account,
so one naming
+ // another account would otherwise sign this account's
requests.
+ name: "shared key for another account",
+ props: map[string]string{
+ icebergio.ADLSSharedKeyAccountName:
"otheraccount",
+ icebergio.ADLSSharedKeyAccountKey: "YQ==",
+ },
+ },
+ }
+
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ acct, ok := adlsauth.AccountFor(location)
+ require.True(t, ok)
+ props := map[string]string{icebergio.ADLSToken: token}
+ maps.Copy(props, tt.props)
+ adlsauth.ClearTokenShadowingCredentials(props, acct)
+
+ parsed, err := url.Parse(location)
+ require.NoError(t, err)
+ factories := azureCredentialFactories{
+ newDefaultCredential:
func(*azidentity.DefaultAzureCredentialOptions) (azcore.TokenCredential, error)
{
+ t.Error("default credential chain must
not be selected")
+
+ return &fake.TokenCredential{}, nil
+ },
+ newManagedIdentity:
func(*azidentity.ManagedIdentityCredentialOptions) (azcore.TokenCredential,
error) {
+ t.Error("managed identity must not be
selected")
+
+ return &fake.TokenCredential{}, nil
+ },
+ }
+ requests := 0
+ options := &container.ClientOptions{ClientOptions:
azcore.ClientOptions{
+ Retry: policy.RetryOptions{MaxRetries: -1},
+ Transport: tokenTestTransport(func(req
*http.Request) (*http.Response, error) {
+ requests++
+ assert.Equal(t, wantAuth,
req.Header.Get("Authorization"))
+ assert.Empty(t,
req.URL.Query().Get("sig"))
+ header := http.Header{}
+ header.Set("Last-Modified", "Mon, 21
Sep 2026 00:00:00 GMT")
+ header.Set("x-ms-creation-time", "Mon,
21 Sep 2026 00:00:00 GMT")
+
+ return &http.Response{
+ StatusCode: http.StatusOK,
+ Header: header,
+ Body:
io.NopCloser(strings.NewReader("")),
+ Request: req,
+ }, nil
+ }),
+ }}
+
+ bucket, err :=
createAzureBucketWithOptions(t.Context(), parsed, props, factories, options)
+ require.NoError(t, err)
+ defer bucket.Close()
+ _, err = bucket.Exists(t.Context(), "file.parquet")
+ require.NoError(t, err)
+ assert.Equal(t, 1, requests)
+ })
+ }
+}
diff --git a/website/src/configuration.md b/website/src/configuration.md
index 0faebce78..a249117ca 100644
--- a/website/src/configuration.md
+++ b/website/src/configuration.md
@@ -227,15 +227,22 @@ Tuning properties:
### Azure Data Lake Storage / Blob
-Authentication is selected in the following order
(`io/gocloud/azure/azure.go`); the first applicable option wins, and if none of
1-4 apply the default credential chain (5) is used:
+Authentication is selected in the following order
(`io/gocloud/azure/azure.go`); the first applicable option wins, and if none of
1-5 apply the default credential chain (6) is used:
1. Shared key: a nonempty `adls.auth.shared-key.account.name` selects this
path and requires a nonempty `adls.auth.shared-key.account.key`.
2. Per-host SAS token: `adls.sas-token.<hostname>`, where `<hostname>` exactly
matches the storage account hostname (for example,
`myaccount.dfs.core.windows.net`).
3. Per-account connection string: `adls.connection-string.<account-name>` (for
example, `adls.connection-string.myaccount`).
-4. Managed identity: `adls.auth.managed-identity.enabled` set to exactly
`"true"` selects this path and uses `ManagedIdentityCredential` directly. Set
`adls.client-id` to select a user-assigned managed identity; otherwise, the
system-assigned managed identity is used.
-5. Default credential chain: `DefaultAzureCredential`, which includes managed
identity among its credential sources.
+4. Static access token: a nonempty `adls.token`, supplied without the `Bearer
` prefix.
+5. Managed identity: `adls.auth.managed-identity.enabled` set to exactly
`"true"` selects this path and uses `ManagedIdentityCredential` directly. Set
`adls.client-id` to select a user-assigned managed identity; otherwise, the
system-assigned managed identity is used.
+6. Default credential chain: `DefaultAzureCredential`, which includes managed
identity among its credential sources.
-> **Note on cross-client parity.** The two properties in step 4 diverge from
the
+`adls.token` accepts an OAuth2 access token for Azure Storage, including a
token supplied by a catalog. An empty value is ignored. A rejected or expired
token does not fall back to managed identity or the default credential chain.
+
+The token is static and is not refreshed. As in
[Java](https://github.com/apache/iceberg/blob/main/azure/src/main/java/org/apache/iceberg/azure/AzureProperties.java)
and
[PyIceberg](https://github.com/apache/iceberg-python/blob/main/pyiceberg/io/fsspec.py),
the credential reports a one-hour cache window to the Azure SDK each time the
SDK requests the token. This does not extend the token's actual validity: Azure
enforces its expiration. Callers must obtain a new token and recreate the Fi
[...]
+
+A REST catalog can also vend `adls.token` as a storage credential. The token
carries no expiry of its own, so a vended token's expiry comes from
`expiration-time` in the same credential when the catalog supplies one. Without
it, a table FileIO that can fetch new credentials assumes a 60-minute lifetime
and renews them shortly before it ends, and one that cannot, such as a FileIO
for a scan plan's credentials, treats them as never expiring. A token whose
actual lifetime is shorter than th [...]
+
+> **Note on cross-client parity.** The two properties in step 5 diverge from
the
> other clients. `adls.auth.managed-identity.enabled` is Go-specific: neither
> Iceberg Java nor PyIceberg defines a property that forces
> `ManagedIdentityCredential`, so catalog properties that set it do not carry
@@ -244,7 +251,7 @@ Authentication is selected in the following order
(`io/gocloud/azure/azure.go`);
> `adls.tenant-id`, neither of which iceberg-go defines (see the
> [PyIceberg ADLS
> configuration](https://py.iceberg.apache.org/configuration/#azure-data-lake)),
> while Java's `AzureProperties` defines no `adls.client-id` at all. Here it is
-> only ever read as a user-assigned managed identity ID, and only when step 4
is
+> only ever read as a user-assigned managed identity ID, and only when step 5
is
> selected.
Tuning properties:
@@ -255,6 +262,7 @@ Tuning properties:
| `adls.auth.shared-key.account.key` (`io.ADLSSharedKeyAccountKey`) | Account
key. |
| `adls.sas-token.<host>` (prefix `io.ADLSSasTokenPrefix`) | Per-host SAS
token. |
| `adls.connection-string.<account-name>` (prefix
`io.ADLSConnectionStringPrefix`) | Per-account connection string. |
+| `adls.token` (`io.ADLSToken`) | Static Azure Storage access token, without
the `Bearer ` prefix. Subject to the precedence above; not refreshed. |
| `adls.client-id` (`io.ADLSClientID`) | Client ID of a user-assigned managed
identity. Used only when the explicit managed-identity authentication path is
selected. |
| `adls.endpoint` (`io.ADLSEndpoint`) | Storage domain (e.g.
`blob.core.windows.net`). |
| `adls.protocol` (`io.ADLSProtocol`) | `http` or `https`. |
