zeroshade commented on code in PR #313: URL: https://github.com/apache/iceberg-go/pull/313#discussion_r2021316142
########## io/azure.go: ########## @@ -0,0 +1,120 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package io + +import ( + "context" + "errors" + "fmt" + "net/url" + + "github.com/Azure/azure-sdk-for-go/sdk/storage/azblob" + "github.com/Azure/azure-sdk-for-go/sdk/storage/azblob/container" + "gocloud.dev/blob" + "gocloud.dev/blob/azureblob" +) + +// Constants for Azure configuration options +const ( + AdlsSasTokenPrefix = "adls.sas-token." + AdlsConnectionStringPrefix = "adls.connection-string." + AdlsSharedKeyAccountName = "adls.auth.shared-key.account.name" + AdlsSharedKeyAccountKey = "adls.auth.shared-key.account.key" + AdlsEndpoint = "adls.endpoint" + AdlsProtocol = "adls.protocol" + + // Not in use yet + // AdlsReadBlockSize = "adls.read.block-size-bytes" + // AdlsWriteBlockSize = "adls.write.block-size-bytes" +) + +// Construct a Azure bucket from a URL +func createAzureBucket(ctx context.Context, parsed *url.URL, props map[string]string) (*blob.Bucket, error) { + adlsSasTokens := propertiesWithPrefix(props, AdlsSasTokenPrefix) + adlsConnectionStrings := propertiesWithPrefix(props, AdlsConnectionStringPrefix) + + // Construct the client + accountName := props[AdlsSharedKeyAccountName] + endpoint := props[AdlsEndpoint] + protocol := props[AdlsProtocol] Review Comment: are there defaults we should use for these? ########## io/azure.go: ########## @@ -0,0 +1,120 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package io + +import ( + "context" + "errors" + "fmt" + "net/url" + + "github.com/Azure/azure-sdk-for-go/sdk/storage/azblob" + "github.com/Azure/azure-sdk-for-go/sdk/storage/azblob/container" + "gocloud.dev/blob" + "gocloud.dev/blob/azureblob" +) + +// Constants for Azure configuration options +const ( + AdlsSasTokenPrefix = "adls.sas-token." + AdlsConnectionStringPrefix = "adls.connection-string." + AdlsSharedKeyAccountName = "adls.auth.shared-key.account.name" + AdlsSharedKeyAccountKey = "adls.auth.shared-key.account.key" + AdlsEndpoint = "adls.endpoint" + AdlsProtocol = "adls.protocol" + + // Not in use yet + // AdlsReadBlockSize = "adls.read.block-size-bytes" + // AdlsWriteBlockSize = "adls.write.block-size-bytes" +) + +// Construct a Azure bucket from a URL +func createAzureBucket(ctx context.Context, parsed *url.URL, props map[string]string) (*blob.Bucket, error) { + adlsSasTokens := propertiesWithPrefix(props, AdlsSasTokenPrefix) + adlsConnectionStrings := propertiesWithPrefix(props, AdlsConnectionStringPrefix) + + // Construct the client + accountName := props[AdlsSharedKeyAccountName] + endpoint := props[AdlsEndpoint] + protocol := props[AdlsProtocol] + + var client *container.Client + + if accountName != "" { + var sharedKeyCred *azblob.SharedKeyCredential + var err error Review Comment: i don't think you need to declare these separately outside of the conditions. ########## io/azure.go: ########## @@ -0,0 +1,120 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package io + +import ( + "context" + "errors" + "fmt" + "net/url" + + "github.com/Azure/azure-sdk-for-go/sdk/storage/azblob" + "github.com/Azure/azure-sdk-for-go/sdk/storage/azblob/container" + "gocloud.dev/blob" + "gocloud.dev/blob/azureblob" +) + +// Constants for Azure configuration options +const ( + AdlsSasTokenPrefix = "adls.sas-token." + AdlsConnectionStringPrefix = "adls.connection-string." + AdlsSharedKeyAccountName = "adls.auth.shared-key.account.name" + AdlsSharedKeyAccountKey = "adls.auth.shared-key.account.key" + AdlsEndpoint = "adls.endpoint" + AdlsProtocol = "adls.protocol" + + // Not in use yet + // AdlsReadBlockSize = "adls.read.block-size-bytes" + // AdlsWriteBlockSize = "adls.write.block-size-bytes" +) + +// Construct a Azure bucket from a URL +func createAzureBucket(ctx context.Context, parsed *url.URL, props map[string]string) (*blob.Bucket, error) { Review Comment: use `iceberg.Properties` instead of `map[string]string` ########## io/azure.go: ########## @@ -0,0 +1,120 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package io + +import ( + "context" + "errors" + "fmt" + "net/url" + + "github.com/Azure/azure-sdk-for-go/sdk/storage/azblob" + "github.com/Azure/azure-sdk-for-go/sdk/storage/azblob/container" + "gocloud.dev/blob" + "gocloud.dev/blob/azureblob" +) + +// Constants for Azure configuration options +const ( + AdlsSasTokenPrefix = "adls.sas-token." + AdlsConnectionStringPrefix = "adls.connection-string." + AdlsSharedKeyAccountName = "adls.auth.shared-key.account.name" + AdlsSharedKeyAccountKey = "adls.auth.shared-key.account.key" + AdlsEndpoint = "adls.endpoint" + AdlsProtocol = "adls.protocol" + + // Not in use yet + // AdlsReadBlockSize = "adls.read.block-size-bytes" + // AdlsWriteBlockSize = "adls.write.block-size-bytes" +) + +// Construct a Azure bucket from a URL +func createAzureBucket(ctx context.Context, parsed *url.URL, props map[string]string) (*blob.Bucket, error) { + adlsSasTokens := propertiesWithPrefix(props, AdlsSasTokenPrefix) + adlsConnectionStrings := propertiesWithPrefix(props, AdlsConnectionStringPrefix) + + // Construct the client + accountName := props[AdlsSharedKeyAccountName] + endpoint := props[AdlsEndpoint] + protocol := props[AdlsProtocol] + + var client *container.Client + + if accountName != "" { + var sharedKeyCred *azblob.SharedKeyCredential + var err error + + if accountKey, ok := props[AdlsSharedKeyAccountKey]; ok { + svcURL, err := azureblob.NewServiceURL(&azureblob.ServiceURLOptions{ + AccountName: accountName, + Protocol: protocol, + StorageDomain: endpoint, + }) + if err != nil { + return nil, err + } + containerURL, err := url.JoinPath(string(svcURL), parsed.Host) + if err != nil { + return nil, err + } + sharedKeyCred, err = azblob.NewSharedKeyCredential(accountName, accountKey) + if err != nil { + return nil, fmt.Errorf("failed azblob.NewSharedKeyCredential: %w", err) + } + + client, err = container.NewClientWithSharedKeyCredential(containerURL, sharedKeyCred, nil) + if err != nil { + return nil, fmt.Errorf("failed container.NewClientWithSharedKeyCredential: %w", err) + } + } else if sasToken, ok := adlsSasTokens[accountName]; ok { + svcURL, err := azureblob.NewServiceURL(&azureblob.ServiceURLOptions{ + AccountName: accountName, + SASToken: sasToken, + Protocol: protocol, + StorageDomain: endpoint, + }) + if err != nil { + return nil, err + } + + containerURL, err := url.JoinPath(string(svcURL), parsed.Host) + if err != nil { + return nil, err + } + + client, err = container.NewClientWithNoCredential(containerURL, nil) + if err != nil { + return nil, fmt.Errorf("failed container.NewClientWithNoCredential: %w", err) + } + } else if connectionString, ok := adlsConnectionStrings[accountName]; ok { + client, err = container.NewClientFromConnectionString(connectionString, parsed.Host, nil) + if err != nil { + return nil, fmt.Errorf("failed container.NewClientFromConnectionString: %w", err) + } + } + + bucket, err := azureblob.OpenBucket(ctx, client, nil) + if err != nil { + return nil, err + } + + return bucket, nil Review Comment: maybe just `return azureblob.OpenBucket(ctx, client, nil)` ? ########## io/azure_integration_test.go: ########## @@ -0,0 +1,148 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +//go:build integration + +package io_test + +import ( + "context" + "fmt" + "testing" + + "github.com/Azure/azure-sdk-for-go/sdk/storage/azblob" + "github.com/Azure/azure-sdk-for-go/sdk/storage/azblob/bloberror" + "github.com/apache/iceberg-go" + "github.com/apache/iceberg-go/catalog" + sqlcat "github.com/apache/iceberg-go/catalog/sql" + "github.com/apache/iceberg-go/io" + "github.com/stretchr/testify/suite" + "github.com/uptrace/bun/driver/sqliteshim" + "gocloud.dev/blob/azureblob" +) + +const ( + accountName = "devstoreaccount1" + accountKey = "Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==" + endpoint = "127.0.0.1:10010" + protocol = "http" + containerName = "warehouse" + connectionString = "DefaultEndpointsProtocol=http;AccountName=devstoreaccount1;AccountKey=Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==;BlobEndpoint=http://127.0.0.1:10010/devstoreaccount1;QueueEndpoint=http://127.0.0.1:10001/devstoreaccount1;TableEndpoint=http://127.0.0.1:10002/devstoreaccount1;"; +) + +type AzureBlobIOTestSuite struct { + suite.Suite + + ctx context.Context +} + +func (s *AzureBlobIOTestSuite) SetupTest() { + s.ctx = context.Background() + + err := createContainerIfNotExist(containerName) + s.NoError(err) Review Comment: ```go s.Require().NoError(createContainerIfNotExist(containerName) ``` ########## io/azure_integration_test.go: ########## @@ -0,0 +1,148 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +//go:build integration + +package io_test + +import ( + "context" + "fmt" + "testing" + + "github.com/Azure/azure-sdk-for-go/sdk/storage/azblob" + "github.com/Azure/azure-sdk-for-go/sdk/storage/azblob/bloberror" + "github.com/apache/iceberg-go" + "github.com/apache/iceberg-go/catalog" + sqlcat "github.com/apache/iceberg-go/catalog/sql" + "github.com/apache/iceberg-go/io" + "github.com/stretchr/testify/suite" + "github.com/uptrace/bun/driver/sqliteshim" + "gocloud.dev/blob/azureblob" +) + +const ( + accountName = "devstoreaccount1" + accountKey = "Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==" + endpoint = "127.0.0.1:10010" + protocol = "http" + containerName = "warehouse" + connectionString = "DefaultEndpointsProtocol=http;AccountName=devstoreaccount1;AccountKey=Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==;BlobEndpoint=http://127.0.0.1:10010/devstoreaccount1;QueueEndpoint=http://127.0.0.1:10001/devstoreaccount1;TableEndpoint=http://127.0.0.1:10002/devstoreaccount1;"; +) + +type AzureBlobIOTestSuite struct { + suite.Suite + + ctx context.Context +} + +func (s *AzureBlobIOTestSuite) SetupTest() { + s.ctx = context.Background() + + err := createContainerIfNotExist(containerName) + s.NoError(err) +} + +func (s *AzureBlobIOTestSuite) TestAzureBlobWarehouseKey() { + path := "iceberg-test-azure/test-table-azure" + containerName := "warehouse" + + cat, err := catalog.Load(context.Background(), "default", iceberg.Properties{ + "uri": ":memory:", + sqlcat.DriverKey: sqliteshim.ShimName, + sqlcat.DialectKey: string(sqlcat.SQLite), + "type": "sql", + io.AdlsSharedKeyAccountName: accountName, + io.AdlsSharedKeyAccountKey: accountKey, + io.AdlsEndpoint: endpoint, + io.AdlsProtocol: protocol, + }) + s.NoError(err) + s.NotNil(cat) + + c := cat.(*sqlcat.Catalog) + s.NoError(c.CreateNamespace(s.ctx, catalog.ToIdentifier("iceberg-test-azure"), nil)) + + tbl, err := c.CreateTable(s.ctx, + catalog.ToIdentifier("iceberg-test-azure", "test-table-azure"), + iceberg.NewSchema(0, iceberg.NestedField{ + Name: "id", Type: iceberg.PrimitiveTypes.Int32, Required: true, ID: 1, + }), catalog.WithLocation(fmt.Sprintf("abfs://%s/iceberg/%s", containerName, path))) + s.NoError(err) + s.NotNil(tbl) Review Comment: should add a test here afterwards to load the table and ensure the data matches. Right now you're only testing that there's no errors when it tries to perform the write, there's no confirmation that it actually wrote the right things by attempting to read it back. ########## io/azure_integration_test.go: ########## @@ -0,0 +1,148 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +//go:build integration + +package io_test + +import ( + "context" + "fmt" + "testing" + + "github.com/Azure/azure-sdk-for-go/sdk/storage/azblob" + "github.com/Azure/azure-sdk-for-go/sdk/storage/azblob/bloberror" + "github.com/apache/iceberg-go" + "github.com/apache/iceberg-go/catalog" + sqlcat "github.com/apache/iceberg-go/catalog/sql" + "github.com/apache/iceberg-go/io" + "github.com/stretchr/testify/suite" + "github.com/uptrace/bun/driver/sqliteshim" + "gocloud.dev/blob/azureblob" +) + +const ( + accountName = "devstoreaccount1" + accountKey = "Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==" + endpoint = "127.0.0.1:10010" + protocol = "http" + containerName = "warehouse" + connectionString = "DefaultEndpointsProtocol=http;AccountName=devstoreaccount1;AccountKey=Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==;BlobEndpoint=http://127.0.0.1:10010/devstoreaccount1;QueueEndpoint=http://127.0.0.1:10001/devstoreaccount1;TableEndpoint=http://127.0.0.1:10002/devstoreaccount1;"; +) + +type AzureBlobIOTestSuite struct { + suite.Suite + + ctx context.Context +} + +func (s *AzureBlobIOTestSuite) SetupTest() { + s.ctx = context.Background() + + err := createContainerIfNotExist(containerName) + s.NoError(err) +} + +func (s *AzureBlobIOTestSuite) TestAzureBlobWarehouseKey() { + path := "iceberg-test-azure/test-table-azure" + containerName := "warehouse" + + cat, err := catalog.Load(context.Background(), "default", iceberg.Properties{ + "uri": ":memory:", + sqlcat.DriverKey: sqliteshim.ShimName, + sqlcat.DialectKey: string(sqlcat.SQLite), + "type": "sql", + io.AdlsSharedKeyAccountName: accountName, + io.AdlsSharedKeyAccountKey: accountKey, + io.AdlsEndpoint: endpoint, + io.AdlsProtocol: protocol, + }) + s.NoError(err) Review Comment: use `s.Require().` for conditions like this where the test cannot continue if it fails. ########## io/azure.go: ########## @@ -0,0 +1,120 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package io + +import ( + "context" + "errors" + "fmt" + "net/url" + + "github.com/Azure/azure-sdk-for-go/sdk/storage/azblob" + "github.com/Azure/azure-sdk-for-go/sdk/storage/azblob/container" + "gocloud.dev/blob" + "gocloud.dev/blob/azureblob" +) + +// Constants for Azure configuration options +const ( + AdlsSasTokenPrefix = "adls.sas-token." + AdlsConnectionStringPrefix = "adls.connection-string." + AdlsSharedKeyAccountName = "adls.auth.shared-key.account.name" + AdlsSharedKeyAccountKey = "adls.auth.shared-key.account.key" + AdlsEndpoint = "adls.endpoint" + AdlsProtocol = "adls.protocol" + + // Not in use yet + // AdlsReadBlockSize = "adls.read.block-size-bytes" + // AdlsWriteBlockSize = "adls.write.block-size-bytes" +) + +// Construct a Azure bucket from a URL +func createAzureBucket(ctx context.Context, parsed *url.URL, props map[string]string) (*blob.Bucket, error) { + adlsSasTokens := propertiesWithPrefix(props, AdlsSasTokenPrefix) + adlsConnectionStrings := propertiesWithPrefix(props, AdlsConnectionStringPrefix) + + // Construct the client + accountName := props[AdlsSharedKeyAccountName] + endpoint := props[AdlsEndpoint] + protocol := props[AdlsProtocol] + + var client *container.Client + + if accountName != "" { Review Comment: let's make this more readable by inverting this. ```go if accountName == "" { return nil, errors.New("account name is required for azure bucket") } // rest of code ``` Just makes it easier to read ########## io/utils.go: ########## @@ -0,0 +1,31 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package io + +import "strings" + +func propertiesWithPrefix(props map[string]string, prefix string) map[string]string { Review Comment: use `iceberg.Properties` instead please. ########## io/azure_integration_test.go: ########## @@ -0,0 +1,148 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +//go:build integration + +package io_test + +import ( + "context" + "fmt" + "testing" + + "github.com/Azure/azure-sdk-for-go/sdk/storage/azblob" + "github.com/Azure/azure-sdk-for-go/sdk/storage/azblob/bloberror" + "github.com/apache/iceberg-go" + "github.com/apache/iceberg-go/catalog" + sqlcat "github.com/apache/iceberg-go/catalog/sql" + "github.com/apache/iceberg-go/io" + "github.com/stretchr/testify/suite" + "github.com/uptrace/bun/driver/sqliteshim" + "gocloud.dev/blob/azureblob" +) + +const ( + accountName = "devstoreaccount1" + accountKey = "Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==" + endpoint = "127.0.0.1:10010" + protocol = "http" + containerName = "warehouse" + connectionString = "DefaultEndpointsProtocol=http;AccountName=devstoreaccount1;AccountKey=Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==;BlobEndpoint=http://127.0.0.1:10010/devstoreaccount1;QueueEndpoint=http://127.0.0.1:10001/devstoreaccount1;TableEndpoint=http://127.0.0.1:10002/devstoreaccount1;"; +) + +type AzureBlobIOTestSuite struct { + suite.Suite + + ctx context.Context +} + +func (s *AzureBlobIOTestSuite) SetupTest() { + s.ctx = context.Background() + + err := createContainerIfNotExist(containerName) + s.NoError(err) +} + +func (s *AzureBlobIOTestSuite) TestAzureBlobWarehouseKey() { + path := "iceberg-test-azure/test-table-azure" + containerName := "warehouse" + + cat, err := catalog.Load(context.Background(), "default", iceberg.Properties{ + "uri": ":memory:", + sqlcat.DriverKey: sqliteshim.ShimName, + sqlcat.DialectKey: string(sqlcat.SQLite), + "type": "sql", + io.AdlsSharedKeyAccountName: accountName, + io.AdlsSharedKeyAccountKey: accountKey, + io.AdlsEndpoint: endpoint, + io.AdlsProtocol: protocol, + }) + s.NoError(err) + s.NotNil(cat) + + c := cat.(*sqlcat.Catalog) + s.NoError(c.CreateNamespace(s.ctx, catalog.ToIdentifier("iceberg-test-azure"), nil)) + + tbl, err := c.CreateTable(s.ctx, + catalog.ToIdentifier("iceberg-test-azure", "test-table-azure"), + iceberg.NewSchema(0, iceberg.NestedField{ + Name: "id", Type: iceberg.PrimitiveTypes.Int32, Required: true, ID: 1, + }), catalog.WithLocation(fmt.Sprintf("abfs://%s/iceberg/%s", containerName, path))) + s.NoError(err) + s.NotNil(tbl) +} + +func (s *AzureBlobIOTestSuite) TestAzuriteWarehouseConnectionString() { + path := "iceberg-test-azure/test-table-azure" + containerName := "warehouse" + + cat, err := catalog.Load(context.Background(), "default", iceberg.Properties{ + "uri": ":memory:", + sqlcat.DriverKey: sqliteshim.ShimName, + sqlcat.DialectKey: string(sqlcat.SQLite), + "type": "sql", + io.AdlsSharedKeyAccountName: accountName, + io.AdlsConnectionStringPrefix + accountName: connectionString, + }) + s.NoError(err) + + s.NotNil(cat) + + c := cat.(*sqlcat.Catalog) + ctx := context.Background() + s.NoError(c.CreateNamespace(ctx, catalog.ToIdentifier("iceberg-test-azure"), nil)) + + tbl, err := c.CreateTable(ctx, + catalog.ToIdentifier("iceberg-test-azure", "test-table-azure"), + iceberg.NewSchema(0, iceberg.NestedField{ + Name: "id", Type: iceberg.PrimitiveTypes.Int32, Required: true, ID: 1, + }), catalog.WithLocation(fmt.Sprintf("abfs://%s/iceberg/%s", containerName, path))) + s.NoError(err) + s.NotNil(tbl) Review Comment: same comment as above, add a LoadTable call ########## io/io.go: ########## @@ -258,6 +258,11 @@ func inferFileIOFromSchema(ctx context.Context, path string, props map[string]st bucket = memblob.OpenBucket(nil) case "file", "": return LocalFS{}, nil + case "abfs", "abfss", "wasb", "wasbs": Review Comment: do the different schemes mean anything important? -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: issues-unsubscr...@iceberg.apache.org For queries about this service, please contact Infrastructure at: us...@infra.apache.org --------------------------------------------------------------------- To unsubscribe, e-mail: issues-unsubscr...@iceberg.apache.org For additional commands, e-mail: issues-h...@iceberg.apache.org
