nastra commented on code in PR #12808: URL: https://github.com/apache/iceberg/pull/12808#discussion_r2059745760
########## bigquery/src/test/java/org/apache/iceberg/gcp/bigquery/TestBigQueryCatalog.java: ########## @@ -0,0 +1,673 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.iceberg.gcp.bigquery; + +import static org.apache.iceberg.CatalogUtil.ICEBERG_CATALOG_TYPE; +import static org.apache.iceberg.CatalogUtil.ICEBERG_CATALOG_TYPE_BIGQUERY; +import static org.apache.iceberg.gcp.bigquery.BigQueryMetastoreCatalog.PROJECT_ID; +import static org.apache.iceberg.gcp.bigquery.BigQueryMetastoreCatalog.TESTING_ENABLED; +import static org.apache.iceberg.types.Types.NestedField.optional; +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; + +import java.io.File; +import java.util.List; +import java.util.Map; +import org.apache.iceberg.CatalogProperties; +import org.apache.iceberg.CatalogUtil; +import org.apache.iceberg.PartitionSpec; +import org.apache.iceberg.Schema; +import org.apache.iceberg.SortOrder; +import org.apache.iceberg.Table; +import org.apache.iceberg.TableUtil; +import org.apache.iceberg.Transaction; +import org.apache.iceberg.catalog.CatalogTests; +import org.apache.iceberg.catalog.Namespace; +import org.apache.iceberg.catalog.TableIdentifier; +import org.apache.iceberg.exceptions.NoSuchNamespaceException; +import org.apache.iceberg.exceptions.NoSuchTableException; +import org.apache.iceberg.expressions.Expressions; +import org.apache.iceberg.relocated.com.google.common.collect.ImmutableMap; +import org.apache.iceberg.types.Types; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Disabled; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.ValueSource; + +public class TestBigQueryCatalog extends CatalogTests<BigQueryMetastoreCatalog> { + + private static final String GCP_PROJECT = "project-id"; + private static final String CATALOG_ID = "catalog-name"; + + // another schema that is not the same + protected static final Schema OTHER_SCHEMA_OPTIONAL = + new Schema(optional(1, "some_id", Types.IntegerType.get())); + + // Schema passed to create tables + protected static final Schema SCHEMA_OPTIONAL = + new Schema( + optional(3, "id", Types.IntegerType.get(), "unique ID 🤪"), + optional(4, "data", Types.StringType.get())); + + // This is the actual schema for the table, with column IDs reassigned + protected static final Schema REPLACE_SCHEMA_OPTIONAL = + new Schema( + optional(2, "id", Types.IntegerType.get(), "unique ID 🤪"), + optional(3, "data", Types.StringType.get())); + + protected static final PartitionSpec REPLACE_SPEC_OPTIONAL = + PartitionSpec.builderFor(REPLACE_SCHEMA_OPTIONAL).bucket("id", 16).withSpecId(1).build(); + + protected static final SortOrder REPLACE_WRITE_ORDER_OPTIONAL = + SortOrder.builderFor(REPLACE_SCHEMA_OPTIONAL) + .asc(Expressions.bucket("id", 16)) + .asc("id") + .build(); + + @TempDir private File tempFolder; + private BigQueryMetastoreCatalog catalog; + private String warehouseLocation; + + @BeforeEach + public void before() throws Exception { + catalog = initCatalog(CATALOG_ID, ImmutableMap.of()); + } + + @AfterEach + public void after() throws Exception { + // Drop all tables in all datasets + List<Namespace> namespaces = catalog().listNamespaces(Namespace.empty()); + for (Namespace namespace : namespaces) { + List<TableIdentifier> tables = catalog().listTables(namespace); + for (TableIdentifier table : tables) { + try { + catalog().dropTable(table, true); // drop with purge + } catch (NoSuchTableException e) { + // Table might be dropped by a previous iteration, ignore + } + } + + // Drop all datasets (except any initial ones) + try { + catalog().dropNamespace(namespace); + } catch (NoSuchNamespaceException e) { + // Namespace might be dropped by a previous iteration, ignore + } + } + } + + @Override + protected boolean requiresNamespaceCreate() { + return true; + } + + @Override + protected boolean supportsNamesWithSlashes() { + return false; + } + + @Override + protected boolean supportsNamesWithDot() { + return false; + } + + @Override + public BigQueryMetastoreCatalog catalog() { + return catalog; + } + + @Override + protected BigQueryMetastoreCatalog initCatalog( + String catalogName, Map<String, String> additionalProperties) { + + warehouseLocation = tempFolder.toPath().resolve("hive-warehouse").toString(); + + Map<String, String> properties = + Map.of( + ICEBERG_CATALOG_TYPE, + ICEBERG_CATALOG_TYPE_BIGQUERY, + PROJECT_ID, + GCP_PROJECT, + CatalogProperties.WAREHOUSE_LOCATION, + warehouseLocation, + TESTING_ENABLED, + "true", + CatalogProperties.TABLE_DEFAULT_PREFIX + "default-key1", + "catalog-default-key1", + CatalogProperties.TABLE_DEFAULT_PREFIX + "default-key2", + "catalog-default-key2", + CatalogProperties.TABLE_DEFAULT_PREFIX + "override-key3", + "catalog-default-key3", + CatalogProperties.TABLE_OVERRIDE_PREFIX + "override-key3", + "catalog-override-key3", + CatalogProperties.TABLE_OVERRIDE_PREFIX + "override-key4", + "catalog-override-key4"); + + BigQueryMetastoreCatalog tmpCatalog = + (BigQueryMetastoreCatalog) + CatalogUtil.buildIcebergCatalog( + catalogName, + ImmutableMap.<String, String>builder() + .putAll(properties) + .putAll(additionalProperties) + .build(), + null); + + return tmpCatalog; + } + + @Test + public void testRenameTable() { + if (requiresNamespaceCreate()) { + catalog.createNamespace(NS); + } + + assertThat(catalog.tableExists(TABLE)) + .as("Source table should not exist before create") + .isFalse(); + + catalog.buildTable(TABLE, SCHEMA).create(); + assertThat(catalog.tableExists(TABLE)).as("Table should exist after create").isTrue(); + + assertThat(catalog.tableExists(RENAMED_TABLE)) + .as("Destination table should not exist before rename") + .isFalse(); + + assertThatThrownBy(() -> catalog.renameTable(TABLE, RENAMED_TABLE)) + .isInstanceOf(UnsupportedOperationException.class) + .hasMessageContaining("Table rename operation is unsupported"); + + catalog.dropTable(TABLE); + } + + @Disabled( + "BigQuery does not allow adding a REQUIRED column to an existing table schema. " + + "When this operation is attempted, an exception gets thrown.") + @Test + public void testOverrideTablePropertiesReplaceTransaction() {} Review Comment: given that the tests here that live in the Iceberg codebase are always running against the Fake client, I would suggest to remove all of those `Disabled` tests, since they all pass against the Fake client. If you want to run the catalog tests against an actual client, you can then basically just create a new `TestRealBigQueryCatalog` (I'm sure there's a better name) that initializes the correct client where you disable the tests that don't work -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: issues-unsubscr...@iceberg.apache.org For queries about this service, please contact Infrastructure at: us...@infra.apache.org --------------------------------------------------------------------- To unsubscribe, e-mail: issues-unsubscr...@iceberg.apache.org For additional commands, e-mail: issues-h...@iceberg.apache.org
