rdblue commented on code in PR #11039: URL: https://github.com/apache/iceberg/pull/11039#discussion_r1755564516
########## gcp/src/main/java/org/apache/iceberg/gcp/bigquery/BigQueryClientImpl.java: ########## @@ -0,0 +1,481 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.iceberg.gcp.bigquery; + +import com.google.api.client.googleapis.javanet.GoogleNetHttpTransport; +import com.google.api.client.http.HttpHeaders; +import com.google.api.client.http.HttpResponse; +import com.google.api.client.http.HttpResponseException; +import com.google.api.client.http.HttpStatusCodes; +import com.google.api.client.json.gson.GsonFactory; +import com.google.api.client.util.Data; +import com.google.api.services.bigquery.Bigquery; +import com.google.api.services.bigquery.BigqueryScopes; +import com.google.api.services.bigquery.model.Dataset; +import com.google.api.services.bigquery.model.DatasetList; +import com.google.api.services.bigquery.model.DatasetList.Datasets; +import com.google.api.services.bigquery.model.DatasetReference; +import com.google.api.services.bigquery.model.ExternalCatalogDatasetOptions; +import com.google.api.services.bigquery.model.ExternalCatalogTableOptions; +import com.google.api.services.bigquery.model.Table; +import com.google.api.services.bigquery.model.TableList; +import com.google.api.services.bigquery.model.TableList.Tables; +import com.google.api.services.bigquery.model.TableReference; +import com.google.api.services.bigquery.model.TableSchema; +import com.google.auth.http.HttpCredentialsAdapter; +import com.google.auth.oauth2.GoogleCredentials; +import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.security.GeneralSecurityException; +import java.util.List; +import java.util.Locale; +import java.util.Map; +import java.util.Set; +import java.util.stream.Collectors; +import java.util.stream.Stream; +import org.apache.iceberg.BaseMetastoreTableOperations; +import org.apache.iceberg.exceptions.BadRequestException; +import org.apache.iceberg.exceptions.ForbiddenException; +import org.apache.iceberg.exceptions.NoSuchIcebergTableException; +import org.apache.iceberg.exceptions.NoSuchNamespaceException; +import org.apache.iceberg.exceptions.NoSuchTableException; +import org.apache.iceberg.exceptions.NotAuthorizedException; +import org.apache.iceberg.exceptions.NotFoundException; +import org.apache.iceberg.exceptions.RuntimeIOException; +import org.apache.iceberg.exceptions.ServiceFailureException; +import org.apache.iceberg.exceptions.ServiceUnavailableException; +import org.apache.iceberg.exceptions.ValidationException; +import org.apache.iceberg.relocated.com.google.common.annotations.VisibleForTesting; +import org.apache.iceberg.relocated.com.google.common.base.Preconditions; +import org.apache.iceberg.relocated.com.google.common.collect.Lists; +import org.apache.iceberg.relocated.com.google.common.collect.Maps; + +/** A client of Google Bigquery Metastore functions over the BigQuery service. */ +public final class BigQueryClientImpl implements BigQueryClient { + + public static final String NOT_AUTHORIZED_ERROR_MESSAGE = + "Not authorized to call the BigQuery API or access this resource"; + private final Bigquery client; + + /** Constructs a client of the Google BigQuery service. */ + public BigQueryClientImpl() throws IOException, GeneralSecurityException { + // Initialize client that will be used to send requests. This client only needs to be created + // once, and can be reused for multiple requests + HttpCredentialsAdapter httpCredentialsAdapter = + new HttpCredentialsAdapter( + GoogleCredentials.getApplicationDefault().createScoped(BigqueryScopes.all())); + this.client = + new Bigquery.Builder( + GoogleNetHttpTransport.newTrustedTransport(), + GsonFactory.getDefaultInstance(), + httpRequest -> { + httpCredentialsAdapter.initialize(httpRequest); + httpRequest.setThrowExceptionOnExecuteError( + false); // Less catching of exceptions, more analysis of the same HttpResponse + // object, inspecting its status code + }) + .setApplicationName("BigQuery Iceberg Catalog Plugin") + .build(); + } + + @VisibleForTesting + BigQueryClientImpl(Bigquery client) { + this.client = client; + } + + @Override + public Dataset createDataset(Dataset dataset) { + try { + HttpResponse response = + client + .datasets() + .insert(dataset.getDatasetReference().getProjectId(), dataset) + .executeUnparsed(); + return convertExceptionIfUnsuccessful(response).parseAs(Dataset.class); + } catch (IOException e) { + throw new RuntimeIOException(e); + } + } + + @Override + @SuppressWarnings("FormatStringAnnotation") + public Dataset getDataset(DatasetReference datasetReference) { + try { + HttpResponse response = + client + .datasets() + .get(datasetReference.getProjectId(), datasetReference.getDatasetId()) + .executeUnparsed(); + if (response.getStatusCode() == HttpStatusCodes.STATUS_CODE_NOT_FOUND) { + throw new NoSuchNamespaceException(response.getStatusMessage()); + } + return convertExceptionIfUnsuccessful(response).parseAs(Dataset.class); + } catch (IOException e) { + throw new RuntimeIOException(e); + } + } + + @Override + @SuppressWarnings("FormatStringAnnotation") + public void deleteDataset(DatasetReference datasetReference) { + try { + HttpResponse response = + client + .datasets() + .delete(datasetReference.getProjectId(), datasetReference.getDatasetId()) + .executeUnparsed(); + if (response.getStatusCode() == HttpStatusCodes.STATUS_CODE_NOT_FOUND) { + throw new NoSuchNamespaceException(response.getStatusMessage()); + } + convertExceptionIfUnsuccessful(response); + } catch (IOException e) { + throw new RuntimeIOException(e); + } + } + + @Override + public Dataset setDatasetParameters( + DatasetReference datasetReference, Map<String, String> parameters) { + Dataset dataset = getDataset(datasetReference); + ExternalCatalogDatasetOptions externalCatalogDatasetOptions = + dataset.getExternalCatalogDatasetOptions() == null + ? new ExternalCatalogDatasetOptions() + : dataset.getExternalCatalogDatasetOptions(); + Map<String, String> finalParameters = + externalCatalogDatasetOptions.getParameters() == null + ? Maps.newHashMap() + : externalCatalogDatasetOptions.getParameters(); + finalParameters.putAll(parameters); + + dataset.setExternalCatalogDatasetOptions( + externalCatalogDatasetOptions.setParameters(finalParameters)); + + return updateDataset(dataset); + } + + @Override + public Dataset removeDatasetParameters( + DatasetReference datasetReference, Set<String> parameters) { + Dataset dataset = getDataset(datasetReference); + ExternalCatalogDatasetOptions externalCatalogDatasetOptions = + dataset.getExternalCatalogDatasetOptions() == null + ? new ExternalCatalogDatasetOptions() + : dataset.getExternalCatalogDatasetOptions(); + Map<String, String> finalParameters = + externalCatalogDatasetOptions.getParameters() == null + ? Maps.newHashMap() + : externalCatalogDatasetOptions.getParameters(); + parameters.forEach(finalParameters::remove); + + dataset.setExternalCatalogDatasetOptions( + externalCatalogDatasetOptions.setParameters(finalParameters)); + + return updateDataset(dataset); + } + + @Override + public List<Datasets> listDatasets(String projectId) { + try { + String nextPageToken = null; + List<Datasets> datasets = Lists.newArrayList(); + do { + HttpResponse pageResponse = + client.datasets().list(projectId).setPageToken(nextPageToken).executeUnparsed(); + DatasetList result = + convertExceptionIfUnsuccessful(pageResponse).parseAs(DatasetList.class); + nextPageToken = result.getNextPageToken(); + datasets.addAll(result.getDatasets()); + } while (nextPageToken != null && !nextPageToken.isEmpty()); + return datasets; + } catch (IOException e) { + throw new RuntimeIOException(e); + } + } + + @Override + public Table createTable(Table table) { + try { + validateTable(table); + HttpResponse response = + client + .tables() + .insert( + Preconditions.checkNotNull(table.getTableReference()).getProjectId(), + Preconditions.checkNotNull(table.getTableReference()).getDatasetId(), + table) + .executeUnparsed(); + return convertExceptionIfUnsuccessful(response).parseAs(Table.class); + } catch (IOException e) { + throw new RuntimeIOException(e); + } + } + + @Override + @SuppressWarnings("FormatStringAnnotation") + public Table getTable(TableReference tableReference) { + try { + HttpResponse response = + client + .tables() + .get( + tableReference.getProjectId(), + tableReference.getDatasetId(), + tableReference.getTableId()) + .executeUnparsed(); + if (response.getStatusCode() == HttpStatusCodes.STATUS_CODE_NOT_FOUND) { + throw new NoSuchTableException(response.getStatusMessage()); + } + return validateTable(convertExceptionIfUnsuccessful(response).parseAs(Table.class)); + } catch (IOException e) { + throw new RuntimeIOException(e); + } + } + + @Override + @SuppressWarnings("FormatStringAnnotation") + public Table patchTable(TableReference tableReference, Table table) { + validateTable(table); // Ensure it is an Iceberg table supported by the catalog + + ExternalCatalogTableOptions newExternalCatalogTableOptions = + new ExternalCatalogTableOptions() + .setStorageDescriptor(table.getExternalCatalogTableOptions().getStorageDescriptor()) + .setConnectionId(table.getExternalCatalogTableOptions().getConnectionId()) + .setParameters(table.getExternalCatalogTableOptions().getParameters()); + Table patch = + new Table() + .setExternalCatalogTableOptions(newExternalCatalogTableOptions) + // TODO(b/341933455): Update this once the server side accepts schema. + // Must set the schema as null for using schema auto-detect + .setSchema(Data.nullOf(TableSchema.class)); + + try { + HttpResponse response = + client + .tables() + .patch( + tableReference.getProjectId(), + tableReference.getDatasetId(), + tableReference.getTableId(), + patch) + .setRequestHeaders(new HttpHeaders().setIfMatch(table.getEtag())) + .executeUnparsed(); + + if (response.getStatusCode() == HttpStatusCodes.STATUS_CODE_NOT_FOUND) { + String responseString = response.parseAsString(); + if (responseString.toLowerCase(Locale.ROOT).contains("not found: connection")) { + throw new BadRequestException(responseString); + } + throw new NoSuchTableException(response.getStatusMessage()); + } + return convertExceptionIfUnsuccessful(response).parseAs(Table.class); + } catch (IOException e) { + throw new RuntimeIOException(e); + } + } + + @Override + @SuppressWarnings("FormatStringAnnotation") + public Table renameTable(TableReference tableToRename, String newTableId) { + Table table = getTable(tableToRename); // Verify table first + Table patch = + new Table() + .setTableReference( + new TableReference() + .setProjectId(table.getTableReference().getProjectId()) + .setDatasetId(table.getTableReference().getDatasetId()) + .setTableId(newTableId)); + + try { + HttpResponse response = + client + .tables() + .patch( + tableToRename.getProjectId(), + tableToRename.getDatasetId(), + tableToRename.getTableId(), + patch) + .setRequestHeaders(new HttpHeaders().setIfMatch(table.getEtag())) + .executeUnparsed(); + + if (response.getStatusCode() == HttpStatusCodes.STATUS_CODE_NOT_FOUND) { + throw new NoSuchTableException(response.getStatusMessage()); + } + return convertExceptionIfUnsuccessful(response).parseAs(Table.class); + } catch (IOException e) { + throw new RuntimeIOException(e); Review Comment: Why not add context here, like what the request was? -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: [email protected] For queries about this service, please contact Infrastructure at: [email protected] --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
