This is an automated email from the ASF dual-hosted git repository.
davsclaus pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/camel.git
The following commit(s) were added to refs/heads/main by this push:
new bdd495048da CAMEL-17885 camel google bigquery: Allow to read service
account key file from resolver (#7336)
bdd495048da is described below
commit bdd495048da891a2e856a116355e9ca6b6f58c72
Author: Claudio Miranda <clau...@claudius.com.br>
AuthorDate: Tue Apr 5 01:28:44 2022 -0300
CAMEL-17885 camel google bigquery: Allow to read service account key file
from resolver (#7336)
https://issues.apache.org/jira/browse/CAMEL-17885
* Added the serviceAccountKey as query parameter to the endpoint
* Use resolver to load the service account key content from a variety of
sources: http, base64, classpath, etc.
* Some tests were failing, related to number of expected messages, it
was due to bigquery api was using result.getTotalRows() but for DML
operations, it doesn't work. The JobStatistics.QueryStatistics should
be used instead.
* Removed unused log4j.properties
---
.../bigquery/GoogleBigQueryEndpointConfigurer.java | 6 ++++
.../bigquery/GoogleBigQueryEndpointUriFactory.java | 3 +-
.../sql/GoogleBigQuerySQLEndpointConfigurer.java | 6 ++++
.../sql/GoogleBigQuerySQLEndpointUriFactory.java | 3 +-
.../component/google/bigquery/google-bigquery.json | 1 +
.../google/bigquery/sql/google-bigquery-sql.json | 1 +
.../src/main/docs/google-bigquery-component.adoc | 21 +++++++++++-
.../main/docs/google-bigquery-sql-component.adoc | 21 +++++++++++-
.../google/bigquery/GoogleBigQueryComponent.java | 26 +++++++++------
.../bigquery/GoogleBigQueryConfiguration.java | 30 +++++++++++++++++-
.../bigquery/GoogleBigQueryConnectionFactory.java | 32 +++++++++++++------
.../google/bigquery/GoogleBigQueryEndpoint.java | 19 +++++++++--
.../bigquery/sql/GoogleBigQuerySQLComponent.java | 26 +++++++++------
.../sql/GoogleBigQuerySQLConfiguration.java | 28 +++++++++++++++-
.../bigquery/sql/GoogleBigQuerySQLEndpoint.java | 22 ++++++++++---
.../bigquery/sql/GoogleBigQuerySQLProducer.java | 13 +++++---
.../bigquery/integration/BigQueryITSupport.java | 27 +++++++++++-----
.../sql/GoogleBigQuerySQLProducerBaseTest.java | 12 +++++++
.../GoogleBigQuerySQLProducerWithParamersTest.java | 21 ++++++------
.../GoogleBigQuerySQLProducerWithPatternTest.java | 21 ++++++------
.../src/test/resources/log4j.properties | 37 ----------------------
.../src/test/resources/simple.properties | 1 +
22 files changed, 265 insertions(+), 112 deletions(-)
diff --git
a/components/camel-google/camel-google-bigquery/src/generated/java/org/apache/camel/component/google/bigquery/GoogleBigQueryEndpointConfigurer.java
b/components/camel-google/camel-google-bigquery/src/generated/java/org/apache/camel/component/google/bigquery/GoogleBigQueryEndpointConfigurer.java
index 65a33c5506e..bf8a1f67b42 100644
---
a/components/camel-google/camel-google-bigquery/src/generated/java/org/apache/camel/component/google/bigquery/GoogleBigQueryEndpointConfigurer.java
+++
b/components/camel-google/camel-google-bigquery/src/generated/java/org/apache/camel/component/google/bigquery/GoogleBigQueryEndpointConfigurer.java
@@ -25,6 +25,8 @@ public class GoogleBigQueryEndpointConfigurer extends
PropertyConfigurerSupport
case "connectionFactory":
target.getConfiguration().setConnectionFactory(property(camelContext,
org.apache.camel.component.google.bigquery.GoogleBigQueryConnectionFactory.class,
value)); return true;
case "lazystartproducer":
case "lazyStartProducer":
target.setLazyStartProducer(property(camelContext, boolean.class, value));
return true;
+ case "serviceaccountkey":
+ case "serviceAccountKey":
target.getConfiguration().setServiceAccountKey(property(camelContext,
java.lang.String.class, value)); return true;
case "useasinsertid":
case "useAsInsertId":
target.getConfiguration().setUseAsInsertId(property(camelContext,
java.lang.String.class, value)); return true;
default: return false;
@@ -43,6 +45,8 @@ public class GoogleBigQueryEndpointConfigurer extends
PropertyConfigurerSupport
case "connectionFactory": return
org.apache.camel.component.google.bigquery.GoogleBigQueryConnectionFactory.class;
case "lazystartproducer":
case "lazyStartProducer": return boolean.class;
+ case "serviceaccountkey":
+ case "serviceAccountKey": return java.lang.String.class;
case "useasinsertid":
case "useAsInsertId": return java.lang.String.class;
default: return null;
@@ -57,6 +61,8 @@ public class GoogleBigQueryEndpointConfigurer extends
PropertyConfigurerSupport
case "connectionFactory": return
target.getConfiguration().getConnectionFactory();
case "lazystartproducer":
case "lazyStartProducer": return target.isLazyStartProducer();
+ case "serviceaccountkey":
+ case "serviceAccountKey": return
target.getConfiguration().getServiceAccountKey();
case "useasinsertid":
case "useAsInsertId": return
target.getConfiguration().getUseAsInsertId();
default: return null;
diff --git
a/components/camel-google/camel-google-bigquery/src/generated/java/org/apache/camel/component/google/bigquery/GoogleBigQueryEndpointUriFactory.java
b/components/camel-google/camel-google-bigquery/src/generated/java/org/apache/camel/component/google/bigquery/GoogleBigQueryEndpointUriFactory.java
index f5b1066f207..aaea30d2099 100644
---
a/components/camel-google/camel-google-bigquery/src/generated/java/org/apache/camel/component/google/bigquery/GoogleBigQueryEndpointUriFactory.java
+++
b/components/camel-google/camel-google-bigquery/src/generated/java/org/apache/camel/component/google/bigquery/GoogleBigQueryEndpointUriFactory.java
@@ -21,11 +21,12 @@ public class GoogleBigQueryEndpointUriFactory extends
org.apache.camel.support.c
private static final Set<String> SECRET_PROPERTY_NAMES;
private static final Set<String> MULTI_VALUE_PREFIXES;
static {
- Set<String> props = new HashSet<>(6);
+ Set<String> props = new HashSet<>(7);
props.add("connectionFactory");
props.add("datasetId");
props.add("lazyStartProducer");
props.add("projectId");
+ props.add("serviceAccountKey");
props.add("tableId");
props.add("useAsInsertId");
PROPERTY_NAMES = Collections.unmodifiableSet(props);
diff --git
a/components/camel-google/camel-google-bigquery/src/generated/java/org/apache/camel/component/google/bigquery/sql/GoogleBigQuerySQLEndpointConfigurer.java
b/components/camel-google/camel-google-bigquery/src/generated/java/org/apache/camel/component/google/bigquery/sql/GoogleBigQuerySQLEndpointConfigurer.java
index 5bc9b814a95..ba62f6316bb 100644
---
a/components/camel-google/camel-google-bigquery/src/generated/java/org/apache/camel/component/google/bigquery/sql/GoogleBigQuerySQLEndpointConfigurer.java
+++
b/components/camel-google/camel-google-bigquery/src/generated/java/org/apache/camel/component/google/bigquery/sql/GoogleBigQuerySQLEndpointConfigurer.java
@@ -25,6 +25,8 @@ public class GoogleBigQuerySQLEndpointConfigurer extends
PropertyConfigurerSuppo
case "connectionFactory":
target.getConfiguration().setConnectionFactory(property(camelContext,
org.apache.camel.component.google.bigquery.GoogleBigQueryConnectionFactory.class,
value)); return true;
case "lazystartproducer":
case "lazyStartProducer":
target.setLazyStartProducer(property(camelContext, boolean.class, value));
return true;
+ case "serviceaccountkey":
+ case "serviceAccountKey":
target.getConfiguration().setServiceAccountKey(property(camelContext,
java.lang.String.class, value)); return true;
default: return false;
}
}
@@ -41,6 +43,8 @@ public class GoogleBigQuerySQLEndpointConfigurer extends
PropertyConfigurerSuppo
case "connectionFactory": return
org.apache.camel.component.google.bigquery.GoogleBigQueryConnectionFactory.class;
case "lazystartproducer":
case "lazyStartProducer": return boolean.class;
+ case "serviceaccountkey":
+ case "serviceAccountKey": return java.lang.String.class;
default: return null;
}
}
@@ -53,6 +57,8 @@ public class GoogleBigQuerySQLEndpointConfigurer extends
PropertyConfigurerSuppo
case "connectionFactory": return
target.getConfiguration().getConnectionFactory();
case "lazystartproducer":
case "lazyStartProducer": return target.isLazyStartProducer();
+ case "serviceaccountkey":
+ case "serviceAccountKey": return
target.getConfiguration().getServiceAccountKey();
default: return null;
}
}
diff --git
a/components/camel-google/camel-google-bigquery/src/generated/java/org/apache/camel/component/google/bigquery/sql/GoogleBigQuerySQLEndpointUriFactory.java
b/components/camel-google/camel-google-bigquery/src/generated/java/org/apache/camel/component/google/bigquery/sql/GoogleBigQuerySQLEndpointUriFactory.java
index c92c0beb5f2..01978941255 100644
---
a/components/camel-google/camel-google-bigquery/src/generated/java/org/apache/camel/component/google/bigquery/sql/GoogleBigQuerySQLEndpointUriFactory.java
+++
b/components/camel-google/camel-google-bigquery/src/generated/java/org/apache/camel/component/google/bigquery/sql/GoogleBigQuerySQLEndpointUriFactory.java
@@ -21,11 +21,12 @@ public class GoogleBigQuerySQLEndpointUriFactory extends
org.apache.camel.suppor
private static final Set<String> SECRET_PROPERTY_NAMES;
private static final Set<String> MULTI_VALUE_PREFIXES;
static {
- Set<String> props = new HashSet<>(4);
+ Set<String> props = new HashSet<>(5);
props.add("connectionFactory");
props.add("lazyStartProducer");
props.add("projectId");
props.add("queryString");
+ props.add("serviceAccountKey");
PROPERTY_NAMES = Collections.unmodifiableSet(props);
SECRET_PROPERTY_NAMES = Collections.emptySet();
MULTI_VALUE_PREFIXES = Collections.emptySet();
diff --git
a/components/camel-google/camel-google-bigquery/src/generated/resources/org/apache/camel/component/google/bigquery/google-bigquery.json
b/components/camel-google/camel-google-bigquery/src/generated/resources/org/apache/camel/component/google/bigquery/google-bigquery.json
index 2649c56bfe7..067cfbc63ac 100644
---
a/components/camel-google/camel-google-bigquery/src/generated/resources/org/apache/camel/component/google/bigquery/google-bigquery.json
+++
b/components/camel-google/camel-google-bigquery/src/generated/resources/org/apache/camel/component/google/bigquery/google-bigquery.json
@@ -39,6 +39,7 @@
"datasetId": { "kind": "path", "displayName": "Dataset Id", "group":
"common", "label": "common", "required": true, "type": "string", "javaType":
"java.lang.String", "deprecated": false, "deprecationNote": "", "autowired":
false, "secret": false, "configurationClass":
"org.apache.camel.component.google.bigquery.GoogleBigQueryConfiguration",
"configurationField": "configuration", "description": "BigQuery Dataset Id" },
"tableId": { "kind": "path", "displayName": "Table Id", "group": "common",
"label": "common", "required": false, "type": "string", "javaType":
"java.lang.String", "deprecated": false, "deprecationNote": "", "autowired":
false, "secret": false, "configurationClass":
"org.apache.camel.component.google.bigquery.GoogleBigQueryConfiguration",
"configurationField": "configuration", "description": "BigQuery table id" },
"connectionFactory": { "kind": "parameter", "displayName": "Connection
Factory", "group": "producer", "label": "", "required": false, "type":
"object", "javaType":
"org.apache.camel.component.google.bigquery.GoogleBigQueryConnectionFactory",
"deprecated": false, "deprecationNote": "", "autowired": true, "secret": false,
"configurationClass":
"org.apache.camel.component.google.bigquery.GoogleBigQueryConfiguration",
"configurationField": "configuration", "description": "ConnectionFacto [...]
+ "serviceAccountKey": { "kind": "parameter", "displayName": "Service
Account Key", "group": "common", "label": "common", "required": false, "type":
"string", "javaType": "java.lang.String", "deprecated": false,
"deprecationNote": "", "autowired": false, "secret": false,
"configurationClass":
"org.apache.camel.component.google.bigquery.GoogleBigQueryConfiguration",
"configurationField": "configuration", "description": "Service account key in
json format to authenticate an application a [...]
"lazyStartProducer": { "kind": "parameter", "displayName": "Lazy Start
Producer", "group": "producer", "label": "producer", "required": false, "type":
"boolean", "javaType": "boolean", "deprecated": false, "autowired": false,
"secret": false, "defaultValue": false, "description": "Whether the producer
should be started lazy (on the first message). By starting lazy you can use
this to allow CamelContext and routes to startup in situations where a producer
may otherwise fail during sta [...]
"useAsInsertId": { "kind": "parameter", "displayName": "Use As Insert Id",
"group": "producer", "label": "", "required": false, "type": "string",
"javaType": "java.lang.String", "deprecated": false, "autowired": false,
"secret": false, "configurationClass":
"org.apache.camel.component.google.bigquery.GoogleBigQueryConfiguration",
"configurationField": "configuration", "description": "Field name to use as
insert id" }
}
diff --git
a/components/camel-google/camel-google-bigquery/src/generated/resources/org/apache/camel/component/google/bigquery/sql/google-bigquery-sql.json
b/components/camel-google/camel-google-bigquery/src/generated/resources/org/apache/camel/component/google/bigquery/sql/google-bigquery-sql.json
index fb7ffde7a0d..fce30c740a8 100644
---
a/components/camel-google/camel-google-bigquery/src/generated/resources/org/apache/camel/component/google/bigquery/sql/google-bigquery-sql.json
+++
b/components/camel-google/camel-google-bigquery/src/generated/resources/org/apache/camel/component/google/bigquery/sql/google-bigquery-sql.json
@@ -35,6 +35,7 @@
"projectId": { "kind": "path", "displayName": "Project Id", "group":
"common", "label": "common", "required": true, "type": "string", "javaType":
"java.lang.String", "deprecated": false, "deprecationNote": "", "autowired":
false, "secret": false, "configurationClass":
"org.apache.camel.component.google.bigquery.sql.GoogleBigQuerySQLConfiguration",
"configurationField": "configuration", "description": "Google Cloud Project
Id" },
"queryString": { "kind": "path", "displayName": "Query String", "group":
"common", "label": "common", "required": true, "type": "string", "javaType":
"java.lang.String", "deprecated": false, "deprecationNote": "", "autowired":
false, "secret": false, "configurationClass":
"org.apache.camel.component.google.bigquery.sql.GoogleBigQuerySQLConfiguration",
"configurationField": "configuration", "description": "BigQuery standard SQL
query" },
"connectionFactory": { "kind": "parameter", "displayName": "Connection
Factory", "group": "producer", "label": "", "required": false, "type":
"object", "javaType":
"org.apache.camel.component.google.bigquery.GoogleBigQueryConnectionFactory",
"deprecated": false, "deprecationNote": "", "autowired": true, "secret": false,
"configurationClass":
"org.apache.camel.component.google.bigquery.sql.GoogleBigQuerySQLConfiguration",
"configurationField": "configuration", "description": "Connecti [...]
+ "serviceAccountKey": { "kind": "parameter", "displayName": "Service
Account Key", "group": "common", "label": "common", "required": false, "type":
"string", "javaType": "java.lang.String", "deprecated": false,
"deprecationNote": "", "autowired": false, "secret": false,
"configurationClass":
"org.apache.camel.component.google.bigquery.sql.GoogleBigQuerySQLConfiguration",
"configurationField": "configuration", "description": "Service account key in
json format to authenticate an applic [...]
"lazyStartProducer": { "kind": "parameter", "displayName": "Lazy Start
Producer", "group": "producer", "label": "producer", "required": false, "type":
"boolean", "javaType": "boolean", "deprecated": false, "autowired": false,
"secret": false, "defaultValue": false, "description": "Whether the producer
should be started lazy (on the first message). By starting lazy you can use
this to allow CamelContext and routes to startup in situations where a producer
may otherwise fail during sta [...]
}
}
diff --git
a/components/camel-google/camel-google-bigquery/src/main/docs/google-bigquery-component.adoc
b/components/camel-google/camel-google-bigquery/src/main/docs/google-bigquery-component.adoc
index edb5100d028..f304940450e 100644
---
a/components/camel-google/camel-google-bigquery/src/main/docs/google-bigquery-component.adoc
+++
b/components/camel-google/camel-google-bigquery/src/main/docs/google-bigquery-component.adoc
@@ -48,7 +48,26 @@ Google security credentials can be set explicitly by
providing the path to the G
Or they are set implicitly, where the connection factory falls back on
https://developers.google.com/identity/protocols/application-default-credentials#howtheywork[Application
Default Credentials].
-*OBS!* The location of the default credentials file is configurable - via
GOOGLE_APPLICATION_CREDENTIALS environment variable.
+When you have the **service account key** you can provide authentication
credentials to your application code.
+Google security credentials can be set through the component endpoint:
+
+[source,java]
+--------------------------------------------------------
+String endpoint =
"google-bigquery://project-id:datasetId[:tableId]?serviceAccountKey=/home/user/Downloads/my-key.json";
+--------------------------------------------------------
+
+You can also use the base64 encoded content of the authentication credentials
file if you don't want to set a file system path.
+
+[source,java]
+--------------------------------------------------------
+String endpoint =
"google-bigquery://project-id:datasetId[:tableId]?serviceAccountKey=base64:<base64
encoded>";
+--------------------------------------------------------
+
+Or by setting the environment variable `GOOGLE_APPLICATION_CREDENTIALS` :
+
+--------------------------------------------------------
+export GOOGLE_APPLICATION_CREDENTIALS="/home/user/Downloads/my-key.json"
+--------------------------------------------------------
== URI Format
diff --git
a/components/camel-google/camel-google-bigquery/src/main/docs/google-bigquery-sql-component.adoc
b/components/camel-google/camel-google-bigquery/src/main/docs/google-bigquery-sql-component.adoc
index 9ddd143cb80..8a44abc01a7 100644
---
a/components/camel-google/camel-google-bigquery/src/main/docs/google-bigquery-sql-component.adoc
+++
b/components/camel-google/camel-google-bigquery/src/main/docs/google-bigquery-sql-component.adoc
@@ -47,7 +47,26 @@ Google security credentials can be set explicitly by
providing the path to the G
Or they are set implicitly, where the connection factory falls back on
https://developers.google.com/identity/protocols/application-default-credentials#howtheywork[Application
Default Credentials].
-*OBS!* The location of the default credentials file is configurable - via
GOOGLE_APPLICATION_CREDENTIALS environment variable.
+When you have the **service account key** you can provide authentication
credentials to your application code.
+Google security credentials can be set through the component endpoint:
+
+[source,java]
+--------------------------------------------------------
+String endpoint =
"google-bigquery-sql://project-id:query?serviceAccountKey=/home/user/Downloads/my-key.json";
+--------------------------------------------------------
+
+You can also use the base64 encoded content of the authentication credentials
file if you don't want to set a file system path.
+
+[source,java]
+--------------------------------------------------------
+String endpoint =
"google-bigquery-sql://project-id:query?serviceAccountKey=base64:<base64
encoded>";
+--------------------------------------------------------
+
+Or by setting the environment variable `GOOGLE_APPLICATION_CREDENTIALS` :
+
+--------------------------------------------------------
+export GOOGLE_APPLICATION_CREDENTIALS="/home/user/Downloads/my-key.json"
+--------------------------------------------------------
== URI Format
diff --git
a/components/camel-google/camel-google-bigquery/src/main/java/org/apache/camel/component/google/bigquery/GoogleBigQueryComponent.java
b/components/camel-google/camel-google-bigquery/src/main/java/org/apache/camel/component/google/bigquery/GoogleBigQueryComponent.java
index 8737c081805..86256646252 100644
---
a/components/camel-google/camel-google-bigquery/src/main/java/org/apache/camel/component/google/bigquery/GoogleBigQueryComponent.java
+++
b/components/camel-google/camel-google-bigquery/src/main/java/org/apache/camel/component/google/bigquery/GoogleBigQueryComponent.java
@@ -29,14 +29,23 @@ public class GoogleBigQueryComponent extends
DefaultComponent {
@Metadata
private String projectId;
+
@Metadata
private String datasetId;
+
+ @Metadata
+ private GoogleBigQueryConfiguration configuration;
+
@Metadata(autowired = true)
private GoogleBigQueryConnectionFactory connectionFactory;
public GoogleBigQueryComponent() {
}
+ public GoogleBigQueryComponent(GoogleBigQueryConfiguration configuration) {
+ this.configuration = configuration;
+ }
+
public GoogleBigQueryComponent(CamelContext camelContext) {
super(camelContext);
}
@@ -50,17 +59,10 @@ public class GoogleBigQueryComponent extends
DefaultComponent {
throw new IllegalArgumentException("Google BigQuery Endpoint
format \"projectId:datasetId:tableName\"");
}
- GoogleBigQueryConfiguration configuration = new
GoogleBigQueryConfiguration();
- configuration.parseRemaining(remaining);
-
- if (configuration.getConnectionFactory() == null) {
- if (connectionFactory == null) {
- connectionFactory = new GoogleBigQueryConnectionFactory();
- }
- configuration.setConnectionFactory(getConnectionFactory());
- }
+ GoogleBigQueryConfiguration conf = configuration != null ?
configuration.copy() : new GoogleBigQueryConfiguration();
+ conf.parseRemaining(remaining);
- GoogleBigQueryEndpoint endpoint = new GoogleBigQueryEndpoint(uri,
this, configuration);
+ GoogleBigQueryEndpoint endpoint = new GoogleBigQueryEndpoint(uri,
this, conf);
setProperties(endpoint, parameters);
return endpoint;
}
@@ -97,4 +99,8 @@ public class GoogleBigQueryComponent extends DefaultComponent
{
public void setConnectionFactory(GoogleBigQueryConnectionFactory
connectionFactory) {
this.connectionFactory = connectionFactory;
}
+
+ public GoogleBigQueryConfiguration getConfiguration() {
+ return configuration;
+ }
}
diff --git
a/components/camel-google/camel-google-bigquery/src/main/java/org/apache/camel/component/google/bigquery/GoogleBigQueryConfiguration.java
b/components/camel-google/camel-google-bigquery/src/main/java/org/apache/camel/component/google/bigquery/GoogleBigQueryConfiguration.java
index fbc70bce69c..283a4c558f1 100644
---
a/components/camel-google/camel-google-bigquery/src/main/java/org/apache/camel/component/google/bigquery/GoogleBigQueryConfiguration.java
+++
b/components/camel-google/camel-google-bigquery/src/main/java/org/apache/camel/component/google/bigquery/GoogleBigQueryConfiguration.java
@@ -16,29 +16,39 @@
*/
package org.apache.camel.component.google.bigquery;
+import org.apache.camel.RuntimeCamelException;
import org.apache.camel.spi.Metadata;
import org.apache.camel.spi.UriParam;
import org.apache.camel.spi.UriParams;
import org.apache.camel.spi.UriPath;
@UriParams
-public class GoogleBigQueryConfiguration {
+public class GoogleBigQueryConfiguration implements Cloneable {
@UriParam(description = "ConnectionFactory to obtain connection to
Bigquery Service. If not provided the default one will be used")
@Metadata(autowired = true)
private GoogleBigQueryConnectionFactory connectionFactory;
+
@UriParam(description = "Field name to use as insert id")
private String useAsInsertId;
+
@UriPath(label = "common", description = "Google Cloud Project Id")
@Metadata(required = true)
private String projectId;
+
@UriPath(label = "common", description = "BigQuery Dataset Id")
@Metadata(required = true)
private String datasetId;
+
@UriPath(label = "common", description = "BigQuery table id")
@Metadata(required = false)
private String tableId;
+ @UriParam(label = "security",
+ description = "Service account key in json format to
authenticate an application as a service account to google cloud platform")
+ @Metadata(required = false)
+ private String serviceAccountKey;
+
public void parseRemaining(String remaining) {
String[] parts = remaining.split(":");
@@ -100,4 +110,22 @@ public class GoogleBigQueryConfiguration {
this.tableId = tableId;
return this;
}
+
+ public String getServiceAccountKey() {
+ return serviceAccountKey;
+ }
+
+ public GoogleBigQueryConfiguration setServiceAccountKey(String
serviceAccountKey) {
+ this.serviceAccountKey = serviceAccountKey;
+ return this;
+ }
+
+ public GoogleBigQueryConfiguration copy() {
+ try {
+ return (GoogleBigQueryConfiguration) super.clone();
+ } catch (CloneNotSupportedException e) {
+ throw new RuntimeCamelException(e);
+ }
+ }
+
}
diff --git
a/components/camel-google/camel-google-bigquery/src/main/java/org/apache/camel/component/google/bigquery/GoogleBigQueryConnectionFactory.java
b/components/camel-google/camel-google-bigquery/src/main/java/org/apache/camel/component/google/bigquery/GoogleBigQueryConnectionFactory.java
index 3662cdf4d8a..2e7c2a596e7 100644
---
a/components/camel-google/camel-google-bigquery/src/main/java/org/apache/camel/component/google/bigquery/GoogleBigQueryConnectionFactory.java
+++
b/components/camel-google/camel-google-bigquery/src/main/java/org/apache/camel/component/google/bigquery/GoogleBigQueryConnectionFactory.java
@@ -16,7 +16,6 @@
*/
package org.apache.camel.component.google.bigquery;
-import java.io.FileInputStream;
import java.io.InputStream;
import java.util.Collection;
import java.util.Collections;
@@ -26,7 +25,11 @@ import com.google.api.services.bigquery.BigqueryScopes;
import com.google.auth.oauth2.GoogleCredentials;
import com.google.cloud.bigquery.BigQuery;
import com.google.cloud.bigquery.BigQueryOptions;
+import org.apache.camel.CamelContext;
+import org.apache.camel.CamelException;
+import org.apache.camel.support.ResourceHelper;
import org.apache.camel.util.ObjectHelper;
+import org.apache.camel.util.StringHelper;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -34,9 +37,10 @@ public class GoogleBigQueryConnectionFactory {
private final Logger logger =
LoggerFactory.getLogger(GoogleBigQueryConnectionFactory.class);
- private String credentialsFileLocation;
+ private String serviceAccountKeyFile;
private String serviceURL;
private BigQuery client;
+ private CamelContext camelContext;
public GoogleBigQueryConnectionFactory() {
}
@@ -56,9 +60,10 @@ public class GoogleBigQueryConnectionFactory {
GoogleCredentials credentials = null;
- if (credentials == null &&
!Strings.isNullOrEmpty(credentialsFileLocation)) {
+ if (!Strings.isNullOrEmpty(serviceAccountKeyFile)) {
logger.debug("Key File Name has been set explicitly. Initialising
BigQuery using Key File {}",
- credentialsFileLocation);
+ // limit the output as the value could be a long base64
string, we don't want to show it whole
+ StringHelper.limitLength(serviceAccountKeyFile, 70));
credentials = createFromFile();
}
@@ -81,7 +86,11 @@ public class GoogleBigQueryConnectionFactory {
}
private GoogleCredentials createFromFile() throws Exception {
- try (InputStream is = new FileInputStream(credentialsFileLocation)) {
+ if (camelContext == null) {
+ throw new CamelException("CamelContext is null, but must be set
when creating GoogleBigQueryConnectionFactory.");
+ }
+ try (InputStream is
+ =
ResourceHelper.resolveMandatoryResourceAsInputStream(camelContext,
serviceAccountKeyFile);) {
GoogleCredentials credentials = GoogleCredentials.fromStream(is);
if (credentials.createScopedRequired()) {
@@ -104,12 +113,12 @@ public class GoogleBigQueryConnectionFactory {
return credentials;
}
- public String getCredentialsFileLocation() {
- return credentialsFileLocation;
+ public String getServiceAccountKeyFile() {
+ return serviceAccountKeyFile;
}
- public GoogleBigQueryConnectionFactory setCredentialsFileLocation(String
credentialsFileLocation) {
- this.credentialsFileLocation = credentialsFileLocation;
+ public GoogleBigQueryConnectionFactory setServiceAccountKeyFile(String
serviceAccountKeyFile) {
+ this.serviceAccountKeyFile = serviceAccountKeyFile;
resetClient();
return this;
}
@@ -127,4 +136,9 @@ public class GoogleBigQueryConnectionFactory {
private synchronized void resetClient() {
this.client = null;
}
+
+ public GoogleBigQueryConnectionFactory setCamelContext(CamelContext
camelContext) {
+ this.camelContext = camelContext;
+ return this;
+ }
}
diff --git
a/components/camel-google/camel-google-bigquery/src/main/java/org/apache/camel/component/google/bigquery/GoogleBigQueryEndpoint.java
b/components/camel-google/camel-google-bigquery/src/main/java/org/apache/camel/component/google/bigquery/GoogleBigQueryEndpoint.java
index e7033b3366c..1d9d331164f 100644
---
a/components/camel-google/camel-google-bigquery/src/main/java/org/apache/camel/component/google/bigquery/GoogleBigQueryEndpoint.java
+++
b/components/camel-google/camel-google-bigquery/src/main/java/org/apache/camel/component/google/bigquery/GoogleBigQueryEndpoint.java
@@ -47,6 +47,8 @@ public class GoogleBigQueryEndpoint extends DefaultEndpoint {
@UriParam
protected final GoogleBigQueryConfiguration configuration;
+ private BigQuery bigQuery;
+
protected GoogleBigQueryEndpoint(String endpointUri,
GoogleBigQueryComponent component,
GoogleBigQueryConfiguration
configuration) {
super(endpointUri, component);
@@ -55,9 +57,20 @@ public class GoogleBigQueryEndpoint extends DefaultEndpoint {
@Override
public Producer createProducer() throws Exception {
- BigQuery bigquery =
getConfiguration().getConnectionFactory().getDefaultClient();
- GoogleBigQueryProducer producer = new GoogleBigQueryProducer(bigquery,
this, configuration);
- return producer;
+ return new GoogleBigQueryProducer(bigQuery, this, configuration);
+ }
+
+ @Override
+ protected void doStart() throws Exception {
+ super.doStart();
+ GoogleBigQueryConnectionFactory connFactory =
configuration.getConnectionFactory();
+ if (connFactory == null) {
+ connFactory = new GoogleBigQueryConnectionFactory()
+ .setCamelContext(getCamelContext())
+
.setServiceAccountKeyFile(configuration.getServiceAccountKey());
+ configuration.setConnectionFactory(connFactory);
+ }
+ bigQuery = connFactory.getDefaultClient();
}
@Override
diff --git
a/components/camel-google/camel-google-bigquery/src/main/java/org/apache/camel/component/google/bigquery/sql/GoogleBigQuerySQLComponent.java
b/components/camel-google/camel-google-bigquery/src/main/java/org/apache/camel/component/google/bigquery/sql/GoogleBigQuerySQLComponent.java
index 31e75a53e85..bc38966e2f4 100644
---
a/components/camel-google/camel-google-bigquery/src/main/java/org/apache/camel/component/google/bigquery/sql/GoogleBigQuerySQLComponent.java
+++
b/components/camel-google/camel-google-bigquery/src/main/java/org/apache/camel/component/google/bigquery/sql/GoogleBigQuerySQLComponent.java
@@ -30,12 +30,20 @@ public class GoogleBigQuerySQLComponent extends
DefaultComponent {
@Metadata
private String projectId;
+
+ @Metadata
+ private GoogleBigQuerySQLConfiguration configuration;
+
@Metadata(autowired = true)
private GoogleBigQueryConnectionFactory connectionFactory;
public GoogleBigQuerySQLComponent() {
}
+ public GoogleBigQuerySQLComponent(GoogleBigQuerySQLConfiguration
configuration) {
+ this.configuration = configuration;
+ }
+
public GoogleBigQuerySQLComponent(CamelContext camelContext) {
super(camelContext);
}
@@ -49,17 +57,11 @@ public class GoogleBigQuerySQLComponent extends
DefaultComponent {
throw new IllegalArgumentException("Google BigQuery Endpoint
format \"projectId:<query>\"");
}
- GoogleBigQuerySQLConfiguration configuration = new
GoogleBigQuerySQLConfiguration();
- configuration.parseRemaining(remaining);
+ GoogleBigQuerySQLConfiguration conf
+ = configuration != null ? configuration.copy() : new
GoogleBigQuerySQLConfiguration();
+ conf.parseRemaining(remaining);
- if (configuration.getConnectionFactory() == null) {
- if (connectionFactory == null) {
- connectionFactory = new GoogleBigQueryConnectionFactory();
- }
- configuration.setConnectionFactory(getConnectionFactory());
- }
-
- GoogleBigQuerySQLEndpoint endpoint = new
GoogleBigQuerySQLEndpoint(uri, this, configuration);
+ GoogleBigQuerySQLEndpoint endpoint = new
GoogleBigQuerySQLEndpoint(uri, this, conf);
setProperties(endpoint, parameters);
return endpoint;
}
@@ -85,4 +87,8 @@ public class GoogleBigQuerySQLComponent extends
DefaultComponent {
public void setConnectionFactory(GoogleBigQueryConnectionFactory
connectionFactory) {
this.connectionFactory = connectionFactory;
}
+
+ public GoogleBigQuerySQLConfiguration getConfiguration() {
+ return configuration;
+ }
}
diff --git
a/components/camel-google/camel-google-bigquery/src/main/java/org/apache/camel/component/google/bigquery/sql/GoogleBigQuerySQLConfiguration.java
b/components/camel-google/camel-google-bigquery/src/main/java/org/apache/camel/component/google/bigquery/sql/GoogleBigQuerySQLConfiguration.java
index 6287405a496..507a4a118c3 100644
---
a/components/camel-google/camel-google-bigquery/src/main/java/org/apache/camel/component/google/bigquery/sql/GoogleBigQuerySQLConfiguration.java
+++
b/components/camel-google/camel-google-bigquery/src/main/java/org/apache/camel/component/google/bigquery/sql/GoogleBigQuerySQLConfiguration.java
@@ -16,6 +16,7 @@
*/
package org.apache.camel.component.google.bigquery.sql;
+import org.apache.camel.RuntimeCamelException;
import
org.apache.camel.component.google.bigquery.GoogleBigQueryConnectionFactory;
import org.apache.camel.spi.Metadata;
import org.apache.camel.spi.UriParam;
@@ -23,18 +24,25 @@ import org.apache.camel.spi.UriParams;
import org.apache.camel.spi.UriPath;
@UriParams
-public class GoogleBigQuerySQLConfiguration {
+public class GoogleBigQuerySQLConfiguration implements Cloneable {
@UriParam(description = "ConnectionFactory to obtain connection to
Bigquery Service. If not provided the default one will be used")
@Metadata(autowired = true)
private GoogleBigQueryConnectionFactory connectionFactory;
+
@UriPath(label = "common", description = "Google Cloud Project Id")
@Metadata(required = true)
private String projectId;
+
@UriPath(label = "common", description = "BigQuery standard SQL query")
@Metadata(required = true)
private String queryString;
+ @UriParam(label = "security",
+ description = "Service account key in json format to
authenticate an application as a service account to google cloud platform")
+ @Metadata(required = false)
+ private String serviceAccountKey;
+
public void parseRemaining(String remaining) {
int indexOfColon = remaining.indexOf(':');
@@ -74,4 +82,22 @@ public class GoogleBigQuerySQLConfiguration {
this.projectId = projectId;
return this;
}
+
+ public String getServiceAccountKey() {
+ return serviceAccountKey;
+ }
+
+ public GoogleBigQuerySQLConfiguration setServiceAccountKey(String
serviceAccountKey) {
+ this.serviceAccountKey = serviceAccountKey;
+ return this;
+ }
+
+ public GoogleBigQuerySQLConfiguration copy() {
+ try {
+ return (GoogleBigQuerySQLConfiguration) super.clone();
+ } catch (CloneNotSupportedException e) {
+ throw new RuntimeCamelException(e);
+ }
+ }
+
}
diff --git
a/components/camel-google/camel-google-bigquery/src/main/java/org/apache/camel/component/google/bigquery/sql/GoogleBigQuerySQLEndpoint.java
b/components/camel-google/camel-google-bigquery/src/main/java/org/apache/camel/component/google/bigquery/sql/GoogleBigQuerySQLEndpoint.java
index 35da8fc3288..84dd172ed43 100644
---
a/components/camel-google/camel-google-bigquery/src/main/java/org/apache/camel/component/google/bigquery/sql/GoogleBigQuerySQLEndpoint.java
+++
b/components/camel-google/camel-google-bigquery/src/main/java/org/apache/camel/component/google/bigquery/sql/GoogleBigQuerySQLEndpoint.java
@@ -20,6 +20,7 @@ import com.google.cloud.bigquery.BigQuery;
import org.apache.camel.Consumer;
import org.apache.camel.Processor;
import org.apache.camel.Producer;
+import
org.apache.camel.component.google.bigquery.GoogleBigQueryConnectionFactory;
import org.apache.camel.component.google.bigquery.GoogleBigQueryConstants;
import org.apache.camel.spi.UriEndpoint;
import org.apache.camel.spi.UriParam;
@@ -47,6 +48,8 @@ public class GoogleBigQuerySQLEndpoint extends
DefaultEndpoint {
@UriParam
protected final GoogleBigQuerySQLConfiguration configuration;
+ private BigQuery bigQuery;
+
protected GoogleBigQuerySQLEndpoint(String endpointUri,
GoogleBigQuerySQLComponent component,
GoogleBigQuerySQLConfiguration
configuration) {
super(endpointUri, component);
@@ -54,10 +57,21 @@ public class GoogleBigQuerySQLEndpoint extends
DefaultEndpoint {
}
@Override
- public Producer createProducer() throws Exception {
- BigQuery bigquery =
getConfiguration().getConnectionFactory().getDefaultClient();
- GoogleBigQuerySQLProducer producer = new
GoogleBigQuerySQLProducer(bigquery, this, configuration);
- return producer;
+ public Producer createProducer() {
+ return new GoogleBigQuerySQLProducer(bigQuery, this, configuration);
+ }
+
+ @Override
+ protected void doStart() throws Exception {
+ super.doStart();
+ GoogleBigQueryConnectionFactory connFactory =
configuration.getConnectionFactory();
+ if (connFactory == null) {
+ connFactory = new GoogleBigQueryConnectionFactory()
+ .setCamelContext(getCamelContext())
+
.setServiceAccountKeyFile(configuration.getServiceAccountKey());
+ configuration.setConnectionFactory(connFactory);
+ }
+ bigQuery = connFactory.getDefaultClient();
}
@Override
diff --git
a/components/camel-google/camel-google-bigquery/src/main/java/org/apache/camel/component/google/bigquery/sql/GoogleBigQuerySQLProducer.java
b/components/camel-google/camel-google-bigquery/src/main/java/org/apache/camel/component/google/bigquery/sql/GoogleBigQuerySQLProducer.java
index ba49a7af96e..883d90c7522 100644
---
a/components/camel-google/camel-google-bigquery/src/main/java/org/apache/camel/component/google/bigquery/sql/GoogleBigQuerySQLProducer.java
+++
b/components/camel-google/camel-google-bigquery/src/main/java/org/apache/camel/component/google/bigquery/sql/GoogleBigQuerySQLProducer.java
@@ -23,8 +23,11 @@ import java.util.UUID;
import com.google.cloud.bigquery.BigQuery;
import com.google.cloud.bigquery.BigQueryException;
+import com.google.cloud.bigquery.Job;
import com.google.cloud.bigquery.JobException;
import com.google.cloud.bigquery.JobId;
+import com.google.cloud.bigquery.JobInfo;
+import com.google.cloud.bigquery.JobStatistics;
import com.google.cloud.bigquery.QueryJobConfiguration;
import com.google.cloud.bigquery.QueryParameterValue;
import com.google.cloud.bigquery.StandardSQLTypeName;
@@ -102,13 +105,15 @@ public class GoogleBigQuerySQLProducer extends
DefaultProducer {
queryJobId = JobId.of(configuration.getProjectId(),
UUID.randomUUID().toString());
}
- TableResult result = bigquery.query(queryJobConfiguration,
queryJobId);
+ Job job = bigquery.create(JobInfo.of(queryJobId,
queryJobConfiguration)).waitFor();
+ JobStatistics.QueryStatistics statistics = job.getStatistics();
+ TableResult result = job.getQueryResults();
+ long numAffectedRows = statistics.getNumDmlAffectedRows();
if (LOG.isTraceEnabled()) {
- LOG.trace("Result of query {} is {}", translatedQuery,
result.toString());
+ LOG.trace("Query {} - Affected rows {} - Result {}",
translatedQuery, numAffectedRows, result.toString());
}
-
- return result.getTotalRows();
+ return numAffectedRows;
} catch (JobException e) {
throw new Exception("Query " + translatedQuery + " failed: " +
e.getErrors(), e);
} catch (BigQueryException e) {
diff --git
a/components/camel-google/camel-google-bigquery/src/test/java/org/apache/camel/component/google/bigquery/integration/BigQueryITSupport.java
b/components/camel-google/camel-google-bigquery/src/test/java/org/apache/camel/component/google/bigquery/integration/BigQueryITSupport.java
index acf99d4e99a..829843fa4ea 100644
---
a/components/camel-google/camel-google-bigquery/src/test/java/org/apache/camel/component/google/bigquery/integration/BigQueryITSupport.java
+++
b/components/camel-google/camel-google-bigquery/src/test/java/org/apache/camel/component/google/bigquery/integration/BigQueryITSupport.java
@@ -22,7 +22,7 @@ import java.util.Properties;
import java.util.UUID;
import java.util.stream.Collectors;
-import com.google.api.client.googleapis.json.GoogleJsonResponseException;
+import com.google.cloud.bigquery.BigQueryException;
import com.google.cloud.bigquery.Field;
import com.google.cloud.bigquery.FieldList;
import com.google.cloud.bigquery.JobId;
@@ -38,8 +38,10 @@ import org.apache.camel.BindToRegistry;
import org.apache.camel.CamelContext;
import org.apache.camel.RuntimeCamelException;
import org.apache.camel.component.google.bigquery.GoogleBigQueryComponent;
+import org.apache.camel.component.google.bigquery.GoogleBigQueryConfiguration;
import
org.apache.camel.component.google.bigquery.GoogleBigQueryConnectionFactory;
import
org.apache.camel.component.google.bigquery.sql.GoogleBigQuerySQLComponent;
+import
org.apache.camel.component.google.bigquery.sql.GoogleBigQuerySQLConfiguration;
import org.apache.camel.test.junit5.CamelTestSupport;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -87,10 +89,15 @@ public class BigQueryITSupport extends CamelTestSupport {
protected void addBigqueryComponent(CamelContext context) {
+ GoogleBigQueryConfiguration configuration = new
GoogleBigQueryConfiguration()
+ .setServiceAccountKey(CREDENTIALS_FILE_LOCATION);
+
connectionFactory = new GoogleBigQueryConnectionFactory()
- .setCredentialsFileLocation(CREDENTIALS_FILE_LOCATION);
+ .setServiceAccountKeyFile(configuration.getServiceAccountKey())
+ .setCamelContext(context);
+ configuration.setConnectionFactory(connectionFactory);
- GoogleBigQueryComponent component = new GoogleBigQueryComponent();
+ GoogleBigQueryComponent component = new
GoogleBigQueryComponent(configuration);
component.setConnectionFactory(connectionFactory);
context.addComponent("google-bigquery", component);
@@ -99,10 +106,14 @@ public class BigQueryITSupport extends CamelTestSupport {
protected void addBigquerySqlComponent(CamelContext context) {
+ GoogleBigQuerySQLConfiguration configuration = new
GoogleBigQuerySQLConfiguration()
+ .setServiceAccountKey(CREDENTIALS_FILE_LOCATION);
connectionFactory = new GoogleBigQueryConnectionFactory()
- .setCredentialsFileLocation(CREDENTIALS_FILE_LOCATION);
+ .setServiceAccountKeyFile(configuration.getServiceAccountKey())
+ .setCamelContext(context);
+ configuration.setConnectionFactory(connectionFactory);
- GoogleBigQuerySQLComponent component = new
GoogleBigQuerySQLComponent();
+ GoogleBigQuerySQLComponent component = new
GoogleBigQuerySQLComponent(configuration);
component.setConnectionFactory(connectionFactory);
context.addComponent("google-bigquery-sql", component);
@@ -146,9 +157,9 @@ public class BigQueryITSupport extends CamelTestSupport {
TableInfo tableInfo = TableInfo.of(id, builder.build());
try {
getConnectionFactory().getDefaultClient().create(tableInfo);
- } catch (GoogleJsonResponseException e) {
- if (e.getDetails().getCode() == 409) {
- LOGGER.info("Table {} already exist");
+ } catch (BigQueryException e) {
+ if (e.getCode() == 409) {
+ LOGGER.info("TableId {} already exists", tableId);
} else {
throw e;
}
diff --git
a/components/camel-google/camel-google-bigquery/src/test/java/org/apache/camel/component/google/bigquery/unit/sql/GoogleBigQuerySQLProducerBaseTest.java
b/components/camel-google/camel-google-bigquery/src/test/java/org/apache/camel/component/google/bigquery/unit/sql/GoogleBigQuerySQLProducerBaseTest.java
index 351d7074e50..0f394ace2ef 100644
---
a/components/camel-google/camel-google-bigquery/src/test/java/org/apache/camel/component/google/bigquery/unit/sql/GoogleBigQuerySQLProducerBaseTest.java
+++
b/components/camel-google/camel-google-bigquery/src/test/java/org/apache/camel/component/google/bigquery/unit/sql/GoogleBigQuerySQLProducerBaseTest.java
@@ -17,7 +17,10 @@
package org.apache.camel.component.google.bigquery.unit.sql;
import com.google.cloud.bigquery.BigQuery;
+import com.google.cloud.bigquery.Job;
import com.google.cloud.bigquery.JobId;
+import com.google.cloud.bigquery.JobInfo;
+import com.google.cloud.bigquery.JobStatistics;
import com.google.cloud.bigquery.QueryJobConfiguration;
import com.google.cloud.bigquery.TableResult;
import
org.apache.camel.component.google.bigquery.sql.GoogleBigQuerySQLConfiguration;
@@ -37,6 +40,8 @@ public abstract class GoogleBigQuerySQLProducerBaseTest
extends CamelTestSupport
protected GoogleBigQuerySQLConfiguration configuration = new
GoogleBigQuerySQLConfiguration();
protected BigQuery bigquery;
protected TableResult tableResult;
+ protected Job job;
+ protected JobStatistics.QueryStatistics statistics;
protected GoogleBigQuerySQLProducer createAndStartProducer() {
configuration.setProjectId(projectId);
@@ -50,6 +55,13 @@ public abstract class GoogleBigQuerySQLProducerBaseTest
extends CamelTestSupport
protected void setupBigqueryMock() throws Exception {
bigquery = mock(BigQuery.class);
tableResult = mock(TableResult.class);
+ job = mock(Job.class);
+ statistics = mock(JobStatistics.QueryStatistics.class);
when(bigquery.query(any(QueryJobConfiguration.class),
any(JobId.class))).thenReturn(tableResult);
+ when(bigquery.create(any(JobInfo.class))).thenReturn(job);
+ when(job.waitFor()).thenReturn(job);
+ when(job.getQueryResults()).thenReturn(tableResult);
+ when(job.getStatistics()).thenReturn(statistics);
+ when(statistics.getNumDmlAffectedRows()).thenReturn(1L);
}
}
diff --git
a/components/camel-google/camel-google-bigquery/src/test/java/org/apache/camel/component/google/bigquery/unit/sql/GoogleBigQuerySQLProducerWithParamersTest.java
b/components/camel-google/camel-google-bigquery/src/test/java/org/apache/camel/component/google/bigquery/unit/sql/GoogleBigQuerySQLProducerWithParamersTest.java
index 6521bd29ea7..9eb0982db56 100644
---
a/components/camel-google/camel-google-bigquery/src/test/java/org/apache/camel/component/google/bigquery/unit/sql/GoogleBigQuerySQLProducerWithParamersTest.java
+++
b/components/camel-google/camel-google-bigquery/src/test/java/org/apache/camel/component/google/bigquery/unit/sql/GoogleBigQuerySQLProducerWithParamersTest.java
@@ -20,6 +20,7 @@ import java.util.HashMap;
import java.util.Map;
import com.google.cloud.bigquery.JobId;
+import com.google.cloud.bigquery.JobInfo;
import com.google.cloud.bigquery.QueryJobConfiguration;
import com.google.cloud.bigquery.QueryParameterValue;
import org.apache.camel.Exchange;
@@ -34,8 +35,6 @@ import static
org.apache.camel.component.google.bigquery.integration.BigQueryITS
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertThrows;
import static org.junit.jupiter.api.Assertions.assertTrue;
-import static org.mockito.ArgumentMatchers.any;
-import static org.mockito.ArgumentMatchers.eq;
import static org.mockito.Mockito.verify;
public class GoogleBigQuerySQLProducerWithParamersTest extends
GoogleBigQuerySQLProducerBaseTest {
@@ -54,10 +53,10 @@ public class GoogleBigQuerySQLProducerWithParamersTest
extends GoogleBigQuerySQL
body.put("data", "some data");
producer.process(createExchangeWithBody(body));
- ArgumentCaptor<QueryJobConfiguration> dataCaptor =
ArgumentCaptor.forClass(QueryJobConfiguration.class);
- verify(bigquery).query(dataCaptor.capture(), any(JobId.class));
+ ArgumentCaptor<JobInfo> dataCaptor =
ArgumentCaptor.forClass(JobInfo.class);
+ verify(bigquery).create(dataCaptor.capture());
- QueryJobConfiguration request = dataCaptor.getValue();
+ QueryJobConfiguration request =
dataCaptor.getValue().getConfiguration();
assertEquals(sql, request.getQuery());
Map<String, QueryParameterValue> namedParameters =
request.getNamedParameters();
@@ -82,10 +81,10 @@ public class GoogleBigQuerySQLProducerWithParamersTest
extends GoogleBigQuerySQL
producer.process(exchange);
- ArgumentCaptor<QueryJobConfiguration> dataCaptor =
ArgumentCaptor.forClass(QueryJobConfiguration.class);
- verify(bigquery).query(dataCaptor.capture(), any(JobId.class));
+ ArgumentCaptor<JobInfo> dataCaptor =
ArgumentCaptor.forClass(JobInfo.class);
+ verify(bigquery).create(dataCaptor.capture());
- QueryJobConfiguration request = dataCaptor.getValue();
+ QueryJobConfiguration request =
dataCaptor.getValue().getConfiguration();
assertEquals(sql, request.getQuery());
Map<String, QueryParameterValue> namedParameters =
request.getNamedParameters();
@@ -113,10 +112,10 @@ public class GoogleBigQuerySQLProducerWithParamersTest
extends GoogleBigQuerySQL
producer.process(exchange);
- ArgumentCaptor<QueryJobConfiguration> dataCaptor =
ArgumentCaptor.forClass(QueryJobConfiguration.class);
- verify(bigquery).query(dataCaptor.capture(), eq(jobId));
+ ArgumentCaptor<JobInfo> dataCaptor =
ArgumentCaptor.forClass(JobInfo.class);
+ verify(bigquery).create(dataCaptor.capture());
- QueryJobConfiguration request = dataCaptor.getValue();
+ QueryJobConfiguration request =
dataCaptor.getValue().getConfiguration();
assertEquals(sql, request.getQuery());
Map<String, QueryParameterValue> namedParameters =
request.getNamedParameters();
diff --git
a/components/camel-google/camel-google-bigquery/src/test/java/org/apache/camel/component/google/bigquery/unit/sql/GoogleBigQuerySQLProducerWithPatternTest.java
b/components/camel-google/camel-google-bigquery/src/test/java/org/apache/camel/component/google/bigquery/unit/sql/GoogleBigQuerySQLProducerWithPatternTest.java
index 4b510467036..2e838d05705 100644
---
a/components/camel-google/camel-google-bigquery/src/test/java/org/apache/camel/component/google/bigquery/unit/sql/GoogleBigQuerySQLProducerWithPatternTest.java
+++
b/components/camel-google/camel-google-bigquery/src/test/java/org/apache/camel/component/google/bigquery/unit/sql/GoogleBigQuerySQLProducerWithPatternTest.java
@@ -18,7 +18,7 @@ package org.apache.camel.component.google.bigquery.unit.sql;
import java.util.List;
-import com.google.cloud.bigquery.JobId;
+import com.google.cloud.bigquery.JobInfo;
import com.google.cloud.bigquery.QueryJobConfiguration;
import org.apache.camel.Exchange;
import org.junit.jupiter.api.BeforeEach;
@@ -26,7 +26,6 @@ import org.junit.jupiter.api.Test;
import org.mockito.ArgumentCaptor;
import static org.junit.jupiter.api.Assertions.assertEquals;
-import static org.mockito.ArgumentMatchers.any;
import static org.mockito.Mockito.verify;
public class GoogleBigQuerySQLProducerWithPatternTest extends
GoogleBigQuerySQLProducerBaseTest {
@@ -45,12 +44,13 @@ public class GoogleBigQuerySQLProducerWithPatternTest
extends GoogleBigQuerySQLP
String expected = "insert into dataset.testTableId(id, data) values(1,
'test')";
producer.process(exchange);
- ArgumentCaptor<QueryJobConfiguration> dataCaptor =
ArgumentCaptor.forClass(QueryJobConfiguration.class);
- verify(bigquery).query(dataCaptor.capture(), any(JobId.class));
+ ArgumentCaptor<JobInfo> dataCaptor =
ArgumentCaptor.forClass(JobInfo.class);
+ verify(bigquery).create(dataCaptor.capture());
- List<QueryJobConfiguration> requests = dataCaptor.getAllValues();
+ List<JobInfo> requests = dataCaptor.getAllValues();
assertEquals(1, requests.size());
- assertEquals(expected, requests.get(0).getQuery());
+ QueryJobConfiguration conf = requests.get(0).getConfiguration();
+ assertEquals(expected, conf.getQuery());
}
@Test
@@ -60,12 +60,13 @@ public class GoogleBigQuerySQLProducerWithPatternTest
extends GoogleBigQuerySQLP
String expected = "insert into dataset.testTableId(id, data) values(1,
'test')";
producer.process(exchange);
- ArgumentCaptor<QueryJobConfiguration> dataCaptor =
ArgumentCaptor.forClass(QueryJobConfiguration.class);
- verify(bigquery).query(dataCaptor.capture(), any(JobId.class));
+ ArgumentCaptor<JobInfo> dataCaptor =
ArgumentCaptor.forClass(JobInfo.class);
+ verify(bigquery).create(dataCaptor.capture());
- List<QueryJobConfiguration> requests = dataCaptor.getAllValues();
+ List<JobInfo> requests = dataCaptor.getAllValues();
assertEquals(1, requests.size());
- assertEquals(expected, requests.get(0).getQuery());
+ QueryJobConfiguration conf = requests.get(0).getConfiguration();
+ assertEquals(expected, conf.getQuery());
}
}
diff --git
a/components/camel-google/camel-google-bigquery/src/test/resources/log4j.properties
b/components/camel-google/camel-google-bigquery/src/test/resources/log4j.properties
deleted file mode 100644
index 4180b0d91fe..00000000000
---
a/components/camel-google/camel-google-bigquery/src/test/resources/log4j.properties
+++ /dev/null
@@ -1,37 +0,0 @@
-## ---------------------------------------------------------------------------
-## Licensed to the Apache Software Foundation (ASF) under one or more
-## contributor license agreements. See the NOTICE file distributed with
-## this work for additional information regarding copyright ownership.
-## The ASF licenses this file to You under the Apache License, Version 2.0
-## (the "License"); you may not use this file except in compliance with
-## the License. You may obtain a copy of the License at
-##
-## http://www.apache.org/licenses/LICENSE-2.0
-##
-## Unless required by applicable law or agreed to in writing, software
-## distributed under the License is distributed on an "AS IS" BASIS,
-## WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-## See the License for the specific language governing permissions and
-## limitations under the License.
-## ---------------------------------------------------------------------------
-
-#
-# The logging properties used during tests..
-#
-log4j.rootLogger=INFO, out
-
-#log4j.logger.org.apache.camel.component.google.pubsub=TRACE
-#log4j.logger.org.apache.camel=DEBUG
-#log4j.logger.org.apache.camel.impl=WARN
-
-# CONSOLE appender not used by default
-log4j.appender.stdout=org.apache.log4j.ConsoleAppender
-log4j.appender.stdout.layout=org.apache.log4j.PatternLayout
-log4j.appender.stdout.layout.ConversionPattern=%d [%-15.15t] %-5p %-30.30c{1}
- %m%n
-
-# File appender
-log4j.appender.out=org.apache.log4j.FileAppender
-log4j.appender.out.layout=org.apache.log4j.PatternLayout
-log4j.appender.out.layout.ConversionPattern=%d [%-15.15t] %-5p %-30.30c{1} -
%m%n
-log4j.appender.out.file=target/camel-google-bigquery-test.log
-log4j.appender.out.append=true
diff --git
a/components/camel-google/camel-google-bigquery/src/test/resources/simple.properties
b/components/camel-google/camel-google-bigquery/src/test/resources/simple.properties
index 88c4916240a..f1563effaef 100644
---
a/components/camel-google/camel-google-bigquery/src/test/resources/simple.properties
+++
b/components/camel-google/camel-google-bigquery/src/test/resources/simple.properties
@@ -15,6 +15,7 @@
## limitations under the License.
## ---------------------------------------------------------------------------
+# change the dataset id in insert.sql and delete.sql
project.id=scg-bi-sandbox
bigquery.datasetId=test_dataset
# Service Account Name and Key do not belong to any project
