This is an automated email from the ASF dual-hosted git repository.
gnodet pushed a commit to branch camel-4.22.x
in repository https://gitbox.apache.org/repos/asf/camel.git
The following commit(s) were added to refs/heads/camel-4.22.x by this push:
new 374b56f4b184 [backport camel-4.22.x] CAMEL-26923: camel-duckdb - copy
loads one local file unless options allow URLs or globs (#30159)
374b56f4b184 is described below
commit 374b56f4b18417dc7f11477e9fb7073f2ad87bc2
Author: Claus Ibsen <[email protected]>
AuthorDate: Mon Nov 16 13:39:18 2026 +0100
[backport camel-4.22.x] CAMEL-26923: camel-duckdb - copy loads one local
file unless options allow URLs or globs (#30159)
CAMEL-26923: camel-duckdb - copy loads one local file unless options allow
URLs or globs (#29844)
(cherry picked from commit 777f009d22024ed8407d7e0bc67d5d36f29888c8)
Signed-off-by: Claus Ibsen <[email protected]>
Co-authored-by: Claude Opus 5.5 <[email protected]>
---
.../apache/camel/catalog/components/duckdb.json | 5 +-
.../camel/catalog/docs/duckdb-component.adoc | 48 +++-
.../component/duckdb/DuckDbEndpointConfigurer.java | 18 ++
.../component/duckdb/DuckDbEndpointUriFactory.java | 5 +-
.../org/apache/camel/component/duckdb/duckdb.json | 5 +-
.../src/main/docs/duckdb-component.adoc | 48 +++-
.../camel/component/duckdb/DuckDbEndpoint.java | 40 ++++
.../camel/component/duckdb/DuckDbProducer.java | 75 ++++++-
.../duckdb/DuckDbProducerCopySourceTest.java | 241 +++++++++++++++++++++
.../java/org/apache/camel/util/SecurityUtils.java | 1 +
.../endpoint/dsl/DuckDbEndpointBuilderFactory.java | 90 ++++++++
11 files changed, 570 insertions(+), 6 deletions(-)
diff --git
a/catalog/camel-catalog/src/generated/resources/org/apache/camel/catalog/components/duckdb.json
b/catalog/camel-catalog/src/generated/resources/org/apache/camel/catalog/components/duckdb.json
index 555250e2327b..b22df589e5bf 100644
---
a/catalog/camel-catalog/src/generated/resources/org/apache/camel/catalog/components/duckdb.json
+++
b/catalog/camel-catalog/src/generated/resources/org/apache/camel/catalog/components/duckdb.json
@@ -49,6 +49,9 @@
"readOnly": { "index": 6, "kind": "parameter", "displayName": "Read Only",
"group": "producer", "label": "", "required": false, "type": "boolean",
"javaType": "boolean", "deprecated": false, "autowired": false, "secret":
false, "defaultValue": false, "description": "Open the embedded database file
in read-only mode. Only applies to connections created by this endpoint from
databasePath or jdbcUrl, and requires an existing database file." },
"resultFormat": { "index": 7, "kind": "parameter", "displayName": "Result
Format", "group": "producer", "label": "", "required": false, "type": "enum",
"javaType": "org.apache.camel.component.duckdb.DuckDbResultFormat", "enum": [
"LIST_MAP", "JSON" ], "deprecated": false, "autowired": false, "secret": false,
"defaultValue": "LIST_MAP", "description": "How query results are returned:
LIST_MAP (List of Map) or JSON array string." },
"table": { "index": 8, "kind": "parameter", "displayName": "Table",
"group": "producer", "label": "", "required": false, "type": "string",
"javaType": "java.lang.String", "deprecated": false, "autowired": false,
"secret": false, "description": "Target table for insert and copy operations.
Can be overridden per message with the CamelDuckDbTable header." },
- "lazyStartProducer": { "index": 9, "kind": "parameter", "displayName":
"Lazy Start Producer", "group": "producer (advanced)", "label":
"producer,advanced", "required": false, "type": "boolean", "javaType":
"boolean", "deprecated": false, "autowired": false, "secret": false,
"defaultValue": false, "description": "Whether the producer should be started
lazy (on the first message). By starting lazy you can use this to allow
CamelContext and routes to startup in situations where a produc [...]
+ "lazyStartProducer": { "index": 9, "kind": "parameter", "displayName":
"Lazy Start Producer", "group": "producer (advanced)", "label":
"producer,advanced", "required": false, "type": "boolean", "javaType":
"boolean", "deprecated": false, "autowired": false, "secret": false,
"defaultValue": false, "description": "Whether the producer should be started
lazy (on the first message). By starting lazy you can use this to allow
CamelContext and routes to startup in situations where a produc [...]
+ "allowGlobCopySource": { "index": 10, "kind": "parameter", "displayName":
"Allow Glob Copy Source", "group": "security", "label": "producer,security",
"required": false, "type": "boolean", "javaType": "boolean", "deprecated":
false, "autowired": false, "secret": false, "defaultValue": false,
"description": "Whether the copy operation accepts a source with glob pattern
characters (asterisk, question mark, square bracket or curly brace), which
DuckDB expands to all the matching files. [...]
+ "allowUrlCopySource": { "index": 11, "kind": "parameter", "displayName":
"Allow Url Copy Source", "group": "security", "label": "producer,security",
"required": false, "type": "boolean", "javaType": "boolean", "deprecated":
false, "autowired": false, "secret": false, "security": "insecure:dev",
"insecureValue": "true", "defaultValue": false, "description": "Whether the
copy operation accepts a source that starts with a URL scheme (such as
https:\/\/ or s3:\/\/), which DuckDB reads as [...]
+ "copyBaseDirectory": { "index": 12, "kind": "parameter", "displayName":
"Copy Base Directory", "group": "security", "label": "producer,security",
"required": false, "type": "string", "javaType": "java.lang.String",
"deprecated": false, "autowired": false, "secret": false, "description": "A
directory that the local source of the copy operation must be inside, after
resolving symbolic links. A relative source is resolved against this directory.
For a glob pattern, the directory before [...]
}
}
diff --git
a/catalog/camel-catalog/src/generated/resources/org/apache/camel/catalog/docs/duckdb-component.adoc
b/catalog/camel-catalog/src/generated/resources/org/apache/camel/catalog/docs/duckdb-component.adoc
index f049b3ffca6f..f39d42680905 100644
---
a/catalog/camel-catalog/src/generated/resources/org/apache/camel/catalog/docs/duckdb-component.adoc
+++
b/catalog/camel-catalog/src/generated/resources/org/apache/camel/catalog/docs/duckdb-component.adoc
@@ -56,7 +56,7 @@ The producer supports:
* `execute` (default) — run DDL/DML from the message body (or `query` option /
`CamelDuckDbQuery` header)
* `query` — run a SELECT and return rows as `List<Map>` (default) or JSON
(`resultFormat=JSON`)
* `insert` — insert a `List` of `Map` rows into `table`
-* `copy` — bulk load from a `File` or path `String` using DuckDB `read_csv` /
`read_parquet` / `read_json`
+* `copy` — bulk load one local file, from a `File` or a path `String`, using
DuckDB `read_csv` / `read_parquet` / `read_json`
* `ping` — `SELECT 1` connectivity check; sets `CamelDuckDbPingOk`
The endpoint reuses one JDBC connection for its configured `databasePath` /
`jdbcUrl` and serializes access for
@@ -64,6 +64,52 @@ thread safety. Prefer a shared `DataSource` when you need
concurrent producers.
`CamelDuckDbDatabasePath` header to open a short-lived connection to a
different database file for a single
message.
+== Copy sources
+
+The `copy` operation gives the path from the message body to the DuckDB
`read_csv`, `read_parquet` or `read_json`
+function. DuckDB reads a value that starts with a URL scheme (such as
`https://` or `s3://`) as a remote file, and
+expands a value with glob pattern characters (`*`, `?`, `[` or `{`) to all the
matching files. So by default `copy`
+loads a single local file, and fails with an `IllegalArgumentException` that
names the option to set otherwise:
+
+* `allowUrlCopySource=true` passes a source with a URL scheme to DuckDB as it
is.
+* `allowGlobCopySource=true` loads all the files that a glob pattern matches.
This also applies to a `File` body
+ whose name has one of these characters.
+* `copyBaseDirectory` sets the directory that a local source must be inside,
after resolving symbolic links. A
+ relative source is resolved against this directory, and for a glob pattern
the directory before the first glob
+ character must be inside it.
+
+[tabs]
+====
+Java::
++
+[source,java]
+----
+from("direct:load")
+
.to("duckdb:warehouse.db?operation=copy&table=staging&allowGlobCopySource=true©BaseDirectory=/data/incoming");
+----
+
+YAML::
++
+[source,yaml]
+----
+- route:
+ from:
+ uri: direct:load
+ steps:
+ - to:
+ uri: duckdb:warehouse.db
+ parameters:
+ operation: copy
+ table: staging
+ allowGlobCopySource: true
+ copyBaseDirectory: /data/incoming
+----
+====
+
+These options apply to `copy` only. The SQL of `execute` and `query` can call
the read functions itself; to have
+DuckDB restrict file and network access for every statement, configure the
DuckDB settings
+`enable_external_access`, `allowed_directories` and
`autoinstall_known_extensions` (see the DuckDB documentation).
+
== Example
[source,java]
diff --git
a/components/camel-duckdb/src/generated/java/org/apache/camel/component/duckdb/DuckDbEndpointConfigurer.java
b/components/camel-duckdb/src/generated/java/org/apache/camel/component/duckdb/DuckDbEndpointConfigurer.java
index 17a8c1116f8a..8b50ebccf234 100644
---
a/components/camel-duckdb/src/generated/java/org/apache/camel/component/duckdb/DuckDbEndpointConfigurer.java
+++
b/components/camel-duckdb/src/generated/java/org/apache/camel/component/duckdb/DuckDbEndpointConfigurer.java
@@ -23,8 +23,14 @@ public class DuckDbEndpointConfigurer extends
PropertyConfigurerSupport implemen
public boolean configure(CamelContext camelContext, Object obj, String
name, Object value, boolean ignoreCase) {
DuckDbEndpoint target = (DuckDbEndpoint) obj;
switch (ignoreCase ? name.toLowerCase() : name) {
+ case "allowglobcopysource":
+ case "allowGlobCopySource":
target.setAllowGlobCopySource(property(camelContext, boolean.class, value));
return true;
+ case "allowurlcopysource":
+ case "allowUrlCopySource":
target.setAllowUrlCopySource(property(camelContext, boolean.class, value));
return true;
case "batchsize":
case "batchSize": target.setBatchSize(property(camelContext,
int.class, value)); return true;
+ case "copybasedirectory":
+ case "copyBaseDirectory":
target.setCopyBaseDirectory(property(camelContext, java.lang.String.class,
value)); return true;
case "format": target.setFormat(property(camelContext,
java.lang.String.class, value)); return true;
case "jdbcurl":
case "jdbcUrl": target.setJdbcUrl(property(camelContext,
java.lang.String.class, value)); return true;
@@ -44,8 +50,14 @@ public class DuckDbEndpointConfigurer extends
PropertyConfigurerSupport implemen
@Override
public Class<?> getOptionType(String name, boolean ignoreCase) {
switch (ignoreCase ? name.toLowerCase() : name) {
+ case "allowglobcopysource":
+ case "allowGlobCopySource": return boolean.class;
+ case "allowurlcopysource":
+ case "allowUrlCopySource": return boolean.class;
case "batchsize":
case "batchSize": return int.class;
+ case "copybasedirectory":
+ case "copyBaseDirectory": return java.lang.String.class;
case "format": return java.lang.String.class;
case "jdbcurl":
case "jdbcUrl": return java.lang.String.class;
@@ -66,8 +78,14 @@ public class DuckDbEndpointConfigurer extends
PropertyConfigurerSupport implemen
public Object getOptionValue(Object obj, String name, boolean ignoreCase) {
DuckDbEndpoint target = (DuckDbEndpoint) obj;
switch (ignoreCase ? name.toLowerCase() : name) {
+ case "allowglobcopysource":
+ case "allowGlobCopySource": return target.isAllowGlobCopySource();
+ case "allowurlcopysource":
+ case "allowUrlCopySource": return target.isAllowUrlCopySource();
case "batchsize":
case "batchSize": return target.getBatchSize();
+ case "copybasedirectory":
+ case "copyBaseDirectory": return target.getCopyBaseDirectory();
case "format": return target.getFormat();
case "jdbcurl":
case "jdbcUrl": return target.getJdbcUrl();
diff --git
a/components/camel-duckdb/src/generated/java/org/apache/camel/component/duckdb/DuckDbEndpointUriFactory.java
b/components/camel-duckdb/src/generated/java/org/apache/camel/component/duckdb/DuckDbEndpointUriFactory.java
index d88a588be3e9..ea94910f8fb2 100644
---
a/components/camel-duckdb/src/generated/java/org/apache/camel/component/duckdb/DuckDbEndpointUriFactory.java
+++
b/components/camel-duckdb/src/generated/java/org/apache/camel/component/duckdb/DuckDbEndpointUriFactory.java
@@ -24,8 +24,11 @@ public class DuckDbEndpointUriFactory extends
org.apache.camel.support.component
private static final Set<String> ENDPOINT_IDENTITY_PROPERTY_NAMES;
private static final Map<String, String> MULTI_VALUE_PREFIXES;
static {
- Set<String> props = new HashSet<>(10);
+ Set<String> props = new HashSet<>(13);
+ props.add("allowGlobCopySource");
+ props.add("allowUrlCopySource");
props.add("batchSize");
+ props.add("copyBaseDirectory");
props.add("databasePath");
props.add("format");
props.add("jdbcUrl");
diff --git
a/components/camel-duckdb/src/generated/resources/META-INF/org/apache/camel/component/duckdb/duckdb.json
b/components/camel-duckdb/src/generated/resources/META-INF/org/apache/camel/component/duckdb/duckdb.json
index 555250e2327b..b22df589e5bf 100644
---
a/components/camel-duckdb/src/generated/resources/META-INF/org/apache/camel/component/duckdb/duckdb.json
+++
b/components/camel-duckdb/src/generated/resources/META-INF/org/apache/camel/component/duckdb/duckdb.json
@@ -49,6 +49,9 @@
"readOnly": { "index": 6, "kind": "parameter", "displayName": "Read Only",
"group": "producer", "label": "", "required": false, "type": "boolean",
"javaType": "boolean", "deprecated": false, "autowired": false, "secret":
false, "defaultValue": false, "description": "Open the embedded database file
in read-only mode. Only applies to connections created by this endpoint from
databasePath or jdbcUrl, and requires an existing database file." },
"resultFormat": { "index": 7, "kind": "parameter", "displayName": "Result
Format", "group": "producer", "label": "", "required": false, "type": "enum",
"javaType": "org.apache.camel.component.duckdb.DuckDbResultFormat", "enum": [
"LIST_MAP", "JSON" ], "deprecated": false, "autowired": false, "secret": false,
"defaultValue": "LIST_MAP", "description": "How query results are returned:
LIST_MAP (List of Map) or JSON array string." },
"table": { "index": 8, "kind": "parameter", "displayName": "Table",
"group": "producer", "label": "", "required": false, "type": "string",
"javaType": "java.lang.String", "deprecated": false, "autowired": false,
"secret": false, "description": "Target table for insert and copy operations.
Can be overridden per message with the CamelDuckDbTable header." },
- "lazyStartProducer": { "index": 9, "kind": "parameter", "displayName":
"Lazy Start Producer", "group": "producer (advanced)", "label":
"producer,advanced", "required": false, "type": "boolean", "javaType":
"boolean", "deprecated": false, "autowired": false, "secret": false,
"defaultValue": false, "description": "Whether the producer should be started
lazy (on the first message). By starting lazy you can use this to allow
CamelContext and routes to startup in situations where a produc [...]
+ "lazyStartProducer": { "index": 9, "kind": "parameter", "displayName":
"Lazy Start Producer", "group": "producer (advanced)", "label":
"producer,advanced", "required": false, "type": "boolean", "javaType":
"boolean", "deprecated": false, "autowired": false, "secret": false,
"defaultValue": false, "description": "Whether the producer should be started
lazy (on the first message). By starting lazy you can use this to allow
CamelContext and routes to startup in situations where a produc [...]
+ "allowGlobCopySource": { "index": 10, "kind": "parameter", "displayName":
"Allow Glob Copy Source", "group": "security", "label": "producer,security",
"required": false, "type": "boolean", "javaType": "boolean", "deprecated":
false, "autowired": false, "secret": false, "defaultValue": false,
"description": "Whether the copy operation accepts a source with glob pattern
characters (asterisk, question mark, square bracket or curly brace), which
DuckDB expands to all the matching files. [...]
+ "allowUrlCopySource": { "index": 11, "kind": "parameter", "displayName":
"Allow Url Copy Source", "group": "security", "label": "producer,security",
"required": false, "type": "boolean", "javaType": "boolean", "deprecated":
false, "autowired": false, "secret": false, "security": "insecure:dev",
"insecureValue": "true", "defaultValue": false, "description": "Whether the
copy operation accepts a source that starts with a URL scheme (such as
https:\/\/ or s3:\/\/), which DuckDB reads as [...]
+ "copyBaseDirectory": { "index": 12, "kind": "parameter", "displayName":
"Copy Base Directory", "group": "security", "label": "producer,security",
"required": false, "type": "string", "javaType": "java.lang.String",
"deprecated": false, "autowired": false, "secret": false, "description": "A
directory that the local source of the copy operation must be inside, after
resolving symbolic links. A relative source is resolved against this directory.
For a glob pattern, the directory before [...]
}
}
diff --git a/components/camel-duckdb/src/main/docs/duckdb-component.adoc
b/components/camel-duckdb/src/main/docs/duckdb-component.adoc
index f049b3ffca6f..f39d42680905 100644
--- a/components/camel-duckdb/src/main/docs/duckdb-component.adoc
+++ b/components/camel-duckdb/src/main/docs/duckdb-component.adoc
@@ -56,7 +56,7 @@ The producer supports:
* `execute` (default) — run DDL/DML from the message body (or `query` option /
`CamelDuckDbQuery` header)
* `query` — run a SELECT and return rows as `List<Map>` (default) or JSON
(`resultFormat=JSON`)
* `insert` — insert a `List` of `Map` rows into `table`
-* `copy` — bulk load from a `File` or path `String` using DuckDB `read_csv` /
`read_parquet` / `read_json`
+* `copy` — bulk load one local file, from a `File` or a path `String`, using
DuckDB `read_csv` / `read_parquet` / `read_json`
* `ping` — `SELECT 1` connectivity check; sets `CamelDuckDbPingOk`
The endpoint reuses one JDBC connection for its configured `databasePath` /
`jdbcUrl` and serializes access for
@@ -64,6 +64,52 @@ thread safety. Prefer a shared `DataSource` when you need
concurrent producers.
`CamelDuckDbDatabasePath` header to open a short-lived connection to a
different database file for a single
message.
+== Copy sources
+
+The `copy` operation gives the path from the message body to the DuckDB
`read_csv`, `read_parquet` or `read_json`
+function. DuckDB reads a value that starts with a URL scheme (such as
`https://` or `s3://`) as a remote file, and
+expands a value with glob pattern characters (`*`, `?`, `[` or `{`) to all the
matching files. So by default `copy`
+loads a single local file, and fails with an `IllegalArgumentException` that
names the option to set otherwise:
+
+* `allowUrlCopySource=true` passes a source with a URL scheme to DuckDB as it
is.
+* `allowGlobCopySource=true` loads all the files that a glob pattern matches.
This also applies to a `File` body
+ whose name has one of these characters.
+* `copyBaseDirectory` sets the directory that a local source must be inside,
after resolving symbolic links. A
+ relative source is resolved against this directory, and for a glob pattern
the directory before the first glob
+ character must be inside it.
+
+[tabs]
+====
+Java::
++
+[source,java]
+----
+from("direct:load")
+
.to("duckdb:warehouse.db?operation=copy&table=staging&allowGlobCopySource=true©BaseDirectory=/data/incoming");
+----
+
+YAML::
++
+[source,yaml]
+----
+- route:
+ from:
+ uri: direct:load
+ steps:
+ - to:
+ uri: duckdb:warehouse.db
+ parameters:
+ operation: copy
+ table: staging
+ allowGlobCopySource: true
+ copyBaseDirectory: /data/incoming
+----
+====
+
+These options apply to `copy` only. The SQL of `execute` and `query` can call
the read functions itself; to have
+DuckDB restrict file and network access for every statement, configure the
DuckDB settings
+`enable_external_access`, `allowed_directories` and
`autoinstall_known_extensions` (see the DuckDB documentation).
+
== Example
[source,java]
diff --git
a/components/camel-duckdb/src/main/java/org/apache/camel/component/duckdb/DuckDbEndpoint.java
b/components/camel-duckdb/src/main/java/org/apache/camel/component/duckdb/DuckDbEndpoint.java
index 80895ed53157..00d62d2de444 100644
---
a/components/camel-duckdb/src/main/java/org/apache/camel/component/duckdb/DuckDbEndpoint.java
+++
b/components/camel-duckdb/src/main/java/org/apache/camel/component/duckdb/DuckDbEndpoint.java
@@ -72,6 +72,22 @@ public class DuckDbEndpoint extends DefaultEndpoint {
@UriParam(defaultValue = "LIST_MAP",
description = "How query results are returned: LIST_MAP (List of
Map) or JSON array string.")
private DuckDbResultFormat resultFormat = DuckDbResultFormat.LIST_MAP;
+ @UriParam(defaultValue = "false", label = "producer,security", security =
"insecure:dev", insecureValue = "true",
+ description = "Whether the copy operation accepts a source that
starts with a URL scheme (such as https://"
+ + " or s3://), which DuckDB reads as a remote
file. Disabled by default, so copy loads a"
+ + " local file only; enable it only when the
source of the message body is trusted.")
+ private boolean allowUrlCopySource;
+ @UriParam(defaultValue = "false", label = "producer,security",
+ description = "Whether the copy operation accepts a source with
glob pattern characters (asterisk,"
+ + " question mark, square bracket or curly brace),
which DuckDB expands to all the matching"
+ + " files. Disabled by default, so copy loads a
single file.")
+ private boolean allowGlobCopySource;
+ @UriParam(label = "producer,security",
+ description = "A directory that the local source of the copy
operation must be inside, after resolving"
+ + " symbolic links. A relative source is resolved
against this directory. For a glob"
+ + " pattern, the directory before the first glob
character must be inside it. Not set by"
+ + " default, so copy loads a file from any
directory.")
+ private String copyBaseDirectory;
public DuckDbEndpoint() {
}
@@ -236,6 +252,30 @@ public class DuckDbEndpoint extends DefaultEndpoint {
this.resultFormat = resultFormat;
}
+ public boolean isAllowUrlCopySource() {
+ return allowUrlCopySource;
+ }
+
+ public void setAllowUrlCopySource(boolean allowUrlCopySource) {
+ this.allowUrlCopySource = allowUrlCopySource;
+ }
+
+ public boolean isAllowGlobCopySource() {
+ return allowGlobCopySource;
+ }
+
+ public void setAllowGlobCopySource(boolean allowGlobCopySource) {
+ this.allowGlobCopySource = allowGlobCopySource;
+ }
+
+ public String getCopyBaseDirectory() {
+ return copyBaseDirectory;
+ }
+
+ public void setCopyBaseDirectory(String copyBaseDirectory) {
+ this.copyBaseDirectory = copyBaseDirectory;
+ }
+
String resolvedJdbcUrl() {
return DuckDbJdbcSupport.resolveJdbcUrl(jdbcUrl, databasePath);
}
diff --git
a/components/camel-duckdb/src/main/java/org/apache/camel/component/duckdb/DuckDbProducer.java
b/components/camel-duckdb/src/main/java/org/apache/camel/component/duckdb/DuckDbProducer.java
index 405cec5c5db5..059ded3b195a 100644
---
a/components/camel-duckdb/src/main/java/org/apache/camel/component/duckdb/DuckDbProducer.java
+++
b/components/camel-duckdb/src/main/java/org/apache/camel/component/duckdb/DuckDbProducer.java
@@ -17,12 +17,16 @@
package org.apache.camel.component.duckdb;
import java.io.File;
+import java.io.IOException;
+import java.nio.file.Files;
+import java.nio.file.Path;
import java.sql.Connection;
import java.sql.ResultSet;
import java.sql.Statement;
import java.util.List;
import java.util.Locale;
import java.util.Map;
+import java.util.regex.Pattern;
import org.apache.camel.Exchange;
import org.apache.camel.Message;
@@ -35,6 +39,8 @@ import org.slf4j.LoggerFactory;
public class DuckDbProducer extends DefaultProducer {
private static final Logger LOG =
LoggerFactory.getLogger(DuckDbProducer.class);
+ private static final Pattern URL_SCHEME =
Pattern.compile("^[A-Za-z][A-Za-z0-9+.-]*://");
+ private static final String GLOB_CHARS = "*?[{";
private final DuckDbEndpoint endpoint;
@@ -112,7 +118,7 @@ public class DuckDbProducer extends DefaultProducer {
if (ObjectHelper.isEmpty(table)) {
throw new DuckDbException("A table is required for copy
operations");
}
- String path = resolveFilePath(exchange);
+ String path = resolveCopySource(resolveFilePath(exchange));
String format = resolveFormat(path);
String sql = buildCopySql(table, path, format);
long updated;
@@ -176,6 +182,73 @@ public class DuckDbProducer extends DefaultProducer {
throw new DuckDbException("Copy operation expects a file path in the
message body (File or String)");
}
+ /**
+ * DuckDB reads the source of a copy as a remote file when it starts with
a URL scheme, and expands it to many files
+ * when it holds glob characters. Both need an option, and
copyBaseDirectory keeps a local source inside a
+ * directory.
+ *
+ * @return the source to give to DuckDB
+ */
+ private String resolveCopySource(String source) throws IOException {
+ if (URL_SCHEME.matcher(source).find()) {
+ if (!endpoint.isAllowUrlCopySource()) {
+ throw new IllegalArgumentException(
+ "The copy source is a URL, which is loaded only when
allowUrlCopySource=true: " + source);
+ }
+ return source;
+ }
+ int glob = indexOfGlob(source);
+ if (glob >= 0 && !endpoint.isAllowGlobCopySource()) {
+ throw new IllegalArgumentException(
+ "The copy source contains a glob pattern character (*, ?,
[ or {), which is expanded only when"
+ + " allowGlobCopySource=true: "
+ source);
+ }
+ String base = endpoint.getCopyBaseDirectory();
+ if (ObjectHelper.isEmpty(base)) {
+ return source;
+ }
+ Path baseDir = Path.of(base).toRealPath();
+ if (glob < 0) {
+ return resolveWithinBaseDirectory(baseDir, source,
source).toString();
+ }
+ // a pattern: check the directory before the first glob character, and
keep the pattern itself as is
+ for (String segment : source.split("[/\\\\]")) {
+ if ("..".equals(segment)) {
+ throw new IllegalArgumentException(
+ "The copy source must not contain .. when
copyBaseDirectory is set: " + source);
+ }
+ }
+ int sep = source.lastIndexOf('/', glob);
+ if (File.separatorChar != '/') {
+ sep = Math.max(sep, source.lastIndexOf(File.separatorChar, glob));
+ }
+ String dir = resolveWithinBaseDirectory(baseDir, source.substring(0,
sep + 1), source).toString();
+ String pattern = source.substring(sep + 1);
+ return dir.endsWith(File.separator) ? dir + pattern : dir +
File.separator + pattern;
+ }
+
+ private static Path resolveWithinBaseDirectory(Path baseDir, String path,
String source) throws IOException {
+ Path resolved = baseDir.resolve(path).normalize();
+ if (Files.exists(resolved)) {
+ // follow symbolic links, so a link inside the directory to a file
outside it is not loaded
+ resolved = resolved.toRealPath();
+ }
+ if (!resolved.startsWith(baseDir)) {
+ throw new IllegalArgumentException(
+ "The copy source is outside the directory of the
copyBaseDirectory option: " + source);
+ }
+ return resolved;
+ }
+
+ private static int indexOfGlob(String source) {
+ for (int i = 0; i < source.length(); i++) {
+ if (GLOB_CHARS.indexOf(source.charAt(i)) >= 0) {
+ return i;
+ }
+ }
+ return -1;
+ }
+
private void setRowsWritten(Exchange exchange, long rows) {
exchange.getMessage().setHeader(DuckDbConstants.ROWS_WRITTEN, rows);
exchange.getMessage().setBody(rows);
diff --git
a/components/camel-duckdb/src/test/java/org/apache/camel/component/duckdb/DuckDbProducerCopySourceTest.java
b/components/camel-duckdb/src/test/java/org/apache/camel/component/duckdb/DuckDbProducerCopySourceTest.java
new file mode 100644
index 000000000000..a60805b65fba
--- /dev/null
+++
b/components/camel-duckdb/src/test/java/org/apache/camel/component/duckdb/DuckDbProducerCopySourceTest.java
@@ -0,0 +1,241 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.camel.component.duckdb;
+
+import java.net.URLEncoder;
+import java.nio.charset.StandardCharsets;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.sql.Connection;
+import java.sql.DriverManager;
+import java.sql.SQLException;
+import java.sql.Statement;
+import java.util.List;
+import java.util.Map;
+import java.util.Properties;
+
+import org.apache.camel.CamelExecutionException;
+import org.apache.camel.test.junit6.CamelTestSupport;
+import org.junit.jupiter.api.AfterEach;
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.condition.DisabledOnOs;
+import org.junit.jupiter.api.condition.OS;
+import org.junit.jupiter.api.io.TempDir;
+
+import static org.assertj.core.api.Assertions.assertThat;
+import static org.assertj.core.api.Assertions.assertThatThrownBy;
+
+/**
+ * The copy operation loads a single local file unless an option allows a URL
or a glob pattern, and copyBaseDirectory
+ * keeps the source inside a directory.
+ */
+class DuckDbProducerCopySourceTest extends CamelTestSupport {
+
+ @TempDir
+ Path tempDir;
+
+ private String uri;
+ private Connection offlineConnection;
+
+ private String endpointUri() throws Exception {
+ if (uri == null) {
+ String jdbcUrl = "jdbc:duckdb:" +
tempDir.resolve("test.db").toAbsolutePath();
+ uri = "duckdb::memory:?jdbcUrl=" + URLEncoder.encode(jdbcUrl,
StandardCharsets.UTF_8);
+ template.sendBody(uri + "&operation=execute", "CREATE TABLE events
(id INTEGER, name VARCHAR)");
+ }
+ return uri;
+ }
+
+ private List<Object> names() throws Exception {
+ List<?> rows = template.requestBody(endpointUri() +
"&operation=query", "SELECT name FROM events ORDER BY id",
+ List.class);
+ return rows.stream().<Object> map(row -> ((Map<?, ?>)
row).get("name")).toList();
+ }
+
+ private Path csv(Path dir, String fileName, int id, String name) throws
Exception {
+ Files.createDirectories(dir);
+ return Files.writeString(dir.resolve(fileName), "id,name\n" + id + ","
+ name + "\n");
+ }
+
+ private static String encode(Path path) {
+ return URLEncoder.encode(path.toAbsolutePath().toString(),
StandardCharsets.UTF_8);
+ }
+
+ @Test
+ void copyRefusesUrlSourceByDefault() throws Exception {
+ String copyUri = "duckdb::memory:?operation=copy&table=events";
+ useConnectionWithoutExtensionLoading(copyUri);
+
+ assertThatThrownBy(() -> template.sendBody(copyUri,
"https://example.com/data.csv"))
+ .isInstanceOf(CamelExecutionException.class)
+ .cause()
+ .isInstanceOf(IllegalArgumentException.class)
+ .hasMessageContaining("allowUrlCopySource=true");
+ }
+
+ @Test
+ void copyPassesUrlSourceToDuckDbWhenAllowed() throws Exception {
+ String copyUri =
"duckdb::memory:?operation=copy&table=events&allowUrlCopySource=true";
+ useConnectionWithoutExtensionLoading(copyUri);
+
+ // DuckDB gets the URL: it fails here only because the connection may
not load the extension that reads it
+ assertThatThrownBy(() -> template.sendBody(copyUri,
"https://example.com/data.csv"))
+ .isInstanceOf(CamelExecutionException.class)
+ .rootCause()
+ .isInstanceOf(SQLException.class);
+ }
+
+ @Test
+ void copyRefusesGlobPatternByDefault() throws Exception {
+ csv(tempDir, "one.csv", 1, "ada");
+ csv(tempDir, "two.csv", 2, "bob");
+ String copyUri = endpointUri() + "&operation=copy&table=events";
+
+ assertThatThrownBy(() -> template.sendBody(copyUri,
tempDir.toAbsolutePath() + "/*.csv"))
+ .isInstanceOf(CamelExecutionException.class)
+ .cause()
+ .isInstanceOf(IllegalArgumentException.class)
+ .hasMessageContaining("allowGlobCopySource=true");
+ assertThat(names()).isEmpty();
+ }
+
+ @Test
+ void copyLoadsGlobPatternWhenAllowed() throws Exception {
+ csv(tempDir, "one.csv", 1, "ada");
+ csv(tempDir, "two.csv", 2, "bob");
+
+ Long written = template.requestBody(endpointUri() +
"&operation=copy&table=events&allowGlobCopySource=true",
+ tempDir.toAbsolutePath() + "/*.csv", Long.class);
+
+ assertThat(written).isEqualTo(2L);
+ assertThat(names()).containsExactly("ada", "bob");
+ }
+
+ @Test
+ void copyRefusesFileNameWithGlobCharacters() throws Exception {
+ csv(tempDir, "data1.csv", 1, "ada");
+ Path file = csv(tempDir, "data[1].csv", 2, "bob");
+ String copyUri = endpointUri() + "&operation=copy&table=events";
+
+ assertThatThrownBy(() -> template.sendBody(copyUri, file.toFile()))
+ .isInstanceOf(CamelExecutionException.class)
+ .cause()
+ .isInstanceOf(IllegalArgumentException.class)
+ .hasMessageContaining("allowGlobCopySource=true");
+ assertThat(names()).isEmpty();
+ }
+
+ @Test
+ void copyLoadsFileInsideBaseDirectory() throws Exception {
+ Path base = tempDir.resolve("in");
+ Path file = csv(base, "one.csv", 1, "ada");
+ csv(base.resolve("sub"), "two.csv", 2, "bob");
+ String copyUri = endpointUri() +
"&operation=copy&table=events©BaseDirectory=" + encode(base);
+
+ template.sendBody(copyUri, file.toFile());
+ // a relative source is resolved against the base directory
+ template.sendBody(copyUri, "sub/two.csv");
+
+ assertThat(names()).containsExactly("ada", "bob");
+ }
+
+ @Test
+ void copyRefusesFileOutsideBaseDirectory() throws Exception {
+ Path base = tempDir.resolve("in");
+ Files.createDirectories(base);
+ Path outside = csv(tempDir.resolve("out"), "one.csv", 1, "ada");
+ // a sibling directory whose name starts with the name of the base
directory is outside it too
+ Path sibling = csv(tempDir.resolve("in-other"), "two.csv", 2, "bob");
+ String copyUri = endpointUri() +
"&operation=copy&table=events©BaseDirectory=" + encode(base);
+
+ for (Object body : List.of(outside.toAbsolutePath().toString(),
outside.toFile(), "../out/one.csv",
+ sibling.toAbsolutePath().toString())) {
+ assertThatThrownBy(() -> template.sendBody(copyUri, body))
+ .isInstanceOf(CamelExecutionException.class)
+ .cause()
+ .isInstanceOf(IllegalArgumentException.class)
+ .hasMessageContaining("copyBaseDirectory");
+ }
+ assertThat(names()).isEmpty();
+ }
+
+ @Test
+ @DisabledOnOs(OS.WINDOWS)
+ void copyRefusesSymbolicLinkLeavingBaseDirectory() throws Exception {
+ Path base = tempDir.resolve("in");
+ Files.createDirectories(base);
+ Path outside = csv(tempDir.resolve("out"), "one.csv", 1, "ada");
+ Files.createSymbolicLink(base.resolve("link.csv"), outside);
+ Files.createSymbolicLink(base.resolve("linkdir"), outside.getParent());
+ String copyUri = endpointUri() +
"&operation=copy&table=events©BaseDirectory=" + encode(base);
+
+ for (String body : List.of("link.csv", "linkdir/one.csv")) {
+ assertThatThrownBy(() -> template.sendBody(copyUri, body))
+ .isInstanceOf(CamelExecutionException.class)
+ .cause()
+ .isInstanceOf(IllegalArgumentException.class)
+ .hasMessageContaining("copyBaseDirectory");
+ }
+ assertThat(names()).isEmpty();
+ }
+
+ @Test
+ void copyLoadsGlobPatternOnlyInsideBaseDirectory() throws Exception {
+ Path base = tempDir.resolve("in");
+ csv(base, "one.csv", 1, "ada");
+ csv(base, "two.csv", 2, "bob");
+ csv(tempDir.resolve("out"), "three.csv", 3, "carl");
+ String copyUri = endpointUri() +
"&operation=copy&table=events&allowGlobCopySource=true©BaseDirectory="
+ + encode(base);
+
+ for (String body : List.of("../out/*.csv",
tempDir.resolve("out").toAbsolutePath() + "/*.csv",
+ "*/../../out/*.csv")) {
+ assertThatThrownBy(() -> template.sendBody(copyUri, body))
+ .isInstanceOf(CamelExecutionException.class)
+ .cause()
+ .isInstanceOf(IllegalArgumentException.class)
+ .hasMessageContaining("copyBaseDirectory");
+ }
+
+ Long written = template.requestBody(copyUri, "*.csv", Long.class);
+ assertThat(written).isEqualTo(2L);
+ assertThat(names()).containsExactly("ada", "bob");
+ }
+
+ /**
+ * An in-memory connection that may not install or load DuckDB extensions,
so a URL source never reaches the
+ * network.
+ */
+ private void useConnectionWithoutExtensionLoading(String copyUri) throws
Exception {
+ Properties properties = new Properties();
+ properties.setProperty("autoinstall_known_extensions", "false");
+ properties.setProperty("autoload_known_extensions", "false");
+ Connection connection = DriverManager.getConnection("jdbc:duckdb:",
properties);
+ try (Statement statement = connection.createStatement()) {
+ statement.execute("CREATE TABLE events (id INTEGER, name
VARCHAR)");
+ }
+ context.getEndpoint(copyUri,
DuckDbEndpoint.class).setConnectionForTesting(connection);
+ offlineConnection = connection;
+ }
+
+ @AfterEach
+ void closeConnection() throws SQLException {
+ if (offlineConnection != null) {
+ offlineConnection.close();
+ }
+ }
+}
diff --git
a/core/camel-util/src/main/java/org/apache/camel/util/SecurityUtils.java
b/core/camel-util/src/main/java/org/apache/camel/util/SecurityUtils.java
index 33cfc1c638d9..694bef22cad5 100644
--- a/core/camel-util/src/main/java/org/apache/camel/util/SecurityUtils.java
+++ b/core/camel-util/src/main/java/org/apache/camel/util/SecurityUtils.java
@@ -65,6 +65,7 @@ public final class SecurityUtils {
map.put("allowpredicatefrommessage", new SecurityOption(INSECURE_DEV,
"true"));
map.put("allowserializedheaders", new
SecurityOption(INSECURE_SERIALIZATION, "true"));
map.put("allowurlbody", new SecurityOption(INSECURE_DEV, "true"));
+ map.put("allowurlcopysource", new SecurityOption(INSECURE_DEV,
"true"));
map.put("allowurlsource", new SecurityOption(INSECURE_DEV, "true"));
map.put("devconsoleenabled", new SecurityOption(INSECURE_DEV, "true"));
map.put("downloadenabled", new SecurityOption(INSECURE_DEV, "true"));
diff --git
a/dsl/camel-endpointdsl/src/generated/java/org/apache/camel/builder/endpoint/dsl/DuckDbEndpointBuilderFactory.java
b/dsl/camel-endpointdsl/src/generated/java/org/apache/camel/builder/endpoint/dsl/DuckDbEndpointBuilderFactory.java
index af1d5b494565..780612970221 100644
---
a/dsl/camel-endpointdsl/src/generated/java/org/apache/camel/builder/endpoint/dsl/DuckDbEndpointBuilderFactory.java
+++
b/dsl/camel-endpointdsl/src/generated/java/org/apache/camel/builder/endpoint/dsl/DuckDbEndpointBuilderFactory.java
@@ -239,6 +239,96 @@ public interface DuckDbEndpointBuilderFactory {
doSetProperty("table", table);
return this;
}
+ /**
+ * Whether the copy operation accepts a source with glob pattern
+ * characters (asterisk, question mark, square bracket or curly brace),
+ * which DuckDB expands to all the matching files. Disabled by default,
+ * so copy loads a single file.
+ *
+ * The option is a: <code>boolean</code> type.
+ *
+ * Default: false
+ * Group: security
+ *
+ * @param allowGlobCopySource the value to set
+ * @return the dsl builder
+ */
+ default DuckDbEndpointBuilder allowGlobCopySource(boolean
allowGlobCopySource) {
+ doSetProperty("allowGlobCopySource", allowGlobCopySource);
+ return this;
+ }
+ /**
+ * Whether the copy operation accepts a source with glob pattern
+ * characters (asterisk, question mark, square bracket or curly brace),
+ * which DuckDB expands to all the matching files. Disabled by default,
+ * so copy loads a single file.
+ *
+ * The option will be converted to a <code>boolean</code> type.
+ *
+ * Default: false
+ * Group: security
+ *
+ * @param allowGlobCopySource the value to set
+ * @return the dsl builder
+ */
+ default DuckDbEndpointBuilder allowGlobCopySource(String
allowGlobCopySource) {
+ doSetProperty("allowGlobCopySource", allowGlobCopySource);
+ return this;
+ }
+ /**
+ * Whether the copy operation accepts a source that starts with a URL
+ * scheme (such as https:// or s3://), which DuckDB reads as a remote
+ * file. Disabled by default, so copy loads a local file only; enable
it
+ * only when the source of the message body is trusted.
+ *
+ * The option is a: <code>boolean</code> type.
+ *
+ * Default: false
+ * Group: security
+ *
+ * @param allowUrlCopySource the value to set
+ * @return the dsl builder
+ */
+ default DuckDbEndpointBuilder allowUrlCopySource(boolean
allowUrlCopySource) {
+ doSetProperty("allowUrlCopySource", allowUrlCopySource);
+ return this;
+ }
+ /**
+ * Whether the copy operation accepts a source that starts with a URL
+ * scheme (such as https:// or s3://), which DuckDB reads as a remote
+ * file. Disabled by default, so copy loads a local file only; enable
it
+ * only when the source of the message body is trusted.
+ *
+ * The option will be converted to a <code>boolean</code> type.
+ *
+ * Default: false
+ * Group: security
+ *
+ * @param allowUrlCopySource the value to set
+ * @return the dsl builder
+ */
+ default DuckDbEndpointBuilder allowUrlCopySource(String
allowUrlCopySource) {
+ doSetProperty("allowUrlCopySource", allowUrlCopySource);
+ return this;
+ }
+ /**
+ * A directory that the local source of the copy operation must be
+ * inside, after resolving symbolic links. A relative source is
resolved
+ * against this directory. For a glob pattern, the directory before the
+ * first glob character must be inside it. Not set by default, so copy
+ * loads a file from any directory.
+ *
+ * The option is a: <code>java.lang.String</code> type.
+ *
+ * Group: security
+ *
+ * @param copyBaseDirectory the value to set
+ * @return the dsl builder
+ */
+ default DuckDbEndpointBuilder copyBaseDirectory(String
copyBaseDirectory) {
+ doSetProperty("copyBaseDirectory", copyBaseDirectory);
+ return this;
+ }
}
/**