This is an automated email from the ASF dual-hosted git repository. davsclaus pushed a commit to branch main in repository https://gitbox.apache.org/repos/asf/camel.git
The following commit(s) were added to refs/heads/main by this push: new 65123ab6ccd CAMEL-21845: camel-sql - Improve performance of batch inserts (#17390) 65123ab6ccd is described below commit 65123ab6ccd5b62c69edb81aff52bb2b3c54245f Author: Claus Ibsen <claus.ib...@gmail.com> AuthorDate: Fri Mar 7 15:33:11 2025 +0000 CAMEL-21845: camel-sql - Improve performance of batch inserts (#17390) * CAMEL-21845: camel-sql - Improve performance of batch inserts --- .../org/apache/camel/catalog/components/sql.json | 66 +++++---- .../component/sql/SqlComponentConfigurer.java | 12 ++ .../camel/component/sql/SqlEndpointConfigurer.java | 12 ++ .../camel/component/sql/SqlEndpointUriFactory.java | 4 +- .../org/apache/camel/component/sql/sql.json | 66 +++++---- .../camel/component/sql/DefaultSqlEndpoint.java | 21 +++ .../apache/camel/component/sql/SqlComponent.java | 37 +++++ .../apache/camel/component/sql/SqlConsumer.java | 4 + .../apache/camel/component/sql/SqlEndpoint.java | 15 ++ .../apache/camel/component/sql/SqlProducer.java | 41 +++++- .../component/sql/SqlConsumerFetchSizeTest.java | 87 +++++++++++ .../component/sql/SqlProducerFetchSizeTest.java | 87 +++++++++++ .../ROOT/pages/camel-4x-upgrade-guide-4_10.adoc | 7 + .../ROOT/pages/camel-4x-upgrade-guide-4_11.adoc | 7 + .../component/dsl/SqlComponentBuilderFactory.java | 43 ++++++ .../endpoint/dsl/SqlEndpointBuilderFactory.java | 160 +++++++++++++++++++++ 16 files changed, 601 insertions(+), 68 deletions(-) diff --git a/catalog/camel-catalog/src/generated/resources/org/apache/camel/catalog/components/sql.json b/catalog/camel-catalog/src/generated/resources/org/apache/camel/catalog/components/sql.json index 7594b0197ef..1e4bcc94236 100644 --- a/catalog/camel-catalog/src/generated/resources/org/apache/camel/catalog/components/sql.json +++ b/catalog/camel-catalog/src/generated/resources/org/apache/camel/catalog/components/sql.json @@ -26,13 +26,15 @@ "componentProperties": { "dataSource": { "index": 0, "kind": "property", "displayName": "Data Source", "group": "common", "label": "", "required": false, "type": "object", "javaType": "javax.sql.DataSource", "deprecated": false, "autowired": true, "secret": false, "description": "Sets the DataSource to use to communicate with the database." }, "bridgeErrorHandler": { "index": 1, "kind": "property", "displayName": "Bridge Error Handler", "group": "consumer", "label": "consumer", "required": false, "type": "boolean", "javaType": "boolean", "deprecated": false, "autowired": false, "secret": false, "defaultValue": false, "description": "Allows for bridging the consumer to the Camel routing Error Handler, which mean any exceptions (if possible) occurred while the Camel consumer is trying to pickup incoming messages, or the like [...] - "lazyStartProducer": { "index": 2, "kind": "property", "displayName": "Lazy Start Producer", "group": "producer", "label": "producer", "required": false, "type": "boolean", "javaType": "boolean", "deprecated": false, "autowired": false, "secret": false, "defaultValue": false, "description": "Whether the producer should be started lazy (on the first message). By starting lazy you can use this to allow CamelContext and routes to startup in situations where a producer may otherwise fail [...] - "autowiredEnabled": { "index": 3, "kind": "property", "displayName": "Autowired Enabled", "group": "advanced", "label": "advanced", "required": false, "type": "boolean", "javaType": "boolean", "deprecated": false, "autowired": false, "secret": false, "defaultValue": true, "description": "Whether autowiring is enabled. This is used for automatic autowiring options (the option must be marked as autowired) by looking up in the registry to find if there is a single instance of matching t [...] - "rowMapperFactory": { "index": 4, "kind": "property", "displayName": "Row Mapper Factory", "group": "advanced", "label": "advanced", "required": false, "type": "object", "javaType": "org.apache.camel.component.sql.RowMapperFactory", "deprecated": false, "autowired": true, "secret": false, "description": "Factory for creating RowMapper" }, - "serviceLocationEnabled": { "index": 5, "kind": "property", "displayName": "Service Location Enabled", "group": "advanced", "label": "advanced", "required": false, "type": "boolean", "javaType": "boolean", "deprecated": false, "autowired": false, "secret": false, "defaultValue": true, "description": "Whether to detect the network address location of the JMS broker on startup. This information is gathered via reflection on the ConnectionFactory, and is vendor specific. This option can [...] - "usePlaceholder": { "index": 6, "kind": "property", "displayName": "Use Placeholder", "group": "advanced", "label": "advanced", "required": false, "type": "boolean", "javaType": "boolean", "deprecated": false, "autowired": false, "secret": false, "defaultValue": true, "description": "Sets whether to use placeholder and replace all placeholder characters with sign in the SQL queries. This option is default true" }, - "healthCheckConsumerEnabled": { "index": 7, "kind": "property", "displayName": "Health Check Consumer Enabled", "group": "health", "label": "health", "required": false, "type": "boolean", "javaType": "boolean", "deprecated": false, "autowired": false, "secret": false, "defaultValue": true, "description": "Used for enabling or disabling all consumer based health checks from this component" }, - "healthCheckProducerEnabled": { "index": 8, "kind": "property", "displayName": "Health Check Producer Enabled", "group": "health", "label": "health", "required": false, "type": "boolean", "javaType": "boolean", "deprecated": false, "autowired": false, "secret": false, "defaultValue": true, "description": "Used for enabling or disabling all producer based health checks from this component. Notice: Camel has by default disabled all producer based health-checks. You can turn on producer [...] + "batchAutoCommitDisabled": { "index": 2, "kind": "property", "displayName": "Batch Auto Commit Disabled", "group": "producer", "label": "producer", "required": false, "type": "boolean", "javaType": "boolean", "deprecated": false, "autowired": false, "secret": false, "defaultValue": true, "description": "Whether to optimize batch by turning off auto-commit which can dramatic improve performance, and instead execute as a manual commit after the entire batch operation is complete" }, + "lazyStartProducer": { "index": 3, "kind": "property", "displayName": "Lazy Start Producer", "group": "producer", "label": "producer", "required": false, "type": "boolean", "javaType": "boolean", "deprecated": false, "autowired": false, "secret": false, "defaultValue": false, "description": "Whether the producer should be started lazy (on the first message). By starting lazy you can use this to allow CamelContext and routes to startup in situations where a producer may otherwise fail [...] + "autowiredEnabled": { "index": 4, "kind": "property", "displayName": "Autowired Enabled", "group": "advanced", "label": "advanced", "required": false, "type": "boolean", "javaType": "boolean", "deprecated": false, "autowired": false, "secret": false, "defaultValue": true, "description": "Whether autowiring is enabled. This is used for automatic autowiring options (the option must be marked as autowired) by looking up in the registry to find if there is a single instance of matching t [...] + "fetchSize": { "index": 5, "kind": "property", "displayName": "Fetch Size", "group": "advanced", "label": "advanced", "required": false, "type": "integer", "javaType": "int", "deprecated": false, "autowired": false, "secret": false, "description": "Gives the JDBC driver a hint as to the number of rows that should be fetched from the database when more rows are needed for ResultSet objects generated by this Statement. If the value specified is zero, then the hint is ignored. The defau [...] + "rowMapperFactory": { "index": 6, "kind": "property", "displayName": "Row Mapper Factory", "group": "advanced", "label": "advanced", "required": false, "type": "object", "javaType": "org.apache.camel.component.sql.RowMapperFactory", "deprecated": false, "autowired": true, "secret": false, "description": "Factory for creating RowMapper" }, + "serviceLocationEnabled": { "index": 7, "kind": "property", "displayName": "Service Location Enabled", "group": "advanced", "label": "advanced", "required": false, "type": "boolean", "javaType": "boolean", "deprecated": false, "autowired": false, "secret": false, "defaultValue": true, "description": "Whether to detect the network address location of the JMS broker on startup. This information is gathered via reflection on the ConnectionFactory, and is vendor specific. This option can [...] + "usePlaceholder": { "index": 8, "kind": "property", "displayName": "Use Placeholder", "group": "advanced", "label": "advanced", "required": false, "type": "boolean", "javaType": "boolean", "deprecated": false, "autowired": false, "secret": false, "defaultValue": true, "description": "Sets whether to use placeholder and replace all placeholder characters with sign in the SQL queries. This option is default true" }, + "healthCheckConsumerEnabled": { "index": 9, "kind": "property", "displayName": "Health Check Consumer Enabled", "group": "health", "label": "health", "required": false, "type": "boolean", "javaType": "boolean", "deprecated": false, "autowired": false, "secret": false, "defaultValue": true, "description": "Used for enabling or disabling all consumer based health checks from this component" }, + "healthCheckProducerEnabled": { "index": 10, "kind": "property", "displayName": "Health Check Producer Enabled", "group": "health", "label": "health", "required": false, "type": "boolean", "javaType": "boolean", "deprecated": false, "autowired": false, "secret": false, "defaultValue": true, "description": "Used for enabling or disabling all producer based health checks from this component. Notice: Camel has by default disabled all producer based health-checks. You can turn on produce [...] }, "headers": { "CamelSqlQuery": { "index": 0, "kind": "header", "displayName": "", "group": "producer", "label": "producer", "required": false, "javaType": "String", "deprecated": false, "deprecationNote": "", "autowired": false, "secret": false, "description": "Query to execute. This query takes precedence over the query specified in the endpoint URI. Note that query parameters in the header _are_ represented by a instead of a pass:# symbol", "constantName": "org.apache.camel.component.sql.SqlCons [...] @@ -68,29 +70,31 @@ "pollStrategy": { "index": 20, "kind": "parameter", "displayName": "Poll Strategy", "group": "consumer (advanced)", "label": "consumer,advanced", "required": false, "type": "object", "javaType": "org.apache.camel.spi.PollingConsumerPollStrategy", "deprecated": false, "autowired": false, "secret": false, "description": "A pluggable org.apache.camel.PollingConsumerPollingStrategy allowing you to provide your custom implementation to control error handling usually occurred during the po [...] "processingStrategy": { "index": 21, "kind": "parameter", "displayName": "Processing Strategy", "group": "consumer (advanced)", "label": "consumer,advanced", "required": false, "type": "object", "javaType": "org.apache.camel.component.sql.SqlProcessingStrategy", "deprecated": false, "autowired": false, "secret": false, "description": "Allows to plugin to use a custom org.apache.camel.component.sql.SqlProcessingStrategy to execute queries when the consumer has processed the rows\/batch." }, "batch": { "index": 22, "kind": "parameter", "displayName": "Batch", "group": "producer", "label": "producer", "required": false, "type": "boolean", "javaType": "boolean", "deprecated": false, "autowired": false, "secret": false, "defaultValue": false, "description": "Enables or disables batch mode" }, - "noop": { "index": 23, "kind": "parameter", "displayName": "Noop", "group": "producer", "label": "producer", "required": false, "type": "boolean", "javaType": "boolean", "deprecated": false, "autowired": false, "secret": false, "defaultValue": false, "description": "If set, will ignore the results of the SQL query and use the existing IN message as the OUT message for the continuation of processing" }, - "useMessageBodyForSql": { "index": 24, "kind": "parameter", "displayName": "Use Message Body For Sql", "group": "producer", "label": "producer", "required": false, "type": "boolean", "javaType": "boolean", "deprecated": false, "autowired": false, "secret": false, "defaultValue": false, "description": "Whether to use the message body as the SQL and then headers for parameters. If this option is enabled then the SQL in the uri is not used. Note that query parameters in the message body [...] - "lazyStartProducer": { "index": 25, "kind": "parameter", "displayName": "Lazy Start Producer", "group": "producer (advanced)", "label": "producer,advanced", "required": false, "type": "boolean", "javaType": "boolean", "deprecated": false, "autowired": false, "secret": false, "defaultValue": false, "description": "Whether the producer should be started lazy (on the first message). By starting lazy you can use this to allow CamelContext and routes to startup in situations where a produ [...] - "alwaysPopulateStatement": { "index": 26, "kind": "parameter", "displayName": "Always Populate Statement", "group": "advanced", "label": "advanced", "required": false, "type": "boolean", "javaType": "boolean", "deprecated": false, "autowired": false, "secret": false, "defaultValue": false, "description": "If enabled then the populateStatement method from org.apache.camel.component.sql.SqlPrepareStatementStrategy is always invoked, also if there is no expected parameters to be prepare [...] - "parametersCount": { "index": 27, "kind": "parameter", "displayName": "Parameters Count", "group": "advanced", "label": "advanced", "required": false, "type": "integer", "javaType": "int", "deprecated": false, "autowired": false, "secret": false, "description": "If set greater than zero, then Camel will use this count value of parameters to replace instead of querying via JDBC metadata API. This is useful if the JDBC vendor could not return correct parameters count, then user may ove [...] - "placeholder": { "index": 28, "kind": "parameter", "displayName": "Placeholder", "group": "advanced", "label": "advanced", "required": false, "type": "string", "javaType": "java.lang.String", "deprecated": false, "autowired": false, "secret": false, "defaultValue": "#", "description": "Specifies a character that will be replaced to in SQL query. Notice, that it is simple String.replaceAll() operation and no SQL parsing is involved (quoted strings will also change)." }, - "prepareStatementStrategy": { "index": 29, "kind": "parameter", "displayName": "Prepare Statement Strategy", "group": "advanced", "label": "advanced", "required": false, "type": "object", "javaType": "org.apache.camel.component.sql.SqlPrepareStatementStrategy", "deprecated": false, "autowired": false, "secret": false, "description": "Allows to plugin to use a custom org.apache.camel.component.sql.SqlPrepareStatementStrategy to control preparation of the query and prepared statement." }, - "rowMapperFactory": { "index": 30, "kind": "parameter", "displayName": "Row Mapper Factory", "group": "advanced", "label": "advanced", "required": false, "type": "object", "javaType": "org.apache.camel.component.sql.RowMapperFactory", "deprecated": false, "autowired": false, "secret": false, "description": "Factory for creating RowMapper" }, - "templateOptions": { "index": 31, "kind": "parameter", "displayName": "Template Options", "group": "advanced", "label": "advanced", "required": false, "type": "object", "javaType": "java.util.Map<java.lang.String, java.lang.Object>", "prefix": "template.", "multiValue": true, "deprecated": false, "autowired": false, "secret": false, "description": "Configures the Spring JdbcTemplate with the key\/values from the Map" }, - "usePlaceholder": { "index": 32, "kind": "parameter", "displayName": "Use Placeholder", "group": "advanced", "label": "advanced", "required": false, "type": "boolean", "javaType": "boolean", "deprecated": false, "autowired": false, "secret": false, "defaultValue": true, "description": "Sets whether to use placeholder and replace all placeholder characters with sign in the SQL queries." }, - "backoffErrorThreshold": { "index": 33, "kind": "parameter", "displayName": "Backoff Error Threshold", "group": "scheduler", "label": "consumer,scheduler", "required": false, "type": "integer", "javaType": "int", "deprecated": false, "autowired": false, "secret": false, "description": "The number of subsequent error polls (failed due some error) that should happen before the backoffMultipler should kick-in." }, - "backoffIdleThreshold": { "index": 34, "kind": "parameter", "displayName": "Backoff Idle Threshold", "group": "scheduler", "label": "consumer,scheduler", "required": false, "type": "integer", "javaType": "int", "deprecated": false, "autowired": false, "secret": false, "description": "The number of subsequent idle polls that should happen before the backoffMultipler should kick-in." }, - "backoffMultiplier": { "index": 35, "kind": "parameter", "displayName": "Backoff Multiplier", "group": "scheduler", "label": "consumer,scheduler", "required": false, "type": "integer", "javaType": "int", "deprecated": false, "autowired": false, "secret": false, "description": "To let the scheduled polling consumer backoff if there has been a number of subsequent idles\/errors in a row. The multiplier is then the number of polls that will be skipped before the next actual attempt is h [...] - "delay": { "index": 36, "kind": "parameter", "displayName": "Delay", "group": "scheduler", "label": "consumer,scheduler", "required": false, "type": "integer", "javaType": "long", "deprecated": false, "autowired": false, "secret": false, "defaultValue": 500, "description": "Milliseconds before the next poll." }, - "greedy": { "index": 37, "kind": "parameter", "displayName": "Greedy", "group": "scheduler", "label": "consumer,scheduler", "required": false, "type": "boolean", "javaType": "boolean", "deprecated": false, "autowired": false, "secret": false, "defaultValue": false, "description": "If greedy is enabled, then the ScheduledPollConsumer will run immediately again, if the previous run polled 1 or more messages." }, - "initialDelay": { "index": 38, "kind": "parameter", "displayName": "Initial Delay", "group": "scheduler", "label": "consumer,scheduler", "required": false, "type": "integer", "javaType": "long", "deprecated": false, "autowired": false, "secret": false, "defaultValue": 1000, "description": "Milliseconds before the first poll starts." }, - "repeatCount": { "index": 39, "kind": "parameter", "displayName": "Repeat Count", "group": "scheduler", "label": "consumer,scheduler", "required": false, "type": "integer", "javaType": "long", "deprecated": false, "autowired": false, "secret": false, "defaultValue": 0, "description": "Specifies a maximum limit of number of fires. So if you set it to 1, the scheduler will only fire once. If you set it to 5, it will only fire five times. A value of zero or negative means fire forever." }, - "runLoggingLevel": { "index": 40, "kind": "parameter", "displayName": "Run Logging Level", "group": "scheduler", "label": "consumer,scheduler", "required": false, "type": "object", "javaType": "org.apache.camel.LoggingLevel", "enum": [ "TRACE", "DEBUG", "INFO", "WARN", "ERROR", "OFF" ], "deprecated": false, "autowired": false, "secret": false, "defaultValue": "TRACE", "description": "The consumer logs a start\/complete log line when it polls. This option allows you to configure the l [...] - "scheduledExecutorService": { "index": 41, "kind": "parameter", "displayName": "Scheduled Executor Service", "group": "scheduler", "label": "consumer,scheduler", "required": false, "type": "object", "javaType": "java.util.concurrent.ScheduledExecutorService", "deprecated": false, "autowired": false, "secret": false, "description": "Allows for configuring a custom\/shared thread pool to use for the consumer. By default each consumer has its own single threaded thread pool." }, - "scheduler": { "index": 42, "kind": "parameter", "displayName": "Scheduler", "group": "scheduler", "label": "consumer,scheduler", "required": false, "type": "object", "javaType": "java.lang.Object", "deprecated": false, "autowired": false, "secret": false, "defaultValue": "none", "description": "To use a cron scheduler from either camel-spring or camel-quartz component. Use value spring or quartz for built in scheduler" }, - "schedulerProperties": { "index": 43, "kind": "parameter", "displayName": "Scheduler Properties", "group": "scheduler", "label": "consumer,scheduler", "required": false, "type": "object", "javaType": "java.util.Map<java.lang.String, java.lang.Object>", "prefix": "scheduler.", "multiValue": true, "deprecated": false, "autowired": false, "secret": false, "description": "To configure additional properties when using a custom scheduler or any of the Quartz, Spring based scheduler." }, - "startScheduler": { "index": 44, "kind": "parameter", "displayName": "Start Scheduler", "group": "scheduler", "label": "consumer,scheduler", "required": false, "type": "boolean", "javaType": "boolean", "deprecated": false, "autowired": false, "secret": false, "defaultValue": true, "description": "Whether the scheduler should be auto started." }, - "timeUnit": { "index": 45, "kind": "parameter", "displayName": "Time Unit", "group": "scheduler", "label": "consumer,scheduler", "required": false, "type": "object", "javaType": "java.util.concurrent.TimeUnit", "enum": [ "NANOSECONDS", "MICROSECONDS", "MILLISECONDS", "SECONDS", "MINUTES", "HOURS", "DAYS" ], "deprecated": false, "autowired": false, "secret": false, "defaultValue": "MILLISECONDS", "description": "Time unit for initialDelay and delay options." }, - "useFixedDelay": { "index": 46, "kind": "parameter", "displayName": "Use Fixed Delay", "group": "scheduler", "label": "consumer,scheduler", "required": false, "type": "boolean", "javaType": "boolean", "deprecated": false, "autowired": false, "secret": false, "defaultValue": true, "description": "Controls if fixed delay or fixed rate is used. See ScheduledExecutorService in JDK for details." } + "batchAutoCommitDisabled": { "index": 23, "kind": "parameter", "displayName": "Batch Auto Commit Disabled", "group": "producer", "label": "producer", "required": false, "type": "boolean", "javaType": "boolean", "deprecated": false, "autowired": false, "secret": false, "defaultValue": true, "description": "Whether to optimize batch by turning off auto-commit which can dramatic improve performance, and instead execute as a manual commit after the entire batch operation is complete" }, + "noop": { "index": 24, "kind": "parameter", "displayName": "Noop", "group": "producer", "label": "producer", "required": false, "type": "boolean", "javaType": "boolean", "deprecated": false, "autowired": false, "secret": false, "defaultValue": false, "description": "If set, will ignore the results of the SQL query and use the existing IN message as the OUT message for the continuation of processing" }, + "useMessageBodyForSql": { "index": 25, "kind": "parameter", "displayName": "Use Message Body For Sql", "group": "producer", "label": "producer", "required": false, "type": "boolean", "javaType": "boolean", "deprecated": false, "autowired": false, "secret": false, "defaultValue": false, "description": "Whether to use the message body as the SQL and then headers for parameters. If this option is enabled then the SQL in the uri is not used. Note that query parameters in the message body [...] + "lazyStartProducer": { "index": 26, "kind": "parameter", "displayName": "Lazy Start Producer", "group": "producer (advanced)", "label": "producer,advanced", "required": false, "type": "boolean", "javaType": "boolean", "deprecated": false, "autowired": false, "secret": false, "defaultValue": false, "description": "Whether the producer should be started lazy (on the first message). By starting lazy you can use this to allow CamelContext and routes to startup in situations where a produ [...] + "alwaysPopulateStatement": { "index": 27, "kind": "parameter", "displayName": "Always Populate Statement", "group": "advanced", "label": "advanced", "required": false, "type": "boolean", "javaType": "boolean", "deprecated": false, "autowired": false, "secret": false, "defaultValue": false, "description": "If enabled then the populateStatement method from org.apache.camel.component.sql.SqlPrepareStatementStrategy is always invoked, also if there is no expected parameters to be prepare [...] + "fetchSize": { "index": 28, "kind": "parameter", "displayName": "Fetch Size", "group": "advanced", "label": "advanced", "required": false, "type": "integer", "javaType": "int", "deprecated": false, "autowired": false, "secret": false, "description": "Gives the JDBC driver a hint as to the number of rows that should be fetched from the database when more rows are needed for ResultSet objects generated by this Statement. If the value specified is zero, then the hint is ignored. The def [...] + "parametersCount": { "index": 29, "kind": "parameter", "displayName": "Parameters Count", "group": "advanced", "label": "advanced", "required": false, "type": "integer", "javaType": "int", "deprecated": false, "autowired": false, "secret": false, "description": "If set greater than zero, then Camel will use this count value of parameters to replace instead of querying via JDBC metadata API. This is useful if the JDBC vendor could not return correct parameters count, then user may ove [...] + "placeholder": { "index": 30, "kind": "parameter", "displayName": "Placeholder", "group": "advanced", "label": "advanced", "required": false, "type": "string", "javaType": "java.lang.String", "deprecated": false, "autowired": false, "secret": false, "defaultValue": "#", "description": "Specifies a character that will be replaced to in SQL query. Notice, that it is simple String.replaceAll() operation and no SQL parsing is involved (quoted strings will also change)." }, + "prepareStatementStrategy": { "index": 31, "kind": "parameter", "displayName": "Prepare Statement Strategy", "group": "advanced", "label": "advanced", "required": false, "type": "object", "javaType": "org.apache.camel.component.sql.SqlPrepareStatementStrategy", "deprecated": false, "autowired": false, "secret": false, "description": "Allows to plugin to use a custom org.apache.camel.component.sql.SqlPrepareStatementStrategy to control preparation of the query and prepared statement." }, + "rowMapperFactory": { "index": 32, "kind": "parameter", "displayName": "Row Mapper Factory", "group": "advanced", "label": "advanced", "required": false, "type": "object", "javaType": "org.apache.camel.component.sql.RowMapperFactory", "deprecated": false, "autowired": false, "secret": false, "description": "Factory for creating RowMapper" }, + "templateOptions": { "index": 33, "kind": "parameter", "displayName": "Template Options", "group": "advanced", "label": "advanced", "required": false, "type": "object", "javaType": "java.util.Map<java.lang.String, java.lang.Object>", "prefix": "template.", "multiValue": true, "deprecated": false, "autowired": false, "secret": false, "description": "Configures the Spring JdbcTemplate with the key\/values from the Map" }, + "usePlaceholder": { "index": 34, "kind": "parameter", "displayName": "Use Placeholder", "group": "advanced", "label": "advanced", "required": false, "type": "boolean", "javaType": "boolean", "deprecated": false, "autowired": false, "secret": false, "defaultValue": true, "description": "Sets whether to use placeholder and replace all placeholder characters with sign in the SQL queries." }, + "backoffErrorThreshold": { "index": 35, "kind": "parameter", "displayName": "Backoff Error Threshold", "group": "scheduler", "label": "consumer,scheduler", "required": false, "type": "integer", "javaType": "int", "deprecated": false, "autowired": false, "secret": false, "description": "The number of subsequent error polls (failed due some error) that should happen before the backoffMultipler should kick-in." }, + "backoffIdleThreshold": { "index": 36, "kind": "parameter", "displayName": "Backoff Idle Threshold", "group": "scheduler", "label": "consumer,scheduler", "required": false, "type": "integer", "javaType": "int", "deprecated": false, "autowired": false, "secret": false, "description": "The number of subsequent idle polls that should happen before the backoffMultipler should kick-in." }, + "backoffMultiplier": { "index": 37, "kind": "parameter", "displayName": "Backoff Multiplier", "group": "scheduler", "label": "consumer,scheduler", "required": false, "type": "integer", "javaType": "int", "deprecated": false, "autowired": false, "secret": false, "description": "To let the scheduled polling consumer backoff if there has been a number of subsequent idles\/errors in a row. The multiplier is then the number of polls that will be skipped before the next actual attempt is h [...] + "delay": { "index": 38, "kind": "parameter", "displayName": "Delay", "group": "scheduler", "label": "consumer,scheduler", "required": false, "type": "integer", "javaType": "long", "deprecated": false, "autowired": false, "secret": false, "defaultValue": 500, "description": "Milliseconds before the next poll." }, + "greedy": { "index": 39, "kind": "parameter", "displayName": "Greedy", "group": "scheduler", "label": "consumer,scheduler", "required": false, "type": "boolean", "javaType": "boolean", "deprecated": false, "autowired": false, "secret": false, "defaultValue": false, "description": "If greedy is enabled, then the ScheduledPollConsumer will run immediately again, if the previous run polled 1 or more messages." }, + "initialDelay": { "index": 40, "kind": "parameter", "displayName": "Initial Delay", "group": "scheduler", "label": "consumer,scheduler", "required": false, "type": "integer", "javaType": "long", "deprecated": false, "autowired": false, "secret": false, "defaultValue": 1000, "description": "Milliseconds before the first poll starts." }, + "repeatCount": { "index": 41, "kind": "parameter", "displayName": "Repeat Count", "group": "scheduler", "label": "consumer,scheduler", "required": false, "type": "integer", "javaType": "long", "deprecated": false, "autowired": false, "secret": false, "defaultValue": 0, "description": "Specifies a maximum limit of number of fires. So if you set it to 1, the scheduler will only fire once. If you set it to 5, it will only fire five times. A value of zero or negative means fire forever." }, + "runLoggingLevel": { "index": 42, "kind": "parameter", "displayName": "Run Logging Level", "group": "scheduler", "label": "consumer,scheduler", "required": false, "type": "object", "javaType": "org.apache.camel.LoggingLevel", "enum": [ "TRACE", "DEBUG", "INFO", "WARN", "ERROR", "OFF" ], "deprecated": false, "autowired": false, "secret": false, "defaultValue": "TRACE", "description": "The consumer logs a start\/complete log line when it polls. This option allows you to configure the l [...] + "scheduledExecutorService": { "index": 43, "kind": "parameter", "displayName": "Scheduled Executor Service", "group": "scheduler", "label": "consumer,scheduler", "required": false, "type": "object", "javaType": "java.util.concurrent.ScheduledExecutorService", "deprecated": false, "autowired": false, "secret": false, "description": "Allows for configuring a custom\/shared thread pool to use for the consumer. By default each consumer has its own single threaded thread pool." }, + "scheduler": { "index": 44, "kind": "parameter", "displayName": "Scheduler", "group": "scheduler", "label": "consumer,scheduler", "required": false, "type": "object", "javaType": "java.lang.Object", "deprecated": false, "autowired": false, "secret": false, "defaultValue": "none", "description": "To use a cron scheduler from either camel-spring or camel-quartz component. Use value spring or quartz for built in scheduler" }, + "schedulerProperties": { "index": 45, "kind": "parameter", "displayName": "Scheduler Properties", "group": "scheduler", "label": "consumer,scheduler", "required": false, "type": "object", "javaType": "java.util.Map<java.lang.String, java.lang.Object>", "prefix": "scheduler.", "multiValue": true, "deprecated": false, "autowired": false, "secret": false, "description": "To configure additional properties when using a custom scheduler or any of the Quartz, Spring based scheduler." }, + "startScheduler": { "index": 46, "kind": "parameter", "displayName": "Start Scheduler", "group": "scheduler", "label": "consumer,scheduler", "required": false, "type": "boolean", "javaType": "boolean", "deprecated": false, "autowired": false, "secret": false, "defaultValue": true, "description": "Whether the scheduler should be auto started." }, + "timeUnit": { "index": 47, "kind": "parameter", "displayName": "Time Unit", "group": "scheduler", "label": "consumer,scheduler", "required": false, "type": "object", "javaType": "java.util.concurrent.TimeUnit", "enum": [ "NANOSECONDS", "MICROSECONDS", "MILLISECONDS", "SECONDS", "MINUTES", "HOURS", "DAYS" ], "deprecated": false, "autowired": false, "secret": false, "defaultValue": "MILLISECONDS", "description": "Time unit for initialDelay and delay options." }, + "useFixedDelay": { "index": 48, "kind": "parameter", "displayName": "Use Fixed Delay", "group": "scheduler", "label": "consumer,scheduler", "required": false, "type": "boolean", "javaType": "boolean", "deprecated": false, "autowired": false, "secret": false, "defaultValue": true, "description": "Controls if fixed delay or fixed rate is used. See ScheduledExecutorService in JDK for details." } } } diff --git a/components/camel-sql/src/generated/java/org/apache/camel/component/sql/SqlComponentConfigurer.java b/components/camel-sql/src/generated/java/org/apache/camel/component/sql/SqlComponentConfigurer.java index 1f9d62383c0..f04aba0caa6 100644 --- a/components/camel-sql/src/generated/java/org/apache/camel/component/sql/SqlComponentConfigurer.java +++ b/components/camel-sql/src/generated/java/org/apache/camel/component/sql/SqlComponentConfigurer.java @@ -25,10 +25,14 @@ public class SqlComponentConfigurer extends PropertyConfigurerSupport implements switch (ignoreCase ? name.toLowerCase() : name) { case "autowiredenabled": case "autowiredEnabled": target.setAutowiredEnabled(property(camelContext, boolean.class, value)); return true; + case "batchautocommitdisabled": + case "batchAutoCommitDisabled": target.setBatchAutoCommitDisabled(property(camelContext, boolean.class, value)); return true; case "bridgeerrorhandler": case "bridgeErrorHandler": target.setBridgeErrorHandler(property(camelContext, boolean.class, value)); return true; case "datasource": case "dataSource": target.setDataSource(property(camelContext, javax.sql.DataSource.class, value)); return true; + case "fetchsize": + case "fetchSize": target.setFetchSize(property(camelContext, int.class, value)); return true; case "healthcheckconsumerenabled": case "healthCheckConsumerEnabled": target.setHealthCheckConsumerEnabled(property(camelContext, boolean.class, value)); return true; case "healthcheckproducerenabled": @@ -55,10 +59,14 @@ public class SqlComponentConfigurer extends PropertyConfigurerSupport implements switch (ignoreCase ? name.toLowerCase() : name) { case "autowiredenabled": case "autowiredEnabled": return boolean.class; + case "batchautocommitdisabled": + case "batchAutoCommitDisabled": return boolean.class; case "bridgeerrorhandler": case "bridgeErrorHandler": return boolean.class; case "datasource": case "dataSource": return javax.sql.DataSource.class; + case "fetchsize": + case "fetchSize": return int.class; case "healthcheckconsumerenabled": case "healthCheckConsumerEnabled": return boolean.class; case "healthcheckproducerenabled": @@ -81,10 +89,14 @@ public class SqlComponentConfigurer extends PropertyConfigurerSupport implements switch (ignoreCase ? name.toLowerCase() : name) { case "autowiredenabled": case "autowiredEnabled": return target.isAutowiredEnabled(); + case "batchautocommitdisabled": + case "batchAutoCommitDisabled": return target.isBatchAutoCommitDisabled(); case "bridgeerrorhandler": case "bridgeErrorHandler": return target.isBridgeErrorHandler(); case "datasource": case "dataSource": return target.getDataSource(); + case "fetchsize": + case "fetchSize": return target.getFetchSize(); case "healthcheckconsumerenabled": case "healthCheckConsumerEnabled": return target.isHealthCheckConsumerEnabled(); case "healthcheckproducerenabled": diff --git a/components/camel-sql/src/generated/java/org/apache/camel/component/sql/SqlEndpointConfigurer.java b/components/camel-sql/src/generated/java/org/apache/camel/component/sql/SqlEndpointConfigurer.java index f4d10612e14..6805419d4f9 100644 --- a/components/camel-sql/src/generated/java/org/apache/camel/component/sql/SqlEndpointConfigurer.java +++ b/components/camel-sql/src/generated/java/org/apache/camel/component/sql/SqlEndpointConfigurer.java @@ -34,6 +34,8 @@ public class SqlEndpointConfigurer extends PropertyConfigurerSupport implements case "backoffmultiplier": case "backoffMultiplier": target.setBackoffMultiplier(property(camelContext, int.class, value)); return true; case "batch": target.setBatch(property(camelContext, boolean.class, value)); return true; + case "batchautocommitdisabled": + case "batchAutoCommitDisabled": target.setBatchAutoCommitDisabled(property(camelContext, boolean.class, value)); return true; case "breakbatchonconsumefail": case "breakBatchOnConsumeFail": target.setBreakBatchOnConsumeFail(property(camelContext, boolean.class, value)); return true; case "bridgeerrorhandler": @@ -47,6 +49,8 @@ public class SqlEndpointConfigurer extends PropertyConfigurerSupport implements case "exchangePattern": target.setExchangePattern(property(camelContext, org.apache.camel.ExchangePattern.class, value)); return true; case "expectedupdatecount": case "expectedUpdateCount": target.setExpectedUpdateCount(property(camelContext, int.class, value)); return true; + case "fetchsize": + case "fetchSize": target.setFetchSize(property(camelContext, int.class, value)); return true; case "greedy": target.setGreedy(property(camelContext, boolean.class, value)); return true; case "initialdelay": case "initialDelay": target.setInitialDelay(property(camelContext, long.class, value)); return true; @@ -130,6 +134,8 @@ public class SqlEndpointConfigurer extends PropertyConfigurerSupport implements case "backoffmultiplier": case "backoffMultiplier": return int.class; case "batch": return boolean.class; + case "batchautocommitdisabled": + case "batchAutoCommitDisabled": return boolean.class; case "breakbatchonconsumefail": case "breakBatchOnConsumeFail": return boolean.class; case "bridgeerrorhandler": @@ -143,6 +149,8 @@ public class SqlEndpointConfigurer extends PropertyConfigurerSupport implements case "exchangePattern": return org.apache.camel.ExchangePattern.class; case "expectedupdatecount": case "expectedUpdateCount": return int.class; + case "fetchsize": + case "fetchSize": return int.class; case "greedy": return boolean.class; case "initialdelay": case "initialDelay": return long.class; @@ -222,6 +230,8 @@ public class SqlEndpointConfigurer extends PropertyConfigurerSupport implements case "backoffmultiplier": case "backoffMultiplier": return target.getBackoffMultiplier(); case "batch": return target.isBatch(); + case "batchautocommitdisabled": + case "batchAutoCommitDisabled": return target.isBatchAutoCommitDisabled(); case "breakbatchonconsumefail": case "breakBatchOnConsumeFail": return target.isBreakBatchOnConsumeFail(); case "bridgeerrorhandler": @@ -235,6 +245,8 @@ public class SqlEndpointConfigurer extends PropertyConfigurerSupport implements case "exchangePattern": return target.getExchangePattern(); case "expectedupdatecount": case "expectedUpdateCount": return target.getExpectedUpdateCount(); + case "fetchsize": + case "fetchSize": return target.getFetchSize(); case "greedy": return target.isGreedy(); case "initialdelay": case "initialDelay": return target.getInitialDelay(); diff --git a/components/camel-sql/src/generated/java/org/apache/camel/component/sql/SqlEndpointUriFactory.java b/components/camel-sql/src/generated/java/org/apache/camel/component/sql/SqlEndpointUriFactory.java index b2f450002f1..34e417f5aa8 100644 --- a/components/camel-sql/src/generated/java/org/apache/camel/component/sql/SqlEndpointUriFactory.java +++ b/components/camel-sql/src/generated/java/org/apache/camel/component/sql/SqlEndpointUriFactory.java @@ -23,13 +23,14 @@ public class SqlEndpointUriFactory extends org.apache.camel.support.component.En private static final Set<String> SECRET_PROPERTY_NAMES; private static final Set<String> MULTI_VALUE_PREFIXES; static { - Set<String> props = new HashSet<>(47); + Set<String> props = new HashSet<>(49); props.add("allowNamedParameters"); props.add("alwaysPopulateStatement"); props.add("backoffErrorThreshold"); props.add("backoffIdleThreshold"); props.add("backoffMultiplier"); props.add("batch"); + props.add("batchAutoCommitDisabled"); props.add("breakBatchOnConsumeFail"); props.add("bridgeErrorHandler"); props.add("dataSource"); @@ -37,6 +38,7 @@ public class SqlEndpointUriFactory extends org.apache.camel.support.component.En props.add("exceptionHandler"); props.add("exchangePattern"); props.add("expectedUpdateCount"); + props.add("fetchSize"); props.add("greedy"); props.add("initialDelay"); props.add("lazyStartProducer"); diff --git a/components/camel-sql/src/generated/resources/META-INF/org/apache/camel/component/sql/sql.json b/components/camel-sql/src/generated/resources/META-INF/org/apache/camel/component/sql/sql.json index 7594b0197ef..1e4bcc94236 100644 --- a/components/camel-sql/src/generated/resources/META-INF/org/apache/camel/component/sql/sql.json +++ b/components/camel-sql/src/generated/resources/META-INF/org/apache/camel/component/sql/sql.json @@ -26,13 +26,15 @@ "componentProperties": { "dataSource": { "index": 0, "kind": "property", "displayName": "Data Source", "group": "common", "label": "", "required": false, "type": "object", "javaType": "javax.sql.DataSource", "deprecated": false, "autowired": true, "secret": false, "description": "Sets the DataSource to use to communicate with the database." }, "bridgeErrorHandler": { "index": 1, "kind": "property", "displayName": "Bridge Error Handler", "group": "consumer", "label": "consumer", "required": false, "type": "boolean", "javaType": "boolean", "deprecated": false, "autowired": false, "secret": false, "defaultValue": false, "description": "Allows for bridging the consumer to the Camel routing Error Handler, which mean any exceptions (if possible) occurred while the Camel consumer is trying to pickup incoming messages, or the like [...] - "lazyStartProducer": { "index": 2, "kind": "property", "displayName": "Lazy Start Producer", "group": "producer", "label": "producer", "required": false, "type": "boolean", "javaType": "boolean", "deprecated": false, "autowired": false, "secret": false, "defaultValue": false, "description": "Whether the producer should be started lazy (on the first message). By starting lazy you can use this to allow CamelContext and routes to startup in situations where a producer may otherwise fail [...] - "autowiredEnabled": { "index": 3, "kind": "property", "displayName": "Autowired Enabled", "group": "advanced", "label": "advanced", "required": false, "type": "boolean", "javaType": "boolean", "deprecated": false, "autowired": false, "secret": false, "defaultValue": true, "description": "Whether autowiring is enabled. This is used for automatic autowiring options (the option must be marked as autowired) by looking up in the registry to find if there is a single instance of matching t [...] - "rowMapperFactory": { "index": 4, "kind": "property", "displayName": "Row Mapper Factory", "group": "advanced", "label": "advanced", "required": false, "type": "object", "javaType": "org.apache.camel.component.sql.RowMapperFactory", "deprecated": false, "autowired": true, "secret": false, "description": "Factory for creating RowMapper" }, - "serviceLocationEnabled": { "index": 5, "kind": "property", "displayName": "Service Location Enabled", "group": "advanced", "label": "advanced", "required": false, "type": "boolean", "javaType": "boolean", "deprecated": false, "autowired": false, "secret": false, "defaultValue": true, "description": "Whether to detect the network address location of the JMS broker on startup. This information is gathered via reflection on the ConnectionFactory, and is vendor specific. This option can [...] - "usePlaceholder": { "index": 6, "kind": "property", "displayName": "Use Placeholder", "group": "advanced", "label": "advanced", "required": false, "type": "boolean", "javaType": "boolean", "deprecated": false, "autowired": false, "secret": false, "defaultValue": true, "description": "Sets whether to use placeholder and replace all placeholder characters with sign in the SQL queries. This option is default true" }, - "healthCheckConsumerEnabled": { "index": 7, "kind": "property", "displayName": "Health Check Consumer Enabled", "group": "health", "label": "health", "required": false, "type": "boolean", "javaType": "boolean", "deprecated": false, "autowired": false, "secret": false, "defaultValue": true, "description": "Used for enabling or disabling all consumer based health checks from this component" }, - "healthCheckProducerEnabled": { "index": 8, "kind": "property", "displayName": "Health Check Producer Enabled", "group": "health", "label": "health", "required": false, "type": "boolean", "javaType": "boolean", "deprecated": false, "autowired": false, "secret": false, "defaultValue": true, "description": "Used for enabling or disabling all producer based health checks from this component. Notice: Camel has by default disabled all producer based health-checks. You can turn on producer [...] + "batchAutoCommitDisabled": { "index": 2, "kind": "property", "displayName": "Batch Auto Commit Disabled", "group": "producer", "label": "producer", "required": false, "type": "boolean", "javaType": "boolean", "deprecated": false, "autowired": false, "secret": false, "defaultValue": true, "description": "Whether to optimize batch by turning off auto-commit which can dramatic improve performance, and instead execute as a manual commit after the entire batch operation is complete" }, + "lazyStartProducer": { "index": 3, "kind": "property", "displayName": "Lazy Start Producer", "group": "producer", "label": "producer", "required": false, "type": "boolean", "javaType": "boolean", "deprecated": false, "autowired": false, "secret": false, "defaultValue": false, "description": "Whether the producer should be started lazy (on the first message). By starting lazy you can use this to allow CamelContext and routes to startup in situations where a producer may otherwise fail [...] + "autowiredEnabled": { "index": 4, "kind": "property", "displayName": "Autowired Enabled", "group": "advanced", "label": "advanced", "required": false, "type": "boolean", "javaType": "boolean", "deprecated": false, "autowired": false, "secret": false, "defaultValue": true, "description": "Whether autowiring is enabled. This is used for automatic autowiring options (the option must be marked as autowired) by looking up in the registry to find if there is a single instance of matching t [...] + "fetchSize": { "index": 5, "kind": "property", "displayName": "Fetch Size", "group": "advanced", "label": "advanced", "required": false, "type": "integer", "javaType": "int", "deprecated": false, "autowired": false, "secret": false, "description": "Gives the JDBC driver a hint as to the number of rows that should be fetched from the database when more rows are needed for ResultSet objects generated by this Statement. If the value specified is zero, then the hint is ignored. The defau [...] + "rowMapperFactory": { "index": 6, "kind": "property", "displayName": "Row Mapper Factory", "group": "advanced", "label": "advanced", "required": false, "type": "object", "javaType": "org.apache.camel.component.sql.RowMapperFactory", "deprecated": false, "autowired": true, "secret": false, "description": "Factory for creating RowMapper" }, + "serviceLocationEnabled": { "index": 7, "kind": "property", "displayName": "Service Location Enabled", "group": "advanced", "label": "advanced", "required": false, "type": "boolean", "javaType": "boolean", "deprecated": false, "autowired": false, "secret": false, "defaultValue": true, "description": "Whether to detect the network address location of the JMS broker on startup. This information is gathered via reflection on the ConnectionFactory, and is vendor specific. This option can [...] + "usePlaceholder": { "index": 8, "kind": "property", "displayName": "Use Placeholder", "group": "advanced", "label": "advanced", "required": false, "type": "boolean", "javaType": "boolean", "deprecated": false, "autowired": false, "secret": false, "defaultValue": true, "description": "Sets whether to use placeholder and replace all placeholder characters with sign in the SQL queries. This option is default true" }, + "healthCheckConsumerEnabled": { "index": 9, "kind": "property", "displayName": "Health Check Consumer Enabled", "group": "health", "label": "health", "required": false, "type": "boolean", "javaType": "boolean", "deprecated": false, "autowired": false, "secret": false, "defaultValue": true, "description": "Used for enabling or disabling all consumer based health checks from this component" }, + "healthCheckProducerEnabled": { "index": 10, "kind": "property", "displayName": "Health Check Producer Enabled", "group": "health", "label": "health", "required": false, "type": "boolean", "javaType": "boolean", "deprecated": false, "autowired": false, "secret": false, "defaultValue": true, "description": "Used for enabling or disabling all producer based health checks from this component. Notice: Camel has by default disabled all producer based health-checks. You can turn on produce [...] }, "headers": { "CamelSqlQuery": { "index": 0, "kind": "header", "displayName": "", "group": "producer", "label": "producer", "required": false, "javaType": "String", "deprecated": false, "deprecationNote": "", "autowired": false, "secret": false, "description": "Query to execute. This query takes precedence over the query specified in the endpoint URI. Note that query parameters in the header _are_ represented by a instead of a pass:# symbol", "constantName": "org.apache.camel.component.sql.SqlCons [...] @@ -68,29 +70,31 @@ "pollStrategy": { "index": 20, "kind": "parameter", "displayName": "Poll Strategy", "group": "consumer (advanced)", "label": "consumer,advanced", "required": false, "type": "object", "javaType": "org.apache.camel.spi.PollingConsumerPollStrategy", "deprecated": false, "autowired": false, "secret": false, "description": "A pluggable org.apache.camel.PollingConsumerPollingStrategy allowing you to provide your custom implementation to control error handling usually occurred during the po [...] "processingStrategy": { "index": 21, "kind": "parameter", "displayName": "Processing Strategy", "group": "consumer (advanced)", "label": "consumer,advanced", "required": false, "type": "object", "javaType": "org.apache.camel.component.sql.SqlProcessingStrategy", "deprecated": false, "autowired": false, "secret": false, "description": "Allows to plugin to use a custom org.apache.camel.component.sql.SqlProcessingStrategy to execute queries when the consumer has processed the rows\/batch." }, "batch": { "index": 22, "kind": "parameter", "displayName": "Batch", "group": "producer", "label": "producer", "required": false, "type": "boolean", "javaType": "boolean", "deprecated": false, "autowired": false, "secret": false, "defaultValue": false, "description": "Enables or disables batch mode" }, - "noop": { "index": 23, "kind": "parameter", "displayName": "Noop", "group": "producer", "label": "producer", "required": false, "type": "boolean", "javaType": "boolean", "deprecated": false, "autowired": false, "secret": false, "defaultValue": false, "description": "If set, will ignore the results of the SQL query and use the existing IN message as the OUT message for the continuation of processing" }, - "useMessageBodyForSql": { "index": 24, "kind": "parameter", "displayName": "Use Message Body For Sql", "group": "producer", "label": "producer", "required": false, "type": "boolean", "javaType": "boolean", "deprecated": false, "autowired": false, "secret": false, "defaultValue": false, "description": "Whether to use the message body as the SQL and then headers for parameters. If this option is enabled then the SQL in the uri is not used. Note that query parameters in the message body [...] - "lazyStartProducer": { "index": 25, "kind": "parameter", "displayName": "Lazy Start Producer", "group": "producer (advanced)", "label": "producer,advanced", "required": false, "type": "boolean", "javaType": "boolean", "deprecated": false, "autowired": false, "secret": false, "defaultValue": false, "description": "Whether the producer should be started lazy (on the first message). By starting lazy you can use this to allow CamelContext and routes to startup in situations where a produ [...] - "alwaysPopulateStatement": { "index": 26, "kind": "parameter", "displayName": "Always Populate Statement", "group": "advanced", "label": "advanced", "required": false, "type": "boolean", "javaType": "boolean", "deprecated": false, "autowired": false, "secret": false, "defaultValue": false, "description": "If enabled then the populateStatement method from org.apache.camel.component.sql.SqlPrepareStatementStrategy is always invoked, also if there is no expected parameters to be prepare [...] - "parametersCount": { "index": 27, "kind": "parameter", "displayName": "Parameters Count", "group": "advanced", "label": "advanced", "required": false, "type": "integer", "javaType": "int", "deprecated": false, "autowired": false, "secret": false, "description": "If set greater than zero, then Camel will use this count value of parameters to replace instead of querying via JDBC metadata API. This is useful if the JDBC vendor could not return correct parameters count, then user may ove [...] - "placeholder": { "index": 28, "kind": "parameter", "displayName": "Placeholder", "group": "advanced", "label": "advanced", "required": false, "type": "string", "javaType": "java.lang.String", "deprecated": false, "autowired": false, "secret": false, "defaultValue": "#", "description": "Specifies a character that will be replaced to in SQL query. Notice, that it is simple String.replaceAll() operation and no SQL parsing is involved (quoted strings will also change)." }, - "prepareStatementStrategy": { "index": 29, "kind": "parameter", "displayName": "Prepare Statement Strategy", "group": "advanced", "label": "advanced", "required": false, "type": "object", "javaType": "org.apache.camel.component.sql.SqlPrepareStatementStrategy", "deprecated": false, "autowired": false, "secret": false, "description": "Allows to plugin to use a custom org.apache.camel.component.sql.SqlPrepareStatementStrategy to control preparation of the query and prepared statement." }, - "rowMapperFactory": { "index": 30, "kind": "parameter", "displayName": "Row Mapper Factory", "group": "advanced", "label": "advanced", "required": false, "type": "object", "javaType": "org.apache.camel.component.sql.RowMapperFactory", "deprecated": false, "autowired": false, "secret": false, "description": "Factory for creating RowMapper" }, - "templateOptions": { "index": 31, "kind": "parameter", "displayName": "Template Options", "group": "advanced", "label": "advanced", "required": false, "type": "object", "javaType": "java.util.Map<java.lang.String, java.lang.Object>", "prefix": "template.", "multiValue": true, "deprecated": false, "autowired": false, "secret": false, "description": "Configures the Spring JdbcTemplate with the key\/values from the Map" }, - "usePlaceholder": { "index": 32, "kind": "parameter", "displayName": "Use Placeholder", "group": "advanced", "label": "advanced", "required": false, "type": "boolean", "javaType": "boolean", "deprecated": false, "autowired": false, "secret": false, "defaultValue": true, "description": "Sets whether to use placeholder and replace all placeholder characters with sign in the SQL queries." }, - "backoffErrorThreshold": { "index": 33, "kind": "parameter", "displayName": "Backoff Error Threshold", "group": "scheduler", "label": "consumer,scheduler", "required": false, "type": "integer", "javaType": "int", "deprecated": false, "autowired": false, "secret": false, "description": "The number of subsequent error polls (failed due some error) that should happen before the backoffMultipler should kick-in." }, - "backoffIdleThreshold": { "index": 34, "kind": "parameter", "displayName": "Backoff Idle Threshold", "group": "scheduler", "label": "consumer,scheduler", "required": false, "type": "integer", "javaType": "int", "deprecated": false, "autowired": false, "secret": false, "description": "The number of subsequent idle polls that should happen before the backoffMultipler should kick-in." }, - "backoffMultiplier": { "index": 35, "kind": "parameter", "displayName": "Backoff Multiplier", "group": "scheduler", "label": "consumer,scheduler", "required": false, "type": "integer", "javaType": "int", "deprecated": false, "autowired": false, "secret": false, "description": "To let the scheduled polling consumer backoff if there has been a number of subsequent idles\/errors in a row. The multiplier is then the number of polls that will be skipped before the next actual attempt is h [...] - "delay": { "index": 36, "kind": "parameter", "displayName": "Delay", "group": "scheduler", "label": "consumer,scheduler", "required": false, "type": "integer", "javaType": "long", "deprecated": false, "autowired": false, "secret": false, "defaultValue": 500, "description": "Milliseconds before the next poll." }, - "greedy": { "index": 37, "kind": "parameter", "displayName": "Greedy", "group": "scheduler", "label": "consumer,scheduler", "required": false, "type": "boolean", "javaType": "boolean", "deprecated": false, "autowired": false, "secret": false, "defaultValue": false, "description": "If greedy is enabled, then the ScheduledPollConsumer will run immediately again, if the previous run polled 1 or more messages." }, - "initialDelay": { "index": 38, "kind": "parameter", "displayName": "Initial Delay", "group": "scheduler", "label": "consumer,scheduler", "required": false, "type": "integer", "javaType": "long", "deprecated": false, "autowired": false, "secret": false, "defaultValue": 1000, "description": "Milliseconds before the first poll starts." }, - "repeatCount": { "index": 39, "kind": "parameter", "displayName": "Repeat Count", "group": "scheduler", "label": "consumer,scheduler", "required": false, "type": "integer", "javaType": "long", "deprecated": false, "autowired": false, "secret": false, "defaultValue": 0, "description": "Specifies a maximum limit of number of fires. So if you set it to 1, the scheduler will only fire once. If you set it to 5, it will only fire five times. A value of zero or negative means fire forever." }, - "runLoggingLevel": { "index": 40, "kind": "parameter", "displayName": "Run Logging Level", "group": "scheduler", "label": "consumer,scheduler", "required": false, "type": "object", "javaType": "org.apache.camel.LoggingLevel", "enum": [ "TRACE", "DEBUG", "INFO", "WARN", "ERROR", "OFF" ], "deprecated": false, "autowired": false, "secret": false, "defaultValue": "TRACE", "description": "The consumer logs a start\/complete log line when it polls. This option allows you to configure the l [...] - "scheduledExecutorService": { "index": 41, "kind": "parameter", "displayName": "Scheduled Executor Service", "group": "scheduler", "label": "consumer,scheduler", "required": false, "type": "object", "javaType": "java.util.concurrent.ScheduledExecutorService", "deprecated": false, "autowired": false, "secret": false, "description": "Allows for configuring a custom\/shared thread pool to use for the consumer. By default each consumer has its own single threaded thread pool." }, - "scheduler": { "index": 42, "kind": "parameter", "displayName": "Scheduler", "group": "scheduler", "label": "consumer,scheduler", "required": false, "type": "object", "javaType": "java.lang.Object", "deprecated": false, "autowired": false, "secret": false, "defaultValue": "none", "description": "To use a cron scheduler from either camel-spring or camel-quartz component. Use value spring or quartz for built in scheduler" }, - "schedulerProperties": { "index": 43, "kind": "parameter", "displayName": "Scheduler Properties", "group": "scheduler", "label": "consumer,scheduler", "required": false, "type": "object", "javaType": "java.util.Map<java.lang.String, java.lang.Object>", "prefix": "scheduler.", "multiValue": true, "deprecated": false, "autowired": false, "secret": false, "description": "To configure additional properties when using a custom scheduler or any of the Quartz, Spring based scheduler." }, - "startScheduler": { "index": 44, "kind": "parameter", "displayName": "Start Scheduler", "group": "scheduler", "label": "consumer,scheduler", "required": false, "type": "boolean", "javaType": "boolean", "deprecated": false, "autowired": false, "secret": false, "defaultValue": true, "description": "Whether the scheduler should be auto started." }, - "timeUnit": { "index": 45, "kind": "parameter", "displayName": "Time Unit", "group": "scheduler", "label": "consumer,scheduler", "required": false, "type": "object", "javaType": "java.util.concurrent.TimeUnit", "enum": [ "NANOSECONDS", "MICROSECONDS", "MILLISECONDS", "SECONDS", "MINUTES", "HOURS", "DAYS" ], "deprecated": false, "autowired": false, "secret": false, "defaultValue": "MILLISECONDS", "description": "Time unit for initialDelay and delay options." }, - "useFixedDelay": { "index": 46, "kind": "parameter", "displayName": "Use Fixed Delay", "group": "scheduler", "label": "consumer,scheduler", "required": false, "type": "boolean", "javaType": "boolean", "deprecated": false, "autowired": false, "secret": false, "defaultValue": true, "description": "Controls if fixed delay or fixed rate is used. See ScheduledExecutorService in JDK for details." } + "batchAutoCommitDisabled": { "index": 23, "kind": "parameter", "displayName": "Batch Auto Commit Disabled", "group": "producer", "label": "producer", "required": false, "type": "boolean", "javaType": "boolean", "deprecated": false, "autowired": false, "secret": false, "defaultValue": true, "description": "Whether to optimize batch by turning off auto-commit which can dramatic improve performance, and instead execute as a manual commit after the entire batch operation is complete" }, + "noop": { "index": 24, "kind": "parameter", "displayName": "Noop", "group": "producer", "label": "producer", "required": false, "type": "boolean", "javaType": "boolean", "deprecated": false, "autowired": false, "secret": false, "defaultValue": false, "description": "If set, will ignore the results of the SQL query and use the existing IN message as the OUT message for the continuation of processing" }, + "useMessageBodyForSql": { "index": 25, "kind": "parameter", "displayName": "Use Message Body For Sql", "group": "producer", "label": "producer", "required": false, "type": "boolean", "javaType": "boolean", "deprecated": false, "autowired": false, "secret": false, "defaultValue": false, "description": "Whether to use the message body as the SQL and then headers for parameters. If this option is enabled then the SQL in the uri is not used. Note that query parameters in the message body [...] + "lazyStartProducer": { "index": 26, "kind": "parameter", "displayName": "Lazy Start Producer", "group": "producer (advanced)", "label": "producer,advanced", "required": false, "type": "boolean", "javaType": "boolean", "deprecated": false, "autowired": false, "secret": false, "defaultValue": false, "description": "Whether the producer should be started lazy (on the first message). By starting lazy you can use this to allow CamelContext and routes to startup in situations where a produ [...] + "alwaysPopulateStatement": { "index": 27, "kind": "parameter", "displayName": "Always Populate Statement", "group": "advanced", "label": "advanced", "required": false, "type": "boolean", "javaType": "boolean", "deprecated": false, "autowired": false, "secret": false, "defaultValue": false, "description": "If enabled then the populateStatement method from org.apache.camel.component.sql.SqlPrepareStatementStrategy is always invoked, also if there is no expected parameters to be prepare [...] + "fetchSize": { "index": 28, "kind": "parameter", "displayName": "Fetch Size", "group": "advanced", "label": "advanced", "required": false, "type": "integer", "javaType": "int", "deprecated": false, "autowired": false, "secret": false, "description": "Gives the JDBC driver a hint as to the number of rows that should be fetched from the database when more rows are needed for ResultSet objects generated by this Statement. If the value specified is zero, then the hint is ignored. The def [...] + "parametersCount": { "index": 29, "kind": "parameter", "displayName": "Parameters Count", "group": "advanced", "label": "advanced", "required": false, "type": "integer", "javaType": "int", "deprecated": false, "autowired": false, "secret": false, "description": "If set greater than zero, then Camel will use this count value of parameters to replace instead of querying via JDBC metadata API. This is useful if the JDBC vendor could not return correct parameters count, then user may ove [...] + "placeholder": { "index": 30, "kind": "parameter", "displayName": "Placeholder", "group": "advanced", "label": "advanced", "required": false, "type": "string", "javaType": "java.lang.String", "deprecated": false, "autowired": false, "secret": false, "defaultValue": "#", "description": "Specifies a character that will be replaced to in SQL query. Notice, that it is simple String.replaceAll() operation and no SQL parsing is involved (quoted strings will also change)." }, + "prepareStatementStrategy": { "index": 31, "kind": "parameter", "displayName": "Prepare Statement Strategy", "group": "advanced", "label": "advanced", "required": false, "type": "object", "javaType": "org.apache.camel.component.sql.SqlPrepareStatementStrategy", "deprecated": false, "autowired": false, "secret": false, "description": "Allows to plugin to use a custom org.apache.camel.component.sql.SqlPrepareStatementStrategy to control preparation of the query and prepared statement." }, + "rowMapperFactory": { "index": 32, "kind": "parameter", "displayName": "Row Mapper Factory", "group": "advanced", "label": "advanced", "required": false, "type": "object", "javaType": "org.apache.camel.component.sql.RowMapperFactory", "deprecated": false, "autowired": false, "secret": false, "description": "Factory for creating RowMapper" }, + "templateOptions": { "index": 33, "kind": "parameter", "displayName": "Template Options", "group": "advanced", "label": "advanced", "required": false, "type": "object", "javaType": "java.util.Map<java.lang.String, java.lang.Object>", "prefix": "template.", "multiValue": true, "deprecated": false, "autowired": false, "secret": false, "description": "Configures the Spring JdbcTemplate with the key\/values from the Map" }, + "usePlaceholder": { "index": 34, "kind": "parameter", "displayName": "Use Placeholder", "group": "advanced", "label": "advanced", "required": false, "type": "boolean", "javaType": "boolean", "deprecated": false, "autowired": false, "secret": false, "defaultValue": true, "description": "Sets whether to use placeholder and replace all placeholder characters with sign in the SQL queries." }, + "backoffErrorThreshold": { "index": 35, "kind": "parameter", "displayName": "Backoff Error Threshold", "group": "scheduler", "label": "consumer,scheduler", "required": false, "type": "integer", "javaType": "int", "deprecated": false, "autowired": false, "secret": false, "description": "The number of subsequent error polls (failed due some error) that should happen before the backoffMultipler should kick-in." }, + "backoffIdleThreshold": { "index": 36, "kind": "parameter", "displayName": "Backoff Idle Threshold", "group": "scheduler", "label": "consumer,scheduler", "required": false, "type": "integer", "javaType": "int", "deprecated": false, "autowired": false, "secret": false, "description": "The number of subsequent idle polls that should happen before the backoffMultipler should kick-in." }, + "backoffMultiplier": { "index": 37, "kind": "parameter", "displayName": "Backoff Multiplier", "group": "scheduler", "label": "consumer,scheduler", "required": false, "type": "integer", "javaType": "int", "deprecated": false, "autowired": false, "secret": false, "description": "To let the scheduled polling consumer backoff if there has been a number of subsequent idles\/errors in a row. The multiplier is then the number of polls that will be skipped before the next actual attempt is h [...] + "delay": { "index": 38, "kind": "parameter", "displayName": "Delay", "group": "scheduler", "label": "consumer,scheduler", "required": false, "type": "integer", "javaType": "long", "deprecated": false, "autowired": false, "secret": false, "defaultValue": 500, "description": "Milliseconds before the next poll." }, + "greedy": { "index": 39, "kind": "parameter", "displayName": "Greedy", "group": "scheduler", "label": "consumer,scheduler", "required": false, "type": "boolean", "javaType": "boolean", "deprecated": false, "autowired": false, "secret": false, "defaultValue": false, "description": "If greedy is enabled, then the ScheduledPollConsumer will run immediately again, if the previous run polled 1 or more messages." }, + "initialDelay": { "index": 40, "kind": "parameter", "displayName": "Initial Delay", "group": "scheduler", "label": "consumer,scheduler", "required": false, "type": "integer", "javaType": "long", "deprecated": false, "autowired": false, "secret": false, "defaultValue": 1000, "description": "Milliseconds before the first poll starts." }, + "repeatCount": { "index": 41, "kind": "parameter", "displayName": "Repeat Count", "group": "scheduler", "label": "consumer,scheduler", "required": false, "type": "integer", "javaType": "long", "deprecated": false, "autowired": false, "secret": false, "defaultValue": 0, "description": "Specifies a maximum limit of number of fires. So if you set it to 1, the scheduler will only fire once. If you set it to 5, it will only fire five times. A value of zero or negative means fire forever." }, + "runLoggingLevel": { "index": 42, "kind": "parameter", "displayName": "Run Logging Level", "group": "scheduler", "label": "consumer,scheduler", "required": false, "type": "object", "javaType": "org.apache.camel.LoggingLevel", "enum": [ "TRACE", "DEBUG", "INFO", "WARN", "ERROR", "OFF" ], "deprecated": false, "autowired": false, "secret": false, "defaultValue": "TRACE", "description": "The consumer logs a start\/complete log line when it polls. This option allows you to configure the l [...] + "scheduledExecutorService": { "index": 43, "kind": "parameter", "displayName": "Scheduled Executor Service", "group": "scheduler", "label": "consumer,scheduler", "required": false, "type": "object", "javaType": "java.util.concurrent.ScheduledExecutorService", "deprecated": false, "autowired": false, "secret": false, "description": "Allows for configuring a custom\/shared thread pool to use for the consumer. By default each consumer has its own single threaded thread pool." }, + "scheduler": { "index": 44, "kind": "parameter", "displayName": "Scheduler", "group": "scheduler", "label": "consumer,scheduler", "required": false, "type": "object", "javaType": "java.lang.Object", "deprecated": false, "autowired": false, "secret": false, "defaultValue": "none", "description": "To use a cron scheduler from either camel-spring or camel-quartz component. Use value spring or quartz for built in scheduler" }, + "schedulerProperties": { "index": 45, "kind": "parameter", "displayName": "Scheduler Properties", "group": "scheduler", "label": "consumer,scheduler", "required": false, "type": "object", "javaType": "java.util.Map<java.lang.String, java.lang.Object>", "prefix": "scheduler.", "multiValue": true, "deprecated": false, "autowired": false, "secret": false, "description": "To configure additional properties when using a custom scheduler or any of the Quartz, Spring based scheduler." }, + "startScheduler": { "index": 46, "kind": "parameter", "displayName": "Start Scheduler", "group": "scheduler", "label": "consumer,scheduler", "required": false, "type": "boolean", "javaType": "boolean", "deprecated": false, "autowired": false, "secret": false, "defaultValue": true, "description": "Whether the scheduler should be auto started." }, + "timeUnit": { "index": 47, "kind": "parameter", "displayName": "Time Unit", "group": "scheduler", "label": "consumer,scheduler", "required": false, "type": "object", "javaType": "java.util.concurrent.TimeUnit", "enum": [ "NANOSECONDS", "MICROSECONDS", "MILLISECONDS", "SECONDS", "MINUTES", "HOURS", "DAYS" ], "deprecated": false, "autowired": false, "secret": false, "defaultValue": "MILLISECONDS", "description": "Time unit for initialDelay and delay options." }, + "useFixedDelay": { "index": 48, "kind": "parameter", "displayName": "Use Fixed Delay", "group": "scheduler", "label": "consumer,scheduler", "required": false, "type": "boolean", "javaType": "boolean", "deprecated": false, "autowired": false, "secret": false, "defaultValue": true, "description": "Controls if fixed delay or fixed rate is used. See ScheduledExecutorService in JDK for details." } } } diff --git a/components/camel-sql/src/main/java/org/apache/camel/component/sql/DefaultSqlEndpoint.java b/components/camel-sql/src/main/java/org/apache/camel/component/sql/DefaultSqlEndpoint.java index 473d1ed24ee..a8565aca61d 100644 --- a/components/camel-sql/src/main/java/org/apache/camel/component/sql/DefaultSqlEndpoint.java +++ b/components/camel-sql/src/main/java/org/apache/camel/component/sql/DefaultSqlEndpoint.java @@ -63,6 +63,10 @@ public abstract class DefaultSqlEndpoint extends DefaultPollingEndpoint implemen @UriParam(label = "consumer,advanced", description = "Allows to plugin to use a custom org.apache.camel.component.sql.SqlProcessingStrategy to execute queries when the consumer has processed the rows/batch.") private SqlProcessingStrategy processingStrategy; + @UriParam(label = "advanced", + description = "Gives the JDBC driver a hint as to the number of rows that should be fetched from the database when more rows are needed for ResultSet objects generated by this Statement. If the value specified is zero, then the hint is ignored. The default value is zero." + + " This is important for processing large result sets: Setting this higher than the default value will increase processing speed at the cost of memory consumption; setting this lower can avoid transferring row data that will never be read by the application.") + private int fetchSize; @UriParam(label = "advanced", description = "Allows to plugin to use a custom org.apache.camel.component.sql.SqlPrepareStatementStrategy to control preparation of the query and prepared statement.") private SqlPrepareStatementStrategy prepareStatementStrategy; @@ -199,6 +203,23 @@ public abstract class DefaultSqlEndpoint extends DefaultPollingEndpoint implemen this.maxMessagesPerPoll = maxMessagesPerPoll; } + public int getFetchSize() { + return fetchSize; + } + + /** + * Gives the JDBC driver a hint as to the number of rows that should be fetched from the database when more rows are + * needed for ResultSet objects generated by this Statement. If the value specified is zero, then the hint is + * ignored. The default value is zero. + * + * This is important for processing large result sets: Setting this higher than the default value will increase + * processing speed at the cost of memory consumption; setting this lower can avoid transferring row data that will + * never be read by the application. + */ + public void setFetchSize(int fetchSize) { + this.fetchSize = fetchSize; + } + public SqlProcessingStrategy getProcessingStrategy() { return processingStrategy; } diff --git a/components/camel-sql/src/main/java/org/apache/camel/component/sql/SqlComponent.java b/components/camel-sql/src/main/java/org/apache/camel/component/sql/SqlComponent.java index 7f92e5092b3..507ab55eb00 100644 --- a/components/camel-sql/src/main/java/org/apache/camel/component/sql/SqlComponent.java +++ b/components/camel-sql/src/main/java/org/apache/camel/component/sql/SqlComponent.java @@ -49,6 +49,13 @@ public class SqlComponent extends HealthCheckComponent { + " This option can be used to turn this off.", defaultValue = "true") private boolean serviceLocationEnabled = true; + @Metadata(label = "producer", defaultValue = "true", + description = "Whether to optimize batch by turning off auto-commit which can dramatic improve performance, and instead execute as a manual commit after the entire batch operation is complete") + private boolean batchAutoCommitDisabled = true; + @Metadata(label = "advanced", + description = "Gives the JDBC driver a hint as to the number of rows that should be fetched from the database when more rows are needed for ResultSet objects generated by this Statement. If the value specified is zero, then the hint is ignored. The default value is zero." + + " This is important for processing large result sets: Setting this higher than the default value will increase processing speed at the cost of memory consumption; setting this lower can avoid transferring row data that will never be read by the application.") + private int fetchSize; public SqlComponent() { } @@ -106,6 +113,8 @@ public class SqlComponent extends HealthCheckComponent { // create endpoint SqlEndpoint endpoint = new SqlEndpoint(uri, this); + endpoint.setBatchAutoCommitDisabled(isBatchAutoCommitDisabled()); + endpoint.setFetchSize(getFetchSize()); endpoint.setServiceLocationEnabled(serviceLocationEnabled); endpoint.setQuery(query); endpoint.setPlaceholder(parameterPlaceholderSubstitute); @@ -185,4 +194,32 @@ public class SqlComponent extends HealthCheckComponent { this.serviceLocationEnabled = serviceLocationEnabled; } + public boolean isBatchAutoCommitDisabled() { + return batchAutoCommitDisabled; + } + + /** + * Whether to optimize batch by turning off auto-commit which can dramatic improve performance, and instead execute + * as a manual commit after the entire batch operation is complete + */ + public void setBatchAutoCommitDisabled(boolean batchAutoCommitDisabled) { + this.batchAutoCommitDisabled = batchAutoCommitDisabled; + } + + public int getFetchSize() { + return fetchSize; + } + + /** + * Gives the JDBC driver a hint as to the number of rows that should be fetched from the database when more rows are + * needed for ResultSet objects generated by this Statement. If the value specified is zero, then the hint is + * ignored. The default value is zero. + * + * This is important for processing large result sets: Setting this higher than the default value will increase + * processing speed at the cost of memory consumption; setting this lower can avoid transferring row data that will + * never be read by the application. + */ + public void setFetchSize(int fetchSize) { + this.fetchSize = fetchSize; + } } diff --git a/components/camel-sql/src/main/java/org/apache/camel/component/sql/SqlConsumer.java b/components/camel-sql/src/main/java/org/apache/camel/component/sql/SqlConsumer.java index 2c3132338d8..1ae2d6ecbce 100644 --- a/components/camel-sql/src/main/java/org/apache/camel/component/sql/SqlConsumer.java +++ b/components/camel-sql/src/main/java/org/apache/camel/component/sql/SqlConsumer.java @@ -141,6 +141,10 @@ public class SqlConsumer extends ScheduledBatchPollingConsumer { final PreparedStatementCallback<Integer> callback = new PreparedStatementCallback<Integer>() { @Override public Integer doInPreparedStatement(PreparedStatement ps) throws SQLException, DataAccessException { + if (getEndpoint().getFetchSize() > 0) { + ps.setFetchSize(getEndpoint().getFetchSize()); + } + Queue<DataHolder> answer = new LinkedList<>(); int expected = parametersCount > 0 ? parametersCount : ps.getParameterMetaData().getParameterCount(); diff --git a/components/camel-sql/src/main/java/org/apache/camel/component/sql/SqlEndpoint.java b/components/camel-sql/src/main/java/org/apache/camel/component/sql/SqlEndpoint.java index be0aad03b47..fd6affe1e4d 100644 --- a/components/camel-sql/src/main/java/org/apache/camel/component/sql/SqlEndpoint.java +++ b/components/camel-sql/src/main/java/org/apache/camel/component/sql/SqlEndpoint.java @@ -23,6 +23,7 @@ import org.apache.camel.Processor; import org.apache.camel.Producer; import org.apache.camel.spi.Metadata; import org.apache.camel.spi.UriEndpoint; +import org.apache.camel.spi.UriParam; import org.apache.camel.spi.UriPath; import org.apache.camel.util.UnsafeUriCharactersEncoder; @@ -36,6 +37,9 @@ public class SqlEndpoint extends DefaultSqlEndpoint { @UriPath(description = "Sets the SQL query to perform. You can externalize the query by using file: or classpath: as prefix and specify the location of the file.") @Metadata(required = true, supportFileReference = true, largeInput = true, inputLanguage = "sql") private String query; + @UriParam(label = "producer", defaultValue = "true", + description = "Whether to optimize batch by turning off auto-commit which can dramatic improve performance, and instead execute as a manual commit after the entire batch operation is complete") + private boolean batchAutoCommitDisabled = true; public SqlEndpoint() { } @@ -94,4 +98,15 @@ public class SqlEndpoint extends DefaultSqlEndpoint { this.query = query; } + public boolean isBatchAutoCommitDisabled() { + return batchAutoCommitDisabled; + } + + /** + * Whether to optimize batch by turning off auto-commit which can dramatic improve performance, and instead execute + * as a manual commit after the entire batch operation is complete + */ + public void setBatchAutoCommitDisabled(boolean batchAutoCommitDisabled) { + this.batchAutoCommitDisabled = batchAutoCommitDisabled; + } } diff --git a/components/camel-sql/src/main/java/org/apache/camel/component/sql/SqlProducer.java b/components/camel-sql/src/main/java/org/apache/camel/component/sql/SqlProducer.java index 3bcc5359ea1..3a771c290a4 100644 --- a/components/camel-sql/src/main/java/org/apache/camel/component/sql/SqlProducer.java +++ b/components/camel-sql/src/main/java/org/apache/camel/component/sql/SqlProducer.java @@ -52,12 +52,14 @@ public class SqlProducer extends DefaultProducer { private final boolean alwaysPopulateStatement; private final SqlPrepareStatementStrategy sqlPrepareStatementStrategy; private final boolean useMessageBodyForSql; + private final boolean manualCommit; private int parametersCount; public SqlProducer(SqlEndpoint endpoint, String query, JdbcTemplate jdbcTemplate, SqlPrepareStatementStrategy sqlPrepareStatementStrategy, boolean batch, boolean alwaysPopulateStatement, boolean useMessageBodyForSql) { super(endpoint); + this.manualCommit = endpoint.isBatchAutoCommitDisabled(); this.jdbcTemplate = jdbcTemplate; this.sqlPrepareStatementStrategy = sqlPrepareStatementStrategy; this.query = query; @@ -154,14 +156,38 @@ public class SqlProducer extends DefaultProducer { try { populateStatement(ps, exchange, sql, preparedQuery); boolean isResultSet = false; + boolean restoreAutoCommit = true; if (batch) { - int[] updateCounts = ps.executeBatch(); - int total = 0; - for (int count : updateCounts) { - total += count; + if (manualCommit) { + // optimize batch by turning off auto-commit + restoreAutoCommit = ps.getConnection().getAutoCommit(); + ps.getConnection().setAutoCommit(false); + } + try { + int[] updateCounts = ps.executeBatch(); + int total = 0; + for (int count : updateCounts) { + total += count; + } + exchange.getIn().setHeader(SqlConstants.SQL_UPDATE_COUNT, total); + if (manualCommit) { + // optimize batch by commit after done + ps.getConnection().commit(); + } + } catch (Exception e) { + if (manualCommit) { + // we failed so rollback + ps.getConnection().rollback(); + } + throw e; + } finally { + if (manualCommit && restoreAutoCommit) { + // restore auto commit on connection as it may be used + // in another kind of query (connection pooling) + ps.getConnection().setAutoCommit(true); + } } - exchange.getIn().setHeader(SqlConstants.SQL_UPDATE_COUNT, total); } else { isResultSet = ps.execute(); if (isResultSet) { @@ -250,6 +276,11 @@ public class SqlProducer extends DefaultProducer { private void populateStatement(PreparedStatement ps, Exchange exchange, String sql, String preparedQuery) throws SQLException { + + if (getEndpoint().getFetchSize() > 0) { + ps.setFetchSize(getEndpoint().getFetchSize()); + } + int expected; if (parametersCount > 0) { expected = parametersCount; diff --git a/components/camel-sql/src/test/java/org/apache/camel/component/sql/SqlConsumerFetchSizeTest.java b/components/camel-sql/src/test/java/org/apache/camel/component/sql/SqlConsumerFetchSizeTest.java new file mode 100644 index 00000000000..29e6f164649 --- /dev/null +++ b/components/camel-sql/src/test/java/org/apache/camel/component/sql/SqlConsumerFetchSizeTest.java @@ -0,0 +1,87 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.camel.component.sql; + +import java.util.List; +import java.util.Map; + +import org.apache.camel.Exchange; +import org.apache.camel.builder.RouteBuilder; +import org.apache.camel.component.mock.MockEndpoint; +import org.apache.camel.test.junit5.CamelTestSupport; +import org.junit.jupiter.api.Test; +import org.springframework.jdbc.datasource.embedded.EmbeddedDatabase; +import org.springframework.jdbc.datasource.embedded.EmbeddedDatabaseBuilder; +import org.springframework.jdbc.datasource.embedded.EmbeddedDatabaseType; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; + +public class SqlConsumerFetchSizeTest extends CamelTestSupport { + + private EmbeddedDatabase db; + + @Override + public void doPreSetup() throws Exception { + db = new EmbeddedDatabaseBuilder() + .setName(getClass().getSimpleName()) + .setType(EmbeddedDatabaseType.H2) + .addScript("sql/createAndPopulateDatabase.sql").build(); + + } + + @Override + public void doPostTearDown() throws Exception { + if (db != null) { + db.shutdown(); + } + } + + @Test + public void testConsume() throws Exception { + MockEndpoint mock = getMockEndpoint("mock:result"); + mock.expectedMinimumMessageCount(3); + + MockEndpoint.assertIsSatisfied(context); + + List<Exchange> exchanges = mock.getReceivedExchanges(); + assertTrue(exchanges.size() >= 3); + + assertEquals(1, exchanges.get(0).getIn().getBody(Map.class).get("ID")); + assertEquals("Camel", exchanges.get(0).getIn().getBody(Map.class).get("PROJECT")); + assertEquals(2, exchanges.get(1).getIn().getBody(Map.class).get("ID")); + assertEquals("AMQ", exchanges.get(1).getIn().getBody(Map.class).get("PROJECT")); + assertEquals(3, exchanges.get(2).getIn().getBody(Map.class).get("ID")); + assertEquals("Linux", exchanges.get(2).getIn().getBody(Map.class).get("PROJECT")); + } + + @Override + protected RouteBuilder createRouteBuilder() { + return new RouteBuilder() { + @Override + public void configure() { + // required for the sql component + SqlComponent sql = getContext().getComponent("sql", SqlComponent.class); + sql.setDataSource(db); + sql.setFetchSize(10); + + from("sql:select * from projects order by id?initialDelay=0&delay=50") + .to("mock:result"); + } + }; + } +} diff --git a/components/camel-sql/src/test/java/org/apache/camel/component/sql/SqlProducerFetchSizeTest.java b/components/camel-sql/src/test/java/org/apache/camel/component/sql/SqlProducerFetchSizeTest.java new file mode 100644 index 00000000000..49f580572b1 --- /dev/null +++ b/components/camel-sql/src/test/java/org/apache/camel/component/sql/SqlProducerFetchSizeTest.java @@ -0,0 +1,87 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.camel.component.sql; + +import java.util.List; +import java.util.Map; + +import org.apache.camel.builder.RouteBuilder; +import org.apache.camel.component.mock.MockEndpoint; +import org.apache.camel.test.junit5.CamelTestSupport; +import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Test; +import org.springframework.jdbc.datasource.embedded.EmbeddedDatabase; +import org.springframework.jdbc.datasource.embedded.EmbeddedDatabaseBuilder; +import org.springframework.jdbc.datasource.embedded.EmbeddedDatabaseType; + +import static org.junit.jupiter.api.Assertions.assertEquals; + +public class SqlProducerFetchSizeTest extends CamelTestSupport { + + static EmbeddedDatabase db; + + @BeforeAll + public static void setupDatabase() { + db = new EmbeddedDatabaseBuilder() + .setName(SqlProducerFetchSizeTest.class.getSimpleName()) + .setType(EmbeddedDatabaseType.H2) + .addScript("sql/createAndPopulateDatabase.sql").build(); + } + + @AfterAll + public static void cleanupDatabase() { + if (db != null) { + db.shutdown(); + } + } + + @Test + public void testFetchSize() throws InterruptedException { + MockEndpoint mock = getMockEndpoint("mock:query"); + mock.expectedMessageCount(1); + + template.requestBody("direct:query", new String[] { "Camel", "AMQ" }); + + MockEndpoint.assertIsSatisfied(context); + + List list = mock.getReceivedExchanges().get(0).getIn().getBody(List.class); + assertEquals(2, list.size()); + Map row = (Map) list.get(0); + assertEquals("Camel", row.get("PROJECT")); + row = (Map) list.get(1); + assertEquals("AMQ", row.get("PROJECT")); + } + + @Override + protected RouteBuilder createRouteBuilder() { + return new RouteBuilder() { + @Override + public void configure() { + // required for the sql component + SqlComponent sql = getContext().getComponent("sql", SqlComponent.class); + sql.setDataSource(db); + sql.setFetchSize(10); + + from("direct:query") + .to("sql:classpath:sql/selectProjectsInBody.sql") + .to("log:query") + .to("mock:query"); + } + }; + } +} diff --git a/docs/user-manual/modules/ROOT/pages/camel-4x-upgrade-guide-4_10.adoc b/docs/user-manual/modules/ROOT/pages/camel-4x-upgrade-guide-4_10.adoc index 7fd56f991b1..9d2f5d7609b 100644 --- a/docs/user-manual/modules/ROOT/pages/camel-4x-upgrade-guide-4_10.adoc +++ b/docs/user-manual/modules/ROOT/pages/camel-4x-upgrade-guide-4_10.adoc @@ -23,6 +23,13 @@ When MDC is enabled, then the WireTap and OnCompletion (in parallel mode) will n context when creating threads to process the exchanges. This makes these EIPs similar to how other EIPs such as Multicast EIP already does this. +=== camel-sql + +When inserting or updating many rows in `batch=true` mode (producer) then this component +has been optimized to execute the entire batch operation in a single transaction; by turning off auto-commit +on the SQL Connection, and doing a manual `commit` or `rollback`. This can dramatically improve performance +on some databases. The old behaviour can be restored by setting the `batchAutoCommitDisabled=false` on the component or endpoint. + == Upgrading from 4.10.0 to 4.10.1 === camel-api diff --git a/docs/user-manual/modules/ROOT/pages/camel-4x-upgrade-guide-4_11.adoc b/docs/user-manual/modules/ROOT/pages/camel-4x-upgrade-guide-4_11.adoc index 5d0b7ec757d..4462efaaa59 100644 --- a/docs/user-manual/modules/ROOT/pages/camel-4x-upgrade-guide-4_11.adoc +++ b/docs/user-manual/modules/ROOT/pages/camel-4x-upgrade-guide-4_11.adoc @@ -78,3 +78,10 @@ work in more situations out of the box. From this version onward the `export` command add the `camel-observability-services` dependency (which includes telemetry, metrics, health and JMX management services out of the box). The `--health` and `--metrics` flags of `run` command have been deprecated in favor of the newly `--observe` flag which add the `camel-observability-services` dependency (hence including telemetry, metrics, health and JMX management out of the box). For the run command, this has to be explicitly enabled (ie, `camel run ... --observe`). + +=== camel-sql + +When inserting or updating many rows in `batch=true` mode (producer) then this component +has been optimized to execute the entire batch operation in a single transaction; by turning off auto-commit +on the SQL Connection, and doing a manual `commit` or `rollback`. This can dramatically improve performance +on some databases. The old behaviour can be restored by setting the `batchAutoCommitDisabled=false` on the component or endpoint. diff --git a/dsl/camel-componentdsl/src/generated/java/org/apache/camel/builder/component/dsl/SqlComponentBuilderFactory.java b/dsl/camel-componentdsl/src/generated/java/org/apache/camel/builder/component/dsl/SqlComponentBuilderFactory.java index 2a9f78c6800..df2f9b38d9b 100644 --- a/dsl/camel-componentdsl/src/generated/java/org/apache/camel/builder/component/dsl/SqlComponentBuilderFactory.java +++ b/dsl/camel-componentdsl/src/generated/java/org/apache/camel/builder/component/dsl/SqlComponentBuilderFactory.java @@ -94,6 +94,25 @@ public interface SqlComponentBuilderFactory { } + /** + * Whether to optimize batch by turning off auto-commit which can + * dramatic improve performance, and instead execute as a manual commit + * after the entire batch operation is complete. + * + * The option is a: <code>boolean</code> type. + * + * Default: true + * Group: producer + * + * @param batchAutoCommitDisabled the value to set + * @return the dsl builder + */ + default SqlComponentBuilder batchAutoCommitDisabled(boolean batchAutoCommitDisabled) { + doSetProperty("batchAutoCommitDisabled", batchAutoCommitDisabled); + return this; + } + + /** * Whether the producer should be started lazy (on the first message). * By starting lazy you can use this to allow CamelContext and routes to @@ -140,6 +159,28 @@ public interface SqlComponentBuilderFactory { return this; } + /** + * Gives the JDBC driver a hint as to the number of rows that should be + * fetched from the database when more rows are needed for ResultSet + * objects generated by this Statement. If the value specified is zero, + * then the hint is ignored. The default value is zero. This is + * important for processing large result sets: Setting this higher than + * the default value will increase processing speed at the cost of + * memory consumption; setting this lower can avoid transferring row + * data that will never be read by the application. + * + * The option is a: <code>int</code> type. + * + * Group: advanced + * + * @param fetchSize the value to set + * @return the dsl builder + */ + default SqlComponentBuilder fetchSize(int fetchSize) { + doSetProperty("fetchSize", fetchSize); + return this; + } + /** * Factory for creating RowMapper. * @@ -248,8 +289,10 @@ public interface SqlComponentBuilderFactory { switch (name) { case "dataSource": ((SqlComponent) component).setDataSource((javax.sql.DataSource) value); return true; case "bridgeErrorHandler": ((SqlComponent) component).setBridgeErrorHandler((boolean) value); return true; + case "batchAutoCommitDisabled": ((SqlComponent) component).setBatchAutoCommitDisabled((boolean) value); return true; case "lazyStartProducer": ((SqlComponent) component).setLazyStartProducer((boolean) value); return true; case "autowiredEnabled": ((SqlComponent) component).setAutowiredEnabled((boolean) value); return true; + case "fetchSize": ((SqlComponent) component).setFetchSize((int) value); return true; case "rowMapperFactory": ((SqlComponent) component).setRowMapperFactory((org.apache.camel.component.sql.RowMapperFactory) value); return true; case "serviceLocationEnabled": ((SqlComponent) component).setServiceLocationEnabled((boolean) value); return true; case "usePlaceholder": ((SqlComponent) component).setUsePlaceholder((boolean) value); return true; diff --git a/dsl/camel-endpointdsl/src/generated/java/org/apache/camel/builder/endpoint/dsl/SqlEndpointBuilderFactory.java b/dsl/camel-endpointdsl/src/generated/java/org/apache/camel/builder/endpoint/dsl/SqlEndpointBuilderFactory.java index 0e9676e8e96..c1b25fcc227 100644 --- a/dsl/camel-endpointdsl/src/generated/java/org/apache/camel/builder/endpoint/dsl/SqlEndpointBuilderFactory.java +++ b/dsl/camel-endpointdsl/src/generated/java/org/apache/camel/builder/endpoint/dsl/SqlEndpointBuilderFactory.java @@ -1189,6 +1189,48 @@ public interface SqlEndpointBuilderFactory { doSetProperty("alwaysPopulateStatement", alwaysPopulateStatement); return this; } + /** + * Gives the JDBC driver a hint as to the number of rows that should be + * fetched from the database when more rows are needed for ResultSet + * objects generated by this Statement. If the value specified is zero, + * then the hint is ignored. The default value is zero. This is + * important for processing large result sets: Setting this higher than + * the default value will increase processing speed at the cost of + * memory consumption; setting this lower can avoid transferring row + * data that will never be read by the application. + * + * The option is a: <code>int</code> type. + * + * Group: advanced + * + * @param fetchSize the value to set + * @return the dsl builder + */ + default AdvancedSqlEndpointConsumerBuilder fetchSize(int fetchSize) { + doSetProperty("fetchSize", fetchSize); + return this; + } + /** + * Gives the JDBC driver a hint as to the number of rows that should be + * fetched from the database when more rows are needed for ResultSet + * objects generated by this Statement. If the value specified is zero, + * then the hint is ignored. The default value is zero. This is + * important for processing large result sets: Setting this higher than + * the default value will increase processing speed at the cost of + * memory consumption; setting this lower can avoid transferring row + * data that will never be read by the application. + * + * The option will be converted to a <code>int</code> type. + * + * Group: advanced + * + * @param fetchSize the value to set + * @return the dsl builder + */ + default AdvancedSqlEndpointConsumerBuilder fetchSize(String fetchSize) { + doSetProperty("fetchSize", fetchSize); + return this; + } /** * If set greater than zero, then Camel will use this count value of * parameters to replace instead of querying via JDBC metadata API. This @@ -1601,6 +1643,40 @@ public interface SqlEndpointBuilderFactory { doSetProperty("batch", batch); return this; } + /** + * Whether to optimize batch by turning off auto-commit which can + * dramatic improve performance, and instead execute as a manual commit + * after the entire batch operation is complete. + * + * The option is a: <code>boolean</code> type. + * + * Default: true + * Group: producer + * + * @param batchAutoCommitDisabled the value to set + * @return the dsl builder + */ + default SqlEndpointProducerBuilder batchAutoCommitDisabled(boolean batchAutoCommitDisabled) { + doSetProperty("batchAutoCommitDisabled", batchAutoCommitDisabled); + return this; + } + /** + * Whether to optimize batch by turning off auto-commit which can + * dramatic improve performance, and instead execute as a manual commit + * after the entire batch operation is complete. + * + * The option will be converted to a <code>boolean</code> type. + * + * Default: true + * Group: producer + * + * @param batchAutoCommitDisabled the value to set + * @return the dsl builder + */ + default SqlEndpointProducerBuilder batchAutoCommitDisabled(String batchAutoCommitDisabled) { + doSetProperty("batchAutoCommitDisabled", batchAutoCommitDisabled); + return this; + } /** * If set, will ignore the results of the SQL query and use the existing * IN message as the OUT message for the continuation of processing. @@ -1765,6 +1841,48 @@ public interface SqlEndpointBuilderFactory { doSetProperty("alwaysPopulateStatement", alwaysPopulateStatement); return this; } + /** + * Gives the JDBC driver a hint as to the number of rows that should be + * fetched from the database when more rows are needed for ResultSet + * objects generated by this Statement. If the value specified is zero, + * then the hint is ignored. The default value is zero. This is + * important for processing large result sets: Setting this higher than + * the default value will increase processing speed at the cost of + * memory consumption; setting this lower can avoid transferring row + * data that will never be read by the application. + * + * The option is a: <code>int</code> type. + * + * Group: advanced + * + * @param fetchSize the value to set + * @return the dsl builder + */ + default AdvancedSqlEndpointProducerBuilder fetchSize(int fetchSize) { + doSetProperty("fetchSize", fetchSize); + return this; + } + /** + * Gives the JDBC driver a hint as to the number of rows that should be + * fetched from the database when more rows are needed for ResultSet + * objects generated by this Statement. If the value specified is zero, + * then the hint is ignored. The default value is zero. This is + * important for processing large result sets: Setting this higher than + * the default value will increase processing speed at the cost of + * memory consumption; setting this lower can avoid transferring row + * data that will never be read by the application. + * + * The option will be converted to a <code>int</code> type. + * + * Group: advanced + * + * @param fetchSize the value to set + * @return the dsl builder + */ + default AdvancedSqlEndpointProducerBuilder fetchSize(String fetchSize) { + doSetProperty("fetchSize", fetchSize); + return this; + } /** * If set greater than zero, then Camel will use this count value of * parameters to replace instead of querying via JDBC metadata API. This @@ -2201,6 +2319,48 @@ public interface SqlEndpointBuilderFactory { doSetProperty("alwaysPopulateStatement", alwaysPopulateStatement); return this; } + /** + * Gives the JDBC driver a hint as to the number of rows that should be + * fetched from the database when more rows are needed for ResultSet + * objects generated by this Statement. If the value specified is zero, + * then the hint is ignored. The default value is zero. This is + * important for processing large result sets: Setting this higher than + * the default value will increase processing speed at the cost of + * memory consumption; setting this lower can avoid transferring row + * data that will never be read by the application. + * + * The option is a: <code>int</code> type. + * + * Group: advanced + * + * @param fetchSize the value to set + * @return the dsl builder + */ + default AdvancedSqlEndpointBuilder fetchSize(int fetchSize) { + doSetProperty("fetchSize", fetchSize); + return this; + } + /** + * Gives the JDBC driver a hint as to the number of rows that should be + * fetched from the database when more rows are needed for ResultSet + * objects generated by this Statement. If the value specified is zero, + * then the hint is ignored. The default value is zero. This is + * important for processing large result sets: Setting this higher than + * the default value will increase processing speed at the cost of + * memory consumption; setting this lower can avoid transferring row + * data that will never be read by the application. + * + * The option will be converted to a <code>int</code> type. + * + * Group: advanced + * + * @param fetchSize the value to set + * @return the dsl builder + */ + default AdvancedSqlEndpointBuilder fetchSize(String fetchSize) { + doSetProperty("fetchSize", fetchSize); + return this; + } /** * If set greater than zero, then Camel will use this count value of * parameters to replace instead of querying via JDBC metadata API. This