This is an automated email from the ASF dual-hosted git repository.
morningman pushed a commit to branch master
in repository
https://gitbox.apache.org/repos/asf/incubator-doris-spark-connector.git
commit 115ae27a5d3f747e2cec0e23f0567ac7a5a49a3f
Author: jiafeng.zhang <zhang...@gmail.com>
AuthorDate: Sun Dec 26 11:13:47 2021 +0800
[improvement](spark-connector)(flink-connector) Modify the max num of batch
written by Spark/Flink connector each time. (#7485)
Increase the default batch size and flush interval
---
src/main/java/org/apache/doris/spark/cfg/ConfigurationOptions.java | 6 ++----
src/main/scala/org/apache/doris/spark/sql/DorisStreamLoadSink.scala | 2 +-
2 files changed, 3 insertions(+), 5 deletions(-)
diff --git a/src/main/java/org/apache/doris/spark/cfg/ConfigurationOptions.java
b/src/main/java/org/apache/doris/spark/cfg/ConfigurationOptions.java
index 7ba46b2..9a0cead 100644
--- a/src/main/java/org/apache/doris/spark/cfg/ConfigurationOptions.java
+++ b/src/main/java/org/apache/doris/spark/cfg/ConfigurationOptions.java
@@ -66,11 +66,9 @@ public interface ConfigurationOptions {
String DORIS_WRITE_FIELDS = "doris.write.fields";
- String SINK_BATCH_SIZE = "sink.batch.size";
String DORIS_SINK_BATCH_SIZE = "doris.sink.batch.size";
- int SINK_BATCH_SIZE_DEFAULT = 1024;
+ int SINK_BATCH_SIZE_DEFAULT = 10000;
- String SINK_MAX_RETRIES = "sink.max-retries";
String DORIS_SINK_MAX_RETRIES = "doris.sink.max-retries";
- int SINK_MAX_RETRIES_DEFAULT = 3;
+ int SINK_MAX_RETRIES_DEFAULT = 1;
}
diff --git
a/src/main/scala/org/apache/doris/spark/sql/DorisStreamLoadSink.scala
b/src/main/scala/org/apache/doris/spark/sql/DorisStreamLoadSink.scala
index b53a23a..edd08f1 100644
--- a/src/main/scala/org/apache/doris/spark/sql/DorisStreamLoadSink.scala
+++ b/src/main/scala/org/apache/doris/spark/sql/DorisStreamLoadSink.scala
@@ -72,7 +72,7 @@ private[sql] class DorisStreamLoadSink(sqlContext:
SQLContext, settings: SparkSe
val loop = new Breaks
loop.breakable {
- for (i <- 1 to maxRetryTimes) {
+ for (i <- 0 to maxRetryTimes) {
try {
dorisStreamLoader.load(rowsBuffer)
rowsBuffer.clear()
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org
For additional commands, e-mail: commits-h...@doris.apache.org
