gnehil commented on code in PR #305:
URL:
https://github.com/apache/doris-spark-connector/pull/305#discussion_r2027083710
##########
spark-doris-connector/spark-doris-connector-base/src/main/java/org/apache/doris/spark/client/write/AbstractStreamLoadProcessor.java:
##########
@@ -132,57 +117,82 @@ public AbstractStreamLoadProcessor(DorisConfig config)
throws Exception {
// init stream load props
this.isTwoPhaseCommitEnabled =
config.getValue(DorisOptions.DORIS_SINK_ENABLE_2PC);
this.format = DataFormat.valueOf(properties.getOrDefault("format",
"csv").toUpperCase());
- this.isGzipCompressionEnabled =
properties.containsKey("compress_type") &&
"gzip".equals(properties.get("compress_type"));
+ this.isGzipCompressionEnabled =
+ properties.containsKey("compress_type") &&
"gzip".equals(properties.get("compress_type"));
if (properties.containsKey(GROUP_COMMIT)) {
String message = "";
- if (isTwoPhaseCommitEnabled) message = "group commit does not
support two-phase commit";
- if (properties.containsKey(PARTIAL_COLUMNS) &&
"true".equalsIgnoreCase(properties.get(PARTIAL_COLUMNS)))
+ if (isTwoPhaseCommitEnabled) {
+ message = "group commit does not support two-phase commit";
+ }
+ if (properties.containsKey(PARTIAL_COLUMNS) &&
"true".equalsIgnoreCase(properties.get(PARTIAL_COLUMNS))) {
message = "group commit does not support partial column
updates";
- if
(!VALID_GROUP_MODE.contains(properties.get(GROUP_COMMIT).toLowerCase()))
+ }
+ if
(!VALID_GROUP_MODE.contains(properties.get(GROUP_COMMIT).toLowerCase())) {
message = "Unsupported group commit mode: " +
properties.get(GROUP_COMMIT);
- if (!message.isEmpty()) throw new
IllegalArgumentException(message);
+ }
+ if (!message.isEmpty()) {
+ throw new IllegalArgumentException(message);
+ }
groupCommit = properties.get(GROUP_COMMIT).toLowerCase();
}
this.isPassThrough =
config.getValue(DorisOptions.DORIS_SINK_STREAMING_PASSTHROUGH);
}
public void load(R row) throws Exception {
+ if (unexpectedException != null) {
+ throw unexpectedException;
+ }
if (createNewBatch) {
+ createNewBatch = false;
+ isStopped = false;
if (autoRedirect) {
requestFuture = frontend.requestFrontends((frontEnd,
httpClient) ->
- buildReqAndExec(frontEnd.getHost(),
frontEnd.getHttpPort(), httpClient));
+ {
+ try {
+ return buildReqAndExec(frontEnd.getHost(),
frontEnd.getHttpPort(), httpClient);
+ } catch (OptionRequiredException e) {
+ throw new RuntimeException(e);
+ }
+ });
} else {
requestFuture = backendHttpClient.executeReq((backend,
httpClient) ->
- buildReqAndExec(backend.getHost(),
backend.getHttpPort(), httpClient));
+ {
+ try {
+ return buildReqAndExec(backend.getHost(),
backend.getHttpPort(), httpClient);
+ } catch (OptionRequiredException e) {
+ throw new RuntimeException(e);
+ }
+ });
Review Comment:
Modified, the buffer size is initialized in the constructor, no exception is
thrown here
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org
For queries about this service, please contact Infrastructure at:
us...@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org
For additional commands, e-mail: commits-h...@doris.apache.org
