This is an automated email from the ASF dual-hosted git repository.
morningman pushed a commit to branch compaction_opt
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/compaction_opt by this push:
new 996ba873e2 [bugfix](cherry-pick) 14190 fix csv split_line core (#14336)
996ba873e2 is described below
commit 996ba873e2c1764b7df1297cc587756b3859c29b
Author: yixiutt <102007456+yixi...@users.noreply.github.com>
AuthorDate: Wed Nov 16 20:46:00 2022 +0800
[bugfix](cherry-pick) 14190 fix csv split_line core (#14336)
---
be/src/exec/broker_scanner.cpp | 2 +-
be/src/vec/exec/format/csv/csv_reader.cpp | 2 +-
.../stream_load/test_txt_special_delimiter.csv | 1 +
.../stream_load/test_txt_special_delimiter.out | 7 +++
.../stream_load/test_txt_special_delimiter.groovy | 58 ++++++++++++++++++++++
5 files changed, 68 insertions(+), 2 deletions(-)
diff --git a/be/src/exec/broker_scanner.cpp b/be/src/exec/broker_scanner.cpp
index d65b413927..4b5a0c49ee 100644
--- a/be/src/exec/broker_scanner.cpp
+++ b/be/src/exec/broker_scanner.cpp
@@ -282,7 +282,7 @@ void BrokerScanner::split_line(const Slice& line) {
// curpos
while (curpos < line.size) {
- if (*(value + curpos + p1) != _value_separator[p1]) {
+ if (curpos + p1 == line.size || *(value + curpos + p1) !=
_value_separator[p1]) {
// Not match, move forward:
curpos += (p1 == 0 ? 1 : p1);
p1 = 0;
diff --git a/be/src/vec/exec/format/csv/csv_reader.cpp
b/be/src/vec/exec/format/csv/csv_reader.cpp
index 2a115d8f07..6b118b85b7 100644
--- a/be/src/vec/exec/format/csv/csv_reader.cpp
+++ b/be/src/vec/exec/format/csv/csv_reader.cpp
@@ -395,7 +395,7 @@ void CsvReader::_split_line(const Slice& line) {
// curpos
while (curpos < line.size) {
- if (*(value + curpos + p1) != _value_separator[p1]) {
+ if (curpos + p1 == line.size || *(value + curpos + p1) !=
_value_separator[p1]) {
// Not match, move forward:
curpos += (p1 == 0 ? 1 : p1);
p1 = 0;
diff --git
a/regression-test/data/load_p0/stream_load/test_txt_special_delimiter.csv
b/regression-test/data/load_p0/stream_load/test_txt_special_delimiter.csv
new file mode 100644
index 0000000000..c2f8816941
--- /dev/null
+++ b/regression-test/data/load_p0/stream_load/test_txt_special_delimiter.csv
@@ -0,0 +1 @@
+101030204165996924763600000104020301101030204165996924763600000104020301
\ No newline at end of file
diff --git
a/regression-test/data/load_p0/stream_load/test_txt_special_delimiter.out
b/regression-test/data/load_p0/stream_load/test_txt_special_delimiter.out
new file mode 100644
index 0000000000..864c58075f
--- /dev/null
+++ b/regression-test/data/load_p0/stream_load/test_txt_special_delimiter.out
@@ -0,0 +1,7 @@
+-- This file is automatically generated. You should know what you did if you
want to edit this
+-- !sql --
+1 1659969247636000001
+1 1659969247636000001
+1 1659969247636000001
+1 1659969247636000001
+
diff --git
a/regression-test/suites/load_p0/stream_load/test_txt_special_delimiter.groovy
b/regression-test/suites/load_p0/stream_load/test_txt_special_delimiter.groovy
new file mode 100644
index 0000000000..fff343078b
--- /dev/null
+++
b/regression-test/suites/load_p0/stream_load/test_txt_special_delimiter.groovy
@@ -0,0 +1,58 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+suite("test_txt_special_delimiter", "p0") {
+ sql "show tables"
+
+ def tableName = "test_txt_special_delimiter"
+
+ sql """ DROP TABLE IF EXISTS ${tableName} """
+ sql """
+ CREATE TABLE IF NOT EXISTS ${tableName} (
+ `k1` bigint(20) NULL,
+ `k2` bigint(20) NULL
+ ) ENGINE=OLAP
+ DUPLICATE KEY(`k1`, `k2`)
+ DISTRIBUTED BY HASH(`k1`, `k2`) BUCKETS 3
+ PROPERTIES ("replication_allocation" = "tag.location.default: 1");
+ """
+ for ( i in 0..1 ) {
+ // should be deleted after new_load_scan is ready
+ if (i == 1) {
+ sql """ADMIN SET FRONTEND CONFIG ("enable_new_load_scan_node" =
"false");"""
+ } else {
+ sql """ADMIN SET FRONTEND CONFIG ("enable_new_load_scan_node" =
"true");"""
+ }
+
+ // test special_delimiter success
+ streamLoad {
+ table "${tableName}"
+
+ set 'column_separator', '01030204'
+ set 'line_delimiter', '04020301'
+ set 'columns', 'k1, k2'
+ set 'strict_mode', 'true'
+
+ file 'test_txt_special_delimiter.csv'
+ time 10000 // limit inflight 10s
+ }
+
+ sql "sync"
+ }
+ qt_sql "select * from ${tableName} order by k1, k2"
+}
+
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org
For additional commands, e-mail: commits-h...@doris.apache.org
