This is an automated email from the ASF dual-hosted git repository.
yiguolei pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/master by this push:
new 94a8fa6bc9 [bug](function) fix explode_number function return wrong
rows (#23603)
94a8fa6bc9 is described below
commit 94a8fa6bc940636e93e96b19205d76c90a762d42
Author: zhangstar333 <87313068+zhangstar...@users.noreply.github.com>
AuthorDate: Tue Aug 29 19:02:49 2023 +0800
[bug](function) fix explode_number function return wrong rows (#23603)
before the explode_number function result is random with const value.
because the _cur_size is reset, so it's can't insert values to column.
---
be/src/vec/exec/vtable_function_node.h | 4 +-
be/src/vec/exprs/table_function/table_function.h | 6 +-
be/src/vec/exprs/table_function/vexplode.cpp | 2 +-
be/src/vec/exprs/table_function/vexplode.h | 2 +-
.../vec/exprs/table_function/vexplode_bitmap.cpp | 2 +-
be/src/vec/exprs/table_function/vexplode_bitmap.h | 2 +-
.../exprs/table_function/vexplode_json_array.cpp | 2 +-
.../vec/exprs/table_function/vexplode_json_array.h | 2 +-
.../vec/exprs/table_function/vexplode_numbers.cpp | 10 +-
be/src/vec/exprs/table_function/vexplode_numbers.h | 6 +-
be/src/vec/exprs/table_function/vexplode_split.cpp | 2 +-
be/src/vec/exprs/table_function/vexplode_split.h | 2 +-
be/test/vec/function/function_test_util.cpp | 4 +-
.../sql_functions/table_function/explode.out | 229 +++++++++++++++++++++
.../sql_functions/table_function/explode.groovy | 21 ++
15 files changed, 276 insertions(+), 20 deletions(-)
diff --git a/be/src/vec/exec/vtable_function_node.h
b/be/src/vec/exec/vtable_function_node.h
index 040ca3f7af..db88147cf4 100644
--- a/be/src/vec/exec/vtable_function_node.h
+++ b/be/src/vec/exec/vtable_function_node.h
@@ -69,14 +69,14 @@ public:
ExecNode::release_resource(state);
}
- Status push(RuntimeState*, Block* input_block, bool eos) override {
+ Status push(RuntimeState* state, Block* input_block, bool eos) override {
_child_eos = eos;
if (input_block->rows() == 0) {
return Status::OK();
}
for (TableFunction* fn : _fns) {
- RETURN_IF_ERROR(fn->process_init(input_block));
+ RETURN_IF_ERROR(fn->process_init(input_block, state));
}
RETURN_IF_ERROR(_process_next_child_row());
return Status::OK();
diff --git a/be/src/vec/exprs/table_function/table_function.h
b/be/src/vec/exprs/table_function/table_function.h
index 4b31a681b1..7ba6379a20 100644
--- a/be/src/vec/exprs/table_function/table_function.h
+++ b/be/src/vec/exprs/table_function/table_function.h
@@ -36,10 +36,12 @@ public:
virtual Status open() { return Status::OK(); }
- virtual Status process_init(Block* block) = 0;
+ virtual Status process_init(Block* block, RuntimeState* state) = 0;
virtual Status process_row(size_t row_idx) {
- _cur_size = 0;
+ if (!_is_const) {
+ _cur_size = 0;
+ }
return reset();
}
diff --git a/be/src/vec/exprs/table_function/vexplode.cpp
b/be/src/vec/exprs/table_function/vexplode.cpp
index 1340ff74bb..6f13a710e2 100644
--- a/be/src/vec/exprs/table_function/vexplode.cpp
+++ b/be/src/vec/exprs/table_function/vexplode.cpp
@@ -36,7 +36,7 @@ VExplodeTableFunction::VExplodeTableFunction() {
_fn_name = "vexplode";
}
-Status VExplodeTableFunction::process_init(Block* block) {
+Status VExplodeTableFunction::process_init(Block* block, RuntimeState* state) {
CHECK(_expr_context->root()->children().size() == 1)
<< "VExplodeTableFunction only support 1 child but has "
<< _expr_context->root()->children().size();
diff --git a/be/src/vec/exprs/table_function/vexplode.h
b/be/src/vec/exprs/table_function/vexplode.h
index 26771bd2b2..2da97d11a4 100644
--- a/be/src/vec/exprs/table_function/vexplode.h
+++ b/be/src/vec/exprs/table_function/vexplode.h
@@ -40,7 +40,7 @@ public:
~VExplodeTableFunction() override = default;
- Status process_init(Block* block) override;
+ Status process_init(Block* block, RuntimeState* state) override;
Status process_row(size_t row_idx) override;
Status process_close() override;
void get_value(MutableColumnPtr& column) override;
diff --git a/be/src/vec/exprs/table_function/vexplode_bitmap.cpp
b/be/src/vec/exprs/table_function/vexplode_bitmap.cpp
index 152566d00b..9ce7771454 100644
--- a/be/src/vec/exprs/table_function/vexplode_bitmap.cpp
+++ b/be/src/vec/exprs/table_function/vexplode_bitmap.cpp
@@ -40,7 +40,7 @@ VExplodeBitmapTableFunction::VExplodeBitmapTableFunction() {
_fn_name = "vexplode_bitmap";
}
-Status VExplodeBitmapTableFunction::process_init(Block* block) {
+Status VExplodeBitmapTableFunction::process_init(Block* block, RuntimeState*
state) {
CHECK(_expr_context->root()->children().size() == 1)
<< "VExplodeNumbersTableFunction must be have 1 children but have "
<< _expr_context->root()->children().size();
diff --git a/be/src/vec/exprs/table_function/vexplode_bitmap.h
b/be/src/vec/exprs/table_function/vexplode_bitmap.h
index 0dfde605de..18cdc6bb22 100644
--- a/be/src/vec/exprs/table_function/vexplode_bitmap.h
+++ b/be/src/vec/exprs/table_function/vexplode_bitmap.h
@@ -45,7 +45,7 @@ public:
void get_value(MutableColumnPtr& column) override;
Status forward(int step = 1) override;
- Status process_init(Block* block) override;
+ Status process_init(Block* block, RuntimeState* state) override;
Status process_row(size_t row_idx) override;
Status process_close() override;
diff --git a/be/src/vec/exprs/table_function/vexplode_json_array.cpp
b/be/src/vec/exprs/table_function/vexplode_json_array.cpp
index 840923d018..7c8c48733f 100644
--- a/be/src/vec/exprs/table_function/vexplode_json_array.cpp
+++ b/be/src/vec/exprs/table_function/vexplode_json_array.cpp
@@ -162,7 +162,7 @@
VExplodeJsonArrayTableFunction::VExplodeJsonArrayTableFunction(ExplodeJsonArrayT
_fn_name = "vexplode_json_array";
}
-Status VExplodeJsonArrayTableFunction::process_init(Block* block) {
+Status VExplodeJsonArrayTableFunction::process_init(Block* block,
RuntimeState* state) {
CHECK(_expr_context->root()->children().size() == 1)
<< _expr_context->root()->children().size();
diff --git a/be/src/vec/exprs/table_function/vexplode_json_array.h
b/be/src/vec/exprs/table_function/vexplode_json_array.h
index db1784e5ad..6b5a0f7651 100644
--- a/be/src/vec/exprs/table_function/vexplode_json_array.h
+++ b/be/src/vec/exprs/table_function/vexplode_json_array.h
@@ -111,7 +111,7 @@ public:
VExplodeJsonArrayTableFunction(ExplodeJsonArrayType type);
~VExplodeJsonArrayTableFunction() override = default;
- Status process_init(Block* block) override;
+ Status process_init(Block* block, RuntimeState* state) override;
Status process_row(size_t row_idx) override;
Status process_close() override;
void get_value(MutableColumnPtr& column) override;
diff --git a/be/src/vec/exprs/table_function/vexplode_numbers.cpp
b/be/src/vec/exprs/table_function/vexplode_numbers.cpp
index fe450c8096..8149d4d958 100644
--- a/be/src/vec/exprs/table_function/vexplode_numbers.cpp
+++ b/be/src/vec/exprs/table_function/vexplode_numbers.cpp
@@ -23,6 +23,7 @@
#include <vector>
#include "common/status.h"
+#include "runtime/runtime_state.h"
#include "vec/columns/column.h"
#include "vec/columns/column_const.h"
#include "vec/columns/column_nullable.h"
@@ -40,7 +41,7 @@ VExplodeNumbersTableFunction::VExplodeNumbersTableFunction() {
_fn_name = "vexplode_numbers";
}
-Status VExplodeNumbersTableFunction::process_init(Block* block) {
+Status VExplodeNumbersTableFunction::process_init(Block* block, RuntimeState*
state) {
CHECK(_expr_context->root()->children().size() == 1)
<< "VExplodeSplitTableFunction must be have 1 children but have "
<< _expr_context->root()->children().size();
@@ -61,9 +62,10 @@ Status VExplodeNumbersTableFunction::process_init(Block*
block) {
} else {
_cur_size = column_nested->get_int(0);
}
-
- if (_cur_size && _cur_size <= block->rows()) { // avoid
elements_column too big or empty
- _is_const = true; // use const optimize
+ ((ColumnInt32*)_elements_column.get())->clear();
+ if (_cur_size &&
+ _cur_size <= state->batch_size()) { // avoid elements_column too
big or empty
+ _is_const = true; // use const optimize
for (int i = 0; i < _cur_size; i++) {
((ColumnInt32*)_elements_column.get())->insert_value(i);
}
diff --git a/be/src/vec/exprs/table_function/vexplode_numbers.h
b/be/src/vec/exprs/table_function/vexplode_numbers.h
index f4f86e8d4d..7bdde9278b 100644
--- a/be/src/vec/exprs/table_function/vexplode_numbers.h
+++ b/be/src/vec/exprs/table_function/vexplode_numbers.h
@@ -43,7 +43,7 @@ public:
VExplodeNumbersTableFunction();
~VExplodeNumbersTableFunction() override = default;
- Status process_init(Block* block) override;
+ Status process_init(Block* block, RuntimeState* state) override;
Status process_row(size_t row_idx) override;
Status process_close() override;
void get_value(MutableColumnPtr& column) override;
@@ -53,13 +53,13 @@ public:
if (_is_nullable) {
static_cast<ColumnInt32*>(
static_cast<ColumnNullable*>(column.get())->get_nested_column_ptr().get())
- ->insert_many_from(*_elements_column, _cur_offset,
max_step);
+ ->insert_range_from(*_elements_column, _cur_offset,
max_step);
static_cast<ColumnUInt8*>(
static_cast<ColumnNullable*>(column.get())->get_null_map_column_ptr().get())
->insert_many_defaults(max_step);
} else {
static_cast<ColumnInt32*>(column.get())
- ->insert_many_from(*_elements_column, _cur_offset,
max_step);
+ ->insert_range_from(*_elements_column, _cur_offset,
max_step);
}
forward(max_step);
diff --git a/be/src/vec/exprs/table_function/vexplode_split.cpp
b/be/src/vec/exprs/table_function/vexplode_split.cpp
index c3ae83f9f3..c2a48f456a 100644
--- a/be/src/vec/exprs/table_function/vexplode_split.cpp
+++ b/be/src/vec/exprs/table_function/vexplode_split.cpp
@@ -43,7 +43,7 @@ Status VExplodeSplitTableFunction::open() {
return Status::OK();
}
-Status VExplodeSplitTableFunction::process_init(Block* block) {
+Status VExplodeSplitTableFunction::process_init(Block* block, RuntimeState*
state) {
CHECK(_expr_context->root()->children().size() == 2)
<< "VExplodeSplitTableFunction must be have 2 children but have "
<< _expr_context->root()->children().size();
diff --git a/be/src/vec/exprs/table_function/vexplode_split.h
b/be/src/vec/exprs/table_function/vexplode_split.h
index 1155090bb1..629e2259b6 100644
--- a/be/src/vec/exprs/table_function/vexplode_split.h
+++ b/be/src/vec/exprs/table_function/vexplode_split.h
@@ -45,7 +45,7 @@ public:
~VExplodeSplitTableFunction() override = default;
Status open() override;
- Status process_init(Block* block) override;
+ Status process_init(Block* block, RuntimeState* state) override;
Status process_row(size_t row_idx) override;
Status process_close() override;
void get_value(MutableColumnPtr& column) override;
diff --git a/be/test/vec/function/function_test_util.cpp
b/be/test/vec/function/function_test_util.cpp
index f75be08910..c61d272cca 100644
--- a/be/test/vec/function/function_test_util.cpp
+++ b/be/test/vec/function/function_test_util.cpp
@@ -23,6 +23,7 @@
#include <iostream>
#include "runtime/jsonb_value.h"
+#include "runtime/runtime_state.h"
#include "util/binary_cast.hpp"
#include "util/bitmap_value.h"
#include "vec/data_types/data_type_array.h"
@@ -341,8 +342,9 @@ Block* process_table_function(TableFunction* fn, Block*
input_block,
return nullptr;
}
+ RuntimeState runtime_state((TQueryGlobals()));
// process table function init
- if (fn->process_init(input_block) != Status::OK()) {
+ if (fn->process_init(input_block, &runtime_state) != Status::OK()) {
LOG(WARNING) << "TableFunction process_init failed";
return nullptr;
}
diff --git
a/regression-test/data/query_p0/sql_functions/table_function/explode.out
b/regression-test/data/query_p0/sql_functions/table_function/explode.out
index c3e327625a..2c06dc5d81 100644
--- a/regression-test/data/query_p0/sql_functions/table_function/explode.out
+++ b/regression-test/data/query_p0/sql_functions/table_function/explode.out
@@ -46,3 +46,232 @@
3 1
4 1
+-- !test4 --
+1 0
+1 1
+1 2
+1 3
+1 4
+2 0
+2 1
+2 2
+2 3
+2 4
+3 0
+3 1
+3 2
+3 3
+3 4
+4 0
+4 1
+4 2
+4 3
+4 4
+5 0
+5 1
+5 2
+5 3
+5 4
+6 0
+6 1
+6 2
+6 3
+6 4
+7 0
+7 1
+7 2
+7 3
+7 4
+8 0
+8 1
+8 2
+8 3
+8 4
+9 0
+9 1
+9 2
+9 3
+9 4
+10 0
+10 1
+10 2
+10 3
+10 4
+11 0
+11 1
+11 2
+11 3
+11 4
+12 0
+12 1
+12 2
+12 3
+12 4
+13 0
+13 1
+13 2
+13 3
+13 4
+14 0
+14 1
+14 2
+14 3
+14 4
+15 0
+15 1
+15 2
+15 3
+15 4
+
+-- !test5 --
+1 0
+1 1
+1 2
+1 3
+1 4
+1 5
+1 6
+1 7
+1 8
+1 9
+2 0
+2 1
+2 2
+2 3
+2 4
+2 5
+2 6
+2 7
+2 8
+2 9
+3 0
+3 1
+3 2
+3 3
+3 4
+3 5
+3 6
+3 7
+3 8
+3 9
+4 0
+4 1
+4 2
+4 3
+4 4
+4 5
+4 6
+4 7
+4 8
+4 9
+5 0
+5 1
+5 2
+5 3
+5 4
+5 5
+5 6
+5 7
+5 8
+5 9
+6 0
+6 1
+6 2
+6 3
+6 4
+6 5
+6 6
+6 7
+6 8
+6 9
+7 0
+7 1
+7 2
+7 3
+7 4
+7 5
+7 6
+7 7
+7 8
+7 9
+8 0
+8 1
+8 2
+8 3
+8 4
+8 5
+8 6
+8 7
+8 8
+8 9
+9 0
+9 1
+9 2
+9 3
+9 4
+9 5
+9 6
+9 7
+9 8
+9 9
+10 0
+10 1
+10 2
+10 3
+10 4
+10 5
+10 6
+10 7
+10 8
+10 9
+11 0
+11 1
+11 2
+11 3
+11 4
+11 5
+11 6
+11 7
+11 8
+11 9
+12 0
+12 1
+12 2
+12 3
+12 4
+12 5
+12 6
+12 7
+12 8
+12 9
+13 0
+13 1
+13 2
+13 3
+13 4
+13 5
+13 6
+13 7
+13 8
+13 9
+14 0
+14 1
+14 2
+14 3
+14 4
+14 5
+14 6
+14 7
+14 8
+14 9
+15 0
+15 1
+15 2
+15 3
+15 4
+15 5
+15 6
+15 7
+15 8
+15 9
+
diff --git
a/regression-test/suites/query_p0/sql_functions/table_function/explode.groovy
b/regression-test/suites/query_p0/sql_functions/table_function/explode.groovy
index 7118ebf43b..fabb847189 100644
---
a/regression-test/suites/query_p0/sql_functions/table_function/explode.groovy
+++
b/regression-test/suites/query_p0/sql_functions/table_function/explode.groovy
@@ -49,4 +49,25 @@ suite("explode") {
qt_test1 """select e1 from (select k1 from d_table) as t lateral view
explode_numbers(5) tmp1 as e1;"""
qt_test2 """select e1 from (select k1 from d_table) as t lateral view
explode_numbers(5) tmp1 as e1 where e1=k1;"""
qt_test3 """select e1,k1 from (select k1 from d_table) as t lateral view
explode_numbers(5) tmp1 as e1;"""
+
+ sql """ DROP TABLE IF EXISTS baseall_explode_numbers; """
+ sql """
+ CREATE TABLE `baseall_explode_numbers` (
+ `k3` int(11) NULL
+ ) ENGINE=OLAP
+ duplicate KEY(`k3`)
+ COMMENT 'OLAP'
+ DISTRIBUTED BY HASH(`k3`) BUCKETS 5
+ PROPERTIES (
+ "replication_allocation" = "tag.location.default: 1",
+ "is_being_synced" = "false",
+ "storage_format" = "V2",
+ "light_schema_change" = "true",
+ "disable_auto_compaction" = "false",
+ "enable_single_replica_compaction" = "false"
+ );
+ """
+ sql "insert into baseall_explode_numbers
values(1),(2),(3),(4),(5),(6),(7),(8),(9),(10),(11),(12),(13),(14),(15);"
+ qt_test4 """select k3,e from baseall_explode_numbers as U lateral view
explode_numbers(5) tmp1 as e order by k3,e;"""
+ qt_test5 """select k3,e from baseall_explode_numbers as U lateral view
explode_numbers(10) tmp1 as e order by k3,e;"""
}
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org
For additional commands, e-mail: commits-h...@doris.apache.org
