This is an automated email from the ASF dual-hosted git repository.
eldenmoon pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/master by this push:
new 1acbbbdd8e8 [improve](ip)improve data type ip for rowstore and
trycatch in segment_flusher (#39258)
1acbbbdd8e8 is described below
commit 1acbbbdd8e89f319914ecf835f596653a565b371
Author: amory <wangqian...@selectdb.com>
AuthorDate: Wed Aug 14 09:57:02 2024 +0800
[improve](ip)improve data type ip for rowstore and trycatch in
segment_flusher (#39258)
before we not support ipv6 with rowstore, and can make coredump like
this:
```
terminate called after throwing an instance of 'doris::Exception'
what(): [E3] write_one_cell_to_jsonb with type IPv6
*** Query id: e5e020df4ea04a70-95a27c948db5a518 ***
*** is nereids: 0 ***
*** tablet id: 9008975 ***
*** Aborted at 1723184823 (unix time) try "date -d @1723184823" if you are
using GNU date ***
*** Current BE git commitID: a2baaf3174 ***
*** SIGABRT unknown detail explain (@0x29f794) received by PID 2750356 (TID
2752049 OR 0x7f6064dec640) from PID 2750356; stack trace: ***
0# doris::signal::(anonymous namespace)::FailureSignalHandler(int,
siginfo_t*, void*) at
/home/zcp/repo_center/doris_master/doris/be/src/common/signal_handler.h:421
1# 0x00007F6499586520 in /lib/x86_64-linux-gnu/libc.so.6
2# pthread_kill at ./nptl/pthread_kill.c:89
3# raise at ../sysdeps/posix/raise.c:27
4# abort at ./stdlib/abort.c:81
5# __gnu_cxx::__verbose_terminate_handler() [clone .cold] at
../../../../libstdc++-v3/libsupc++/vterminate.cc:75
6# __cxxabiv1::__terminate(void (*)()) at
../../../../libstdc++-v3/libsupc++/eh_terminate.cc:48
7# 0x00005610407949C1 in /mnt/hdd01/PERFORMANCE_ENV/be/lib/doris_be
8# 0x0000561040794B14 in /mnt/hdd01/PERFORMANCE_ENV/be/lib/doris_be
9# doris::vectorized::DataTypeNumberSerDe<unsigned
__int128>::write_one_cell_to_jsonb(doris::vectorized::IColumn const&,
doris::JsonbWriterT<doris::JsonbOutStream>&, doris::vectorized::Arena*, int,
int) const in /mnt/hdd01/PERFORMANCE_ENV/be/lib/doris_be
10#
doris::vectorized::JsonbSerializeUtil::block_to_jsonb(doris::TabletSchema
const&, doris::vectorized::Block const&, doris::vectorized::ColumnStr<unsigned
int>&, int, std::vector<std::shared_ptr<doris::vectorized::DataTypeSerDe>,
std::allocator<std::shared_ptr<doris::vectorized::DataTypeSerDe> > > const&,
std::unordered_set<int, std::hash<int>, std::equal_to<int>, std::allocator<int>
> const&) at
/home/zcp/repo_center/doris_master/doris/be/src/vec/jsonb/serialize.cpp:57
11#
doris::segment_v2::VerticalSegmentWriter::_serialize_block_to_row_column(doris::vectorized::Block&)
at
/home/zcp/repo_center/doris_master/doris/be/src/olap/rowset/segment_v2/vertical_segment_writer.cpp:302
12# doris::segment_v2::VerticalSegmentWriter::write_batch() at
/home/zcp/repo_center/doris_master/doris/be/src/olap/rowset/segment_v2/vertical_segment_writer.cpp:839
13#
doris::SegmentFlusher::_add_rows(std::unique_ptr<doris::segment_v2::VerticalSegmentWriter,
std::default_delete<doris::segment_v2::VerticalSegmentWriter> >&,
doris::vectorized::Block const*, unsigned long, unsigned long) at
/home/zcp/repo_center/doris_master/doris/be/src/olap/rowset/segment_creator.cpp:130
14# doris::SegmentFlusher::flush_single_block(doris::vectorized::Block
const*, int, long*) at
/home/zcp/repo_center/doris_master/doris/be/src/olap/rowset/segment_creator.cpp:73
15# doris::SegmentCreator::flush_single_block(doris::vectorized::Block
const*, int, long*) at
/home/zcp/repo_center/doris_master/doris/be/src/olap/rowset/segment_creator.cpp:395
16# doris::BetaRowsetWriterV2::flush_memtable(doris::vectorized::Block*,
int, long*) at
/home/zcp/repo_center/doris_master/doris/be/src/olap/rowset/beta_rowset_writer_v2.cpp:101
17# doris::FlushToken::_do_flush_memtable(doris::MemTable*, int, long*) at
/home/zcp/repo_center/doris_master/doris/be/src/olap/memtable_flush_executor.cpp:148
18# doris::FlushToken::_flush_memtable(std::unique_ptr<doris::MemTable,
std::default_delete<doris::MemTable> >, int, long) in
/mnt/hdd01/PERFORMANCE_ENV/be/lib/doris_be
19# doris::MemtableFlushTask::run() at
/home/zcp/repo_center/doris_master/doris/be/src/olap/memtable_flush_executor.cpp:63
20# doris::ThreadPool::dispatch_thread() in
/mnt/hdd01/PERFORMANCE_ENV/be/lib/doris_be
21# doris::Thread::supervise_thread(void*) at
/home/zcp/repo_center/doris_master/doris/be/src/util/thread.cpp:499
22# start_thread at ./nptl/pthread_create.c:442
23# 0x00007F649966A850 at ../sysdeps/unix/sysv/linux/x86_64/clone3.S:83
```
---
be/src/olap/rowset/segment_creator.cpp | 5 +++--
.../vec/data_types/serde/data_type_ipv6_serde.cpp | 22 ++++++++++++++++++++++
be/src/vec/data_types/serde/data_type_ipv6_serde.h | 3 +++
.../data/mv_p0/test_row_store/test_row_store.out | 4 ++++
.../mv_p0/test_row_store/test_row_store.groovy | 19 +++++++++++++++++++
5 files changed, 51 insertions(+), 2 deletions(-)
diff --git a/be/src/olap/rowset/segment_creator.cpp
b/be/src/olap/rowset/segment_creator.cpp
index 40f6e8303fe..5e5dfc3733f 100644
--- a/be/src/olap/rowset/segment_creator.cpp
+++ b/be/src/olap/rowset/segment_creator.cpp
@@ -27,6 +27,7 @@
#include "common/compiler_util.h" // IWYU pragma: keep
#include "common/config.h"
+#include "common/exception.h"
#include "common/logging.h"
#include "common/status.h"
#include "io/fs/file_writer.h"
@@ -70,12 +71,12 @@ Status SegmentFlusher::flush_single_block(const
vectorized::Block* block, int32_
_context.tablet_schema->cluster_key_idxes().empty()) {
std::unique_ptr<segment_v2::VerticalSegmentWriter> writer;
RETURN_IF_ERROR(_create_segment_writer(writer, segment_id,
no_compression));
- RETURN_IF_ERROR(_add_rows(writer, &flush_block, 0,
flush_block.rows()));
+ RETURN_IF_ERROR_OR_CATCH_EXCEPTION(_add_rows(writer, &flush_block, 0,
flush_block.rows()));
RETURN_IF_ERROR(_flush_segment_writer(writer, writer->flush_schema(),
flush_size));
} else {
std::unique_ptr<segment_v2::SegmentWriter> writer;
RETURN_IF_ERROR(_create_segment_writer(writer, segment_id,
no_compression));
- RETURN_IF_ERROR(_add_rows(writer, &flush_block, 0,
flush_block.rows()));
+ RETURN_IF_ERROR_OR_CATCH_EXCEPTION(_add_rows(writer, &flush_block, 0,
flush_block.rows()));
RETURN_IF_ERROR(_flush_segment_writer(writer, writer->flush_schema(),
flush_size));
}
return Status::OK();
diff --git a/be/src/vec/data_types/serde/data_type_ipv6_serde.cpp
b/be/src/vec/data_types/serde/data_type_ipv6_serde.cpp
index 88251a5d5d3..f09b6feb4a2 100644
--- a/be/src/vec/data_types/serde/data_type_ipv6_serde.cpp
+++ b/be/src/vec/data_types/serde/data_type_ipv6_serde.cpp
@@ -68,6 +68,28 @@ Status DataTypeIPv6SerDe::write_column_to_mysql(const
IColumn& column,
return _write_column_to_mysql(column, row_buffer, row_idx, col_const,
options);
}
+void DataTypeIPv6SerDe::read_one_cell_from_jsonb(IColumn& column, const
JsonbValue* arg) const {
+ IPv6 val = 0;
+ const auto* str_value = static_cast<const JsonbStringVal*>(arg);
+ ReadBuffer rb(reinterpret_cast<const unsigned char*>(str_value->getBlob()),
+ str_value->getBlobLen());
+ if (!read_ipv6_text_impl(val, rb)) {
+ throw doris::Exception(ErrorCode::INVALID_ARGUMENT, "parse ipv6 fail,
string: '{}'",
+ rb.to_string());
+ }
+ assert_cast<ColumnIPv6&>(column).insert_value(val);
+}
+
+void DataTypeIPv6SerDe::write_one_cell_to_jsonb(const IColumn& column,
+ JsonbWriterT<JsonbOutStream>&
result,
+ Arena* mem_pool, int col_id,
int row_num) const {
+ // we make ipv6 as string in jsonb
+ result.writeKey(col_id);
+ IPv6 data = assert_cast<const ColumnIPv6&>(column).get_element(row_num);
+ IPv6Value ipv6_value(data);
+ result.writeString(ipv6_value.to_string());
+}
+
Status DataTypeIPv6SerDe::serialize_one_cell_to_json(const IColumn& column,
int row_num,
BufferWritable& bw,
FormatOptions& options)
const {
diff --git a/be/src/vec/data_types/serde/data_type_ipv6_serde.h
b/be/src/vec/data_types/serde/data_type_ipv6_serde.h
index 71f86f383b4..d308da41859 100644
--- a/be/src/vec/data_types/serde/data_type_ipv6_serde.h
+++ b/be/src/vec/data_types/serde/data_type_ipv6_serde.h
@@ -67,6 +67,9 @@ public:
const cctz::time_zone& ctz) const override;
void read_column_from_arrow(IColumn& column, const arrow::Array*
arrow_array, int start,
int end, const cctz::time_zone& ctz) const
override;
+ void read_one_cell_from_jsonb(IColumn& column, const JsonbValue* arg)
const override;
+ void write_one_cell_to_jsonb(const IColumn& column,
JsonbWriterT<JsonbOutStream>& result,
+ Arena* mem_pool, int unique_id, int row_num)
const override;
private:
template <bool is_binary_format>
diff --git a/regression-test/data/mv_p0/test_row_store/test_row_store.out
b/regression-test/data/mv_p0/test_row_store/test_row_store.out
new file mode 100644
index 00000000000..42f6adba1f1
--- /dev/null
+++ b/regression-test/data/mv_p0/test_row_store/test_row_store.out
@@ -0,0 +1,4 @@
+-- This file is automatically generated. You should know what you did if you
want to edit this
+-- !sql --
+1 :: 0.0.0.0
+
diff --git a/regression-test/suites/mv_p0/test_row_store/test_row_store.groovy
b/regression-test/suites/mv_p0/test_row_store/test_row_store.groovy
index 99cc3ba227e..b9ee72c5053 100644
--- a/regression-test/suites/mv_p0/test_row_store/test_row_store.groovy
+++ b/regression-test/suites/mv_p0/test_row_store/test_row_store.groovy
@@ -29,4 +29,23 @@ suite ("test_row_store") {
sql "create materialized view kavg as select k1,k4,avg(k2) from
d_table group by k1,k4;"
exception "RowStore table can't create materialized view"
}
+
+ // test ip type with row store
+ sql """ DROP TABLE IF EXISTS ipv_row; """
+ sql """ CREATE TABLE ipv_row (
+ `id` int,
+ `ip_v6` ipv6,
+ `ip_v4` ipv4
+ ) ENGINE=OLAP
+ DISTRIBUTED BY HASH(`id`) BUCKETS 4
+ PROPERTIES (
+ "store_row_column"="true",
+ "replication_allocation" = "tag.location.default: 1"
+ ); """
+
+ // rowstore column can't make inde
+ sql """ insert into ipv_row values(2, '2001:16a0:2:200a::2',
'192.1.2.0');"""
+ sql """ insert into ipv_row values(1, '::', '0.0.0.0');"""
+ qt_sql """select * from ipv_row where id = 1;"""
+
}
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org
For additional commands, e-mail: commits-h...@doris.apache.org
