This is an automated email from the ASF dual-hosted git repository.
panxiaolei pushed a commit to branch branch-2.1
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/branch-2.1 by this push:
new 43c646363ef [Bug](runtime-filter) support ip rf and use exception to
replace dche… (#41531)
43c646363ef is described below
commit 43c646363ef353fbef3c6fc6bd6929b348ba1bab
Author: Pxl <x...@selectdb.com>
AuthorDate: Mon Dec 30 20:56:11 2024 +0800
[Bug](runtime-filter) support ip rf and use exception to replace dche…
(#41531)
…ck when PrimitiveType to PColumnType (#39985)
use exception to replace dcheck when PrimitiveType to PColumnType
```cpp
*** SIGABRT unknown detail explain (@0x11d3f) received by PID 73023 (TID
74292 OR 0x7fd758225640) from PID 73023; stack trace: ***
0# doris::signal::(anonymous namespace)::FailureSignalHandler(int,
siginfo_t*, void*) at
/home/zcp/repo_center/doris_master/doris/be/src/common/signal_handler.h:421
1# 0x00007FDDBE6B9520 in /lib/x86_64-linux-gnu/libc.so.6
2# pthread_kill at ./nptl/pthread_kill.c:89
3# raise at ../sysdeps/posix/raise.c:27
4# abort at ./stdlib/abort.c:81
5# 0x000056123F81A94D in /root/output/be/lib/doris_be
6# 0x000056123F80CF8A in /root/output/be/lib/doris_be
7# google::LogMessage::SendToLog() in /root/output/be/lib/doris_be
8# google::LogMessage::Flush() in /root/output/be/lib/doris_be
9# google::LogMessageFatal::~LogMessageFatal() in
/root/output/be/lib/doris_be
10# doris::to_proto(doris::PrimitiveType) at
/home/zcp/repo_center/doris_master/doris/be/src/exprs/runtime_filter.cpp:114
11# doris::IRuntimeFilter::push_to_remote(doris::TNetworkAddress const*) at
/home/zcp/repo_center/doris_master/doris/be/src/exprs/runtime_filter.cpp:1143
12#
doris::IRuntimeFilter::publish(bool)::$_0::operator()(doris::IRuntimeFilter*)
const at
/home/zcp/repo_center/doris_master/doris/be/src/exprs/runtime_filter.cpp:959
13# doris::IRuntimeFilter::publish(bool)::$_2::operator()() const at
/home/zcp/repo_center/doris_master/doris/be/src/exprs/runtime_filter.cpp:983
14# doris::IRuntimeFilter::publish(bool) at
/home/zcp/repo_center/doris_master/doris/be/src/exprs/runtime_filter.cpp:997
```
## Proposed changes
pick from #39985
---
be/src/exprs/runtime_filter.cpp | 110 ++++++++++++++++-----
be/src/olap/predicate_creator.h | 4 +-
be/src/runtime/large_int_value.h | 5 +
be/src/runtime/primitive_type.h | 4 +-
be/src/vec/core/types.h | 6 +-
be/src/vec/data_types/data_type_ipv6.cpp | 2 +-
be/src/vec/data_types/data_type_ipv6.h | 2 +-
be/src/vec/exprs/vexpr.cpp | 13 ++-
be/src/vec/exprs/vexpr.h | 16 +++
be/src/vec/olap/olap_data_convertor.cpp | 4 +-
be/src/vec/runtime/ipv4_value.h | 18 ++--
be/src/vec/runtime/ipv6_value.h | 18 ++--
.../serde/data_type_serde_mysql_test.cpp | 4 +-
be/test/vec/runtime/ip_value_test.cpp | 16 +--
gensrc/proto/internal_service.proto | 6 +-
.../data/datatype_p0/ip/ip_rf/test_ip_rf.out | 31 ++++++
.../suites/datatype_p0/ip/ip_rf/test_ip_rf.groovy | 73 ++++++++++++++
17 files changed, 266 insertions(+), 66 deletions(-)
diff --git a/be/src/exprs/runtime_filter.cpp b/be/src/exprs/runtime_filter.cpp
index d7bef524f01..aa0d69ae5f0 100644
--- a/be/src/exprs/runtime_filter.cpp
+++ b/be/src/exprs/runtime_filter.cpp
@@ -110,11 +110,14 @@ PColumnType to_proto(PrimitiveType type) {
return PColumnType::COLUMN_TYPE_VARCHAR;
case TYPE_STRING:
return PColumnType::COLUMN_TYPE_STRING;
+ case TYPE_IPV4:
+ return PColumnType::COLUMN_TYPE_IPV4;
+ case TYPE_IPV6:
+ return PColumnType::COLUMN_TYPE_IPV6;
default:
- DCHECK(false) << "Invalid type.";
+ throw Exception(ErrorCode::INTERNAL_ERROR,
+ "runtime filter meet invalid PrimitiveType type {}",
int(type));
}
- DCHECK(false);
- return PColumnType::COLUMN_TYPE_INT;
}
// PColumnType->PrimitiveType
@@ -160,10 +163,14 @@ PrimitiveType to_primitive_type(PColumnType type) {
return TYPE_CHAR;
case PColumnType::COLUMN_TYPE_STRING:
return TYPE_STRING;
+ case PColumnType::COLUMN_TYPE_IPV4:
+ return TYPE_IPV4;
+ case PColumnType::COLUMN_TYPE_IPV6:
+ return TYPE_IPV6;
default:
- DCHECK(false);
+ throw Exception(ErrorCode::INTERNAL_ERROR,
+ "runtime filter meet invalid PColumnType type {}",
int(type));
}
- return TYPE_INT;
}
// PFilterType -> RuntimeFilterType
@@ -559,14 +566,13 @@ public:
}
Status assign(const PInFilter* in_filter, bool contain_null) {
- PrimitiveType type = to_primitive_type(in_filter->column_type());
- _context->hybrid_set.reset(create_set(type));
+ _context->hybrid_set.reset(create_set(_column_return_type));
if (contain_null) {
_context->hybrid_set->set_null_aware(true);
_context->hybrid_set->insert((const void*)nullptr);
}
- switch (type) {
+ switch (_column_return_type) {
case TYPE_BOOLEAN: {
batch_assign(in_filter, [](std::shared_ptr<HybridSetBase>& set,
PColumnValue& column,
ObjectPool* pool) {
@@ -723,9 +729,29 @@ public:
});
break;
}
+ case TYPE_IPV4: {
+ batch_assign(in_filter, [](std::shared_ptr<HybridSetBase>& set,
PColumnValue& column,
+ ObjectPool* pool) {
+ int32_t tmp = column.intval();
+ set->insert(&tmp);
+ });
+ break;
+ }
+ case TYPE_IPV6: {
+ batch_assign(in_filter, [](std::shared_ptr<HybridSetBase>& set,
PColumnValue& column,
+ ObjectPool* pool) {
+ auto string_val = column.stringval();
+ StringParser::ParseResult result;
+ auto int128_val = StringParser::string_to_int<uint128_t>(
+ string_val.c_str(), string_val.length(), &result);
+ DCHECK(result == StringParser::PARSE_SUCCESS);
+ set->insert(&int128_val);
+ });
+ break;
+ }
default: {
return Status::InternalError("not support assign to in filter,
type: " +
- type_to_string(type));
+ type_to_string(_column_return_type));
}
}
return Status::OK();
@@ -748,15 +774,14 @@ public:
// used by shuffle runtime filter
// assign this filter by protobuf
Status assign(const PMinMaxFilter* minmax_filter, bool contain_null) {
- PrimitiveType type = to_primitive_type(minmax_filter->column_type());
- _context->minmax_func.reset(create_minmax_filter(type));
+ _context->minmax_func.reset(create_minmax_filter(_column_return_type));
if (contain_null) {
_context->minmax_func->set_null_aware(true);
_context->minmax_func->set_contain_null();
}
- switch (type) {
+ switch (_column_return_type) {
case TYPE_BOOLEAN: {
bool min_val = minmax_filter->min_val().boolval();
bool max_val = minmax_filter->max_val().boolval();
@@ -876,6 +901,23 @@ public:
StringRef max_val(max_val_ptr->c_str(), max_val_ptr->length());
return _context->minmax_func->assign(&min_val, &max_val);
}
+ case TYPE_IPV4: {
+ int tmp_min = minmax_filter->min_val().intval();
+ int tmp_max = minmax_filter->max_val().intval();
+ return _context->minmax_func->assign(&tmp_min, &tmp_max);
+ }
+ case TYPE_IPV6: {
+ auto min_string_val = minmax_filter->min_val().stringval();
+ auto max_string_val = minmax_filter->max_val().stringval();
+ StringParser::ParseResult result;
+ auto min_val =
StringParser::string_to_int<uint128_t>(min_string_val.c_str(),
+
min_string_val.length(), &result);
+ DCHECK(result == StringParser::PARSE_SUCCESS);
+ auto max_val =
StringParser::string_to_int<uint128_t>(max_string_val.c_str(),
+
max_string_val.length(), &result);
+ DCHECK(result == StringParser::PARSE_SUCCESS);
+ return _context->minmax_func->assign(&min_val, &max_val);
+ }
default:
break;
}
@@ -1168,7 +1210,7 @@ Status IRuntimeFilter::push_to_remote(const
TNetworkAddress* addr, bool opt_remo
merge_filter_request->set_opt_remote_rf(opt_remote_rf);
merge_filter_request->set_is_pipeline(_state->enable_pipeline_exec);
auto column_type = _wrapper->column_type();
- merge_filter_request->set_column_type(to_proto(column_type));
+
RETURN_IF_CATCH_EXCEPTION(merge_filter_request->set_column_type(to_proto(column_type)));
merge_filter_callback->cntl_->set_timeout_ms(wait_time_ms());
merge_filter_callback->cntl_->ignore_eovercrowded();
@@ -1530,13 +1572,10 @@ template <class T>
Status IRuntimeFilter::_create_wrapper(const T* param, ObjectPool* pool,
std::unique_ptr<RuntimePredicateWrapper>* wrapper) {
int filter_type = param->request->filter_type();
- PrimitiveType column_type = PrimitiveType::INVALID_TYPE;
- if (param->request->has_in_filter()) {
- column_type =
to_primitive_type(param->request->in_filter().column_type());
- }
- if (param->request->has_column_type()) {
- column_type = to_primitive_type(param->request->column_type());
+ if (!param->request->has_column_type()) {
+ return Status::InternalError("unknown filter column type");
}
+ PrimitiveType column_type =
to_primitive_type(param->request->column_type());
*wrapper = std::make_unique<RuntimePredicateWrapper>(pool, column_type,
get_type(filter_type),
param->request->filter_id());
@@ -1754,9 +1793,21 @@ void IRuntimeFilter::to_protobuf(PInFilter* filter) {
});
return;
}
+ case TYPE_IPV4: {
+ batch_copy<IPv4>(filter, it, [](PColumnValue* column, const IPv4*
value) {
+ column->set_intval(*reinterpret_cast<const int32_t*>(value));
+ });
+ return;
+ }
+ case TYPE_IPV6: {
+ batch_copy<IPv6>(filter, it, [](PColumnValue* column, const IPv6*
value) {
+ column->set_stringval(LargeIntValue::to_string(*value));
+ });
+ return;
+ }
default: {
- DCHECK(false) << "unknown type";
- break;
+ throw Exception(ErrorCode::INTERNAL_ERROR,
+ "runtime filter meet invalid PrimitiveType type {}",
int(column_type));
}
}
}
@@ -1872,9 +1923,22 @@ void IRuntimeFilter::to_protobuf(PMinMaxFilter* filter) {
std::string(max_string_value->data, max_string_value->size));
break;
}
+ case TYPE_IPV4: {
+ filter->mutable_min_val()->set_intval(*reinterpret_cast<const
int32_t*>(min_data));
+ filter->mutable_max_val()->set_intval(*reinterpret_cast<const
int32_t*>(max_data));
+ return;
+ }
+ case TYPE_IPV6: {
+ filter->mutable_min_val()->set_stringval(
+ LargeIntValue::to_string(*reinterpret_cast<const
uint128_t*>(min_data)));
+ filter->mutable_max_val()->set_stringval(
+ LargeIntValue::to_string(*reinterpret_cast<const
uint128_t*>(max_data)));
+ return;
+ }
default: {
- DCHECK(false) << "unknown type";
- break;
+ throw Exception(ErrorCode::INTERNAL_ERROR,
+ "runtime filter meet invalid PrimitiveType type {}",
+ int(_wrapper->column_type()));
}
}
}
diff --git a/be/src/olap/predicate_creator.h b/be/src/olap/predicate_creator.h
index c47a1694f17..1020e985248 100644
--- a/be/src/olap/predicate_creator.h
+++ b/be/src/olap/predicate_creator.h
@@ -242,7 +242,7 @@ std::unique_ptr<PredicateCreator<ConditionType>>
get_creator(const FieldType& ty
case FieldType::OLAP_FIELD_TYPE_IPV4: {
return std::make_unique<CustomPredicateCreator<TYPE_IPV4, PT,
ConditionType>>(
[](const std::string& condition) {
- vectorized::IPv4 value;
+ IPv4 value;
bool res = IPv4Value::from_string(value, condition);
DCHECK(res);
return value;
@@ -251,7 +251,7 @@ std::unique_ptr<PredicateCreator<ConditionType>>
get_creator(const FieldType& ty
case FieldType::OLAP_FIELD_TYPE_IPV6: {
return std::make_unique<CustomPredicateCreator<TYPE_IPV6, PT,
ConditionType>>(
[](const std::string& condition) {
- vectorized::IPv6 value;
+ IPv6 value;
bool res = IPv6Value::from_string(value, condition);
DCHECK(res);
return value;
diff --git a/be/src/runtime/large_int_value.h b/be/src/runtime/large_int_value.h
index 7bc89ea765c..fbe37ac1f40 100644
--- a/be/src/runtime/large_int_value.h
+++ b/be/src/runtime/large_int_value.h
@@ -24,6 +24,8 @@
#include <iostream>
#include <string>
+#include "olap/olap_common.h"
+
namespace doris {
inline const __int128 MAX_INT128 = ~((__int128)0x01 << 127);
@@ -36,6 +38,9 @@ public:
}
static std::string to_string(__int128 value) { return
fmt::format(FMT_COMPILE("{}"), value); }
+ static std::string to_string(__uint128_t value) {
+ return fmt::format(FMT_COMPILE("{}"), value);
+ }
};
std::ostream& operator<<(std::ostream& os, __int128 const& value);
diff --git a/be/src/runtime/primitive_type.h b/be/src/runtime/primitive_type.h
index 59c0d91e432..fb7c6e317f1 100644
--- a/be/src/runtime/primitive_type.h
+++ b/be/src/runtime/primitive_type.h
@@ -243,13 +243,13 @@ struct PrimitiveTypeTraits<TYPE_LARGEINT> {
};
template <>
struct PrimitiveTypeTraits<TYPE_IPV4> {
- using CppType = vectorized::IPv4;
+ using CppType = IPv4;
using StorageFieldType = CppType;
using ColumnType = vectorized::ColumnIPv4;
};
template <>
struct PrimitiveTypeTraits<TYPE_IPV6> {
- using CppType = vectorized::IPv6;
+ using CppType = IPv6;
using StorageFieldType = CppType;
using ColumnType = vectorized::ColumnIPv6;
};
diff --git a/be/src/vec/core/types.h b/be/src/vec/core/types.h
index 73025b166e5..f5943fa6e1d 100644
--- a/be/src/vec/core/types.h
+++ b/be/src/vec/core/types.h
@@ -42,6 +42,9 @@ struct decimal12_t;
struct uint24_t;
struct StringRef;
+using IPv4 = uint32_t;
+using IPv6 = uint128_t;
+
namespace vectorized {
/// Data types for representing elementary values from a database in RAM.
@@ -296,9 +299,6 @@ struct TypeId<String> {
/// Not a data type in database, defined just for convenience.
using Strings = std::vector<String>;
-using IPv4 = uint32_t;
-using IPv6 = uint128_t;
-
template <>
inline constexpr bool IsNumber<IPv6> = true;
template <>
diff --git a/be/src/vec/data_types/data_type_ipv6.cpp
b/be/src/vec/data_types/data_type_ipv6.cpp
index 11b8d250071..e74564eb3f8 100755
--- a/be/src/vec/data_types/data_type_ipv6.cpp
+++ b/be/src/vec/data_types/data_type_ipv6.cpp
@@ -42,7 +42,7 @@ std::string DataTypeIPv6::to_string(const IColumn& column,
size_t row_num) const
return value.to_string();
}
-std::string DataTypeIPv6::to_string(const IPv6& ipv6_val) const {
+std::string DataTypeIPv6::to_string(const IPv6& ipv6_val) {
auto value = IPv6Value(ipv6_val);
return value.to_string();
}
diff --git a/be/src/vec/data_types/data_type_ipv6.h
b/be/src/vec/data_types/data_type_ipv6.h
index 34c97c4a1e9..93b3c93847b 100755
--- a/be/src/vec/data_types/data_type_ipv6.h
+++ b/be/src/vec/data_types/data_type_ipv6.h
@@ -57,7 +57,7 @@ public:
bool equals(const IDataType& rhs) const override;
std::string to_string(const IColumn& column, size_t row_num) const
override;
void to_string(const IColumn& column, size_t row_num, BufferWritable&
ostr) const override;
- std::string to_string(const IPv6& value) const;
+ static std::string to_string(const IPv6& value);
Status from_string(ReadBuffer& rb, IColumn* column) const override;
Field get_field(const TExprNode& node) const override {
diff --git a/be/src/vec/exprs/vexpr.cpp b/be/src/vec/exprs/vexpr.cpp
index 6f4ac281c5a..0035b2a292f 100644
--- a/be/src/vec/exprs/vexpr.cpp
+++ b/be/src/vec/exprs/vexpr.cpp
@@ -31,6 +31,7 @@
#include "common/config.h"
#include "common/exception.h"
#include "common/status.h"
+#include "runtime/define_primitive_type.h"
#include "vec/columns/column_vector.h"
#include "vec/columns/columns_number.h"
#include "vec/data_types/data_type_array.h"
@@ -147,9 +148,17 @@ TExprNode create_texpr_node_from(const void* data, const
PrimitiveType& type, in
THROW_IF_ERROR(create_texpr_literal_node<TYPE_STRING>(data, &node));
break;
}
+ case TYPE_IPV4: {
+ THROW_IF_ERROR(create_texpr_literal_node<TYPE_IPV4>(data, &node));
+ break;
+ }
+ case TYPE_IPV6: {
+ THROW_IF_ERROR(create_texpr_literal_node<TYPE_IPV6>(data, &node));
+ break;
+ }
default:
- DCHECK(false);
- throw std::invalid_argument("Invalid type!");
+ throw Exception(ErrorCode::INTERNAL_ERROR, "runtime filter meet
invalid type {}",
+ int(type));
}
return node;
}
diff --git a/be/src/vec/exprs/vexpr.h b/be/src/vec/exprs/vexpr.h
index 51f3c2b2523..953780aac8b 100644
--- a/be/src/vec/exprs/vexpr.h
+++ b/be/src/vec/exprs/vexpr.h
@@ -38,8 +38,10 @@
#include "vec/columns/column.h"
#include "vec/core/block.h"
#include "vec/core/column_with_type_and_name.h"
+#include "vec/core/types.h"
#include "vec/core/wide_integer.h"
#include "vec/data_types/data_type.h"
+#include "vec/data_types/data_type_ipv6.h"
#include "vec/exprs/vexpr_fwd.h"
#include "vec/functions/function.h"
@@ -456,6 +458,20 @@ Status create_texpr_literal_node(const void* data,
TExprNode* node, int precisio
string_literal.__set_value(origin_value->to_string());
(*node).__set_string_literal(string_literal);
(*node).__set_type(create_type_desc(PrimitiveType::TYPE_STRING));
+ } else if constexpr (T == TYPE_IPV4) {
+ const auto* origin_value = reinterpret_cast<const IPv4*>(data);
+ (*node).__set_node_type(TExprNodeType::IPV4_LITERAL);
+ TIPv4Literal literal;
+ literal.__set_value(*origin_value);
+ (*node).__set_ipv4_literal(literal);
+ (*node).__set_type(create_type_desc(PrimitiveType::TYPE_IPV4));
+ } else if constexpr (T == TYPE_IPV6) {
+ const auto* origin_value = reinterpret_cast<const IPv6*>(data);
+ (*node).__set_node_type(TExprNodeType::IPV6_LITERAL);
+ TIPv6Literal literal;
+
literal.__set_value(vectorized::DataTypeIPv6::to_string(*origin_value));
+ (*node).__set_ipv6_literal(literal);
+ (*node).__set_type(create_type_desc(PrimitiveType::TYPE_IPV6));
} else {
return Status::InvalidArgument("Invalid argument type!");
}
diff --git a/be/src/vec/olap/olap_data_convertor.cpp
b/be/src/vec/olap/olap_data_convertor.cpp
index 501c6f184aa..5bb00832cdf 100644
--- a/be/src/vec/olap/olap_data_convertor.cpp
+++ b/be/src/vec/olap/olap_data_convertor.cpp
@@ -160,10 +160,10 @@
OlapBlockDataConvertor::create_olap_column_data_convertor(const TabletColumn& co
return
std::make_unique<OlapColumnDataConvertorSimple<vectorized::Int128>>();
}
case FieldType::OLAP_FIELD_TYPE_IPV4: {
- return
std::make_unique<OlapColumnDataConvertorSimple<vectorized::IPv4>>();
+ return std::make_unique<OlapColumnDataConvertorSimple<IPv4>>();
}
case FieldType::OLAP_FIELD_TYPE_IPV6: {
- return
std::make_unique<OlapColumnDataConvertorSimple<vectorized::IPv6>>();
+ return std::make_unique<OlapColumnDataConvertorSimple<IPv6>>();
}
case FieldType::OLAP_FIELD_TYPE_FLOAT: {
return
std::make_unique<OlapColumnDataConvertorSimple<vectorized::Float32>>();
diff --git a/be/src/vec/runtime/ipv4_value.h b/be/src/vec/runtime/ipv4_value.h
index 9fd53afd4c7..d5d749c9628 100644
--- a/be/src/vec/runtime/ipv4_value.h
+++ b/be/src/vec/runtime/ipv4_value.h
@@ -32,19 +32,19 @@ class IPv4Value {
public:
IPv4Value() = default;
- explicit IPv4Value(vectorized::IPv4 ipv4) { _value = ipv4; }
+ explicit IPv4Value(IPv4 ipv4) { _value = ipv4; }
- const vectorized::IPv4& value() const { return _value; }
+ const IPv4& value() const { return _value; }
- vectorized::IPv4& value() { return _value; }
+ IPv4& value() { return _value; }
- void set_value(vectorized::IPv4 ipv4) { _value = ipv4; }
+ void set_value(IPv4 ipv4) { _value = ipv4; }
bool from_string(const std::string& ipv4_str) { return from_string(_value,
ipv4_str); }
std::string to_string() const { return to_string(_value); }
- static bool from_string(vectorized::IPv4& value, const char* ipv4_str,
size_t len) {
+ static bool from_string(IPv4& value, const char* ipv4_str, size_t len) {
if (len == 0) {
return false;
}
@@ -61,15 +61,15 @@ public:
reinterpret_cast<unsigned
char*>(&parse_value))) {
return false;
}
- value = static_cast<vectorized::IPv4>(parse_value);
+ value = static_cast<IPv4>(parse_value);
return true;
}
- static bool from_string(vectorized::IPv4& value, const std::string&
ipv4_str) {
+ static bool from_string(IPv4& value, const std::string& ipv4_str) {
return from_string(value, ipv4_str.c_str(), ipv4_str.size());
}
- static std::string to_string(vectorized::IPv4 value) {
+ static std::string to_string(IPv4 value) {
char buf[IPV4_MAX_TEXT_LENGTH + 1];
char* start = buf;
char* end = buf;
@@ -97,7 +97,7 @@ public:
}
private:
- vectorized::IPv4 _value;
+ IPv4 _value;
};
} // namespace doris
diff --git a/be/src/vec/runtime/ipv6_value.h b/be/src/vec/runtime/ipv6_value.h
index 953b71baf84..5a26af25243 100644
--- a/be/src/vec/runtime/ipv6_value.h
+++ b/be/src/vec/runtime/ipv6_value.h
@@ -32,17 +32,17 @@ class IPv6Value {
public:
IPv6Value() { _value = 0; }
- explicit IPv6Value(vectorized::IPv6 ipv6) { _value = ipv6; }
+ explicit IPv6Value(IPv6 ipv6) { _value = ipv6; }
- const vectorized::IPv6& value() const { return _value; }
+ const IPv6& value() const { return _value; }
- vectorized::IPv6& value() { return _value; }
+ IPv6& value() { return _value; }
- void set_value(vectorized::IPv6 ipv6) { _value = ipv6; }
+ void set_value(IPv6 ipv6) { _value = ipv6; }
bool from_string(const std::string& ipv6_str) { return from_string(_value,
ipv6_str); }
- static bool from_string(vectorized::IPv6& value, const char* ipv6_str,
size_t len) {
+ static bool from_string(IPv6& value, const char* ipv6_str, size_t len) {
if (len == 0) {
return false;
}
@@ -59,13 +59,13 @@ public:
reinterpret_cast<unsigned
char*>(&value));
}
- static bool from_string(vectorized::IPv6& value, const std::string&
ipv6_str) {
+ static bool from_string(IPv6& value, const std::string& ipv6_str) {
return from_string(value, ipv6_str.c_str(), ipv6_str.size());
}
std::string to_string() const { return to_string(_value); }
- static std::string to_string(vectorized::IPv6 value) {
+ static std::string to_string(IPv6 value) {
char buf[IPV6_MAX_TEXT_LENGTH + 1];
char* start = buf;
char* end = buf;
@@ -80,7 +80,7 @@ public:
if (len == 0 || len > IPV6_MAX_TEXT_LENGTH) {
return false;
}
- vectorized::IPv6 value;
+ IPv6 value;
size_t begin = 0;
size_t end = len - 1;
while (begin < len && std::isspace(ipv6_str[begin])) {
@@ -94,7 +94,7 @@ public:
}
private:
- vectorized::IPv6 _value;
+ IPv6 _value;
};
} // namespace doris
diff --git a/be/test/vec/data_types/serde/data_type_serde_mysql_test.cpp
b/be/test/vec/data_types/serde/data_type_serde_mysql_test.cpp
index 5ba8af8b81f..7e17431d172 100644
--- a/be/test/vec/data_types/serde/data_type_serde_mysql_test.cpp
+++ b/be/test/vec/data_types/serde/data_type_serde_mysql_test.cpp
@@ -250,7 +250,7 @@ void serialize_and_deserialize_mysql_test() {
case TYPE_IPV4:
tslot.__set_slotType(type_desc.to_thrift());
{
- auto column_vector_ipv4 =
vectorized::ColumnVector<vectorized::IPv4>::create();
+ auto column_vector_ipv4 =
vectorized::ColumnVector<IPv4>::create();
auto& ipv4_data = column_vector_ipv4->get_data();
for (int i = 0; i < row_num; ++i) {
IPv4Value ipv4_value;
@@ -267,7 +267,7 @@ void serialize_and_deserialize_mysql_test() {
case TYPE_IPV6:
tslot.__set_slotType(type_desc.to_thrift());
{
- auto column_vector_ipv6 =
vectorized::ColumnVector<vectorized::IPv6>::create();
+ auto column_vector_ipv6 =
vectorized::ColumnVector<IPv6>::create();
auto& ipv6_data = column_vector_ipv6->get_data();
for (int i = 0; i < row_num; ++i) {
IPv6Value ipv6_value;
diff --git a/be/test/vec/runtime/ip_value_test.cpp
b/be/test/vec/runtime/ip_value_test.cpp
index cbeebcf24b6..f3b3b9ac9cd 100644
--- a/be/test/vec/runtime/ip_value_test.cpp
+++ b/be/test/vec/runtime/ip_value_test.cpp
@@ -49,8 +49,8 @@ static void print_bytes(T num) {
TEST(IPValueTest, IPv4ValueTest) {
const std::string ipv4_str1 = "192.168.103.254";
const std::string ipv4_str2 = "193.168.103.255";
- vectorized::IPv4 ipv4_val1;
- vectorized::IPv4 ipv4_val2;
+ IPv4 ipv4_val1;
+ IPv4 ipv4_val2;
ASSERT_TRUE(IPv4Value::from_string(ipv4_val1, ipv4_str1.c_str(),
ipv4_str1.size()));
ASSERT_TRUE(IPv4Value::from_string(ipv4_val2, ipv4_str2.c_str(),
ipv4_str2.size()));
ASSERT_TRUE(ipv4_val1 < ipv4_val2);
@@ -65,8 +65,8 @@ TEST(IPValueTest, IPv4ValueTest) {
TEST(IPValueTest, IPv6ValueTest) {
const std::string ipv6_str1 = "2001:418:0:5000::c2d";
const std::string ipv6_str2 = "2001:428::205:171:200:230";
- vectorized::IPv6 ipv6_val1;
- vectorized::IPv6 ipv6_val2;
+ IPv6 ipv6_val1;
+ IPv6 ipv6_val2;
ASSERT_TRUE(IPv6Value::from_string(ipv6_val1, ipv6_str1.c_str(),
ipv6_str1.size()));
ASSERT_TRUE(IPv6Value::from_string(ipv6_val2, ipv6_str2.c_str(),
ipv6_str2.size()));
ASSERT_TRUE(ipv6_val1 < ipv6_val2);
@@ -91,12 +91,12 @@ static void apply_cidr_mask(const char* __restrict src,
char* __restrict dst_low
TEST(IPValueTest, IPv6CIDRTest) {
const std::string ipv6_str1 = "2001:0db8:0000:85a3:0000:0000:ac1f:8001";
const std::string ipv6_str2 = "2001:0db8:0000:85a3:ffff:ffff:ffff:ffff";
- vectorized::IPv6 ipv6_val1; // little-endian
- vectorized::IPv6 ipv6_val2; // little-endian
+ IPv6 ipv6_val1; // little-endian
+ IPv6 ipv6_val2; // little-endian
ASSERT_TRUE(IPv6Value::from_string(ipv6_val1, ipv6_str1.c_str(),
ipv6_str1.size()));
ASSERT_TRUE(IPv6Value::from_string(ipv6_val2, ipv6_str2.c_str(),
ipv6_str2.size()));
- vectorized::IPv6 min_range1, max_range1;
- vectorized::IPv6 min_range2, max_range2;
+ IPv6 min_range1, max_range1;
+ IPv6 min_range2, max_range2;
apply_cidr_mask(reinterpret_cast<const char*>(&ipv6_val1),
reinterpret_cast<char*>(&min_range1),
reinterpret_cast<char*>(&max_range1), 0);
apply_cidr_mask(reinterpret_cast<const char*>(&ipv6_val2),
reinterpret_cast<char*>(&min_range2),
diff --git a/gensrc/proto/internal_service.proto
b/gensrc/proto/internal_service.proto
index 9c993bb9a93..72b11e6e2ed 100644
--- a/gensrc/proto/internal_service.proto
+++ b/gensrc/proto/internal_service.proto
@@ -513,16 +513,18 @@ enum PColumnType {
COLUMN_TYPE_DECIMAL64 = 18;
COLUMN_TYPE_DECIMAL128I = 19;
COLUMN_TYPE_DECIMAL256 = 20;
+ COLUMN_TYPE_IPV4 = 21;
+ COLUMN_TYPE_IPV6 = 22;
}
message PMinMaxFilter {
- required PColumnType column_type = 1;
+ required PColumnType column_type = 1; // Deprecated
required PColumnValue min_val = 2;
required PColumnValue max_val = 3;
};
message PInFilter {
- required PColumnType column_type = 1;
+ required PColumnType column_type = 1; // Deprecated
repeated PColumnValue values = 2;
optional string ignored_msg = 3;
}
diff --git a/regression-test/data/datatype_p0/ip/ip_rf/test_ip_rf.out
b/regression-test/data/datatype_p0/ip/ip_rf/test_ip_rf.out
new file mode 100644
index 00000000000..7e8a59791f6
--- /dev/null
+++ b/regression-test/data/datatype_p0/ip/ip_rf/test_ip_rf.out
@@ -0,0 +1,31 @@
+-- This file is automatically generated. You should know what you did if you
want to edit this
+-- !sql --
+2
+
+-- !sql --
+2
+
+-- !sql --
+2
+
+-- !sql --
+2
+
+-- !sql --
+2
+
+-- !sql --
+2
+
+-- !sql --
+2
+
+-- !sql --
+2
+
+-- !sql --
+2
+
+-- !sql --
+2
+
diff --git a/regression-test/suites/datatype_p0/ip/ip_rf/test_ip_rf.groovy
b/regression-test/suites/datatype_p0/ip/ip_rf/test_ip_rf.groovy
new file mode 100644
index 00000000000..c7f6c30481e
--- /dev/null
+++ b/regression-test/suites/datatype_p0/ip/ip_rf/test_ip_rf.groovy
@@ -0,0 +1,73 @@
+
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+suite("test_ip_rf") {
+ sql """ DROP TABLE IF EXISTS ip_test """
+ sql """ DROP TABLE IF EXISTS ip_test2 """
+ sql """
+ CREATE TABLE ip_test (
+ `id` int,
+ `ip_v4` ipv4,
+ `ip_v6` ipv6
+ ) ENGINE=OLAP
+ DISTRIBUTED BY HASH(`id`) BUCKETS 4
+ PROPERTIES (
+ "replication_allocation" = "tag.location.default: 1"
+ );
+ """
+
+ sql """
+ CREATE TABLE ip_test2 (
+ `id` int,
+ `ip_v4` ipv4,
+ `ip_v6` ipv6
+ ) ENGINE=OLAP
+ DISTRIBUTED BY HASH(`id`) BUCKETS 4
+ PROPERTIES (
+ "replication_allocation" = "tag.location.default: 1"
+ );
+ """
+
+ sql """
+ insert into ip_test values(1, '0.0.0.0','2001:16a0:2:200a::2'),(2,
'127.0.0.1','2a02:e980:83:5b09:ecb8:c669:b336:650e');
+ """
+
+ sql """
+ insert into ip_test2 values(1, '0.0.0.0','2001:16a0:2:200a::2'),(2,
'127.0.0.1','2a02:e980:83:5b09:ecb8:c669:b336:650e'),(3,
'59.50.185.152','2001:4888:1f:e891:161:26::'),(4,
'255.255.255.255','2001:4888:1f:e891:161:26::');
+
+ """
+
+ sql "ANALYZE TABLE ip_test WITH sync;"
+ sql "ANALYZE TABLE ip_test2 WITH sync;"
+
+ sql "set runtime_filter_type=0;"
+ qt_sql "select count(*) from ip_test a, ip_test2 b where a.ip_v4=b.ip_v4;"
+ qt_sql "select count(*) from ip_test a, ip_test2 b where a.ip_v6=b.ip_v6;"
+ sql "set runtime_filter_type=1;"
+ qt_sql "select count(*) from ip_test a, ip_test2 b where a.ip_v4=b.ip_v4;"
+ qt_sql "select count(*) from ip_test a, ip_test2 b where a.ip_v6=b.ip_v6;"
+ sql "set runtime_filter_type=2;"
+ qt_sql "select count(*) from ip_test a, ip_test2 b where a.ip_v4=b.ip_v4;"
+ qt_sql "select count(*) from ip_test a, ip_test2 b where a.ip_v6=b.ip_v6;"
+ sql "set runtime_filter_type=4;"
+ qt_sql "select count(*) from ip_test a, ip_test2 b where a.ip_v4=b.ip_v4;"
+ qt_sql "select count(*) from ip_test a, ip_test2 b where a.ip_v6=b.ip_v6;"
+ sql "set runtime_filter_type=8;"
+ qt_sql "select count(*) from ip_test a, ip_test2 b where a.ip_v4=b.ip_v4;"
+ qt_sql "select count(*) from ip_test a, ip_test2 b where a.ip_v6=b.ip_v6;"
+
+}
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org
For additional commands, e-mail: commits-h...@doris.apache.org
