This is an automated email from the ASF dual-hosted git repository.
panxiaolei pushed a commit to branch branch-3.0
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/branch-3.0 by this push:
new 2010f9f63e1 [Fix-3.0](smooth-upgrade) Fix smooth upgrade of function
is_ip_address_in_range (#45428)
2010f9f63e1 is described below
commit 2010f9f63e15cbff388d24cea7c3c11d4bfd1ff0
Author: zclllyybb <zhaochan...@selectdb.com>
AuthorDate: Tue Dec 17 11:55:42 2024 +0800
[Fix-3.0](smooth-upgrade) Fix smooth upgrade of function
is_ip_address_in_range (#45428)
### What problem does this PR solve?
Issue Number: close #xxx
Related PR: #xxx
Problem Summary:
The origin PR is https://github.com/apache/doris/pull/35239. for
branch-3.0 it was merged in 3.0.0 but forgot to register old version.
now in branch-3.0 we fix it in this PR. and will pick it to branch-2.0
in https://github.com/apache/doris/pull/45358 which must be merged in
2.1.8.
then:
```
FROM TO result
217- 218+ ✅
217- 303- 💥
218+ 303- ✅
218+ 304+ ✅
303- 304+ ✅
```
this is our best result.
### Release note
None
### Check List (For Author)
- Test <!-- At least one of them must be included. -->
- [ ] Regression test
- [ ] Unit Test
- [ ] Manual test (add detailed scripts or steps below)
- [x] No need to test or manual test. Explain why:
- [ ] This is a refactor/code format and no logic has been changed.
- [x] Previous test can cover this change.
- [ ] No code files have been changed.
- [ ] Other reason <!-- Add your reason? -->
- Behavior changed:
- [x] No.
- [ ] Yes. <!-- Explain the behavior change -->
- Does this need documentation?
- [x] No.
- [ ] Yes. <!-- Add document PR link here. eg:
https://github.com/apache/doris-website/pull/1214 -->
### Check List (For Reviewer who merge this PR)
- [ ] Confirm the release note
- [ ] Confirm test cases
- [ ] Confirm document
- [ ] Add branch pick label <!-- Add branch pick label that this PR
should merge into -->
---
be/src/runtime/runtime_state.h | 5 ++
be/src/vec/exprs/vectorized_fn_call.cpp | 4 +-
be/src/vec/functions/function.h | 1 +
be/src/vec/functions/function_ip.cpp | 2 +
be/src/vec/functions/function_ip.h | 75 ++++++++++++++++++++++
be/src/vec/functions/simple_function_factory.h | 5 ++
.../java/org/apache/doris/qe/SessionVariable.java | 9 +++
gensrc/thrift/PaloInternalService.thrift | 4 ++
8 files changed, 104 insertions(+), 1 deletion(-)
diff --git a/be/src/runtime/runtime_state.h b/be/src/runtime/runtime_state.h
index fcbe8519ef1..e1779895bba 100644
--- a/be/src/runtime/runtime_state.h
+++ b/be/src/runtime/runtime_state.h
@@ -183,6 +183,11 @@ public:
return _query_options.__isset.enable_decimal256 &&
_query_options.enable_decimal256;
}
+ bool new_is_ip_address_in_range() const {
+ return _query_options.__isset.new_is_ip_address_in_range &&
+ _query_options.new_is_ip_address_in_range;
+ }
+
bool enable_common_expr_pushdown() const {
return _query_options.__isset.enable_common_expr_pushdown &&
_query_options.enable_common_expr_pushdown;
diff --git a/be/src/vec/exprs/vectorized_fn_call.cpp
b/be/src/vec/exprs/vectorized_fn_call.cpp
index 3192653a816..427059bba41 100644
--- a/be/src/vec/exprs/vectorized_fn_call.cpp
+++ b/be/src/vec/exprs/vectorized_fn_call.cpp
@@ -107,7 +107,9 @@ Status VectorizedFnCall::prepare(RuntimeState* state, const
RowDescriptor& desc,
// get the function. won't prepare function.
_function = SimpleFunctionFactory::instance().get_function(
_fn.name.function_name, argument_template, _data_type,
- {.enable_decimal256 = state->enable_decimal256()},
state->be_exec_version());
+ {.enable_decimal256 = state->enable_decimal256(),
+ .new_is_ip_address_in_range =
state->new_is_ip_address_in_range()},
+ state->be_exec_version());
}
if (_function == nullptr) {
return Status::InternalError("Could not find function {}, arg {}
return {} ",
diff --git a/be/src/vec/functions/function.h b/be/src/vec/functions/function.h
index 4702a4b7af0..ff987d130cd 100644
--- a/be/src/vec/functions/function.h
+++ b/be/src/vec/functions/function.h
@@ -49,6 +49,7 @@ namespace doris::vectorized {
struct FunctionAttr {
bool enable_decimal256 {false};
+ bool new_is_ip_address_in_range {false};
};
#define RETURN_REAL_TYPE_FOR_DATEV2_FUNCTION(TYPE)
\
diff --git a/be/src/vec/functions/function_ip.cpp
b/be/src/vec/functions/function_ip.cpp
index ae5a2399981..30b31901624 100644
--- a/be/src/vec/functions/function_ip.cpp
+++ b/be/src/vec/functions/function_ip.cpp
@@ -47,6 +47,8 @@ void register_function_ip(SimpleFunctionFactory& factory) {
factory.register_function<FunctionIsIPString<IPv6>>();
factory.register_function<FunctionIsIPAddressInRange>();
+ factory.register_function<FunctionIsIPAddressInRangeOld>();
+
/// CIDR part
factory.register_function<FunctionIPv4CIDRToRange>();
factory.register_function<FunctionIPv6CIDRToRange>();
diff --git a/be/src/vec/functions/function_ip.h
b/be/src/vec/functions/function_ip.h
index b60e1f393f8..864d0cbc220 100644
--- a/be/src/vec/functions/function_ip.h
+++ b/be/src/vec/functions/function_ip.h
@@ -789,6 +789,81 @@ public:
}
};
+// old version throw exception when meet null value
+class FunctionIsIPAddressInRangeOld : public IFunction {
+public:
+ static constexpr auto name = "is_ip_address_in_range";
+ static FunctionPtr create() { return
std::make_shared<FunctionIsIPAddressInRange>(); }
+
+ String get_name() const override { return name; }
+
+ size_t get_number_of_arguments() const override { return 2; }
+
+ DataTypePtr get_return_type_impl(const DataTypes& arguments) const
override {
+ return std::make_shared<DataTypeUInt8>();
+ }
+
+ bool use_default_implementation_for_nulls() const override { return false;
}
+ Status execute_impl(FunctionContext* context, Block& block, const
ColumnNumbers& arguments,
+ size_t result, size_t input_rows_count) const override
{
+ const auto& addr_column_with_type_and_name =
block.get_by_position(arguments[0]);
+ const auto& cidr_column_with_type_and_name =
block.get_by_position(arguments[1]);
+ WhichDataType addr_type(addr_column_with_type_and_name.type);
+ WhichDataType cidr_type(cidr_column_with_type_and_name.type);
+ const auto& [addr_column, addr_const] =
+ unpack_if_const(addr_column_with_type_and_name.column);
+ const auto& [cidr_column, cidr_const] =
+ unpack_if_const(cidr_column_with_type_and_name.column);
+ const ColumnString* str_addr_column = nullptr;
+ const ColumnString* str_cidr_column = nullptr;
+ const NullMap* null_map_addr = nullptr;
+ const NullMap* null_map_cidr = nullptr;
+ if (addr_type.is_nullable()) {
+ const auto* addr_column_nullable =
+ assert_cast<const ColumnNullable*>(addr_column.get());
+ str_addr_column = assert_cast<const ColumnString*>(
+ addr_column_nullable->get_nested_column_ptr().get());
+ null_map_addr = &addr_column_nullable->get_null_map_data();
+ } else {
+ str_addr_column = assert_cast<const
ColumnString*>(addr_column.get());
+ }
+
+ if (cidr_type.is_nullable()) {
+ const auto* cidr_column_nullable =
+ assert_cast<const ColumnNullable*>(cidr_column.get());
+ str_cidr_column = assert_cast<const ColumnString*>(
+ cidr_column_nullable->get_nested_column_ptr().get());
+ null_map_cidr = &cidr_column_nullable->get_null_map_data();
+ } else {
+ str_cidr_column = assert_cast<const
ColumnString*>(cidr_column.get());
+ }
+
+ auto col_res = ColumnUInt8::create(input_rows_count, 0);
+ auto& col_res_data = col_res->get_data();
+ for (size_t i = 0; i < input_rows_count; ++i) {
+ auto addr_idx = index_check_const(i, addr_const);
+ auto cidr_idx = index_check_const(i, cidr_const);
+ if (null_map_addr && (*null_map_addr)[addr_idx]) [[unlikely]] {
+ throw Exception(ErrorCode::INVALID_ARGUMENT,
+ "The arguments of function {} must be String,
not NULL",
+ get_name());
+ }
+ if (null_map_cidr && (*null_map_cidr)[cidr_idx]) [[unlikely]] {
+ throw Exception(ErrorCode::INVALID_ARGUMENT,
+ "The arguments of function {} must be String,
not NULL",
+ get_name());
+ }
+ const auto addr =
+
IPAddressVariant(str_addr_column->get_data_at(addr_idx).to_string_view());
+ const auto cidr =
+
parse_ip_with_cidr(str_cidr_column->get_data_at(cidr_idx).to_string_view());
+ col_res_data[i] = is_address_in_range(addr, cidr) ? 1 : 0;
+ }
+ block.replace_by_position(result, std::move(col_res));
+ return Status::OK();
+ }
+};
+
class FunctionIPv4CIDRToRange : public IFunction {
public:
static constexpr auto name = "ipv4_cidr_to_range";
diff --git a/be/src/vec/functions/simple_function_factory.h
b/be/src/vec/functions/simple_function_factory.h
index dfcb756bba5..5a1718a0283 100644
--- a/be/src/vec/functions/simple_function_factory.h
+++ b/be/src/vec/functions/simple_function_factory.h
@@ -157,6 +157,11 @@ public:
int be_version =
BeExecVersionManager::get_newest_version()) {
std::string key_str = name;
+ // special function replacement
+ if (key_str == "is_ip_address_in_range" &&
!attr.new_is_ip_address_in_range) [[unlikely]] {
+ key_str = "__is_ip_address_in_range_OLD__";
+ }
+
if (function_alias.contains(name)) {
key_str = function_alias[name];
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java
b/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java
index 675bcbaf798..0b9ac4c89e4 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java
@@ -681,6 +681,8 @@ public class SessionVariable implements Serializable,
Writable {
public static final String ENABLE_COOLDOWN_REPLICA_AFFINITY =
"enable_cooldown_replica_affinity";
+
+ public static final String NEW_IS_IP_ADDRESS_IN_RANGE =
"new_is_ip_address_in_range";
/**
* Inserting overwrite for auto partition table allows creating partition
for
* datas which cannot find partition to overwrite.
@@ -2309,6 +2311,11 @@ public class SessionVariable implements Serializable,
Writable {
})
public int adaptivePipelineTaskSerialReadOnLimit = 10000;
+ // only to control some function behaviour. not visible or mutable.
+ @VariableMgr.VarAttr(name = NEW_IS_IP_ADDRESS_IN_RANGE, needForward =
true, flag = VariableMgr.INVISIBLE
+ | VariableMgr.READ_ONLY)
+ public boolean newIsIpAddressInRange = true;
+
public void setEnableEsParallelScroll(boolean enableESParallelScroll) {
this.enableESParallelScroll = enableESParallelScroll;
}
@@ -3957,6 +3964,8 @@ public class SessionVariable implements Serializable,
Writable {
tResult.setOrcOnceMaxReadBytes(orcOnceMaxReadBytes);
tResult.setIgnoreRuntimeFilterError(ignoreRuntimeFilterError);
+ tResult.setNewIsIpAddressInRange(newIsIpAddressInRange);
+
return tResult;
}
diff --git a/gensrc/thrift/PaloInternalService.thrift
b/gensrc/thrift/PaloInternalService.thrift
index 354ab4d3c84..e29ad1c1a04 100644
--- a/gensrc/thrift/PaloInternalService.thrift
+++ b/gensrc/thrift/PaloInternalService.thrift
@@ -355,6 +355,10 @@ struct TQueryOptions {
140: optional i64 orc_max_merge_distance_bytes = 1048576;
141: optional bool ignore_runtime_filter_error = false;
+
+ // upgrade options. keep them same in every branch.
+ 200: optional bool new_is_ip_address_in_range = false;
+
// For cloud, to control if the content would be written into file cache
// In write path, to control if the content would be written into file cache.
// In read path, read from file cache or remote storage when execute query.
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org
For additional commands, e-mail: commits-h...@doris.apache.org
