This is an automated email from the ASF dual-hosted git repository.
mrhhsg pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/master by this push:
new f70306ce044 [Opt](bitmap) Optimize bitmap op count rewrites (#65483)
f70306ce044 is described below
commit f70306ce0449de8f591956ca95632bae8e5e85ff
Author: linrrarity <[email protected]>
AuthorDate: Mon Jul 27 16:03:34 2026 +0800
[Opt](bitmap) Optimize bitmap op count rewrites (#65483)
Problem Summary:
### Background
`bitmap_count(bitmap_op(...))` currently materializes an intermediate
bitmap before counting. For patterns such as
`bitmap_count(bitmap_and(...))`, `bitmap_count(bitmap_or(...))`,
`bitmap_count(bitmap_xor(...))`, and
`bitmap_count(bitmap_and_not(...))`, Doris already has corresponding
`bitmap_*_count` functions that can compute the cardinality directly.
Also, `bitmap_and_count`, `bitmap_or_count`, `bitmap_xor_count`, and
`bitmap_and_not_count` are semantically always not null, but their
previous metadata could be marked nullable when bitmap arguments were
nullable.
### Changes
- Add Nereids expression rewrite:
- `bitmap_count(bitmap_and(...))` -> `bitmap_and_count(...)`
- `bitmap_count(bitmap_or(...))` -> `bitmap_or_count(...)`
- `bitmap_count(bitmap_xor(...))` -> `bitmap_xor_count(...)`
- `bitmap_count(bitmap_not(...))` / `bitmap_and_not(...)` ->
`bitmap_and_not_count(...)`
- Mark bitmap op count scalar functions as always not nullable in
Nereids.
- Add BE v2 implementations for bitmap op count return type inference:
- `bitmap_and_count_v2`
- `bitmap_or_count_v2`
- `bitmap_xor_count_v2`
- `bitmap_and_not_count_v2`
- Add `new_version_bitmap_op_count` query option to select the v2 BE
implementations.
- Optimize binary bitmap count execution paths to avoid intermediate
bitmap materialization.
- Fix `BitmapValue::or_cardinality` for SET/SET input.
- Merge `bitmap_and_not` / `bitmap_andnot` aliases into `bitmap_not`.
---
be/src/core/value/bitmap_value.h | 61 ++++++++
be/src/exprs/function/function.h | 1 +
be/src/exprs/function/function_bitmap.cpp | 101 +++----------
be/src/exprs/function/function_bitmap_variadic.cpp | 166 +++++++++++++++++----
be/src/exprs/function/simple_function_factory.h | 5 +
be/src/exprs/vectorized_fn_call.cpp | 5 +-
be/src/format_v2/expr/cast.cpp | 5 +-
be/test/core/value/bitmap_value_test.cpp | 5 +-
be/test/exprs/function/function_bitmap_test.cpp | 81 ++++++++--
.../function/simple_function_factory_test.cpp | 35 +++++
.../doris/catalog/BuiltinScalarFunctions.java | 10 +-
.../nereids/load/NereidsStreamLoadPlanner.java | 1 +
.../rules/expression/ExpressionNormalization.java | 4 +-
.../rules/expression/ExpressionRuleType.java | 3 +-
.../rules/BitmapCountToBitmapOpCount.java | 67 +++++++++
.../expression/rules/FoldConstantRuleOnBE.java | 1 +
.../functions/scalar/BitmapAndCount.java | 8 +-
.../expressions/functions/scalar/BitmapAndNot.java | 73 ---------
.../functions/scalar/BitmapAndNotAlias.java | 73 ---------
.../functions/scalar/BitmapAndNotCount.java | 8 +-
.../functions/scalar/BitmapAndNotCountAlias.java | 78 ----------
.../functions/scalar/BitmapOrCount.java | 7 +-
.../functions/scalar/BitmapXorCount.java | 8 +-
.../expressions/visitor/ScalarFunctionVisitor.java | 15 --
.../main/java/org/apache/doris/qe/Coordinator.java | 3 +
.../org/apache/doris/qe/CoordinatorContext.java | 1 +
.../org/apache/doris/qe/NereidsCoordinator.java | 1 +
.../java/org/apache/doris/qe/SessionVariable.java | 1 +
.../doris/cloud/catalog/CloudEnvFactoryTest.java | 32 ++++
.../rules/analysis/FunctionRegistryTest.java | 21 +++
.../rules/BitmapCountToBitmapOpCountTest.java | 115 ++++++++++++++
.../doris/qe/ShortCircuitQueryContextTest.java | 33 ++++
gensrc/thrift/PaloInternalService.thrift | 1 +
.../bitmap_functions/test_bitmap_function.out | 18 +++
.../bitmap_functions/test_bitmap_function.groovy | 86 +++++++++++
35 files changed, 736 insertions(+), 397 deletions(-)
diff --git a/be/src/core/value/bitmap_value.h b/be/src/core/value/bitmap_value.h
index f18a7eb8fd3..ae4249c5b19 100644
--- a/be/src/core/value/bitmap_value.h
+++ b/be/src/core/value/bitmap_value.h
@@ -2281,6 +2281,67 @@ public:
return 0;
}
+ uint64_t xor_cardinality(const BitmapValue& rhs) const {
+ switch (rhs._type) {
+ case EMPTY:
+ return cardinality();
+ case SINGLE:
+ switch (_type) {
+ case EMPTY:
+ return 1;
+ case SINGLE:
+ return 2 * (_sv != rhs._sv);
+ case BITMAP:
+ return cardinality() + 1 - 2 * _bitmap->contains(rhs._sv);
+ case SET:
+ return _set.size() + 1 - 2 * _set.contains(rhs._sv);
+ }
+ break;
+ case BITMAP:
+ switch (_type) {
+ case EMPTY:
+ return rhs.cardinality();
+ case SINGLE:
+ return rhs.cardinality() + 1 - 2 * rhs._bitmap->contains(_sv);
+ case BITMAP:
+ return _bitmap->xorCardinality(*rhs._bitmap);
+ case SET: {
+ uint64_t cardinality = rhs._bitmap->cardinality();
+ for (auto v : _set) {
+ if (rhs._bitmap->contains(v)) {
+ --cardinality;
+ } else {
+ ++cardinality;
+ }
+ }
+ return cardinality;
+ }
+ }
+ break;
+ case SET:
+ switch (_type) {
+ case EMPTY:
+ return rhs.cardinality();
+ case SINGLE:
+ return rhs.cardinality() + 1 - 2 * rhs._set.contains(_sv);
+ case BITMAP: {
+ uint64_t cardinality = _bitmap->cardinality();
+ for (auto v : rhs._set) {
+ if (_bitmap->contains(v)) {
+ --cardinality;
+ } else {
+ ++cardinality;
+ }
+ }
+ return cardinality;
+ }
+ case SET:
+ return _set.size() + rhs._set.size() - 2 *
and_cardinality(rhs);
+ }
+ }
+ return 0;
+ }
+
uint64_t andnot_cardinality(const BitmapValue& rhs) const {
switch (rhs._type) {
case EMPTY:
diff --git a/be/src/exprs/function/function.h b/be/src/exprs/function/function.h
index 13d4bfd6bb5..d26cab4d783 100644
--- a/be/src/exprs/function/function.h
+++ b/be/src/exprs/function/function.h
@@ -57,6 +57,7 @@ namespace doris {
struct FunctionAttr {
bool new_version_unix_timestamp {false};
+ bool new_version_bitmap_op_count {false};
};
#define RETURN_REAL_TYPE_FOR_DATEV2_FUNCTION(TYPE)
\
diff --git a/be/src/exprs/function/function_bitmap.cpp
b/be/src/exprs/function/function_bitmap.cpp
index 6213066e0e1..6ba5974cd41 100644
--- a/be/src/exprs/function/function_bitmap.cpp
+++ b/be/src/exprs/function/function_bitmap.cpp
@@ -582,52 +582,6 @@ struct BitmapNot {
}
};
-struct NameBitmapAndNot {
- static constexpr auto name = "bitmap_and_not";
-};
-
-template <typename LeftDataType, typename RightDataType>
-struct BitmapAndNot {
- using ResultDataType = DataTypeBitMap;
- using T0 = typename LeftDataType::FieldType;
- using T1 = typename RightDataType::FieldType;
- using TData = std::vector<BitmapValue>;
-
- static void vector_vector(const TData& lvec, const TData& rvec, TData&
res) {
- size_t size = lvec.size();
- BitmapValue mid_data;
- for (size_t i = 0; i < size; ++i) {
- mid_data = lvec[i];
- mid_data &= rvec[i];
- res[i] = lvec[i];
- res[i] -= mid_data;
- mid_data.reset();
- }
- }
- static void vector_scalar(const TData& lvec, const BitmapValue& rval,
TData& res) {
- size_t size = lvec.size();
- BitmapValue mid_data;
- for (size_t i = 0; i < size; ++i) {
- mid_data = lvec[i];
- mid_data &= rval;
- res[i] = lvec[i];
- res[i] -= mid_data;
- mid_data.reset();
- }
- }
- static void scalar_vector(const BitmapValue& lval, const TData& rvec,
TData& res) {
- size_t size = rvec.size();
- BitmapValue mid_data;
- for (size_t i = 0; i < size; ++i) {
- mid_data = lval;
- mid_data &= rvec[i];
- res[i] = lval;
- res[i] -= mid_data;
- mid_data.reset();
- }
- }
-};
-
struct NameBitmapAndNotCount {
static constexpr auto name = "bitmap_and_not_count";
};
@@ -642,32 +596,20 @@ struct BitmapAndNotCount {
static void vector_vector(const TData& lvec, const TData& rvec, ResTData*
res) {
size_t size = lvec.size();
- BitmapValue mid_data;
for (size_t i = 0; i < size; ++i) {
- mid_data = lvec[i];
- mid_data &= rvec[i];
- res[i] = lvec[i].andnot_cardinality(mid_data);
- mid_data.reset();
+ res[i] = lvec[i].andnot_cardinality(rvec[i]);
}
}
static void scalar_vector(const BitmapValue& lval, const TData& rvec,
ResTData* res) {
size_t size = rvec.size();
- BitmapValue mid_data;
for (size_t i = 0; i < size; ++i) {
- mid_data = lval;
- mid_data &= rvec[i];
- res[i] = lval.andnot_cardinality(mid_data);
- mid_data.reset();
+ res[i] = lval.andnot_cardinality(rvec[i]);
}
}
static void vector_scalar(const TData& lvec, const BitmapValue& rval,
ResTData* res) {
size_t size = lvec.size();
- BitmapValue mid_data;
for (size_t i = 0; i < size; ++i) {
- mid_data = lvec[i];
- mid_data &= rval;
- res[i] = lvec[i].andnot_cardinality(mid_data);
- mid_data.reset();
+ res[i] = lvec[i].andnot_cardinality(rval);
}
}
};
@@ -687,8 +629,10 @@ ColumnPtr handle_bitmap_op_count_null_value(ColumnPtr&
src, const Block& block,
const ColumnNumbers& args,
uint32_t result,
size_t input_rows_count) {
MutableColumnPtr mutable_src = IColumn::mutate(std::move(src));
- auto* nullable = assert_cast<ColumnNullable*>(mutable_src.get());
- auto* src_not_nullable_mutable = &nullable->get_nested_column();
+ auto* src_not_nullable_mutable = mutable_src.get();
+ if (auto* nullable = check_and_get_column<ColumnNullable>(*mutable_src)) {
+ src_not_nullable_mutable = &nullable->get_nested_column();
+ }
auto* __restrict count_data =
assert_cast<ColumnInt64*>(src_not_nullable_mutable)->get_data().data();
@@ -739,28 +683,32 @@ Status execute_bitmap_op_count_null_to_zero(
return Status::OK();
}
-template <typename FunctionName>
+template <typename FunctionName, bool NewVersion = false>
class FunctionBitmapAndNotCount : public IFunction {
public:
using LeftDataType = DataTypeBitMap;
using RightDataType = DataTypeBitMap;
using ResultDataType = typename BitmapAndNotCount<LeftDataType,
RightDataType>::ResultDataType;
- static constexpr auto name = FunctionName::name;
+ static constexpr auto name = NewVersion ? "bitmap_and_not_count_v2" :
FunctionName::name;
static FunctionPtr create() { return
std::make_shared<FunctionBitmapAndNotCount>(); }
String get_name() const override { return name; }
size_t get_number_of_arguments() const override { return 2; }
DataTypePtr get_return_type_impl(const DataTypes& arguments) const
override {
- bool return_nullable = false;
- // result is nullable only when any columns is nullable for
bitmap_and_not_count
- for (size_t i = 0; i < arguments.size(); ++i) {
- if (arguments[i]->is_nullable()) {
- return_nullable = true;
- break;
+ if constexpr (NewVersion) {
+ return std::make_shared<ResultDataType>();
+ } else {
+ bool return_nullable = false;
+ // result is nullable only when any columns is nullable for
bitmap_and_not_count
+ for (size_t i = 0; i < arguments.size(); ++i) {
+ if (arguments[i]->is_nullable()) {
+ return_nullable = true;
+ break;
+ }
}
+ auto result_type = std::make_shared<ResultDataType>();
+ return return_nullable ? make_nullable(result_type) : result_type;
}
- auto result_type = std::make_shared<ResultDataType>();
- return return_nullable ? make_nullable(result_type) : result_type;
}
bool use_default_implementation_for_nulls() const override {
@@ -1254,8 +1202,6 @@ using FunctionBitmapToBase64 =
FunctionUnaryToType<BitmapToBase64, NameBitmapToB
using FunctionBitmapFromBase64 = FunctionBitmapAlwaysNull<BitmapFromBase64>;
using FunctionBitmapNot =
FunctionBinaryToType<DataTypeBitMap, DataTypeBitMap, BitmapNot,
NameBitmapNot>;
-using FunctionBitmapAndNot =
- FunctionBinaryToType<DataTypeBitMap, DataTypeBitMap, BitmapAndNot,
NameBitmapAndNot>;
using FunctionBitmapContains =
FunctionBinaryToType<DataTypeBitMap, DataTypeInt64, BitmapContains,
NameBitmapContains>;
using FunctionBitmapRemove =
@@ -1284,9 +1230,10 @@ void register_function_bitmap(SimpleFunctionFactory&
factory) {
factory.register_function<FunctionBitmapMax>();
factory.register_function<FunctionBitmapToString>();
factory.register_function<FunctionBitmapNot>();
- factory.register_function<FunctionBitmapAndNot>();
- factory.register_alias(NameBitmapAndNot::name, "bitmap_andnot");
+ factory.register_alias(NameBitmapNot::name, "bitmap_and_not");
+ factory.register_alias(NameBitmapNot::name, "bitmap_andnot");
factory.register_function<FunctionBitmapAndNotCount<NameBitmapAndNotCount>>();
+ factory.register_function<FunctionBitmapAndNotCount<NameBitmapAndNotCount,
true>>();
factory.register_alias(NameBitmapAndNotCount::name, "bitmap_andnot_count");
factory.register_function<FunctionBitmapContains>();
factory.register_function<FunctionBitmapRemove>();
diff --git a/be/src/exprs/function/function_bitmap_variadic.cpp
b/be/src/exprs/function/function_bitmap_variadic.cpp
index 11cfcb9ef81..b9749c115d8 100644
--- a/be/src/exprs/function/function_bitmap_variadic.cpp
+++ b/be/src/exprs/function/function_bitmap_variadic.cpp
@@ -114,47 +114,117 @@ namespace doris {
}
\
}
-#define BITMAP_FUNCTION_COUNT_VARIADIC(CLASS, FUNCTION_NAME, OP)
\
+#define BITMAP_FUNCTION_COUNT_VARIADIC(CLASS, FUNCTION_NAME, OP, COUNT_OP)
\
struct CLASS {
\
+ public:
\
static constexpr auto name = #FUNCTION_NAME;
\
using ResultDataType = DataTypeInt64;
\
using TData = std::vector<BitmapValue>;
\
using ResTData = typename ColumnInt64::Container;
\
static Status vector_vector(ColumnPtr argument_columns[], size_t
col_size, \
size_t input_rows_count, ResTData& res,
IColumn* res_nulls) { \
- TData vals;
\
- if (auto* nullable =
check_and_get_column<ColumnNullable>(*argument_columns[0])) { \
- vals.resize(input_rows_count);
\
- BITMAP_OR_NULLABLE(nullable, input_rows_count, vals, =);
\
- } else {
\
- vals = assert_cast<const
ColumnBitmap*>(argument_columns[0].get())->get_data(); \
+ DCHECK_GE(col_size, 2);
\
+ if (col_size == 2) {
\
+ const auto lhs =
make_bitmap_column_accessor(argument_columns[0]); \
+ const auto rhs =
make_bitmap_column_accessor(argument_columns[1]); \
+ for (size_t row = 0; row < input_rows_count; ++row) {
\
+ const auto* lhs_value = lhs.is_null_at(row) ? nullptr :
&lhs.get_value(row); \
+ const auto* rhs_value = rhs.is_null_at(row) ? nullptr :
&rhs.get_value(row); \
+ if (!lhs_value || !rhs_value) {
\
+ if constexpr (std::is_same_v<CLASS, BitmapOrCount>) {
\
+ const auto* value = lhs_value ? lhs_value :
rhs_value; \
+ res[row] = value ? value->cardinality() : 0;
\
+ } else {
\
+ res[row] = 0;
\
+ }
\
+ } else {
\
+ res[row] = lhs_value->COUNT_OP(*rhs_value);
\
+ }
\
+ }
\
+ return Status::OK();
\
}
\
- for (size_t col = 1; col < col_size; ++col) {
\
- if (auto* nullable =
\
-
check_and_get_column<ColumnNullable>(*argument_columns[col])) { \
- BITMAP_OR_NULLABLE(nullable, input_rows_count, vals, OP);
\
- } else {
\
- const auto& col_data =
\
- assert_cast<const
ColumnBitmap*>(argument_columns[col].get()) \
- ->get_data();
\
- for (size_t row = 0; row < input_rows_count; ++row) {
\
- vals[row] OP col_data[row];
\
+ TData vals(input_rows_count);
\
+ const auto first_column =
make_bitmap_column_accessor(argument_columns[0]); \
+ for (size_t row = 0; row < input_rows_count; ++row) {
\
+ if (!first_column.is_null_at(row)) {
\
+ vals[row] = first_column.get_value(row);
\
+ }
\
+ }
\
+ for (size_t col = 1; col + 1 < col_size; ++col) {
\
+ const auto column =
make_bitmap_column_accessor(argument_columns[col]); \
+ for (size_t row = 0; row < input_rows_count; ++row) {
\
+ if (!column.is_null_at(row)) {
\
+ vals[row] OP column.get_value(row);
\
}
\
}
\
}
\
+ const auto last_column =
make_bitmap_column_accessor(argument_columns[col_size - 1]); \
for (size_t row = 0; row < input_rows_count; ++row) {
\
- res[row] = vals[row].cardinality();
\
+ const auto* lhs_value = &vals[row];
\
+ const auto* rhs_value =
\
+ last_column.is_null_at(row) ? nullptr :
&last_column.get_value(row); \
+ if (!rhs_value) {
\
+ if constexpr (std::is_same_v<CLASS, BitmapOrCount>) {
\
+ res[row] = lhs_value->cardinality();
\
+ } else {
\
+ res[row] = 0;
\
+ }
\
+ } else {
\
+ res[row] = lhs_value->COUNT_OP(*rhs_value);
\
+ }
\
}
\
return Status::OK();
\
+ }
\
+
\
+ private:
\
+ struct BitmapColumnAccessor {
\
+ const std::vector<BitmapValue>* values = nullptr;
\
+ const BitmapValue* const_value = nullptr;
\
+ const ColumnUInt8::value_type* null_map_data = nullptr;
\
+ bool is_const = false;
\
+ bool is_const_null = false;
\
+
\
+ bool is_null_at(size_t row) const {
\
+ return is_const ? is_const_null : (null_map_data &&
null_map_data[row]); \
+ }
\
+
\
+ const BitmapValue& get_value(size_t row) const {
\
+ return is_const ? *const_value : (*values)[row];
\
+ }
\
+ };
\
+
\
+ static BitmapColumnAccessor make_bitmap_column_accessor(const
ColumnPtr& column) { \
+ BitmapColumnAccessor accessor;
\
+ const auto& [data_column_ptr, is_const] = unpack_if_const(column);
\
+ accessor.is_const = is_const;
\
+ const IColumn* data_column = data_column_ptr.get();
\
+
\
+ if (const auto* nullable =
check_and_get_column<ColumnNullable>(*data_column)) { \
+ if (accessor.is_const) {
\
+ accessor.is_const_null = nullable->is_null_at(0);
\
+ } else {
\
+ accessor.null_map_data =
nullable->get_null_map_data().data(); \
+ }
\
+ data_column = nullable->get_nested_column_ptr().get();
\
+ }
\
+
\
+ const auto* bitmap_column = assert_cast<const
ColumnBitmap*>(data_column); \
+ if (accessor.is_const) {
\
+ accessor.const_value = &bitmap_column->get_data()[0];
\
+ } else {
\
+ accessor.values = &bitmap_column->get_data();
\
+ }
\
+
\
+ return accessor;
\
}
\
}
BITMAP_FUNCTION_VARIADIC(BitmapOr, bitmap_or, |=);
BITMAP_FUNCTION_VARIADIC(BitmapAnd, bitmap_and, &=);
BITMAP_FUNCTION_VARIADIC(BitmapXor, bitmap_xor, ^=);
-BITMAP_FUNCTION_COUNT_VARIADIC(BitmapOrCount, bitmap_or_count, |=);
-BITMAP_FUNCTION_COUNT_VARIADIC(BitmapAndCount, bitmap_and_count, &=);
-BITMAP_FUNCTION_COUNT_VARIADIC(BitmapXorCount, bitmap_xor_count, ^=);
+BITMAP_FUNCTION_COUNT_VARIADIC(BitmapOrCount, bitmap_or_count, |=,
or_cardinality);
+BITMAP_FUNCTION_COUNT_VARIADIC(BitmapAndCount, bitmap_and_count, &=,
and_cardinality);
+BITMAP_FUNCTION_COUNT_VARIADIC(BitmapXorCount, bitmap_xor_count, ^=,
xor_cardinality);
Status execute_bitmap_op_count_null_to_zero(
FunctionContext* context, Block& block, const ColumnNumbers&
arguments, uint32_t result,
@@ -162,12 +232,30 @@ Status execute_bitmap_op_count_null_to_zero(
const std::function<Status(FunctionContext*, Block&, const
ColumnNumbers&, size_t, size_t)>&
exec_impl_func);
-template <typename Impl>
+template <typename Impl, bool NewVersion = false>
class FunctionBitMapVariadic : public IFunction {
+ static_assert(!NewVersion || (NewVersion && (std::is_same_v<Impl,
BitmapOrCount> ||
+ std::is_same_v<Impl,
BitmapAndCount> ||
+ std::is_same_v<Impl,
BitmapXorCount>)));
+
public:
- static constexpr auto name = Impl::name;
+ static constexpr auto name = []() constexpr {
+ if constexpr (!NewVersion) {
+ return Impl::name;
+ } else if constexpr (std::is_same_v<Impl, BitmapOrCount>) {
+ return "bitmap_or_count_v2";
+ } else if constexpr (std::is_same_v<Impl, BitmapAndCount>) {
+ return "bitmap_and_count_v2";
+ } else if constexpr (std::is_same_v<Impl, BitmapXorCount>) {
+ return "bitmap_xor_count_v2";
+ } else {
+ return Impl::name;
+ }
+ }();
- static FunctionPtr create() { return
std::make_shared<FunctionBitMapVariadic>(); }
+ static FunctionPtr create() {
+ return std::make_shared<FunctionBitMapVariadic<Impl, NewVersion>>();
+ }
String get_name() const override { return name; }
@@ -177,6 +265,9 @@ public:
DataTypePtr get_return_type_impl(const DataTypes& arguments) const
override {
using ResultDataType = typename Impl::ResultDataType;
+ if constexpr (NewVersion) {
+ return std::make_shared<ResultDataType>();
+ }
if (std::is_same_v<Impl, BitmapOr> || is_count()) {
bool return_nullable = false;
// result is nullable only when any columns is nullable for
bitmap_or and bitmap_or_count
@@ -202,7 +293,8 @@ public:
Status execute_impl(FunctionContext* context, Block& block, const
ColumnNumbers& arguments,
uint32_t result, size_t input_rows_count) const
override {
- if (std::is_same_v<Impl, BitmapAndCount> || std::is_same_v<Impl,
BitmapXorCount>) {
+ if constexpr (std::is_same_v<Impl, BitmapAndCount> ||
+ std::is_same_v<Impl, BitmapXorCount>) {
auto impl_func = [&](FunctionContext* context, Block& block,
const ColumnNumbers& arguments, uint32_t
result,
size_t input_rows_count) {
@@ -219,12 +311,6 @@ public:
const ColumnNumbers& arguments, uint32_t
result,
size_t input_rows_count) const {
size_t argument_size = arguments.size();
- std::vector<ColumnPtr> argument_columns(argument_size);
-
- for (size_t i = 0; i < argument_size; ++i) {
- argument_columns[i] =
-
block.get_by_position(arguments[i]).column->convert_to_full_column_if_const();
- }
using ResultDataType = typename Impl::ResultDataType; //DataTypeBitMap
or DataTypeInt64
using ColVecResult =
std::conditional_t<is_complex_v<ResultDataType::PType>,
@@ -244,8 +330,18 @@ public:
auto& vec_res = col_res->get_data();
vec_res.resize(input_rows_count);
+ std::vector<ColumnPtr> argument_columns(argument_size);
+ for (size_t i = 0; i < argument_size; ++i) {
+ argument_columns[i] = block.get_by_position(arguments[i]).column;
+ if constexpr (!std::is_same_v<Impl, BitmapOrCount> &&
+ !std::is_same_v<Impl, BitmapAndCount> &&
+ !std::is_same_v<Impl, BitmapXorCount>) {
+ argument_columns[i] =
argument_columns[i]->convert_to_full_column_if_const();
+ }
+ }
RETURN_IF_ERROR(Impl::vector_vector(argument_columns.data(),
argument_size,
input_rows_count, vec_res,
col_res_nulls.get()));
+
if (!use_default_implementation_for_nulls() &&
result_info.type->is_nullable()) {
block.replace_by_position(
result, ColumnNullable::create(std::move(col_res),
std::move(col_res_nulls)));
@@ -268,6 +364,9 @@ using FunctionBitmapAnd = FunctionBitMapVariadic<BitmapAnd>;
using FunctionBitmapOrCount = FunctionBitMapVariadic<BitmapOrCount>;
using FunctionBitmapAndCount = FunctionBitMapVariadic<BitmapAndCount>;
using FunctionBitmapXorCount = FunctionBitMapVariadic<BitmapXorCount>;
+using FunctionBitmapOrCountV2 = FunctionBitMapVariadic<BitmapOrCount, true>;
+using FunctionBitmapAndCountV2 = FunctionBitMapVariadic<BitmapAndCount, true>;
+using FunctionBitmapXorCountV2 = FunctionBitMapVariadic<BitmapXorCount, true>;
void register_function_bitmap_variadic(SimpleFunctionFactory& factory) {
factory.register_function<FunctionBitmapOr>();
@@ -276,5 +375,8 @@ void
register_function_bitmap_variadic(SimpleFunctionFactory& factory) {
factory.register_function<FunctionBitmapOrCount>();
factory.register_function<FunctionBitmapAndCount>();
factory.register_function<FunctionBitmapXorCount>();
+ factory.register_function<FunctionBitmapOrCountV2>();
+ factory.register_function<FunctionBitmapAndCountV2>();
+ factory.register_function<FunctionBitmapXorCountV2>();
}
-} // namespace doris
\ No newline at end of file
+} // namespace doris
diff --git a/be/src/exprs/function/simple_function_factory.h
b/be/src/exprs/function/simple_function_factory.h
index c0c3fc1ad60..1b7ac1d8c70 100644
--- a/be/src/exprs/function/simple_function_factory.h
+++ b/be/src/exprs/function/simple_function_factory.h
@@ -204,6 +204,11 @@ public:
attr.new_version_unix_timestamp) {
key_str += "_new";
}
+ if (attr.new_version_bitmap_op_count &&
+ (key_str == "bitmap_and_count" || key_str == "bitmap_or_count" ||
+ key_str == "bitmap_xor_count" || key_str ==
"bitmap_and_not_count")) {
+ key_str += "_v2";
+ }
temporary_function_update(be_version, key_str);
diff --git a/be/src/exprs/vectorized_fn_call.cpp
b/be/src/exprs/vectorized_fn_call.cpp
index 56ba95cd755..0b78c49bb7e 100644
--- a/be/src/exprs/vectorized_fn_call.cpp
+++ b/be/src/exprs/vectorized_fn_call.cpp
@@ -216,7 +216,10 @@ Status VectorizedFnCall::prepare(RuntimeState* state,
const RowDescriptor& desc,
// get the function. won't prepare function.
_function = SimpleFunctionFactory::instance().get_function(
_fn.name.function_name, argument_template, _data_type,
- {.new_version_unix_timestamp =
state->query_options().new_version_unix_timestamp},
+ {.new_version_unix_timestamp =
state->query_options().new_version_unix_timestamp,
+ .new_version_bitmap_op_count =
+
state->query_options().__isset.new_version_bitmap_op_count &&
+ state->query_options().new_version_bitmap_op_count},
state->be_exec_version());
}
if (_function == nullptr) {
diff --git a/be/src/format_v2/expr/cast.cpp b/be/src/format_v2/expr/cast.cpp
index efeb9d851de..c1435732a9e 100644
--- a/be/src/format_v2/expr/cast.cpp
+++ b/be/src/format_v2/expr/cast.cpp
@@ -56,7 +56,10 @@ Status Cast::prepare(RuntimeState* state, const
RowDescriptor& desc, VExprContex
// get the function. won't prepare function.
_function = SimpleFunctionFactory::instance().get_function(
"CAST", argument_template, _data_type,
- {.new_version_unix_timestamp =
state->query_options().new_version_unix_timestamp},
+ {.new_version_unix_timestamp =
state->query_options().new_version_unix_timestamp,
+ .new_version_bitmap_op_count =
+
state->query_options().__isset.new_version_bitmap_op_count &&
+ state->query_options().new_version_bitmap_op_count},
state->be_exec_version());
if (_function == nullptr) {
return Status::InternalError("Could not find function {} ",
_expr_name);
diff --git a/be/test/core/value/bitmap_value_test.cpp
b/be/test/core/value/bitmap_value_test.cpp
index b281d71a443..97cbdef56cb 100644
--- a/be/test/core/value/bitmap_value_test.cpp
+++ b/be/test/core/value/bitmap_value_test.cpp
@@ -448,6 +448,7 @@ void check_bitmap_value_operator(const BitmapValue& left,
const BitmapValue& rig
auto right_cardinality = right.cardinality();
auto and_cardinality = left.and_cardinality(right);
auto and_not_cardinality = left.andnot_cardinality(right);
+ auto xor_cardinality = left.xor_cardinality(right);
std::cout << "left_cardinality: " << left_cardinality
<< ", right_cardinality: " << right_cardinality
@@ -475,7 +476,8 @@ void check_bitmap_value_operator(const BitmapValue& left,
const BitmapValue& rig
copy = left;
copy ^= right;
- EXPECT_EQ(copy.cardinality(), left_cardinality + right_cardinality -
and_cardinality * 2);
+ EXPECT_EQ(xor_cardinality, left_cardinality + right_cardinality -
and_cardinality * 2);
+ EXPECT_EQ(copy.cardinality(), xor_cardinality);
}
// '='
@@ -559,6 +561,7 @@ TEST(BitmapValueTest, operators) {
check_bitmap_value_operator(left_empty, right_bitmap);
check_bitmap_value_operator(left_single, right_emtpy);
+ check_bitmap_value_operator(left_single, left_single);
check_bitmap_value_operator(left_single, right_single);
check_bitmap_value_operator(left_single, right_set);
check_bitmap_value_operator(left_single, right_bitmap);
diff --git a/be/test/exprs/function/function_bitmap_test.cpp
b/be/test/exprs/function/function_bitmap_test.cpp
index 95813289427..aba66d453ba 100644
--- a/be/test/exprs/function/function_bitmap_test.cpp
+++ b/be/test/exprs/function/function_bitmap_test.cpp
@@ -346,9 +346,9 @@ TEST(function_bitmap_test, function_bitmap_and_count) {
BitmapValue bitmap1({1, 2, 3});
BitmapValue bitmap2({3, 4, 5});
BitmapValue empty_bitmap;
- DataSet data_set = {{{&bitmap1, &empty_bitmap}, (int64_t)0},
- {{&bitmap1, &bitmap1}, (int64_t)3},
- {{&bitmap1, &bitmap2}, (int64_t)1}};
+ DataSet data_set = {{{&bitmap1, &empty_bitmap}, (int64_t)0}, {{&bitmap1,
&bitmap1}, (int64_t)3},
+ {{&bitmap1, &bitmap2}, (int64_t)1}, {{Null(),
&bitmap1}, (int64_t)0},
+ {{&bitmap1, Null()}, (int64_t)0}, {{Null(),
Null()}, (int64_t)0}};
static_cast<void>(check_function<DataTypeInt64, true>(func_name,
input_types, data_set));
@@ -377,12 +377,26 @@ TEST(function_bitmap_test, function_bitmap_or_count) {
BitmapValue bitmap2({1, 2, 3, 4});
BitmapValue bitmap3({2, 3});
BitmapValue empty_bitmap;
- DataSet data_set = {{{&bitmap1, &empty_bitmap}, (int64_t)3},
- {{&bitmap2, &bitmap3}, (int64_t)4},
- {{&bitmap1, &bitmap3}, (int64_t)3}};
+ DataSet data_set = {{{&bitmap1, &empty_bitmap}, (int64_t)3}, {{&bitmap2,
&bitmap3}, (int64_t)4},
+ {{&bitmap1, &bitmap3}, (int64_t)3}, {{Null(),
&bitmap1}, (int64_t)3},
+ {{&bitmap1, Null()}, (int64_t)3}, {{Null(),
Null()}, (int64_t)0}};
static_cast<void>(check_function<DataTypeInt64, true>(func_name,
input_types, data_set));
+ {
+ const auto old_config = config::enable_set_in_bitmap_value;
+ config::enable_set_in_bitmap_value = true;
+ BitmapValue short_bitmap1({1, 2});
+ BitmapValue short_bitmap2({2, 3, 4, 5});
+ EXPECT_EQ(short_bitmap1.get_type_code(), BitmapTypeCode::SET);
+ EXPECT_EQ(short_bitmap2.get_type_code(), BitmapTypeCode::SET);
+
+ DataSet set_data_set = {{{&short_bitmap1, &short_bitmap2},
(int64_t)5}};
+ static_cast<void>(
+ check_function<DataTypeInt64, true>(func_name, input_types,
set_data_set));
+ config::enable_set_in_bitmap_value = old_config;
+ }
+
{
InputTypeSet input_types = {PrimitiveType::TYPE_BITMAP,
PrimitiveType::TYPE_BITMAP,
PrimitiveType::TYPE_BITMAP};
@@ -411,9 +425,9 @@ TEST(function_bitmap_test, function_bitmap_xor_count) {
BitmapValue bitmap3({2, 3});
BitmapValue bitmap4({1, 2, 6});
BitmapValue empty_bitmap;
- DataSet data_set = {{{&bitmap1, &empty_bitmap}, (int64_t)3},
- {{&bitmap2, &bitmap3}, (int64_t)2},
- {{&bitmap1, &bitmap4}, (int64_t)2}};
+ DataSet data_set = {{{&bitmap1, &empty_bitmap}, (int64_t)3}, {{&bitmap2,
&bitmap3}, (int64_t)2},
+ {{&bitmap1, &bitmap4}, (int64_t)2}, {{Null(),
&bitmap1}, (int64_t)0},
+ {{&bitmap1, Null()}, (int64_t)0}, {{Null(),
Null()}, (int64_t)0}};
static_cast<void>(check_function<DataTypeInt64, true>(func_name,
input_types, data_set));
@@ -443,13 +457,54 @@ TEST(function_bitmap_test, function_bitmap_and_not_count)
{
BitmapValue bitmap3({33, 5, std::numeric_limits<uint64_t>::max()});
BitmapValue empty_bitmap;
- DataSet data_set = {{{&bitmap1, &empty_bitmap}, (int64_t)3}, //1,2,3
- {{&bitmap2, Null()}, (int64_t)0},
- {{&bitmap2, &bitmap3}, (int64_t)3}, //0,3,4
- {{&bitmap1, &bitmap2}, (int64_t)2}}; //1,2
+ DataSet data_set = {
+ {{&bitmap1, &empty_bitmap}, (int64_t)3}, //1,2,3
+ {{Null(), &bitmap1}, (int64_t)0}, {{&bitmap2, Null()},
(int64_t)0},
+ {{Null(), Null()}, (int64_t)0}, {{&bitmap2, &bitmap3},
(int64_t)3}, //0,3,4
+ {{&bitmap1, &bitmap2}, (int64_t)2}};
//1,2
static_cast<void>(check_function<DataTypeInt64, true>(func_name,
input_types, data_set));
}
+
+TEST(function_bitmap_test, function_bitmap_op_count_v2_nullable_args) {
+ InputTypeSet input_types = {PrimitiveType::TYPE_BITMAP,
PrimitiveType::TYPE_BITMAP};
+ BitmapValue bitmap1({1, 2, 3});
+ BitmapValue bitmap2({3, 4, 5});
+
+ {
+ DataSet data_set = {{{&bitmap1, &bitmap2}, (int64_t)1},
+ {{Null(), &bitmap1}, (int64_t)0},
+ {{&bitmap1, Null()}, (int64_t)0},
+ {{Null(), Null()}, (int64_t)0}};
+ static_cast<void>(
+ check_function<DataTypeInt64, false>("bitmap_and_count_v2",
input_types, data_set));
+ }
+ {
+ DataSet data_set = {{{&bitmap1, &bitmap2}, (int64_t)5},
+ {{Null(), &bitmap1}, (int64_t)3},
+ {{&bitmap1, Null()}, (int64_t)3},
+ {{Null(), Null()}, (int64_t)0}};
+ static_cast<void>(
+ check_function<DataTypeInt64, false>("bitmap_or_count_v2",
input_types, data_set));
+ }
+ {
+ DataSet data_set = {{{&bitmap1, &bitmap2}, (int64_t)4},
+ {{Null(), &bitmap1}, (int64_t)0},
+ {{&bitmap1, Null()}, (int64_t)0},
+ {{Null(), Null()}, (int64_t)0}};
+ static_cast<void>(
+ check_function<DataTypeInt64, false>("bitmap_xor_count_v2",
input_types, data_set));
+ }
+ {
+ DataSet data_set = {{{&bitmap1, &bitmap2}, (int64_t)2},
+ {{Null(), &bitmap1}, (int64_t)0},
+ {{&bitmap1, Null()}, (int64_t)0},
+ {{Null(), Null()}, (int64_t)0}};
+ static_cast<void>(check_function<DataTypeInt64,
false>("bitmap_and_not_count_v2",
+ input_types,
data_set));
+ }
+}
+
TEST(function_bitmap_test, function_bitmap_and_not_count_alias) {
std::string func_name = "bitmap_andnot_count";
InputTypeSet input_types = {PrimitiveType::TYPE_BITMAP,
PrimitiveType::TYPE_BITMAP};
diff --git a/be/test/exprs/function/simple_function_factory_test.cpp
b/be/test/exprs/function/simple_function_factory_test.cpp
index 86f344a3f03..dc8dcdf8583 100644
--- a/be/test/exprs/function/simple_function_factory_test.cpp
+++ b/be/test/exprs/function/simple_function_factory_test.cpp
@@ -20,7 +20,12 @@
#include <gtest/gtest.h>
#include <memory>
+#include <string>
+#include <utility>
+#include <vector>
+#include "core/data_type/data_type_bitmap.h"
+#include "core/data_type/data_type_nullable.h"
#include "core/data_type/data_type_number.h"
namespace doris {
@@ -95,4 +100,34 @@ TEST_F(SimpleFunctionFactoryTest, test_return_all) {
}
}
+TEST_F(SimpleFunctionFactoryTest, test_bitmap_count_new_version_return_type) {
+ ColumnsWithTypeAndName arguments = {
+ {nullptr, make_nullable(std::make_shared<DataTypeBitMap>()), ""},
+ {nullptr, make_nullable(std::make_shared<DataTypeBitMap>()), ""}};
+ auto& factory = SimpleFunctionFactory::instance();
+
+ for (const auto& [function_name, new_function_name] :
+ std::vector<std::pair<std::string, std::string>> {
+ {"bitmap_and_count", "bitmap_and_count_v2"},
+ {"bitmap_or_count", "bitmap_or_count_v2"},
+ {"bitmap_xor_count", "bitmap_xor_count_v2"},
+ {"bitmap_and_not_count", "bitmap_and_not_count_v2"}}) {
+ auto old_builder = std::dynamic_pointer_cast<FunctionBuilderImpl>(
+ factory.function_creators.find(function_name)->second());
+ ASSERT_NE(old_builder, nullptr);
+ EXPECT_TRUE(old_builder->get_return_type(arguments)->is_nullable());
+
+ auto new_builder = std::dynamic_pointer_cast<FunctionBuilderImpl>(
+ factory.function_creators.find(new_function_name)->second());
+ ASSERT_NE(new_builder, nullptr);
+ EXPECT_FALSE(new_builder->get_return_type(arguments)->is_nullable());
+
+ auto new_func = SimpleFunctionFactory::instance().get_function(
+ function_name, arguments, std::make_shared<DataTypeInt64>(),
+ {.new_version_bitmap_op_count = true},
BeExecVersionManager::get_newest_version());
+ EXPECT_EQ(new_func->get_name(), new_function_name);
+ EXPECT_FALSE(new_func->get_return_type()->is_nullable());
+ }
+}
+
} // namespace doris
\ No newline at end of file
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/catalog/BuiltinScalarFunctions.java
b/fe/fe-core/src/main/java/org/apache/doris/catalog/BuiltinScalarFunctions.java
index 3b1171a376c..a1326326778 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/catalog/BuiltinScalarFunctions.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/catalog/BuiltinScalarFunctions.java
@@ -103,10 +103,7 @@ import
org.apache.doris.nereids.trees.expressions.functions.scalar.BitShiftRight
import org.apache.doris.nereids.trees.expressions.functions.scalar.BitTest;
import org.apache.doris.nereids.trees.expressions.functions.scalar.BitmapAnd;
import
org.apache.doris.nereids.trees.expressions.functions.scalar.BitmapAndCount;
-import
org.apache.doris.nereids.trees.expressions.functions.scalar.BitmapAndNot;
-import
org.apache.doris.nereids.trees.expressions.functions.scalar.BitmapAndNotAlias;
import
org.apache.doris.nereids.trees.expressions.functions.scalar.BitmapAndNotCount;
-import
org.apache.doris.nereids.trees.expressions.functions.scalar.BitmapAndNotCountAlias;
import
org.apache.doris.nereids.trees.expressions.functions.scalar.BitmapContains;
import org.apache.doris.nereids.trees.expressions.functions.scalar.BitmapCount;
import org.apache.doris.nereids.trees.expressions.functions.scalar.BitmapEmpty;
@@ -678,10 +675,7 @@ public class BuiltinScalarFunctions implements
FunctionHelper {
scalar(BitLength.class, "bit_length"),
scalar(BitmapAnd.class, "bitmap_and"),
scalar(BitmapAndCount.class, "bitmap_and_count"),
- scalar(BitmapAndNot.class, "bitmap_and_not"),
- scalar(BitmapAndNotAlias.class, "bitmap_andnot"),
- scalar(BitmapAndNotCount.class, "bitmap_and_not_count"),
- scalar(BitmapAndNotCountAlias.class, "bitmap_andnot_count"),
+ scalar(BitmapAndNotCount.class, "bitmap_and_not_count",
"bitmap_andnot_count"),
scalar(BitmapContains.class, "bitmap_contains"),
scalar(BitmapCount.class, "bitmap_count"),
scalar(BitmapEmpty.class, "bitmap_empty"),
@@ -694,7 +688,7 @@ public class BuiltinScalarFunctions implements
FunctionHelper {
scalar(BitmapHash64.class, "bitmap_hash64"),
scalar(BitmapMax.class, "bitmap_max"),
scalar(BitmapMin.class, "bitmap_min"),
- scalar(BitmapNot.class, "bitmap_not"),
+ scalar(BitmapNot.class, "bitmap_not", "bitmap_and_not",
"bitmap_andnot"),
scalar(BitmapOr.class, "bitmap_or"),
scalar(BitmapOrCount.class, "bitmap_or_count"),
scalar(BitmapRemove.class, "bitmap_remove"),
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/nereids/load/NereidsStreamLoadPlanner.java
b/fe/fe-core/src/main/java/org/apache/doris/nereids/load/NereidsStreamLoadPlanner.java
index 6bc4a4974e9..ce581ad2010 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/nereids/load/NereidsStreamLoadPlanner.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/nereids/load/NereidsStreamLoadPlanner.java
@@ -323,6 +323,7 @@ public class NereidsStreamLoadPlanner {
queryOptions.setEnableMemtableOnSinkNode(enableMemtableOnSinkNode);
queryOptions.setNewVersionUnixTimestamp(true);
queryOptions.setNewVersionPercentile(true);
+ queryOptions.setNewVersionBitmapOpCount(true);
params.setQueryOptions(queryOptions);
TQueryGlobals queryGlobals = new TQueryGlobals();
queryGlobals.setNowString(TimeUtils.getDatetimeFormatWithTimeZone().format(LocalDateTime.now()));
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/ExpressionNormalization.java
b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/ExpressionNormalization.java
index c6b129e8c50..6155734100e 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/ExpressionNormalization.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/ExpressionNormalization.java
@@ -18,6 +18,7 @@
package org.apache.doris.nereids.rules.expression;
import org.apache.doris.nereids.rules.expression.check.CheckCast;
+import
org.apache.doris.nereids.rules.expression.rules.BitmapCountToBitmapOpCount;
import org.apache.doris.nereids.rules.expression.rules.ConcatWsMultiArrayToOne;
import org.apache.doris.nereids.rules.expression.rules.ConvertAggStateCast;
import org.apache.doris.nereids.rules.expression.rules.DigitalMaskingConvert;
@@ -76,7 +77,8 @@ public class ExpressionNormalization extends
ExpressionRewrite {
MergeDateTrunc.INSTANCE,
NormalizeElementAt.INSTANCE,
CheckCast.INSTANCE,
- SimplifyEqualBooleanLiteral.INSTANCE
+ SimplifyEqualBooleanLiteral.INSTANCE,
+ BitmapCountToBitmapOpCount.INSTANCE
)
);
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/ExpressionRuleType.java
b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/ExpressionRuleType.java
index b2e44c81c16..a3409004ca7 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/ExpressionRuleType.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/ExpressionRuleType.java
@@ -69,7 +69,8 @@ public enum ExpressionRuleType {
TIMESTAMP_TO_ADD_TIME,
TOPN_TO_MAX,
REWRITE_DEFAULT_EXPRESSION,
- ADD_SESSION_VAR_GUARD;
+ ADD_SESSION_VAR_GUARD,
+ BITMAP_COUNT_TO_BITMAP_OP_COUNT;
public int type() {
return ordinal();
}
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/rules/BitmapCountToBitmapOpCount.java
b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/rules/BitmapCountToBitmapOpCount.java
new file mode 100644
index 00000000000..8292d5cf62d
--- /dev/null
+++
b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/rules/BitmapCountToBitmapOpCount.java
@@ -0,0 +1,67 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.doris.nereids.rules.expression.rules;
+
+import org.apache.doris.nereids.rules.expression.ExpressionPatternMatcher;
+import org.apache.doris.nereids.rules.expression.ExpressionPatternRuleFactory;
+import org.apache.doris.nereids.rules.expression.ExpressionRuleType;
+import org.apache.doris.nereids.trees.expressions.Expression;
+import org.apache.doris.nereids.trees.expressions.functions.scalar.BitmapAnd;
+import
org.apache.doris.nereids.trees.expressions.functions.scalar.BitmapAndCount;
+import
org.apache.doris.nereids.trees.expressions.functions.scalar.BitmapAndNotCount;
+import org.apache.doris.nereids.trees.expressions.functions.scalar.BitmapCount;
+import org.apache.doris.nereids.trees.expressions.functions.scalar.BitmapNot;
+import org.apache.doris.nereids.trees.expressions.functions.scalar.BitmapOr;
+import
org.apache.doris.nereids.trees.expressions.functions.scalar.BitmapOrCount;
+import org.apache.doris.nereids.trees.expressions.functions.scalar.BitmapXor;
+import
org.apache.doris.nereids.trees.expressions.functions.scalar.BitmapXorCount;
+
+import com.google.common.collect.ImmutableList;
+
+import java.util.List;
+
+/** Rewrite bitmap_count(bitmap_op(...)) to corresponding bitmap_op_count
function. */
+public class BitmapCountToBitmapOpCount implements
ExpressionPatternRuleFactory {
+ public static final BitmapCountToBitmapOpCount INSTANCE = new
BitmapCountToBitmapOpCount();
+
+ @Override
+ public List<ExpressionPatternMatcher<? extends Expression>> buildRules() {
+ return ImmutableList.of(
+
matchesType(BitmapCount.class).then(BitmapCountToBitmapOpCount::rewrite)
+
.toRule(ExpressionRuleType.BITMAP_COUNT_TO_BITMAP_OP_COUNT)
+ );
+ }
+
+ private static Expression rewrite(BitmapCount bitmapCount) {
+ Expression child = bitmapCount.child(0);
+ if (child instanceof BitmapAnd) {
+ return new BitmapAndCount(child.child(0), child.child(1),
varArgs(child));
+ } else if (child instanceof BitmapOr) {
+ return new BitmapOrCount(child.child(0), child.child(1),
varArgs(child));
+ } else if (child instanceof BitmapXor) {
+ return new BitmapXorCount(child.child(0), child.child(1),
varArgs(child));
+ } else if (child instanceof BitmapNot) {
+ return new BitmapAndNotCount(child.child(0), child.child(1));
+ }
+ return bitmapCount;
+ }
+
+ private static Expression[] varArgs(Expression expression) {
+ return expression.children().subList(2,
expression.arity()).toArray(new Expression[0]);
+ }
+}
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/rules/FoldConstantRuleOnBE.java
b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/rules/FoldConstantRuleOnBE.java
index 50eee509628..3f6812405e8 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/rules/FoldConstantRuleOnBE.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/rules/FoldConstantRuleOnBE.java
@@ -322,6 +322,7 @@ public class FoldConstantRuleOnBE implements
ExpressionPatternRuleFactory {
tQueryOptions.setEnableDecimal256(context.getSessionVariable().isEnableDecimal256());
tQueryOptions.setNewVersionUnixTimestamp(true);
tQueryOptions.setNewVersionPercentile(true);
+ tQueryOptions.setNewVersionBitmapOpCount(true);
tQueryOptions.setEnableStrictCast(SessionVariable.enableStrictCast());
TFoldConstantParams tParams = new TFoldConstantParams(paramMap,
queryGlobals);
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/BitmapAndCount.java
b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/BitmapAndCount.java
index 6266b02735f..70a62d3a41a 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/BitmapAndCount.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/BitmapAndCount.java
@@ -19,6 +19,7 @@ package
org.apache.doris.nereids.trees.expressions.functions.scalar;
import org.apache.doris.catalog.FunctionSignature;
import org.apache.doris.nereids.trees.expressions.Expression;
+import org.apache.doris.nereids.trees.expressions.functions.AlwaysNotNullable;
import
org.apache.doris.nereids.trees.expressions.functions.ExplicitlyCastableSignature;
import org.apache.doris.nereids.trees.expressions.visitor.ExpressionVisitor;
import org.apache.doris.nereids.types.BigIntType;
@@ -34,7 +35,7 @@ import java.util.List;
* ScalarFunction 'bitmap_and_count'. This class is generated by
GenerateFunction.
*/
public class BitmapAndCount extends ScalarFunction
- implements ExplicitlyCastableSignature {
+ implements ExplicitlyCastableSignature, AlwaysNotNullable {
public static final List<FunctionSignature> SIGNATURES = ImmutableList.of(
FunctionSignature.ret(BigIntType.INSTANCE).varArgs(BitmapType.INSTANCE,
BitmapType.INSTANCE)
@@ -70,9 +71,4 @@ public class BitmapAndCount extends ScalarFunction
public <R, C> R accept(ExpressionVisitor<R, C> visitor, C context) {
return visitor.visitBitmapAndCount(this, context);
}
-
- @Override
- public boolean nullable() {
- return children().stream().anyMatch(Expression::nullable);
- }
}
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/BitmapAndNot.java
b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/BitmapAndNot.java
deleted file mode 100644
index 835b141027d..00000000000
---
a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/BitmapAndNot.java
+++ /dev/null
@@ -1,73 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-package org.apache.doris.nereids.trees.expressions.functions.scalar;
-
-import org.apache.doris.catalog.FunctionSignature;
-import org.apache.doris.nereids.trees.expressions.Expression;
-import
org.apache.doris.nereids.trees.expressions.functions.ExplicitlyCastableSignature;
-import org.apache.doris.nereids.trees.expressions.functions.PropagateNullable;
-import org.apache.doris.nereids.trees.expressions.shape.BinaryExpression;
-import org.apache.doris.nereids.trees.expressions.visitor.ExpressionVisitor;
-import org.apache.doris.nereids.types.BitmapType;
-
-import com.google.common.base.Preconditions;
-import com.google.common.collect.ImmutableList;
-
-import java.util.List;
-
-/**
- * ScalarFunction 'bitmap_and_not'. This class is generated by
GenerateFunction.
- */
-public class BitmapAndNot extends ScalarFunction
- implements BinaryExpression, ExplicitlyCastableSignature,
PropagateNullable {
-
- public static final List<FunctionSignature> SIGNATURES = ImmutableList.of(
-
FunctionSignature.ret(BitmapType.INSTANCE).args(BitmapType.INSTANCE,
BitmapType.INSTANCE)
- );
-
- /**
- * constructor with 2 arguments.
- */
- public BitmapAndNot(Expression arg0, Expression arg1) {
- super("bitmap_and_not", arg0, arg1);
- }
-
- /** constructor for withChildren and reuse signature */
- private BitmapAndNot(ScalarFunctionParams functionParams) {
- super(functionParams);
- }
-
- /**
- * withChildren.
- */
- @Override
- public BitmapAndNot withChildren(List<Expression> children) {
- Preconditions.checkArgument(children.size() == 2);
- return new BitmapAndNot(getFunctionParams(children));
- }
-
- @Override
- public List<FunctionSignature> getSignatures() {
- return SIGNATURES;
- }
-
- @Override
- public <R, C> R accept(ExpressionVisitor<R, C> visitor, C context) {
- return visitor.visitBitmapAndNot(this, context);
- }
-}
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/BitmapAndNotAlias.java
b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/BitmapAndNotAlias.java
deleted file mode 100644
index 16045904184..00000000000
---
a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/BitmapAndNotAlias.java
+++ /dev/null
@@ -1,73 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-package org.apache.doris.nereids.trees.expressions.functions.scalar;
-
-import org.apache.doris.catalog.FunctionSignature;
-import org.apache.doris.nereids.trees.expressions.Expression;
-import
org.apache.doris.nereids.trees.expressions.functions.ExplicitlyCastableSignature;
-import org.apache.doris.nereids.trees.expressions.functions.PropagateNullable;
-import org.apache.doris.nereids.trees.expressions.shape.BinaryExpression;
-import org.apache.doris.nereids.trees.expressions.visitor.ExpressionVisitor;
-import org.apache.doris.nereids.types.BitmapType;
-
-import com.google.common.base.Preconditions;
-import com.google.common.collect.ImmutableList;
-
-import java.util.List;
-
-/**
- * ScalarFunction 'bitmap_and_not'. This class is generated by
GenerateFunction.
- */
-public class BitmapAndNotAlias extends ScalarFunction
- implements BinaryExpression, ExplicitlyCastableSignature,
PropagateNullable {
-
- public static final List<FunctionSignature> SIGNATURES = ImmutableList.of(
-
FunctionSignature.ret(BitmapType.INSTANCE).args(BitmapType.INSTANCE,
BitmapType.INSTANCE)
- );
-
- /**
- * constructor with 2 arguments.
- */
- public BitmapAndNotAlias(Expression arg0, Expression arg1) {
- super("bitmap_andnot", arg0, arg1);
- }
-
- /** constructor for withChildren and reuse signature */
- private BitmapAndNotAlias(ScalarFunctionParams functionParams) {
- super(functionParams);
- }
-
- /**
- * withChildren.
- */
- @Override
- public BitmapAndNotAlias withChildren(List<Expression> children) {
- Preconditions.checkArgument(children.size() == 2);
- return new BitmapAndNotAlias(getFunctionParams(children));
- }
-
- @Override
- public List<FunctionSignature> getSignatures() {
- return SIGNATURES;
- }
-
- @Override
- public <R, C> R accept(ExpressionVisitor<R, C> visitor, C context) {
- return visitor.visitBitmapAndNotAlias(this, context);
- }
-}
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/BitmapAndNotCount.java
b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/BitmapAndNotCount.java
index 0976bc6fe05..ac00f957a18 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/BitmapAndNotCount.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/BitmapAndNotCount.java
@@ -19,6 +19,7 @@ package
org.apache.doris.nereids.trees.expressions.functions.scalar;
import org.apache.doris.catalog.FunctionSignature;
import org.apache.doris.nereids.trees.expressions.Expression;
+import org.apache.doris.nereids.trees.expressions.functions.AlwaysNotNullable;
import
org.apache.doris.nereids.trees.expressions.functions.ExplicitlyCastableSignature;
import org.apache.doris.nereids.trees.expressions.shape.BinaryExpression;
import org.apache.doris.nereids.trees.expressions.visitor.ExpressionVisitor;
@@ -34,7 +35,7 @@ import java.util.List;
* ScalarFunction 'bitmap_and_not_count'. This class is generated by
GenerateFunction.
*/
public class BitmapAndNotCount extends ScalarFunction
- implements BinaryExpression, ExplicitlyCastableSignature {
+ implements BinaryExpression, ExplicitlyCastableSignature,
AlwaysNotNullable {
public static final List<FunctionSignature> SIGNATURES = ImmutableList.of(
FunctionSignature.ret(BigIntType.INSTANCE).args(BitmapType.INSTANCE,
BitmapType.INSTANCE)
@@ -70,9 +71,4 @@ public class BitmapAndNotCount extends ScalarFunction
public <R, C> R accept(ExpressionVisitor<R, C> visitor, C context) {
return visitor.visitBitmapAndNotCount(this, context);
}
-
- @Override
- public boolean nullable() {
- return children().stream().anyMatch(Expression::nullable);
- }
}
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/BitmapAndNotCountAlias.java
b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/BitmapAndNotCountAlias.java
deleted file mode 100644
index 36205c847e2..00000000000
---
a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/BitmapAndNotCountAlias.java
+++ /dev/null
@@ -1,78 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-package org.apache.doris.nereids.trees.expressions.functions.scalar;
-
-import org.apache.doris.catalog.FunctionSignature;
-import org.apache.doris.nereids.trees.expressions.Expression;
-import
org.apache.doris.nereids.trees.expressions.functions.ExplicitlyCastableSignature;
-import org.apache.doris.nereids.trees.expressions.shape.BinaryExpression;
-import org.apache.doris.nereids.trees.expressions.visitor.ExpressionVisitor;
-import org.apache.doris.nereids.types.BigIntType;
-import org.apache.doris.nereids.types.BitmapType;
-
-import com.google.common.base.Preconditions;
-import com.google.common.collect.ImmutableList;
-
-import java.util.List;
-
-/**
- * ScalarFunction 'bitmap_and_not_count'. This class is generated by
GenerateFunction.
- */
-public class BitmapAndNotCountAlias extends ScalarFunction
- implements BinaryExpression, ExplicitlyCastableSignature {
-
- public static final List<FunctionSignature> SIGNATURES = ImmutableList.of(
-
FunctionSignature.ret(BigIntType.INSTANCE).args(BitmapType.INSTANCE,
BitmapType.INSTANCE)
- );
-
- /**
- * constructor with 2 arguments.
- */
- public BitmapAndNotCountAlias(Expression arg0, Expression arg1) {
- super("bitmap_andnot_count", arg0, arg1);
- }
-
- /** constructor for withChildren and reuse signature */
- private BitmapAndNotCountAlias(ScalarFunctionParams functionParams) {
- super(functionParams);
- }
-
- /**
- * withChildren.
- */
- @Override
- public BitmapAndNotCountAlias withChildren(List<Expression> children) {
- Preconditions.checkArgument(children.size() == 2);
- return new BitmapAndNotCountAlias(getFunctionParams(children));
- }
-
- @Override
- public List<FunctionSignature> getSignatures() {
- return SIGNATURES;
- }
-
- @Override
- public <R, C> R accept(ExpressionVisitor<R, C> visitor, C context) {
- return visitor.visitBitmapAndNotCountAlias(this, context);
- }
-
- @Override
- public boolean nullable() {
- return children().stream().anyMatch(Expression::nullable);
- }
-}
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/BitmapOrCount.java
b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/BitmapOrCount.java
index 92c51db51ac..5165325d03f 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/BitmapOrCount.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/BitmapOrCount.java
@@ -19,6 +19,7 @@ package
org.apache.doris.nereids.trees.expressions.functions.scalar;
import org.apache.doris.catalog.FunctionSignature;
import org.apache.doris.nereids.trees.expressions.Expression;
+import org.apache.doris.nereids.trees.expressions.functions.AlwaysNotNullable;
import
org.apache.doris.nereids.trees.expressions.functions.ExplicitlyCastableSignature;
import org.apache.doris.nereids.trees.expressions.visitor.ExpressionVisitor;
import org.apache.doris.nereids.types.BigIntType;
@@ -34,7 +35,7 @@ import java.util.List;
* ScalarFunction 'bitmap_or_count'. This class is generated by
GenerateFunction.
*/
public class BitmapOrCount extends ScalarFunction
- implements ExplicitlyCastableSignature {
+ implements ExplicitlyCastableSignature, AlwaysNotNullable {
public static final List<FunctionSignature> SIGNATURES = ImmutableList.of(
FunctionSignature.ret(BigIntType.INSTANCE).varArgs(BitmapType.INSTANCE,
BitmapType.INSTANCE)
@@ -70,8 +71,4 @@ public class BitmapOrCount extends ScalarFunction
public <R, C> R accept(ExpressionVisitor<R, C> visitor, C context) {
return visitor.visitBitmapOrCount(this, context);
}
-
- public boolean nullable() {
- return children().stream().anyMatch(Expression::nullable);
- }
}
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/BitmapXorCount.java
b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/BitmapXorCount.java
index ff6742ebafc..a532aa87e9f 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/BitmapXorCount.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/BitmapXorCount.java
@@ -19,6 +19,7 @@ package
org.apache.doris.nereids.trees.expressions.functions.scalar;
import org.apache.doris.catalog.FunctionSignature;
import org.apache.doris.nereids.trees.expressions.Expression;
+import org.apache.doris.nereids.trees.expressions.functions.AlwaysNotNullable;
import
org.apache.doris.nereids.trees.expressions.functions.ExplicitlyCastableSignature;
import org.apache.doris.nereids.trees.expressions.visitor.ExpressionVisitor;
import org.apache.doris.nereids.types.BigIntType;
@@ -34,7 +35,7 @@ import java.util.List;
* ScalarFunction 'bitmap_xor_count'. This class is generated by
GenerateFunction.
*/
public class BitmapXorCount extends ScalarFunction
- implements ExplicitlyCastableSignature {
+ implements ExplicitlyCastableSignature, AlwaysNotNullable {
public static final List<FunctionSignature> SIGNATURES = ImmutableList.of(
FunctionSignature.ret(BigIntType.INSTANCE).varArgs(BitmapType.INSTANCE,
BitmapType.INSTANCE)
@@ -70,9 +71,4 @@ public class BitmapXorCount extends ScalarFunction
public <R, C> R accept(ExpressionVisitor<R, C> visitor, C context) {
return visitor.visitBitmapXorCount(this, context);
}
-
- @Override
- public boolean nullable() {
- return children().stream().anyMatch(Expression::nullable);
- }
}
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/visitor/ScalarFunctionVisitor.java
b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/visitor/ScalarFunctionVisitor.java
index 7b1724ae085..6b73a00b854 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/visitor/ScalarFunctionVisitor.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/visitor/ScalarFunctionVisitor.java
@@ -113,10 +113,7 @@ import
org.apache.doris.nereids.trees.expressions.functions.scalar.BitShiftRight
import org.apache.doris.nereids.trees.expressions.functions.scalar.BitTest;
import org.apache.doris.nereids.trees.expressions.functions.scalar.BitmapAnd;
import
org.apache.doris.nereids.trees.expressions.functions.scalar.BitmapAndCount;
-import
org.apache.doris.nereids.trees.expressions.functions.scalar.BitmapAndNot;
-import
org.apache.doris.nereids.trees.expressions.functions.scalar.BitmapAndNotAlias;
import
org.apache.doris.nereids.trees.expressions.functions.scalar.BitmapAndNotCount;
-import
org.apache.doris.nereids.trees.expressions.functions.scalar.BitmapAndNotCountAlias;
import
org.apache.doris.nereids.trees.expressions.functions.scalar.BitmapContains;
import org.apache.doris.nereids.trees.expressions.functions.scalar.BitmapCount;
import org.apache.doris.nereids.trees.expressions.functions.scalar.BitmapEmpty;
@@ -929,22 +926,10 @@ public interface ScalarFunctionVisitor<R, C> {
return visitScalarFunction(bitmapAndCount, context);
}
- default R visitBitmapAndNot(BitmapAndNot bitmapAndNot, C context) {
- return visitScalarFunction(bitmapAndNot, context);
- }
-
default R visitBitmapAndNotCount(BitmapAndNotCount bitmapAndNotCount, C
context) {
return visitScalarFunction(bitmapAndNotCount, context);
}
- default R visitBitmapAndNotAlias(BitmapAndNotAlias bitmapAndNotAlias, C
context) {
- return visitScalarFunction(bitmapAndNotAlias, context);
- }
-
- default R visitBitmapAndNotCountAlias(BitmapAndNotCountAlias
bitmapAndNotCountAlias, C context) {
- return visitScalarFunction(bitmapAndNotCountAlias, context);
- }
-
default R visitBitmapContains(BitmapContains bitmapContains, C context) {
return visitScalarFunction(bitmapContains, context);
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/Coordinator.java
b/fe/fe-core/src/main/java/org/apache/doris/qe/Coordinator.java
index 74e0128e8da..c8b8d6000df 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/qe/Coordinator.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/qe/Coordinator.java
@@ -402,6 +402,9 @@ public class Coordinator implements CoordInterface {
this.queryGlobals.setTimeZone(timezone);
this.queryGlobals.setLoadZeroTolerance(loadZeroTolerance);
this.queryOptions.setBeExecVersion(Config.be_exec_version);
+ this.queryOptions.setNewVersionUnixTimestamp(true);
+ this.queryOptions.setNewVersionPercentile(true);
+ this.queryOptions.setNewVersionBitmapOpCount(true);
List<Integer> fragmentIds = new ArrayList<>();
for (PlanFragment fragment : fragments) {
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/qe/CoordinatorContext.java
b/fe/fe-core/src/main/java/org/apache/doris/qe/CoordinatorContext.java
index 1caae547f06..f619d7f4917 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/qe/CoordinatorContext.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/qe/CoordinatorContext.java
@@ -310,6 +310,7 @@ public class CoordinatorContext {
queryOptions.setBeExecVersion(Config.be_exec_version);
queryOptions.setNewVersionUnixTimestamp(true);
queryOptions.setNewVersionPercentile(true);
+ queryOptions.setNewVersionBitmapOpCount(true);
TQueryGlobals queryGlobals = new TQueryGlobals();
queryGlobals.setNowString(TimeUtils.getDatetimeFormatWithTimeZone().format(LocalDateTime.now()));
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/qe/NereidsCoordinator.java
b/fe/fe-core/src/main/java/org/apache/doris/qe/NereidsCoordinator.java
index 19c2eaec3ad..bc4c74335ca 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/qe/NereidsCoordinator.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/qe/NereidsCoordinator.java
@@ -504,6 +504,7 @@ public class NereidsCoordinator extends Coordinator {
this.coordinatorContext.queryOptions.setDisableFileCache(true);
this.coordinatorContext.queryOptions.setNewVersionUnixTimestamp(true);
this.coordinatorContext.queryOptions.setNewVersionPercentile(true);
+ this.coordinatorContext.queryOptions.setNewVersionBitmapOpCount(true);
}
private void setForQuery() {
diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java
b/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java
index df3fcbac6e0..ca2e3a6394f 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java
@@ -5866,6 +5866,7 @@ public class SessionVariable implements Serializable,
Writable {
tResult.setEnableInsertStrict(enableInsertStrict);
tResult.setNewVersionUnixTimestamp(true); // once FE upgraded, always
use new version
tResult.setNewVersionPercentile(true);
+ tResult.setNewVersionBitmapOpCount(true);
tResult.setHnswEfSearch(hnswEFSearch);
tResult.setHnswCheckRelativeDistance(hnswCheckRelativeDistance);
diff --git
a/fe/fe-core/src/test/java/org/apache/doris/cloud/catalog/CloudEnvFactoryTest.java
b/fe/fe-core/src/test/java/org/apache/doris/cloud/catalog/CloudEnvFactoryTest.java
index e1687ae21d1..c7af444c99c 100644
---
a/fe/fe-core/src/test/java/org/apache/doris/cloud/catalog/CloudEnvFactoryTest.java
+++
b/fe/fe-core/src/test/java/org/apache/doris/cloud/catalog/CloudEnvFactoryTest.java
@@ -17,16 +17,23 @@
package org.apache.doris.cloud.catalog;
+import org.apache.doris.analysis.DescriptorTable;
import org.apache.doris.catalog.Env;
import org.apache.doris.catalog.EnvFactory;
import org.apache.doris.cloud.datasource.CloudInternalCatalog;
+import org.apache.doris.cloud.qe.CloudCoordinator;
import org.apache.doris.common.Config;
+import org.apache.doris.common.FeConstants;
import org.apache.doris.common.util.PropertyAnalyzer;
+import org.apache.doris.qe.ConnectContext;
+import org.apache.doris.qe.Coordinator;
+import org.apache.doris.thrift.TUniqueId;
import com.google.common.collect.Maps;
import org.junit.Assert;
import org.junit.Test;
+import java.util.Collections;
import java.util.Map;
public class CloudEnvFactoryTest {
@@ -51,4 +58,29 @@ public class CloudEnvFactoryTest {
Assert.assertEquals("1",
properties.get(PropertyAnalyzer.PROPERTIES_REPLICATION_NUM));
}
+ @Test
+ public void testLegacyLoadCoordinatorSetsFunctionVersionOptions() {
+ boolean runningUnitTest = FeConstants.runningUnitTest;
+ FeConstants.runningUnitTest = true;
+ try {
+ ConnectContext context = new ConnectContext();
+
context.getSessionVariable().setEnableNereidsDistributePlanner(false);
+ context.setThreadLocalInfo();
+ Coordinator coordinator = new CloudEnvFactory().createCoordinator(
+ 1L, new TUniqueId(1L, 1L), new DescriptorTable(),
+ Collections.emptyList(), Collections.emptyList(), "UTC",
false, false);
+
+ Assert.assertTrue(coordinator instanceof CloudCoordinator);
+
Assert.assertTrue(coordinator.getQueryOptions().isSetNewVersionUnixTimestamp());
+
Assert.assertTrue(coordinator.getQueryOptions().isNewVersionUnixTimestamp());
+
Assert.assertTrue(coordinator.getQueryOptions().isSetNewVersionPercentile());
+
Assert.assertTrue(coordinator.getQueryOptions().isNewVersionPercentile());
+
Assert.assertTrue(coordinator.getQueryOptions().isSetNewVersionBitmapOpCount());
+
Assert.assertTrue(coordinator.getQueryOptions().isNewVersionBitmapOpCount());
+ } finally {
+ ConnectContext.remove();
+ FeConstants.runningUnitTest = runningUnitTest;
+ }
+ }
+
}
diff --git
a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/analysis/FunctionRegistryTest.java
b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/analysis/FunctionRegistryTest.java
index 0b3e4cb19f6..df7f4ca319e 100644
---
a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/analysis/FunctionRegistryTest.java
+++
b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/analysis/FunctionRegistryTest.java
@@ -21,16 +21,19 @@ import org.apache.doris.catalog.FunctionRegistry;
import org.apache.doris.catalog.FunctionSignature;
import org.apache.doris.nereids.exceptions.AnalysisException;
import org.apache.doris.nereids.trees.expressions.Expression;
+import org.apache.doris.nereids.trees.expressions.SlotReference;
import org.apache.doris.nereids.trees.expressions.functions.BoundFunction;
import
org.apache.doris.nereids.trees.expressions.functions.BuiltinFunctionBuilder;
import
org.apache.doris.nereids.trees.expressions.functions.ExplicitlyCastableSignature;
import org.apache.doris.nereids.trees.expressions.functions.FunctionBuilder;
import org.apache.doris.nereids.trees.expressions.functions.PropagateNullable;
+import
org.apache.doris.nereids.trees.expressions.functions.scalar.BitmapAndNotCount;
import
org.apache.doris.nereids.trees.expressions.functions.scalar.ScalarFunction;
import org.apache.doris.nereids.trees.expressions.functions.scalar.Substring;
import org.apache.doris.nereids.trees.expressions.functions.scalar.Year;
import org.apache.doris.nereids.trees.expressions.literal.Literal;
import org.apache.doris.nereids.trees.expressions.shape.UnaryExpression;
+import org.apache.doris.nereids.types.BitmapType;
import org.apache.doris.nereids.types.IntegerType;
import org.apache.doris.nereids.util.MemoPatternMatchSupported;
import org.apache.doris.nereids.util.MemoTestUtils;
@@ -87,6 +90,24 @@ public class FunctionRegistryTest implements
MemoPatternMatchSupported {
);
}
+ @Test
+ public void testBitmapAndNotCountAliasNotNullable() {
+ FunctionRegistry functionRegistry = new FunctionRegistry();
+ ImmutableList<Expression> arguments = ImmutableList.of(
+ new SlotReference("nullableBitmap1", BitmapType.INSTANCE,
true),
+ new SlotReference("nullableBitmap2", BitmapType.INSTANCE,
true));
+
+ FunctionBuilder aliasBuilder =
functionRegistry.findFunctionBuilder("bitmap_andnot_count", arguments);
+ Expression aliasFunction = aliasBuilder.build("bitmap_andnot_count",
arguments).first;
+ Assertions.assertInstanceOf(BitmapAndNotCount.class, aliasFunction);
+ Assertions.assertFalse(aliasFunction.nullable());
+
+ FunctionBuilder canonicalBuilder =
functionRegistry.findFunctionBuilder("bitmap_and_not_count", arguments);
+ Expression canonicalFunction =
canonicalBuilder.build("bitmap_and_not_count", arguments).first;
+ Assertions.assertInstanceOf(BitmapAndNotCount.class,
canonicalFunction);
+ Assertions.assertFalse(canonicalFunction.nullable());
+ }
+
@Test
public void testOverrideArity() {
// the substring function has 2 override functions:
diff --git
a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/expression/rules/BitmapCountToBitmapOpCountTest.java
b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/expression/rules/BitmapCountToBitmapOpCountTest.java
new file mode 100644
index 00000000000..7275e9eedb7
--- /dev/null
+++
b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/expression/rules/BitmapCountToBitmapOpCountTest.java
@@ -0,0 +1,115 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.doris.nereids.rules.expression.rules;
+
+import org.apache.doris.nereids.rules.expression.ExpressionRewriteTestHelper;
+import org.apache.doris.nereids.rules.expression.ExpressionRuleExecutor;
+import org.apache.doris.nereids.trees.expressions.Expression;
+import org.apache.doris.nereids.trees.expressions.SlotReference;
+import org.apache.doris.nereids.trees.expressions.functions.scalar.BitmapAnd;
+import
org.apache.doris.nereids.trees.expressions.functions.scalar.BitmapAndCount;
+import
org.apache.doris.nereids.trees.expressions.functions.scalar.BitmapAndNotCount;
+import org.apache.doris.nereids.trees.expressions.functions.scalar.BitmapCount;
+import org.apache.doris.nereids.trees.expressions.functions.scalar.BitmapNot;
+import org.apache.doris.nereids.trees.expressions.functions.scalar.BitmapOr;
+import
org.apache.doris.nereids.trees.expressions.functions.scalar.BitmapOrCount;
+import org.apache.doris.nereids.trees.expressions.functions.scalar.BitmapXor;
+import
org.apache.doris.nereids.trees.expressions.functions.scalar.BitmapXorCount;
+import org.apache.doris.nereids.types.BitmapType;
+
+import com.google.common.collect.ImmutableList;
+import org.junit.jupiter.api.Assertions;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Test;
+
+class BitmapCountToBitmapOpCountTest extends ExpressionRewriteTestHelper {
+
+ private final SlotReference bitmap1 = new SlotReference("bitmap1",
BitmapType.INSTANCE, true);
+ private final SlotReference bitmap2 = new SlotReference("bitmap2",
BitmapType.INSTANCE, true);
+ private final SlotReference bitmap3 = new SlotReference("bitmap3",
BitmapType.INSTANCE, true);
+
+ @BeforeEach
+ void setUp() {
+ executor = new ExpressionRuleExecutor(ImmutableList.of(
+ bottomUp(BitmapCountToBitmapOpCount.INSTANCE)
+ ));
+ }
+
+ @Test
+ void testRewriteBitmapAnd() {
+ assertRewrite(
+ new BitmapCount(new BitmapAnd(bitmap1, bitmap2)),
+ new BitmapAndCount(bitmap1, bitmap2)
+ );
+ }
+
+ @Test
+ void testRewriteBitmapAndWithVarArgs() {
+ assertRewrite(
+ new BitmapCount(new BitmapAnd(bitmap1, bitmap2, bitmap3)),
+ new BitmapAndCount(bitmap1, bitmap2, bitmap3)
+ );
+ }
+
+ @Test
+ void testRewriteBitmapOr() {
+ assertRewrite(
+ new BitmapCount(new BitmapOr(bitmap1, bitmap2, bitmap3)),
+ new BitmapOrCount(bitmap1, bitmap2, bitmap3)
+ );
+ }
+
+ @Test
+ void testRewriteBitmapXor() {
+ assertRewrite(
+ new BitmapCount(new BitmapXor(bitmap1, bitmap2)),
+ new BitmapXorCount(bitmap1, bitmap2)
+ );
+ }
+
+ @Test
+ void testRewriteBitmapNot() {
+ assertRewrite(
+ new BitmapCount(new BitmapNot(bitmap1, bitmap2)),
+ new BitmapAndNotCount(bitmap1, bitmap2)
+ );
+ }
+
+ @Test
+ void testKeepPlainBitmapCount() {
+ assertRewrite(
+ new BitmapCount(bitmap1),
+ new BitmapCount(bitmap1)
+ );
+ }
+
+ @Test
+ void testBitmapOpCountNotNullableMetadata() {
+ Assertions.assertFalse(new BitmapAndCount(bitmap1,
bitmap2).nullable());
+ Assertions.assertFalse(new BitmapOrCount(bitmap1, bitmap2).nullable());
+ Assertions.assertFalse(new BitmapXorCount(bitmap1,
bitmap2).nullable());
+ Assertions.assertFalse(new BitmapAndNotCount(bitmap1,
bitmap2).nullable());
+ }
+
+ @Test
+ void testRewriteToNotNullableBitmapOpCount() {
+ Expression rewrittenExpression = executor.rewrite(new BitmapCount(new
BitmapAnd(bitmap1, bitmap2)), context);
+ Assertions.assertInstanceOf(BitmapAndCount.class, rewrittenExpression);
+ Assertions.assertFalse(rewrittenExpression.nullable());
+ }
+}
diff --git
a/fe/fe-core/src/test/java/org/apache/doris/qe/ShortCircuitQueryContextTest.java
b/fe/fe-core/src/test/java/org/apache/doris/qe/ShortCircuitQueryContextTest.java
index 45e4927c403..e24a57ee465 100644
---
a/fe/fe-core/src/test/java/org/apache/doris/qe/ShortCircuitQueryContextTest.java
+++
b/fe/fe-core/src/test/java/org/apache/doris/qe/ShortCircuitQueryContextTest.java
@@ -17,12 +17,20 @@
package org.apache.doris.qe;
+import org.apache.doris.analysis.DescriptorTable;
+import org.apache.doris.analysis.Queriable;
import org.apache.doris.catalog.OlapTable;
+import org.apache.doris.planner.OlapScanNode;
+import org.apache.doris.planner.Planner;
+import org.apache.doris.thrift.TQueryOptions;
+import org.apache.thrift.TDeserializer;
import org.junit.jupiter.api.Assertions;
import org.junit.jupiter.api.Test;
import org.mockito.Mockito;
+import java.util.Collections;
+
public class ShortCircuitQueryContextTest {
private OlapTable table(String name, int schemaVersion) {
OlapTable table = Mockito.spy(new OlapTable());
@@ -55,4 +63,29 @@ public class ShortCircuitQueryContextTest {
Assertions.assertFalse(context.isReusable(connectContext(0)));
}
+
+ @Test
+ public void testSerializedQueryOptionsKeepBitmapOpCountVersion() throws
Exception {
+ TQueryOptions queryOptions = new SessionVariable().toThrift();
+ Planner planner = Mockito.mock(Planner.class);
+ Mockito.when(planner.getQueryOptions()).thenReturn(queryOptions);
+ DescriptorTable descriptorTable = new DescriptorTable();
+ descriptorTable.createTupleDescriptor();
+ Mockito.when(planner.getDescTable()).thenReturn(descriptorTable);
+
+ OlapScanNode scanNode = Mockito.mock(OlapScanNode.class);
+ OlapTable table = table("tbl", 10);
+
Mockito.when(scanNode.getPointQueryProjectList()).thenReturn(Collections.emptyList());
+ Mockito.when(scanNode.getOlapTable()).thenReturn(table);
+ Mockito.when(scanNode.getTableNameInPlan()).thenReturn("tbl");
+
Mockito.when(planner.getScanNodes()).thenReturn(Collections.singletonList(scanNode));
+
+ ShortCircuitQueryContext context =
+ new ShortCircuitQueryContext(planner,
Mockito.mock(Queriable.class));
+ TQueryOptions serializedQueryOptions = new TQueryOptions();
+ new TDeserializer().deserialize(serializedQueryOptions,
context.serializedQueryOptions.toByteArray());
+
+
Assertions.assertTrue(serializedQueryOptions.isSetNewVersionBitmapOpCount());
+
Assertions.assertTrue(serializedQueryOptions.isNewVersionBitmapOpCount());
+ }
}
diff --git a/gensrc/thrift/PaloInternalService.thrift
b/gensrc/thrift/PaloInternalService.thrift
index 578ff2fca79..d15765ecf82 100644
--- a/gensrc/thrift/PaloInternalService.thrift
+++ b/gensrc/thrift/PaloInternalService.thrift
@@ -507,6 +507,7 @@ struct TQueryOptions {
225: optional i64 runtime_filter_tree_publish_max_send_bytes = 268435456
226: optional bool enable_prune_nested_column = false;
+ 227: optional bool new_version_bitmap_op_count = false;
// For cloud, to control if the content would be written into file cache
// In write path, to control if the content would be written into file cache.
// In read path, read from file cache or remote storage when execute query.
diff --git
a/regression-test/data/query_p0/sql_functions/bitmap_functions/test_bitmap_function.out
b/regression-test/data/query_p0/sql_functions/bitmap_functions/test_bitmap_function.out
index 8fa306872e2..2df96803a0f 100644
---
a/regression-test/data/query_p0/sql_functions/bitmap_functions/test_bitmap_function.out
+++
b/regression-test/data/query_p0/sql_functions/bitmap_functions/test_bitmap_function.out
@@ -228,6 +228,9 @@ true
-- !sql_bitmap_and_count6 --
0
+-- !sql_bitmap_and_count_nulls --
+0 0 0
+
-- !sql_bitmap_or_count1 --
3
@@ -237,12 +240,21 @@ true
-- !sql_bitmap_or_count3 --
5
+-- !sql_bitmap_or_count_set_diff_length --
+5
+
-- !sql_bitmap_or_count4 --
6
-- !sql_bitmap_or_count5 --
6
+-- !sql_bitmap_or_count_nulls --
+3 3 0
+
+-- !sql_bitmap_or_count_nulls_multi --
+5
+
-- !sql --
2
@@ -276,6 +288,12 @@ true
-- !sql_bitmap_xor_count6 --
0
+-- !sql_bitmap_xor_count_nulls --
+0 0 0
+
+-- !sql_bitmap_count_rewrite_nullable --
+3 3 0 0 0 0
+
-- !sql_bitmap_and_count7 --
1 1
2 1
diff --git
a/regression-test/suites/query_p0/sql_functions/bitmap_functions/test_bitmap_function.groovy
b/regression-test/suites/query_p0/sql_functions/bitmap_functions/test_bitmap_function.groovy
index 2c6640a6515..540b396f37f 100644
---
a/regression-test/suites/query_p0/sql_functions/bitmap_functions/test_bitmap_function.groovy
+++
b/regression-test/suites/query_p0/sql_functions/bitmap_functions/test_bitmap_function.groovy
@@ -307,13 +307,17 @@ suite("test_bitmap_function") {
qt_sql_bitmap_and_count4 """ select
bitmap_and_count(bitmap_from_string('1,2,3'), bitmap_from_string('1,2'),
bitmap_from_string('1,2,3,4,5')) """
qt_sql_bitmap_and_count5 """ select
bitmap_and_count(bitmap_from_string('1,2,3'), bitmap_from_string('1,2'),
bitmap_from_string('1,2,3,4,5'),bitmap_empty()) """
qt_sql_bitmap_and_count6 """ select
bitmap_and_count(bitmap_from_string('1,2,3'), bitmap_from_string('1,2'),
bitmap_from_string('1,2,3,4,5'), NULL) """
+ qt_sql_bitmap_and_count_nulls """ select bitmap_and_count(cast(NULL as
bitmap), bitmap_from_string('1,2,3')),
bitmap_and_count(bitmap_from_string('1,2,3'), cast(NULL as bitmap)),
bitmap_and_count(cast(NULL as bitmap), cast(NULL as bitmap)) """
// bitmap_or_count
qt_sql_bitmap_or_count1 """ select
bitmap_or_count(bitmap_from_string('1,2,3'),bitmap_empty()) """
qt_sql_bitmap_or_count2 """ select
bitmap_or_count(bitmap_from_string('1,2,3'),bitmap_from_string('1,2,3'))"""
qt_sql_bitmap_or_count3 """ select
bitmap_or_count(bitmap_from_string('1,2,3'),bitmap_from_string('3,4,5')) """
+ qt_sql_bitmap_or_count_set_diff_length """ select
bitmap_or_count(bitmap_from_string('1,2'), bitmap_from_string('2,3,4,5')) """
qt_sql_bitmap_or_count4 """ select
bitmap_or_count(bitmap_from_string('1,2,3'), bitmap_from_string('3,4,5'),
to_bitmap(100), bitmap_empty()) """
qt_sql_bitmap_or_count5 """ select
bitmap_or_count(bitmap_from_string('1,2,3'), bitmap_from_string('3,4,5'),
to_bitmap(100), NULL) """
+ qt_sql_bitmap_or_count_nulls """ select bitmap_or_count(cast(NULL as
bitmap), bitmap_from_string('1,2,3')),
bitmap_or_count(bitmap_from_string('1,2,3'), cast(NULL as bitmap)),
bitmap_or_count(cast(NULL as bitmap), cast(NULL as bitmap)) """
+ qt_sql_bitmap_or_count_nulls_multi """ select bitmap_or_count(cast(NULL as
bitmap), bitmap_from_string('1,2,3'), bitmap_from_string('3,4,5'), cast(NULL as
bitmap)) """
// BITMAP_XOR
qt_sql """ select
bitmap_count(bitmap_xor(bitmap_from_string('2,3'),bitmap_from_string('1,2,3,4')))
cnt """
@@ -329,6 +333,88 @@ suite("test_bitmap_function") {
qt_sql_bitmap_xor_count4 """ select
(bitmap_xor_count(bitmap_from_string('2,3'),bitmap_from_string('1,2,3,4'),bitmap_from_string('3,4,5')))
"""
qt_sql_bitmap_xor_count5 """ select
(bitmap_xor_count(bitmap_from_string('2,3'),bitmap_from_string('1,2,3,4'),bitmap_from_string('3,4,5'),bitmap_empty()))
"""
qt_sql_bitmap_xor_count6 """ select
(bitmap_xor_count(bitmap_from_string('2,3'),bitmap_from_string('1,2,3,4'),bitmap_from_string('3,4,5'),NULL))
"""
+ qt_sql_bitmap_xor_count_nulls """ select bitmap_xor_count(cast(NULL as
bitmap), bitmap_from_string('1,2,3')),
bitmap_xor_count(bitmap_from_string('1,2,3'), cast(NULL as bitmap)),
bitmap_xor_count(cast(NULL as bitmap), cast(NULL as bitmap)) """
+ qt_sql_bitmap_count_rewrite_nullable """ select
bitmap_count(bitmap_or(cast(NULL as bitmap), bitmap_from_string('1,2,3'))),
bitmap_or_count(cast(NULL as bitmap), bitmap_from_string('1,2,3')),
bitmap_count(bitmap_and(bitmap_from_string('1,2,3'), cast(NULL as bitmap))),
bitmap_and_count(bitmap_from_string('1,2,3'), cast(NULL as bitmap)),
bitmap_count(bitmap_xor(bitmap_from_string('1,2,3'), cast(NULL as bitmap))),
bitmap_xor_count(bitmap_from_string('1,2,3'), cast(NULL as bitmap)) """
+
+ sql """ DROP TABLE IF EXISTS test_bitmap_op_count_short_circuit """
+ sql """
+ CREATE TABLE test_bitmap_op_count_short_circuit (
+ id BIGINT NOT NULL,
+ lhs BITMAP NOT NULL,
+ rhs BITMAP NOT NULL
+ ) ENGINE=OLAP
+ UNIQUE KEY(id)
+ DISTRIBUTED BY HASH(id) BUCKETS 1
+ PROPERTIES (
+ "replication_num" = "1",
+ "store_row_column" = "true",
+ "enable_unique_key_merge_on_write" = "true",
+ "light_schema_change" = "true"
+ )
+ """
+ sql """
+ INSERT INTO test_bitmap_op_count_short_circuit VALUES
+ (1, bitmap_from_string('1,2,3'), bitmap_from_string('3,4'))
+ """
+ sql "SET enable_nereids_planner=true"
+ sql "SET enable_short_circuit_query=true"
+ explain {
+ sql """
+ SELECT
+ bitmap_count(bitmap_and(lhs, rhs)),
+ bitmap_count(bitmap_or(lhs, rhs)),
+ bitmap_count(bitmap_xor(lhs, rhs)),
+ bitmap_count(bitmap_and_not(lhs, rhs))
+ FROM test_bitmap_op_count_short_circuit
+ WHERE id = 1
+ """
+ contains "SHORT-CIRCUIT"
+ contains "bitmap_and_count"
+ contains "bitmap_or_count"
+ contains "bitmap_xor_count"
+ contains "bitmap_and_not_count"
+ }
+ def shortCircuitResult = sql """
+ SELECT
+ bitmap_count(bitmap_and(lhs, rhs)),
+ bitmap_count(bitmap_or(lhs, rhs)),
+ bitmap_count(bitmap_xor(lhs, rhs)),
+ bitmap_count(bitmap_and_not(lhs, rhs))
+ FROM test_bitmap_op_count_short_circuit
+ WHERE id = 1
+ """
+ sql "SET enable_short_circuit_query=false"
+ def regularResult = sql """
+ SELECT
+ bitmap_count(bitmap_and(lhs, rhs)),
+ bitmap_count(bitmap_or(lhs, rhs)),
+ bitmap_count(bitmap_xor(lhs, rhs)),
+ bitmap_count(bitmap_and_not(lhs, rhs))
+ FROM test_bitmap_op_count_short_circuit
+ WHERE id = 1
+ """
+ check2_doris(shortCircuitResult, regularResult)
+
+ sql "SET enable_short_circuit_query=true"
+ def shortCircuitNullableResult = sql """
+ SELECT
+ bitmap_count(bitmap_and(lhs, CAST(NULL AS BITMAP))),
+ bitmap_count(bitmap_or(lhs, CAST(NULL AS BITMAP))),
+ bitmap_count(bitmap_xor(lhs, CAST(NULL AS BITMAP)))
+ FROM test_bitmap_op_count_short_circuit
+ WHERE id = 1
+ """
+ sql "SET enable_short_circuit_query=false"
+ def regularNullableResult = sql """
+ SELECT
+ bitmap_count(bitmap_and(lhs, CAST(NULL AS BITMAP))),
+ bitmap_count(bitmap_or(lhs, CAST(NULL AS BITMAP))),
+ bitmap_count(bitmap_xor(lhs, CAST(NULL AS BITMAP)))
+ FROM test_bitmap_op_count_short_circuit
+ WHERE id = 1
+ """
+ check2_doris(shortCircuitNullableResult, regularNullableResult)
+ sql "SET enable_short_circuit_query=true"
// bitmap_and_count, bitmap_xor_count, bitmap_and_not_count of all
nullable column
sql """ DROP TABLE IF EXISTS test_bitmap1 """
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
