This is an automated email from the ASF dual-hosted git repository.
morningman pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/master by this push:
new 2b6f85ab96 [chore](macOS) Fix BE UT (#14307)
2b6f85ab96 is described below
commit 2b6f85ab96b6bd38700fbf26bffdafbf2ec4520c
Author: Adonis Ling <adonis0...@gmail.com>
AuthorDate: Fri Nov 18 10:13:38 2022 +0800
[chore](macOS) Fix BE UT (#14307)
#13195 left some unresolved issues. One of them is that some BE unit tests
fail.
This PR fixes this issue. Now, we can run the command ./run-be-ut.sh --run
successfully on macOS.
---
be/src/exprs/encryption_functions.h | 4 +-
be/src/util/string_util.h | 2 +-
be/test/CMakeLists.txt | 5 +-
be/test/env/env_posix_test.cpp | 4 ++
be/test/runtime/memory_scratch_sink_test.cpp | 2 +-
be/test/testutil/any_type.h | 86 ++++++++++++++++++++++++++++
be/test/testutil/test_util.cpp | 10 ++++
be/test/util/arrow/arrow_work_flow_test.cpp | 2 +-
be/test/util/path_util_test.cpp | 4 ++
be/test/util/threadpool_test.cpp | 6 ++
be/test/vec/function/function_test_util.cpp | 60 +++++++++----------
be/test/vec/function/function_test_util.h | 21 +++----
12 files changed, 160 insertions(+), 46 deletions(-)
diff --git a/be/src/exprs/encryption_functions.h
b/be/src/exprs/encryption_functions.h
index e4976ab519..1f72b96eb2 100644
--- a/be/src/exprs/encryption_functions.h
+++ b/be/src/exprs/encryption_functions.h
@@ -28,7 +28,7 @@ namespace doris {
class Expr;
struct ExprValue;
class TupleRow;
-static StringCaseUnorderedMap<EncryptionMode> aes_mode_map {
+inline StringCaseUnorderedMap<EncryptionMode> aes_mode_map {
{"AES_128_ECB", EncryptionMode::AES_128_ECB},
{"AES_192_ECB", EncryptionMode::AES_192_ECB},
{"AES_256_ECB", EncryptionMode::AES_256_ECB},
@@ -53,7 +53,7 @@ static StringCaseUnorderedMap<EncryptionMode> aes_mode_map {
{"AES_128_OFB", EncryptionMode::AES_128_OFB},
{"AES_192_OFB", EncryptionMode::AES_192_OFB},
{"AES_256_OFB", EncryptionMode::AES_256_OFB}};
-static StringCaseUnorderedMap<EncryptionMode> sm4_mode_map {
+inline StringCaseUnorderedMap<EncryptionMode> sm4_mode_map {
{"SM4_128_ECB", EncryptionMode::SM4_128_ECB},
{"SM4_128_CBC", EncryptionMode::SM4_128_CBC},
{"SM4_128_CFB128", EncryptionMode::SM4_128_CFB128},
diff --git a/be/src/util/string_util.h b/be/src/util/string_util.h
index 0d2fe7e3ee..ac19448164 100644
--- a/be/src/util/string_util.h
+++ b/be/src/util/string_util.h
@@ -104,7 +104,7 @@ public:
if (lhs.size() != rhs.size()) {
return false;
}
- return strncasecmp(lhs.c_str(), rhs.c_str(), 0) == 0;
+ return strncasecmp(lhs.c_str(), rhs.c_str(), lhs.size()) == 0;
}
};
diff --git a/be/test/CMakeLists.txt b/be/test/CMakeLists.txt
index bfae619cbf..ec4a252872 100644
--- a/be/test/CMakeLists.txt
+++ b/be/test/CMakeLists.txt
@@ -324,6 +324,10 @@ set(UTIL_TEST_FILES
util/interval_tree_test.cpp
util/key_util_test.cpp
)
+if (OS_MACOSX)
+ list(REMOVE_ITEM UTIL_TEST_FILES util/system_metrics_test.cpp)
+endif()
+
set(VEC_TEST_FILES
vec/aggregate_functions/agg_collect_test.cpp
vec/aggregate_functions/agg_test.cpp
@@ -372,7 +376,6 @@ set(VEC_TEST_FILES
vec/utils/arrow_column_to_doris_column_test.cpp
vec/olap/char_type_padding_test.cpp
)
-
add_executable(doris_be_test
${AGENT_TEST_FILES}
${COMMON_TEST_FILES}
diff --git a/be/test/env/env_posix_test.cpp b/be/test/env/env_posix_test.cpp
index f4343b7ba8..b325e66ad9 100644
--- a/be/test/env/env_posix_test.cpp
+++ b/be/test/env/env_posix_test.cpp
@@ -55,7 +55,11 @@ TEST_F(EnvPosixTest, random_access) {
auto st = env->new_writable_file(fname, &wfile);
EXPECT_TRUE(st.ok());
st = wfile->pre_allocate(1024);
+#ifndef __APPLE__
EXPECT_TRUE(st.ok());
+#else
+ EXPECT_FALSE(st.ok());
+#endif
// write data
Slice field1("123456789");
st = wfile->append(field1);
diff --git a/be/test/runtime/memory_scratch_sink_test.cpp
b/be/test/runtime/memory_scratch_sink_test.cpp
index bf9c4a2954..6ebf608231 100644
--- a/be/test/runtime/memory_scratch_sink_test.cpp
+++ b/be/test/runtime/memory_scratch_sink_test.cpp
@@ -71,7 +71,7 @@ public:
EXPECT_EQ(system("mkdir -p ./test_run/output/"), 0);
EXPECT_EQ(system("pwd"), 0);
- EXPECT_EQ(system("cp -r ./be/test/runtime/test_data/ ./test_run/."),
0);
+ EXPECT_EQ(system("cp -r ./be/test/runtime/test_data ./test_run/."), 0);
init();
}
diff --git a/be/test/testutil/any_type.h b/be/test/testutil/any_type.h
new file mode 100644
index 0000000000..96cb228e15
--- /dev/null
+++ b/be/test/testutil/any_type.h
@@ -0,0 +1,86 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#pragma once
+
+#include <any>
+#include <typeinfo>
+#include <variant>
+
+namespace doris {
+
+// Typeinfo for __int128 is missing on macOS, we can't use std::any directly.
+struct AnyType : public std::variant<std::any, __int128_t> {
+ AnyType() = default;
+
+ template <typename T>
+ AnyType(T value) {
+ this->emplace<std::any>(std::move(value));
+ }
+
+ AnyType(__int128_t value) { this->emplace<__int128_t>(value); }
+
+ const std::type_info* type() const {
+ if (std::holds_alternative<std::any>(*this)) {
+ const auto& type_info = std::get<std::any>(*this).type();
+ return &type_info;
+ } else {
+#ifdef __APPLE__
+ return nullptr;
+#else
+ static const auto& type_info = typeid(__int128_t);
+ return &type_info;
+#endif
+ }
+ }
+
+ template <class ValueType>
+ friend ValueType any_cast(const AnyType& operand);
+ template <class ValueType>
+ friend ValueType any_cast(AnyType& operand);
+ template <class ValueType>
+ friend ValueType any_cast(AnyType&& operand);
+};
+
+template <class ValueType>
+inline ValueType any_cast(const AnyType& operand) {
+ return std::any_cast<ValueType>(std::get<std::any>(operand));
+}
+template <>
+inline __int128_t any_cast(const AnyType& operand) {
+ return std::get<__int128_t>(operand);
+}
+
+template <class ValueType>
+inline ValueType any_cast(AnyType& operand) {
+ return std::any_cast<ValueType>(std::get<std::any>(operand));
+}
+template <>
+inline __int128_t any_cast(AnyType& operand) {
+ return std::get<__int128_t>(operand);
+}
+
+template <class ValueType>
+inline ValueType any_cast(AnyType&& operand) {
+ return std::any_cast<ValueType>(std::get<std::any>(operand));
+}
+template <>
+inline __int128_t any_cast(AnyType&& operand) {
+ return std::get<__int128_t>(operand);
+}
+
+} // namespace doris
diff --git a/be/test/testutil/test_util.cpp b/be/test/testutil/test_util.cpp
index faf81a9ca8..d7883fc9e2 100644
--- a/be/test/testutil/test_util.cpp
+++ b/be/test/testutil/test_util.cpp
@@ -19,6 +19,8 @@
#ifndef __APPLE__
#include <linux/limits.h>
+#else
+#include <mach-o/dyld.h>
#endif
#include <common/configbase.h>
@@ -56,9 +58,17 @@ std::string GetCurrentRunningDir() {
char exe[PATH_MAX];
ssize_t r;
+#ifdef __APPLE__
+ uint32_t size = PATH_MAX;
+ if (_NSGetExecutablePath(exe, &size) < 0) {
+ return std::string();
+ }
+ r = strlen(exe) + 1;
+#else
if ((r = readlink("/proc/self/exe", exe, PATH_MAX)) < 0) {
return std::string();
}
+#endif
if (r == PATH_MAX) {
r -= 1;
diff --git a/be/test/util/arrow/arrow_work_flow_test.cpp
b/be/test/util/arrow/arrow_work_flow_test.cpp
index 7acd5e58b7..21a70ffcf9 100644
--- a/be/test/util/arrow/arrow_work_flow_test.cpp
+++ b/be/test/util/arrow/arrow_work_flow_test.cpp
@@ -53,7 +53,7 @@ protected:
EXPECT_EQ(system("mkdir -p ./test_run/output/"), 0);
EXPECT_EQ(system("pwd"), 0);
- EXPECT_EQ(system("cp -r ./be/test/util/test_data/ ./test_run/."), 0);
+ EXPECT_EQ(system("cp -r ./be/test/util/test_data ./test_run/."), 0);
init();
}
diff --git a/be/test/util/path_util_test.cpp b/be/test/util/path_util_test.cpp
index 7be3e543bc..1962a5b2b7 100644
--- a/be/test/util/path_util_test.cpp
+++ b/be/test/util/path_util_test.cpp
@@ -58,7 +58,11 @@ TEST(TestPathUtil, DirNameTest) {
EXPECT_EQ(".", path_util::dir_name("."));
EXPECT_EQ(".", path_util::dir_name(".."));
EXPECT_EQ("/", path_util::dir_name("/"));
+#ifndef __APPLE__
EXPECT_EQ("//", path_util::dir_name("//"));
+#else
+ EXPECT_EQ("/", path_util::dir_name("//"));
+#endif
EXPECT_EQ(".", path_util::dir_name("a"));
EXPECT_EQ(".", path_util::dir_name("ab"));
EXPECT_EQ(".", path_util::dir_name("ab/"));
diff --git a/be/test/util/threadpool_test.cpp b/be/test/util/threadpool_test.cpp
index 67924e8711..a46386fe1f 100644
--- a/be/test/util/threadpool_test.cpp
+++ b/be/test/util/threadpool_test.cpp
@@ -327,9 +327,15 @@ TEST_F(ThreadPoolTest, TestZeroQueueSize) {
#ifndef THREAD_SANITIZER
TEST_F(ThreadPoolTest, TestDeadlocks) {
::testing::FLAGS_gtest_death_test_style = "threadsafe";
+#ifdef __APPLE__
+ const char* death_msg =
+
"_ZNSt3__1L8__invokeIRNS_6__bindIMN5doris10ThreadPoolEFvvEJPS3_EEEJEEEDTclscT_fp_"
+ "spscT0_fp0_EEOS9_DpOSA_";
+#else
const char* death_msg =
"_ZNSt5_BindIFMN5doris10ThreadPoolEFvvEPS1_EE6__callIvJEJLm0EEEET_OSt5tupleIJDpT0_"
"EESt12_Index_tupleIJXspT1_EEE";
+#endif
EXPECT_DEATH(
{
EXPECT_TRUE(rebuild_pool_with_min_max(1, 1).ok());
diff --git a/be/test/vec/function/function_test_util.cpp
b/be/test/vec/function/function_test_util.cpp
index a2e002acec..0dbea78d02 100644
--- a/be/test/vec/function/function_test_util.cpp
+++ b/be/test/vec/function/function_test_util.cpp
@@ -48,7 +48,7 @@ uint64_t str_to_datetime_v2(std::string datetime_str,
std::string datetime_forma
return binary_cast<DateV2Value<DateTimeV2ValueType>, UInt64>(v);
}
-size_t type_index_to_data_type(const std::vector<std::any>& input_types,
size_t index,
+size_t type_index_to_data_type(const std::vector<AnyType>& input_types, size_t
index,
ut_type::UTDataTypeDesc& ut_desc, DataTypePtr&
type) {
doris_udf::FunctionContext::TypeDesc& desc = ut_desc.type_desc;
if (index < 0 || index >= input_types.size()) {
@@ -56,19 +56,19 @@ size_t type_index_to_data_type(const std::vector<std::any>&
input_types, size_t
}
TypeIndex tp;
- if (input_types[index].type() == typeid(Consted)) {
- tp = std::any_cast<Consted>(input_types[index]).tp;
- } else if (input_types[index].type() == typeid(ConstedNotnull)) {
- tp = std::any_cast<ConstedNotnull>(input_types[index]).tp;
+ if (input_types[index].type() == &typeid(Consted)) {
+ tp = any_cast<Consted>(input_types[index]).tp;
+ } else if (input_types[index].type() == &typeid(ConstedNotnull)) {
+ tp = any_cast<ConstedNotnull>(input_types[index]).tp;
ut_desc.is_nullable = false;
- } else if (input_types[index].type() == typeid(Nullable)) {
- tp = std::any_cast<Nullable>(input_types[index]).tp;
+ } else if (input_types[index].type() == &typeid(Nullable)) {
+ tp = any_cast<Nullable>(input_types[index]).tp;
ut_desc.is_nullable = true;
- } else if (input_types[index].type() == typeid(Notnull)) {
- tp = std::any_cast<Notnull>(input_types[index]).tp;
+ } else if (input_types[index].type() == &typeid(Notnull)) {
+ tp = any_cast<Notnull>(input_types[index]).tp;
ut_desc.is_nullable = false;
} else {
- tp = std::any_cast<TypeIndex>(input_types[index]);
+ tp = any_cast<TypeIndex>(input_types[index]);
}
switch (tp) {
@@ -164,13 +164,13 @@ size_t type_index_to_data_type(const
std::vector<std::any>& input_types, size_t
return 0;
}
}
-bool parse_ut_data_type(const std::vector<std::any>& input_types,
ut_type::UTDataTypeDescs& descs) {
+bool parse_ut_data_type(const std::vector<AnyType>& input_types,
ut_type::UTDataTypeDescs& descs) {
descs.clear();
descs.reserve(input_types.size());
for (size_t i = 0; i < input_types.size();) {
ut_type::UTDataTypeDesc desc;
- if (input_types[i].type() == typeid(Consted) ||
- input_types[i].type() == typeid(ConstedNotnull)) {
+ if (input_types[i].type() == &typeid(Consted) ||
+ input_types[i].type() == &typeid(ConstedNotnull)) {
desc.is_const = true;
}
size_t res = type_index_to_data_type(input_types, i, desc,
desc.data_type);
@@ -188,8 +188,8 @@ bool parse_ut_data_type(const std::vector<std::any>&
input_types, ut_type::UTDat
}
template <typename Date, TypeIndex type_index = TypeIndex::Nothing>
-bool insert_date_cell(MutableColumnPtr& column, const std::string& format,
const std::any& cell) {
- auto datetime_str = std::any_cast<std::string>(cell);
+bool insert_date_cell(MutableColumnPtr& column, const std::string& format,
const AnyType& cell) {
+ auto datetime_str = any_cast<std::string>(cell);
Date v;
auto result = v.from_date_format_str(format.c_str(), format.size(),
datetime_str.c_str(),
datetime_str.size());
@@ -208,8 +208,8 @@ bool insert_date_cell(MutableColumnPtr& column, const
std::string& format, const
return true;
}
-bool insert_cell(MutableColumnPtr& column, DataTypePtr type_ptr, const
std::any& cell) {
- if (cell.type() == typeid(Null)) {
+bool insert_cell(MutableColumnPtr& column, DataTypePtr type_ptr, const
AnyType& cell) {
+ if (cell.type() == &typeid(Null)) {
column->insert_data(nullptr, 0);
return true;
}
@@ -219,41 +219,41 @@ bool insert_cell(MutableColumnPtr& column, DataTypePtr
type_ptr, const std::any&
WhichDataType type(type_ptr);
if (type.is_string()) {
- auto str = std::any_cast<ut_type::STRING>(cell);
+ auto str = any_cast<ut_type::STRING>(cell);
column->insert_data(str.c_str(), str.size());
} else if (type.is_json()) {
- auto str = std::any_cast<ut_type::STRING>(cell);
+ auto str = any_cast<ut_type::STRING>(cell);
JsonBinaryValue jsonb_val(str.c_str(), str.size());
column->insert_data(jsonb_val.value(), jsonb_val.size());
} else if (type.idx == TypeIndex::BitMap) {
- BitmapValue* bitmap = std::any_cast<BitmapValue*>(cell);
+ BitmapValue* bitmap = any_cast<BitmapValue*>(cell);
column->insert_data((char*)bitmap, sizeof(BitmapValue));
} else if (type.is_uint8()) {
- auto value = std::any_cast<ut_type::BOOLEAN>(cell);
+ auto value = any_cast<ut_type::BOOLEAN>(cell);
column->insert_data(reinterpret_cast<char*>(&value), 0);
} else if (type.is_int8()) {
- auto value = std::any_cast<ut_type::TINYINT>(cell);
+ auto value = any_cast<ut_type::TINYINT>(cell);
column->insert_data(reinterpret_cast<char*>(&value), 0);
} else if (type.is_int16()) {
- auto value = std::any_cast<ut_type::SMALLINT>(cell);
+ auto value = any_cast<ut_type::SMALLINT>(cell);
column->insert_data(reinterpret_cast<char*>(&value), 0);
} else if (type.is_int32()) {
- auto value = std::any_cast<ut_type::INT>(cell);
+ auto value = any_cast<ut_type::INT>(cell);
column->insert_data(reinterpret_cast<char*>(&value), 0);
} else if (type.is_int64()) {
- auto value = std::any_cast<ut_type::BIGINT>(cell);
+ auto value = any_cast<ut_type::BIGINT>(cell);
column->insert_data(reinterpret_cast<char*>(&value), 0);
} else if (type.is_int128()) {
- auto value = std::any_cast<ut_type::LARGEINT>(cell);
+ auto value = any_cast<ut_type::LARGEINT>(cell);
column->insert_data(reinterpret_cast<char*>(&value), 0);
} else if (type.is_float32()) {
- auto value = std::any_cast<ut_type::FLOAT>(cell);
+ auto value = any_cast<ut_type::FLOAT>(cell);
column->insert_data(reinterpret_cast<char*>(&value), 0);
} else if (type.is_float64()) {
- auto value = std::any_cast<ut_type::DOUBLE>(cell);
+ auto value = any_cast<ut_type::DOUBLE>(cell);
column->insert_data(reinterpret_cast<char*>(&value), 0);
} else if (type.is_decimal128()) {
- auto value = std::any_cast<Decimal<Int128>>(cell);
+ auto value = any_cast<Decimal<Int128>>(cell);
column->insert_data(reinterpret_cast<char*>(&value), 0);
} else if (type.is_date_time()) {
static std::string date_time_format("%Y-%m-%d %H:%i:%s");
@@ -272,7 +272,7 @@ bool insert_cell(MutableColumnPtr& column, DataTypePtr
type_ptr, const std::any&
RETURN_IF_FALSE((insert_date_cell<DateV2Value<DateTimeV2ValueType>>(
column, date_time_format, cell)));
} else if (type.is_array()) {
- auto v = std::any_cast<Array>(cell);
+ auto v = any_cast<Array>(cell);
column->insert(v);
} else {
LOG(WARNING) << "dataset not supported for TypeIndex:" <<
(int)type.idx;
diff --git a/be/test/vec/function/function_test_util.h
b/be/test/vec/function/function_test_util.h
index 8da9e6f1b0..2603d224a4 100644
--- a/be/test/vec/function/function_test_util.h
+++ b/be/test/vec/function/function_test_util.h
@@ -23,6 +23,7 @@
#include <string>
#include "exprs/table_function/table_function.h"
+#include "testutil/any_type.h"
#include "testutil/function_utils.h"
#include "udf/udf.h"
#include "udf/udf_internal.h"
@@ -38,12 +39,12 @@
namespace doris::vectorized {
-using InputDataSet = std::vector<std::vector<std::any>>; // without result
-using CellSet = std::vector<std::any>;
-using Expect = std::any;
+using InputDataSet = std::vector<std::vector<AnyType>>; // without result
+using CellSet = std::vector<AnyType>;
+using Expect = AnyType;
using Row = std::pair<CellSet, Expect>;
using DataSet = std::vector<Row>;
-using InputTypeSet = std::vector<std::any>;
+using InputTypeSet = std::vector<AnyType>;
int64_t str_to_date_time(std::string datetime_str, bool data_time = true);
uint32_t str_to_date_v2(std::string datetime_str, std::string datetime_format);
@@ -157,11 +158,11 @@ using UTDataTypeDescs = std::vector<UTDataTypeDesc>;
} // namespace ut_type
-size_t type_index_to_data_type(const std::vector<std::any>& input_types,
size_t index,
+size_t type_index_to_data_type(const std::vector<AnyType>& input_types, size_t
index,
ut_type::UTDataTypeDesc& ut_desc, DataTypePtr&
type);
-bool parse_ut_data_type(const std::vector<std::any>& input_types,
ut_type::UTDataTypeDescs& descs);
+bool parse_ut_data_type(const std::vector<AnyType>& input_types,
ut_type::UTDataTypeDescs& descs);
-bool insert_cell(MutableColumnPtr& column, DataTypePtr type_ptr, const
std::any& cell);
+bool insert_cell(MutableColumnPtr& column, DataTypePtr type_ptr, const
AnyType& cell);
Block* create_block_from_inputset(const InputTypeSet& input_types, const
InputDataSet& input_set);
@@ -271,7 +272,7 @@ Status check_function(const std::string& func_name, const
InputTypeSet& input_ty
for (int i = 0; i < row_size; ++i) {
auto check_column_data = [&]() {
if constexpr (std::is_same_v<ReturnType, DataTypeJsonb>) {
- const auto& expect_data =
std::any_cast<String>(data_set[i].second);
+ const auto& expect_data = any_cast<String>(data_set[i].second);
auto s = column->get_data_at(i);
if (expect_data.size() == 0) {
// zero size result means invalid
@@ -286,7 +287,7 @@ Status check_function(const std::string& func_name, const
InputTypeSet& input_ty
column->get(i, field);
const auto& expect_data =
- std::any_cast<typename
ReturnType::FieldType>(data_set[i].second);
+ any_cast<typename
ReturnType::FieldType>(data_set[i].second);
if constexpr (std::is_same_v<ReturnType,
DataTypeDecimal<Decimal128>>) {
const auto& column_data =
field.get<DecimalField<Decimal128>>().get_value();
@@ -303,7 +304,7 @@ Status check_function(const std::string& func_name, const
InputTypeSet& input_ty
};
if constexpr (nullable) {
- bool is_null = data_set[i].second.type() == typeid(Null);
+ bool is_null = data_set[i].second.type() == &typeid(Null);
EXPECT_EQ(is_null, column->is_null_at(i)) << " at row " << i;
if (!is_null) check_column_data();
} else {
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org
For additional commands, e-mail: commits-h...@doris.apache.org
