This is an automated email from the ASF dual-hosted git repository.
kxiao pushed a commit to branch branch-2.0
in repository https://gitbox.apache.org/repos/asf/doris.git
commit c43c95e6223d3820ecc1a3f5c40ae951525614cf
Author: GoGoWen <82132356+gogo...@users.noreply.github.com>
AuthorDate: Sat Sep 2 12:46:33 2023 +0800
[Feature](Multi-Catalog) support query doris bitmap column in external jdbc
catalog (#23021)
---
be/src/runtime/descriptors.h | 9 ++
be/src/util/bitmap_value.h | 98 ++++++++++++----
be/src/vec/data_types/data_type_bitmap.cpp | 12 ++
be/src/vec/data_types/data_type_bitmap.h | 3 +-
be/src/vec/exec/vjdbc_connector.cpp | 82 +++++++++++++-
be/src/vec/exec/vjdbc_connector.h | 7 ++
be/src/vec/functions/function_cast.h | 24 ++++
docs/en/docs/lakehouse/multi-catalog/jdbc.md | 1 +
docs/zh-CN/docs/lakehouse/multi-catalog/jdbc.md | 3 +-
.../java/org/apache/doris/jdbc/JdbcExecutor.java | 50 +++++++++
.../datasource/jdbc/client/JdbcMySQLClient.java | 4 +
.../jdbc/test_doris_jdbc_catalog_query_bitmap.out | 51 +++++++++
.../test_doris_jdbc_catalog_query_bitmap.groovy | 124 +++++++++++++++++++++
13 files changed, 442 insertions(+), 26 deletions(-)
diff --git a/be/src/runtime/descriptors.h b/be/src/runtime/descriptors.h
index aff3b03a0f..ad1199f3f1 100644
--- a/be/src/runtime/descriptors.h
+++ b/be/src/runtime/descriptors.h
@@ -363,6 +363,15 @@ public:
return false;
}
+ bool has_bitmap_slot() const {
+ for (auto slot : _slots) {
+ if (slot->type().is_bitmap_type()) {
+ return true;
+ }
+ }
+ return false;
+ }
+
TupleId id() const { return _id; }
std::string debug_string() const;
diff --git a/be/src/util/bitmap_value.h b/be/src/util/bitmap_value.h
index 02a5595440..e67e665155 100644
--- a/be/src/util/bitmap_value.h
+++ b/be/src/util/bitmap_value.h
@@ -1176,48 +1176,102 @@ public:
BitmapValue(const BitmapValue& other) {
_type = other._type;
- _sv = other._sv;
- _bitmap = other._bitmap;
- _set = other._set;
- _is_shared = true;
- // should also set other's state to shared, so that other bitmap value
will
- // create a new bitmap when it wants to modify it.
- const_cast<BitmapValue&>(other)._is_shared = true;
+ switch (other._type) {
+ case EMPTY:
+ break;
+ case SINGLE:
+ _sv = other._sv;
+ break;
+ case BITMAP:
+ _bitmap = other._bitmap;
+ break;
+ case SET:
+ _set = other._set;
+ break;
+ }
+
+ if (other._type != EMPTY) {
+ _is_shared = true;
+ // should also set other's state to shared, so that other bitmap
value will
+ // create a new bitmap when it wants to modify it.
+ const_cast<BitmapValue&>(other)._is_shared = true;
+ }
}
BitmapValue(BitmapValue&& other) {
_type = other._type;
- _sv = other._sv;
+ switch (other._type) {
+ case EMPTY:
+ break;
+ case SINGLE:
+ _sv = other._sv;
+ break;
+ case BITMAP:
+ _bitmap = std::move(other._bitmap);
+ other._bitmap = nullptr;
+ break;
+ case SET:
+ _set = std::move(other._set);
+ break;
+ }
_is_shared = other._is_shared;
- _bitmap = std::move(other._bitmap);
- _set = std::move(other._set);
-
other._type = EMPTY;
other._is_shared = false;
- other._bitmap = nullptr;
}
BitmapValue& operator=(const BitmapValue& other) {
_type = other._type;
- _sv = other._sv;
- _bitmap = other._bitmap;
- _is_shared = true;
- _set = other._set;
- // should also set other's state to shared, so that other bitmap value
will
- // create a new bitmap when it wants to modify it.
- const_cast<BitmapValue&>(other)._is_shared = true;
+ switch (other._type) {
+ case EMPTY:
+ break;
+ case SINGLE:
+ _sv = other._sv;
+ break;
+ case BITMAP:
+ _bitmap = other._bitmap;
+ break;
+ case SET:
+ _set = other._set;
+ break;
+ }
+
+ if (other._type != EMPTY) {
+ _is_shared = true;
+ // should also set other's state to shared, so that other bitmap
value will
+ // create a new bitmap when it wants to modify it.
+ const_cast<BitmapValue&>(other)._is_shared = true;
+ }
return *this;
}
+ static std::string empty_bitmap() {
+ static BitmapValue btmap;
+ std::string buf;
+ buf.resize(btmap.getSizeInBytes());
+ btmap.write_to((char*)buf.c_str());
+ return buf;
+ }
+
BitmapValue& operator=(BitmapValue&& other) {
if (this == &other) {
return *this;
}
_type = other._type;
- _sv = other._sv;
- _bitmap = std::move(other._bitmap);
- _set = std::move(other._set);
+ switch (other._type) {
+ case EMPTY:
+ break;
+ case SINGLE:
+ _sv = other._sv;
+ break;
+ case BITMAP:
+ _bitmap = std::move(other._bitmap);
+ other._bitmap = nullptr;
+ break;
+ case SET:
+ _set = std::move(other._set);
+ break;
+ }
_is_shared = other._is_shared;
return *this;
}
diff --git a/be/src/vec/data_types/data_type_bitmap.cpp
b/be/src/vec/data_types/data_type_bitmap.cpp
index b7b5ba1aa1..ccb39b080a 100644
--- a/be/src/vec/data_types/data_type_bitmap.cpp
+++ b/be/src/vec/data_types/data_type_bitmap.cpp
@@ -121,4 +121,16 @@ void DataTypeBitMap::to_string(const IColumn& column,
size_t row_num, BufferWrit
data.write_to(const_cast<char*>(buffer.data()));
ostr.write(buffer.c_str(), buffer.size());
}
+
+Status DataTypeBitMap::from_string(ReadBuffer& rb, IColumn* column) const {
+ auto& data_column = assert_cast<ColumnBitmap&>(*column);
+ auto& data = data_column.get_data();
+
+ BitmapValue value;
+ if (!value.deserialize(rb.to_string().c_str())) {
+ return Status::InternalError("deserialize BITMAP from string fail!");
+ }
+ data.push_back(std::move(value));
+ return Status::OK();
+}
} // namespace doris::vectorized
diff --git a/be/src/vec/data_types/data_type_bitmap.h
b/be/src/vec/data_types/data_type_bitmap.h
index 6dabd87f34..af1c9fbf2a 100644
--- a/be/src/vec/data_types/data_type_bitmap.h
+++ b/be/src/vec/data_types/data_type_bitmap.h
@@ -93,8 +93,9 @@ public:
return "BitMap()";
}
void to_string(const IColumn& column, size_t row_num, BufferWritable&
ostr) const override;
+ Status from_string(ReadBuffer& rb, IColumn* column) const override;
- Field get_default() const override { return BitmapValue(); }
+ Field get_default() const override { return BitmapValue::empty_bitmap(); }
[[noreturn]] Field get_field(const TExprNode& node) const override {
LOG(FATAL) << "Unimplemented get_field for BitMap";
diff --git a/be/src/vec/exec/vjdbc_connector.cpp
b/be/src/vec/exec/vjdbc_connector.cpp
index 73dc0a9117..1fcb4b95af 100644
--- a/be/src/vec/exec/vjdbc_connector.cpp
+++ b/be/src/vec/exec/vjdbc_connector.cpp
@@ -379,6 +379,23 @@ Status JdbcConnector::_check_type(SlotDescriptor*
slot_desc, const std::string&
->create_column());
break;
}
+ case TYPE_OBJECT: {
+ if (type_str != "java.lang.String") {
+ return Status::InternalError(error_msg);
+ }
+
+ _map_column_idx_to_cast_idx_bitmap[column_index] =
_input_bitmap_string_types.size();
+ if (slot_desc->is_nullable()) {
+
_input_bitmap_string_types.push_back(make_nullable(std::make_shared<DataTypeString>()));
+ } else {
+
_input_bitmap_string_types.push_back(std::make_shared<DataTypeString>());
+ }
+
+ str_bitmap_cols.push_back(
+
_input_bitmap_string_types[_map_column_idx_to_cast_idx_bitmap[column_index]]
+ ->create_column());
+ break;
+ }
default: {
return Status::InternalError(error_msg);
}
@@ -403,7 +420,7 @@ Status JdbcConnector::get_next(bool* eos,
std::vector<MutableColumnPtr>& columns
jobject block_obj;
// if contain HLL column, pass the column type to jni env
- if (_tuple_desc->has_hll_slot()) {
+ if (_tuple_desc->has_hll_slot() || _tuple_desc->has_bitmap_slot()) {
auto column_size = _tuple_desc->slots().size();
// Find ArrayList and Integer
jclass arrayListClass = env->FindClass("java/util/ArrayList");
@@ -418,7 +435,7 @@ Status JdbcConnector::get_next(bool* eos,
std::vector<MutableColumnPtr>& columns
jobject arrayListObject = env->NewObject(arrayListClass,
arrayListConstructor);
for (int column_index = 0; column_index < column_size; ++column_index)
{
auto slot_desc = _tuple_desc->slots()[column_index];
- if (slot_desc->type().is_hll_type()) {
+ if (slot_desc->type().is_hll_type() ||
slot_desc->type().is_bitmap_type()) {
// Create an Integer object
jobject integerObject = env->NewObject(
integerClass, env->GetMethodID(integerClass, "<init>",
"(I)V"),
@@ -465,6 +482,8 @@ Status JdbcConnector::get_next(bool* eos,
std::vector<MutableColumnPtr>& columns
_cast_string_to_hll(slot_desc, block, column_index, num_rows);
} else if (slot_desc->type().is_json_type()) {
_cast_string_to_json(slot_desc, block, column_index, num_rows);
+ } else if (slot_desc->type().is_bitmap_type()) {
+ _cast_string_to_bitmap(slot_desc, block, column_index, num_rows);
}
materialized_column_index++;
}
@@ -685,6 +704,27 @@ Status JdbcConnector::_convert_batch_result_set(JNIEnv*
env, jobject jcolumn_dat
address[1], chars_addres);
break;
}
+ //BITMAP
+ case TYPE_OBJECT: {
+
str_bitmap_cols[_map_column_idx_to_cast_idx_bitmap[column_index]]->resize(num_rows);
+ if (column_is_nullable) {
+ auto* nullable_column =
reinterpret_cast<vectorized::ColumnNullable*>(
+
str_bitmap_cols[_map_column_idx_to_cast_idx_bitmap[column_index]].get());
+ auto& null_map = nullable_column->get_null_map_data();
+ memset(null_map.data(), 0, num_rows);
+ address[0] = reinterpret_cast<int64_t>(null_map.data());
+ col_ptr = &nullable_column->get_nested_column();
+ } else {
+ col_ptr =
str_bitmap_cols[_map_column_idx_to_cast_idx_bitmap[column_index]].get();
+ }
+ auto column_string =
reinterpret_cast<vectorized::ColumnString*>(col_ptr);
+ address[1] =
reinterpret_cast<int64_t>(column_string->get_offsets().data());
+ auto chars_addres =
reinterpret_cast<int64_t>(&column_string->get_chars());
+ env->CallNonvirtualVoidMethod(_executor_obj, _executor_clazz,
_executor_get_bitmap_result,
+ jcolumn_data, column_is_nullable,
num_rows, address[0],
+ address[1], chars_addres);
+ break;
+ }
default: {
const std::string& error_msg =
fmt::format("Fail to convert jdbc value to {} on column: {}",
@@ -742,6 +782,8 @@ Status JdbcConnector::_register_func_id(JNIEnv* env) {
"(Ljava/lang/Object;ZIJJJ)V",
_executor_get_array_result));
RETURN_IF_ERROR(register_id(_executor_clazz, "copyBatchHllResult",
"(Ljava/lang/Object;ZIJJJ)V",
_executor_get_hll_result));
+ RETURN_IF_ERROR(register_id(_executor_clazz, "copyBatchBitMapResult",
+ "(Ljava/lang/Object;ZIJJJ)V",
_executor_get_bitmap_result));
RETURN_IF_ERROR(register_id(_executor_clazz, "copyBatchJsonResult",
"(Ljava/lang/Object;ZIJJJ)V",
_executor_get_json_result));
RETURN_IF_ERROR(register_id(_executor_clazz, "copyBatchCharResult",
@@ -825,6 +867,42 @@ Status JdbcConnector::_cast_string_to_hll(const
SlotDescriptor* slot_desc, Block
return Status::OK();
}
+Status JdbcConnector::_cast_string_to_bitmap(const SlotDescriptor* slot_desc,
Block* block,
+ int column_index, int rows) {
+ DataTypePtr _target_data_type = slot_desc->get_data_type_ptr();
+ std::string _target_data_type_name = _target_data_type->get_name();
+ DataTypePtr _cast_param_data_type = _target_data_type;
+ ColumnPtr _cast_param =
_cast_param_data_type->create_column_const_with_default_value(1);
+
+ ColumnsWithTypeAndName argument_template;
+ argument_template.reserve(2);
+ argument_template.emplace_back(
+
std::move(str_bitmap_cols[_map_column_idx_to_cast_idx_bitmap[column_index]]),
+
_input_bitmap_string_types[_map_column_idx_to_cast_idx_bitmap[column_index]],
+ "java.sql.String");
+ argument_template.emplace_back(_cast_param, _cast_param_data_type,
_target_data_type_name);
+ FunctionBasePtr func_cast = SimpleFunctionFactory::instance().get_function(
+ "CAST", argument_template, make_nullable(_target_data_type));
+
+ Block cast_block(argument_template);
+ int result_idx = cast_block.columns();
+ cast_block.insert({nullptr, make_nullable(_target_data_type),
"cast_result"});
+ func_cast->execute(nullptr, cast_block, {0, 1}, result_idx, rows);
+
+ auto res_col = cast_block.get_by_position(result_idx).column;
+ if (_target_data_type->is_nullable()) {
+ block->replace_by_position(column_index, res_col);
+ } else {
+ auto nested_ptr = reinterpret_cast<const
vectorized::ColumnNullable*>(res_col.get())
+ ->get_nested_column_ptr();
+ block->replace_by_position(column_index, nested_ptr);
+ }
+ str_bitmap_cols[_map_column_idx_to_cast_idx_bitmap[column_index]] =
+
_input_bitmap_string_types[_map_column_idx_to_cast_idx_bitmap[column_index]]
+ ->create_column();
+ return Status::OK();
+}
+
Status JdbcConnector::_cast_string_to_array(const SlotDescriptor* slot_desc,
Block* block,
int column_index, int rows) {
DataTypePtr _target_data_type = slot_desc->get_data_type_ptr();
diff --git a/be/src/vec/exec/vjdbc_connector.h
b/be/src/vec/exec/vjdbc_connector.h
index df53eb5347..0c1f690a76 100644
--- a/be/src/vec/exec/vjdbc_connector.h
+++ b/be/src/vec/exec/vjdbc_connector.h
@@ -104,6 +104,8 @@ private:
int rows);
Status _cast_string_to_hll(const SlotDescriptor* slot_desc, Block* block,
int column_index,
int rows);
+ Status _cast_string_to_bitmap(const SlotDescriptor* slot_desc, Block*
block, int column_index,
+ int rows);
Status _cast_string_to_json(const SlotDescriptor* slot_desc, Block* block,
int column_index,
int rows);
Status _convert_batch_result_set(JNIEnv* env, jobject jobj, const
SlotDescriptor* slot_desc,
@@ -146,6 +148,7 @@ private:
jmethodID _executor_get_array_result;
jmethodID _executor_get_json_result;
jmethodID _executor_get_hll_result;
+ jmethodID _executor_get_bitmap_result;
jmethodID _executor_get_types_id;
jmethodID _executor_close_id;
jmethodID _executor_get_list_id;
@@ -163,6 +166,10 @@ private:
std::vector<DataTypePtr> _input_hll_string_types;
std::vector<MutableColumnPtr> str_hll_cols; // for hll type to save data
like string
+ std::map<int, int> _map_column_idx_to_cast_idx_bitmap;
+ std::vector<DataTypePtr> _input_bitmap_string_types;
+ std::vector<MutableColumnPtr> str_bitmap_cols; // for bitmap type to save
data like string
+
std::map<int, int> _map_column_idx_to_cast_idx_json;
std::vector<DataTypePtr> _input_json_string_types;
std::vector<MutableColumnPtr> str_json_cols; // for json type to save data
like string
diff --git a/be/src/vec/functions/function_cast.h
b/be/src/vec/functions/function_cast.h
index b64d5ba9fd..ed86e436a6 100644
--- a/be/src/vec/functions/function_cast.h
+++ b/be/src/vec/functions/function_cast.h
@@ -70,6 +70,7 @@
#include "vec/core/types.h"
#include "vec/data_types/data_type.h"
#include "vec/data_types/data_type_array.h"
+#include "vec/data_types/data_type_bitmap.h"
#include "vec/data_types/data_type_date.h"
#include "vec/data_types/data_type_date_time.h"
#include "vec/data_types/data_type_decimal.h"
@@ -1695,6 +1696,26 @@ private:
return nullptr;
}
+ WrapperType create_bitmap_wrapper(FunctionContext* context,
+ const DataTypePtr& from_type_untyped,
+ const DataTypeBitMap& to_type) const {
+ /// Conversion from String through parsing.
+ if (check_and_get_data_type<DataTypeString>(from_type_untyped.get())) {
+ return &ConvertImplGenericFromString::execute;
+ }
+
+ //TODO if from is not string, it must be BITMAP?
+ const auto* from_type =
check_and_get_data_type<DataTypeBitMap>(from_type_untyped.get());
+
+ if (!from_type) {
+ return create_unsupport_wrapper(
+ "CAST AS BITMAP can only be performed between BITMAP,
String "
+ "types");
+ }
+
+ return nullptr;
+ }
+
WrapperType create_array_wrapper(FunctionContext* context, const
DataTypePtr& from_type_untyped,
const DataTypeArray& to_type) const {
/// Conversion from String through parsing.
@@ -2085,6 +2106,9 @@ private:
case TypeIndex::HLL:
return create_hll_wrapper(context, from_type,
static_cast<const
DataTypeHLL&>(*to_type));
+ case TypeIndex::BitMap:
+ return create_bitmap_wrapper(context, from_type,
+ static_cast<const
DataTypeBitMap&>(*to_type));
default:
break;
}
diff --git a/docs/en/docs/lakehouse/multi-catalog/jdbc.md
b/docs/en/docs/lakehouse/multi-catalog/jdbc.md
index dded188564..7200752064 100644
--- a/docs/en/docs/lakehouse/multi-catalog/jdbc.md
+++ b/docs/en/docs/lakehouse/multi-catalog/jdbc.md
@@ -393,6 +393,7 @@ CREATE CATALOG jdbc_doris PROPERTIES (
| TEXT | STRING |
|
| HLL | HLL | Query HLL needs to set
`return_object_data_as_binary=true` |
| Array | Array | The internal type adaptation logic of
Array refers to the above types, and nested complex types are not supported
|
+| BITMAP | BITMAP | Query BITMAP needs to set
`return_object_data_as_binary=true` |
| Other | UNSUPPORTED |
|
### Clickhouse
diff --git a/docs/zh-CN/docs/lakehouse/multi-catalog/jdbc.md
b/docs/zh-CN/docs/lakehouse/multi-catalog/jdbc.md
index 91b742867d..79301cd1e1 100644
--- a/docs/zh-CN/docs/lakehouse/multi-catalog/jdbc.md
+++ b/docs/zh-CN/docs/lakehouse/multi-catalog/jdbc.md
@@ -392,8 +392,9 @@ CREATE CATALOG jdbc_doris PROPERTIES (
| VARCHAR | VARCHAR |
|
| STRING | STRING |
|
| TEXT | STRING |
|
-| HLL | HLL |
查询HLL需要设置`return_object_data_as_binary=true` |
+| HLL | HLL |
查询HLL需要设置`return_object_data_as_binary=true` |
| Array | Array | Array内部类型适配逻辑参考上述类型，不支持嵌套复杂类型 |
+| BITMAP | BITMAP |
查询BITMAP需要设置`return_object_data_as_binary=true` |
| Other | UNSUPPORTED |
|
### Clickhouse
diff --git
a/fe/be-java-extensions/jdbc-scanner/src/main/java/org/apache/doris/jdbc/JdbcExecutor.java
b/fe/be-java-extensions/jdbc-scanner/src/main/java/org/apache/doris/jdbc/JdbcExecutor.java
index 6b22fb1be0..f84761a20e 100644
---
a/fe/be-java-extensions/jdbc-scanner/src/main/java/org/apache/doris/jdbc/JdbcExecutor.java
+++
b/fe/be-java-extensions/jdbc-scanner/src/main/java/org/apache/doris/jdbc/JdbcExecutor.java
@@ -1473,6 +1473,43 @@ public class JdbcExecutor {
UdfUtils.copyMemory(bytes, UdfUtils.BYTE_ARRAY_OFFSET, null,
bytesAddr, offsets[numRows - 1]);
}
+ private void bitMapPutToString(Object[] column, boolean isNullable, int
numRows, long nullMapAddr,
+ long offsetsAddr, long charsAddr) {
+ int[] offsets = new int[numRows];
+ byte[][] byteRes = new byte[numRows][];
+ int offset = 0;
+ if (isNullable == true) {
+ // Here can not loop from startRowForNullable,
+ // because byteRes will be used later
+ for (int i = 0; i < numRows; i++) {
+ if (column[i] == null) {
+ byteRes[i] = emptyBytes;
+ UdfUtils.UNSAFE.putByte(nullMapAddr + i, (byte) 1);
+ } else {
+ byteRes[i] = (byte[]) column[i];
+ }
+ offset += byteRes[i].length;
+ offsets[i] = offset;
+ }
+ } else {
+ for (int i = 0; i < numRows; i++) {
+ byteRes[i] = (byte[]) column[i];
+ offset += byteRes[i].length;
+ offsets[i] = offset;
+ }
+ }
+ byte[] bytes = new byte[offsets[numRows - 1]];
+ long bytesAddr = JNINativeMethod.resizeStringColumn(charsAddr,
offsets[numRows - 1]);
+ int dst = 0;
+ for (int i = 0; i < numRows; i++) {
+ for (int j = 0; j < byteRes[i].length; j++) {
+ bytes[dst++] = byteRes[i][j];
+ }
+ }
+ UdfUtils.copyMemory(offsets, UdfUtils.INT_ARRAY_OFFSET, null,
offsetsAddr, numRows * 4L);
+ UdfUtils.copyMemory(bytes, UdfUtils.BYTE_ARRAY_OFFSET, null,
bytesAddr, offsets[numRows - 1]);
+ }
+
public void copyBatchHllResult(Object columnObj, boolean isNullable, int
numRows, long nullMapAddr,
long offsetsAddr, long charsAddr) {
Object[] column = (Object[]) columnObj;
@@ -1486,6 +1523,19 @@ public class JdbcExecutor {
hllPutToString(column, isNullable, numRows, nullMapAddr, offsetsAddr,
charsAddr);
}
+ public void copyBatchBitMapResult(Object columnObj, boolean isNullable,
int numRows, long nullMapAddr,
+ long offsetsAddr, long charsAddr) {
+ Object[] column = (Object[]) columnObj;
+ int firstNotNullIndex = 0;
+ if (isNullable) {
+ firstNotNullIndex = getFirstNotNullObject(column, numRows,
nullMapAddr);
+ }
+ if (firstNotNullIndex == numRows) {
+ return;
+ }
+ bitMapPutToString(column, isNullable, numRows, nullMapAddr,
offsetsAddr, charsAddr);
+ }
+
private static String simplifyIPv6Address(String address) {
// Replace longest sequence of zeros with "::"
String[] parts = address.split(":");
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/datasource/jdbc/client/JdbcMySQLClient.java
b/fe/fe-core/src/main/java/org/apache/doris/datasource/jdbc/client/JdbcMySQLClient.java
index aed3136161..7004309d2d 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/datasource/jdbc/client/JdbcMySQLClient.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/datasource/jdbc/client/JdbcMySQLClient.java
@@ -136,6 +136,8 @@ public class JdbcMySQLClient extends JdbcClient {
// in mysql-jdbc-connector-8.0.*, TYPE_NAME of the HLL column
in doris will be "UNKNOWN"
// in mysql-jdbc-connector-5.1.*, TYPE_NAME of the HLL column
in doris will be "HLL"
+ // in mysql-jdbc-connector-8.0.*, TYPE_NAME of BITMAP column
in doris will be "BIT"
+ // in mysql-jdbc-connector-5.1.*, TYPE_NAME of BITMAP column
in doris will be "BITMAP"
field.setDataTypeName(rs.getString("TYPE_NAME"));
if (isDoris) {
mapFieldtoType = getColumnsDataTypeUseQuery(dbName,
tableName);
@@ -417,6 +419,8 @@ public class JdbcMySQLClient extends JdbcClient {
return ScalarType.createJsonbType();
case "HLL":
return ScalarType.createHllType();
+ case "BITMAP":
+ return Type.BITMAP;
default:
return Type.UNSUPPORTED;
}
diff --git
a/regression-test/data/external_table_p0/jdbc/test_doris_jdbc_catalog_query_bitmap.out
b/regression-test/data/external_table_p0/jdbc/test_doris_jdbc_catalog_query_bitmap.out
new file mode 100644
index 0000000000..5d18206414
--- /dev/null
+++
b/regression-test/data/external_table_p0/jdbc/test_doris_jdbc_catalog_query_bitmap.out
@@ -0,0 +1,51 @@
+-- This file is automatically generated. You should know what you did if you
want to edit this
+-- !sql --
+internal
+
+-- !sql --
+internal
+
+-- !ex_tb1 --
+1 doris1
+2 doris2
+3 doris3
+4 doris4
+5 doris5
+6 doris6
+
+-- !sql --
+internal
+
+-- !sql --
+doris_jdbc_catalog_query_bitmap
+
+-- !ex_tb1 --
+1 doris1
+2 doris2
+3 doris3
+4 doris4
+5 doris5
+6 doris6
+
+-- !tb1 --
+1 1
+2 7
+3 8
+
+-- !sql --
+internal
+
+-- !sql --
+doris_jdbc_catalog_query_bitmap
+
+-- !tb2 --
+1 1
+2 7
+3 8
+
+-- !sql --
+doris_jdbc_catalog_query_bitmap
+
+-- !sql --
+internal
+
diff --git
a/regression-test/suites/external_table_p0/jdbc/test_doris_jdbc_catalog_query_bitmap.groovy
b/regression-test/suites/external_table_p0/jdbc/test_doris_jdbc_catalog_query_bitmap.groovy
new file mode 100644
index 0000000000..59066d6099
--- /dev/null
+++
b/regression-test/suites/external_table_p0/jdbc/test_doris_jdbc_catalog_query_bitmap.groovy
@@ -0,0 +1,124 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+suite("test_doris_jdbc_catalog_query_bitmap",
"p0,external,doris,external_docker,external_docker_doris") {
+ qt_sql """select current_catalog()"""
+
+ String jdbcUrl = context.config.jdbcUrl +
"&sessionVariables=return_object_data_as_binary=true"
+ String jdbcUser = context.config.jdbcUser
+ String jdbcPassword = context.config.jdbcPassword
+ String s3_endpoint = getS3Endpoint()
+ String bucket = getS3BucketName()
+ String driver_url =
"https://${bucket}.${s3_endpoint}/regression/jdbc_driver/mysql-connector-java-8.0.25.jar";
+ String externalEnvIp = context.config.otherConfigs.get("externalEnvIp")
+
+
+ String resource_name = "jdbc_resource_catalog_doris_query_bitmap"
+ String catalog_name = "doris_jdbc_catalog_query_bitmap";
+ String internal_db_name = "regression_test_jdbc_catalog_p0_query_bitmap";
+ String doris_port = 9030;
+ String inDorisTable = "test_doris_jdbc_doris_in_tb_metric_table";
+ String bitmapTable = "metric_table"
+
+ sql """create database if not exists ${internal_db_name}; """
+
+ qt_sql """select current_catalog()"""
+ sql """drop catalog if exists ${catalog_name} """
+
+ sql """ CREATE CATALOG `${catalog_name}` PROPERTIES (
+ "user" = "${jdbcUser}",
+ "type" = "jdbc",
+ "password" = "${jdbcPassword}",
+ "jdbc_url" = "${jdbcUrl}",
+ "driver_url" = "${driver_url}",
+ "driver_class" = "com.mysql.cj.jdbc.Driver"
+ )"""
+ sql """use ${internal_db_name}"""
+ sql """ drop table if exists ${internal_db_name}.${inDorisTable} """
+ sql """
+ CREATE TABLE ${internal_db_name}.${inDorisTable} (
+ `id` INT NULL COMMENT "主键id",
+ `name` string NULL COMMENT "名字"
+ ) DISTRIBUTED BY HASH(id) BUCKETS 10
+ PROPERTIES("replication_num" = "1");
+ """
+ sql """ insert into ${inDorisTable} values (1, 'doris1')"""
+ sql """ insert into ${inDorisTable} values (2, 'doris2')"""
+ sql """ insert into ${inDorisTable} values (3, 'doris3')"""
+ sql """ insert into ${inDorisTable} values (4, 'doris4')"""
+ sql """ insert into ${inDorisTable} values (5, 'doris5')"""
+ sql """ insert into ${inDorisTable} values (6, 'doris6')"""
+
+ order_qt_ex_tb1 """ select * from
internal.${internal_db_name}.${inDorisTable} order by id; """
+
+ qt_sql """select current_catalog()"""
+ sql "switch ${catalog_name}"
+ qt_sql """select current_catalog()"""
+ sql """ use ${internal_db_name}"""
+ order_qt_ex_tb1 """ select * from ${inDorisTable} order by id; """
+
+ // test hll query
+ sql "switch internal"
+ sql "use ${internal_db_name}"
+
+ sql """ drop table if exists ${bitmapTable} """
+ sql """ create table `${bitmapTable}` (
+ datekey int,
+ hour int,
+ device_id bitmap BITMAP_UNION
+ )
+ aggregate key (datekey, hour)
+ distributed by hash(datekey, hour) buckets 1
+ properties(
+ "replication_num" = "1"
+ ); """
+
+ sql """ insert into ${bitmapTable} values(20200622, 1, to_bitmap(243));"""
+ sql """ insert into ${bitmapTable} values(20200622, 2,
bitmap_from_array([1,2,3,4,5,434543]));"""
+ sql """ insert into ${bitmapTable} values(20200622, 3,
to_bitmap(287667876573));"""
+
+ sql """ set return_object_data_as_binary=true """
+ order_qt_tb1 """ select hour, BITMAP_UNION_COUNT(pv) over(order by hour)
uv from(
+ select hour, BITMAP_UNION(device_id) as pv
+ from `${bitmapTable}`
+ where datekey=20200622
+ group by hour order by 1
+ ) final; """
+
+ // query with jdbc external table
+ sql """ refresh catalog ${catalog_name} """
+ qt_sql """select current_catalog()"""
+ sql """ switch ${catalog_name} """
+ qt_sql """select current_catalog()"""
+ sql """ use ${internal_db_name} """
+ //order_qt_tb2 """ select pin_id, hll_union_agg(user_log_acct) from
${catalog_name}.${internal_db_name}.${hllTable} group by pin_id; """
+ order_qt_tb2 """ select hour, BITMAP_UNION_COUNT(pv) over(order by hour)
uv from(
+ select hour, BITMAP_UNION(device_id) as pv
+ from ${catalog_name}.${internal_db_name}.${bitmapTable}
+ where datekey=20200622
+ group by hour order by 1
+ ) final; """
+
+ //clean
+ qt_sql """select current_catalog()"""
+ sql "switch internal"
+ qt_sql """select current_catalog()"""
+ sql "use ${internal_db_name}"
+ sql """ drop table if exists ${inDorisTable} """
+ sql """ drop table if exists ${bitmapTable} """
+
+}
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org
For additional commands, e-mail: commits-h...@doris.apache.org
