xy720 commented on code in PR #15966:
URL: https://github.com/apache/doris/pull/15966#discussion_r1092726577
##########
be/src/olap/rowset/segment_v2/column_reader.cpp:
##########
@@ -485,13 +512,91 @@ Status ColumnReader::new_iterator(ColumnIterator**
iterator) {
null_iterator);
return Status::OK();
}
+ case FieldType::OLAP_FIELD_TYPE_MAP: {
+ ColumnIterator* key_iterator = nullptr;
+ RETURN_IF_ERROR(_sub_readers[0]->new_iterator(&key_iterator));
+ ColumnIterator* val_iterator = nullptr;
+ RETURN_IF_ERROR(_sub_readers[1]->new_iterator(&val_iterator));
+ ColumnIterator* null_iterator = nullptr;
+ if (is_nullable()) {
+ RETURN_IF_ERROR(_sub_readers[2]->new_iterator(&null_iterator));
+ }
+ *iterator = new MapFileColumnIterator(this, null_iterator,
key_iterator, val_iterator);
+ return Status::OK();
+ }
default:
return Status::NotSupported("unsupported type to create iterator:
{}",
std::to_string(type));
}
}
}
+///====================== MapFileColumnIterator
============================////
+MapFileColumnIterator::MapFileColumnIterator(ColumnReader* reader,
ColumnIterator* null_iterator,
+ ColumnIterator* key_iterator,
+ ColumnIterator* val_iterator)
+ : _map_reader(reader) {
+ _key_iterator.reset(key_iterator);
+ _val_iterator.reset(val_iterator);
+ if (_map_reader->is_nullable()) {
+ _null_iterator.reset(null_iterator);
+ }
+}
+
+Status MapFileColumnIterator::init(const ColumnIteratorOptions& opts) {
+ RETURN_IF_ERROR(_key_iterator->init(opts));
+ RETURN_IF_ERROR(_val_iterator->init(opts));
+ if (_map_reader->is_nullable()) {
+ RETURN_IF_ERROR(_null_iterator->init(opts));
+ }
+ return Status::OK();
+}
+
+Status MapFileColumnIterator::next_batch(size_t* n, ColumnBlockView* dst,
bool* has_null) {
+ return Status::NotSupported("Not support next_batch");
+}
+
+Status MapFileColumnIterator::seek_to_ordinal(ordinal_t ord) {
+ RETURN_IF_ERROR(_key_iterator->seek_to_ordinal(ord));
+ RETURN_IF_ERROR(_val_iterator->seek_to_ordinal(ord));
+ if (_map_reader->is_nullable()) {
+ RETURN_IF_ERROR(_null_iterator->seek_to_ordinal(ord));
+ }
+ return Status::OK();
+}
+
+Status MapFileColumnIterator::next_batch(size_t* n,
vectorized::MutableColumnPtr& dst,
+ bool* has_null) {
+ const auto* column_map =
vectorized::check_and_get_column<vectorized::ColumnMap>(
+ dst->is_nullable() ?
static_cast<vectorized::ColumnNullable&>(*dst).get_nested_column()
+ : *dst);
+ size_t num_read = *n;
+ auto column_key_ptr = column_map->get_keys().assume_mutable();
+ auto column_val_ptr = column_map->get_values().assume_mutable();
+ RETURN_IF_ERROR(_key_iterator->next_batch(num_read, column_key_ptr,
has_null));
Review Comment:
```suggestion
RETURN_IF_ERROR(_key_iterator->next_batch(&num_read, column_key_ptr,
has_null));
```
##########
be/src/olap/types.h:
##########
@@ -431,6 +432,93 @@ class ArrayTypeInfo : public TypeInfo {
TypeInfoPtr _item_type_info;
const size_t _item_size;
};
+///====================== MapType Info ==========================///
+class MapTypeInfo : public TypeInfo {
+public:
+ explicit MapTypeInfo(TypeInfoPtr key_type_info, TypeInfoPtr
value_type_info)
+ : _key_type_info(std::move(key_type_info)),
+ _value_type_info(std::move(value_type_info)) {}
+ ~MapTypeInfo() override = default;
+
+ inline bool equal(const void* left, const void* right) const override {
+ auto l_value = reinterpret_cast<const MapValue*>(left);
+ auto r_value = reinterpret_cast<const MapValue*>(right);
+ return l_value->size() == r_value->size();
+ }
+
+ int cmp(const void* left, const void* right) const override {
+ auto l_value = reinterpret_cast<const MapValue*>(left);
+ auto r_value = reinterpret_cast<const MapValue*>(right);
+ uint32_t l_size = l_value->size();
+ uint32_t r_size = r_value->size();
+ if (l_size < r_size) {
+ return -1;
+ } else if (l_size > r_size) {
+ return 1;
+ } else {
+ return 0;
+ }
+ }
+
+ void shallow_copy(void* dest, const void* src) const override {
+ auto dest_value = reinterpret_cast<MapValue*>(dest);
+ auto src_value = reinterpret_cast<const MapValue*>(src);
+ dest_value->shallow_copy(src_value);
+ }
+
+ void deep_copy(void* dest, const void* src, MemPool* mem_pool) const
override { DCHECK(false); }
+
+ void copy_object(void* dest, const void* src, MemPool* mem_pool) const
override {
+ deep_copy(dest, src, mem_pool);
+ }
+
+ void direct_copy(void* dest, const void* src) const override {
CHECK(false); }
+
+ void direct_copy(uint8_t** base, void* dest, const void* src) const {
CHECK(false); }
+
+ void direct_copy_may_cut(void* dest, const void* src) const override {
direct_copy(dest, src); }
+
+ Status convert_from(void* dest, const void* src, const TypeInfo* src_type,
MemPool* mem_pool,
+ size_t variable_len = 0) const override {
+ return Status::Error<ErrorCode::NOT_IMPLEMENTED_ERROR>();
+ }
+
+ Status from_string(void* buf, const std::string& scan_key, const int
precision = 0,
+ const int scale = 0) const override {
+ return Status::Error<ErrorCode::NOT_IMPLEMENTED_ERROR>();
+ }
+
+ std::string to_string(const void* src) const override { return "{}"; }
+
+ void set_to_max(void* buf) const override {
+ DCHECK(false) << "set_to_max of list is not implemented.";
+ }
+
+ void set_to_min(void* buf) const override {
+ DCHECK(false) << "set_to_min of list is not implemented.";
+ }
+
+ uint32_t hash_code(const void* data, uint32_t seed) const override {
+ auto map_value = reinterpret_cast<const MapValue*>(data);
+ auto size = map_value->size();
+ uint32_t result = HashUtil::hash(&size, sizeof(size), seed);
+ result = seed * result +
_key_type_info->hash_code(map_value->key_data(), seed) +
+ _value_type_info->hash_code(map_value->value_data(), seed);
+ return result;
+ }
+
+ // todo . is here only to need return 16 for two ptr?
+ const size_t size() const override { return 16; }
Review Comment:
should use sizeof(MapValue)
16 bytes(2 pointers) + 4 bytes(_length) = 20 bytes => align to 8 bytes =>
total 24 bytes
##########
be/src/runtime/types.cpp:
##########
@@ -223,6 +227,15 @@ TypeDescriptor::TypeDescriptor(const
google::protobuf::RepeatedPtrField<PTypeNod
children.push_back(TypeDescriptor(types, idx));
break;
}
+ case TTypeNodeType::MAP: {
+ type = TYPE_MAP;
+ if (node.has_contains_null()) {
+ contains_null = node.contains_null();
Review Comment:
contains_null has been modified by struct, please check this logic.
##########
be/src/vec/olap/olap_data_convertor.cpp:
##########
@@ -129,6 +128,20 @@
OlapBlockDataConvertor::create_olap_column_data_convertor(const TabletColumn& co
return std::make_unique<OlapColumnDataConvertorArray>(
create_olap_column_data_convertor(sub_column));
}
+ case FieldType::OLAP_FIELD_TYPE_MAP: {
+ const auto& key_column = column.get_sub_column(0);
+ const auto& value_column = column.get_sub_column(1);
+ return std::make_unique<OlapColumnDataConvertorMap>(
+ std::make_unique<OlapColumnDataConvertorArray>(
+ create_olap_column_data_convertor(key_column)),
+ std::make_unique<OlapColumnDataConvertorArray>(
+ create_olap_column_data_convertor(value_column)));
+ //const auto& key_column = column.get_sub_column(0);
Review Comment:
remove unused code
##########
be/src/olap/rowset/segment_v2/column_reader.cpp:
##########
@@ -485,13 +512,91 @@ Status ColumnReader::new_iterator(ColumnIterator**
iterator) {
null_iterator);
return Status::OK();
}
+ case FieldType::OLAP_FIELD_TYPE_MAP: {
+ ColumnIterator* key_iterator = nullptr;
+ RETURN_IF_ERROR(_sub_readers[0]->new_iterator(&key_iterator));
+ ColumnIterator* val_iterator = nullptr;
+ RETURN_IF_ERROR(_sub_readers[1]->new_iterator(&val_iterator));
+ ColumnIterator* null_iterator = nullptr;
+ if (is_nullable()) {
+ RETURN_IF_ERROR(_sub_readers[2]->new_iterator(&null_iterator));
+ }
+ *iterator = new MapFileColumnIterator(this, null_iterator,
key_iterator, val_iterator);
+ return Status::OK();
+ }
default:
return Status::NotSupported("unsupported type to create iterator:
{}",
std::to_string(type));
}
}
}
+///====================== MapFileColumnIterator
============================////
+MapFileColumnIterator::MapFileColumnIterator(ColumnReader* reader,
ColumnIterator* null_iterator,
+ ColumnIterator* key_iterator,
+ ColumnIterator* val_iterator)
+ : _map_reader(reader) {
+ _key_iterator.reset(key_iterator);
+ _val_iterator.reset(val_iterator);
+ if (_map_reader->is_nullable()) {
+ _null_iterator.reset(null_iterator);
+ }
+}
+
+Status MapFileColumnIterator::init(const ColumnIteratorOptions& opts) {
+ RETURN_IF_ERROR(_key_iterator->init(opts));
+ RETURN_IF_ERROR(_val_iterator->init(opts));
+ if (_map_reader->is_nullable()) {
+ RETURN_IF_ERROR(_null_iterator->init(opts));
+ }
+ return Status::OK();
+}
+
+Status MapFileColumnIterator::next_batch(size_t* n, ColumnBlockView* dst,
bool* has_null) {
+ return Status::NotSupported("Not support next_batch");
+}
+
+Status MapFileColumnIterator::seek_to_ordinal(ordinal_t ord) {
+ RETURN_IF_ERROR(_key_iterator->seek_to_ordinal(ord));
+ RETURN_IF_ERROR(_val_iterator->seek_to_ordinal(ord));
+ if (_map_reader->is_nullable()) {
+ RETURN_IF_ERROR(_null_iterator->seek_to_ordinal(ord));
+ }
+ return Status::OK();
+}
+
+Status MapFileColumnIterator::next_batch(size_t* n,
vectorized::MutableColumnPtr& dst,
+ bool* has_null) {
+ const auto* column_map =
vectorized::check_and_get_column<vectorized::ColumnMap>(
+ dst->is_nullable() ?
static_cast<vectorized::ColumnNullable&>(*dst).get_nested_column()
+ : *dst);
+ size_t num_read = *n;
+ auto column_key_ptr = column_map->get_keys().assume_mutable();
+ auto column_val_ptr = column_map->get_values().assume_mutable();
+ RETURN_IF_ERROR(_key_iterator->next_batch(num_read, column_key_ptr,
has_null));
+ RETURN_IF_ERROR(_val_iterator->next_batch(num_read, column_val_ptr,
has_null));
Review Comment:
```suggestion
RETURN_IF_ERROR(_val_iterator->next_batch(&num_read, column_val_ptr,
has_null));
```
##########
fe/fe-core/src/main/java/org/apache/doris/analysis/CastExpr.java:
##########
@@ -318,6 +318,11 @@ public void analyze() throws AnalysisException {
type, Function.NullableMode.ALWAYS_NULLABLE,
Lists.newArrayList(Type.VARCHAR), false,
"doris::CastFunctions::cast_to_array_val", null, null,
true);
+ } else if (type.isMapType()) {
+ fn = ScalarFunction.createBuiltin(getFnName(Type.MAP),
+ type, Function.NullableMode.ALWAYS_NULLABLE,
+ Lists.newArrayList(Type.VARCHAR), false,
+ "doris::CastFunctions::cast_to_map_val", null, null, true);
Review Comment:
I see. No need to change here.
##########
be/src/vec/sink/vmysql_result_writer.cpp:
##########
@@ -715,6 +732,18 @@ Status VMysqlResultWriter::append_block(Block&
input_block) {
}
break;
}
+ case TYPE_MAP: {
+ if (type_ptr->is_nullable()) {
+ auto& nested_type =
+ assert_cast<const
DataTypeNullable&>(*type_ptr).get_nested_type(); //for map
+ status = _add_one_column<PrimitiveType::TYPE_MAP,
true>(column_ptr, result,
Review Comment:
_add_one_column has been modified by struct, please check this logic.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org
For queries about this service, please contact Infrastructure at:
us...@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org
For additional commands, e-mail: commits-h...@doris.apache.org
