github-actions[bot] commented on code in PR #35628:
URL: https://github.com/apache/doris/pull/35628#discussion_r1620524841
##########
be/src/olap/rowset/segment_v2/column_writer.cpp:
##########
@@ -88,267 +88,305 @@
RleEncoder<bool> _rle_encoder;
};
+Status ColumnWriter::create_struct_writer(const ColumnWriterOptions& opts,
+ const TabletColumn* column,
io::FileWriter* file_writer,
+ std::unique_ptr<ColumnWriter>*
writer) {
+ // not support empty struct
+ DCHECK(column->get_subtype_count() >= 1);
+ std::vector<std::unique_ptr<ColumnWriter>> sub_column_writers;
+ sub_column_writers.reserve(column->get_subtype_count());
+ for (uint32_t i = 0; i < column->get_subtype_count(); i++) {
+ const TabletColumn& sub_column = column->get_sub_column(i);
+
+ // create sub writer
+ ColumnWriterOptions column_options;
+ column_options.meta = opts.meta->mutable_children_columns(i);
+ column_options.need_zone_map = false;
+ column_options.need_bloom_filter = sub_column.is_bf_column();
+ column_options.need_bitmap_index = sub_column.has_bitmap_index();
+ if (sub_column.type() == FieldType::OLAP_FIELD_TYPE_STRUCT) {
+ if (column_options.need_bloom_filter) {
+ return Status::NotSupported("Do not support bloom filter for
struct type");
+ }
+ if (column_options.need_bitmap_index) {
+ return Status::NotSupported("Do not support bitmap index for
struct type");
+ }
+ }
+ if (sub_column.type() == FieldType::OLAP_FIELD_TYPE_ARRAY) {
+ if (column_options.need_bloom_filter) {
+ return Status::NotSupported("Do not support bloom filter for
array type");
+ }
+ if (column_options.need_bitmap_index) {
+ return Status::NotSupported("Do not support bitmap index for
array type");
+ }
+ }
+ std::unique_ptr<ColumnWriter> sub_column_writer;
+ RETURN_IF_ERROR(
+ ColumnWriter::create(column_options, &sub_column, file_writer,
&sub_column_writer));
+ sub_column_writers.push_back(std::move(sub_column_writer));
+ }
+
+ // if nullable, create null writer
+ ScalarColumnWriter* null_writer = nullptr;
+ if (opts.meta->is_nullable()) {
+ FieldType null_type = FieldType::OLAP_FIELD_TYPE_TINYINT;
+ ColumnWriterOptions null_options;
+ null_options.meta = opts.meta->add_children_columns();
+ null_options.meta->set_column_id(column->get_subtype_count() + 1);
+ null_options.meta->set_unique_id(column->get_subtype_count() + 1);
+ null_options.meta->set_type(int(null_type));
+ null_options.meta->set_is_nullable(false);
+ null_options.meta->set_length(
+
get_scalar_type_info<FieldType::OLAP_FIELD_TYPE_TINYINT>()->size());
+ null_options.meta->set_encoding(DEFAULT_ENCODING);
+ null_options.meta->set_compression(opts.meta->compression());
+
+ null_options.need_zone_map = false;
+ null_options.need_bloom_filter = false;
+ null_options.need_bitmap_index = false;
+
+ TabletColumn null_column =
+
TabletColumn(FieldAggregationMethod::OLAP_FIELD_AGGREGATION_NONE, null_type,
+ null_options.meta->is_nullable(),
null_options.meta->unique_id(),
+ null_options.meta->length());
+ null_column.set_name("nullable");
+ null_column.set_index_length(-1); // no short key index
+ std::unique_ptr<Field> null_field(FieldFactory::create(null_column));
+ null_writer = new ScalarColumnWriter(null_options,
std::move(null_field), file_writer);
+ }
+
+ *writer = std::unique_ptr<ColumnWriter>(
+ new StructColumnWriter(opts,
std::unique_ptr<Field>(FieldFactory::create(*column)),
+ null_writer, sub_column_writers));
+ return Status::OK();
+}
+
+Status ColumnWriter::create_array_writer(const ColumnWriterOptions& opts,
+ const TabletColumn* column,
io::FileWriter* file_writer,
+ std::unique_ptr<ColumnWriter>*
writer) {
+ DCHECK(column->get_subtype_count() == 1);
+ const TabletColumn& item_column = column->get_sub_column(0);
+
+ // create item writer
+ ColumnWriterOptions item_options;
+ item_options.meta = opts.meta->mutable_children_columns(0);
+ item_options.need_zone_map = false;
+ item_options.need_bloom_filter = item_column.is_bf_column();
+ item_options.need_bitmap_index = item_column.has_bitmap_index();
+ if (item_column.type() == FieldType::OLAP_FIELD_TYPE_ARRAY) {
+ if (item_options.need_bloom_filter) {
+ return Status::NotSupported("Do not support bloom filter for array
type");
+ }
+ if (item_options.need_bitmap_index) {
+ return Status::NotSupported("Do not support bitmap index for array
type");
+ }
+ }
+ std::unique_ptr<ColumnWriter> item_writer;
+ RETURN_IF_ERROR(ColumnWriter::create(item_options, &item_column,
file_writer, &item_writer));
+
+ // create length writer
+ FieldType length_type = FieldType::OLAP_FIELD_TYPE_UNSIGNED_BIGINT;
+
+ ColumnWriterOptions length_options;
+ length_options.meta = opts.meta->add_children_columns();
+ length_options.meta->set_column_id(2);
+ length_options.meta->set_unique_id(2);
+ length_options.meta->set_type(int(length_type));
+ length_options.meta->set_is_nullable(false);
+ length_options.meta->set_length(
+
get_scalar_type_info<FieldType::OLAP_FIELD_TYPE_UNSIGNED_BIGINT>()->size());
+ length_options.meta->set_encoding(DEFAULT_ENCODING);
+ length_options.meta->set_compression(opts.meta->compression());
+
+ length_options.need_zone_map = false;
+ length_options.need_bloom_filter = false;
+ length_options.need_bitmap_index = false;
+
+ TabletColumn length_column =
+ TabletColumn(FieldAggregationMethod::OLAP_FIELD_AGGREGATION_NONE,
length_type,
+ length_options.meta->is_nullable(),
length_options.meta->unique_id(),
+ length_options.meta->length());
+ length_column.set_name("length");
+ length_column.set_index_length(-1); // no short key index
+ std::unique_ptr<Field> bigint_field(FieldFactory::create(length_column));
+ auto* length_writer =
+ new OffsetColumnWriter(length_options, std::move(bigint_field),
file_writer);
+
+ // if nullable, create null writer
+ ScalarColumnWriter* null_writer = nullptr;
+ if (opts.meta->is_nullable()) {
+ FieldType null_type = FieldType::OLAP_FIELD_TYPE_TINYINT;
+ ColumnWriterOptions null_options;
+ null_options.meta = opts.meta->add_children_columns();
+ null_options.meta->set_column_id(3);
+ null_options.meta->set_unique_id(3);
+ null_options.meta->set_type(int(null_type));
+ null_options.meta->set_is_nullable(false);
+ null_options.meta->set_length(
+
get_scalar_type_info<FieldType::OLAP_FIELD_TYPE_TINYINT>()->size());
+ null_options.meta->set_encoding(DEFAULT_ENCODING);
+ null_options.meta->set_compression(opts.meta->compression());
+
+ null_options.need_zone_map = false;
+ null_options.need_bloom_filter = false;
+ null_options.need_bitmap_index = false;
+
+ TabletColumn null_column =
+
TabletColumn(FieldAggregationMethod::OLAP_FIELD_AGGREGATION_NONE, null_type,
+ length_options.meta->is_nullable(),
null_options.meta->unique_id(),
+ null_options.meta->length());
+ null_column.set_name("nullable");
+ null_column.set_index_length(-1); // no short key index
+ std::unique_ptr<Field> null_field(FieldFactory::create(null_column));
+ null_writer = new ScalarColumnWriter(null_options,
std::move(null_field), file_writer);
+ }
+
+ *writer = std::unique_ptr<ColumnWriter>(
+ new ArrayColumnWriter(opts,
std::unique_ptr<Field>(FieldFactory::create(*column)),
+ length_writer, null_writer,
std::move(item_writer)));
+ return Status::OK();
+}
+
+Status ColumnWriter::create_map_writer(const ColumnWriterOptions& opts, const
TabletColumn* column,
Review Comment:
warning: function 'create_map_writer' exceeds recommended size/complexity
thresholds [readability-function-size]
```cpp
Status ColumnWriter::create_map_writer(const ColumnWriterOptions& opts,
const TabletColumn* column,
^
```
<details>
<summary>Additional context</summary>
**be/src/olap/rowset/segment_v2/column_writer.cpp:249:** 88 lines including
whitespace and comments (threshold 80)
```cpp
Status ColumnWriter::create_map_writer(const ColumnWriterOptions& opts,
const TabletColumn* column,
^
```
</details>
##########
be/src/olap/rowset/segment_v2/column_writer.cpp:
##########
@@ -88,267 +88,305 @@ class NullBitmapBuilder {
RleEncoder<bool> _rle_encoder;
};
+Status ColumnWriter::create_struct_writer(const ColumnWriterOptions& opts,
+ const TabletColumn* column,
io::FileWriter* file_writer,
+ std::unique_ptr<ColumnWriter>*
writer) {
+ // not support empty struct
+ DCHECK(column->get_subtype_count() >= 1);
+ std::vector<std::unique_ptr<ColumnWriter>> sub_column_writers;
+ sub_column_writers.reserve(column->get_subtype_count());
+ for (uint32_t i = 0; i < column->get_subtype_count(); i++) {
+ const TabletColumn& sub_column = column->get_sub_column(i);
+
+ // create sub writer
+ ColumnWriterOptions column_options;
+ column_options.meta = opts.meta->mutable_children_columns(i);
+ column_options.need_zone_map = false;
+ column_options.need_bloom_filter = sub_column.is_bf_column();
+ column_options.need_bitmap_index = sub_column.has_bitmap_index();
+ if (sub_column.type() == FieldType::OLAP_FIELD_TYPE_STRUCT) {
+ if (column_options.need_bloom_filter) {
+ return Status::NotSupported("Do not support bloom filter for
struct type");
+ }
+ if (column_options.need_bitmap_index) {
+ return Status::NotSupported("Do not support bitmap index for
struct type");
+ }
+ }
+ if (sub_column.type() == FieldType::OLAP_FIELD_TYPE_ARRAY) {
+ if (column_options.need_bloom_filter) {
+ return Status::NotSupported("Do not support bloom filter for
array type");
+ }
+ if (column_options.need_bitmap_index) {
+ return Status::NotSupported("Do not support bitmap index for
array type");
+ }
+ }
+ std::unique_ptr<ColumnWriter> sub_column_writer;
+ RETURN_IF_ERROR(
+ ColumnWriter::create(column_options, &sub_column, file_writer,
&sub_column_writer));
+ sub_column_writers.push_back(std::move(sub_column_writer));
+ }
+
+ // if nullable, create null writer
+ ScalarColumnWriter* null_writer = nullptr;
+ if (opts.meta->is_nullable()) {
+ FieldType null_type = FieldType::OLAP_FIELD_TYPE_TINYINT;
+ ColumnWriterOptions null_options;
+ null_options.meta = opts.meta->add_children_columns();
+ null_options.meta->set_column_id(column->get_subtype_count() + 1);
+ null_options.meta->set_unique_id(column->get_subtype_count() + 1);
+ null_options.meta->set_type(int(null_type));
+ null_options.meta->set_is_nullable(false);
+ null_options.meta->set_length(
+
get_scalar_type_info<FieldType::OLAP_FIELD_TYPE_TINYINT>()->size());
+ null_options.meta->set_encoding(DEFAULT_ENCODING);
+ null_options.meta->set_compression(opts.meta->compression());
+
+ null_options.need_zone_map = false;
+ null_options.need_bloom_filter = false;
+ null_options.need_bitmap_index = false;
+
+ TabletColumn null_column =
+
TabletColumn(FieldAggregationMethod::OLAP_FIELD_AGGREGATION_NONE, null_type,
+ null_options.meta->is_nullable(),
null_options.meta->unique_id(),
+ null_options.meta->length());
+ null_column.set_name("nullable");
+ null_column.set_index_length(-1); // no short key index
+ std::unique_ptr<Field> null_field(FieldFactory::create(null_column));
+ null_writer = new ScalarColumnWriter(null_options,
std::move(null_field), file_writer);
+ }
+
+ *writer = std::unique_ptr<ColumnWriter>(
+ new StructColumnWriter(opts,
std::unique_ptr<Field>(FieldFactory::create(*column)),
+ null_writer, sub_column_writers));
+ return Status::OK();
+}
+
+Status ColumnWriter::create_array_writer(const ColumnWriterOptions& opts,
Review Comment:
warning: function 'create_array_writer' exceeds recommended size/complexity
thresholds [readability-function-size]
```cpp
Status ColumnWriter::create_array_writer(const ColumnWriterOptions& opts,
^
```
<details>
<summary>Additional context</summary>
**be/src/olap/rowset/segment_v2/column_writer.cpp:163:** 82 lines including
whitespace and comments (threshold 80)
```cpp
Status ColumnWriter::create_array_writer(const ColumnWriterOptions& opts,
^
```
</details>
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org
For queries about this service, please contact Infrastructure at:
us...@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org
For additional commands, e-mail: commits-h...@doris.apache.org
