This is an automated email from the ASF dual-hosted git repository.
yiguolei pushed a commit to branch branch-4.2
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/branch-4.2 by this push:
new 807c3a02a37 branch-4.2: [Opt](variant) Stream the VARIANT root column
and bound segment-buffering writers (#69362)
807c3a02a37 is described below
commit 807c3a02a37d667cc632b6d5096ce2e1e22b8b3b
Author: lihangyu <[email protected]>
AuthorDate: Wed Oct 21 18:31:52 2026 +0800
branch-4.2: [Opt](variant) Stream the VARIANT root column and bound
segment-buffering writers (#69362)
cherry-pick #69221
---
be/src/storage/segment/column_writer.h | 25 +
.../segment/variant/v2/variant_column_writer.cpp | 41 +-
.../segment/variant/v2/variant_column_writer.h | 9 +-
.../segment/variant/v2/variant_path_builder.cpp | 75 ++-
.../segment/variant/v2/variant_path_builder.h | 16 +-
.../segment/variant/variant_column_writer_impl.cpp | 212 +++++--
.../segment/variant/variant_writer_helpers.cpp | 32 +-
.../segment/variant/variant_writer_helpers.h | 7 +
.../variant/variant_column_writer_reader_test.cpp | 628 ++++++++++++++++++++-
9 files changed, 962 insertions(+), 83 deletions(-)
diff --git a/be/src/storage/segment/column_writer.h
b/be/src/storage/segment/column_writer.h
index bcfbcb2c7d1..da9e96741b5 100644
--- a/be/src/storage/segment/column_writer.h
+++ b/be/src/storage/segment/column_writer.h
@@ -647,6 +647,14 @@ public:
size_t get_non_null_size() const { return none_null_size; }
+#ifdef BE_TEST
+ // Force small chunk budgets so tests can exercise multi-chunk flushes
cheaply.
+ static void set_chunk_bytes_for_test(size_t bytes);
+ static void set_chunk_values_for_test(size_t values);
+ size_t num_chunks_for_test() const;
+ size_t max_chunk_bytes_for_test() const;
+#endif
+
Status finalize();
private:
@@ -655,6 +663,17 @@ private:
std::span<const uint8_t> outer_nulls);
Status _ensure_input_format(const VariantColumnData& column);
Status _initialize_v2_builder();
+ Status _promote_chunks(DataTypePtr* inferred_type);
+ Status _seal_chunk();
+ // Bounded value chunks; each chunk is finalized and flushed independently
so the writer never
+ // holds a segment-sized ColumnString.
+ struct ValueChunk {
+ std::unique_ptr<VariantPathBuilder> builder;
+ size_t row_begin = 0;
+ size_t rows = 0;
+ };
+ static constexpr size_t kChunkBytes = 256ULL << 20;
+ static constexpr size_t kChunkValues = 4ULL << 20;
bool is_finalized() const;
bool _is_finalized = false;
ordinal_t _next_rowid = 0;
@@ -662,6 +681,12 @@ private:
VariantWriterInputFormat _input_format = VariantWriterInputFormat::UNSET;
ColumnVariant::MutablePtr _v1_column;
std::unique_ptr<VariantPathBuilder> _v2_builder;
+ std::vector<ValueChunk> _chunks;
+ size_t _chunk_row_begin = 0;
+ size_t _chunk_rows = 0;
+ // Type carried into the next chunk. Inferred promotions also check and
update sealed chunks
+ // so fallback follows the same sequence as one uninterrupted builder.
+ DataTypePtr _running_type;
size_t _num_rows = 0;
ColumnWriterOptions _opts;
std::unique_ptr<ColumnWriter> _writer;
diff --git a/be/src/storage/segment/variant/v2/variant_column_writer.cpp
b/be/src/storage/segment/variant/v2/variant_column_writer.cpp
index a641fae2492..e7c8da11f2e 100644
--- a/be/src/storage/segment/variant/v2/variant_column_writer.cpp
+++ b/be/src/storage/segment/variant/v2/variant_column_writer.cpp
@@ -107,7 +107,12 @@ Status VariantV2ColumnWriter::init() {
_root_only = variant_writer_helpers::has_extracted_variant_columns(
*_opts.rowset_ctx->tablet_schema, _tablet_column->unique_id());
if (_root_only) {
- _root_jsonb = ColumnString::create();
+ // Stream the root column instead of buffering a whole segment: the
JSONB ColumnString
+ // would otherwise have to hold every row of the segment and could
exceed the
+ // ColumnStr<uint32_t> 4 GiB limit before finalize.
+ _root_writer = std::make_unique<ScalarColumnWriter>(
+ _opts, std::make_shared<TabletColumn>(*_tablet_column),
_opts.file_writer);
+ RETURN_IF_ERROR(_root_writer->init());
return Status::OK();
}
@@ -127,15 +132,14 @@ Status VariantV2ColumnWriter::append(const
VariantColumnData& column, size_t num
return Status::InternalError("Cannot append ColumnVariantV2 after
writer finalization");
}
if (_root_only) {
- DORIS_CHECK(_root_jsonb);
+ DORIS_CHECK(_root_writer);
if (num_rows > std::numeric_limits<size_t>::max() - _num_rows) {
return Status::InvalidArgument("Variant writer row count overflows
size_t");
}
ColumnString::MutablePtr batch_root_jsonb;
RETURN_IF_ERROR(build_root_only_batch(column, num_rows, outer_nulls,
&batch_root_jsonb));
DORIS_CHECK_EQ(batch_root_jsonb->size(), num_rows);
- RETURN_IF_CATCH_EXCEPTION(
- { _root_jsonb->insert_range_from(*batch_root_jsonb, 0,
num_rows); });
+ RETURN_IF_ERROR(_append_root_batch(*batch_root_jsonb, outer_nulls));
} else {
RETURN_IF_ERROR(variant_writer_helpers::append_variant_v2_to_shredder(
_shredder.get(), column, num_rows, outer_nulls));
@@ -150,6 +154,25 @@ Status VariantV2ColumnWriter::append(const
VariantColumnData& column, size_t num
return Status::OK();
}
+Status VariantV2ColumnWriter::_append_root_batch(const ColumnString& batch,
+ std::span<const uint8_t>
outer_nulls) {
+ DORIS_CHECK(_root_writer != nullptr);
+ if (batch.size() == 0) {
+ return Status::OK();
+ }
+ if (!outer_nulls.empty() && outer_nulls.size() != batch.size()) {
+ return Status::InvalidArgument("Variant root outer-null span has {}
rows, expected {}",
+ outer_nulls.size(), batch.size());
+ }
+ DorisVector<Slice> slices(batch.size());
+ for (size_t row = 0; row < batch.size(); ++row) {
+ slices[row] = batch.get_data_at(row).to_slice();
+ }
+ const uint8_t* null_map =
+ _tablet_column->is_nullable() && !outer_nulls.empty() ?
outer_nulls.data() : nullptr;
+ return _root_writer->append(null_map, slices.data(), batch.size());
+}
+
Status VariantV2ColumnWriter::_write_root(const IColumn* root_jsonb, int&
column_id) {
if (!root_jsonb || root_jsonb->size() != _num_rows) {
return Status::InvalidArgument("Variant root column has {} rows,
expected {}",
@@ -229,10 +252,8 @@ Status VariantV2ColumnWriter::finalize() {
}
if (_root_only) {
DORIS_CHECK(_shredder == nullptr);
- DORIS_CHECK(_root_jsonb);
- int column_id = 0;
- RETURN_IF_ERROR(_write_root(_root_jsonb.get(), column_id));
- _root_jsonb.reset();
+ DORIS_CHECK(_root_writer != nullptr);
+ _opts.meta->set_num_rows(_num_rows);
_is_finalized = true;
return Status::OK();
}
@@ -341,8 +362,8 @@ Status VariantV2ColumnWriter::write_bloom_filter_index() {
uint64_t VariantV2ColumnWriter::estimate_buffer_size() {
if (!_is_finalized) {
if (_root_only) {
- DORIS_CHECK(_root_jsonb);
- return _outer_nulls->byte_size() + _root_jsonb->byte_size();
+ DORIS_CHECK(_root_writer != nullptr);
+ return _outer_nulls->byte_size() +
_root_writer->estimate_buffer_size();
}
DORIS_CHECK(_shredder != nullptr);
return _outer_nulls->byte_size() + _shredder->byte_size();
diff --git a/be/src/storage/segment/variant/v2/variant_column_writer.h
b/be/src/storage/segment/variant/v2/variant_column_writer.h
index ea982f58a87..491cdf5a964 100644
--- a/be/src/storage/segment/variant/v2/variant_column_writer.h
+++ b/be/src/storage/segment/variant/v2/variant_column_writer.h
@@ -38,9 +38,9 @@ class VariantBinaryWriter;
class VariantShredder;
struct VariantShreddedColumns;
-// Native storage writer for ColumnVariantV2. Full values are incrementally
shredded at each append;
-// a parent with extracted children retains only its root JSONB column. The
writer never retains a
-// segment-sized copy of the input execution column.
+// Native storage writer for ColumnVariantV2. Full values are incrementally
shredded at each append.
+// A parent with extracted children streams its root JSONB column into the
root column writer batch
+// by batch. The writer never retains a segment-sized copy of the input
execution column.
class VariantV2ColumnWriter {
public:
VariantV2ColumnWriter(ColumnWriterOptions opts, const TabletColumn*
column);
@@ -65,6 +65,8 @@ public:
private:
Status _write_root(const IColumn* root_jsonb, int& column_id);
+ // Streams one batch of the root-only JSONB column into the root column
writer.
+ Status _append_root_batch(const ColumnString& batch, std::span<const
uint8_t> outer_nulls);
Status _write_materialized(const VariantShreddedColumns& shredded,
OlapBlockDataConvertor* converter, int&
column_id);
Status _write_binary(const VariantShreddedColumns& shredded,
OlapBlockDataConvertor* converter,
@@ -74,7 +76,6 @@ private:
ColumnWriterOptions _opts;
const TabletColumn* _tablet_column = nullptr;
std::unique_ptr<VariantShredder> _shredder;
- ColumnString::MutablePtr _root_jsonb;
ColumnUInt8::MutablePtr _outer_nulls;
size_t _num_rows = 0;
bool _root_only = false;
diff --git a/be/src/storage/segment/variant/v2/variant_path_builder.cpp
b/be/src/storage/segment/variant/v2/variant_path_builder.cpp
index 9c79d604894..6ad8be228bd 100644
--- a/be/src/storage/segment/variant/v2/variant_path_builder.cpp
+++ b/be/src/storage/segment/variant/v2/variant_path_builder.cpp
@@ -371,6 +371,11 @@ DataTypePtr path_least_common_type(const DataTypePtr&
left, const DataTypePtr& r
if (left.get() == right.get() || left->equals(*right)) {
return left;
}
+ // Chunk builders expose Nullable(Array(...)). Unwrap before dispatching
so decimal array
+ // elements use the scale-aware rules below instead of the generic
first-primitive match.
+ if (left->is_nullable() || right->is_nullable()) {
+ return make_nullable(path_least_common_type(remove_nullable(left),
remove_nullable(right)));
+ }
const auto* left_array = typeid_cast<const DataTypeArray*>(left.get());
const auto* right_array = typeid_cast<const DataTypeArray*>(right.get());
if (left_array != nullptr || right_array != nullptr) {
@@ -1042,6 +1047,10 @@ bool cast_introduced_null(const IColumn& source, const
IColumn& target) {
} // namespace
+DataTypePtr variant_path_least_common_type(const DataTypePtr& left, const
DataTypePtr& right) {
+ return path_least_common_type(left, right);
+}
+
DataTypePtr normalize_variant_path_integer_widths(const DataTypePtr& type) {
const DataTypePtr base = remove_nullable(type);
if (const auto* array = typeid_cast<const DataTypeArray*>(base.get())) {
@@ -1082,39 +1091,54 @@ struct VariantPathBuilder::Impl {
return Status::OK();
}
- Status promote(const DataTypePtr& target_type, bool filter_cast_nulls) {
- DataTypePtr target = remove_nullable(target_type);
- if (target->equals(*type)) {
- return Status::OK();
- }
- ColumnPtr promoted;
+ Status cast_to_type(const DataTypePtr& target, ColumnPtr* promoted) const {
if (target->get_primitive_type() == TYPE_STRING &&
type->get_primitive_type() == TYPE_ARRAY) {
RETURN_IF_ERROR(stringify_complex_column(
- type, assert_cast<const ColumnNullable&>(*column),
&promoted));
+ type, assert_cast<const ColumnNullable&>(*column),
promoted));
} else if (type->get_primitive_type() == TYPE_ARRAY &&
target->get_primitive_type() == TYPE_ARRAY &&
variant_path_type_contains_nothing(type)) {
MutableColumnPtr materialized;
RETURN_IF_ERROR(replace_array_nothing(nullable_type, *column,
make_nullable(target),
&materialized));
- promoted = std::move(materialized);
+ *promoted = std::move(materialized);
} else {
RETURN_IF_ERROR(
variant_util::cast_column({column->get_ptr(),
nullable_type, path.get_path()},
- make_nullable(target),
&promoted));
+ make_nullable(target),
promoted));
}
+ return Status::OK();
+ }
+
+ Status promote(const DataTypePtr& target_type, bool filter_cast_nulls,
+ const InferredPromotion& on_promotion = {}) {
+ DataTypePtr target = remove_nullable(target_type);
+ if (target->equals(*type)) {
+ return Status::OK();
+ }
+ ColumnPtr promoted;
+ RETURN_IF_ERROR(cast_to_type(target, &promoted));
// Inferred widening is only a storage representation choice. If CAST
loses a valid value
// at any array depth, preserve the whole path as JSONB instead.
Forced typed-path
// conversion intentionally retains its existing cast-null filtering
below.
if (!filter_cast_nulls && target->get_primitive_type() != TYPE_JSONB &&
cast_introduced_null(*column, *promoted)) {
target = jsonb_type();
- RETURN_IF_ERROR(
- variant_util::cast_column({column->get_ptr(),
nullable_type, path.get_path()},
- make_nullable(target),
&promoted));
+ RETURN_IF_ERROR(cast_to_type(target, &promoted));
DORIS_CHECK(!assert_cast<const
ColumnNullable&>(*promoted).has_null());
}
+ if (on_promotion) {
+ const DataTypePtr proposed = target;
+ RETURN_IF_ERROR(on_promotion(&target));
+ target = remove_nullable(target);
+ if (!target->equals(*proposed)) {
+ // For example, the active chunk's 0.1 fits scale 38, but a
sealed large Decimal
+ // does not. Recast the original active values to JSONB,
preserving 0.1's scale.
+ DORIS_CHECK_EQ(target->get_primitive_type(), TYPE_JSONB);
+ RETURN_IF_ERROR(cast_to_type(target, &promoted));
+ }
+ }
const auto& nullable = assert_cast<const ColumnNullable&>(*promoted);
if (nullable.has_null()) {
DORIS_CHECK(filter_cast_nulls);
@@ -1164,7 +1188,8 @@ VariantPathBuilder::~VariantPathBuilder() = default;
VariantPathBuilder::VariantPathBuilder(VariantPathBuilder&&) noexcept =
default;
VariantPathBuilder& VariantPathBuilder::operator=(VariantPathBuilder&&)
noexcept = default;
-Status VariantPathBuilder::append(VariantRef value, size_t row) {
+Status VariantPathBuilder::append(VariantRef value, size_t row,
+ const InferredPromotion& on_promotion) {
try {
const ScalarPhysical physical = scalar_physical(value);
if (physical.kind == ScalarPhysicalKind::NULL_VALUE) {
@@ -1193,14 +1218,14 @@ Status VariantPathBuilder::append(VariantRef value,
size_t row) {
DataTypePtr common_type =
path_least_common_type(_impl->type, inferred_type);
if (_impl->type.get() != common_type.get() &&
!_impl->type->equals(*common_type)) {
- RETURN_IF_ERROR(_impl->promote(common_type, false));
+ RETURN_IF_ERROR(_impl->promote(common_type, false,
on_promotion));
}
}
}
}
const bool is_array = !stable_scalar && value_kind(value) ==
ValueKind::ARRAY;
if (is_array && !value_is_representable(value, _impl->type)) {
- RETURN_IF_ERROR(_impl->promote(jsonb_type(), false));
+ RETURN_IF_ERROR(_impl->promote(jsonb_type(), false, on_promotion));
}
try {
@@ -1211,7 +1236,7 @@ Status VariantPathBuilder::append(VariantRef value,
size_t row) {
if (is_array || _impl->type->get_primitive_type() == TYPE_JSONB) {
throw;
}
- RETURN_IF_ERROR(_impl->promote(jsonb_type(), false));
+ RETURN_IF_ERROR(_impl->promote(jsonb_type(), false, on_promotion));
auto& nullable = assert_cast<ColumnNullable&>(*_impl->column);
append_value(value, _impl->type, &nullable.get_nested_column());
nullable.get_null_map_data().push_back(0);
@@ -1236,11 +1261,25 @@ Status VariantPathBuilder::complete_rows(size_t rows) {
return Status::OK();
}
-Status VariantPathBuilder::convert_to(const DataTypePtr& storage_type) {
+Status VariantPathBuilder::convert_to(const DataTypePtr& storage_type, bool
filter_cast_nulls) {
if (!_impl->column) {
return _impl->initialize(storage_type);
}
- return _impl->promote(storage_type, true);
+ return _impl->promote(storage_type, filter_cast_nulls);
+}
+
+Status VariantPathBuilder::check_inferred_promotion(const DataTypePtr&
target_type,
+ bool* needs_jsonb) const {
+ const DataTypePtr target = remove_nullable(target_type);
+ *needs_jsonb = false;
+ if (!_impl->column || target->equals(*_impl->type) ||
+ target->get_primitive_type() == TYPE_JSONB) {
+ return Status::OK();
+ }
+ ColumnPtr promoted;
+ RETURN_IF_ERROR(_impl->cast_to_type(target, &promoted));
+ *needs_jsonb = cast_introduced_null(*_impl->column, *promoted);
+ return Status::OK();
}
const PathInData& VariantPathBuilder::path() const {
diff --git a/be/src/storage/segment/variant/v2/variant_path_builder.h
b/be/src/storage/segment/variant/v2/variant_path_builder.h
index 7b467a17ae7..cc21594840a 100644
--- a/be/src/storage/segment/variant/v2/variant_path_builder.h
+++ b/be/src/storage/segment/variant/v2/variant_path_builder.h
@@ -19,6 +19,7 @@
#include <cstddef>
#include <cstdint>
+#include <functional>
#include <memory>
#include <span>
@@ -54,6 +55,10 @@ bool variant_path_type_contains_nothing(const DataTypePtr&
type);
// returns.
class VariantPathBuilder final {
public:
+ // Resolve inferred widening against values held outside this builder,
before either side
+ // commits its cast. The resolver may replace the proposed type with JSONB.
+ using InferredPromotion = std::function<Status(DataTypePtr*)>;
+
explicit VariantPathBuilder(PathInData path, size_t prefix_rows = 0);
~VariantPathBuilder();
@@ -62,9 +67,13 @@ public:
VariantPathBuilder(const VariantPathBuilder&) = delete;
VariantPathBuilder& operator=(const VariantPathBuilder&) = delete;
- Status append(VariantRef value, size_t row);
+ Status append(VariantRef value, size_t row, const InferredPromotion&
on_promotion = {});
Status complete_rows(size_t rows);
- Status convert_to(const DataTypePtr& storage_type);
+ // Inferred conversions preserve values through JSONB fallback; explicit
storage coercion
+ // retains the default cast-null filtering.
+ Status convert_to(const DataTypePtr& storage_type, bool filter_cast_nulls
= true);
+ // Probe inferred widening without changing values, types or rowids in
retained chunks.
+ Status check_inferred_promotion(const DataTypePtr& target_type, bool*
needs_jsonb) const;
const PathInData& path() const;
const DataTypePtr& type() const;
@@ -107,4 +116,7 @@ VariantPathSelection select_variant_paths(std::span<const
VariantPathSelectionCa
size_t
max_dynamic_materialized_paths,
bool typed_paths_to_sparse);
+// Least common type for a variant path. Used to merge types inferred by
independent value chunks.
+DataTypePtr variant_path_least_common_type(const DataTypePtr& left, const
DataTypePtr& right);
+
} // namespace doris::segment_v2
diff --git a/be/src/storage/segment/variant/variant_column_writer_impl.cpp
b/be/src/storage/segment/variant/variant_column_writer_impl.cpp
index 6e044cf7ccc..b04235b60e6 100644
--- a/be/src/storage/segment/variant/variant_column_writer_impl.cpp
+++ b/be/src/storage/segment/variant/variant_column_writer_impl.cpp
@@ -1979,6 +1979,11 @@ uint64_t
VariantColumnWriterImpl::get_total_data_pages_bytes(
return 0;
}
+#ifdef BE_TEST
+static size_t g_chunk_bytes_for_test = 0;
+static size_t g_chunk_values_for_test = 0;
+#endif
+
VariantSubcolumnWriter::VariantSubcolumnWriter(const ColumnWriterOptions& opts,
TabletColumnPtr column)
: ColumnWriter(std::move(column), opts.meta->is_nullable(), opts.meta)
{
@@ -2019,7 +2024,6 @@ Status VariantSubcolumnWriter::_ensure_input_format(const
VariantColumnData& col
Status VariantSubcolumnWriter::_append_v2(const VariantColumnData& column,
size_t num_rows,
std::span<const uint8_t>
outer_nulls) {
- DORIS_CHECK(_v2_builder != nullptr);
DORIS_CHECK(column.column_data != nullptr);
const auto* source =
check_and_get_column<ColumnVariantV2>(*column.column_data);
DORIS_CHECK(source != nullptr);
@@ -2028,7 +2032,23 @@ Status VariantSubcolumnWriter::_append_v2(const
VariantColumnData& column, size_
column.row_pos, column.row_pos +
num_rows, source->size());
}
DORIS_CHECK(outer_nulls.empty() || outer_nulls.size() == num_rows);
+ if (_v2_builder == nullptr) {
+ RETURN_IF_ERROR(_initialize_v2_builder());
+ if (_running_type != nullptr) {
+ // Continue from the type inferred by the sealed chunks. Appending
a value that the
+ // previous chunks' type cannot hold (for example a bool after
integers) then takes
+ // the builder's own JSONB fallback inside this chunk instead of
being merged away.
+ RETURN_IF_ERROR(_v2_builder->convert_to(_running_type));
+ }
+ _chunk_row_begin = _num_rows;
+ _chunk_rows = 0;
+ }
+ VariantPathBuilder::InferredPromotion on_promotion;
+ // Until a chunk is sealed, the active builder already holds every value
to be checked.
+ if (!_chunks.empty()) {
+ on_promotion = [this](DataTypePtr* type) { return
_promote_chunks(type); };
+ }
const auto append_encoded = [&](const ColumnVariantV2::ReadView& view,
size_t begin) -> Status {
DORIS_CHECK(!view.is_typed());
for (size_t offset = 0; offset < num_rows; ++offset) {
@@ -2038,7 +2058,8 @@ Status VariantSubcolumnWriter::_append_v2(const
VariantColumnData& column, size_
const size_t input_row = begin + offset;
const VariantRef value = view.value_at(input_row);
if (!value.is_null()) {
- RETURN_IF_ERROR(_v2_builder->append(value, _num_rows +
offset));
+ RETURN_IF_ERROR(_v2_builder->append(value, _num_rows + offset
- _chunk_row_begin,
+ on_promotion));
}
}
return Status::OK();
@@ -2046,13 +2067,69 @@ Status VariantSubcolumnWriter::_append_v2(const
VariantColumnData& column, size_
const auto view = source->read_view();
if (!view.is_typed()) {
- return append_encoded(view, column.row_pos);
+ RETURN_IF_ERROR(append_encoded(view, column.row_pos));
+ } else {
+ auto encoded_batch = ColumnVariantV2::create();
+ RETURN_IF_CATCH_EXCEPTION({
+ encoded_batch->insert_range_from(*source, column.row_pos,
num_rows);
+ encoded_batch->ensure_encoded();
+ });
+ RETURN_IF_ERROR(append_encoded(encoded_batch->read_view(), 0));
+ }
+ _chunk_rows = _num_rows + num_rows - _chunk_row_begin;
+ const size_t chunk_bytes =
+#ifdef BE_TEST
+ g_chunk_bytes_for_test != 0 ? g_chunk_bytes_for_test : kChunkBytes;
+#else
+ kChunkBytes;
+#endif
+ const size_t chunk_values =
+#ifdef BE_TEST
+ g_chunk_values_for_test != 0 ? g_chunk_values_for_test :
kChunkValues;
+#else
+ kChunkValues;
+#endif
+ if (_v2_builder->byte_size() >= chunk_bytes ||
_v2_builder->non_null_rows() >= chunk_values) {
+ RETURN_IF_ERROR(_seal_chunk());
}
+ return Status::OK();
+}
- auto encoded_batch = ColumnVariantV2::create();
- RETURN_IF_CATCH_EXCEPTION(
- { encoded_batch->insert_range_from(*source, column.row_pos,
num_rows); });
- return append_encoded(encoded_batch->read_view(), 0);
+Status VariantSubcolumnWriter::_promote_chunks(DataTypePtr* inferred_type) {
+ // Decide against every retained value before changing any chunk. A small
scale-1 Decimal may
+ // fit scale 38 while a later large Decimal does not; committing the first
cast would change
+ // its JSONB representation before discovering that the whole path needs
fallback.
+ for (const auto& chunk : _chunks) {
+ if (chunk.builder->non_null_rows() == 0) {
+ continue;
+ }
+ bool needs_jsonb = false;
+
RETURN_IF_ERROR(chunk.builder->check_inferred_promotion(*inferred_type,
&needs_jsonb));
+ if (needs_jsonb) {
+ *inferred_type = std::make_shared<DataTypeJsonb>();
+ break;
+ }
+ }
+ for (auto& chunk : _chunks) {
+ if (chunk.builder->non_null_rows() != 0) {
+ RETURN_IF_ERROR(
+ chunk.builder->convert_to(*inferred_type, false /*
filter_cast_nulls */));
+ }
+ }
+ return Status::OK();
+}
+
+Status VariantSubcolumnWriter::_seal_chunk() {
+ if (_v2_builder == nullptr) {
+ return Status::OK();
+ }
+ RETURN_IF_ERROR(_v2_builder->complete_rows(_chunk_rows));
+ _running_type = _v2_builder->type();
+ _chunks.push_back(ValueChunk {
+ .builder = std::move(_v2_builder), .row_begin = _chunk_row_begin,
.rows = _chunk_rows});
+ _v2_builder.reset();
+ _chunk_rows = 0;
+ return Status::OK();
}
Status VariantSubcolumnWriter::_append(const uint8_t* null_map, const
uint8_t** ptr,
@@ -2089,14 +2166,40 @@ Status VariantSubcolumnWriter::append_data(const
uint8_t** ptr, size_t num_rows)
return _append(nullptr, ptr, num_rows);
}
+#ifdef BE_TEST
+void VariantSubcolumnWriter::set_chunk_bytes_for_test(size_t bytes) {
+ g_chunk_bytes_for_test = bytes;
+}
+
+void VariantSubcolumnWriter::set_chunk_values_for_test(size_t values) {
+ g_chunk_values_for_test = values;
+}
+
+size_t VariantSubcolumnWriter::num_chunks_for_test() const {
+ return _chunks.size() + (_v2_builder != nullptr ? 1 : 0);
+}
+
+size_t VariantSubcolumnWriter::max_chunk_bytes_for_test() const {
+ size_t max = _v2_builder != nullptr ? _v2_builder->byte_size() : 0;
+ for (const auto& chunk : _chunks) {
+ max = std::max(max, chunk.builder->byte_size());
+ }
+ return max;
+}
+#endif
+
uint64_t VariantSubcolumnWriter::estimate_buffer_size() {
if (is_finalized()) {
return _writer ? _writer->estimate_buffer_size() : 0;
}
- if (_input_format == VariantWriterInputFormat::V2) {
- return _v2_builder->byte_size();
+ if (_input_format != VariantWriterInputFormat::V2) {
+ return _v1_column->byte_size();
}
- return _v1_column->byte_size();
+ size_t bytes = _v2_builder ? _v2_builder->byte_size() : 0;
+ for (const auto& chunk : _chunks) {
+ bytes += chunk.builder->byte_size();
+ }
+ return bytes;
}
bool VariantSubcolumnWriter::is_finalized() const {
@@ -2110,14 +2213,16 @@ Status VariantSubcolumnWriter::finalize() {
if (_input_format == VariantWriterInputFormat::UNSET) {
_input_format = VariantWriterInputFormat::V1;
}
+ RETURN_IF_ERROR(_seal_chunk());
+
const auto& parent_column =
_opts.rowset_ctx->tablet_schema->column_by_uid(get_column()->parent_unique_id());
DORIS_CHECK(!parent_column.variant_enable_nested_group());
DataTypePtr flush_type;
ColumnPtr flush_values;
- DorisVector<uint32_t> flush_rowids;
- int64_t non_null_value_size = 0;
+ size_t non_null_value_size = 0;
+ bool publish_builder = false;
if (_input_format == VariantWriterInputFormat::V1) {
_v1_column->finalize();
DCHECK(_v1_column->is_finalized());
@@ -2133,41 +2238,51 @@ Status VariantSubcolumnWriter::finalize() {
non_null_value_size =
_v1_column->get_subcolumns().get_root()->data.get_non_null_value_size();
} else {
- DORIS_CHECK(_v2_builder != nullptr);
- RETURN_IF_ERROR(_v2_builder->complete_rows(_num_rows));
DataTypePtr storage_type;
TabletSchema::SubColumnInfo subcolumn_info;
- if (variant_util::generate_sub_column_info(
- *_opts.rowset_ctx->tablet_schema,
parent_column.unique_id(),
- _v2_builder->path().get_path(), &subcolumn_info)) {
+ const PathInData relative_path =
get_column()->path_info_ptr()->copy_pop_front();
+ if
(variant_util::generate_sub_column_info(*_opts.rowset_ctx->tablet_schema,
+ parent_column.unique_id(),
+ relative_path.get_path(),
&subcolumn_info)) {
storage_type =
DataTypeFactory::instance().create_data_type(subcolumn_info.column);
}
- bool publish_builder = _v2_builder->non_null_rows() != 0;
+ // Merge the types inferred by independent chunks; the result is what
a single builder would
+ // have inferred for the whole segment.
+ DataTypePtr merged_type;
+ bool has_values = false;
+ for (auto& chunk : _chunks) {
+ if (chunk.builder->non_null_rows() == 0) {
+ continue;
+ }
+ has_values = true;
+ RETURN_IF_ERROR(chunk.builder->convert_to(
+
normalize_variant_path_integer_widths(chunk.builder->type())));
+ merged_type = merged_type == nullptr ? chunk.builder->type()
+ :
variant_path_least_common_type(
+ merged_type,
chunk.builder->type());
+ }
+
+ publish_builder = has_values;
if (publish_builder) {
- RETURN_IF_ERROR(_v2_builder->convert_to(
-
normalize_variant_path_integer_widths(_v2_builder->type())));
publish_builder =
- !variant_path_type_contains_nothing(_v2_builder->type()) ||
+ !variant_path_type_contains_nothing(merged_type) ||
(storage_type != nullptr &&
!variant_path_type_contains_nothing(storage_type));
}
- if (publish_builder && storage_type != nullptr) {
- RETURN_IF_ERROR(_v2_builder->convert_to(storage_type));
- }
if (!publish_builder) {
flush_type =
DataTypeFactory::instance().create_data_type(PrimitiveType::TYPE_TINYINT,
true /*
is_nullable */);
- MutableColumnPtr defaults = flush_type->create_column();
- flush_values = std::move(defaults);
} else {
- flush_type = _v2_builder->type();
- flush_values = _v2_builder->column();
- const std::span<const uint32_t> rowids = _v2_builder->rowids();
- flush_rowids.assign(rowids.begin(), rowids.end());
- non_null_value_size = cast_set<int64_t>(flush_rowids.size());
+ flush_type = storage_type != nullptr ? storage_type : merged_type;
+ for (auto& chunk : _chunks) {
+ if (chunk.builder->non_null_rows() == 0) {
+ continue;
+ }
+ RETURN_IF_ERROR(chunk.builder->convert_to(flush_type));
+ non_null_value_size += chunk.builder->rowids().size();
+ }
}
- _v2_builder.reset();
}
TabletColumn flush_column = variant_util::get_column_by_type(
@@ -2186,12 +2301,35 @@ Status VariantSubcolumnWriter::finalize() {
_indexes, &opts, non_null_value_size,
need_record_none_null_value_size));
_opts = opts;
- RETURN_IF_ERROR(write_variant_subcolumn_values(_input_format,
flush_column, _writer.get(),
- flush_type, flush_values,
flush_rowids,
- _num_rows));
+ // One convertor for the whole column. The ARRAY convertor carries its
accumulated item offset
+ // across calls, so flushing chunk by chunk keeps the absolute array
offsets correct; a fresh
+ // convertor per chunk would restart them at zero and corrupt the column.
+ OlapBlockDataConvertor converter;
+ if (_input_format == VariantWriterInputFormat::V1) {
+ RETURN_IF_ERROR(write_variant_subcolumn_values(_input_format,
flush_column, _writer.get(),
+ flush_type,
flush_values, {}, _num_rows));
+ } else if (!publish_builder) {
+ MutableColumnPtr defaults = flush_type->create_column();
+ RETURN_IF_ERROR(variant_writer_helpers::append_sparse_converted_column(
+ flush_column, _writer.get(), &converter, 0, flush_type,
std::move(defaults), {},
+ _num_rows));
+ } else {
+ for (auto& chunk : _chunks) {
+ if (chunk.builder->non_null_rows() == 0) {
+ MutableColumnPtr empty = flush_type->create_column();
+
RETURN_IF_ERROR(variant_writer_helpers::append_sparse_converted_column(
+ flush_column, _writer.get(), &converter, 0,
flush_type, std::move(empty),
+ {}, chunk.rows));
+ continue;
+ }
+
RETURN_IF_ERROR(variant_writer_helpers::append_sparse_converted_column(
+ flush_column, _writer.get(), &converter, 0, flush_type,
chunk.builder->column(),
+ chunk.builder->rowids(), chunk.rows));
+ }
+ }
_opts.meta->set_num_rows(_num_rows);
- none_null_size = cast_set<size_t>(non_null_value_size);
-
+ none_null_size = non_null_value_size;
+ _chunks.clear();
_is_finalized = true;
return Status::OK();
}
diff --git a/be/src/storage/segment/variant/variant_writer_helpers.cpp
b/be/src/storage/segment/variant/variant_writer_helpers.cpp
index d8e53e405b2..bbaaa06b377 100644
--- a/be/src/storage/segment/variant/variant_writer_helpers.cpp
+++ b/be/src/storage/segment/variant/variant_writer_helpers.cpp
@@ -18,6 +18,7 @@
#include "storage/segment/variant/variant_writer_helpers.h"
#include <algorithm>
+#include <atomic>
#include <memory>
#include <span>
#include <string>
@@ -320,6 +321,20 @@ Status append_sparse_scalar_column(const TabletColumn&
tablet_column, ColumnWrit
} // namespace
+#ifdef BE_TEST
+namespace {
+std::atomic<uint64_t> g_array_all_null_chunks_for_test = 0;
+} // namespace
+
+uint64_t array_all_null_chunks_for_test() {
+ return g_array_all_null_chunks_for_test.load(std::memory_order_relaxed);
+}
+
+void reset_array_all_null_chunks_for_test() {
+ g_array_all_null_chunks_for_test.store(0, std::memory_order_relaxed);
+}
+#endif
+
Status append_sparse_converted_column(const TabletColumn& tablet_column,
ColumnWriter* writer,
OlapBlockDataConvertor* converter, int
column_id,
const DataTypePtr& type, const
ColumnPtr& values_column,
@@ -336,16 +351,23 @@ Status append_sparse_converted_column(const TabletColumn&
tablet_column, ColumnW
if (total_rows == 0) {
return Status::OK();
}
- if (rowids.empty()) {
- DORIS_CHECK(tablet_column.is_nullable());
- return writer->append_nulls(total_rows);
- }
-
const DataTypePtr base_type = remove_nullable(type);
if (base_type->get_primitive_type() == PrimitiveType::TYPE_ARRAY) {
+#ifdef BE_TEST
+ if (rowids.empty()) {
+ g_array_all_null_chunks_for_test.fetch_add(1,
std::memory_order_relaxed);
+ }
+#endif
+ // ARRAY writers materialize a dense N-row column and update offsets,
nulls and the
+ // inverted index together. An all-NULL chunk (empty rowids) must take
this path too:
+ // append_nulls only writes column data and leaves the inverted index
inconsistent.
return append_sparse_array_column(tablet_column, writer, converter,
column_id, type,
values_column, rowids, total_rows);
}
+ if (rowids.empty()) {
+ DORIS_CHECK(tablet_column.is_nullable());
+ return writer->append_nulls(total_rows);
+ }
return append_sparse_scalar_column(tablet_column, writer, converter,
column_id, type,
values_column, rowids, total_rows);
}
diff --git a/be/src/storage/segment/variant/variant_writer_helpers.h
b/be/src/storage/segment/variant/variant_writer_helpers.h
index 5caea664990..98722b9f808 100644
--- a/be/src/storage/segment/variant/variant_writer_helpers.h
+++ b/be/src/storage/segment/variant/variant_writer_helpers.h
@@ -76,6 +76,13 @@ Status convert_and_write_column(OlapBlockDataConvertor*
converter, const TabletC
// Converts only present values. Missing logical rows are appended as NULL;
ARRAY uses a one-path
// full-column fallback because its converted representation is not
fixed-width/strided.
+#ifdef BE_TEST
+// Number of all-NULL ARRAY chunks written through the array path. The
dispatch must never send
+// them to the scalar append_nulls fast path, which does not maintain the
inverted index.
+uint64_t array_all_null_chunks_for_test();
+void reset_array_all_null_chunks_for_test();
+#endif
+
Status append_sparse_converted_column(const TabletColumn& tablet_column,
ColumnWriter* writer,
OlapBlockDataConvertor* converter, int
column_id,
const DataTypePtr& type, const
ColumnPtr& values_column,
diff --git a/be/test/storage/variant/variant_column_writer_reader_test.cpp
b/be/test/storage/variant/variant_column_writer_reader_test.cpp
index 8c5d89d0b1e..aaf3794ddfa 100644
--- a/be/test/storage/variant/variant_column_writer_reader_test.cpp
+++ b/be/test/storage/variant/variant_column_writer_reader_test.cpp
@@ -47,6 +47,7 @@
#include "core/value/uuid_value.h"
#include "core/value/variant/variant_batch_builder.h"
#include "core/value/variant/variant_parquet_encoding.h"
+#include "exec/common/variant_util.h"
#include "exec/rowid_fetcher.h"
#include "exprs/function/search/function_search.h"
#include "exprs/function/search/variant_inverted_index_search.h"
@@ -79,6 +80,7 @@
#include "storage/segment/variant/variant_column_reader.h"
#include "storage/segment/variant/variant_column_writer_impl.h"
#include "storage/segment/variant/variant_doc_snpashot_compact_iterator.h"
+#include "storage/segment/variant/variant_writer_helpers.h"
#include "storage/storage_engine.h"
#include "storage/tablet/tablet_manager.h"
#include "testutil/variant_util.h"
@@ -1463,6 +1465,25 @@ static Status create_variant_root_reader(const
SegmentFooterPB& footer,
class VariantColumnWriterReaderTest : public testing::Test {
public:
+ // Parent VARIANT column with an extracted VARIANT subcolumn `v.payload`,
i.e. the layout
+ // VariantSubcolumnWriter is responsible for.
+ void init_extracted_variant_schema(int64_t tablet_id) {
+ TabletSchemaPB schema_pb;
+ schema_pb.set_keys_type(KeysType::DUP_KEYS);
+ construct_column(schema_pb.add_column(), 1, "VARIANT", "v", 1);
+ _tablet_schema = std::make_shared<TabletSchema>();
+ _tablet_schema->init_from_pb(schema_pb);
+ const TabletColumn& parent_column = _tablet_schema->column_by_uid(1);
+ TabletColumn extracted_column;
+ extracted_column.set_name(parent_column.name_lower_case() +
".payload");
+ extracted_column.set_type(FieldType::OLAP_FIELD_TYPE_VARIANT);
+ extracted_column.set_parent_unique_id(parent_column.unique_id());
+
extracted_column.set_path_info(PathInData(parent_column.name_lower_case() +
".payload"));
+ extracted_column.set_is_nullable(true);
+ _tablet_schema->append_column(extracted_column);
+ init_tablet_from_current_schema(tablet_id);
+ }
+
void SetUp() override {
// absolute dir
char buffer[MAX_PATH_LEN];
@@ -1826,9 +1847,10 @@ protected:
return Status::OK();
}
- Status write_extracted_variant_segment(const ColumnPtr& source, const
DataTypePtr& source_type,
- std::string_view rowset_id,
SegmentFooterPB* footer,
- std::string* file_path, size_t
first_batch_rows = 0) {
+ Status write_extracted_variant_segment(
+ const ColumnPtr& source, const DataTypePtr& source_type,
std::string_view rowset_id,
+ SegmentFooterPB* footer, std::string* file_path, size_t
first_batch_rows = 0,
+ size_t batch_rows = 0, const std::function<void(ColumnWriter*)>&
after_batch = {}) {
DORIS_CHECK(source.get() != nullptr);
DORIS_CHECK(source_type != nullptr);
DORIS_CHECK(footer != nullptr);
@@ -1873,11 +1895,28 @@ protected:
converter.clear_source_content(0);
return Status::OK();
};
- if (first_batch_rows > 0 && first_batch_rows < source->size()) {
+ if (batch_rows > 0) {
+ for (size_t row_pos = 0; row_pos < source->size(); row_pos +=
batch_rows) {
+ const size_t rows = std::min(batch_rows, source->size() -
row_pos);
+ RETURN_IF_ERROR(append_batch(row_pos, rows));
+ if (after_batch) {
+ after_batch(writer.get());
+ }
+ }
+ } else if (first_batch_rows > 0 && first_batch_rows < source->size()) {
RETURN_IF_ERROR(append_batch(0, first_batch_rows));
+ if (after_batch) {
+ after_batch(writer.get());
+ }
RETURN_IF_ERROR(append_batch(first_batch_rows, source->size() -
first_batch_rows));
+ if (after_batch) {
+ after_batch(writer.get());
+ }
} else {
RETURN_IF_ERROR(append_batch(0, source->size()));
+ if (after_batch) {
+ after_batch(writer.get());
+ }
}
RETURN_IF_ERROR(writer->finish());
@@ -1940,9 +1979,15 @@ protected:
RETURN_IF_ERROR(ColumnReader::create(reader_options,
footer.columns(0), footer.num_rows(),
file_reader, &reader));
- TabletColumn read_column = _tablet_schema->column(1);
- read_column.set_type(static_cast<FieldType>(footer.columns(0).type()));
- auto read_type =
DataTypeFactory::instance().create_data_type(read_column, false);
+ // Build the read type and the matching tablet column from the footer
meta so nested types
+ // (ARRAY/MAP/STRUCT) keep their child type information; the schema
column only carries the
+ // top-level type and the array iterator needs the element sub-column.
+ auto read_type =
DataTypeFactory::instance().create_data_type(footer.columns(0));
+ TabletColumn read_column = variant_util::get_column_by_type(
+ read_type, _tablet_schema->column(1).name(),
+ variant_util::ExtraInfo {.unique_id = -1,
+ .parent_unique_id = 1,
+ .path_info =
*_tablet_schema->column(1).path_info_ptr()});
ColumnIteratorUPtr iterator;
RETURN_IF_ERROR(reader->new_iterator(&iterator, &read_column));
OlapReaderStatistics stats;
@@ -2501,6 +2546,104 @@ TEST_F(VariantColumnWriterReaderTest,
test_write_column_variant_v2_accounts_root
EXPECT_EQ(actual[1], "1");
}
+TEST_F(VariantColumnWriterReaderTest,
test_variant_root_writer_streams_per_batch) {
+ // A variant with an extracted subcolumn takes the root-only writer path.
Root scalars all
+ // land in the root JSONB column, so the writer must flush them batch by
batch instead of
+ // retaining the whole segment: buffering SegmentRows * value bytes is
what used to overflow
+ // ColumnStr<uint32_t> at 4 GiB.
+ TabletSchemaPB schema_pb;
+ schema_pb.set_keys_type(KeysType::DUP_KEYS);
+ construct_column(schema_pb.add_column(), 1, "VARIANT", "v", 10, false,
false, 0, false, 0, 0,
+ false);
+ _tablet_schema = std::make_shared<TabletSchema>();
+ _tablet_schema->init_from_pb(schema_pb);
+ const TabletColumn& parent_column = _tablet_schema->column_by_uid(1);
+ TabletColumn extracted_column;
+ extracted_column.set_name(parent_column.name_lower_case() + ".payload");
+ extracted_column.set_type(FieldType::OLAP_FIELD_TYPE_VARIANT);
+ extracted_column.set_parent_unique_id(parent_column.unique_id());
+ extracted_column.set_path_info(PathInData(parent_column.name_lower_case()
+ ".payload"));
+ extracted_column.set_is_nullable(true);
+ _tablet_schema->append_column(extracted_column);
+ init_tablet_from_current_schema(10043);
+
+ constexpr size_t kRows = 4000;
+ constexpr size_t kBatchRows = 1000;
+ constexpr size_t kValueBytes = 4096;
+ const std::string payload(kValueBytes, 'x');
+ auto source = ColumnVariantV2::create();
+ DataTypeVariantV2SerDe serde;
+ DataTypeSerDe::FormatOptions serde_options;
+ for (size_t row = 0; row < kRows; ++row) {
+ const std::string json = "\"" + payload + "\"";
+ Slice slice(json.data(), json.size());
+ ASSERT_TRUE(serde.deserialize_one_cell_from_json(*source, slice,
serde_options).ok());
+ }
+
+ const std::string rowset_id = "v2_root_streaming";
+ SegmentFooterPB footer;
+ std::string file_path = local_segment_path(_tablet->tablet_path(),
rowset_id, 0);
+ ASSERT_TRUE(io::global_local_filesystem()->delete_file(file_path).ok());
+ io::FileWriterPtr file_writer;
+ ASSERT_TRUE(io::global_local_filesystem()->create_file(file_path,
&file_writer).ok());
+
+ RowsetWriterContext rowset_ctx;
+ rowset_ctx.write_type = DataWriteType::TYPE_DIRECT;
+ rowset_ctx.tablet_schema = _tablet_schema;
+ rowset_ctx.tablet = _tablet;
+ rowset_ctx.tablet_path = _tablet->tablet_path();
+
+ TabletColumn column = _tablet_schema->column(0);
+ ColumnWriterOptions opts;
+ opts.meta = footer.add_columns();
+ opts.compression_type = CompressionTypePB::LZ4;
+ opts.file_writer = file_writer.get();
+ opts.footer = &footer;
+ opts.rowset_ctx = &rowset_ctx;
+ opts.storage_format = TabletStorageFormatPB::TABLET_STORAGE_FORMAT_V2;
+ _init_column_meta(opts.meta, 0, column, opts);
+
+ std::unique_ptr<ColumnWriter> writer;
+ ASSERT_TRUE(ColumnWriter::create(opts, &column, file_writer.get(),
&writer).ok());
+ ASSERT_TRUE(writer->init().ok());
+
+ OlapBlockDataConvertor converter;
+ converter.add_column_data_convertor(column);
+ const auto source_type = std::make_shared<DataTypeVariantV2>(2, false);
+ uint64_t max_buffered_bytes = 0;
+ for (size_t row_pos = 0; row_pos < kRows; row_pos += kBatchRows) {
+ ASSERT_TRUE(converter
+ .set_source_content_with_specifid_column(
+ {source->get_ptr(), source_type,
column.name()}, row_pos,
+ kBatchRows, 0)
+ .ok());
+ auto [convert_status, accessor] = converter.convert_column_data(0);
+ ASSERT_TRUE(convert_status.ok());
+ ASSERT_TRUE(writer->append(accessor->get_nullmap(),
accessor->get_data(), kBatchRows).ok());
+ converter.clear_source_content(0);
+ max_buffered_bytes = std::max<uint64_t>(max_buffered_bytes,
writer->estimate_buffer_size());
+ }
+ // The buffering writer retained the whole segment (kRows * kValueBytes ~
16 MiB). The
+ // streaming writer flushes the root column batch by batch, so its own
retained raw buffer is
+ // about one batch; encoded pages and the other variant column writers are
separate.
+ EXPECT_LT(max_buffered_bytes, 2 * kBatchRows * kValueBytes);
+
+ ASSERT_TRUE(writer->finish().ok());
+ ASSERT_TRUE(writer->write_data().ok());
+ ASSERT_TRUE(writer->write_ordinal_index().ok());
+ ASSERT_TRUE(writer->write_zone_map().ok());
+ ASSERT_TRUE(file_writer->close().ok());
+ footer.set_num_rows(kRows);
+
+ std::vector<std::optional<std::string>> actual;
+ ASSERT_TRUE(read_variant_root_rows(footer, file_path, &actual).ok());
+ ASSERT_EQ(actual.size(), kRows);
+ for (const auto& row : actual) {
+ ASSERT_TRUE(row.has_value());
+ EXPECT_EQ(row->size(), kValueBytes + 2);
+ }
+}
+
TEST_F(VariantColumnWriterReaderTest,
test_write_column_variant_v2_typed_state) {
init_variant_tablet(10009, 1);
@@ -8944,4 +9087,475 @@ TEST_F(VariantColumnWriterReaderTest,
test_compaction_nokey_variant_uid0) {
ASSERT_EQ(output_rowset->num_rows(), 4);
}
+TEST_F(VariantColumnWriterReaderTest,
test_variant_subcolumn_writer_chunks_large_values) {
+ // A large VARIANT subcolumn must be flushed in bounded chunks: a single
segment-sized
+ // ColumnString is what overflows ColumnStr<uint32_t> at 4 GiB. Compare
the chunked output with
+ // the single-chunk reference and check the observed chunk budget.
+ init_extracted_variant_schema(11401);
+ constexpr size_t kRows = 3000;
+ constexpr size_t kBatchRows = 300;
+ const std::string value(1024, 'x');
+ const std::string json = "\"" + value + "\"";
+ std::vector<std::string> jsons(kRows, json);
+ ColumnPtr source;
+ DataTypePtr source_type;
+ ASSERT_TRUE(create_variant_writer_source(VariantWriterInput::V2, jsons, 0,
false, {}, &source,
+ &source_type)
+ .ok());
+
+ SegmentFooterPB ref_footer;
+ std::string ref_path;
+ ASSERT_TRUE(write_extracted_variant_segment(source, source_type,
"subcolumn_chunk_ref",
+ &ref_footer, &ref_path)
+ .ok());
+ std::vector<std::string> ref_rows;
+ ASSERT_TRUE(read_extracted_variant_rows(ref_footer, ref_path,
&ref_rows).ok());
+ ASSERT_EQ(ref_rows.size(), kRows);
+
+ struct ChunkGuard {
+ ChunkGuard() { VariantSubcolumnWriter::set_chunk_values_for_test(250);
}
+ ~ChunkGuard() { VariantSubcolumnWriter::set_chunk_values_for_test(0); }
+ } guard;
+ size_t max_chunks = 0;
+ size_t max_chunk_bytes = 0;
+ SegmentFooterPB chunked_footer;
+ std::string chunked_path;
+ ASSERT_TRUE(write_extracted_variant_segment(
+ source, source_type, "subcolumn_chunk_values",
&chunked_footer,
+ &chunked_path, 0, kBatchRows,
+ [&](ColumnWriter* writer) {
+ auto* subcolumn =
dynamic_cast<VariantSubcolumnWriter*>(writer);
+ ASSERT_NE(subcolumn, nullptr);
+ max_chunks = std::max(max_chunks,
subcolumn->num_chunks_for_test());
+ max_chunk_bytes = std::max(max_chunk_bytes,
+
subcolumn->max_chunk_bytes_for_test());
+ })
+ .ok());
+ EXPECT_GE(max_chunks, 2);
+ // A chunk holds at most the configured budget plus the batch that crossed
it.
+ EXPECT_LE(max_chunk_bytes, 250 * 1024 + kBatchRows * 1024);
+
+ std::vector<std::string> chunked_rows;
+ ASSERT_TRUE(read_extracted_variant_rows(chunked_footer, chunked_path,
&chunked_rows).ok());
+ EXPECT_EQ(chunked_rows, ref_rows);
+ EXPECT_EQ(chunked_rows.front(), ref_rows.front());
+ EXPECT_EQ(chunked_rows.back(), ref_rows.back());
+}
+
+TEST_F(VariantColumnWriterReaderTest,
test_variant_subcolumn_writer_chunks_type_promotion) {
+ // Values that widen the inferred type across a chunk boundary (int ->
string -> JSONB) must
+ // still produce the same column as the single-chunk path.
+ init_extracted_variant_schema(11402);
+ constexpr size_t kHalf = 2000;
+ std::vector<std::string> jsons;
+ jsons.reserve(2 * kHalf);
+ for (size_t i = 0; i < kHalf; ++i) {
+ jsons.push_back(std::to_string(i));
+ }
+ for (size_t i = 0; i < kHalf; ++i) {
+ jsons.push_back("\"s" + std::to_string(i) + "\"");
+ }
+ ColumnPtr source;
+ DataTypePtr source_type;
+ ASSERT_TRUE(create_variant_writer_source(VariantWriterInput::V2, jsons, 0,
false, {}, &source,
+ &source_type)
+ .ok());
+
+ SegmentFooterPB ref_footer;
+ std::string ref_path;
+ ASSERT_TRUE(write_extracted_variant_segment(source, source_type,
"subcolumn_promote_ref",
+ &ref_footer, &ref_path)
+ .ok());
+ std::vector<std::string> ref_rows;
+ ASSERT_TRUE(read_extracted_variant_rows(ref_footer, ref_path,
&ref_rows).ok());
+
+ struct ChunkGuard {
+ ChunkGuard() { VariantSubcolumnWriter::set_chunk_values_for_test(500);
}
+ ~ChunkGuard() { VariantSubcolumnWriter::set_chunk_values_for_test(0); }
+ } guard;
+ size_t max_chunks = 0;
+ SegmentFooterPB chunked_footer;
+ std::string chunked_path;
+ ASSERT_TRUE(write_extracted_variant_segment(
+ source, source_type, "subcolumn_promote_chunked",
&chunked_footer,
+ &chunked_path, 0, 300,
+ [&](ColumnWriter* writer) {
+ auto* subcolumn =
dynamic_cast<VariantSubcolumnWriter*>(writer);
+ ASSERT_NE(subcolumn, nullptr);
+ max_chunks = std::max(max_chunks,
subcolumn->num_chunks_for_test());
+ })
+ .ok());
+ EXPECT_GE(max_chunks, 2);
+
+ std::vector<std::string> chunked_rows;
+ ASSERT_TRUE(read_extracted_variant_rows(chunked_footer, chunked_path,
&chunked_rows).ok());
+ EXPECT_EQ(chunked_rows, ref_rows);
+}
+
+TEST_F(VariantColumnWriterReaderTest,
test_variant_subcolumn_writer_chunks_sparse_rows) {
+ // Rows without the subcolumn become nulls. When chunks split a gap the
dense materialization
+ // must still place every value on its logical row.
+ init_extracted_variant_schema(11403);
+ constexpr size_t kRows = 3000;
+ std::vector<std::string> jsons(kRows, "\"v\"");
+ std::vector<UInt8> outer_nulls(kRows, 0);
+ for (size_t row = 0; row < kRows; row += 3) {
+ outer_nulls[row] = 1;
+ }
+ ColumnPtr source;
+ DataTypePtr source_type;
+ ASSERT_TRUE(create_variant_writer_source(VariantWriterInput::V2, jsons, 0,
false, outer_nulls,
+ &source, &source_type)
+ .ok());
+
+ SegmentFooterPB ref_footer;
+ std::string ref_path;
+ ASSERT_TRUE(write_extracted_variant_segment(source, source_type,
"subcolumn_sparse_ref",
+ &ref_footer, &ref_path)
+ .ok());
+ std::vector<std::string> ref_rows;
+ ASSERT_TRUE(read_extracted_variant_rows(ref_footer, ref_path,
&ref_rows).ok());
+
+ struct ChunkGuard {
+ ChunkGuard() { VariantSubcolumnWriter::set_chunk_values_for_test(200);
}
+ ~ChunkGuard() { VariantSubcolumnWriter::set_chunk_values_for_test(0); }
+ } guard;
+ size_t max_chunks = 0;
+ SegmentFooterPB chunked_footer;
+ std::string chunked_path;
+ ASSERT_TRUE(write_extracted_variant_segment(
+ source, source_type, "subcolumn_sparse_chunked",
&chunked_footer,
+ &chunked_path, 0, 300,
+ [&](ColumnWriter* writer) {
+ auto* subcolumn =
dynamic_cast<VariantSubcolumnWriter*>(writer);
+ ASSERT_NE(subcolumn, nullptr);
+ max_chunks = std::max(max_chunks,
subcolumn->num_chunks_for_test());
+ })
+ .ok());
+ EXPECT_GE(max_chunks, 2);
+
+ std::vector<std::string> chunked_rows;
+ ASSERT_TRUE(read_extracted_variant_rows(chunked_footer, chunked_path,
&chunked_rows).ok());
+ EXPECT_EQ(chunked_rows, ref_rows);
+ ASSERT_FALSE(chunked_rows.empty());
+ EXPECT_EQ(chunked_rows[0], "NULL");
+ EXPECT_EQ(chunked_rows[1], ref_rows[1]);
+}
+
+TEST_F(VariantColumnWriterReaderTest,
test_variant_subcolumn_writer_chunks_all_null) {
+ // A subcolumn whose every row is null must still append the right number
of nulls per chunk.
+ init_extracted_variant_schema(11404);
+ constexpr size_t kRows = 2000;
+ std::vector<std::string> jsons(kRows, "\"v\"");
+ std::vector<UInt8> outer_nulls(kRows, 1);
+ ColumnPtr source;
+ DataTypePtr source_type;
+ ASSERT_TRUE(create_variant_writer_source(VariantWriterInput::V2, jsons, 0,
false, outer_nulls,
+ &source, &source_type)
+ .ok());
+
+ SegmentFooterPB ref_footer;
+ std::string ref_path;
+ ASSERT_TRUE(write_extracted_variant_segment(source, source_type,
"subcolumn_null_ref",
+ &ref_footer, &ref_path)
+ .ok());
+ std::vector<std::string> ref_rows;
+ ASSERT_TRUE(read_extracted_variant_rows(ref_footer, ref_path,
&ref_rows).ok());
+
+ struct ChunkGuard {
+ ChunkGuard() { VariantSubcolumnWriter::set_chunk_values_for_test(100);
}
+ ~ChunkGuard() { VariantSubcolumnWriter::set_chunk_values_for_test(0); }
+ } guard;
+ SegmentFooterPB chunked_footer;
+ std::string chunked_path;
+ ASSERT_TRUE(write_extracted_variant_segment(source, source_type,
"subcolumn_null_chunked",
+ &chunked_footer,
&chunked_path, 0, 300)
+ .ok());
+ std::vector<std::string> chunked_rows;
+ ASSERT_TRUE(read_extracted_variant_rows(chunked_footer, chunked_path,
&chunked_rows).ok());
+ EXPECT_EQ(chunked_rows, ref_rows);
+ for (const auto& row : chunked_rows) {
+ EXPECT_EQ(row, "NULL");
+ }
+}
+
+TEST_F(VariantColumnWriterReaderTest,
test_variant_subcolumn_writer_chunks_by_bytes) {
+ // The byte budget (kChunkBytes) seals a chunk even when the value count
is small, e.g. few very
+ // wide strings. The chunk must still be flushed independently and read
back unchanged.
+ init_extracted_variant_schema(11405);
+ constexpr size_t kRows = 3000;
+ constexpr size_t kBatchRows = 200;
+ const std::string value(1024, 'y');
+ const std::string json = "\"" + value + "\"";
+ std::vector<std::string> jsons(kRows, json);
+ ColumnPtr source;
+ DataTypePtr source_type;
+ ASSERT_TRUE(create_variant_writer_source(VariantWriterInput::V2, jsons, 0,
false, {}, &source,
+ &source_type)
+ .ok());
+
+ SegmentFooterPB ref_footer;
+ std::string ref_path;
+ ASSERT_TRUE(write_extracted_variant_segment(source, source_type,
"subcolumn_bytes_ref",
+ &ref_footer, &ref_path)
+ .ok());
+ std::vector<std::string> ref_rows;
+ ASSERT_TRUE(read_extracted_variant_rows(ref_footer, ref_path,
&ref_rows).ok());
+
+ struct ChunkGuard {
+ ChunkGuard() { VariantSubcolumnWriter::set_chunk_bytes_for_test(256 *
1024); }
+ ~ChunkGuard() { VariantSubcolumnWriter::set_chunk_bytes_for_test(0); }
+ } guard;
+ size_t max_chunks = 0;
+ size_t max_chunk_bytes = 0;
+ SegmentFooterPB chunked_footer;
+ std::string chunked_path;
+ ASSERT_TRUE(write_extracted_variant_segment(
+ source, source_type, "subcolumn_bytes_chunked",
&chunked_footer,
+ &chunked_path, 0, kBatchRows,
+ [&](ColumnWriter* writer) {
+ auto* subcolumn =
dynamic_cast<VariantSubcolumnWriter*>(writer);
+ ASSERT_NE(subcolumn, nullptr);
+ max_chunks = std::max(max_chunks,
subcolumn->num_chunks_for_test());
+ max_chunk_bytes = std::max(max_chunk_bytes,
+
subcolumn->max_chunk_bytes_for_test());
+ })
+ .ok());
+ EXPECT_GE(max_chunks, 2);
+ EXPECT_LE(max_chunk_bytes, 256 * 1024 + kBatchRows * 1024);
+
+ std::vector<std::string> chunked_rows;
+ ASSERT_TRUE(read_extracted_variant_rows(chunked_footer, chunked_path,
&chunked_rows).ok());
+ EXPECT_EQ(chunked_rows, ref_rows);
+}
+
+TEST_F(VariantColumnWriterReaderTest,
test_variant_subcolumn_writer_chunks_int_then_bool) {
+ // The single builder falls back to JSONB when a bool follows integers:
appending `true` into
+ // an Int64 column is not a representable promotion. Chunks must carry the
inferred type across
+ // their boundary so the same value sequence keeps the JSONB fallback
instead of coercing
+ // true/false to 1/0 through the least-common-type merge.
+ init_extracted_variant_schema(11406);
+ const std::vector<std::string> jsons {"7", "true", "false", "8"};
+ ColumnPtr source;
+ DataTypePtr source_type;
+ ASSERT_TRUE(create_variant_writer_source(VariantWriterInput::V2, jsons, 0,
false, {}, &source,
+ &source_type)
+ .ok());
+
+ SegmentFooterPB ref_footer;
+ std::string ref_path;
+ ASSERT_TRUE(write_extracted_variant_segment(source, source_type,
"subcolumn_bool_ref",
+ &ref_footer, &ref_path)
+ .ok());
+ std::vector<std::string> ref_rows;
+ ASSERT_TRUE(read_extracted_variant_rows(ref_footer, ref_path,
&ref_rows).ok());
+
+ struct ChunkGuard {
+ ChunkGuard() { VariantSubcolumnWriter::set_chunk_values_for_test(1); }
+ ~ChunkGuard() { VariantSubcolumnWriter::set_chunk_values_for_test(0); }
+ } guard;
+ size_t max_chunks = 0;
+ SegmentFooterPB chunked_footer;
+ std::string chunked_path;
+ ASSERT_TRUE(write_extracted_variant_segment(
+ source, source_type, "subcolumn_bool_chunked",
&chunked_footer,
+ &chunked_path, 0, 1,
+ [&](ColumnWriter* writer) {
+ auto* subcolumn =
dynamic_cast<VariantSubcolumnWriter*>(writer);
+ ASSERT_NE(subcolumn, nullptr);
+ max_chunks = std::max(max_chunks,
subcolumn->num_chunks_for_test());
+ })
+ .ok());
+ EXPECT_GE(max_chunks, 2);
+
+ std::vector<std::string> chunked_rows;
+ ASSERT_TRUE(read_extracted_variant_rows(chunked_footer, chunked_path,
&chunked_rows).ok());
+ EXPECT_EQ(chunked_rows, ref_rows);
+ EXPECT_EQ(chunked_rows, (std::vector<std::string> {"7", "true", "false",
"8"}));
+ EXPECT_EQ(static_cast<FieldType>(ref_footer.columns(0).type()),
+ FieldType::OLAP_FIELD_TYPE_JSONB);
+ EXPECT_EQ(static_cast<FieldType>(chunked_footer.columns(0).type()),
+ FieldType::OLAP_FIELD_TYPE_JSONB);
+}
+
+TEST_F(VariantColumnWriterReaderTest,
test_variant_subcolumn_writer_chunks_array_all_null_tail) {
+ // An all-NULL tail chunk of an ARRAY subcolumn must be flushed through
the array writer (which
+ // updates offsets, nulls and the inverted index together), not through
append_nulls.
+ init_extracted_variant_schema(11407);
+ const std::vector<std::string> jsons {"[1,2]", "[\"a\",\"b\"]", "null",
"null"};
+ ColumnPtr source;
+ DataTypePtr source_type;
+ ASSERT_TRUE(create_variant_writer_source(VariantWriterInput::V2, jsons, 0,
false, {}, &source,
+ &source_type)
+ .ok());
+
+ SegmentFooterPB ref_footer;
+ std::string ref_path;
+ ASSERT_TRUE(write_extracted_variant_segment(source, source_type,
"subcolumn_array_ref",
+ &ref_footer, &ref_path)
+ .ok());
+ std::vector<std::string> ref_rows;
+ ASSERT_TRUE(read_extracted_variant_rows(ref_footer, ref_path,
&ref_rows).ok());
+
+ struct ChunkGuard {
+ ChunkGuard() { VariantSubcolumnWriter::set_chunk_values_for_test(2); }
+ ~ChunkGuard() { VariantSubcolumnWriter::set_chunk_values_for_test(0); }
+ } guard;
+ segment_v2::variant_writer_helpers::reset_array_all_null_chunks_for_test();
+ size_t max_chunks = 0;
+ SegmentFooterPB chunked_footer;
+ std::string chunked_path;
+ ASSERT_TRUE(write_extracted_variant_segment(
+ source, source_type, "subcolumn_array_chunked",
&chunked_footer,
+ &chunked_path, 0, 1,
+ [&](ColumnWriter* writer) {
+ auto* subcolumn =
dynamic_cast<VariantSubcolumnWriter*>(writer);
+ ASSERT_NE(subcolumn, nullptr);
+ max_chunks = std::max(max_chunks,
subcolumn->num_chunks_for_test());
+ })
+ .ok());
+ EXPECT_GE(max_chunks, 2);
+
EXPECT_EQ(segment_v2::variant_writer_helpers::array_all_null_chunks_for_test(),
1u);
+
+ std::vector<std::string> chunked_rows;
+ ASSERT_TRUE(read_extracted_variant_rows(chunked_footer, chunked_path,
&chunked_rows).ok());
+ EXPECT_EQ(chunked_rows, ref_rows);
+ ASSERT_EQ(chunked_rows.size(), 4);
+ EXPECT_EQ(chunked_rows[2], "NULL");
+ EXPECT_EQ(chunked_rows[3], "NULL");
+}
+
+TEST(VariantPathBuilderTest, MergeNullableDecimalArraysPreservesScale) {
+ VariantBatchBuilder value_builder;
+ for (size_t index = 0; index < 2; ++index) {
+ auto row = value_builder.begin_row();
+ auto array = row.start_array();
+ row.add_decimal(index == 0 ? 123 : 12345, index == 0 ? 2 : 4, 16);
+ array.finish();
+ row.finish();
+ }
+ const auto values = value_builder.finish_batch();
+ segment_v2::VariantPathBuilder first(PathInData("metric"));
+ segment_v2::VariantPathBuilder second(PathInData("metric"));
+ ASSERT_TRUE(first.append(values.value_at(0), 0).ok());
+ ASSERT_TRUE(second.append(values.value_at(1), 0).ok());
+ for (bool reverse : {false, true}) {
+ const DataTypePtr common = segment_v2::variant_path_least_common_type(
+ reverse ? second.type() : first.type(), reverse ? first.type()
: second.type());
+ EXPECT_TRUE(common->is_nullable());
+ const auto& array = assert_cast<const
DataTypeArray&>(*remove_nullable(common));
+ EXPECT_EQ(array.get_nested_type()->get_scale(), 4);
+ }
+}
+
+TEST_F(VariantColumnWriterReaderTest,
test_variant_subcolumn_writer_chunks_decimal_overflow) {
+ init_extracted_variant_schema(11408);
+ for (bool array_values : {false, true}) {
+ for (bool double_tail : {false, true}) {
+ SCOPED_TRACE(testing::Message()
+ << "array=" << array_values << ", double_tail=" <<
double_tail);
+ VariantBatchBuilder value_builder;
+ for (size_t index = 0; index < (double_tail ? 5 : 4); ++index) {
+ auto row = value_builder.begin_row();
+ std::optional<VariantBatchBuilder::Row::ArrayScope> array;
+ if (array_values) {
+ array.emplace(row.start_array());
+ }
+ if (index == 1) {
+
row.add_decimal(static_cast<__int128>(VARIANT_DECIMAL16_MAX), 1, 16);
+ } else if (index == 3) {
+ row.add_decimal(1, 38, 16);
+ } else if (index == 4) {
+ row.add_double(2.5);
+ } else {
+ row.add_decimal(1, 1, 16);
+ }
+ if (array_values) {
+ array->finish();
+ }
+ row.finish();
+ }
+ auto source = ColumnVariantV2::create();
+ source->insert_encoded_batch(value_builder.finish_batch());
+ const auto source_type = std::make_shared<DataTypeVariantV2>(0,
false);
+ const auto suffix = fmt::format("{}_{}", array_values,
double_tail);
+ SegmentFooterPB ref_footer;
+ std::string ref_path;
+ ASSERT_TRUE(write_extracted_variant_segment(source->get_ptr(),
source_type,
+
"decimal_overflow_ref_" + suffix,
+ &ref_footer, &ref_path)
+ .ok());
+ std::vector<std::string> ref_rows;
+ ASSERT_TRUE(read_extracted_variant_rows(ref_footer, ref_path,
&ref_rows).ok());
+ ASSERT_EQ(static_cast<FieldType>(ref_footer.columns(0).type()),
+ FieldType::OLAP_FIELD_TYPE_JSONB);
+
+ struct ChunkGuard {
+ explicit ChunkGuard(size_t values) {
+ VariantSubcolumnWriter::set_chunk_values_for_test(values);
+ }
+ ~ChunkGuard() {
VariantSubcolumnWriter::set_chunk_values_for_test(0); }
+ } guard(double_tail ? 2 : 1);
+ SegmentFooterPB chunked_footer;
+ std::string chunked_path;
+ ASSERT_TRUE(write_extracted_variant_segment(source->get_ptr(),
source_type,
+
"decimal_overflow_chunks_" + suffix,
+ &chunked_footer,
&chunked_path, 0, 1)
+ .ok());
+ std::vector<std::string> chunked_rows;
+ ASSERT_TRUE(
+ read_extracted_variant_rows(chunked_footer, chunked_path,
&chunked_rows).ok());
+ // The scale-38 value cannot widen the earlier large value
losslessly. With two rows
+ // per chunk, fallback must also preserve the preceding 0.1 in the
active chunk.
+ EXPECT_EQ(chunked_rows, ref_rows);
+ EXPECT_EQ(static_cast<FieldType>(chunked_footer.columns(0).type()),
+ FieldType::OLAP_FIELD_TYPE_JSONB);
+ }
+ }
+}
+
+TEST_F(VariantColumnWriterReaderTest,
test_variant_subcolumn_writer_chunks_decimal_array_scales) {
+ init_extracted_variant_schema(11409);
+ for (bool reverse : {false, true}) {
+ SCOPED_TRACE(testing::Message() << "reverse=" << reverse);
+ VariantBatchBuilder value_builder;
+ for (size_t index = 0; index < 2; ++index) {
+ auto row = value_builder.begin_row();
+ auto array = row.start_array();
+ const bool high_scale = (index == 1) != reverse;
+ row.add_decimal(high_scale ? 12345 : 123, high_scale ? 4 : 2, 16);
+ array.finish();
+ row.finish();
+ }
+ auto source = ColumnVariantV2::create();
+ source->insert_encoded_batch(value_builder.finish_batch());
+ const auto source_type = std::make_shared<DataTypeVariantV2>(0, false);
+ const auto suffix = std::to_string(reverse);
+ SegmentFooterPB ref_footer;
+ std::string ref_path;
+ ASSERT_TRUE(write_extracted_variant_segment(source->get_ptr(),
source_type,
+ "decimal_array_ref_" +
suffix, &ref_footer,
+ &ref_path)
+ .ok());
+ std::vector<std::string> ref_rows;
+ ASSERT_TRUE(read_extracted_variant_rows(ref_footer, ref_path,
&ref_rows).ok());
+ ASSERT_EQ(ref_footer.columns(0).children_columns(0).frac(), 4);
+
+ struct ChunkGuard {
+ ChunkGuard() {
VariantSubcolumnWriter::set_chunk_values_for_test(1); }
+ ~ChunkGuard() {
VariantSubcolumnWriter::set_chunk_values_for_test(0); }
+ } guard;
+ SegmentFooterPB chunked_footer;
+ std::string chunked_path;
+ ASSERT_TRUE(write_extracted_variant_segment(source->get_ptr(),
source_type,
+ "decimal_array_chunks_" +
suffix,
+ &chunked_footer,
&chunked_path, 0, 1)
+ .ok());
+ std::vector<std::string> chunked_rows;
+ ASSERT_TRUE(read_extracted_variant_rows(chunked_footer, chunked_path,
&chunked_rows).ok());
+ EXPECT_EQ(chunked_rows, ref_rows);
+ EXPECT_EQ(chunked_footer.columns(0).children_columns(0).frac(), 4);
+ }
+}
+
} // namespace doris
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
