csun5285 commented on code in PR #55124:
URL: https://github.com/apache/doris/pull/55124#discussion_r2290466924
##########
fe/fe-core/src/main/java/org/apache/doris/common/util/PropertyAnalyzer.java:
##########
@@ -371,6 +374,12 @@ public static DataProperty analyzeDataProperty(Map<String,
String> properties, f
newStoragePolicy = value;
} else if (key.equalsIgnoreCase(PROPERTIES_IS_BEING_SYNCED)) {
isBeingSynced = Boolean.parseBoolean(value);
+ } else if
(key.equalsIgnoreCase(PROPERTIES_VARIANT_MAX_SPARSE_COLUMN_STATISTICS_SIZE)) {
Review Comment:
这里不用改
##########
be/src/olap/rowset/segment_v2/segment.cpp:
##########
@@ -629,7 +629,8 @@ vectorized::DataTypePtr Segment::get_data_type_of(const
TabletColumn& column,
// 2. OR It's a leaf in the physical column structure AND it doesn't
*also* exist
// in the sparse column (meaning it's purely a materialized leaf).
if (read_flat_leaves || (is_physical_leaf && !exist_in_sparse &&
-
!variant_reader->is_exceeded_sparse_column_limit())) {
+ !variant_reader->is_exceeded_sparse_column_limit(
+
column.variant_max_sparse_column_statistics_size()))) {
Review Comment:
variant_max_sparse_column_statistics_size 这个值是存在variant 本身这个列里的，这里应该用父列里面的值
##########
gensrc/proto/olap_file.proto:
##########
@@ -354,6 +354,8 @@ message ColumnPB {
optional bool variant_enable_typed_paths_to_sparse = 27 [default = false];
// this field is only used during flexible partial update load
optional bool is_on_update_current_timestamp = 28 [default = false];
+ // variant_max_sparse_column_statistics_size
+ optional int32 variant_max_sparse_column_statistics_size = 29 [default =
10000];
Review Comment:
default 值和 fe 对上
##########
be/src/olap/rowset/segment_v2/variant/variant_column_reader.cpp:
##########
@@ -276,9 +276,10 @@ Status
VariantColumnReader::new_iterator(ColumnIteratorUPtr* iterator,
// Otherwise the prefix is not exist and the sparse column size is reached
limit
// which means the path maybe exist in sparse_column
- bool exceeded_sparse_column_limit =
!_statistics->sparse_column_non_null_size.empty() &&
-
_statistics->sparse_column_non_null_size.size() >=
-
config::variant_max_sparse_column_statistics_size;
+ bool exceeded_sparse_column_limit =
+ !_statistics->sparse_column_non_null_size.empty() &&
+ _statistics->sparse_column_non_null_size.size() >=
+ target_col->variant_max_sparse_column_statistics_size();
Review Comment:
这里应该用父列里面的值
##########
fe/fe-core/src/main/java/org/apache/doris/nereids/types/VariantType.java:
##########
@@ -38,21 +38,24 @@
*/
public class VariantType extends PrimitiveType {
- public static final VariantType INSTANCE = new VariantType(0);
+ public static final VariantType INSTANCE = new VariantType(0, 0);
public static final int WIDTH = 24;
private final int variantMaxSubcolumnsCount;
private final boolean enableTypedPathsToSparse;
+ private final int variantMaxSparseColumnStatisticsSize;
+
private final List<VariantField> predefinedFields;
// No predefined fields
- public VariantType(int variantMaxSubcolumnsCount) {
+ public VariantType(int variantMaxSubcolumnsCount, int
variantMaxSparseColumnStatisticsSize) {
Review Comment:
这里不用改
##########
be/src/olap/rowset/segment_v2/variant_stats_calculator.cpp:
##########
@@ -63,7 +65,10 @@ Status VariantStatsCaculator::calculate_variant_stats(const
vectorized::Block* b
// Check if this is a sparse column or sub column
if (column_path.ends_with("__DORIS_VARIANT_SPARSE__")) {
// This is a sparse column from variant column
- _calculate_sparse_column_stats(*column, column_meta, row_pos,
num_rows);
+ _calculate_sparse_column_stats(
+ *column, column_meta,
+
tablet_column.variant_max_sparse_column_statistics_size(), row_pos,
Review Comment:
这里应该用父列里面的值
##########
fe/fe-core/src/main/java/org/apache/doris/nereids/types/VariantType.java:
##########
@@ -38,21 +38,24 @@
*/
public class VariantType extends PrimitiveType {
- public static final VariantType INSTANCE = new VariantType(0);
+ public static final VariantType INSTANCE = new VariantType(0, 0);
Review Comment:
这里不用改
##########
fe/fe-core/src/test/java/org/apache/doris/nereids/trees/expressions/functions/ComputeSignatureHelperTest.java:
##########
@@ -532,7 +532,7 @@ void testNoDynamicComputeVariantArgs() {
@Test
void testDynamicComputeVariantArgsSingleVariant() {
- VariantType variantType = new VariantType(100);
+ VariantType variantType = new VariantType(100, 100);
Review Comment:
这里不用改，DynamicComputeVariantArgsSingleVariant 主要是用来计算max subcolumns count
的，查询的时候 max subcolumns count 要传到be，别的参数不用
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
