github-actions[bot] commented on code in PR #63528: URL: https://github.com/apache/doris/pull/63528#discussion_r4257770321
########## fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/ComplexTypeDefaultValue.java: ########## @@ -0,0 +1,238 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.nereids.trees.plans.commands.info; + +import org.apache.doris.nereids.DorisParser.ArrayLiteralContext; +import org.apache.doris.nereids.DorisParser.ConstantContext; +import org.apache.doris.nereids.DorisParser.MapLiteralContext; +import org.apache.doris.nereids.exceptions.AnalysisException; +import org.apache.doris.nereids.exceptions.ParseException; +import org.apache.doris.nereids.parser.LogicalPlanBuilder; +import org.apache.doris.nereids.parser.NereidsParser; +import org.apache.doris.nereids.trees.expressions.Expression; +import org.apache.doris.nereids.trees.expressions.literal.ArrayLiteral; +import org.apache.doris.nereids.trees.expressions.literal.BooleanLiteral; +import org.apache.doris.nereids.trees.expressions.literal.Literal; +import org.apache.doris.nereids.trees.expressions.literal.MapLiteral; +import org.apache.doris.nereids.trees.expressions.literal.NullLiteral; +import org.apache.doris.nereids.trees.expressions.literal.StructLiteral; +import org.apache.doris.nereids.types.ArrayType; +import org.apache.doris.nereids.types.DataType; +import org.apache.doris.nereids.types.MapType; +import org.apache.doris.nereids.types.StructField; +import org.apache.doris.nereids.types.StructType; + +import com.google.common.base.Preconditions; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMap; +import org.antlr.v4.runtime.ParserRuleContext; + +import java.util.HashSet; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.StringJoiner; + +/** + * Validates a non-null ARRAY/MAP/STRUCT column default and rewrites it into a canonical literal text. + * + * <p>The user supplied text is parsed as a SQL literal whose nested values are kept exactly as written + * (see {@link LiteralBuilder}), and every nested value is then cast to the declared nested type, so a + * type mismatch is rejected at DDL time instead of when old rows are read. The + * canonical text only contains plain nested values (unquoted numbers, double quoted strings, nested + * brackets and NULL). BE parses the stored text in two places that must agree: the complex SerDe + * {@code from_fe_string} used by the default value iterator and schema change, and the + * string-to-complex cast used when INSERT fills an unmentioned column. Neither of them decodes + * escape sequences, and the DDL parser keeps the text of a default value verbatim when SHOW CREATE + * TABLE output is replayed, so string values containing quotes or backslashes are rejected instead + * of stored. + */ +public class ComplexTypeDefaultValue { + private ComplexTypeDefaultValue() { + } + + /** + * Validate the default literal of a complex column and return its canonical text. + */ + public static String canonicalize(DataType type, String defaultValue) throws AnalysisException { + Preconditions.checkArgument(type.isArrayType() || type.isMapType() || type.isStructType(), + "%s is not a complex type", type); + Expression expression; + try { + expression = new NereidsParser().parseExpression(defaultValue, new LiteralBuilder()); + } catch (Exception e) { + throw literalShapeException(type); + } + if (!hasLiteralShape(expression, type)) { + throw literalShapeException(type); + } + try { + return render((Literal) expression, type); + } catch (AnalysisException e) { + throw new AnalysisException(String.format("Invalid default value '%s' for %s column: %s", + defaultValue, type.toSql(), e.getMessage()), e); + } + } + + private static boolean hasLiteralShape(Expression expression, DataType type) { + if (type.isArrayType()) { + return expression instanceof ArrayLiteral; + } + if (type.isMapType()) { + return expression instanceof MapLiteral; + } + // `{}` parses as an empty map literal and means every struct field takes its own default. + return expression instanceof StructLiteral + || (expression instanceof MapLiteral && ((MapLiteral) expression).getValue().isEmpty()); + } + + private static AnalysisException literalShapeException(DataType type) { + String literalKind = type.isArrayType() ? "array" : type.isMapType() ? "map" : "struct"; + return new AnalysisException(String.format("%s type column default value only supports %s literals" + + " or DEFAULT NULL", capitalize(literalKind), literalKind)); + } + + private static String capitalize(String value) { + return Character.toUpperCase(value.charAt(0)) + value.substring(1); + } + + private static String render(Literal literal, DataType type) throws AnalysisException { + if (literal instanceof NullLiteral) { + return "NULL"; Review Comment: [P1] Preserve nested NULL in partial-update defaults For a UNIQUE MoW table with `a ARRAY<STRING> DEFAULT '[NULL]'`, setting `enable_unique_key_partial_update=true` and `enable_insert_strict=false` permits `INSERT INTO t(k) VALUES (new_key)` with only the key. FE skips `a`, and BE fills it through `BaseTablet::generate_default_value_block` -> `DataTypeSerDe::default_from_string` -> JSON SerDe. That nullable parser recognizes only lowercase `null`; this uppercase token falls through to the STRING SerDe, which succeeds and stores `"NULL"` with a non-null flag. The row gets `["NULL"]` instead of `[null]`, unlike ordinary INSERT and Segment default reads. This consumer bypasses the now-case-insensitive `ComplexTypeDeserializeUtil` helpers. Make canonical null text agree with `default_from_string` as well, and cover omitted complex-string defaults on new partial-update keys. ########## be/src/core/data_type_serde/data_type_array_serde.cpp: ########## @@ -593,6 +593,42 @@ Status DataTypeArraySerDe::_from_string(StringRef& str, IColumn& column, return Status::OK(); } +Status DataTypeArraySerDe::from_fe_string(const std::string& str, Field& field) const { + StringRef slice(str); + slice = slice.trim_whitespace(); + if (slice.empty()) { + return Status::InvalidArgument("slice is empty!"); + } + if (slice.front() != '[') { + return Status::InvalidArgument("Array does not start with '[' character, found '{}'", + slice.to_string()); + } + if (slice.back() != ']') { + return Status::InvalidArgument("Array does not end with ']' character, found '{}'", + slice.to_string()); + } + + Array array; + if (slice.size > 2) { + slice = slice.substring(1, slice.size - 2); + slice = slice.trim_whitespace(); + if (!slice.empty()) { + FormatOptions options; + auto split_result = ComplexTypeDeserializeUtil::split_by_delimiter( + slice, [&](char c) { return c == options.collection_delim; }); + array.reserve(split_result.size()); Review Comment: [P2] Account for parsed complex-default buffers After light ADD COLUMN with a large nonempty ARRAY default, each old-segment scan reserves one `Field` per element here. ARRAY/MAP/STRUCT use `std::vector<Field>`, which bypasses Doris `Allocator`/MemTracker, and `ConstantColumnReader::new_iterator` copies the parsed value into a scan-owned `ConstantColumnIterator` without charging its recursive buffers. For example, a 100000-element INT default retains `100000 * sizeof(Field)` untracked bytes per active iterator, in addition to the split workspace; concurrent scanners multiply this allocation before tracked output columns are produced. Account/reserve the parsed value and its retained copies (including the analogous MAP/STRUCT and schema-change paths), or use allocator-aware payload storage, so query memory limits include these new buffers. ########## be/src/storage/segment/segment.cpp: ########## @@ -894,29 +894,22 @@ Status Segment::get_default_value_field(const TabletColumn& tablet_column, Field return Status::OK(); } - if (tablet_column.type() == FieldType::OLAP_FIELD_TYPE_ARRAY) { - if (default_value != "[]") { - return Status::NotSupported("Array default {} is unsupported", default_value); - } + // [] and {} need no nested SerDe, so an ARRAY/MAP TabletColumn built without sub-columns can + // still materialize an empty default. + if (tablet_column.type() == FieldType::OLAP_FIELD_TYPE_ARRAY && default_value == "[]") { *field = Field::create_field<TYPE_ARRAY>(Array {}); return Status::OK(); } - if (tablet_column.type() == FieldType::OLAP_FIELD_TYPE_STRUCT) { - return Status::NotSupported("STRUCT default type is unsupported"); - } - if (tablet_column.type() == FieldType::OLAP_FIELD_TYPE_MAP) { - if (default_value != "{}") { - return Status::NotSupported("Map default {} is unsupported", default_value); - } + if (tablet_column.type() == FieldType::OLAP_FIELD_TYPE_MAP && default_value == "{}") { *field = Field::create_field<TYPE_MAP>(Map {Field::create_field<TYPE_ARRAY>(Array {}), Field::create_field<TYPE_ARRAY>(Array {})}); return Status::OK(); } - const auto serde = DataTypeFactory::instance() - .create_data_type(tablet_column.type(), tablet_column.precision(), - tablet_column.frac(), tablet_column.length()) - ->get_serde(); + // Any other non-null default, including a non-empty ARRAY/MAP or any STRUCT literal, needs the + // SerDe of the declared (possibly nested) type to parse recursively; a bare FieldType cannot + // describe nested item types. + const auto serde = remove_nullable(tablet_column.get_vec_type())->get_serde(); Review Comment: [P1] Project complex defaults to the pruned scan schema After inserting an old row and light-adding `s STRUCT<a:INT,b:STRING> DEFAULT '{7, "x"}'`, `SELECT struct_element(s, 'b')` uses this reduced plan (nested pruning is enabled by default): ```text Project(struct_element(s#1, 'b')) OlapScan(s#1: STRUCT<b:STRING>) ``` This code parses a two-field default using the full `TabletColumn`, while `OlapScanner`/`ReadSchema` and `Segment::get_data_type_of` materialize a one-field destination. `ConstantColumnIterator` does not project its Field for access paths, so insertion reaches `ColumnStruct::insert` and throws `Cannot insert value of different size into struct` instead of returning `x`. STRUCT children inside nonempty ARRAY/MAP defaults can fail similarly. Project the default recursively to the requested type/access paths, or materialize the full type before an explicit projection, and cover field-only reads from old rowsets. -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: [email protected] For queries about this service, please contact Infrastructure at: [email protected] --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
