This is an automated email from the ASF dual-hosted git repository.
yiguolei pushed a commit to branch branch-2.1
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/branch-2.1 by this push:
new f7e844b1f2e branch-2.1: [Enhancement] reject schema change on hidden
columns (#53… (#55387)
f7e844b1f2e is described below
commit f7e844b1f2e7f002a5a78506aff44d7ef057f122
Author: csding <[email protected]>
AuthorDate: Wed Sep 3 19:52:59 2025 +0800
branch-2.1: [Enhancement] reject schema change on hidden columns (#53…
(#55387)
Issue Number: close https://github.com/apache/doris/issues/53019
picked from #53376
### What problem does this PR solve?
Issue Number: close #xxx
Related PR: #xxx
Problem Summary:
### Release note
None
### Check List (For Author)
- Test <!-- At least one of them must be included. -->
- [ ] Regression test
- [ ] Unit Test
- [ ] Manual test (add detailed scripts or steps below)
- [ ] No need to test or manual test. Explain why:
- [ ] This is a refactor/code format and no logic has been changed.
- [ ] Previous test can cover this change.
- [ ] No code files have been changed.
- [ ] Other reason <!-- Add your reason? -->
- Behavior changed:
- [ ] No.
- [ ] Yes. <!-- Explain the behavior change -->
- Does this need documentation?
- [ ] No.
- [ ] Yes. <!-- Add document PR link here. eg:
https://github.com/apache/doris-website/pull/1214 -->
### Check List (For Reviewer who merge this PR)
- [ ] Confirm the release note
- [ ] Confirm test cases
- [ ] Confirm document
- [ ] Add branch pick label <!-- Add branch pick label that this PR
should merge into -->
---
.../java/org/apache/doris/analysis/ColumnDef.java | 47 +++++++++---
.../main/java/org/apache/doris/catalog/Column.java | 1 +
.../java/org/apache/doris/common/FeNameFormat.java | 24 +++++--
.../plans/commands/info/ColumnDefinition.java | 84 ++++++++++++++++++----
4 files changed, 128 insertions(+), 28 deletions(-)
diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/ColumnDef.java
b/fe/fe-core/src/main/java/org/apache/doris/analysis/ColumnDef.java
index 17986662058..4312f706ee3 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/analysis/ColumnDef.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/ColumnDef.java
@@ -185,6 +185,12 @@ public class ColumnDef {
private String comment;
private boolean visible;
private int clusterKeyId = -1;
+ // if add hidden column, must set enableAddHiddenColumn true
+ private boolean enableAddHiddenColumn = false;
+
+ public void setEnableAddHiddenColumn(boolean enableAddHiddenColumn) {
+ this.enableAddHiddenColumn = enableAddHiddenColumn;
+ }
public ColumnDef(String name, TypeDef typeDef) {
this(name, typeDef, false, null, false, -1, DefaultValue.NOT_SET, "");
@@ -219,40 +225,59 @@ public class ColumnDef {
}
public static ColumnDef newDeleteSignColumnDef() {
- return new ColumnDef(Column.DELETE_SIGN,
TypeDef.create(PrimitiveType.TINYINT), false, null, false,
+ ColumnDef columnDef = new ColumnDef(Column.DELETE_SIGN,
TypeDef.create(PrimitiveType.TINYINT),
+ false, null, false,
-1, new ColumnDef.DefaultValue(true, "0"), "doris delete flag
hidden column", false);
+ columnDef.setEnableAddHiddenColumn(true);
+ return columnDef;
}
public static ColumnDef newDeleteSignColumnDef(AggregateType
aggregateType) {
- return new ColumnDef(Column.DELETE_SIGN,
TypeDef.create(PrimitiveType.TINYINT), false, aggregateType, false,
+ ColumnDef columnDef = new ColumnDef(Column.DELETE_SIGN,
TypeDef.create(PrimitiveType.TINYINT),
+ false, aggregateType, false,
-1, new ColumnDef.DefaultValue(true, "0"), "doris delete flag
hidden column", false);
+ columnDef.setEnableAddHiddenColumn(true);
+ return columnDef;
}
public static ColumnDef newSequenceColumnDef(Type type) {
- return new ColumnDef(Column.SEQUENCE_COL, new TypeDef(type), false,
null, true,
+ ColumnDef columnDef = new ColumnDef(Column.SEQUENCE_COL, new
TypeDef(type),
+ false, null, true,
-1, DefaultValue.NULL_DEFAULT_VALUE, "sequence column hidden
column", false);
+ columnDef.setEnableAddHiddenColumn(true);
+ return columnDef;
}
public static ColumnDef newSequenceColumnDef(Type type, AggregateType
aggregateType) {
- return new ColumnDef(Column.SEQUENCE_COL, new TypeDef(type), false,
+ ColumnDef columnDef = new ColumnDef(Column.SEQUENCE_COL, new
TypeDef(type), false,
aggregateType, true, -1, DefaultValue.NULL_DEFAULT_VALUE,
"sequence column hidden column", false);
+ columnDef.setEnableAddHiddenColumn(true);
+ return columnDef;
}
public static ColumnDef newRowStoreColumnDef(AggregateType aggregateType) {
- return new ColumnDef(Column.ROW_STORE_COL,
TypeDef.create(PrimitiveType.STRING), false,
+ ColumnDef columnDef = new ColumnDef(Column.ROW_STORE_COL,
TypeDef.create(PrimitiveType.STRING), false,
aggregateType, false, -1,
new ColumnDef.DefaultValue(true, ""), "doris row store hidden
column", false);
+ columnDef.setEnableAddHiddenColumn(true);
+ return columnDef;
}
public static ColumnDef newVersionColumnDef() {
- return new ColumnDef(Column.VERSION_COL,
TypeDef.create(PrimitiveType.BIGINT), false, null, false, -1,
+ ColumnDef columnDef = new ColumnDef(Column.VERSION_COL,
TypeDef.create(PrimitiveType.BIGINT),
+ false, null, false, -1,
new ColumnDef.DefaultValue(true, "0"), "doris version hidden
column", false);
+ columnDef.setEnableAddHiddenColumn(true);
+ return columnDef;
}
public static ColumnDef newVersionColumnDef(AggregateType aggregateType) {
- return new ColumnDef(Column.VERSION_COL,
TypeDef.create(PrimitiveType.BIGINT), false, aggregateType, false,
+ ColumnDef columnDef = new ColumnDef(Column.VERSION_COL,
TypeDef.create(PrimitiveType.BIGINT),
+ false, aggregateType, false,
-1, new ColumnDef.DefaultValue(true, "0"), "doris version
hidden column", false);
+ columnDef.setEnableAddHiddenColumn(true);
+ return columnDef;
}
public boolean isAllowNull() {
@@ -311,7 +336,13 @@ public class ColumnDef {
if (name == null || typeDef == null) {
throw new AnalysisException("No column name or column type in
column definition.");
}
- FeNameFormat.checkColumnName(name);
+ // if enableAddHiddenColumn is true, can add hidden column.
+ // So does not check if the column name starts with __DORIS_
+ if (enableAddHiddenColumn) {
+ FeNameFormat.checkColumnNameBypassHiddenColumn(name);
+ } else {
+ FeNameFormat.checkColumnName(name);
+ }
typeDef.analyze(null);
diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/Column.java
b/fe/fe-core/src/main/java/org/apache/doris/catalog/Column.java
index 903b7569e60..a99c348810b 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/catalog/Column.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/Column.java
@@ -55,6 +55,7 @@ import java.util.Set;
*/
public class Column implements Writable, GsonPostProcessable {
private static final Logger LOG = LogManager.getLogger(Column.class);
+ public static final String HIDDEN_COLUMN_PREFIX = "__DORIS_";
// NOTE: you should name hidden column start with '__DORIS_'
!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
public static final String DELETE_SIGN = "__DORIS_DELETE_SIGN__";
public static final String WHERE_SIGN = "__DORIS_WHERE_SIGN__";
diff --git a/fe/fe-core/src/main/java/org/apache/doris/common/FeNameFormat.java
b/fe/fe-core/src/main/java/org/apache/doris/common/FeNameFormat.java
index 03600cc3e3b..68855e71c1f 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/common/FeNameFormat.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/common/FeNameFormat.java
@@ -19,6 +19,7 @@ package org.apache.doris.common;
import org.apache.doris.alter.SchemaChangeHandler;
import org.apache.doris.analysis.CreateMaterializedViewStmt;
+import org.apache.doris.catalog.Column;
import org.apache.doris.datasource.InternalCatalog;
import org.apache.doris.mysql.privilege.Role;
import org.apache.doris.mysql.privilege.RoleManager;
@@ -85,18 +86,29 @@ public class FeNameFormat {
}
public static void checkColumnName(String columnName) throws
AnalysisException {
+ // if need check another column name prefix, add in
`checkColumnNameBypassHiddenColumn`
+ checkColumnNameBypassHiddenColumn(columnName);
+ checkColumnNamePrefix(columnName, Column.HIDDEN_COLUMN_PREFIX);
+ }
+
+ public static void checkColumnNameBypassHiddenColumn(String columnName)
throws AnalysisException {
if (Strings.isNullOrEmpty(columnName) ||
!columnName.matches(getColumnNameRegex())) {
ErrorReport.reportAnalysisException(ErrorCode.ERR_WRONG_COLUMN_NAME,
columnName, getColumnNameRegex());
}
- if (columnName.startsWith(SchemaChangeHandler.SHADOW_NAME_PREFIX)) {
-
ErrorReport.reportAnalysisException(ErrorCode.ERR_WRONG_COLUMN_NAME,
- columnName, getColumnNameRegex());
+ checkColumnNamePrefix(columnName,
SchemaChangeHandler.SHADOW_NAME_PREFIX);
+ checkColumnNamePrefix(columnName,
CreateMaterializedViewStmt.MATERIALIZED_VIEW_NAME_PREFIX);
+ checkColumnNamePrefix(columnName,
CreateMaterializedViewStmt.MATERIALIZED_VIEW_AGGREGATE_NAME_PREFIX);
+ }
+
+ private static void checkColumnNamePrefix(String columnName, String
prefix) throws AnalysisException {
+ int prefixLength = prefix.length();
+ if (columnName.length() < prefixLength) {
+ return;
}
- if
(columnName.startsWith(CreateMaterializedViewStmt.MATERIALIZED_VIEW_NAME_PREFIX)
- ||
columnName.startsWith(CreateMaterializedViewStmt.MATERIALIZED_VIEW_AGGREGATE_NAME_PREFIX))
{
+ if (columnName.substring(0, prefixLength).equalsIgnoreCase(prefix)) {
throw new AnalysisException(
- "Incorrect column name " + columnName + ", column name
can't start with 'mv_'/'mva_'");
+ "Incorrect column name " + columnName + ", column name can't
start with '" + prefix + "'");
}
}
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/ColumnDefinition.java
b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/ColumnDefinition.java
index 36140b3b821..28a0b7bdd93 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/ColumnDefinition.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/ColumnDefinition.java
@@ -65,6 +65,8 @@ public class ColumnDefinition {
private boolean aggTypeImplicit = false;
private long autoIncInitValue = -1;
private int clusterKeyId = -1;
+ // if add hidden column, must set enableAddHiddenColumn true
+ private boolean enableAddHiddenColumn = false;
public ColumnDefinition(String name, DataType type, boolean isKey,
AggregateType aggType, boolean isNullable,
Optional<DefaultValue> defaultValue, String comment) {
@@ -131,6 +133,10 @@ public class ColumnDefinition {
return aggType;
}
+ public void setEnableAddHiddenColumn(boolean enableAddHiddenColumn) {
+ this.enableAddHiddenColumn = enableAddHiddenColumn;
+ }
+
public void setAggType(AggregateType aggType) {
this.aggType = aggType;
}
@@ -180,7 +186,13 @@ public class ColumnDefinition {
*/
public void validate(boolean isOlap, Set<String> keysSet, boolean
isEnableMergeOnWrite, KeysType keysType) {
try {
- FeNameFormat.checkColumnName(name);
+ // if enableAddHiddenColumn is true, can add hidden column.
+ // So does not check if the column name starts with __DORIS_
+ if (enableAddHiddenColumn) {
+ FeNameFormat.checkColumnNameBypassHiddenColumn(name);
+ } else {
+ FeNameFormat.checkColumnName(name);
+ }
} catch (Exception e) {
throw new AnalysisException(e.getMessage(), e);
}
@@ -639,35 +651,79 @@ public class ColumnDefinition {
return column;
}
- // hidden column
+ /**
+ * add hidden column
+ */
public static ColumnDefinition newDeleteSignColumnDefinition() {
- return new ColumnDefinition(Column.DELETE_SIGN, TinyIntType.INSTANCE,
false, null, false,
- Optional.of(new DefaultValue(DefaultValue.ZERO_NUMBER)),
"doris delete flag hidden column", false);
+ ColumnDefinition columnDefinition = new
ColumnDefinition(Column.DELETE_SIGN, TinyIntType.INSTANCE, false, null,
+ false, Optional.of(new DefaultValue(DefaultValue.ZERO_NUMBER)),
+ "doris delete flag hidden column", false);
+ columnDefinition.setEnableAddHiddenColumn(true);
+
+ return columnDefinition;
}
+ /**
+ * add hidden column
+ */
public static ColumnDefinition newDeleteSignColumnDefinition(AggregateType
aggregateType) {
- return new ColumnDefinition(Column.DELETE_SIGN, TinyIntType.INSTANCE,
false, aggregateType, false,
- Optional.of(new DefaultValue(DefaultValue.ZERO_NUMBER)),
"doris delete flag hidden column", false);
+ ColumnDefinition columnDefinition = new
ColumnDefinition(Column.DELETE_SIGN, TinyIntType.INSTANCE, false,
+ aggregateType, false, Optional.of(new
DefaultValue(DefaultValue.ZERO_NUMBER)),
+ "doris delete flag hidden column", false);
+ columnDefinition.setEnableAddHiddenColumn(true);
+
+ return columnDefinition;
}
+ /**
+ * add hidden column
+ */
public static ColumnDefinition newSequenceColumnDefinition(DataType type) {
- return new ColumnDefinition(Column.SEQUENCE_COL, type, false, null,
true,
- Optional.empty(), "sequence column hidden column", false);
+ ColumnDefinition columnDefinition = new
ColumnDefinition(Column.SEQUENCE_COL, type, false, null,
+ true, Optional.empty(),
+ "sequence column hidden column", false);
+ columnDefinition.setEnableAddHiddenColumn(true);
+
+ return columnDefinition;
}
+ /**
+ * add hidden column
+ */
public static ColumnDefinition newSequenceColumnDefinition(DataType type,
AggregateType aggregateType) {
- return new ColumnDefinition(Column.SEQUENCE_COL, type, false,
aggregateType, true,
- Optional.empty(), "sequence column hidden column", false);
+ ColumnDefinition columnDefinition = new
ColumnDefinition(Column.SEQUENCE_COL, type, false, aggregateType,
+ true, Optional.empty(),
+ "sequence column hidden column", false);
+ columnDefinition.setEnableAddHiddenColumn(true);
+
+ return columnDefinition;
}
+ /**
+ * add hidden column
+ */
public static ColumnDefinition newRowStoreColumnDefinition(AggregateType
aggregateType) {
- return new ColumnDefinition(Column.ROW_STORE_COL, StringType.INSTANCE,
false, aggregateType, false,
- Optional.of(new DefaultValue("")), "doris row store hidden
column", false);
+ ColumnDefinition columnDefinition = new
ColumnDefinition(Column.ROW_STORE_COL, StringType.INSTANCE, false,
+ aggregateType, false, Optional.of(new
DefaultValue("")),
+ "doris row store hidden column", false);
+ columnDefinition.setEnableAddHiddenColumn(true);
+
+ return columnDefinition;
}
+ /**
+ * add hidden column
+ */
public static ColumnDefinition newVersionColumnDefinition(AggregateType
aggregateType) {
- return new ColumnDefinition(Column.VERSION_COL, BigIntType.INSTANCE,
false, aggregateType, false,
- Optional.of(new DefaultValue(DefaultValue.ZERO_NUMBER)),
"doris version hidden column", false);
+ ColumnDefinition columnDefinition = new
ColumnDefinition(Column.VERSION_COL, BigIntType.INSTANCE, false,
+ aggregateType, false, Optional.of(new
DefaultValue(DefaultValue.ZERO_NUMBER)),
+ "doris version hidden column", false);
+ columnDefinition.setEnableAddHiddenColumn(true);
+
+ return columnDefinition;
}
+ public long getAutoIncInitValue() {
+ return autoIncInitValue;
+ }
}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
