This is an automated email from the ASF dual-hosted git repository.
jianliangqi pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/master by this push:
new ef05e13f888 [improvement](inverted index) Disable the use of skipping
write index on load (#34719)
ef05e13f888 is described below
commit ef05e13f888f68c819e39da8f0f9b4e0a8551701
Author: qiye <jianliang5...@gmail.com>
AuthorDate: Mon May 13 15:27:51 2024 +0800
[improvement](inverted index) Disable the use of skipping write index on
load (#34719)
When `skip_write_index_on_load` is turned on, users will get an error when
querying for the latest data(not compacted), giving them a bad experience. And
we can use `inverted_index_ram_dir_enable = true` and
`inverted_index_storage_format=V2` to reduce IO and CPU consumption. So we
disable it now.
1. Disable setting `skip_write_index_on_load` to `true` in create table
stmt.
2. Disable setting `skip_write_index_on_load` to `true` in alter table
properties stmt. You can still alter `skip_write_index_on_load` to `false`.
Co-authored-by: Luennng <luen...@gmail.com>
---
.../java/org/apache/doris/analysis/ModifyTablePropertiesClause.java | 5 +++++
.../src/main/java/org/apache/doris/common/util/PropertyAnalyzer.java | 3 ++-
.../src/main/java/org/apache/doris/datasource/InternalCatalog.java | 3 +--
regression-test/suites/datatype_p0/scalar_types/load.groovy | 2 +-
4 files changed, 9 insertions(+), 4 deletions(-)
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/analysis/ModifyTablePropertiesClause.java
b/fe/fe-core/src/main/java/org/apache/doris/analysis/ModifyTablePropertiesClause.java
index e0b0914634b..b6ef97815bc 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/analysis/ModifyTablePropertiesClause.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/analysis/ModifyTablePropertiesClause.java
@@ -240,6 +240,11 @@ public class ModifyTablePropertiesClause extends
AlterTableClause {
this.needTableStable = false;
this.opType = AlterOpType.MODIFY_TABLE_PROPERTY_SYNC;
} else if
(properties.containsKey(PropertyAnalyzer.PROPERTIES_SKIP_WRITE_INDEX_ON_LOAD)) {
+ if
(properties.get(PropertyAnalyzer.PROPERTIES_SKIP_WRITE_INDEX_ON_LOAD).equalsIgnoreCase("true"))
{
+ throw new AnalysisException(
+ "Property "
+ + PropertyAnalyzer.PROPERTIES_SKIP_WRITE_INDEX_ON_LOAD + "
is forbidden now");
+ }
if
(!properties.get(PropertyAnalyzer.PROPERTIES_SKIP_WRITE_INDEX_ON_LOAD).equalsIgnoreCase("true")
&& !properties.get(PropertyAnalyzer
.PROPERTIES_SKIP_WRITE_INDEX_ON_LOAD).equalsIgnoreCase("false")) {
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/common/util/PropertyAnalyzer.java
b/fe/fe-core/src/main/java/org/apache/doris/common/util/PropertyAnalyzer.java
index 91848292f89..0972018f572 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/common/util/PropertyAnalyzer.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/common/util/PropertyAnalyzer.java
@@ -798,7 +798,8 @@ public class PropertyAnalyzer {
}
properties.remove(PROPERTIES_SKIP_WRITE_INDEX_ON_LOAD);
if (value.equalsIgnoreCase("true")) {
- return true;
+ throw new AnalysisException("Property " +
PROPERTIES_SKIP_WRITE_INDEX_ON_LOAD
+ + " is forbidden now.");
} else if (value.equalsIgnoreCase("false")) {
return false;
}
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/datasource/InternalCatalog.java
b/fe/fe-core/src/main/java/org/apache/doris/datasource/InternalCatalog.java
index 1f51f1f9c7c..d85d335c45d 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/datasource/InternalCatalog.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/InternalCatalog.java
@@ -2492,8 +2492,7 @@ public class InternalCatalog implements
CatalogIf<Database> {
olapTable.setStoreRowColumn(storeRowColumn);
// set skip inverted index on load
- boolean skipWriteIndexOnLoad =
PropertyAnalyzer.analyzeBooleanProp(properties,
- PropertyAnalyzer.PROPERTIES_SKIP_WRITE_INDEX_ON_LOAD, false);
+ boolean skipWriteIndexOnLoad =
PropertyAnalyzer.analyzeSkipWriteIndexOnLoad(properties);
olapTable.setSkipWriteIndexOnLoad(skipWriteIndexOnLoad);
boolean isMutable = PropertyAnalyzer.analyzeBooleanProp(properties,
PropertyAnalyzer.PROPERTIES_MUTABLE, true);
diff --git a/regression-test/suites/datatype_p0/scalar_types/load.groovy
b/regression-test/suites/datatype_p0/scalar_types/load.groovy
index 91cd4eb1f1f..4c7c7ee9366 100644
--- a/regression-test/suites/datatype_p0/scalar_types/load.groovy
+++ b/regression-test/suites/datatype_p0/scalar_types/load.groovy
@@ -448,7 +448,7 @@ suite("test_scalar_types_load", "p0") {
DUPLICATE KEY(`k1`)
COMMENT 'OLAP'
DISTRIBUTED BY HASH(`k1`) BUCKETS 10
- PROPERTIES("replication_num" = "1", "skip_write_index_on_load" =
"true");
+ PROPERTIES("replication_num" = "1", "skip_write_index_on_load" =
"false");
"""
// insert data into dup table with index
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org
For additional commands, e-mail: commits-h...@doris.apache.org
