This is an automated email from the ASF dual-hosted git repository.
yhcast0 pushed a commit to branch kylin5
in repository https://gitbox.apache.org/repos/asf/kylin.git
The following commit(s) were added to refs/heads/kylin5 by this push:
new ba4659c2c2 KYLIN-6004 check gluten enabled when load table as internal
table (#2257)
ba4659c2c2 is described below
commit ba4659c2c2c40629f61c636f8730032a7022aa81
Author: Guoliang Sun <[email protected]>
AuthorDate: Fri Feb 21 10:30:22 2025 +0800
KYLIN-6004 check gluten enabled when load table as internal table (#2257)
Co-authored-by: Zhong.Zhu <[email protected]>
---
.../src/main/java/org/apache/kylin/common/KylinConfigBase.java | 3 ++-
.../java/org/apache/kylin/common/exception/ServerErrorCode.java | 3 ++-
.../src/main/java/org/apache/kylin/common/msg/CnMessage.java | 5 +++++
.../src/main/java/org/apache/kylin/common/msg/Message.java | 4 ++++
src/core-common/src/main/resources/kylin_errorcode_conf.properties | 1 +
.../src/main/resources/kylin_errorcode_conf_en.properties | 1 +
.../src/main/resources/kylin_errorcode_conf_zh.properties | 1 +
.../main/java/org/apache/kylin/rest/service/TableExtService.java | 6 ++++++
.../java/org/apache/kylin/rest/service/TableExtServiceTest.java | 4 ++++
9 files changed, 26 insertions(+), 2 deletions(-)
diff --git
a/src/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
b/src/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
index 601fc7fb14..cb03a4ca63 100644
--- a/src/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
+++ b/src/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
@@ -4395,7 +4395,8 @@ public abstract class KylinConfigBase implements
Serializable {
}
public boolean queryUseGlutenEnabled() {
- return
Boolean.parseBoolean(this.getOptional("kylin.storage.columnar.spark-conf.spark.gluten.enabled",
FALSE));
+ return
Boolean.parseBoolean(this.getOptional("kylin.storage.columnar.spark-conf.spark.gluten.enabled",
FALSE))
+ &&
this.getOptional("kylin.storage.columnar.spark-conf.spark.plugins",
"").contains("GlutenPlugin");
}
public String getFragmentType() {
diff --git
a/src/core-common/src/main/java/org/apache/kylin/common/exception/ServerErrorCode.java
b/src/core-common/src/main/java/org/apache/kylin/common/exception/ServerErrorCode.java
index 0e142b29e7..18edd49932 100644
---
a/src/core-common/src/main/java/org/apache/kylin/common/exception/ServerErrorCode.java
+++
b/src/core-common/src/main/java/org/apache/kylin/common/exception/ServerErrorCode.java
@@ -106,7 +106,8 @@ public enum ServerErrorCode implements ErrorCodeSupplier {
INVALID_INTERNAL_TABLE_PARAMETER("KE-010007013"), //
INTERNAL_TABLE_NOT_EXIST("KE-010007014"), //
INTERNAL_TABLE_PARTITION_NOT_EXIST("KE-010007015"), //
- INTERNAL_TABLE_RELOAD_ERROR("KE-010007016"),
+ INTERNAL_TABLE_RELOAD_ERROR("KE-010007016"), //
+ GLUTEN_NOT_ENABLED_ERROR("KE-010007017"), //
// 10008XXX database
DATABASE_NOT_EXIST("KE-010008001"), //
diff --git
a/src/core-common/src/main/java/org/apache/kylin/common/msg/CnMessage.java
b/src/core-common/src/main/java/org/apache/kylin/common/msg/CnMessage.java
index f6cda0080a..1fe8898e52 100644
--- a/src/core-common/src/main/java/org/apache/kylin/common/msg/CnMessage.java
+++ b/src/core-common/src/main/java/org/apache/kylin/common/msg/CnMessage.java
@@ -187,6 +187,11 @@ public class CnMessage extends Message {
}
}
+ @Override
+ public String getGlutenNotEnabled() {
+ return "查询Gluten开关未开启，请打开后重试。";
+ }
+
@Override
public String getIncorrectDateformat() {
return "时间分区格式 \"%s\" 不正确。";
diff --git
a/src/core-common/src/main/java/org/apache/kylin/common/msg/Message.java
b/src/core-common/src/main/java/org/apache/kylin/common/msg/Message.java
index 780fe8c99a..429c5966ea 100644
--- a/src/core-common/src/main/java/org/apache/kylin/common/msg/Message.java
+++ b/src/core-common/src/main/java/org/apache/kylin/common/msg/Message.java
@@ -184,6 +184,10 @@ public class Message {
return "Invalid internal table parameters.";
}
+ public String getGlutenNotEnabled() {
+ return "Gluten is disabled in query function, please enable it and try
again.";
+ }
+
public String getDatePartitionColumnNotFound() {
return "Date format partition column must be set in an incremental
load table.";
}
diff --git a/src/core-common/src/main/resources/kylin_errorcode_conf.properties
b/src/core-common/src/main/resources/kylin_errorcode_conf.properties
index 9d2074bf54..26521f84d3 100644
--- a/src/core-common/src/main/resources/kylin_errorcode_conf.properties
+++ b/src/core-common/src/main/resources/kylin_errorcode_conf.properties
@@ -84,6 +84,7 @@ KE-010007204
KE-010007208
KE-010007301
KE-010007011
+KE-010007017
## 100322XX job
KE-010032201
diff --git
a/src/core-common/src/main/resources/kylin_errorcode_conf_en.properties
b/src/core-common/src/main/resources/kylin_errorcode_conf_en.properties
index 9430ae4cc7..a9d7438033 100644
--- a/src/core-common/src/main/resources/kylin_errorcode_conf_en.properties
+++ b/src/core-common/src/main/resources/kylin_errorcode_conf_en.properties
@@ -104,6 +104,7 @@ KE-010007013=Internal Table Parameter Invalid
KE-010007014=Internal Table Not Exist
KE-010007015=Internal Table Partitions Not Exist
KE-010007016=Can't reload none empty internal table
+KE-010007017=Gluten is disabled
KE-010008001=Database Not Exist
KE-010008002=Failed Import SSB Data
KE-010008003=Unsupported Data Source Type
diff --git
a/src/core-common/src/main/resources/kylin_errorcode_conf_zh.properties
b/src/core-common/src/main/resources/kylin_errorcode_conf_zh.properties
index b834cf85f2..ade4fc7a18 100644
--- a/src/core-common/src/main/resources/kylin_errorcode_conf_zh.properties
+++ b/src/core-common/src/main/resources/kylin_errorcode_conf_zh.properties
@@ -104,6 +104,7 @@ KE-010007013=内表无效表格参数
KE-010007014=内表不存在
KE-010007015=内表分区不存在
KE-010007016=无法重载非空内表
+KE-010007017=Gluten未开启
KE-010008001=数据库不存在
KE-010008002=导入SSB数据集失败
KE-010008003=不支持的数据源类型
diff --git
a/src/datasource-service/src/main/java/org/apache/kylin/rest/service/TableExtService.java
b/src/datasource-service/src/main/java/org/apache/kylin/rest/service/TableExtService.java
index edf9ced5a4..5ece80a571 100644
---
a/src/datasource-service/src/main/java/org/apache/kylin/rest/service/TableExtService.java
+++
b/src/datasource-service/src/main/java/org/apache/kylin/rest/service/TableExtService.java
@@ -18,6 +18,7 @@
package org.apache.kylin.rest.service;
+import static
org.apache.kylin.common.exception.ServerErrorCode.GLUTEN_NOT_ENABLED_ERROR;
import static
org.apache.kylin.common.exception.ServerErrorCode.INVALID_TABLE_NAME;
import static
org.apache.kylin.common.exception.code.ErrorCodeServer.EXCLUDED_TABLE_REQUEST_NOT_ALLOWED;
import static
org.apache.kylin.common.exception.code.ErrorCodeServer.ONCE_LOAD_TABLE_LIMIT;
@@ -360,6 +361,11 @@ public class TableExtService extends BasicService {
List<Pair<TableDesc, TableExtDesc>> loadTables, Boolean
loadAsInternal, String storageType) {
int batchSize =
NProjectManager.getProjectConfig(project).getLoadTableBatchSize();
List<List<Pair<TableDesc, TableExtDesc>>> batches =
splitIntoBatches(loadTables, batchSize);
+ // check if gluten enabled before load table
+ if (loadAsInternal &&
!KylinConfig.getInstanceFromEnv().queryUseGlutenEnabled()) {
+ throw new KylinException(GLUTEN_NOT_ENABLED_ERROR,
+ String.format(Locale.ROOT,
MsgPicker.getMsg().getGlutenNotEnabled()));
+ }
for (List<Pair<TableDesc, TableExtDesc>> batch : batches) {
try {
microBatchLoadTable(project, tableResponse, batch,
loadAsInternal, storageType);
diff --git
a/src/datasource-service/src/test/java/org/apache/kylin/rest/service/TableExtServiceTest.java
b/src/datasource-service/src/test/java/org/apache/kylin/rest/service/TableExtServiceTest.java
index 89d6cfb71a..2195d90ca6 100644
---
a/src/datasource-service/src/test/java/org/apache/kylin/rest/service/TableExtServiceTest.java
+++
b/src/datasource-service/src/test/java/org/apache/kylin/rest/service/TableExtServiceTest.java
@@ -585,6 +585,10 @@ public class TableExtServiceTest extends
NLocalFileMetadataTestCase {
request.setTables(new String[] { "TEST_KYLIN_FACT" });
LoadTableResponse response2 =
tableExtService.loadTablesWithShortCircuit(request);
Assert.assertEquals(8, response2.getFailed().size());
+
+ // load as internal table when gluten not enabled
+ request.setLoadAsInternal(true);
+ Assert.assertThrows(KylinException.class, () ->
tableExtService.loadTablesWithShortCircuit(request));
}
private String[] mockInputDBOrTable() {
