This is an automated email from the ASF dual-hosted git repository.
jlfsdtc pushed a commit to branch kylin5
in repository https://gitbox.apache.org/repos/asf/kylin.git
The following commit(s) were added to refs/heads/kylin5 by this push:
new e7ca74ecc6 KYLIN-6013 OpenAPI check gluten is enabled
e7ca74ecc6 is described below
commit e7ca74ecc60c0005388063473d2ba5d8133ff7b6
Author: jlf <[email protected]>
AuthorDate: Tue Oct 15 19:43:27 2024 +0800
KYLIN-6013 OpenAPI check gluten is enabled
---
.../org/apache/kylin/common/msg/CnMessage.java | 7 +-
.../java/org/apache/kylin/common/msg/Message.java | 6 +-
.../apache/kylin/rest/service/TableExtService.java | 2 +-
.../kylin/rest/service/GlutenCacheService.java | 20 ++++-
.../kylin/rest/service/GlutenCacheServiceTest.java | 89 ++++++++++++++++++++++
5 files changed, 120 insertions(+), 4 deletions(-)
diff --git
a/src/core-common/src/main/java/org/apache/kylin/common/msg/CnMessage.java
b/src/core-common/src/main/java/org/apache/kylin/common/msg/CnMessage.java
index 1fe8898e52..0da315c13d 100644
--- a/src/core-common/src/main/java/org/apache/kylin/common/msg/CnMessage.java
+++ b/src/core-common/src/main/java/org/apache/kylin/common/msg/CnMessage.java
@@ -188,7 +188,7 @@ public class CnMessage extends Message {
}
@Override
- public String getGlutenNotEnabled() {
+ public String getGlutenDisabled() {
return "查询Gluten开关未开启，请打开后重试。";
}
@@ -960,6 +960,11 @@ public class CnMessage extends Message {
return "表 \"%s\" 不是一个内表。 请检查后重试。";
}
+ @Override
+ public String getInternalTableDisabled() {
+ return "内表未开启，请打开后重试。";
+ }
+
@Override
public String getTableParamEmpty() {
return "无法找到该表，请检查后重试。";
diff --git
a/src/core-common/src/main/java/org/apache/kylin/common/msg/Message.java
b/src/core-common/src/main/java/org/apache/kylin/common/msg/Message.java
index 429c5966ea..230cffe1d3 100644
--- a/src/core-common/src/main/java/org/apache/kylin/common/msg/Message.java
+++ b/src/core-common/src/main/java/org/apache/kylin/common/msg/Message.java
@@ -184,7 +184,7 @@ public class Message {
return "Invalid internal table parameters.";
}
- public String getGlutenNotEnabled() {
+ public String getGlutenDisabled() {
return "Gluten is disabled in query function, please enable it and try
again.";
}
@@ -462,6 +462,10 @@ public class Message {
return "Table \"%s\" is not an internal table. Please check and try
again.";
}
+ public String getInternalTableDisabled() {
+ return "InternalTable is disabled, please enable it and try again.";
+ }
+
public String getSamplingFailedForIllegalTableName() {
return "The name of table for sampling is invalid. Please enter a
table name like “database.table”. ";
}
diff --git
a/src/datasource-service/src/main/java/org/apache/kylin/rest/service/TableExtService.java
b/src/datasource-service/src/main/java/org/apache/kylin/rest/service/TableExtService.java
index 5ece80a571..019382d96a 100644
---
a/src/datasource-service/src/main/java/org/apache/kylin/rest/service/TableExtService.java
+++
b/src/datasource-service/src/main/java/org/apache/kylin/rest/service/TableExtService.java
@@ -364,7 +364,7 @@ public class TableExtService extends BasicService {
// check if gluten enabled before load table
if (loadAsInternal &&
!KylinConfig.getInstanceFromEnv().queryUseGlutenEnabled()) {
throw new KylinException(GLUTEN_NOT_ENABLED_ERROR,
- String.format(Locale.ROOT,
MsgPicker.getMsg().getGlutenNotEnabled()));
+ String.format(Locale.ROOT,
MsgPicker.getMsg().getGlutenDisabled()));
}
for (List<Pair<TableDesc, TableExtDesc>> batch : batches) {
try {
diff --git
a/src/query-service/src/main/java/org/apache/kylin/rest/service/GlutenCacheService.java
b/src/query-service/src/main/java/org/apache/kylin/rest/service/GlutenCacheService.java
index 5238525f12..d0f95ac024 100644
---
a/src/query-service/src/main/java/org/apache/kylin/rest/service/GlutenCacheService.java
+++
b/src/query-service/src/main/java/org/apache/kylin/rest/service/GlutenCacheService.java
@@ -18,6 +18,8 @@
package org.apache.kylin.rest.service;
+import static
org.apache.kylin.common.exception.ServerErrorCode.GLUTEN_NOT_ENABLED_ERROR;
+import static
org.apache.kylin.common.exception.ServerErrorCode.INTERNAL_TABLE_ERROR;
import static
org.apache.kylin.common.msg.Message.LOAD_GLUTEN_CACHE_EXECUTE_ERROR;
import static org.apache.kylin.metadata.model.SegmentStatusEnum.READY;
import static org.apache.kylin.metadata.model.SegmentStatusEnum.WARNING;
@@ -31,10 +33,14 @@ import javax.servlet.http.HttpServletRequest;
import org.apache.commons.collections4.CollectionUtils;
import org.apache.commons.lang3.StringUtils;
+import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.common.constant.LogConstant;
+import org.apache.kylin.common.exception.KylinException;
import org.apache.kylin.common.exception.KylinRuntimeException;
import org.apache.kylin.common.logging.SetLogCategory;
+import org.apache.kylin.common.msg.MsgPicker;
import org.apache.kylin.guava30.shaded.common.collect.Lists;
+import org.apache.kylin.metadata.project.NProjectManager;
import org.apache.kylin.rest.model.GlutenCacheExecuteResult;
import org.apache.kylin.rest.request.IndexGlutenCacheRequest;
import org.apache.kylin.rest.request.InternalTableGlutenCacheRequest;
@@ -104,10 +110,18 @@ public class GlutenCacheService extends BasicService {
public void internalTableGlutenCache(InternalTableGlutenCacheRequest
request, HttpServletRequest servletRequest)
throws Exception {
+ val config = KylinConfig.getInstanceFromEnv();
+ if (!config.queryUseGlutenEnabled()) {
+ throw new KylinException(GLUTEN_NOT_ENABLED_ERROR,
MsgPicker.getMsg().getGlutenDisabled());
+ }
+ val project = request.getProject();
+ val projectConfig = NProjectManager.getProjectConfig(project);
+ if (!projectConfig.isInternalTableEnabled()) {
+ throw new KylinException(INTERNAL_TABLE_ERROR,
MsgPicker.getMsg().getInternalTableDisabled());
+ }
val columns =
request.getColumns().stream().map(StringUtils::upperCase).collect(Collectors.toList());
val table = StringUtils.upperCase(request.getDatabase() + "." +
request.getTable());
val start = request.getStart();
- val project = request.getProject();
val cacheTableCommand =
GlutenCacheUtils.generateCacheTableCommand(getConfig(), project, table, start,
columns,
true);
log.info("InternalTable[{}] cache command is [{}]", table,
cacheTableCommand);
@@ -115,6 +129,10 @@ public class GlutenCacheService extends BasicService {
}
public void indexGlutenCache(IndexGlutenCacheRequest request,
HttpServletRequest servletRequest) throws Exception {
+ val config = KylinConfig.getInstanceFromEnv();
+ if (!config.queryUseGlutenEnabled()) {
+ throw new KylinException(GLUTEN_NOT_ENABLED_ERROR,
MsgPicker.getMsg().getGlutenDisabled());
+ }
val project = request.getProject();
val modelAlias = request.getModel();
val indexes = request.getIndexes();
diff --git
a/src/query-service/src/test/java/org/apache/kylin/rest/service/GlutenCacheServiceTest.java
b/src/query-service/src/test/java/org/apache/kylin/rest/service/GlutenCacheServiceTest.java
index 4f24747fab..c94d097648 100644
---
a/src/query-service/src/test/java/org/apache/kylin/rest/service/GlutenCacheServiceTest.java
+++
b/src/query-service/src/test/java/org/apache/kylin/rest/service/GlutenCacheServiceTest.java
@@ -31,12 +31,17 @@ import java.util.stream.Collectors;
import org.apache.commons.collections4.CollectionUtils;
import org.apache.commons.lang3.StringUtils;
import org.apache.kylin.common.KylinConfig;
+import org.apache.kylin.common.exception.KylinException;
import org.apache.kylin.common.exception.KylinRuntimeException;
+import org.apache.kylin.common.msg.CnMessage;
+import org.apache.kylin.common.msg.Message;
+import org.apache.kylin.common.msg.MsgPicker;
import org.apache.kylin.common.util.NLocalFileMetadataTestCase;
import org.apache.kylin.guava30.shaded.common.collect.Lists;
import org.apache.kylin.guava30.shaded.common.collect.Sets;
import org.apache.kylin.metadata.model.NDataModel;
import org.apache.kylin.metadata.model.Segments;
+import org.apache.kylin.metadata.project.NProjectManager;
import org.apache.kylin.metadata.streaming.ReflectionUtils;
import org.apache.kylin.rest.model.GlutenCacheExecuteResult;
import org.apache.kylin.rest.request.IndexGlutenCacheRequest;
@@ -118,6 +123,9 @@ public class GlutenCacheServiceTest extends
NLocalFileMetadataTestCase {
public void setUp() throws Exception {
createTestMetadata();
config = getTestConfig();
+
config.setProperty("kylin.storage.columnar.spark-conf.spark.gluten.enabled",
KylinConfig.TRUE);
+ config.setProperty("kylin.storage.columnar.spark-conf.spark.plugins",
"GlutenPlugin");
+ config.setProperty("kylin.internal-table-enabled", KylinConfig.TRUE);
PowerMockito.mockStatic(SparderEnv.class, GlutenCacheUtils.class);
PowerMockito
@@ -374,6 +382,87 @@ public class GlutenCacheServiceTest extends
NLocalFileMetadataTestCase {
Assert.assertEquals(String.format(Locale.ROOT, "Model[%s]
StorageType is V3, start/end need null", MODEL),
e.getMessage());
}
+ }
+
+ @Test
+ public void indexGlutenCacheDisabled() throws Exception {
+ val servletRequest = new MockHttpServletRequest();
+ val request = new IndexGlutenCacheRequest();
+ request.setModel(MODEL);
+ request.setProject(PROJECT);
+ request.setIndexes(INDEXES);
+
+
config.setProperty("kylin.storage.columnar.spark-conf.spark.gluten.enabled",
KylinConfig.FALSE);
+ try {
+ MsgPicker.setMsg("cn");
+ glutenCacheService.indexGlutenCache(request, servletRequest);
+ Assert.fail();
+ } catch (Exception e) {
+ Assert.assertTrue(e instanceof KylinException);
+ Assert.assertEquals(CnMessage.getInstance().getGlutenDisabled(),
e.getMessage());
+ }
+ try {
+ MsgPicker.setMsg("en");
+ glutenCacheService.indexGlutenCache(request, servletRequest);
+ Assert.fail();
+ } catch (Exception e) {
+ Assert.assertTrue(e instanceof KylinException);
+ Assert.assertEquals(Message.getInstance().getGlutenDisabled(),
e.getMessage());
+ }
+
config.setProperty("kylin.storage.columnar.spark-conf.spark.gluten.enabled",
KylinConfig.TRUE);
+ }
+
+ @Test
+ public void internalTableGlutenCacheDisabled() throws Exception {
+ val servletRequest = new MockHttpServletRequest();
+ val request = new InternalTableGlutenCacheRequest();
+ request.setProject(PROJECT);
+ request.setDatabase(DATABASE);
+ request.setTable(TABLE);
+ request.setStart(START);
+ request.setColumns(COLUMNS);
+
+ {
+
config.setProperty("kylin.storage.columnar.spark-conf.spark.gluten.enabled",
KylinConfig.FALSE);
+ try {
+ MsgPicker.setMsg("cn");
+ glutenCacheService.internalTableGlutenCache(request,
servletRequest);
+ Assert.fail();
+ } catch (Exception e) {
+ Assert.assertTrue(e instanceof KylinException);
+
Assert.assertEquals(CnMessage.getInstance().getGlutenDisabled(),
e.getMessage());
+ }
+ try {
+ MsgPicker.setMsg("en");
+ glutenCacheService.internalTableGlutenCache(request,
servletRequest);
+ Assert.fail();
+ } catch (Exception e) {
+ Assert.assertTrue(e instanceof KylinException);
+ Assert.assertEquals(Message.getInstance().getGlutenDisabled(),
e.getMessage());
+ }
+
config.setProperty("kylin.storage.columnar.spark-conf.spark.gluten.enabled",
KylinConfig.TRUE);
+ }
+ {
+ val projectConfig = NProjectManager.getProjectConfig(PROJECT);
+ projectConfig.setProperty("kylin.internal-table-enabled",
KylinConfig.FALSE);
+ try {
+ MsgPicker.setMsg("cn");
+ glutenCacheService.internalTableGlutenCache(request,
servletRequest);
+ Assert.fail();
+ } catch (Exception e) {
+ Assert.assertTrue(e instanceof KylinException);
+
Assert.assertEquals(CnMessage.getInstance().getInternalTableDisabled(),
e.getMessage());
+ }
+ try {
+ MsgPicker.setMsg("en");
+ glutenCacheService.internalTableGlutenCache(request,
servletRequest);
+ Assert.fail();
+ } catch (Exception e) {
+ Assert.assertTrue(e instanceof KylinException);
+
Assert.assertEquals(Message.getInstance().getInternalTableDisabled(),
e.getMessage());
+ }
+ projectConfig.setProperty("kylin.internal-table-enabled",
KylinConfig.TRUE);
+ }
}
}
