This is an automated email from the ASF dual-hosted git repository.
liyang pushed a commit to branch kylin5
in repository https://gitbox.apache.org/repos/asf/kylin.git
commit d7f37675e2de44478da78d5ae7b4b70fd043b13f
Author: haocheni <hao_...@qq.com>
AuthorDate: Fri Sep 22 19:21:38 2023 +0800
KYLIN-5832 Support queryDetect api
---
.../java/org/apache/kylin/common/QueryContext.java | 1 +
.../kylin/rest/controller/NQueryController.java | 9 +
.../kylin/rest/request/QueryDetectRequest.java | 33 +++
.../kylin/rest/response/QueryDetectResponse.java | 156 ++++++++++++
.../apache/kylin/rest/service/QueryService.java | 37 ++-
.../kylin/rest/service/QueryServiceTest.java | 278 ++++++++++++++++++++-
.../kylin/query/engine/QueryRoutingEngine.java | 7 +
.../engine/exec/sparder/SparderQueryPlanExec.java | 6 +
.../kylin/query/engine/QueryRoutingEngineTest.java | 34 +++
9 files changed, 557 insertions(+), 4 deletions(-)
diff --git
a/src/core-common/src/main/java/org/apache/kylin/common/QueryContext.java
b/src/core-common/src/main/java/org/apache/kylin/common/QueryContext.java
index 592df87f68..b112591e27 100644
--- a/src/core-common/src/main/java/org/apache/kylin/common/QueryContext.java
+++ b/src/core-common/src/main/java/org/apache/kylin/common/QueryContext.java
@@ -389,6 +389,7 @@ public class QueryContext implements Closeable {
private boolean isRefused;
private boolean includeHeader;
private boolean isVacant;
+ private boolean isQueryDetect;
}
@Getter
diff --git
a/src/query-server/src/main/java/org/apache/kylin/rest/controller/NQueryController.java
b/src/query-server/src/main/java/org/apache/kylin/rest/controller/NQueryController.java
index 6d69ea475f..1f56073647 100644
---
a/src/query-server/src/main/java/org/apache/kylin/rest/controller/NQueryController.java
+++
b/src/query-server/src/main/java/org/apache/kylin/rest/controller/NQueryController.java
@@ -73,12 +73,14 @@ import org.apache.kylin.rest.exception.ForbiddenException;
import org.apache.kylin.rest.exception.InternalErrorException;
import org.apache.kylin.rest.model.Query;
import org.apache.kylin.rest.request.PrepareSqlRequest;
+import org.apache.kylin.rest.request.QueryDetectRequest;
import org.apache.kylin.rest.request.SQLFormatRequest;
import org.apache.kylin.rest.request.SQLRequest;
import org.apache.kylin.rest.request.SaveSqlRequest;
import org.apache.kylin.rest.request.SyncFileSegmentsRequest;
import org.apache.kylin.rest.response.DataResult;
import org.apache.kylin.rest.response.EnvelopeResponse;
+import org.apache.kylin.rest.response.QueryDetectResponse;
import org.apache.kylin.rest.response.QueryHistoryFiltersResponse;
import org.apache.kylin.rest.response.QueryStatisticsResponse;
import org.apache.kylin.rest.response.SQLResponse;
@@ -662,6 +664,13 @@ public class NQueryController extends NBasicController {
return new EnvelopeResponse<>(KylinException.CODE_SUCCESS,
queryService.format(request.getSqls()), "");
}
+ @ApiOperation(value = "queryDetect", tags = { "QE" })
+ @PostMapping("/detection")
+ public EnvelopeResponse<QueryDetectResponse> queryDetect(@RequestBody
QueryDetectRequest request) {
+ checkProjectName(request.getProject());
+ return new EnvelopeResponse<>(KylinException.CODE_SUCCESS,
queryService.queryDetect(request), "");
+ }
+
private void checkQueryName(String queryName) {
if (!queryNamePattern.matcher(queryName).matches()) {
throw new KylinException(INVALID_NAME,
MsgPicker.getMsg().getInvalidQueryName());
diff --git
a/src/query-service/src/main/java/org/apache/kylin/rest/request/QueryDetectRequest.java
b/src/query-service/src/main/java/org/apache/kylin/rest/request/QueryDetectRequest.java
new file mode 100644
index 0000000000..9ee03f4b3f
--- /dev/null
+++
b/src/query-service/src/main/java/org/apache/kylin/rest/request/QueryDetectRequest.java
@@ -0,0 +1,33 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.kylin.rest.request;
+
+import lombok.AllArgsConstructor;
+import lombok.Data;
+import lombok.NoArgsConstructor;
+
+@Data
+@AllArgsConstructor
+@NoArgsConstructor
+public class QueryDetectRequest {
+ private String sql;
+ private String project;
+ private Integer offset = 0;
+ private Integer limit = 500;
+}
diff --git
a/src/query-service/src/main/java/org/apache/kylin/rest/response/QueryDetectResponse.java
b/src/query-service/src/main/java/org/apache/kylin/rest/response/QueryDetectResponse.java
new file mode 100644
index 0000000000..b6548101d9
--- /dev/null
+++
b/src/query-service/src/main/java/org/apache/kylin/rest/response/QueryDetectResponse.java
@@ -0,0 +1,156 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.kylin.rest.response;
+
+import static com.fasterxml.jackson.annotation.JsonAutoDetect.Visibility.NONE;
+
+import java.util.List;
+import java.util.stream.Collectors;
+
+import org.apache.kylin.common.KylinConfig;
+import org.apache.kylin.common.QueryContext;
+import org.apache.kylin.guava30.shaded.common.collect.Lists;
+import org.apache.kylin.metadata.cube.model.IndexEntity;
+import org.apache.kylin.metadata.cube.model.IndexPlan;
+import org.apache.kylin.metadata.cube.model.LayoutEntity;
+import org.apache.kylin.metadata.cube.model.NIndexPlanManager;
+import org.apache.kylin.metadata.query.NativeQueryRealization;
+import org.apache.kylin.metadata.query.QueryHistory;
+
+import com.fasterxml.jackson.annotation.JsonAutoDetect;
+import com.fasterxml.jackson.annotation.JsonProperty;
+
+import lombok.AllArgsConstructor;
+import lombok.Data;
+import lombok.NoArgsConstructor;
+
+@Data
+@AllArgsConstructor
+@NoArgsConstructor
+@JsonAutoDetect(fieldVisibility = NONE, getterVisibility = NONE,
isGetterVisibility = NONE, setterVisibility = NONE)
+public class QueryDetectResponse {
+ @JsonProperty("is_exception")
+ private boolean isException = false;
+
+ @JsonProperty("exception_message")
+ private String exceptionMessage;
+
+ @JsonProperty("query_id")
+ private String queryId;
+
+ @JsonProperty("is_push_down")
+ private boolean isPushDown = false;
+
+ @JsonProperty("is_post_aggregation")
+ private boolean isPostAggregation = false;
+
+ @JsonProperty("is_table_index")
+ private boolean isTableIndex = false;
+
+ @JsonProperty("is_base_index")
+ private boolean isBaseIndex = false;
+
+ @JsonProperty("is_cache")
+ private boolean isCache = false;
+
+ @JsonProperty("is_constants")
+ private boolean isConstants = false;
+
+ @JsonProperty("realizations")
+ private List<IndexInfo> realizations = Lists.newArrayList();
+
+ public QueryDetectResponse buildExceptionResponse(SQLResponse sqlResponse)
{
+ this.isException = true;
+ this.exceptionMessage = sqlResponse.getExceptionMessage();
+ this.queryId = sqlResponse.getQueryId();
+ this.isCache = sqlResponse.isHitExceptionCache();
+ this.realizations = Lists.newArrayList();
+ return this;
+ }
+
+ public QueryDetectResponse buildResponse(String project, SQLResponse
sqlResponse, QueryContext queryContext) {
+ // build RealizationVO
+ List<QueryDetectResponse.IndexInfo> indexInfoList =
sqlResponse.getNativeRealizations().stream()
+ .map(realization -> new
QueryDetectResponse.IndexInfo().buildResponse(realization, project))
+ .collect(Collectors.toList());
+ boolean isConstantQuery =
QueryHistory.EngineType.CONSTANTS.name().equals(sqlResponse.getEngineType());
+
+ this.isException = sqlResponse.isException;
+ this.exceptionMessage = sqlResponse.getExceptionMessage();
+ this.queryId = sqlResponse.getQueryId();
+ this.isPushDown = sqlResponse.isQueryPushDown();
+ this.isPostAggregation = !sqlResponse.isQueryPushDown() &&
!isConstantQuery
+ && !queryContext.getMetrics().isExactlyMatch();
+ // any realization is TableIndexăbaseIndex
+ this.isTableIndex =
indexInfoList.stream().anyMatch(QueryDetectResponse.IndexInfo::isTableIndex);
+ this.isBaseIndex =
indexInfoList.stream().anyMatch(QueryDetectResponse.IndexInfo::isBaseIndex);
+ this.isCache = sqlResponse.isStorageCacheUsed();
+ this.isConstants = isConstantQuery;
+ this.realizations = indexInfoList;
+ return this;
+ }
+
+ @Data
+ @AllArgsConstructor
+ @NoArgsConstructor
+ @JsonAutoDetect(fieldVisibility = NONE, getterVisibility = NONE,
isGetterVisibility = NONE, setterVisibility = NONE)
+ public static class IndexInfo {
+ @JsonProperty("model_id")
+ private String modelId;
+
+ @JsonProperty("model_alias")
+ private String modelAlias;
+
+ @JsonProperty("layout_id")
+ private long layoutId;
+
+ @JsonProperty("index_type")
+ private String indexType;
+
+ @JsonProperty("partial_match_model")
+ private boolean partialMatchModel = false;
+
+ @JsonProperty("valid")
+ private boolean valid = true;
+
+ @JsonProperty("is_table_index")
+ private boolean isTableIndex = false;
+
+ @JsonProperty("is_base_index")
+ private boolean isBaseIndex = false;
+
+ public IndexInfo buildResponse(NativeQueryRealization realization,
String project) {
+ // calculate isBaseIndex
+ KylinConfig kylinConfig = KylinConfig.getInstanceFromEnv();
+ IndexPlan indexPlan = NIndexPlanManager.getInstance(kylinConfig,
project)
+ .getIndexPlan(realization.getModelId());
+ LayoutEntity layoutEntity = indexPlan == null ? null :
indexPlan.getLayoutEntity(realization.getLayoutId());
+
+ this.modelId = realization.getModelId();
+ this.modelAlias = realization.getModelAlias();
+ this.layoutId = realization.getLayoutId();
+ this.indexType = realization.getIndexType();
+ this.partialMatchModel = realization.isPartialMatchModel();
+ this.valid = realization.isValid();
+ this.isTableIndex =
IndexEntity.isTableIndex(realization.getLayoutId());
+ this.isBaseIndex = layoutEntity != null &&
layoutEntity.isBaseIndex();
+ return this;
+ }
+ }
+}
diff --git
a/src/query-service/src/main/java/org/apache/kylin/rest/service/QueryService.java
b/src/query-service/src/main/java/org/apache/kylin/rest/service/QueryService.java
index 7b10e1eb67..04bd71ff81 100644
---
a/src/query-service/src/main/java/org/apache/kylin/rest/service/QueryService.java
+++
b/src/query-service/src/main/java/org/apache/kylin/rest/service/QueryService.java
@@ -136,6 +136,7 @@ import
org.apache.kylin.query.exception.UserStopQueryException;
import org.apache.kylin.query.relnode.ContextUtil;
import org.apache.kylin.query.relnode.OLAPContext;
import org.apache.kylin.query.util.EscapeDialect;
+import org.apache.kylin.query.util.PrepareSQLUtils;
import org.apache.kylin.query.util.QueryLimiter;
import org.apache.kylin.query.util.QueryModelPriorities;
import org.apache.kylin.query.util.QueryParams;
@@ -150,7 +151,9 @@ import org.apache.kylin.rest.cluster.ClusterManager;
import org.apache.kylin.rest.config.AppConfig;
import org.apache.kylin.rest.model.Query;
import org.apache.kylin.rest.request.PrepareSqlRequest;
+import org.apache.kylin.rest.request.QueryDetectRequest;
import org.apache.kylin.rest.request.SQLRequest;
+import org.apache.kylin.rest.response.QueryDetectResponse;
import org.apache.kylin.rest.response.SQLResponse;
import org.apache.kylin.rest.response.SQLResponseTrace;
import org.apache.kylin.rest.response.TableMetaCacheResult;
@@ -158,7 +161,6 @@ import
org.apache.kylin.rest.response.TableMetaCacheResultV2;
import org.apache.kylin.rest.security.MutableAclRecord;
import org.apache.kylin.rest.util.AclEvaluate;
import org.apache.kylin.rest.util.AclPermissionUtil;
-import org.apache.kylin.query.util.PrepareSQLUtils;
import org.apache.kylin.rest.util.QueryCacheSignatureUtil;
import org.apache.kylin.rest.util.QueryRequestLimits;
import org.apache.kylin.rest.util.QueryUtils;
@@ -645,7 +647,8 @@ public class QueryService extends BasicService implements
CacheSignatureQuerySup
sqlResponse = QueryUtils.handleTempStatement(sqlRequest,
kylinConfig);
// search cache
- if (sqlResponse == null && kylinConfig.isQueryCacheEnabled() &&
!sqlRequest.isForcedToPushDown()) {
+ if (sqlResponse == null && kylinConfig.isQueryCacheEnabled() &&
!sqlRequest.isForcedToPushDown()
+ && !queryContext.getQueryTagInfo().isAsyncQuery()) {
sqlResponse = searchCache(sqlRequest, kylinConfig);
}
@@ -754,6 +757,10 @@ public class QueryService extends BasicService implements
CacheSignatureQuerySup
}
private void addToQueryHistory(SQLRequest sqlRequest, SQLResponse
sqlResponse, String originalSql) {
+ if (QueryContext.current().getQueryTagInfo().isQueryDetect()) {
+ return;
+ }
+
if (!(QueryContext.current().getQueryTagInfo().isAsyncQuery()
&&
NProjectManager.getProjectConfig(sqlRequest.getProject()).isUniqueAsyncQueryYarnQueue()))
{
try {
@@ -936,8 +943,10 @@ public class QueryService extends BasicService implements
CacheSignatureQuerySup
}
private boolean isQueryCacheEnabled(KylinConfig kylinConfig) {
+ boolean isNotQueryDetect =
!QueryContext.current().getQueryTagInfo().isQueryDetect();
return checkCondition(kylinConfig.isQueryCacheEnabled(), "query cache
disabled in KylinConfig") //
- && checkCondition(!BackdoorToggles.getDisableCache(), "query
cache disabled in BackdoorToggles");
+ && checkCondition(!BackdoorToggles.getDisableCache(), "query
cache disabled in BackdoorToggles")
+ && checkCondition(isNotQueryDetect, "query cache disabled
because it is query detect");
}
private boolean isQueryExceptionCacheEnabled(KylinConfig kylinConfig) {
@@ -1168,6 +1177,28 @@ public class QueryService extends BasicService
implements CacheSignatureQuerySup
}).sorted(Comparator.comparingInt(Pair::getFirst)).map(Pair::getSecond).collect(Collectors.toList());
}
+ public QueryDetectResponse queryDetect(QueryDetectRequest
queryDetectRequest) {
+ try (QueryContext queryContext = QueryContext.current()) {
+ // call query method to collect realizations, etc
+ queryContext.getQueryTagInfo().setQueryDetect(true);
+ String project = queryDetectRequest.getProject();
+ SQLRequest sqlRequest = new SQLRequest();
+ sqlRequest.setSql(queryDetectRequest.getSql());
+ sqlRequest.setProject(project);
+ sqlRequest.setLimit(queryDetectRequest.getLimit());
+ sqlRequest.setOffset(queryDetectRequest.getOffset());
+ SQLResponse sqlResponse = queryWithCache(sqlRequest);
+
+ // build exception response
+ if (sqlResponse.isException()) {
+ return new
QueryDetectResponse().buildExceptionResponse(sqlResponse);
+ }
+
+ // build normal response
+ return new QueryDetectResponse().buildResponse(project,
sqlResponse, queryContext);
+ }
+ }
+
@SuppressWarnings("checkstyle:methodlength")
private List<TableMetaWithType> doGetMetadataV2(String project, String
targetModelName) {
aclEvaluate.checkProjectQueryPermission(project);
diff --git
a/src/query-service/src/test/java/org/apache/kylin/rest/service/QueryServiceTest.java
b/src/query-service/src/test/java/org/apache/kylin/rest/service/QueryServiceTest.java
index 89fead0d99..b9c5ab91e2 100644
---
a/src/query-service/src/test/java/org/apache/kylin/rest/service/QueryServiceTest.java
+++
b/src/query-service/src/test/java/org/apache/kylin/rest/service/QueryServiceTest.java
@@ -82,11 +82,13 @@ import org.apache.kylin.metadata.acl.AclTCR;
import org.apache.kylin.metadata.acl.AclTCRManager;
import org.apache.kylin.metadata.cube.cuboid.NLayoutCandidate;
import org.apache.kylin.metadata.cube.model.IndexEntity;
+import org.apache.kylin.metadata.cube.model.IndexPlan;
import org.apache.kylin.metadata.cube.model.LayoutEntity;
import org.apache.kylin.metadata.cube.model.NDataSegment;
import org.apache.kylin.metadata.cube.model.NDataflow;
import org.apache.kylin.metadata.cube.model.NDataflowManager;
import org.apache.kylin.metadata.cube.model.NDataflowUpdate;
+import org.apache.kylin.metadata.cube.model.NIndexPlanManager;
import org.apache.kylin.metadata.model.ColumnDesc;
import org.apache.kylin.metadata.model.ComputedColumnDesc;
import org.apache.kylin.metadata.model.NDataModel;
@@ -99,6 +101,7 @@ import org.apache.kylin.metadata.project.NProjectManager;
import org.apache.kylin.metadata.project.ProjectInstance;
import org.apache.kylin.metadata.query.NativeQueryRealization;
import org.apache.kylin.metadata.query.QueryHistory;
+import org.apache.kylin.metadata.query.QueryMetrics;
import org.apache.kylin.metadata.query.QueryMetricsContext;
import org.apache.kylin.metadata.querymeta.ColumnMeta;
import org.apache.kylin.metadata.querymeta.ColumnMetaWithType;
@@ -127,7 +130,9 @@ import org.apache.kylin.rest.constant.Constant;
import org.apache.kylin.rest.exception.InternalErrorException;
import org.apache.kylin.rest.model.Query;
import org.apache.kylin.rest.request.PrepareSqlRequest;
+import org.apache.kylin.rest.request.QueryDetectRequest;
import org.apache.kylin.rest.request.SQLRequest;
+import org.apache.kylin.rest.response.QueryDetectResponse;
import org.apache.kylin.rest.response.SQLResponse;
import org.apache.kylin.rest.security.AclEntityFactory;
import org.apache.kylin.rest.security.AclEntityType;
@@ -169,7 +174,8 @@ import lombok.val;
* @author xduo
*/
@RunWith(PowerMockRunner.class)
-@PrepareForTest({ SpringContext.class, UserGroupInformation.class,
SparkSession.class, QueryService.class })
+@PrepareForTest({ SpringContext.class, UserGroupInformation.class,
SparkSession.class, QueryService.class,
+ NIndexPlanManager.class, QueryContext.class })
@PowerMockIgnore({ "javax.management.*" })
public class QueryServiceTest extends NLocalFileMetadataTestCase {
@@ -2743,4 +2749,274 @@ public class QueryServiceTest extends
NLocalFileMetadataTestCase {
val queryEntry = result.get();
Assert.assertTrue(queryEntry.getPlannerCancelFlag().isCancelRequested());
}
+
+ @Test
+ public void testQueryDetectException() {
+ QueryDetectRequest queryDetectRequest = new QueryDetectRequest("sql",
"default", 0, 500);
+ // build sqlRequest
+ SQLRequest sqlRequest = new SQLRequest();
+ sqlRequest.setSql(queryDetectRequest.getSql());
+ String project = queryDetectRequest.getProject();
+ sqlRequest.setLimit(queryDetectRequest.getLimit());
+ sqlRequest.setOffset(queryDetectRequest.getOffset());
+ sqlRequest.setProject(project);
+ // build sqlResponse
+ SQLResponse sqlResponse = new SQLResponse();
+ sqlResponse.setException(true);
+ sqlResponse.setExceptionMessage("exceptionMessage");
+ sqlResponse.setQueryId("queryId");
+ // build excepted queryDetectVO
+ QueryDetectResponse exceptedQueryDetectResponse = new
QueryDetectResponse().buildExceptionResponse(sqlResponse);
+
+
Mockito.doReturn(sqlResponse).when(queryService).queryWithCache(sqlRequest);
+
+ QueryDetectResponse queryDetectResponse =
queryService.queryDetect(queryDetectRequest);
+
+ Assert.assertEquals(exceptedQueryDetectResponse, queryDetectResponse);
+ }
+
+ @Test
+ public void testQueryDetectPushDown() {
+ QueryDetectRequest queryDetectRequest = new QueryDetectRequest("sql",
"default", 0, 500);
+ // build sqlRequest
+ SQLRequest sqlRequest = new SQLRequest();
+ sqlRequest.setSql(queryDetectRequest.getSql());
+ String project = queryDetectRequest.getProject();
+ sqlRequest.setLimit(queryDetectRequest.getLimit());
+ sqlRequest.setOffset(queryDetectRequest.getOffset());
+ sqlRequest.setProject(project);
+
+ // build sqlResponse
+ SQLResponse sqlResponse = new SQLResponse();
+ sqlResponse.setException(false);
+ sqlResponse.setQueryId("queryId");
+ sqlResponse.setQueryPushDown(true);
+ List<NativeQueryRealization> nativeRealizations = Lists.newArrayList();
+ sqlResponse.setNativeRealizations(nativeRealizations);
+
+ // build excepted queryDetectVO
+ QueryDetectResponse exceptedQueryDetectResponse = new
QueryDetectResponse().buildResponse(project, sqlResponse,
+ QueryContext.current());
+
+
Mockito.doReturn(sqlResponse).when(queryService).queryWithCache(sqlRequest);
+
+ QueryDetectResponse queryDetectResponse =
queryService.queryDetect(queryDetectRequest);
+
+ Assert.assertEquals(exceptedQueryDetectResponse, queryDetectResponse);
+ }
+
+ @Test
+ public void testQueryDetectAggIndex() {
+ QueryDetectRequest queryDetectRequest = new QueryDetectRequest("sql",
"default", 0, 500);
+ // build sqlRequest
+ SQLRequest sqlRequest = new SQLRequest();
+ sqlRequest.setSql(queryDetectRequest.getSql());
+ String project = queryDetectRequest.getProject();
+ sqlRequest.setLimit(queryDetectRequest.getLimit());
+ sqlRequest.setOffset(queryDetectRequest.getOffset());
+ sqlRequest.setProject(project);
+
+ // build sqlResponse
+ SQLResponse sqlResponse = new SQLResponse();
+ sqlResponse.setException(false);
+ sqlResponse.setQueryId("queryId");
+ sqlResponse.setQueryPushDown(false);
+ NativeQueryRealization realization = new NativeQueryRealization();
+ realization.setModelId("modelId1");
+ realization.setLayoutId(1L);
+ realization.setModelAlias("modelAlias1");
+ realization.setIndexType(QueryMetrics.AGG_INDEX);
+ realization.setPartialMatchModel(false);
+ realization.setValid(true);
+ List<NativeQueryRealization> nativeRealizations =
Lists.newArrayList(realization);
+ sqlResponse.setNativeRealizations(nativeRealizations);
+
+ // mock NIndexPlanManager and IndexPlan
+ PowerMockito.mockStatic(NIndexPlanManager.class);
+ NIndexPlanManager nIndexPlanManager =
Mockito.mock(NIndexPlanManager.class);
+ IndexPlan indexPlan = Mockito.mock(IndexPlan.class);
+ LayoutEntity layoutEntity = new LayoutEntity();
+ layoutEntity.setBase(true);
+
+ // mock method return value
+ Mockito.when(NIndexPlanManager.getInstance(Mockito.any(),
Mockito.any())).thenReturn(nIndexPlanManager);
+
Mockito.when(nIndexPlanManager.getIndexPlan(realization.getModelId())).thenReturn(indexPlan);
+
Mockito.when(indexPlan.getLayoutEntity(realization.getLayoutId())).thenReturn(layoutEntity);
+
Mockito.doReturn(sqlResponse).when(queryService).queryWithCache(sqlRequest);
+
+ // build excepted queryDetectVO
+ QueryDetectResponse exceptedQueryDetectResponse = new
QueryDetectResponse().buildResponse(project, sqlResponse,
+ QueryContext.current());
+
+ QueryDetectResponse queryDetectResponse =
queryService.queryDetect(queryDetectRequest);
+
+ Assert.assertEquals(exceptedQueryDetectResponse, queryDetectResponse);
+ }
+
+ @Test
+ public void testQueryDetectWhenIsConstants() {
+ QueryDetectRequest queryDetectRequest = new QueryDetectRequest("sql",
"default", 0, 500);
+ // build sqlRequest
+ SQLRequest sqlRequest = new SQLRequest();
+ sqlRequest.setSql(queryDetectRequest.getSql());
+ String project = queryDetectRequest.getProject();
+ sqlRequest.setLimit(queryDetectRequest.getLimit());
+ sqlRequest.setOffset(queryDetectRequest.getOffset());
+ sqlRequest.setProject(project);
+
+ // build sqlResponse
+ SQLResponse sqlResponse = new SQLResponse();
+ sqlResponse.setException(false);
+ sqlResponse.setQueryId("queryId");
+ sqlResponse.setQueryPushDown(false);
+ List<NativeQueryRealization> nativeRealizations = Lists.newArrayList();
+ sqlResponse.setNativeRealizations(nativeRealizations);
+ sqlResponse.setEngineType(QueryHistory.EngineType.CONSTANTS.name());
+
+ // mock method return value
+
Mockito.doReturn(sqlResponse).when(queryService).queryWithCache(sqlRequest);
+
+ // build excepted queryDetectVO
+ QueryDetectResponse exceptedQueryDetectResponse = new
QueryDetectResponse().buildResponse(project, sqlResponse,
+ QueryContext.current());
+
+ QueryDetectResponse queryDetectResponse =
queryService.queryDetect(queryDetectRequest);
+
+ Assert.assertEquals(exceptedQueryDetectResponse, queryDetectResponse);
+ }
+
+ @Test
+ public void testQueryDetectWhenIsSnapshot() {
+ QueryDetectRequest queryDetectRequest = new QueryDetectRequest("sql",
"default", 0, 500);
+ // build sqlRequest
+ SQLRequest sqlRequest = new SQLRequest();
+ sqlRequest.setSql(queryDetectRequest.getSql());
+ String project = queryDetectRequest.getProject();
+ sqlRequest.setLimit(queryDetectRequest.getLimit());
+ sqlRequest.setOffset(queryDetectRequest.getOffset());
+ sqlRequest.setProject(project);
+
+ // build sqlResponse
+ SQLResponse sqlResponse = new SQLResponse();
+ sqlResponse.setException(false);
+ sqlResponse.setQueryId("queryId");
+ sqlResponse.setQueryPushDown(false);
+ NativeQueryRealization realization = new NativeQueryRealization();
+ realization.setModelId("modelId1");
+ realization.setLayoutId(-1L);
+ realization.setModelAlias("modelAlias1");
+ realization.setIndexType(QueryMetrics.AGG_INDEX);
+ realization.setPartialMatchModel(false);
+ realization.setValid(true);
+ List<NativeQueryRealization> nativeRealizations =
Lists.newArrayList(realization);
+ sqlResponse.setNativeRealizations(nativeRealizations);
+
+ // mock NIndexPlanManager and IndexPlan
+ PowerMockito.mockStatic(NIndexPlanManager.class);
+ NIndexPlanManager nIndexPlanManager =
Mockito.mock(NIndexPlanManager.class);
+ IndexPlan indexPlan = Mockito.mock(IndexPlan.class);
+
+ // mock method return value
+ Mockito.when(NIndexPlanManager.getInstance(Mockito.any(),
Mockito.any())).thenReturn(nIndexPlanManager);
+
Mockito.when(nIndexPlanManager.getIndexPlan(realization.getModelId())).thenReturn(indexPlan);
+
Mockito.when(indexPlan.getLayoutEntity(realization.getLayoutId())).thenReturn(null);
+
Mockito.doReturn(sqlResponse).when(queryService).queryWithCache(sqlRequest);
+
+ // build excepted queryDetectVO
+ QueryDetectResponse exceptedQueryDetectResponse = new
QueryDetectResponse().buildResponse(project, sqlResponse,
+ QueryContext.current());
+
+ QueryDetectResponse queryDetectResponse =
queryService.queryDetect(queryDetectRequest);
+
+ Assert.assertEquals(exceptedQueryDetectResponse, queryDetectResponse);
+ }
+
+ @Test
+ public void testQueryDetectWhenIndexPlanIsNull() {
+ QueryDetectRequest queryDetectRequest = new QueryDetectRequest("sql",
"default", 0, 500);
+ // build sqlRequest
+ SQLRequest sqlRequest = new SQLRequest();
+ sqlRequest.setSql(queryDetectRequest.getSql());
+ String project = queryDetectRequest.getProject();
+ sqlRequest.setLimit(queryDetectRequest.getLimit());
+ sqlRequest.setOffset(queryDetectRequest.getOffset());
+ sqlRequest.setProject(project);
+
+ // build sqlResponse
+ SQLResponse sqlResponse = new SQLResponse();
+ sqlResponse.setException(false);
+ sqlResponse.setQueryId("queryId");
+ sqlResponse.setQueryPushDown(false);
+ NativeQueryRealization realization = new NativeQueryRealization();
+ realization.setModelId("modelId1");
+ realization.setLayoutId(-1L);
+ realization.setModelAlias("modelAlias1");
+ realization.setIndexType(QueryMetrics.AGG_INDEX);
+ realization.setPartialMatchModel(false);
+ realization.setValid(true);
+ List<NativeQueryRealization> nativeRealizations =
Lists.newArrayList(realization);
+ sqlResponse.setNativeRealizations(nativeRealizations);
+
+ // mock NIndexPlanManager and IndexPlan
+ PowerMockito.mockStatic(NIndexPlanManager.class);
+ NIndexPlanManager nIndexPlanManager =
Mockito.mock(NIndexPlanManager.class);
+
+ // mock method return value
+ Mockito.when(NIndexPlanManager.getInstance(Mockito.any(),
Mockito.any())).thenReturn(nIndexPlanManager);
+
Mockito.when(nIndexPlanManager.getIndexPlan(realization.getModelId())).thenReturn(null);
+
Mockito.doReturn(sqlResponse).when(queryService).queryWithCache(sqlRequest);
+
+ // build excepted queryDetectVO
+ QueryDetectResponse exceptedQueryDetectResponse = new
QueryDetectResponse().buildResponse(project, sqlResponse,
+ QueryContext.current());
+
+ QueryDetectResponse queryDetectResponse =
queryService.queryDetect(queryDetectRequest);
+
+ Assert.assertEquals(exceptedQueryDetectResponse, queryDetectResponse);
+ }
+
+ @Test
+ public void testAddToQueryHistoryIsQueryDetect() throws Exception {
+ try (QueryContext queryContext = QueryContext.current()) {
+ queryContext.getQueryTagInfo().setQueryDetect(true);
+ Method method =
QueryService.class.getDeclaredMethod("addToQueryHistory", SQLRequest.class,
+ SQLResponse.class, String.class);
+ method.setAccessible(true);
+
+ // check void method is ok
+ method.invoke(queryService, null, null, null);
+ }
+ }
+
+ @Test
+ public void testIsQueryCacheEnabledIsQueryDetect() throws Exception {
+ try (QueryContext queryContext = QueryContext.current()) {
+ queryContext.getQueryTagInfo().setQueryDetect(true);
+ Method method =
QueryService.class.getDeclaredMethod("isQueryCacheEnabled", KylinConfig.class);
+ method.setAccessible(true);
+ KylinConfig kylinConfig = KylinConfig.getInstanceFromEnv();
+ boolean isQueryCacheEnabled = (boolean)
method.invoke(queryService, kylinConfig);
+
+ Assert.assertFalse(isQueryCacheEnabled);
+ }
+ }
+
+ @Test
+ public void testNotHitCacheWhenIsAsyncQuery() {
+ try (QueryContext queryContext = QueryContext.current()) {
+ queryContext.getQueryTagInfo().setAsyncQuery(true);
+ String project = "default";
+ String sql = "select * from table";
+ SQLRequest sqlRequest = new SQLRequest();
+ sqlRequest.setProject(project);
+ sqlRequest.setSql(sql);
+
+ queryService.doQueryWithCache(sqlRequest);
+
+ // async query API can not search cache
+
Assert.assertFalse(queryContext.getQueryTagInfo().isStorageCacheUsed());
+
Assert.assertFalse(queryContext.getQueryTagInfo().isHitExceptionCache());
+ Assert.assertTrue(queryContext.getQueryTagInfo().isAsyncQuery());
+ }
+ }
}
diff --git
a/src/query/src/main/java/org/apache/kylin/query/engine/QueryRoutingEngine.java
b/src/query/src/main/java/org/apache/kylin/query/engine/QueryRoutingEngine.java
index 0e3f86b399..f1e5cf9054 100644
---
a/src/query/src/main/java/org/apache/kylin/query/engine/QueryRoutingEngine.java
+++
b/src/query/src/main/java/org/apache/kylin/query/engine/QueryRoutingEngine.java
@@ -54,6 +54,7 @@ import org.apache.kylin.metadata.project.NProjectManager;
import org.apache.kylin.metadata.query.NativeQueryRealization;
import org.apache.kylin.metadata.query.StructField;
import org.apache.kylin.metadata.querymeta.SelectedColumnMeta;
+import org.apache.kylin.metadata.realization.NoRealizationFoundException;
import
org.apache.kylin.metadata.realization.NoStreamingRealizationFoundException;
import org.apache.kylin.query.engine.data.QueryResult;
import org.apache.kylin.query.exception.BusyQueryException;
@@ -260,6 +261,12 @@ public class QueryRoutingEngine {
private QueryResult pushDownQuery(SQLException sqlException, QueryParams
queryParams) throws SQLException {
QueryContext.current().getMetrics().setOlapCause(sqlException);
QueryContext.current().getQueryTagInfo().setPushdown(true);
+ if (QueryContext.current().getQueryTagInfo().isQueryDetect()) {
+ if (sqlException.getCause() instanceof
NoRealizationFoundException) {
+ return new QueryResult();
+ }
+ throw sqlException;
+ }
PushdownResult result = null;
try {
result = tryPushDownSelectQuery(queryParams, sqlException,
BackdoorToggles.getPrepareOnly());
diff --git
a/src/query/src/main/java/org/apache/kylin/query/engine/exec/sparder/SparderQueryPlanExec.java
b/src/query/src/main/java/org/apache/kylin/query/engine/exec/sparder/SparderQueryPlanExec.java
index 988eca606a..07c18197eb 100644
---
a/src/query/src/main/java/org/apache/kylin/query/engine/exec/sparder/SparderQueryPlanExec.java
+++
b/src/query/src/main/java/org/apache/kylin/query/engine/exec/sparder/SparderQueryPlanExec.java
@@ -76,6 +76,7 @@ public class SparderQueryPlanExec implements QueryPlanExec {
QueryContextCutter.selectRealization(QueryContext.current().getProject(), rel,
BackdoorToggles.getIsQueryFromAutoModeling());
+
String msg = "EXECUTION PLAN AFTER (SparderQueryPlanExec) SELECT
REALIZATION IS SET";
ContextUtil.dumpCalcitePlan(msg, rel, log);
@@ -107,6 +108,11 @@ public class SparderQueryPlanExec implements QueryPlanExec
{
// rewrite
rewrite(rel);
+ // query detect
+ if (QueryContext.current().getQueryTagInfo().isQueryDetect()) {
+ return new ExecuteResult(Lists.newArrayList(), 0);
+ }
+
// submit rel and dataContext to query engine
return internalCompute(new SparkEngine(), dataContext,
rel.getInput(0));
}
diff --git
a/src/query/src/test/java/org/apache/kylin/query/engine/QueryRoutingEngineTest.java
b/src/query/src/test/java/org/apache/kylin/query/engine/QueryRoutingEngineTest.java
index dd50f2388f..fa929a34b2 100644
---
a/src/query/src/test/java/org/apache/kylin/query/engine/QueryRoutingEngineTest.java
+++
b/src/query/src/test/java/org/apache/kylin/query/engine/QueryRoutingEngineTest.java
@@ -21,6 +21,7 @@ package org.apache.kylin.query.engine;
import static
org.apache.kylin.query.engine.QueryRoutingEngine.SPARK_JOB_FAILED;
import static
org.apache.kylin.query.engine.QueryRoutingEngine.SPARK_MEM_LIMIT_EXCEEDED;
+import java.lang.reflect.Method;
import java.sql.Date;
import java.sql.SQLException;
import java.sql.Time;
@@ -43,6 +44,7 @@ import org.apache.kylin.common.persistence.InMemResourceStore;
import org.apache.kylin.common.persistence.ResourceStore;
import org.apache.kylin.common.persistence.transaction.TransactionException;
import org.apache.kylin.common.util.NLocalFileMetadataTestCase;
+import org.apache.kylin.metadata.realization.NoRealizationFoundException;
import
org.apache.kylin.metadata.realization.NoStreamingRealizationFoundException;
import org.apache.kylin.query.QueryExtension;
import org.apache.kylin.query.engine.data.QueryResult;
@@ -485,4 +487,36 @@ public class QueryRoutingEngineTest extends
NLocalFileMetadataTestCase {
slowQueryDetector.queryEnd();
}
}
+
+ @Test
+ public void testPushDownQueryWhenIsQueryDetect() throws Exception {
+ try (QueryContext queryContext = QueryContext.current()) {
+ queryContext.getQueryTagInfo().setQueryDetect(true);
+ Method method =
QueryRoutingEngine.class.getDeclaredMethod("pushDownQuery", SQLException.class,
+ QueryParams.class);
+ method.setAccessible(true);
+
+ try {
+ method.invoke(queryRoutingEngine, new SQLException(), null);
+ Assert.fail();
+ } catch (Exception e) {
+ Assert.assertTrue(e.getCause() instanceof SQLException);
+
Assert.assertTrue(QueryContext.current().getQueryTagInfo().isPushdown());
+ }
+ }
+ }
+
+ @Test
+ public void testPushDownQueryWhenIsQueryDetectAndPushDown() throws
Exception {
+ try (QueryContext queryContext = QueryContext.current()) {
+ queryContext.getQueryTagInfo().setQueryDetect(true);
+ Method method =
QueryRoutingEngine.class.getDeclaredMethod("pushDownQuery", SQLException.class,
+ QueryParams.class);
+ method.setAccessible(true);
+
+ Object result = method.invoke(queryRoutingEngine, new
SQLException(new NoRealizationFoundException("")),
+ null);
+ Assert.assertNotNull(result);
+ }
+ }
}
