This is an automated email from the ASF dual-hosted git repository.
liyang pushed a commit to branch kylin5
in repository https://gitbox.apache.org/repos/asf/kylin.git
commit d977ce7bfc9a244d7dbb568ba75a58ab29648908
Author: Hang Jia <754332...@qq.com>
AuthorDate: Wed Jul 5 18:09:55 2023 +0800
KYLIN-5757 No way to deselect query objects after selecting all of them
---
.../metadata/query/JdbcQueryHistoryStore.java | 17 ++++--
.../kylin/metadata/query/QueryHistoryRequest.java | 2 +
.../metadata/query/RDBMSQueryHistoryDaoTest.java | 37 +++++++++++++
.../kylin/rest/controller/NQueryController.java | 12 +++--
.../rest/controller/NQueryControllerTest.java | 2 +-
.../kylin/rest/service/QueryHistoryService.java | 8 +++
.../rest/service/QueryHistoryServiceTest.java | 61 ++++++++++++++++++++++
7 files changed, 132 insertions(+), 7 deletions(-)
diff --git
a/src/core-metadata/src/main/java/org/apache/kylin/metadata/query/JdbcQueryHistoryStore.java
b/src/core-metadata/src/main/java/org/apache/kylin/metadata/query/JdbcQueryHistoryStore.java
index 6a696c0fdd..da5b152073 100644
---
a/src/core-metadata/src/main/java/org/apache/kylin/metadata/query/JdbcQueryHistoryStore.java
+++
b/src/core-metadata/src/main/java/org/apache/kylin/metadata/query/JdbcQueryHistoryStore.java
@@ -29,6 +29,7 @@ import static
org.mybatis.dynamic.sql.SqlBuilder.isLessThanOrEqualTo;
import static org.mybatis.dynamic.sql.SqlBuilder.isLike;
import static org.mybatis.dynamic.sql.SqlBuilder.isLikeCaseInsensitive;
import static org.mybatis.dynamic.sql.SqlBuilder.isNotEqualTo;
+import static org.mybatis.dynamic.sql.SqlBuilder.isNotIn;
import static org.mybatis.dynamic.sql.SqlBuilder.max;
import static org.mybatis.dynamic.sql.SqlBuilder.or;
import static org.mybatis.dynamic.sql.SqlBuilder.select;
@@ -50,6 +51,7 @@ import java.util.stream.Collectors;
import javax.sql.DataSource;
+import org.apache.commons.collections.CollectionUtils;
import org.apache.commons.lang3.StringUtils;
import org.apache.ibatis.jdbc.ScriptRunner;
import org.apache.ibatis.session.ExecutorType;
@@ -743,9 +745,18 @@ public class JdbcQueryHistoryStore {
filterSql = filterSql.and(queryHistoryTable.indexHit,
isEqualTo(false));
}
} else if (selectAllModels) {
- // Process CONSTANTS, HIVE, RDBMS and all model
- filterSql = filterSql.and(queryHistoryTable.engineType,
isIn(realizations),
- or(queryHistoryTable.indexHit, isEqualTo(true)));
+ if (!CollectionUtils.isEmpty(request.getExcludeFilterModelIds())) {
+ // Process CONSTANTS, HIVE, RDBMS and not in model1, model2,
model3...
+ filterSql = filterSql.and(queryHistoryTable.engineType,
isIn(realizations),
+ or(queryHistoryTable.queryId,
+
isIn(selectDistinct(queryHistoryRealizationTable.queryId)
+
.from(queryHistoryRealizationTable).where(queryHistoryRealizationTable.model,
+
isNotIn(request.getExcludeFilterModelIds())))));
+ } else {
+ // Process CONSTANTS, HIVE, RDBMS and all model
+ filterSql = filterSql.and(queryHistoryTable.engineType,
isIn(realizations),
+ or(queryHistoryTable.indexHit, isEqualTo(true)));
+ }
} else if (request.getFilterModelIds() != null &&
!request.getFilterModelIds().isEmpty()) {
// Process CONSTANTS, HIVE, RDBMS and model1, model2, model3...
filterSql = filterSql.and(queryHistoryTable.engineType,
isIn(realizations),
diff --git
a/src/core-metadata/src/main/java/org/apache/kylin/metadata/query/QueryHistoryRequest.java
b/src/core-metadata/src/main/java/org/apache/kylin/metadata/query/QueryHistoryRequest.java
index d406f0864d..a5e10c3e07 100644
---
a/src/core-metadata/src/main/java/org/apache/kylin/metadata/query/QueryHistoryRequest.java
+++
b/src/core-metadata/src/main/java/org/apache/kylin/metadata/query/QueryHistoryRequest.java
@@ -52,6 +52,8 @@ public class QueryHistoryRequest {
private List<String> queryStatus;
List<String> realizations;
+ List<String> excludeRealization;
+ private List<String> excludeFilterModelIds;
public QueryHistoryRequest(String project, String startTimeFrom, String
startTimeTo) {
this.project = project;
diff --git
a/src/core-metadata/src/test/java/org/apache/kylin/metadata/query/RDBMSQueryHistoryDaoTest.java
b/src/core-metadata/src/test/java/org/apache/kylin/metadata/query/RDBMSQueryHistoryDaoTest.java
index b30e13d383..2651f64b81 100644
---
a/src/core-metadata/src/test/java/org/apache/kylin/metadata/query/RDBMSQueryHistoryDaoTest.java
+++
b/src/core-metadata/src/test/java/org/apache/kylin/metadata/query/RDBMSQueryHistoryDaoTest.java
@@ -782,6 +782,43 @@ public class RDBMSQueryHistoryDaoTest extends
NLocalFileMetadataTestCase {
Assert.assertEquals(0, statistics.getMeanDuration(), 0.1);
}
+ @Test
+ public void testGetQueryHistoriesFilterExcludeRealization() throws
Exception {
+ QueryMetrics queryMetrics1 = createQueryMetrics(1580311512000L, 1L,
true, PROJECT, true);
+ queryMetrics1.setSubmitter(ADMIN);
+ queryMetrics1.setEngineType("RDBMS");
+ QueryMetrics queryMetrics2 = createQueryMetrics(1580397912000L, 2L,
false, PROJECT, true);
+ queryMetrics2.setSubmitter(NORMAL_USER);
+ queryMetrics2.setEngineType("HIVE");
+
+ QueryMetrics queryMetrics3 = createQueryMetrics(1580484312000L, 3L,
true, PROJECT, true);
+ queryMetrics3.setSubmitter(NORMAL_USER);
+
+ QueryMetrics queryMetrics4 = createQueryMetrics(1895930712000L, 1L,
false, PROJECT, true);
+ queryMetrics4.setSubmitter(NORMAL_USER);
+ queryMetrics4.setEngineType("CONSTANTS");
+ queryHistoryDAO.insert(queryMetrics1);
+ queryHistoryDAO.insert(queryMetrics2);
+ queryHistoryDAO.insert(queryMetrics3);
+ queryHistoryDAO.insert(queryMetrics4);
+
+ QueryHistoryRequest queryHistoryRequest = new QueryHistoryRequest();
+ queryHistoryRequest.setProject(PROJECT);
+ queryHistoryRequest.setAdmin(true);
+ queryHistoryRequest.setUsername(ADMIN);
+ queryHistoryRequest.setRealizations(Lists.newArrayList("RDBMS",
"HIVE", "CONSTANTS", "modelName"));
+
queryHistoryRequest.setExcludeRealization(Lists.newArrayList("ut_inner_join_cube_partial"));
+
queryHistoryRequest.setExcludeFilterModelIds(Lists.newArrayList("82fa7671-a935-45f5-8779-85703601f49a.json"));
+ List<QueryHistory> queryHistoryList =
queryHistoryDAO.getQueryHistoriesByConditions(queryHistoryRequest, 10, 0);
+ Assert.assertEquals(3, queryHistoryList.size());
+
+ queryHistoryRequest.setExcludeRealization(null);
+ queryHistoryRequest.setExcludeFilterModelIds(null);
+ List<QueryHistory> queryHistoryList2 =
queryHistoryDAO.getQueryHistoriesByConditions(queryHistoryRequest, 10,
+ 0);
+ Assert.assertEquals(4, queryHistoryList2.size());
+ }
+
@Test
public void testGetQueryDailyStatistic() {
// 2022-05-13 10:00:00
diff --git
a/src/query-server/src/main/java/org/apache/kylin/rest/controller/NQueryController.java
b/src/query-server/src/main/java/org/apache/kylin/rest/controller/NQueryController.java
index dc5959aba7..d6ca97cecc 100644
---
a/src/query-server/src/main/java/org/apache/kylin/rest/controller/NQueryController.java
+++
b/src/query-server/src/main/java/org/apache/kylin/rest/controller/NQueryController.java
@@ -286,6 +286,7 @@ public class NQueryController extends NBasicController {
@RequestParam(value = "query_status", required = false)
List<String> queryStatus,
@RequestParam(value = "sql", required = false) String sql,
@RequestParam(value = "realization", required = false)
List<String> realizations,
+ @RequestParam(value = "exclude_realization", required = false)
List<String> excludeRealization,
@RequestParam(value = "server", required = false) String server,
@RequestParam(value = "submitter", required = false) List<String>
submitter, HttpServletResponse response) {
ZoneOffset zoneOffset;
@@ -300,7 +301,8 @@ public class NQueryController extends NBasicController {
}
checkProjectName(project);
QueryHistoryRequest request = new QueryHistoryRequest(project,
startTimeFrom, startTimeTo, latencyFrom,
- latencyTo, sql, server, submitter, null, null, queryStatus,
realizations, false, null, true);
+ latencyTo, sql, server, submitter, null, null, queryStatus,
realizations, excludeRealization, null,
+ false, null, true);
checkGetQueryHistoriesParam(request);
response.setContentType("text/csv;charset=UTF-8");
response.setCharacterEncoding("UTF-8");
@@ -328,11 +330,13 @@ public class NQueryController extends NBasicController {
@RequestParam(value = "query_status", required = false)
List<String> queryStatus,
@RequestParam(value = "sql", required = false) String sql,
@RequestParam(value = "realization", required = false)
List<String> realizations,
+ @RequestParam(value = "exclude_realization", required = false)
List<String> excludeRealization,
@RequestParam(value = "server", required = false) String server,
@RequestParam(value = "submitter", required = false) List<String>
submitter, HttpServletResponse response) {
checkProjectName(project);
QueryHistoryRequest request = new QueryHistoryRequest(project,
startTimeFrom, startTimeTo, latencyFrom,
- latencyTo, sql, server, submitter, null, null, queryStatus,
realizations, false, null, true);
+ latencyTo, sql, server, submitter, null, null, queryStatus,
realizations, excludeRealization, null,
+ false, null, true);
checkGetQueryHistoriesParam(request);
response.setContentType("text/csv;charset=UTF-8");
response.setCharacterEncoding("UTF-8");
@@ -360,13 +364,15 @@ public class NQueryController extends NBasicController {
@RequestParam(value = "query_status", required = false)
List<String> queryStatus,
@RequestParam(value = "sql", required = false) String sql,
@RequestParam(value = "realization", required = false)
List<String> realizations,
+ @RequestParam(value = "exclude_realization", required = false)
List<String> excludeRealization,
@RequestParam(value = "server", required = false) String server,
@RequestParam(value = "offset", required = false, defaultValue =
"0") Integer offset,
@RequestParam(value = "limit", required = false, defaultValue =
"10") Integer limit,
@RequestParam(value = "submitter", required = false) List<String>
submitter) {
checkProjectName(project);
QueryHistoryRequest request = new QueryHistoryRequest(project,
startTimeFrom, startTimeTo, latencyFrom,
- latencyTo, sql, server, submitter, null, null, queryStatus,
realizations, false, null, true);
+ latencyTo, sql, server, submitter, null, null, queryStatus,
realizations, excludeRealization, null,
+ false, null, true);
checkGetQueryHistoriesParam(request);
return new EnvelopeResponse<>(KylinException.CODE_SUCCESS,
QueryHisTransformStandardUtil.transformQueryHistorySqlForDisplay(
diff --git
a/src/query-server/src/test/java/org/apache/kylin/rest/controller/NQueryControllerTest.java
b/src/query-server/src/test/java/org/apache/kylin/rest/controller/NQueryControllerTest.java
index 3f9faf7f5c..51cef2da6f 100644
---
a/src/query-server/src/test/java/org/apache/kylin/rest/controller/NQueryControllerTest.java
+++
b/src/query-server/src/test/java/org/apache/kylin/rest/controller/NQueryControllerTest.java
@@ -498,7 +498,7 @@ public class NQueryControllerTest extends
NLocalFileMetadataTestCase {
Mockito.verify(nQueryController).getQueryHistories(PROJECT,
request.getStartTimeFrom(),
request.getStartTimeTo(), request.getLatencyFrom(),
request.getLatencyTo(), request.getQueryStatus(),
- null, null, null, 2, 3, null);
+ null, null, null, null, 2, 3, null);
// check args
mockMvc.perform(MockMvcRequestBuilders.get("/api/query/history_queries").contentType(MediaType.APPLICATION_JSON)
diff --git
a/src/query-service/src/main/java/org/apache/kylin/rest/service/QueryHistoryService.java
b/src/query-service/src/main/java/org/apache/kylin/rest/service/QueryHistoryService.java
index 339b742aa4..929bc5228d 100644
---
a/src/query-service/src/main/java/org/apache/kylin/rest/service/QueryHistoryService.java
+++
b/src/query-service/src/main/java/org/apache/kylin/rest/service/QueryHistoryService.java
@@ -40,6 +40,7 @@ import java.util.stream.Collectors;
import javax.servlet.http.HttpServletResponse;
+import org.apache.commons.collections.CollectionUtils;
import org.apache.commons.lang3.StringUtils;
import org.apache.commons.lang3.time.DateUtils;
import org.apache.kylin.common.KylinConfig;
@@ -241,6 +242,13 @@ public class QueryHistoryService extends BasicService
implements AsyncTaskQueryH
request.setFilterModelIds(modelNames.stream().filter(modelAliasMap::containsKey).map(modelAliasMap::get)
.collect(Collectors.toList()));
}
+
+ if (realizations != null && realizations.contains("modelName")
+ && !CollectionUtils.isEmpty(request.getExcludeRealization())) {
+ List<String> excludeModelNames =
Lists.newArrayList(request.getExcludeRealization());
+
request.setExcludeFilterModelIds(excludeModelNames.stream().filter(modelAliasMap::containsKey)
+ .map(modelAliasMap::get).collect(Collectors.toList()));
+ }
}
public List<String> getQueryHistoryUsernames(QueryHistoryRequest request,
int size) {
diff --git
a/src/query-service/src/test/java/org/apache/kylin/rest/service/QueryHistoryServiceTest.java
b/src/query-service/src/test/java/org/apache/kylin/rest/service/QueryHistoryServiceTest.java
index 3f2f3cf692..5cd5ff6655 100644
---
a/src/query-service/src/test/java/org/apache/kylin/rest/service/QueryHistoryServiceTest.java
+++
b/src/query-service/src/test/java/org/apache/kylin/rest/service/QueryHistoryServiceTest.java
@@ -598,6 +598,67 @@ public class QueryHistoryServiceTest extends
NLocalFileMetadataTestCase {
Assert.assertEquals("TEST", queryHistories.get(2).getQuerySubmitter());
}
+ @Test
+ public void testGetQueryHistoryFilterExcludeRealization() {
+ QueryHistoryRequest request = new QueryHistoryRequest();
+ request.setProject(PROJECT);
+ // set default values
+ request.setStartTimeFrom("0");
+ request.setStartTimeTo(String.valueOf(Long.MAX_VALUE));
+ request.setLatencyFrom("0");
+ request.setLatencyTo(String.valueOf(Integer.MAX_VALUE));
+ request.setRealizations(Lists.newArrayList("HIVE", "CONSTANTS",
"modelName"));
+
request.setExcludeRealization(Lists.newArrayList("nmodel_basic_inner"));
+ request.setSubmitterExactlyMatch(true);
+
+ // mock query histories
+ QueryHistory queryHistory = new QueryHistory();
+ queryHistory.setSql("select * from test_table_1");
+ queryHistory.setEngineType("NATIVE");
+ queryHistory.setQuerySubmitter("TEST");
+ QueryMetrics.RealizationMetrics noSnapshotMetrics1 = new
QueryMetrics.RealizationMetrics("1", "Agg Index",
+ "741ca86a-1f13-46da-a59f-95fb68615e3a", Lists.newArrayList(new
String[] {}));
+ QueryHistoryInfo queryHistoryInfo = new QueryHistoryInfo();
+
queryHistoryInfo.setRealizationMetrics(Lists.newArrayList(noSnapshotMetrics1));
+ queryHistory.setQueryHistoryInfo(queryHistoryInfo);
+
+ QueryHistory queryHistory1 = new QueryHistory();
+ queryHistory1.setSql("select * from test_table_3");
+ queryHistory1.setEngineType("HIVE");
+ queryHistory1.setQuerySubmitter("TEST");
+
+ QueryHistory queryHistory2 = new QueryHistory();
+ queryHistory2.setSql("select * from test_table_3");
+ queryHistory2.setEngineType("CONSTANTS");
+ queryHistory2.setQuerySubmitter("TEST");
+
+ QueryHistory queryHistory3 = new QueryHistory();
+ queryHistory3.setSql("select * from test_table_2");
+ queryHistory3.setEngineType("NATIVE");
+ queryHistory3.setQuerySubmitter("TEST2");
+ QueryMetrics.RealizationMetrics noSnapshotMetrics2 = new
QueryMetrics.RealizationMetrics("1", "Agg Index",
+ "89af4ee2-2cdb-4b07-b39e-4c29856309aa",
Lists.newArrayList("test_snapshot"));
+ QueryHistoryInfo queryHistoryInfo2 = new QueryHistoryInfo();
+
queryHistoryInfo2.setRealizationMetrics(Lists.newArrayList(noSnapshotMetrics2));
+ queryHistory3.setQueryHistoryInfo(queryHistoryInfo2);
+
+ RDBMSQueryHistoryDAO queryHistoryDAO =
Mockito.mock(RDBMSQueryHistoryDAO.class);
+ Mockito.doReturn(Lists.newArrayList(queryHistory3, queryHistory1,
queryHistory2)).when(queryHistoryDAO)
+ .getQueryHistoriesByConditions(Mockito.any(),
Mockito.anyInt(), Mockito.anyInt());
+
Mockito.doReturn(10L).when(queryHistoryDAO).getQueryHistoriesSize(Mockito.any(),
Mockito.anyString());
+
Mockito.doReturn(queryHistoryDAO).when(queryHistoryService).getQueryHistoryDao();
+
+ Map<String, Object> result =
queryHistoryService.getQueryHistories(request, 10, 0);
+ List<QueryHistory> queryHistories = (List<QueryHistory>)
result.get("query_histories");
+ Assert.assertEquals(3, queryHistories.size());
+ Assert.assertEquals("nmodel_basic",
queryHistories.get(0).getNativeQueryRealizations().get(0).getModelAlias());
+ Assert.assertEquals("HIVE", queryHistories.get(1).getEngineType());
+ Assert.assertEquals("CONSTANTS",
queryHistories.get(2).getEngineType());
+ Assert.assertEquals("TEST2",
queryHistories.get(0).getQuerySubmitter());
+ Assert.assertEquals("TEST", queryHistories.get(1).getQuerySubmitter());
+ Assert.assertEquals("TEST", queryHistories.get(2).getQuerySubmitter());
+ }
+
@Test
public void testGetQueryHistorySubmitters() {
QueryHistoryRequest request = new QueryHistoryRequest();
