This is an automated email from the ASF dual-hosted git repository.
xxyu pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/kylin.git
The following commit(s) were added to refs/heads/main by this push:
new 39ee0b8a6a KYLIN-5183 Customize or get query id for devops
39ee0b8a6a is described below
commit 39ee0b8a6a84dcf27404c838ead379b0342f7b65
Author: yhj <1041247...@qq.com>
AuthorDate: Sat May 14 16:24:25 2022 +0800
KYLIN-5183 Customize or get query id for devops
---
.../java/org/apache/kylin/common/QueryContext.java | 6 +++++-
.../org/apache/kylin/common/debug/BackdoorToggles.java | 5 +++++
.../main/java/org/apache/kylin/jdbc/IRemoteClient.java | 17 ++++++++++++++---
.../main/java/org/apache/kylin/jdbc/KylinClient.java | 4 ++--
.../java/org/apache/kylin/jdbc/KylinResultSet.java | 18 ++++++++++++++++--
.../org/apache/kylin/jdbc/json/SQLResponseStub.java | 10 ++++++++++
.../org/apache/kylin/rest/response/SQLResponse.java | 10 ++++++++++
.../org/apache/kylin/rest/service/QueryService.java | 6 +++++-
.../apache/kylin/rest/response/SQLResponseTest.java | 2 +-
9 files changed, 68 insertions(+), 10 deletions(-)
diff --git
a/core-common/src/main/java/org/apache/kylin/common/QueryContext.java
b/core-common/src/main/java/org/apache/kylin/common/QueryContext.java
index a0ece1ba97..2ba767f357 100644
--- a/core-common/src/main/java/org/apache/kylin/common/QueryContext.java
+++ b/core-common/src/main/java/org/apache/kylin/common/QueryContext.java
@@ -52,7 +52,7 @@ public class QueryContext {
private long queryStartMillis;
- private final String queryId;
+ private String queryId;
private String username;
private Set<String> groups;
private String project;
@@ -102,6 +102,10 @@ public class QueryContext {
}
}
+ public void setQueryId(String queryId) {
+ this.queryId = queryId;
+ }
+
public String getQueryId() {
return queryId == null ? "" : queryId;
}
diff --git
a/core-common/src/main/java/org/apache/kylin/common/debug/BackdoorToggles.java
b/core-common/src/main/java/org/apache/kylin/common/debug/BackdoorToggles.java
index 309271ace1..8b695c7ed7 100644
---
a/core-common/src/main/java/org/apache/kylin/common/debug/BackdoorToggles.java
+++
b/core-common/src/main/java/org/apache/kylin/common/debug/BackdoorToggles.java
@@ -401,4 +401,9 @@ public class BackdoorToggles {
public final static String DEBUG_TOGGLE_HIT_CUBE = "DEBUG_TOGGLE_HIT_CUBE";
public final static String DEBUG_TOGGLE_SPARK_POOL =
"DEBUG_TOGGLE_SPARK_POOL";
+
+ /**
+ * user customize query id
+ */
+ public final static String CUSTOMIZE_QUERY_ID = "CUSTOMIZE_QUERY_ID";
}
diff --git a/jdbc/src/main/java/org/apache/kylin/jdbc/IRemoteClient.java
b/jdbc/src/main/java/org/apache/kylin/jdbc/IRemoteClient.java
index 630eedd1fc..6db1b11529 100644
--- a/jdbc/src/main/java/org/apache/kylin/jdbc/IRemoteClient.java
+++ b/jdbc/src/main/java/org/apache/kylin/jdbc/IRemoteClient.java
@@ -6,15 +6,15 @@
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
- *
+ *
* http://www.apache.org/licenses/LICENSE-2.0
- *
+ *
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
-*/
+ */
package org.apache.kylin.jdbc;
@@ -31,11 +31,22 @@ public interface IRemoteClient extends Closeable {
class QueryResult {
public final List<ColumnMetaData> columnMeta;
public final Iterable<Object> iterable;
+ public String queryId;
+
+ public String getQueryId() {
+ return queryId;
+ }
public QueryResult(List<ColumnMetaData> columnMeta, Iterable<Object>
iterable) {
this.columnMeta = columnMeta;
this.iterable = iterable;
}
+
+ public QueryResult(List<ColumnMetaData> columnMeta, Iterable<Object>
iterable, String queryId) {
+ this.columnMeta = columnMeta;
+ this.iterable = iterable;
+ this.queryId = queryId;
+ }
}
/**
diff --git a/jdbc/src/main/java/org/apache/kylin/jdbc/KylinClient.java
b/jdbc/src/main/java/org/apache/kylin/jdbc/KylinClient.java
index 40844a1a2d..793632d443 100644
--- a/jdbc/src/main/java/org/apache/kylin/jdbc/KylinClient.java
+++ b/jdbc/src/main/java/org/apache/kylin/jdbc/KylinClient.java
@@ -314,7 +314,7 @@ public class KylinClient implements IRemoteClient {
List<KMetaCatalog> catalogs = convertMetaCatalogs(schemas);
return new KMetaProject(project, catalogs);
} finally {
- get.releaseConnection();
+ get.releaseConnection();
}
}
@@ -406,7 +406,7 @@ public class KylinClient implements IRemoteClient {
List<ColumnMetaData> metas = convertColumnMeta(queryResp);
List<Object> data = convertResultData(queryResp, metas);
- return new QueryResult(metas, data);
+ return new QueryResult(metas, data, queryResp.getQueryId());
}
private List<StatementParameter> convertParameters(List<Object>
paramValues) {
diff --git a/jdbc/src/main/java/org/apache/kylin/jdbc/KylinResultSet.java
b/jdbc/src/main/java/org/apache/kylin/jdbc/KylinResultSet.java
index 1654dd01d8..c064180ec6 100644
--- a/jdbc/src/main/java/org/apache/kylin/jdbc/KylinResultSet.java
+++ b/jdbc/src/main/java/org/apache/kylin/jdbc/KylinResultSet.java
@@ -6,9 +6,9 @@
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
- *
+ *
* http://www.apache.org/licenses/LICENSE-2.0
- *
+ *
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
@@ -39,6 +39,16 @@ import org.apache.kylin.jdbc.IRemoteClient.QueryResult;
public class KylinResultSet extends AvaticaResultSet {
+ private String queryId;
+
+ public String getQueryId() {
+ return queryId;
+ }
+
+ public void setQueryId(String queryId) {
+ this.queryId = queryId;
+ }
+
public KylinResultSet(AvaticaStatement statement, QueryState state,
Signature signature, ResultSetMetaData resultSetMetaData, TimeZone timeZone,
Frame firstFrame) throws SQLException {
super(statement, state, signature, resultSetMetaData, timeZone,
firstFrame);
}
@@ -71,6 +81,7 @@ public class KylinResultSet extends AvaticaResultSet {
QueryResult result;
try {
result = client.executeQuery(sql, paramValues, queryToggles);
+ this.setQueryId(result.getQueryId());
} catch (IOException e) {
throw new SQLException(e);
}
@@ -92,6 +103,9 @@ public class KylinResultSet extends AvaticaResultSet {
if (Driver.CLIENT_CALCITE_PROP_NAMES.contains(key)) {
props.put(key, connProps.getProperty(key));
}
+ if (key.startsWith("CUSTOMIZE_")) {
+ queryToggles.put(key, connProps.getProperty(key));
+ }
}
if (props.isEmpty()) {
diff --git a/jdbc/src/main/java/org/apache/kylin/jdbc/json/SQLResponseStub.java
b/jdbc/src/main/java/org/apache/kylin/jdbc/json/SQLResponseStub.java
index a05b6d6230..163d830d4a 100644
--- a/jdbc/src/main/java/org/apache/kylin/jdbc/json/SQLResponseStub.java
+++ b/jdbc/src/main/java/org/apache/kylin/jdbc/json/SQLResponseStub.java
@@ -58,6 +58,8 @@ public class SQLResponseStub implements Serializable {
private boolean storageCacheUsed = false;
+ private String queryId;
+
public SQLResponseStub() {
}
@@ -153,6 +155,14 @@ public class SQLResponseStub implements Serializable {
this.storageCacheUsed = storageCacheUsed;
}
+ public String getQueryId() {
+ return queryId;
+ }
+
+ public void setQueryId(String queryId) {
+ this.queryId = queryId;
+ }
+
@JsonIgnoreProperties(ignoreUnknown = true)
public static class ColumnMetaStub implements Serializable{
diff --git
a/server-base/src/main/java/org/apache/kylin/rest/response/SQLResponse.java
b/server-base/src/main/java/org/apache/kylin/rest/response/SQLResponse.java
index fcfc8ad84b..b4f596bfd6 100644
--- a/server-base/src/main/java/org/apache/kylin/rest/response/SQLResponse.java
+++ b/server-base/src/main/java/org/apache/kylin/rest/response/SQLResponse.java
@@ -97,6 +97,8 @@ public class SQLResponse implements Serializable {
private List<SQLResponseTrace> traces;
+ protected String queryId;
+
public SQLResponse() {
}
@@ -306,6 +308,14 @@ public class SQLResponse implements Serializable {
return traces;
}
+ public String getQueryId() {
+ return queryId;
+ }
+
+ public void setQueryId(String queryId) {
+ this.queryId = queryId;
+ }
+
@JsonIgnore
public List<QueryContext.CubeSegmentStatisticsResult>
getCubeSegmentStatisticsList() {
try {
diff --git
a/server-base/src/main/java/org/apache/kylin/rest/service/QueryService.java
b/server-base/src/main/java/org/apache/kylin/rest/service/QueryService.java
index cc48c838d9..108a9f941c 100644
--- a/server-base/src/main/java/org/apache/kylin/rest/service/QueryService.java
+++ b/server-base/src/main/java/org/apache/kylin/rest/service/QueryService.java
@@ -446,6 +446,10 @@ public class QueryService extends BasicService {
BackdoorToggles.addToggles(sqlRequest.getBackdoorToggles());
}
+ if (sqlRequest.getBackdoorToggles() != null &&
!StringUtil.isEmpty(sqlRequest.getBackdoorToggles().get(BackdoorToggles.CUSTOMIZE_QUERY_ID)))
{
+
queryContext.setQueryId(sqlRequest.getBackdoorToggles().get(BackdoorToggles.CUSTOMIZE_QUERY_ID));
+ }
+
try (SetThreadName ignored = new SetThreadName("Query %s",
queryContext.getQueryId())) {
// force clear the query context before a new query
OLAPContext.clearThreadLocalContexts();
@@ -484,7 +488,7 @@ public class QueryService extends BasicService {
sqlResponse = queryAndUpdateCache(sqlRequest,
isQueryCacheEnabled);
}
}
-
+ sqlResponse.setQueryId(queryContext.getQueryId());
sqlResponse.setDuration(queryContext.getAccumulatedMillis());
if
(QuerySparkMetrics.getInstance().getQueryExecutionMetrics(queryContext.getQueryId())
!= null) {
String sqlTraceUrl = SparderContext.appMasterTrackURL() +
"/SQL/execution/?id=" +
diff --git
a/server-base/src/test/java/org/apache/kylin/rest/response/SQLResponseTest.java
b/server-base/src/test/java/org/apache/kylin/rest/response/SQLResponseTest.java
index deeafc9f27..e62f9d5698 100644
---
a/server-base/src/test/java/org/apache/kylin/rest/response/SQLResponseTest.java
+++
b/server-base/src/test/java/org/apache/kylin/rest/response/SQLResponseTest.java
@@ -36,7 +36,7 @@ public class SQLResponseTest {
"realizationTypes", "affectedRowCount", "isException",
"exceptionMessage", "duration", "partial", "totalScanCount",
"hitExceptionCache",
"storageCacheUsed", "sparkPool", "pushDown", "traceUrl",
"totalScanBytes",
- "totalScanFiles", "metadataTime", "totalSparkScanTime",
"traces"};
+ "totalScanFiles", "metadataTime", "totalSparkScanTime",
"traces", "queryId"};
SQLResponse sqlResponse = new SQLResponse(null, null, "learn_cube",
100, false, null, false, false);
String jsonStr = JsonUtil.writeValueAsString(sqlResponse);
