KYLIN-2318 query cache is not working
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/182c565d Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/182c565d Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/182c565d Branch: refs/heads/master-hbase1.x Commit: 182c565df6b2134ef1302c88988ed27b60fefa42 Parents: fa4a5ee Author: Hongbin Ma <mahong...@apache.org> Authored: Sat Dec 24 15:31:13 2016 +0800 Committer: Hongbin Ma <mahong...@apache.org> Committed: Tue Dec 27 22:15:44 2016 +0800 ---------------------------------------------------------------------- .../org/apache/kylin/common/QueryContext.java | 61 ++++++++++++++++++++ .../kylin/common/debug/BackdoorToggles.java | 11 ++-- .../apache/kylin/rest/request/SQLRequest.java | 23 +++++--- .../apache/kylin/rest/service/QueryService.java | 23 ++++---- .../hbase/cube/v2/CubeHBaseEndpointRPC.java | 4 +- 5 files changed, 93 insertions(+), 29 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/kylin/blob/182c565d/core-common/src/main/java/org/apache/kylin/common/QueryContext.java ---------------------------------------------------------------------- diff --git a/core-common/src/main/java/org/apache/kylin/common/QueryContext.java b/core-common/src/main/java/org/apache/kylin/common/QueryContext.java new file mode 100644 index 0000000..ef0cb14 --- /dev/null +++ b/core-common/src/main/java/org/apache/kylin/common/QueryContext.java @@ -0,0 +1,61 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. +*/ + +package org.apache.kylin.common; + +import java.util.Map; + +import com.google.common.collect.Maps; + +/** + * checkout {@link org.apache.kylin.common.debug.BackdoorToggles} for comparision + */ +public class QueryContext { + private static final ThreadLocal<Map<String, String>> _queryContext = new ThreadLocal<Map<String, String>>(); + + public final static String KEY_QUERY_ID = "QUERY_ID"; + + public static String getQueryId() { + return getString(KEY_QUERY_ID); + } + + public static void setQueryId(String uuid) { + setString(KEY_QUERY_ID, uuid); + } + + private static void setString(String key, String value) { + Map<String, String> context = _queryContext.get(); + if (context == null) { + Map<String, String> newMap = Maps.newHashMap(); + newMap.put(key, value); + _queryContext.set(newMap); + } else { + context.put(key, value); + } + } + + private static String getString(String key) { + Map<String, String> context = _queryContext.get(); + if (context == null) { + return null; + } else { + return context.get(key); + } + } + +} http://git-wip-us.apache.org/repos/asf/kylin/blob/182c565d/core-common/src/main/java/org/apache/kylin/common/debug/BackdoorToggles.java ---------------------------------------------------------------------- diff --git a/core-common/src/main/java/org/apache/kylin/common/debug/BackdoorToggles.java b/core-common/src/main/java/org/apache/kylin/common/debug/BackdoorToggles.java index 28f7697..ca4a19c 100644 --- a/core-common/src/main/java/org/apache/kylin/common/debug/BackdoorToggles.java +++ b/core-common/src/main/java/org/apache/kylin/common/debug/BackdoorToggles.java @@ -24,6 +24,11 @@ import org.apache.commons.lang.StringUtils; import org.apache.kylin.common.util.Pair; /** + * BackdoorToggles and QueryContext are similar because they're both hosting per-query thread local variables. + * The difference is that BackdoorToggles are specified by user input and work for debug purpose. QueryContext + * is used voluntarily by program itself + * + * BackdoorToggles is part of SQLRequest, QueryContext does not belong to SQLRequest */ public class BackdoorToggles { @@ -65,10 +70,6 @@ public class BackdoorToggles { return Integer.valueOf(v); } - public static String getQueryId() { - return getString(KEY_QUERY_ID); - } - public static Pair<Short, Short> getShardAssignment() { String v = getString(DEBUG_TOGGLE_SHARD_ASSIGNMENT); if (v == null) { @@ -104,8 +105,6 @@ public class BackdoorToggles { _backdoorToggles.remove(); } - public final static String KEY_QUERY_ID = "QUERY_ID"; - /** * set DEBUG_TOGGLE_DISABLE_FUZZY_KEY=true to disable fuzzy key for debug/profile usage * http://git-wip-us.apache.org/repos/asf/kylin/blob/182c565d/server-base/src/main/java/org/apache/kylin/rest/request/SQLRequest.java ---------------------------------------------------------------------- diff --git a/server-base/src/main/java/org/apache/kylin/rest/request/SQLRequest.java b/server-base/src/main/java/org/apache/kylin/rest/request/SQLRequest.java index bd8b7e2..1896f4f 100644 --- a/server-base/src/main/java/org/apache/kylin/rest/request/SQLRequest.java +++ b/server-base/src/main/java/org/apache/kylin/rest/request/SQLRequest.java @@ -25,6 +25,7 @@ public class SQLRequest implements Serializable { protected static final long serialVersionUID = 1L; private String sql; + private String project; private Integer offset = 0; private Integer limit = 0; @@ -83,19 +84,25 @@ public class SQLRequest implements Serializable { this.acceptPartial = acceptPartial; } - @Override public boolean equals(Object o) { - if (this == o) return true; - if (o == null || getClass() != o.getClass()) return false; + if (this == o) + return true; + if (o == null || getClass() != o.getClass()) + return false; SQLRequest that = (SQLRequest) o; - if (acceptPartial != that.acceptPartial) return false; - if (sql != null ? !sql.equals(that.sql) : that.sql != null) return false; - if (project != null ? !project.equals(that.project) : that.project != null) return false; - if (offset != null ? !offset.equals(that.offset) : that.offset != null) return false; - if (limit != null ? !limit.equals(that.limit) : that.limit != null) return false; + if (acceptPartial != that.acceptPartial) + return false; + if (sql != null ? !sql.equals(that.sql) : that.sql != null) + return false; + if (project != null ? !project.equals(that.project) : that.project != null) + return false; + if (offset != null ? !offset.equals(that.offset) : that.offset != null) + return false; + if (limit != null ? !limit.equals(that.limit) : that.limit != null) + return false; return backdoorToggles != null ? backdoorToggles.equals(that.backdoorToggles) : that.backdoorToggles == null; } http://git-wip-us.apache.org/repos/asf/kylin/blob/182c565d/server-base/src/main/java/org/apache/kylin/rest/service/QueryService.java ---------------------------------------------------------------------- diff --git a/server-base/src/main/java/org/apache/kylin/rest/service/QueryService.java b/server-base/src/main/java/org/apache/kylin/rest/service/QueryService.java index 82e4a87..0dd5c5f 100644 --- a/server-base/src/main/java/org/apache/kylin/rest/service/QueryService.java +++ b/server-base/src/main/java/org/apache/kylin/rest/service/QueryService.java @@ -53,6 +53,7 @@ import org.apache.hadoop.hbase.client.HTableInterface; import org.apache.hadoop.hbase.client.Put; import org.apache.hadoop.hbase.client.Result; import org.apache.kylin.common.KylinConfig; +import org.apache.kylin.common.QueryContext; import org.apache.kylin.common.debug.BackdoorToggles; import org.apache.kylin.common.util.Bytes; import org.apache.kylin.common.util.DBUtils; @@ -266,7 +267,7 @@ public class QueryService extends BasicService { StringBuilder stringBuilder = new StringBuilder(); stringBuilder.append(newLine); stringBuilder.append("==========================[QUERY]===============================").append(newLine); - stringBuilder.append("Query Id: ").append(BackdoorToggles.getQueryId()).append(newLine); + stringBuilder.append("Query Id: ").append(QueryContext.getQueryId()).append(newLine); stringBuilder.append("SQL: ").append(request.getSql()).append(newLine); stringBuilder.append("User: ").append(user).append(newLine); stringBuilder.append("Success: ").append((null == response.getExceptionMessage())).append(newLine); @@ -324,13 +325,7 @@ public class QueryService extends BasicService { } final String queryId = UUID.randomUUID().toString(); - Map<String, String> toggles = new HashMap<>(); - toggles.put(BackdoorToggles.KEY_QUERY_ID, queryId); - if (sqlRequest.getBackdoorToggles() != null) { - toggles.putAll(sqlRequest.getBackdoorToggles()); - } - sqlRequest.setBackdoorToggles(toggles); - BackdoorToggles.setToggles(toggles); + QueryContext.setQueryId(queryId); try (SetThreadName ignored = new SetThreadName("Query %s", queryId)) { String sql = sqlRequest.getSql(); @@ -442,7 +437,9 @@ public class QueryService extends BasicService { String correctedSql = QueryUtil.massageSql(sqlRequest); if (!correctedSql.equals(sqlRequest.getSql())) { logger.info("The corrected query: " + correctedSql); - sqlRequest.setSql(correctedSql); + + //CAUTION: should not change sqlRequest content! + //sqlRequest.setSql(correctedSql); } // add extra parameters into olap context, like acceptPartial @@ -521,12 +518,12 @@ public class QueryService extends BasicService { } /** - * @param sql + * @param correctedSql * @param sqlRequest * @return * @throws Exception */ - private SQLResponse execute(String sql, SQLRequest sqlRequest) throws Exception { + private SQLResponse execute(String correctedSql, SQLRequest sqlRequest) throws Exception { Connection conn = null; Statement stat = null; ResultSet resultSet = null; @@ -538,7 +535,7 @@ public class QueryService extends BasicService { conn = cacheService.getOLAPDataSource(sqlRequest.getProject()).getConnection(); if (sqlRequest instanceof PrepareSqlRequest) { - PreparedStatement preparedState = conn.prepareStatement(sql); + PreparedStatement preparedState = conn.prepareStatement(correctedSql); processStatementAttr(preparedState, sqlRequest); for (int i = 0; i < ((PrepareSqlRequest) sqlRequest).getParams().length; i++) { @@ -549,7 +546,7 @@ public class QueryService extends BasicService { } else { stat = conn.createStatement(); processStatementAttr(stat, sqlRequest); - resultSet = stat.executeQuery(sql); + resultSet = stat.executeQuery(correctedSql); } ResultSetMetaData metaData = resultSet.getMetaData(); http://git-wip-us.apache.org/repos/asf/kylin/blob/182c565d/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeHBaseEndpointRPC.java ---------------------------------------------------------------------- diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeHBaseEndpointRPC.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeHBaseEndpointRPC.java index ebacb26..df1817e 100644 --- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeHBaseEndpointRPC.java +++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeHBaseEndpointRPC.java @@ -32,7 +32,7 @@ import org.apache.hadoop.hbase.client.coprocessor.Batch; import org.apache.hadoop.hbase.ipc.BlockingRpcCallback; import org.apache.hadoop.hbase.ipc.ServerRpcController; import org.apache.kylin.common.KylinConfig; -import org.apache.kylin.common.debug.BackdoorToggles; +import org.apache.kylin.common.QueryContext; import org.apache.kylin.common.util.Bytes; import org.apache.kylin.common.util.BytesSerializer; import org.apache.kylin.common.util.BytesUtil; @@ -158,7 +158,7 @@ public class CubeHBaseEndpointRPC extends CubeHBaseRPC { } builder.setRowkeyPreambleSize(cubeSeg.getRowKeyPreambleSize()); builder.setKylinProperties(kylinConfig.getConfigAsString()); - final String queryId = BackdoorToggles.getQueryId(); + final String queryId = QueryContext.getQueryId(); if (queryId != null) { builder.setQueryId(queryId); }
