Repository: kylin Updated Branches: refs/heads/1.5.x-CDH5.7 b46029fba -> c7d021661 (forced update)
KYLIN-1908 refactor, extract QueryMetricsFacade Project: http://git-wip-us.apache.org/repos/asf/kylin/repo Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/1b34c38b Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/1b34c38b Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/1b34c38b Branch: refs/heads/1.5.x-CDH5.7 Commit: 1b34c38b2a9ff65950dbac8229af5749ad8403ed Parents: 0188a26 Author: Yang Li <liy...@apache.org> Authored: Sun Aug 21 19:38:11 2016 +0800 Committer: Yang Li <liy...@apache.org> Committed: Sun Aug 21 19:38:11 2016 +0800 ---------------------------------------------------------------------- .../apache/kylin/common/KylinConfigBase.java | 4 + .../common/KylinConfigCannotInitException.java | 1 + .../kylin/rest/controller/QueryController.java | 9 +- .../kylin/rest/init/InitialTaskManager.java | 8 +- .../kylin/rest/metrics/QueryMetricsFacade.java | 108 +++++++++++++++++++ .../kylin/rest/util/QueryMetricsUtil.java | 94 ---------------- .../kylin/rest/metrics/QueryMetricsTest.java | 20 ++-- 7 files changed, 128 insertions(+), 116 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/kylin/blob/1b34c38b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java ---------------------------------------------------------------------- diff --git a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java index 50205fb..de6b977 100644 --- a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java +++ b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java @@ -570,6 +570,10 @@ abstract public class KylinConfigBase implements Serializable { return Long.valueOf(this.getOptional("kylin.query.sequence.expire.time", "86400000"));//default a day } + public boolean getQueryMetricsEnabled() { + return Boolean.parseBoolean(getOptional("kylin.query.metrics.enabled", "false")); + } + public int[] getQueryMetricsPercentilesIntervals() { String[] dft = { "60", "300", "3600" }; return getOptionalIntArray("kylin.query.metrics.percentiles.intervals", dft); http://git-wip-us.apache.org/repos/asf/kylin/blob/1b34c38b/core-common/src/main/java/org/apache/kylin/common/KylinConfigCannotInitException.java ---------------------------------------------------------------------- diff --git a/core-common/src/main/java/org/apache/kylin/common/KylinConfigCannotInitException.java b/core-common/src/main/java/org/apache/kylin/common/KylinConfigCannotInitException.java index 7113872..a4bbeae 100644 --- a/core-common/src/main/java/org/apache/kylin/common/KylinConfigCannotInitException.java +++ b/core-common/src/main/java/org/apache/kylin/common/KylinConfigCannotInitException.java @@ -18,6 +18,7 @@ package org.apache.kylin.common; +@SuppressWarnings("serial") public class KylinConfigCannotInitException extends RuntimeException { public KylinConfigCannotInitException(String message) { super(message); http://git-wip-us.apache.org/repos/asf/kylin/blob/1b34c38b/server-base/src/main/java/org/apache/kylin/rest/controller/QueryController.java ---------------------------------------------------------------------- diff --git a/server-base/src/main/java/org/apache/kylin/rest/controller/QueryController.java b/server-base/src/main/java/org/apache/kylin/rest/controller/QueryController.java index 6f8a7e1..e7847c7 100644 --- a/server-base/src/main/java/org/apache/kylin/rest/controller/QueryController.java +++ b/server-base/src/main/java/org/apache/kylin/rest/controller/QueryController.java @@ -22,7 +22,6 @@ import java.io.IOException; import java.sql.SQLException; import java.util.ArrayList; import java.util.List; -import java.util.concurrent.ConcurrentHashMap; import javax.annotation.PostConstruct; import javax.servlet.http.HttpServletResponse; @@ -33,7 +32,7 @@ import org.apache.kylin.common.debug.BackdoorToggles; import org.apache.kylin.cube.CubeInstance; import org.apache.kylin.rest.constant.Constant; import org.apache.kylin.rest.exception.InternalErrorException; -import org.apache.kylin.rest.metrics.QueryMetrics; +import org.apache.kylin.rest.metrics.QueryMetricsFacade; import org.apache.kylin.rest.model.Query; import org.apache.kylin.rest.model.SelectedColumnMeta; import org.apache.kylin.rest.model.TableMeta; @@ -43,7 +42,6 @@ import org.apache.kylin.rest.request.SQLRequest; import org.apache.kylin.rest.request.SaveSqlRequest; import org.apache.kylin.rest.response.SQLResponse; import org.apache.kylin.rest.service.QueryService; -import org.apache.kylin.rest.util.QueryMetricsUtil; import org.apache.kylin.rest.util.QueryUtil; import org.apache.kylin.storage.exception.ScanOutOfLimitException; import org.slf4j.Logger; @@ -77,9 +75,6 @@ public class QueryController extends BasicController { private static final Logger logger = LoggerFactory.getLogger(QueryController.class); - private ConcurrentHashMap<String, QueryMetrics> metricsMap = - new ConcurrentHashMap<String, QueryMetrics>(); - public static final String SUCCESS_QUERY_CACHE = "StorageCache"; public static final String EXCEPTION_QUERY_CACHE = "ExceptionQueryCache"; @@ -225,7 +220,7 @@ public class QueryController extends BasicController { queryService.logQuery(sqlRequest, sqlResponse); - QueryMetricsUtil.updateMetrics(sqlRequest, sqlResponse, metricsMap); + QueryMetricsFacade.updateMetrics(sqlRequest, sqlResponse); if (sqlResponse.getIsException()) throw new InternalErrorException(sqlResponse.getExceptionMessage()); http://git-wip-us.apache.org/repos/asf/kylin/blob/1b34c38b/server-base/src/main/java/org/apache/kylin/rest/init/InitialTaskManager.java ---------------------------------------------------------------------- diff --git a/server-base/src/main/java/org/apache/kylin/rest/init/InitialTaskManager.java b/server-base/src/main/java/org/apache/kylin/rest/init/InitialTaskManager.java index 430d8a4..5cf48cb 100644 --- a/server-base/src/main/java/org/apache/kylin/rest/init/InitialTaskManager.java +++ b/server-base/src/main/java/org/apache/kylin/rest/init/InitialTaskManager.java @@ -19,8 +19,8 @@ package org.apache.kylin.rest.init; import org.apache.commons.lang.StringUtils; -import org.apache.hadoop.metrics2.lib.DefaultMetricsSystem; import org.apache.kylin.common.KylinConfig; +import org.apache.kylin.rest.metrics.QueryMetricsFacade; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.beans.factory.InitializingBean; @@ -37,12 +37,12 @@ public class InitialTaskManager implements InitializingBean { logger.info("Kylin service is starting....."); runInitialTasks(); - - //init mrtrics system for kylin - DefaultMetricsSystem.initialize("Kylin"); } private void runInitialTasks() { + // init metrics system for kylin + QueryMetricsFacade.init(); + KylinConfig kylinConfig = KylinConfig.getInstanceFromEnv(); String initTasks = kylinConfig.getInitTasks(); if (!StringUtils.isEmpty(initTasks)) { http://git-wip-us.apache.org/repos/asf/kylin/blob/1b34c38b/server-base/src/main/java/org/apache/kylin/rest/metrics/QueryMetricsFacade.java ---------------------------------------------------------------------- diff --git a/server-base/src/main/java/org/apache/kylin/rest/metrics/QueryMetricsFacade.java b/server-base/src/main/java/org/apache/kylin/rest/metrics/QueryMetricsFacade.java new file mode 100644 index 0000000..91aff8b --- /dev/null +++ b/server-base/src/main/java/org/apache/kylin/rest/metrics/QueryMetricsFacade.java @@ -0,0 +1,108 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. +*/ + +package org.apache.kylin.rest.metrics; + +import org.apache.hadoop.metrics2.lib.DefaultMetricsSystem; +import org.apache.kylin.common.KylinConfig; +import org.apache.kylin.rest.request.SQLRequest; +import org.apache.kylin.rest.response.SQLResponse; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import javax.annotation.concurrent.ThreadSafe; +import java.util.concurrent.ConcurrentHashMap; + +/** + * The entrance of metrics features. + */ +@ThreadSafe +public class QueryMetricsFacade { + + private static final Logger logger = LoggerFactory.getLogger(QueryMetricsFacade.class); + + private static boolean enabled = false; + private static ConcurrentHashMap<String, QueryMetrics> metricsMap = new ConcurrentHashMap<String, QueryMetrics>(); + + public static void init() { + enabled = KylinConfig.getInstanceFromEnv().getQueryMetricsEnabled(); + if (!enabled) + return; + + DefaultMetricsSystem.initialize("Kylin"); + } + + public static void updateMetrics(SQLRequest sqlRequest, SQLResponse sqlResponse) { + if (!enabled) + return; + + String projectName = sqlRequest.getProject(); + String cubeName = sqlResponse.getCube(); + + update(getQueryMetrics("Server_Total", metricsMap), sqlResponse); + + update(getQueryMetrics(projectName, metricsMap), sqlResponse); + + String cubeMetricName = projectName + ",sub=" + cubeName; + update(getQueryMetrics(cubeMetricName, metricsMap), sqlResponse); + } + + private static void update(QueryMetrics queryMetrics, SQLResponse sqlResponse) { + try { + incrQueryCount(queryMetrics, sqlResponse); + incrCacheHitCount(queryMetrics, sqlResponse); + + if (!sqlResponse.getIsException()) { + queryMetrics.addQueryLatency(sqlResponse.getDuration()); + queryMetrics.addScanRowCount(sqlResponse.getTotalScanCount()); + queryMetrics.addResultRowCount(sqlResponse.getResults().size()); + } + } catch (Exception e) { + logger.error(e.getMessage()); + } + + } + + private static void incrQueryCount(QueryMetrics queryMetrics, SQLResponse sqlResponse) { + if (!sqlResponse.isHitExceptionCache() && !sqlResponse.getIsException()) { + queryMetrics.incrQuerySuccessCount(); + } else { + queryMetrics.incrQueryFailCount(); + } + queryMetrics.incrQueryCount(); + } + + private static void incrCacheHitCount(QueryMetrics queryMetrics, SQLResponse sqlResponse) { + if (sqlResponse.isStorageCacheUsed()) { + queryMetrics.addCacheHitCount(1); + } + } + + private static QueryMetrics getQueryMetrics(String name, ConcurrentHashMap<String, QueryMetrics> metricsMap) { + KylinConfig config = KylinConfig.getInstanceFromEnv(); + int[] intervals = config.getQueryMetricsPercentilesIntervals(); + + if (metricsMap.containsKey(name)) { + return metricsMap.get(name); + } else { + QueryMetrics queryMetrics = new QueryMetrics(intervals).registerWith(name); + metricsMap.put(name, queryMetrics); + return queryMetrics; + } + } +} http://git-wip-us.apache.org/repos/asf/kylin/blob/1b34c38b/server-base/src/main/java/org/apache/kylin/rest/util/QueryMetricsUtil.java ---------------------------------------------------------------------- diff --git a/server-base/src/main/java/org/apache/kylin/rest/util/QueryMetricsUtil.java b/server-base/src/main/java/org/apache/kylin/rest/util/QueryMetricsUtil.java deleted file mode 100644 index 0c1c780..0000000 --- a/server-base/src/main/java/org/apache/kylin/rest/util/QueryMetricsUtil.java +++ /dev/null @@ -1,94 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. -*/ - -package org.apache.kylin.rest.util; - -import org.apache.kylin.common.KylinConfig; -import org.apache.kylin.rest.metrics.QueryMetrics; -import org.apache.kylin.rest.request.SQLRequest; -import org.apache.kylin.rest.response.SQLResponse; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import javax.annotation.concurrent.ThreadSafe; -import java.util.concurrent.ConcurrentHashMap; - -/** - * update metrics about query - */ -@ThreadSafe -public class QueryMetricsUtil { - - private static final Logger logger = LoggerFactory.getLogger(QueryMetricsUtil.class); - - public static void updateMetrics(SQLRequest sqlRequest, SQLResponse sqlResponse, ConcurrentHashMap<String, QueryMetrics> metricsMap) { - String projectName = sqlRequest.getProject(); - String cubeName = sqlResponse.getCube(); - - update(getQueryMetrics("Server_Total", metricsMap), sqlResponse); - - update(getQueryMetrics(projectName, metricsMap), sqlResponse); - - String cubeMetricName = projectName + ",sub=" + cubeName; - update(getQueryMetrics(cubeMetricName, metricsMap), sqlResponse); - } - - private static void update(QueryMetrics queryMetrics, SQLResponse sqlResponse) { - try { - incrQueryCount(queryMetrics, sqlResponse); - incrCacheHitCount(queryMetrics, sqlResponse); - - if (!sqlResponse.getIsException()) { - queryMetrics.addQueryLatency(sqlResponse.getDuration()); - queryMetrics.addScanRowCount(sqlResponse.getTotalScanCount()); - queryMetrics.addResultRowCount(sqlResponse.getResults().size()); - } - } catch (Exception e) { - logger.error(e.getMessage()); - } - - } - - private static void incrQueryCount(QueryMetrics queryMetrics, SQLResponse sqlResponse) { - if (!sqlResponse.isHitExceptionCache() && !sqlResponse.getIsException()) { - queryMetrics.incrQuerySuccessCount(); - } else { - queryMetrics.incrQueryFailCount(); - } - queryMetrics.incrQueryCount(); - } - - private static void incrCacheHitCount(QueryMetrics queryMetrics, SQLResponse sqlResponse) { - if (sqlResponse.isStorageCacheUsed()) { - queryMetrics.addCacheHitCount(1); - } - } - - private static QueryMetrics getQueryMetrics(String name, ConcurrentHashMap<String, QueryMetrics> metricsMap) { - KylinConfig config = KylinConfig.getInstanceFromEnv(); - int[] intervals = config.getQueryMetricsPercentilesIntervals(); - - if (metricsMap.containsKey(name)) { - return metricsMap.get(name); - } else { - QueryMetrics queryMetrics = new QueryMetrics(intervals).registerWith(name); - metricsMap.put(name, queryMetrics); - return queryMetrics; - } - } -} http://git-wip-us.apache.org/repos/asf/kylin/blob/1b34c38b/server/src/test/java/org/apache/kylin/rest/metrics/QueryMetricsTest.java ---------------------------------------------------------------------- diff --git a/server/src/test/java/org/apache/kylin/rest/metrics/QueryMetricsTest.java b/server/src/test/java/org/apache/kylin/rest/metrics/QueryMetricsTest.java index cd7ff7a..d413f40 100644 --- a/server/src/test/java/org/apache/kylin/rest/metrics/QueryMetricsTest.java +++ b/server/src/test/java/org/apache/kylin/rest/metrics/QueryMetricsTest.java @@ -18,26 +18,24 @@ package org.apache.kylin.rest.metrics; +import java.lang.management.ManagementFactory; +import java.util.ArrayList; +import java.util.List; + +import javax.management.MBeanServer; +import javax.management.ObjectName; + import org.apache.kylin.rest.request.SQLRequest; import org.apache.kylin.rest.response.SQLResponse; import org.apache.kylin.rest.service.ServiceTestBase; -import org.apache.kylin.rest.util.QueryMetricsUtil; import org.junit.Assert; import org.junit.Before; import org.junit.Test; -import javax.management.MBeanServer; -import javax.management.ObjectName; -import java.lang.management.ManagementFactory; -import java.util.ArrayList; -import java.util.List; -import java.util.concurrent.ConcurrentHashMap; - public class QueryMetricsTest extends ServiceTestBase { private static MBeanServer mBeanServer; private static ObjectName objectName; - private ConcurrentHashMap<String, QueryMetrics> metricsMap = new ConcurrentHashMap<String, QueryMetrics>(); @Before public void setup() throws Exception { @@ -71,7 +69,7 @@ public class QueryMetricsTest extends ServiceTestBase { sqlResponse.setResults(results); sqlResponse.setStorageCacheUsed(true); - QueryMetricsUtil.updateMetrics(sqlRequest, sqlResponse, metricsMap); + QueryMetricsFacade.updateMetrics(sqlRequest, sqlResponse); Thread.sleep(2000); @@ -100,7 +98,7 @@ public class QueryMetricsTest extends ServiceTestBase { sqlResponse2.setCube("test_cube"); sqlResponse2.setIsException(true); - QueryMetricsUtil.updateMetrics(sqlRequest, sqlResponse2, metricsMap); + QueryMetricsFacade.updateMetrics(sqlRequest, sqlResponse2); Thread.sleep(2000);
