KYLIN-1546 Refine bad query detect
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/30aa60d6 Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/30aa60d6 Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/30aa60d6 Branch: refs/heads/1.5.x-HBase1.1.3 Commit: 30aa60d6586292bd875e5a6150249bf9b0d6638a Parents: fc14ef7 Author: lidongsjtu <lid...@apache.org> Authored: Fri Apr 1 23:28:50 2016 +0800 Committer: lidongsjtu <lid...@apache.org> Committed: Fri Apr 1 23:28:50 2016 +0800 ---------------------------------------------------------------------- .../apache/kylin/common/KylinConfigBase.java | 6 ++++- .../kylin/metadata/badquery/BadQueryEntry.java | 28 ++++++++++++++++++-- .../badquery/BadQueryHistoryManager.java | 22 ++++++++++++++- .../badquery/BadQueryHistoryManagerTest.java | 21 +++++++++++++-- .../kylin/rest/service/BadQueryDetector.java | 11 +++++--- .../apache/kylin/tool/CubeMetaExtractor.java | 11 +++++--- 6 files changed, 87 insertions(+), 12 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/kylin/blob/30aa60d6/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java ---------------------------------------------------------------------- diff --git a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java index 2fe13a0..6a0f860 100644 --- a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java +++ b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java @@ -451,7 +451,11 @@ abstract public class KylinConfigBase implements Serializable { } public int getBadQueryDefaultDetectIntervalSeconds() { - return Integer.parseInt(getOptional("kylin.query.badquery.default.alerting.seconds", "60")); + return Integer.parseInt(getOptional("kylin.query.badquery.detect.interval.seconds", "60")); + } + + public boolean getBadQueryPersistentEnabled() { + return Boolean.parseBoolean(getOptional("kylin.query.badquery.persistent.enable", "true")); } public int getCachedDictMaxEntrySize() { http://git-wip-us.apache.org/repos/asf/kylin/blob/30aa60d6/core-metadata/src/main/java/org/apache/kylin/metadata/badquery/BadQueryEntry.java ---------------------------------------------------------------------- diff --git a/core-metadata/src/main/java/org/apache/kylin/metadata/badquery/BadQueryEntry.java b/core-metadata/src/main/java/org/apache/kylin/metadata/badquery/BadQueryEntry.java index b15846c..d795954 100644 --- a/core-metadata/src/main/java/org/apache/kylin/metadata/badquery/BadQueryEntry.java +++ b/core-metadata/src/main/java/org/apache/kylin/metadata/badquery/BadQueryEntry.java @@ -103,11 +103,35 @@ public class BadQueryEntry extends RootPersistentEntity implements Comparable<Ba @Override public int compareTo(BadQueryEntry obj) { - return this.startTime >= obj.startTime ? 1 : -1; + if (this.startTime == obj.startTime) { + return 0; + } else if (this.startTime > obj.startTime) { + return 1; + } else { + return -1; + } + } + + @Override + public boolean equals(Object o) { + if (this == o) + return true; + if (o == null || getClass() != o.getClass()) + return false; + + BadQueryEntry entry = (BadQueryEntry) o; + + if (!sql.equals(entry.sql)) + return false; + + if (startTime != entry.startTime) + return false; + + return true; } @Override public String toString() { - return "BadQueryEntry [ adj=" + adj + ", server=" + server + ", startTime=" + DateFormat.formatToTimeStr(startTime) + "]"; + return "BadQueryEntry [ adj=" + adj + ", server=" + server + ", startTime=" + DateFormat.formatToTimeStr(startTime) + " ]"; } } http://git-wip-us.apache.org/repos/asf/kylin/blob/30aa60d6/core-metadata/src/main/java/org/apache/kylin/metadata/badquery/BadQueryHistoryManager.java ---------------------------------------------------------------------- diff --git a/core-metadata/src/main/java/org/apache/kylin/metadata/badquery/BadQueryHistoryManager.java b/core-metadata/src/main/java/org/apache/kylin/metadata/badquery/BadQueryHistoryManager.java index 1945fed..5d532aa 100644 --- a/core-metadata/src/main/java/org/apache/kylin/metadata/badquery/BadQueryHistoryManager.java +++ b/core-metadata/src/main/java/org/apache/kylin/metadata/badquery/BadQueryHistoryManager.java @@ -100,10 +100,30 @@ public class BadQueryHistoryManager { return badQueryHistory; } - public BadQueryHistory addEntryToProject(String sql, String adj, long startTime, int runningSecs, String server, String threadName, String project) throws IOException { + public BadQueryHistory updateEntryToProject(BadQueryEntry badQueryEntry, String project) throws IOException { + if (StringUtils.isEmpty(project) || badQueryEntry.getAdj() == null || badQueryEntry.getSql() == null) + throw new IllegalArgumentException(); + + BadQueryHistory badQueryHistory = getBadQueriesForProject(project); + NavigableSet<BadQueryEntry> entries = badQueryHistory.getEntries(); + BadQueryEntry entry = entries.floor(badQueryEntry); + entry.setAdj(badQueryEntry.getAdj()); + entry.setRunningSec(badQueryEntry.getRunningSec()); + entry.setServer(badQueryEntry.getServer()); + entry.setThread(badQueryEntry.getThread()); + getStore().putResource(badQueryHistory.getResourcePath(), badQueryHistory, BAD_QUERY_INSTANCE_SERIALIZER); + + return badQueryHistory; + } + + public BadQueryHistory addEntryToProject(String sql, long startTime, String adj, int runningSecs, String server, String threadName, String project) throws IOException { return addEntryToProject(new BadQueryEntry(sql, adj, startTime, runningSecs, server, threadName), project); } + public BadQueryHistory updateEntryToProject(String sql, long startTime, String adj, int runningSecs, String server, String threadName, String project) throws IOException { + return updateEntryToProject(new BadQueryEntry(sql, adj, startTime, runningSecs, server, threadName), project); + } + public void removeBadQueryHistory(String project) throws IOException { getStore().deleteResource(getResourcePathForProject(project)); } http://git-wip-us.apache.org/repos/asf/kylin/blob/30aa60d6/core-metadata/src/test/java/org/apache/kylin/metadata/badquery/BadQueryHistoryManagerTest.java ---------------------------------------------------------------------- diff --git a/core-metadata/src/test/java/org/apache/kylin/metadata/badquery/BadQueryHistoryManagerTest.java b/core-metadata/src/test/java/org/apache/kylin/metadata/badquery/BadQueryHistoryManagerTest.java index c684598..949f11f 100644 --- a/core-metadata/src/test/java/org/apache/kylin/metadata/badquery/BadQueryHistoryManagerTest.java +++ b/core-metadata/src/test/java/org/apache/kylin/metadata/badquery/BadQueryHistoryManagerTest.java @@ -63,7 +63,7 @@ public class BadQueryHistoryManagerTest extends LocalFileMetadataTestCase { public void testAddEntryToProject() throws IOException { KylinConfig kylinConfig = getTestConfig(); BadQueryHistoryManager manager = BadQueryHistoryManager.getInstance(kylinConfig); - BadQueryHistory history = manager.addEntryToProject("sql", "adj", 1459362239992L, 100, "server", "t-0", "default"); + BadQueryHistory history = manager.addEntryToProject("sql", 1459362239992L, "adj", 100, "server", "t-0", "default"); NavigableSet<BadQueryEntry> entries = history.getEntries(); assertEquals(3, entries.size()); @@ -77,9 +77,26 @@ public class BadQueryHistoryManagerTest extends LocalFileMetadataTestCase { assertEquals("t-0", newEntry.getThread()); for (int i = 0; i < kylinConfig.getBadQueryHistoryNum(); i++) { - history = manager.addEntryToProject("sql", "adj", 1459362239993L + i, 100 + i, "server", "t-0", "default"); + history = manager.addEntryToProject("sql", 1459362239993L + i, "adj", 100 + i, "server", "t-0", "default"); } assertEquals(kylinConfig.getBadQueryHistoryNum(), history.getEntries().size()); } + @Test + public void testUpdateEntryToProject() throws IOException { + KylinConfig kylinConfig = getTestConfig(); + BadQueryHistoryManager manager = BadQueryHistoryManager.getInstance(kylinConfig); + + manager.addEntryToProject("sql", 1459362239000L, "adj", 100, "server", "t-0", "default"); + BadQueryHistory history = manager.updateEntryToProject("sql", 1459362239000L, "adj2", 120, "server2", "t-1", "default"); + + NavigableSet<BadQueryEntry> entries = history.getEntries(); + BadQueryEntry newEntry = entries.floor(new BadQueryEntry("sql", "adj2", 1459362239000L, 120, "server2", "t-1")); + System.out.println(newEntry); + assertEquals("adj2", newEntry.getAdj()); + assertEquals("server2", newEntry.getServer()); + assertEquals("t-1", newEntry.getThread()); + assertEquals(120, newEntry.getRunningSec()); + } + } http://git-wip-us.apache.org/repos/asf/kylin/blob/30aa60d6/server/src/main/java/org/apache/kylin/rest/service/BadQueryDetector.java ---------------------------------------------------------------------- diff --git a/server/src/main/java/org/apache/kylin/rest/service/BadQueryDetector.java b/server/src/main/java/org/apache/kylin/rest/service/BadQueryDetector.java index 90d095b..9ce7f76 100644 --- a/server/src/main/java/org/apache/kylin/rest/service/BadQueryDetector.java +++ b/server/src/main/java/org/apache/kylin/rest/service/BadQueryDetector.java @@ -24,6 +24,7 @@ import java.net.UnknownHostException; import java.util.ArrayList; import java.util.Collections; import java.util.Comparator; +import java.util.HashSet; import java.util.NavigableSet; import java.util.TreeSet; import java.util.concurrent.ConcurrentMap; @@ -36,6 +37,7 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; import com.google.common.collect.Maps; +import com.google.common.collect.Sets; public class BadQueryDetector extends Thread { @@ -73,7 +75,9 @@ public class BadQueryDetector extends Thread { private void initNotifiers() { this.notifiers.add(new LoggerNotifier()); - this.notifiers.add(new PersistenceNotifier()); + if (kylinConfig.getBadQueryPersistentEnabled()) { + this.notifiers.add(new PersistenceNotifier()); + } } public void registerNotifier(Notifier notifier) { @@ -116,7 +120,6 @@ public class BadQueryDetector extends Thread { } } }); - public PersistenceNotifier() { try { serverHostname = InetAddress.getLocalHost().getHostName(); @@ -132,11 +135,13 @@ public class BadQueryDetector extends Thread { long cachingSeconds = (kylinConfig.getBadQueryDefaultAlertingSeconds() + 1) * 30; Pair<Long, String> sqlPair = new Pair<>(startTime, sql); if (!cacheQueue.contains(sqlPair)) { - badQueryManager.addEntryToProject(sql, adj, startTime, runningSec, serverHostname, t.getName(), project); + badQueryManager.addEntryToProject(sql, startTime, adj, runningSec, serverHostname, t.getName(), project); cacheQueue.add(sqlPair); while (!cacheQueue.isEmpty() && (System.currentTimeMillis() - cacheQueue.first().getFirst() > cachingSeconds * 1000 || cacheQueue.size() > kylinConfig.getBadQueryHistoryNum() * 3)) { cacheQueue.pollFirst(); } + } else { + badQueryManager.updateEntryToProject(sql, startTime, adj, runningSec, serverHostname, t.getName(), project); } } catch (IOException e) { logger.error("Error in bad query persistence.", e); http://git-wip-us.apache.org/repos/asf/kylin/blob/30aa60d6/tool/src/main/java/org/apache/kylin/tool/CubeMetaExtractor.java ---------------------------------------------------------------------- diff --git a/tool/src/main/java/org/apache/kylin/tool/CubeMetaExtractor.java b/tool/src/main/java/org/apache/kylin/tool/CubeMetaExtractor.java index 020f9ca..649eb71 100644 --- a/tool/src/main/java/org/apache/kylin/tool/CubeMetaExtractor.java +++ b/tool/src/main/java/org/apache/kylin/tool/CubeMetaExtractor.java @@ -44,6 +44,7 @@ import org.apache.kylin.job.dao.ExecutableDao; import org.apache.kylin.job.dao.ExecutablePO; import org.apache.kylin.job.exception.PersistentException; import org.apache.kylin.metadata.MetadataManager; +import org.apache.kylin.metadata.badquery.BadQueryHistoryManager; import org.apache.kylin.metadata.model.DataModelDesc; import org.apache.kylin.metadata.model.SegmentStatusEnum; import org.apache.kylin.metadata.model.TableDesc; @@ -98,6 +99,7 @@ public class CubeMetaExtractor extends AbstractApplication { private CubeDescManager cubeDescManager; private ExecutableDao executableDao; private RealizationRegistry realizationRegistry; + private BadQueryHistoryManager badQueryHistoryManager; boolean includeSegments; boolean includeJobs; @@ -158,17 +160,20 @@ public class CubeMetaExtractor extends AbstractApplication { kafkaConfigManager = KafkaConfigManager.getInstance(kylinConfig); executableDao = ExecutableDao.getInstance(kylinConfig); realizationRegistry = RealizationRegistry.getInstance(kylinConfig); + badQueryHistoryManager = BadQueryHistoryManager.getInstance(kylinConfig); if (optionsHelper.hasOption(OPTION_PROJECT)) { - ProjectInstance projectInstance = projectManager.getProject(optionsHelper.getOptionValue(OPTION_PROJECT)); + String projectName = optionsHelper.getOptionValue(OPTION_PROJECT); + ProjectInstance projectInstance = projectManager.getProject(projectName); if (projectInstance == null) { - throw new IllegalArgumentException("Project " + optionsHelper.getOptionValue(OPTION_PROJECT) + " does not exist"); + throw new IllegalArgumentException("Project " + projectName + " does not exist"); } - addRequired(ProjectInstance.concatResourcePath(projectInstance.getName())); + addRequired(projectInstance.getResourcePath()); List<RealizationEntry> realizationEntries = projectInstance.getRealizationEntries(); for (RealizationEntry realizationEntry : realizationEntries) { retrieveResourcePath(getRealization(realizationEntry)); } + addOptional(badQueryHistoryManager.getBadQueriesForProject(projectName).getResourcePath()); } else if (optionsHelper.hasOption(OPTION_CUBE)) { String cubeName = optionsHelper.getOptionValue(OPTION_CUBE); IRealization realization;
