KYLIN-1816 More than one base KylinConfig exist in spring JVM
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/1b0ece4a Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/1b0ece4a Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/1b0ece4a Branch: refs/heads/stream_m1 Commit: 1b0ece4a433ca3417d13a6d5fc7b5c1e503e341b Parents: b514ae1 Author: Hongbin Ma <mahong...@apache.org> Authored: Thu Jun 23 12:04:32 2016 +0800 Committer: Hongbin Ma <mahong...@apache.org> Committed: Thu Jun 23 12:04:43 2016 +0800 ---------------------------------------------------------------------- .../org/apache/kylin/common/KylinConfig.java | 78 +++++++++++++------- .../org/apache/kylin/common/KylinConfigExt.java | 5 ++ .../kylin/common/restclient/Broadcaster.java | 2 +- .../common/util/SerializeToByteBuffer.java | 44 +++++++++++ .../java/org/apache/kylin/cube/CubeManager.java | 9 ++- .../apache/kylin/gridtable/GTScanRequest.java | 13 +++- .../apache/kylin/dict/DictionaryManager.java | 3 + .../kylin/dict/lookup/SnapshotManager.java | 21 +++--- .../org/apache/kylin/job/dao/ExecutableDao.java | 3 + .../kylin/job/execution/AbstractExecutable.java | 2 +- .../kylin/job/manager/ExecutableManager.java | 3 + .../realization/RealizationRegistry.java | 3 + .../kylin/storage/hybrid/HybridManager.java | 2 +- .../engine/streaming/StreamingManager.java | 2 +- .../kylin/source/kafka/KafkaConfigManager.java | 2 +- 15 files changed, 147 insertions(+), 45 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/kylin/blob/1b0ece4a/core-common/src/main/java/org/apache/kylin/common/KylinConfig.java ---------------------------------------------------------------------- diff --git a/core-common/src/main/java/org/apache/kylin/common/KylinConfig.java b/core-common/src/main/java/org/apache/kylin/common/KylinConfig.java index b4b3c1b..a4024ae 100644 --- a/core-common/src/main/java/org/apache/kylin/common/KylinConfig.java +++ b/core-common/src/main/java/org/apache/kylin/common/KylinConfig.java @@ -61,22 +61,28 @@ public class KylinConfig extends KylinConfigBase { private static KylinConfig ENV_INSTANCE = null; public static KylinConfig getInstanceFromEnv() { - if (ENV_INSTANCE == null) { - try { - KylinConfig config = loadKylinConfig(); - ENV_INSTANCE = config; - } catch (IllegalArgumentException e) { - throw new IllegalStateException("Failed to find KylinConfig ", e); + synchronized (KylinConfig.class) { + if (ENV_INSTANCE == null) { + try { + KylinConfig config = loadKylinConfig(); + logger.info("Initialized a new KylinConfig from getInstanceFromEnv : " + System.identityHashCode(config)); + ENV_INSTANCE = config; + } catch (IllegalArgumentException e) { + throw new IllegalStateException("Failed to find KylinConfig ", e); + } } + return ENV_INSTANCE; } - return ENV_INSTANCE; } + //Only used in test cases!!! public static void destroyInstance() { + logger.info("Destory KylinConfig"); + dumpStackTrace(); ENV_INSTANCE = null; } - public static enum UriType { + public enum UriType { PROPERTIES_FILE, REST_ADDR, LOCAL_FOLDER } @@ -150,12 +156,15 @@ public class KylinConfig extends KylinConfigBase { } public static void setKylinConfigFromInputStream(InputStream is) { - if (ENV_INSTANCE == null) { - try { - KylinConfig config = createKylinConfigFromInputStream(is); - ENV_INSTANCE = config; - } catch (IllegalArgumentException e) { - throw new IllegalStateException("Failed to find KylinConfig ", e); + synchronized (KylinConfig.class) { + if (ENV_INSTANCE == null) { + try { + KylinConfig config = createKylinConfigFromInputStream(is); + logger.info("Resetting ENV_INSTANCE by a input stream: " + System.identityHashCode(config)); + ENV_INSTANCE = config; + } catch (IllegalArgumentException e) { + throw new IllegalStateException("Failed to find KylinConfig ", e); + } } } } @@ -254,11 +263,11 @@ public class KylinConfig extends KylinConfigBase { return config; } - + public static void setSandboxEnvIfPossible() { File dir1 = new File("../examples/test_case_data/sandbox"); File dir2 = new File("../../kylin/examples/test_case_data/sandbox"); - + if (dir1.exists()) { logger.info("Setting sandbox env, KYLIN_CONF=" + dir1.getAbsolutePath()); ClassUtil.addClasspath(dir1.getAbsolutePath()); @@ -272,11 +281,13 @@ public class KylinConfig extends KylinConfigBase { // ============================================================================ - public KylinConfig() { + private KylinConfig() { super(); + logger.info("New KylinConfig " + System.identityHashCode(this)); + KylinConfig.dumpStackTrace(); } - public KylinConfig(Properties props) { + protected KylinConfig(Properties props) { super(props); } @@ -305,22 +316,19 @@ public class KylinConfig extends KylinConfigBase { } } - private KylinConfig base() { - if (this instanceof KylinConfigExt) - return ((KylinConfigExt) this).base; - else - return this; + public KylinConfig base() { + return this; } - + private int superHashCode() { return super.hashCode(); } - + @Override public int hashCode() { return base().superHashCode(); } - + @Override public boolean equals(Object another) { if (!(another instanceof KylinConfig)) @@ -329,7 +337,6 @@ public class KylinConfig extends KylinConfigBase { return this.base() == ((KylinConfig) another).base(); } - public static void writeOverrideProperties(Properties properties) throws IOException { File propFile = getKylinProperties(); File overrideFile = new File(propFile.getParentFile(), propFile.getName() + ".override"); @@ -362,4 +369,21 @@ public class KylinConfig extends KylinConfigBase { } } + + private static void dumpStackTrace() { + Thread t = Thread.currentThread(); + int maxStackTraceDepth = 20; + int current = 0; + + StackTraceElement[] stackTrace = t.getStackTrace(); + StringBuilder buf = new StringBuilder(""); + buf.append("\n"); + for (StackTraceElement e : stackTrace) { + if (++current > maxStackTraceDepth) { + break; + } + buf.append("\t").append("at ").append(e.toString()).append("\n"); + } + logger.info(buf.toString()); + } } http://git-wip-us.apache.org/repos/asf/kylin/blob/1b0ece4a/core-common/src/main/java/org/apache/kylin/common/KylinConfigExt.java ---------------------------------------------------------------------- diff --git a/core-common/src/main/java/org/apache/kylin/common/KylinConfigExt.java b/core-common/src/main/java/org/apache/kylin/common/KylinConfigExt.java index 9443a4a..c5eb300 100644 --- a/core-common/src/main/java/org/apache/kylin/common/KylinConfigExt.java +++ b/core-common/src/main/java/org/apache/kylin/common/KylinConfigExt.java @@ -68,4 +68,9 @@ public class KylinConfigExt extends KylinConfig { return result; } + @Override + public KylinConfig base() { + return this.base; + } + } http://git-wip-us.apache.org/repos/asf/kylin/blob/1b0ece4a/core-common/src/main/java/org/apache/kylin/common/restclient/Broadcaster.java ---------------------------------------------------------------------- diff --git a/core-common/src/main/java/org/apache/kylin/common/restclient/Broadcaster.java b/core-common/src/main/java/org/apache/kylin/common/restclient/Broadcaster.java index f8cd3be..b87cc25 100644 --- a/core-common/src/main/java/org/apache/kylin/common/restclient/Broadcaster.java +++ b/core-common/src/main/java/org/apache/kylin/common/restclient/Broadcaster.java @@ -62,7 +62,7 @@ public class Broadcaster { r = new Broadcaster(config); CACHE.put(config, r); if (CACHE.size() > 1) { - logger.warn("More than one cubemanager singleton exist"); + logger.warn("More than one singleton exist"); } return r; } http://git-wip-us.apache.org/repos/asf/kylin/blob/1b0ece4a/core-common/src/main/java/org/apache/kylin/common/util/SerializeToByteBuffer.java ---------------------------------------------------------------------- diff --git a/core-common/src/main/java/org/apache/kylin/common/util/SerializeToByteBuffer.java b/core-common/src/main/java/org/apache/kylin/common/util/SerializeToByteBuffer.java new file mode 100644 index 0000000..a2252a4 --- /dev/null +++ b/core-common/src/main/java/org/apache/kylin/common/util/SerializeToByteBuffer.java @@ -0,0 +1,44 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.kylin.common.util; + +import java.nio.BufferOverflowException; +import java.nio.ByteBuffer; + +public class SerializeToByteBuffer { + + public interface IWriter { + void write(ByteBuffer byteBuffer) throws BufferOverflowException; + } + + public static ByteBuffer retrySerialize(IWriter writer) { + int bufferSize = BytesSerializer.SERIALIZE_BUFFER_SIZE; + + while (true) { + try { + ByteBuffer byteBuffer = ByteBuffer.allocate(bufferSize); + writer.write(byteBuffer); + return byteBuffer; + } catch (BufferOverflowException boe) { + System.out.println("Buffer size cannot hold the raw scans, resizing to 4 times : " + bufferSize); + bufferSize *= 4; + } + } + } +} http://git-wip-us.apache.org/repos/asf/kylin/blob/1b0ece4a/core-cube/src/main/java/org/apache/kylin/cube/CubeManager.java ---------------------------------------------------------------------- diff --git a/core-cube/src/main/java/org/apache/kylin/cube/CubeManager.java b/core-cube/src/main/java/org/apache/kylin/cube/CubeManager.java index caa2765..b62bc05 100644 --- a/core-cube/src/main/java/org/apache/kylin/cube/CubeManager.java +++ b/core-cube/src/main/java/org/apache/kylin/cube/CubeManager.java @@ -101,7 +101,10 @@ public class CubeManager implements IRealizationProvider { r = new CubeManager(config); CACHE.put(config, r); if (CACHE.size() > 1) { - logger.warn("More than one cubemanager singleton exist"); + logger.warn("More than one singleton exist"); + for (KylinConfig kylinConfig : CACHE.keySet()) { + logger.warn("type: " + kylinConfig.getClass() + " reference: " + System.identityHashCode(kylinConfig.base())); + } } return r; } catch (IOException e) { @@ -149,7 +152,7 @@ public class CubeManager implements IRealizationProvider { } return null; } - + /** * Get related Cubes by cubedesc name. By default, the desc name will be * translated into upper case. @@ -411,7 +414,7 @@ public class CubeManager implements IRealizationProvider { public CubeSegment appendSegment(CubeInstance cube, long startDate, long endDate, long startOffset, long endOffset, boolean strictChecking) throws IOException { - if(strictChecking) + if (strictChecking) checkNoBuildingSegment(cube); if (cube.getDescriptor().getModel().getPartitionDesc().isPartitioned()) { http://git-wip-us.apache.org/repos/asf/kylin/blob/1b0ece4a/core-cube/src/main/java/org/apache/kylin/gridtable/GTScanRequest.java ---------------------------------------------------------------------- diff --git a/core-cube/src/main/java/org/apache/kylin/gridtable/GTScanRequest.java b/core-cube/src/main/java/org/apache/kylin/gridtable/GTScanRequest.java index d520efa..5e1e131 100644 --- a/core-cube/src/main/java/org/apache/kylin/gridtable/GTScanRequest.java +++ b/core-cube/src/main/java/org/apache/kylin/gridtable/GTScanRequest.java @@ -19,6 +19,7 @@ package org.apache.kylin.gridtable; import java.io.IOException; +import java.nio.BufferOverflowException; import java.nio.ByteBuffer; import java.util.Arrays; import java.util.List; @@ -29,6 +30,7 @@ import org.apache.kylin.common.util.ByteArray; import org.apache.kylin.common.util.BytesSerializer; import org.apache.kylin.common.util.BytesUtil; import org.apache.kylin.common.util.ImmutableBitSet; +import org.apache.kylin.common.util.SerializeToByteBuffer; import org.apache.kylin.metadata.filter.TupleFilter; import org.apache.kylin.metadata.model.TblColRef; @@ -270,6 +272,16 @@ public class GTScanRequest { return "GTScanRequest [range=" + ranges + ", columns=" + columns + ", filterPushDown=" + filterPushDown + ", aggrGroupBy=" + aggrGroupBy + ", aggrMetrics=" + aggrMetrics + ", aggrMetricsFuncs=" + Arrays.toString(aggrMetricsFuncs) + "]"; } + public byte[] toByteArray() { + ByteBuffer byteBuffer = SerializeToByteBuffer.retrySerialize(new SerializeToByteBuffer.IWriter() { + @Override + public void write(ByteBuffer byteBuffer) throws BufferOverflowException { + GTScanRequest.serializer.serialize(GTScanRequest.this, byteBuffer); + } + }); + return Arrays.copyOf(byteBuffer.array(), byteBuffer.position()); + } + public static final BytesSerializer<GTScanRequest> serializer = new BytesSerializer<GTScanRequest>() { @Override public void serialize(GTScanRequest value, ByteBuffer out) { @@ -340,7 +352,6 @@ public class GTScanRequest { } return new GTRecord(sInfo, sCols); } - }; http://git-wip-us.apache.org/repos/asf/kylin/blob/1b0ece4a/core-dictionary/src/main/java/org/apache/kylin/dict/DictionaryManager.java ---------------------------------------------------------------------- diff --git a/core-dictionary/src/main/java/org/apache/kylin/dict/DictionaryManager.java b/core-dictionary/src/main/java/org/apache/kylin/dict/DictionaryManager.java index 5668bd8..6a1c22c 100644 --- a/core-dictionary/src/main/java/org/apache/kylin/dict/DictionaryManager.java +++ b/core-dictionary/src/main/java/org/apache/kylin/dict/DictionaryManager.java @@ -63,6 +63,9 @@ public class DictionaryManager { if (r == null) { r = new DictionaryManager(config); CACHE.put(config, r); + if (CACHE.size() > 1) { + logger.warn("More than one singleton exist"); + } } } } http://git-wip-us.apache.org/repos/asf/kylin/blob/1b0ece4a/core-dictionary/src/main/java/org/apache/kylin/dict/lookup/SnapshotManager.java ---------------------------------------------------------------------- diff --git a/core-dictionary/src/main/java/org/apache/kylin/dict/lookup/SnapshotManager.java b/core-dictionary/src/main/java/org/apache/kylin/dict/lookup/SnapshotManager.java index b9c17dd..b4b8011 100644 --- a/core-dictionary/src/main/java/org/apache/kylin/dict/lookup/SnapshotManager.java +++ b/core-dictionary/src/main/java/org/apache/kylin/dict/lookup/SnapshotManager.java @@ -20,7 +20,7 @@ package org.apache.kylin.dict.lookup; import java.io.IOException; import java.util.ArrayList; -import java.util.NavigableSet; +import java.util.NavigableSet; import java.util.concurrent.ConcurrentHashMap; import org.apache.kylin.common.KylinConfig; @@ -45,12 +45,15 @@ public class SnapshotManager { public static SnapshotManager getInstance(KylinConfig config) { SnapshotManager r = SERVICE_CACHE.get(config); if (r == null) { - synchronized (SnapshotManager.class) { - r = SERVICE_CACHE.get(config); - if (r == null) { - r = new SnapshotManager(config); - SERVICE_CACHE.put(config, r); - } + synchronized (SnapshotManager.class) { + r = SERVICE_CACHE.get(config); + if (r == null) { + r = new SnapshotManager(config); + SERVICE_CACHE.put(config, r); + if (SERVICE_CACHE.size() > 1) { + logger.warn("More than one singleton exist"); + } + } } } return r; @@ -140,7 +143,7 @@ public class SnapshotManager { private String checkDupByInfo(SnapshotTable snapshot) throws IOException { ResourceStore store = MetadataManager.getInstance(this.config).getStore(); String resourceDir = snapshot.getResourceDir(); - NavigableSet<String> existings = store.listResources(resourceDir); + NavigableSet<String> existings = store.listResources(resourceDir); if (existings == null) return null; @@ -158,7 +161,7 @@ public class SnapshotManager { private String checkDupByContent(SnapshotTable snapshot) throws IOException { ResourceStore store = MetadataManager.getInstance(this.config).getStore(); String resourceDir = snapshot.getResourceDir(); - NavigableSet<String> existings = store.listResources(resourceDir); + NavigableSet<String> existings = store.listResources(resourceDir); if (existings == null) return null; http://git-wip-us.apache.org/repos/asf/kylin/blob/1b0ece4a/core-job/src/main/java/org/apache/kylin/job/dao/ExecutableDao.java ---------------------------------------------------------------------- diff --git a/core-job/src/main/java/org/apache/kylin/job/dao/ExecutableDao.java b/core-job/src/main/java/org/apache/kylin/job/dao/ExecutableDao.java index 58f845d..8808a56 100644 --- a/core-job/src/main/java/org/apache/kylin/job/dao/ExecutableDao.java +++ b/core-job/src/main/java/org/apache/kylin/job/dao/ExecutableDao.java @@ -55,6 +55,9 @@ public class ExecutableDao { if (r == null) { r = new ExecutableDao(config); CACHE.put(config, r); + if (CACHE.size() > 1) { + logger.warn("More than one singleton exist"); + } } } } http://git-wip-us.apache.org/repos/asf/kylin/blob/1b0ece4a/core-job/src/main/java/org/apache/kylin/job/execution/AbstractExecutable.java ---------------------------------------------------------------------- diff --git a/core-job/src/main/java/org/apache/kylin/job/execution/AbstractExecutable.java b/core-job/src/main/java/org/apache/kylin/job/execution/AbstractExecutable.java index 778763c..aee71f7 100644 --- a/core-job/src/main/java/org/apache/kylin/job/execution/AbstractExecutable.java +++ b/core-job/src/main/java/org/apache/kylin/job/execution/AbstractExecutable.java @@ -113,7 +113,7 @@ public abstract class AbstractExecutable implements Executable, Idempotent { try { result = doWork(executableContext); } catch (Throwable e) { - logger.error("error running Executable", e); + logger.error("error running Executable: " + this.toString()); exception = e; } retry++; http://git-wip-us.apache.org/repos/asf/kylin/blob/1b0ece4a/core-job/src/main/java/org/apache/kylin/job/manager/ExecutableManager.java ---------------------------------------------------------------------- diff --git a/core-job/src/main/java/org/apache/kylin/job/manager/ExecutableManager.java b/core-job/src/main/java/org/apache/kylin/job/manager/ExecutableManager.java index 5138fb4..7b4e0f0 100644 --- a/core-job/src/main/java/org/apache/kylin/job/manager/ExecutableManager.java +++ b/core-job/src/main/java/org/apache/kylin/job/manager/ExecutableManager.java @@ -63,6 +63,9 @@ public class ExecutableManager { if (r == null) { r = new ExecutableManager(config); CACHE.put(config, r); + if (CACHE.size() > 1) { + logger.warn("More than one singleton exist"); + } } } } http://git-wip-us.apache.org/repos/asf/kylin/blob/1b0ece4a/core-metadata/src/main/java/org/apache/kylin/metadata/realization/RealizationRegistry.java ---------------------------------------------------------------------- diff --git a/core-metadata/src/main/java/org/apache/kylin/metadata/realization/RealizationRegistry.java b/core-metadata/src/main/java/org/apache/kylin/metadata/realization/RealizationRegistry.java index e5b0033..b447531 100644 --- a/core-metadata/src/main/java/org/apache/kylin/metadata/realization/RealizationRegistry.java +++ b/core-metadata/src/main/java/org/apache/kylin/metadata/realization/RealizationRegistry.java @@ -54,6 +54,9 @@ public class RealizationRegistry { try { r = new RealizationRegistry(config); CACHE.put(config, r); + if (CACHE.size() > 1) { + logger.warn("More than one singleton exist"); + } return r; } catch (IOException e) { throw new IllegalStateException("Failed to init CubeManager from " + config, e); http://git-wip-us.apache.org/repos/asf/kylin/blob/1b0ece4a/core-storage/src/main/java/org/apache/kylin/storage/hybrid/HybridManager.java ---------------------------------------------------------------------- diff --git a/core-storage/src/main/java/org/apache/kylin/storage/hybrid/HybridManager.java b/core-storage/src/main/java/org/apache/kylin/storage/hybrid/HybridManager.java index 5f16b6b..e43331e 100644 --- a/core-storage/src/main/java/org/apache/kylin/storage/hybrid/HybridManager.java +++ b/core-storage/src/main/java/org/apache/kylin/storage/hybrid/HybridManager.java @@ -59,7 +59,7 @@ public class HybridManager implements IRealizationProvider { r = new HybridManager(config); CACHE.put(config, r); if (CACHE.size() > 1) { - logger.warn("More than one Hybrid Manager singleton exist"); + logger.warn("More than one singleton exist"); } return r; } catch (IOException e) { http://git-wip-us.apache.org/repos/asf/kylin/blob/1b0ece4a/engine-streaming/src/main/java/org/apache/kylin/engine/streaming/StreamingManager.java ---------------------------------------------------------------------- diff --git a/engine-streaming/src/main/java/org/apache/kylin/engine/streaming/StreamingManager.java b/engine-streaming/src/main/java/org/apache/kylin/engine/streaming/StreamingManager.java index f652762..b45e313 100644 --- a/engine-streaming/src/main/java/org/apache/kylin/engine/streaming/StreamingManager.java +++ b/engine-streaming/src/main/java/org/apache/kylin/engine/streaming/StreamingManager.java @@ -95,7 +95,7 @@ public class StreamingManager { r = new StreamingManager(config); CACHE.put(config, r); if (CACHE.size() > 1) { - logger.warn("More than one streamingManager singleton exist"); + logger.warn("More than one singleton exist"); } return r; } catch (IOException e) { http://git-wip-us.apache.org/repos/asf/kylin/blob/1b0ece4a/source-kafka/src/main/java/org/apache/kylin/source/kafka/KafkaConfigManager.java ---------------------------------------------------------------------- diff --git a/source-kafka/src/main/java/org/apache/kylin/source/kafka/KafkaConfigManager.java b/source-kafka/src/main/java/org/apache/kylin/source/kafka/KafkaConfigManager.java index 1d07f23..d594873 100644 --- a/source-kafka/src/main/java/org/apache/kylin/source/kafka/KafkaConfigManager.java +++ b/source-kafka/src/main/java/org/apache/kylin/source/kafka/KafkaConfigManager.java @@ -96,7 +96,7 @@ public class KafkaConfigManager { r = new KafkaConfigManager(config); CACHE.put(config, r); if (CACHE.size() > 1) { - logger.warn("More than one KafkaConfigManager singleton exist"); + logger.warn("More than one singleton exist"); } return r; } catch (IOException e) {
