minor, code clean up in KylinConfig
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/4b3a44b9 Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/4b3a44b9 Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/4b3a44b9 Branch: refs/heads/master Commit: 4b3a44b92442e9be056baa953e2543078ea0ec20 Parents: 7f47b51 Author: Li Yang <liy...@apache.org> Authored: Sun Aug 6 21:48:25 2017 +0800 Committer: Hongbin Ma <m...@kyligence.io> Committed: Fri Sep 1 17:51:08 2017 +0800 ---------------------------------------------------------------------- .../org/apache/kylin/common/KylinConfig.java | 227 ++++++++----------- .../apache/kylin/common/KylinConfigBase.java | 4 +- .../org/apache/kylin/common/KylinConfigExt.java | 6 +- .../kylin/common/persistence/ResourceStore.java | 2 +- .../common/HotLoadKylinPropertiesTestCase.java | 64 ++++++ .../apache/kylin/common/KylinConfigTest.java | 3 +- .../util/HotLoadKylinPropertiesTestCase.java | 64 ------ .../kylin/cube/CubeSpecificConfigTest.java | 4 +- .../apache/kylin/job/JobEngineConfigTest.java | 4 +- .../engine/mr/common/AbstractHadoopJob.java | 2 +- .../spark/SparkBatchCubingJobBuilder2.java | 2 +- .../kylin/engine/spark/SparkExecutable.java | 13 +- .../kylin/job/BaseTestDistributedScheduler.java | 4 +- .../kylin/rest/controller/AdminController.java | 2 +- .../kylin/rest/controller/CacheController.java | 2 +- .../rest/controller2/AdminControllerV2.java | 2 +- .../rest/controller2/CacheControllerV2.java | 2 +- .../apache/kylin/rest/service/AdminService.java | 4 +- .../hbase/cube/v2/CubeHBaseEndpointRPC.java | 2 +- .../org/apache/kylin/tool/KylinConfigCLI.java | 3 +- 20 files changed, 186 insertions(+), 230 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/kylin/blob/4b3a44b9/core-common/src/main/java/org/apache/kylin/common/KylinConfig.java ---------------------------------------------------------------------- diff --git a/core-common/src/main/java/org/apache/kylin/common/KylinConfig.java b/core-common/src/main/java/org/apache/kylin/common/KylinConfig.java index dfb9a28..a003638 100644 --- a/core-common/src/main/java/org/apache/kylin/common/KylinConfig.java +++ b/core-common/src/main/java/org/apache/kylin/common/KylinConfig.java @@ -34,12 +34,10 @@ import java.io.FileOutputStream; import java.io.IOException; import java.io.InputStream; import java.io.InputStreamReader; -import java.io.PrintWriter; import java.io.StringReader; import java.net.URL; import java.nio.ByteOrder; import java.nio.charset.Charset; -import java.util.Enumeration; import java.util.Map; import java.util.Properties; @@ -111,7 +109,7 @@ public class KylinConfig extends KylinConfigBase { if (SYS_ENV_INSTANCE == null) { try { config = new KylinConfig(); - config.reloadKylinConfig(getKylinProperties()); + config.reloadKylinConfig(buildSiteProperties()); logger.info("Initialized a new KylinConfig from getInstanceFromEnv : " + System.identityHashCode(config)); @@ -124,7 +122,7 @@ public class KylinConfig extends KylinConfigBase { } } - //Only used in test cases!!! + // Only used in test cases!!! public static void destroyInstance() { synchronized (KylinConfig.class) { logger.info("Destroy KylinConfig"); @@ -134,6 +132,26 @@ public class KylinConfig extends KylinConfigBase { } } + private static void dumpStackTrace() { + + //uncomment below to start debugging + + // Thread t = Thread.currentThread(); + // int maxStackTraceDepth = 20; + // int current = 0; + // + // StackTraceElement[] stackTrace = t.getStackTrace(); + // StringBuilder buf = new StringBuilder("This is not a exception, just for diagnose purpose:"); + // buf.append("\n"); + // for (StackTraceElement e : stackTrace) { + // if (++current > maxStackTraceDepth) { + // break; + // } + // buf.append("\t").append("at ").append(e.toString()).append("\n"); + // } + // logger.info(buf.toString()); + } + public enum UriType { PROPERTIES_FILE, REST_ADDR, LOCAL_FOLDER, HDFS_FILE } @@ -260,25 +278,16 @@ public class KylinConfig extends KylinConfigBase { return kylinConfig; } - public static String getKylinConfPath() { - String kylinConfHome = System.getProperty(KYLIN_CONF); - if (!StringUtils.isEmpty(kylinConfHome)) { - logger.info("Use KYLIN_CONF=" + kylinConfHome); - return kylinConfHome; - } - - String kylinHome = getKylinHome(); - if (StringUtils.isEmpty(kylinHome)) - throw new KylinConfigCannotInitException("Didn't find KYLIN_CONF or KYLIN_HOME, please set one of them"); - - return kylinHome + File.separator + "conf"; + public static File getKylinConfDir() { + return getSitePropertiesFile().getParentFile(); } - public static File getKylinPropertiesFile() { + // should be private; package visible for test only + static File getSitePropertiesFile() { String kylinConfHome = System.getProperty(KYLIN_CONF); if (!StringUtils.isEmpty(kylinConfHome)) { logger.info("Use KYLIN_CONF=" + kylinConfHome); - return getKylinPropertiesFile(kylinConfHome); + return existFile(kylinConfHome); } logger.debug("KYLIN_CONF property was not set, will seek KYLIN_HOME env variable"); @@ -288,13 +297,25 @@ public class KylinConfig extends KylinConfigBase { throw new KylinConfigCannotInitException("Didn't find KYLIN_CONF or KYLIN_HOME, please set one of them"); String path = kylinHome + File.separator + "conf"; - return getKylinPropertiesFile(path); + return existFile(path); } - public static Properties getKylinProperties() { + /** + * Return a File only if it exists + */ + private static File existFile(String path) { + if (path == null) { + return null; + } + + return new File(path, KYLIN_CONF_PROPERTIES_FILE); + } + + // build kylin properties from site deployment, a.k.a KYLIN_HOME/conf/kylin.properties + private static Properties buildSiteProperties() { Properties conf = new Properties(); - OrderedProperties orderedProperties = getKylinOrderedProperties(); + OrderedProperties orderedProperties = buildSiteOrderedProps(); for (Map.Entry<String, String> each : orderedProperties.entrySet()) { conf.put(each.getKey(), each.getValue()); } @@ -302,27 +323,8 @@ public class KylinConfig extends KylinConfigBase { return conf; } - /** - * will close the passed in inputstream - */ - private static void loadPropertiesFromInputStream(InputStream inputStream, OrderedProperties properties) { - Preconditions.checkNotNull(properties); - BufferedReader confReader = null; - try { - confReader = new BufferedReader(new InputStreamReader(inputStream, "UTF-8")); - OrderedProperties temp = new OrderedProperties(); - temp.load(confReader); - temp = BCC.check(temp); - - properties.putAll(temp); - } catch (Exception e) { - throw new RuntimeException(e); - } finally { - IOUtils.closeQuietly(confReader); - } - } - - private static OrderedProperties getKylinOrderedProperties() { + // build kylin properties from site deployment, a.k.a KYLIN_HOME/conf/kylin.properties + private static OrderedProperties buildSiteOrderedProps() { try { // 1. load default configurations from classpath. @@ -344,7 +346,7 @@ public class KylinConfig extends KylinConfigBase { // 2. load site conf, to keep backward compatibility it's still named kylin.properties // actually it's better to be named kylin-site.properties - File propFile = getKylinPropertiesFile(); + File propFile = getSitePropertiesFile(); if (propFile == null || !propFile.exists()) { logger.error("fail to locate " + KYLIN_CONF_PROPERTIES_FILE); throw new RuntimeException("fail to locate " + KYLIN_CONF_PROPERTIES_FILE); @@ -363,39 +365,24 @@ public class KylinConfig extends KylinConfigBase { } } - public String getConfigAsString() throws IOException { - Properties allProps = getAllProperties(); - OrderedProperties orderedProperties = KylinConfig.getKylinOrderedProperties(); - - final StringBuilder sb = new StringBuilder(); - - for (Map.Entry<Object, Object> entry : allProps.entrySet()) { - String key = entry.getKey().toString(); - String value = entry.getValue().toString(); - if (!orderedProperties.containsProperty(key)) { - orderedProperties.setProperty(key, value); - } else if (!orderedProperties.getProperty(key).equalsIgnoreCase(value)) { - orderedProperties.setProperty(key, value); - } - } - for (Map.Entry<String, String> entry : orderedProperties.entrySet()) { - sb.append(entry.getKey() + "=" + entry.getValue()).append('\n'); - } - return sb.toString(); - } - /** - * Check if there is kylin.properties exist - * - * @param path - * @return the properties file + * will close the passed in inputstream */ - private static File getKylinPropertiesFile(String path) { - if (path == null) { - return null; - } + private static void loadPropertiesFromInputStream(InputStream inputStream, OrderedProperties properties) { + Preconditions.checkNotNull(properties); + BufferedReader confReader = null; + try { + confReader = new BufferedReader(new InputStreamReader(inputStream, "UTF-8")); + OrderedProperties temp = new OrderedProperties(); + temp.load(confReader); + temp = BCC.check(temp); - return new File(path, KYLIN_CONF_PROPERTIES_FILE); + properties.putAll(temp); + } catch (Exception e) { + throw new RuntimeException(e); + } finally { + IOUtils.closeQuietly(confReader); + } } public static void setSandboxEnvIfPossible() { @@ -423,17 +410,35 @@ public class KylinConfig extends KylinConfigBase { super(props, force); } - public void writeProperties(Properties props, File file) throws IOException { - FileOutputStream fos = null; - try { - fos = new FileOutputStream(file); - props.store(fos, file.getAbsolutePath()); - } finally { - IOUtils.closeQuietly(fos); + public Properties exportToProperties() { + Properties all = getAllProperties(); + Properties copy = new Properties(); + copy.putAll(all); + return copy; + } + + public String exportToString() throws IOException { + Properties allProps = getAllProperties(); + OrderedProperties orderedProperties = KylinConfig.buildSiteOrderedProps(); + + final StringBuilder sb = new StringBuilder(); + + for (Map.Entry<Object, Object> entry : allProps.entrySet()) { + String key = entry.getKey().toString(); + String value = entry.getValue().toString(); + if (!orderedProperties.containsProperty(key)) { + orderedProperties.setProperty(key, value); + } else if (!orderedProperties.getProperty(key).equalsIgnoreCase(value)) { + orderedProperties.setProperty(key, value); + } + } + for (Map.Entry<String, String> entry : orderedProperties.entrySet()) { + sb.append(entry.getKey() + "=" + entry.getValue()).append('\n'); } + return sb.toString(); } - public void writeProperties(File file) throws IOException { + public void exportToFile(File file) throws IOException { FileOutputStream fos = null; try { fos = new FileOutputStream(file); @@ -443,6 +448,10 @@ public class KylinConfig extends KylinConfigBase { } } + public synchronized void reloadFromSiteProperties() { + reloadKylinConfig(buildSiteProperties()); + } + public KylinConfig base() { return this; } @@ -464,60 +473,4 @@ public class KylinConfig extends KylinConfigBase { return this.base() == ((KylinConfig) another).base(); } - public static void writeOverrideProperties(Properties properties) throws IOException { - File propFile = getKylinPropertiesFile(); - File overrideFile = new File(propFile.getParentFile(), propFile.getName() + ".override"); - overrideFile.createNewFile(); - FileInputStream fis2 = null; - Properties override = new Properties(); - try { - fis2 = new FileInputStream(overrideFile); - override.load(fis2); - for (Map.Entry<Object, Object> entries : properties.entrySet()) { - override.setProperty(entries.getKey().toString(), entries.getValue().toString()); - } - } catch (IOException e) { - throw new RuntimeException(e); - } finally { - IOUtils.closeQuietly(fis2); - } - - PrintWriter pw = null; - try { - pw = new PrintWriter(overrideFile); - Enumeration<?> e = override.propertyNames(); - while (e.hasMoreElements()) { - String key = (String) e.nextElement(); - pw.println(key + "=" + override.getProperty(key)); - } - pw.close(); - } finally { - IOUtils.closeQuietly(pw); - } - - } - - private static void dumpStackTrace() { - - //uncomment below to start debugging - - // Thread t = Thread.currentThread(); - // int maxStackTraceDepth = 20; - // int current = 0; - // - // StackTraceElement[] stackTrace = t.getStackTrace(); - // StringBuilder buf = new StringBuilder("This is not a exception, just for diagnose purpose:"); - // buf.append("\n"); - // for (StackTraceElement e : stackTrace) { - // if (++current > maxStackTraceDepth) { - // break; - // } - // buf.append("\t").append("at ").append(e.toString()).append("\n"); - // } - // logger.info(buf.toString()); - } - - public synchronized void hotLoadKylinProperties() { - reloadKylinConfig(getKylinProperties()); - } } http://git-wip-us.apache.org/repos/asf/kylin/blob/4b3a44b9/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java ---------------------------------------------------------------------- diff --git a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java index 0e990df..e5fc3d6 100644 --- a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java +++ b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java @@ -147,7 +147,7 @@ abstract public class KylinConfigBase implements Serializable { return intArray; } - final public String getRequired(String prop) { + final protected String getRequired(String prop) { String r = getOptional(prop); if (StringUtils.isEmpty(r)) { throw new IllegalArgumentException("missing '" + prop + "' in conf/kylin.properties"); @@ -156,7 +156,7 @@ abstract public class KylinConfigBase implements Serializable { } /** - * Use with care, properties should be read-only. This is for testing mostly. + * Use with care, properties should be read-only. This is for testing only. */ final public void setProperty(String key, String value) { logger.info("Kylin Config was updated with " + key + " : " + value); http://git-wip-us.apache.org/repos/asf/kylin/blob/4b3a44b9/core-common/src/main/java/org/apache/kylin/common/KylinConfigExt.java ---------------------------------------------------------------------- diff --git a/core-common/src/main/java/org/apache/kylin/common/KylinConfigExt.java b/core-common/src/main/java/org/apache/kylin/common/KylinConfigExt.java index 786f467..0e1b6b9 100644 --- a/core-common/src/main/java/org/apache/kylin/common/KylinConfigExt.java +++ b/core-common/src/main/java/org/apache/kylin/common/KylinConfigExt.java @@ -27,7 +27,7 @@ import java.util.Properties; @SuppressWarnings("serial") public class KylinConfigExt extends KylinConfig { - final private Map<String, String> overrides; + final Map<String, String> overrides; final KylinConfig base; public static KylinConfigExt createInstance(KylinConfig kylinConfig, Map<String, String> overrides) { @@ -61,7 +61,7 @@ public class KylinConfigExt extends KylinConfig { return super.getOptional(prop, dft); } - public Properties getAllProperties() { + protected Properties getAllProperties() { Properties result = new Properties(); result.putAll(super.getAllProperties()); result.putAll(overrides); @@ -71,7 +71,7 @@ public class KylinConfigExt extends KylinConfig { public Map<String, String> getExtendedOverrides() { return overrides; } - + @Override public KylinConfig base() { return this.base; http://git-wip-us.apache.org/repos/asf/kylin/blob/4b3a44b9/core-common/src/main/java/org/apache/kylin/common/persistence/ResourceStore.java ---------------------------------------------------------------------- diff --git a/core-common/src/main/java/org/apache/kylin/common/persistence/ResourceStore.java b/core-common/src/main/java/org/apache/kylin/common/persistence/ResourceStore.java index ac29dc0..ae16bca 100644 --- a/core-common/src/main/java/org/apache/kylin/common/persistence/ResourceStore.java +++ b/core-common/src/main/java/org/apache/kylin/common/persistence/ResourceStore.java @@ -454,7 +454,7 @@ abstract public class ResourceStore { // write kylin.properties File kylinPropsFile = new File(metaDir, "kylin.properties"); - kylinConfig.writeProperties(kylinPropsFile); + kylinConfig.exportToFile(kylinPropsFile); ResourceStore from = ResourceStore.getStore(kylinConfig); KylinConfig localConfig = KylinConfig.createInstanceFromUri(metaDir.getAbsolutePath()); http://git-wip-us.apache.org/repos/asf/kylin/blob/4b3a44b9/core-common/src/test/java/org/apache/kylin/common/HotLoadKylinPropertiesTestCase.java ---------------------------------------------------------------------- diff --git a/core-common/src/test/java/org/apache/kylin/common/HotLoadKylinPropertiesTestCase.java b/core-common/src/test/java/org/apache/kylin/common/HotLoadKylinPropertiesTestCase.java new file mode 100644 index 0000000..ecbac33 --- /dev/null +++ b/core-common/src/test/java/org/apache/kylin/common/HotLoadKylinPropertiesTestCase.java @@ -0,0 +1,64 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. +*/ + +package org.apache.kylin.common; + +import java.io.File; +import java.io.FileInputStream; +import java.io.FileOutputStream; +import java.util.Properties; + +import org.apache.kylin.common.util.LocalFileMetadataTestCase; +import org.junit.After; +import org.junit.Before; + +/** + * @author kangkaisen + */ + +public class HotLoadKylinPropertiesTestCase extends LocalFileMetadataTestCase { + @Before + public void setUp() throws Exception { + this.createTestMetadata(); + } + + @After + public void after() throws Exception { + this.cleanupTestMetadata(); + } + + protected void updateProperty(String key, String value) { + File propFile = KylinConfig.getSitePropertiesFile(); + Properties conf = new Properties(); + + //load + try (FileInputStream is = new FileInputStream(propFile)) { + conf.load(is); + conf.setProperty(key, value); + } catch (Exception e) { + System.err.println(e.getMessage()); + } + + //store + try (FileOutputStream out = new FileOutputStream(propFile)) { + conf.store(out, null); + } catch (Exception e) { + System.err.println(e.getMessage()); + } + } +} http://git-wip-us.apache.org/repos/asf/kylin/blob/4b3a44b9/core-common/src/test/java/org/apache/kylin/common/KylinConfigTest.java ---------------------------------------------------------------------- diff --git a/core-common/src/test/java/org/apache/kylin/common/KylinConfigTest.java b/core-common/src/test/java/org/apache/kylin/common/KylinConfigTest.java index f20ed7c..6da854b 100644 --- a/core-common/src/test/java/org/apache/kylin/common/KylinConfigTest.java +++ b/core-common/src/test/java/org/apache/kylin/common/KylinConfigTest.java @@ -27,7 +27,6 @@ import static org.junit.Assert.assertTrue; import java.util.Map; import java.util.Properties; -import org.apache.kylin.common.util.HotLoadKylinPropertiesTestCase; import org.junit.Test; import com.google.common.collect.Maps; @@ -85,7 +84,7 @@ public class KylinConfigTest extends HotLoadKylinPropertiesTestCase { assertEquals("who...@kylin.apache.org", config.getKylinOwner()); updateProperty("kylin.storage.hbase.owner-tag", "ky...@kylin.apache.org"); - KylinConfig.getInstanceFromEnv().hotLoadKylinProperties(); + KylinConfig.getInstanceFromEnv().reloadFromSiteProperties(); assertEquals("ky...@kylin.apache.org", config.getKylinOwner()); } http://git-wip-us.apache.org/repos/asf/kylin/blob/4b3a44b9/core-common/src/test/java/org/apache/kylin/common/util/HotLoadKylinPropertiesTestCase.java ---------------------------------------------------------------------- diff --git a/core-common/src/test/java/org/apache/kylin/common/util/HotLoadKylinPropertiesTestCase.java b/core-common/src/test/java/org/apache/kylin/common/util/HotLoadKylinPropertiesTestCase.java deleted file mode 100644 index 9f5b278..0000000 --- a/core-common/src/test/java/org/apache/kylin/common/util/HotLoadKylinPropertiesTestCase.java +++ /dev/null @@ -1,64 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. -*/ - -package org.apache.kylin.common.util; - -import org.apache.kylin.common.KylinConfig; -import org.junit.After; -import org.junit.Before; - -import java.io.File; -import java.io.FileInputStream; -import java.io.FileOutputStream; -import java.util.Properties; - -/** - * @author kangkaisen - */ - -public class HotLoadKylinPropertiesTestCase extends LocalFileMetadataTestCase { - @Before - public void setUp() throws Exception { - this.createTestMetadata(); - } - - @After - public void after() throws Exception { - this.cleanupTestMetadata(); - } - - protected void updateProperty(String key, String value) { - File propFile = KylinConfig.getKylinPropertiesFile(); - Properties conf = new Properties(); - - //load - try (FileInputStream is = new FileInputStream(propFile)) { - conf.load(is); - conf.setProperty(key, value); - } catch (Exception e) { - System.err.println(e.getMessage()); - } - - //store - try (FileOutputStream out = new FileOutputStream(propFile)) { - conf.store(out, null); - } catch (Exception e) { - System.err.println(e.getMessage()); - } - } -} http://git-wip-us.apache.org/repos/asf/kylin/blob/4b3a44b9/core-cube/src/test/java/org/apache/kylin/cube/CubeSpecificConfigTest.java ---------------------------------------------------------------------- diff --git a/core-cube/src/test/java/org/apache/kylin/cube/CubeSpecificConfigTest.java b/core-cube/src/test/java/org/apache/kylin/cube/CubeSpecificConfigTest.java index 17c02cc..baaf813 100644 --- a/core-cube/src/test/java/org/apache/kylin/cube/CubeSpecificConfigTest.java +++ b/core-cube/src/test/java/org/apache/kylin/cube/CubeSpecificConfigTest.java @@ -20,8 +20,8 @@ package org.apache.kylin.cube; import static org.junit.Assert.assertEquals; +import org.apache.kylin.common.HotLoadKylinPropertiesTestCase; import org.apache.kylin.common.KylinConfig; -import org.apache.kylin.common.util.HotLoadKylinPropertiesTestCase; import org.apache.kylin.cube.model.CubeDesc; import org.junit.Test; @@ -55,7 +55,7 @@ public class CubeSpecificConfigTest extends HotLoadKylinPropertiesTestCase { //hot load Properties updateProperty("kylin.job.max-concurrent-jobs", "20"); - KylinConfig.getInstanceFromEnv().hotLoadKylinProperties(); + KylinConfig.getInstanceFromEnv().reloadFromSiteProperties(); CubeDescManager.getInstance(baseConfig).reloadCubeDescLocal("ssb"); //test cubeDescConfig http://git-wip-us.apache.org/repos/asf/kylin/blob/4b3a44b9/core-job/src/test/java/org/apache/kylin/job/JobEngineConfigTest.java ---------------------------------------------------------------------- diff --git a/core-job/src/test/java/org/apache/kylin/job/JobEngineConfigTest.java b/core-job/src/test/java/org/apache/kylin/job/JobEngineConfigTest.java index 77914ef..6660c47 100644 --- a/core-job/src/test/java/org/apache/kylin/job/JobEngineConfigTest.java +++ b/core-job/src/test/java/org/apache/kylin/job/JobEngineConfigTest.java @@ -18,8 +18,8 @@ package org.apache.kylin.job; +import org.apache.kylin.common.HotLoadKylinPropertiesTestCase; import org.apache.kylin.common.KylinConfig; -import org.apache.kylin.common.util.HotLoadKylinPropertiesTestCase; import org.apache.kylin.job.engine.JobEngineConfig; import org.junit.Test; @@ -40,7 +40,7 @@ public class JobEngineConfigTest extends HotLoadKylinPropertiesTestCase { assertEquals(10, jobEngineConfig.getMaxConcurrentJobLimit()); updateProperty("kylin.job.max-concurrent-jobs", "20"); - KylinConfig.getInstanceFromEnv().hotLoadKylinProperties(); + KylinConfig.getInstanceFromEnv().reloadFromSiteProperties(); assertEquals(20, jobEngineConfig.getMaxConcurrentJobLimit()); } http://git-wip-us.apache.org/repos/asf/kylin/blob/4b3a44b9/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/AbstractHadoopJob.java ---------------------------------------------------------------------- diff --git a/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/AbstractHadoopJob.java b/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/AbstractHadoopJob.java index 54f77c0..9950c69 100644 --- a/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/AbstractHadoopJob.java +++ b/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/AbstractHadoopJob.java @@ -501,7 +501,7 @@ public abstract class AbstractHadoopJob extends Configured implements Tool { // write kylin.properties File kylinPropsFile = new File(metaDir, "kylin.properties"); - kylinConfig.writeProperties(kylinPropsFile); + kylinConfig.exportToFile(kylinPropsFile); if (prj != null) { dumpList.add(ProjectManager.getInstance(kylinConfig).getProject(prj).getResourcePath()); http://git-wip-us.apache.org/repos/asf/kylin/blob/4b3a44b9/engine-spark/src/main/java/org/apache/kylin/engine/spark/SparkBatchCubingJobBuilder2.java ---------------------------------------------------------------------- diff --git a/engine-spark/src/main/java/org/apache/kylin/engine/spark/SparkBatchCubingJobBuilder2.java b/engine-spark/src/main/java/org/apache/kylin/engine/spark/SparkBatchCubingJobBuilder2.java index f1e6aea..779f340 100644 --- a/engine-spark/src/main/java/org/apache/kylin/engine/spark/SparkBatchCubingJobBuilder2.java +++ b/engine-spark/src/main/java/org/apache/kylin/engine/spark/SparkBatchCubingJobBuilder2.java @@ -49,7 +49,7 @@ public class SparkBatchCubingJobBuilder2 extends BatchCubingJobBuilder2 { sparkExecutable.setParam(SparkCubingByLayer.OPTION_SEGMENT_ID.getOpt(), seg.getUuid()); sparkExecutable.setParam(SparkCubingByLayer.OPTION_INPUT_TABLE.getOpt(), seg.getConfig().getHiveDatabaseForIntermediateTable() + "." + flatTableDesc.getTableName()); sparkExecutable.setParam(SparkCubingByLayer.OPTION_META_URL.getOpt(), getSegmentMetadataUrl(seg.getConfig(), seg.getUuid())); - sparkExecutable.setParam(SparkCubingByLayer.OPTION_CONF_PATH.getOpt(), KylinConfig.getKylinConfPath()); + sparkExecutable.setParam(SparkCubingByLayer.OPTION_CONF_PATH.getOpt(), KylinConfig.getKylinConfDir().getAbsolutePath()); sparkExecutable.setParam(SparkCubingByLayer.OPTION_OUTPUT_PATH.getOpt(), cuboidRootPath); StringBuilder jars = new StringBuilder(); http://git-wip-us.apache.org/repos/asf/kylin/blob/4b3a44b9/engine-spark/src/main/java/org/apache/kylin/engine/spark/SparkExecutable.java ---------------------------------------------------------------------- diff --git a/engine-spark/src/main/java/org/apache/kylin/engine/spark/SparkExecutable.java b/engine-spark/src/main/java/org/apache/kylin/engine/spark/SparkExecutable.java index c211ec5..d369e3d 100644 --- a/engine-spark/src/main/java/org/apache/kylin/engine/spark/SparkExecutable.java +++ b/engine-spark/src/main/java/org/apache/kylin/engine/spark/SparkExecutable.java @@ -18,6 +18,7 @@ package org.apache.kylin.engine.spark; import java.io.File; +import java.io.FileOutputStream; import java.io.IOException; import java.util.LinkedHashSet; import java.util.Map; @@ -164,13 +165,15 @@ public class SparkExecutable extends AbstractExecutable { JobRelatedMetaUtil.dumpResources(kylinConfig, metaDir, dumpList); // write kylin.properties - File kylinPropsFile = new File(metaDir, "kylin.properties"); - Properties properties = kylinConfig.getAllProperties(); + Properties props = kylinConfig.exportToProperties(); String metadataUrl = this.getParam(SparkCubingByLayer.OPTION_META_URL.getOpt()); - properties.setProperty("kylin.metadata.url", metadataUrl); - kylinConfig.writeProperties(properties, kylinPropsFile); + props.setProperty("kylin.metadata.url", metadataUrl); + File kylinPropsFile = new File(metaDir, "kylin.properties"); + try (FileOutputStream os = new FileOutputStream(kylinPropsFile)) { + props.store(os, kylinPropsFile.getAbsolutePath()); + } - KylinConfig dstConfig = KylinConfig.createKylinConfig(properties); + KylinConfig dstConfig = KylinConfig.createKylinConfig(props); //upload metadata ResourceTool.copy(KylinConfig.createInstanceFromUri(metaDir.getAbsolutePath()), dstConfig); } http://git-wip-us.apache.org/repos/asf/kylin/blob/4b3a44b9/kylin-it/src/test/java/org/apache/kylin/job/BaseTestDistributedScheduler.java ---------------------------------------------------------------------- diff --git a/kylin-it/src/test/java/org/apache/kylin/job/BaseTestDistributedScheduler.java b/kylin-it/src/test/java/org/apache/kylin/job/BaseTestDistributedScheduler.java index 1ea6507..644b2b3 100644 --- a/kylin-it/src/test/java/org/apache/kylin/job/BaseTestDistributedScheduler.java +++ b/kylin-it/src/test/java/org/apache/kylin/job/BaseTestDistributedScheduler.java @@ -79,8 +79,8 @@ public class BaseTestDistributedScheduler extends HBaseMetadataTestCase { localMetaDir = Files.createTempDir(); backup = srcConfig.getMetadataUrl().toString(); srcConfig.setProperty("kylin.metadata.url", localMetaDir.getAbsolutePath()); - srcConfig.writeProperties(new File(confDstPath1)); - srcConfig.writeProperties(new File(confDstPath2)); + srcConfig.exportToFile(new File(confDstPath1)); + srcConfig.exportToFile(new File(confDstPath2)); kylinConfig1 = KylinConfig.createInstanceFromUri(new File(confDstPath1).getAbsolutePath()); kylinConfig2 = KylinConfig.createInstanceFromUri(new File(confDstPath2).getAbsolutePath()); http://git-wip-us.apache.org/repos/asf/kylin/blob/4b3a44b9/server-base/src/main/java/org/apache/kylin/rest/controller/AdminController.java ---------------------------------------------------------------------- diff --git a/server-base/src/main/java/org/apache/kylin/rest/controller/AdminController.java b/server-base/src/main/java/org/apache/kylin/rest/controller/AdminController.java index 1252775..eec9fa5 100644 --- a/server-base/src/main/java/org/apache/kylin/rest/controller/AdminController.java +++ b/server-base/src/main/java/org/apache/kylin/rest/controller/AdminController.java @@ -75,7 +75,7 @@ public class AdminController extends BasicController { @RequestMapping(value = "/config", method = { RequestMethod.GET }, produces = { "application/json" }) @ResponseBody public GeneralResponse getConfig() throws IOException { - String config = adminService.getConfigAsString(); + String config = adminService.exportToString(); GeneralResponse configRes = new GeneralResponse(); configRes.put("config", config); http://git-wip-us.apache.org/repos/asf/kylin/blob/4b3a44b9/server-base/src/main/java/org/apache/kylin/rest/controller/CacheController.java ---------------------------------------------------------------------- diff --git a/server-base/src/main/java/org/apache/kylin/rest/controller/CacheController.java b/server-base/src/main/java/org/apache/kylin/rest/controller/CacheController.java index 092220c..992094b 100644 --- a/server-base/src/main/java/org/apache/kylin/rest/controller/CacheController.java +++ b/server-base/src/main/java/org/apache/kylin/rest/controller/CacheController.java @@ -69,7 +69,7 @@ public class CacheController extends BasicController { @RequestMapping(value = "/announce/config", method = { RequestMethod.POST }, produces = { "application/json" }) public void hotLoadKylinConfig() throws IOException { - KylinConfig.getInstanceFromEnv().hotLoadKylinProperties(); + KylinConfig.getInstanceFromEnv().reloadFromSiteProperties(); cacheService.notifyMetadataChange(Broadcaster.SYNC_ALL, Broadcaster.Event.UPDATE, Broadcaster.SYNC_ALL); } http://git-wip-us.apache.org/repos/asf/kylin/blob/4b3a44b9/server-base/src/main/java/org/apache/kylin/rest/controller2/AdminControllerV2.java ---------------------------------------------------------------------- diff --git a/server-base/src/main/java/org/apache/kylin/rest/controller2/AdminControllerV2.java b/server-base/src/main/java/org/apache/kylin/rest/controller2/AdminControllerV2.java index a1028d7..9395fb6 100644 --- a/server-base/src/main/java/org/apache/kylin/rest/controller2/AdminControllerV2.java +++ b/server-base/src/main/java/org/apache/kylin/rest/controller2/AdminControllerV2.java @@ -68,7 +68,7 @@ public class AdminControllerV2 extends BasicController { @ResponseBody public EnvelopeResponse getConfigV2() throws IOException { - return new EnvelopeResponse(ResponseCode.CODE_SUCCESS, adminService.getConfigAsString(), ""); + return new EnvelopeResponse(ResponseCode.CODE_SUCCESS, adminService.exportToString(), ""); } @RequestMapping(value = "/metrics/cubes", method = { RequestMethod.GET }, produces = { http://git-wip-us.apache.org/repos/asf/kylin/blob/4b3a44b9/server-base/src/main/java/org/apache/kylin/rest/controller2/CacheControllerV2.java ---------------------------------------------------------------------- diff --git a/server-base/src/main/java/org/apache/kylin/rest/controller2/CacheControllerV2.java b/server-base/src/main/java/org/apache/kylin/rest/controller2/CacheControllerV2.java index fa786b6..73d6a4f 100644 --- a/server-base/src/main/java/org/apache/kylin/rest/controller2/CacheControllerV2.java +++ b/server-base/src/main/java/org/apache/kylin/rest/controller2/CacheControllerV2.java @@ -81,7 +81,7 @@ public class CacheControllerV2 extends BasicController { @ResponseBody public void hotLoadKylinConfigV2() throws IOException { - KylinConfig.getInstanceFromEnv().hotLoadKylinProperties(); + KylinConfig.getInstanceFromEnv().reloadFromSiteProperties(); cacheService.notifyMetadataChange(Broadcaster.SYNC_ALL, Broadcaster.Event.UPDATE, Broadcaster.SYNC_ALL); } http://git-wip-us.apache.org/repos/asf/kylin/blob/4b3a44b9/server-base/src/main/java/org/apache/kylin/rest/service/AdminService.java ---------------------------------------------------------------------- diff --git a/server-base/src/main/java/org/apache/kylin/rest/service/AdminService.java b/server-base/src/main/java/org/apache/kylin/rest/service/AdminService.java index f9bb7e6..ae004cf 100644 --- a/server-base/src/main/java/org/apache/kylin/rest/service/AdminService.java +++ b/server-base/src/main/java/org/apache/kylin/rest/service/AdminService.java @@ -81,10 +81,10 @@ public class AdminService extends BasicService { * Get Java config info as String */ // @PreAuthorize(Constant.ACCESS_HAS_ROLE_ADMIN) // this is a critical security issue, see KYLIN-1664 - public String getConfigAsString() throws IOException { + public String exportToString() throws IOException { logger.debug("Get Kylin Runtime Config"); - return KylinConfig.getInstanceFromEnv().getConfigAsString(); + return KylinConfig.getInstanceFromEnv().exportToString(); } @PreAuthorize(Constant.ACCESS_HAS_ROLE_ADMIN) http://git-wip-us.apache.org/repos/asf/kylin/blob/4b3a44b9/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeHBaseEndpointRPC.java ---------------------------------------------------------------------- diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeHBaseEndpointRPC.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeHBaseEndpointRPC.java index af8754d..4ac237b 100644 --- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeHBaseEndpointRPC.java +++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeHBaseEndpointRPC.java @@ -156,7 +156,7 @@ public class CubeHBaseEndpointRPC extends CubeHBaseRPC { builder.addHbaseColumnsToGT(intList); } builder.setRowkeyPreambleSize(cubeSeg.getRowKeyPreambleSize()); - builder.setKylinProperties(kylinConfig.getConfigAsString()); + builder.setKylinProperties(kylinConfig.exportToString()); final String queryId = queryContext.getQueryId(); if (queryId != null) { builder.setQueryId(queryId); http://git-wip-us.apache.org/repos/asf/kylin/blob/4b3a44b9/tool/src/main/java/org/apache/kylin/tool/KylinConfigCLI.java ---------------------------------------------------------------------- diff --git a/tool/src/main/java/org/apache/kylin/tool/KylinConfigCLI.java b/tool/src/main/java/org/apache/kylin/tool/KylinConfigCLI.java index 88905ef..7c7b318 100644 --- a/tool/src/main/java/org/apache/kylin/tool/KylinConfigCLI.java +++ b/tool/src/main/java/org/apache/kylin/tool/KylinConfigCLI.java @@ -34,7 +34,8 @@ public class KylinConfigCLI { System.exit(1); } - Properties config = KylinConfig.getKylinProperties(); + Properties config = KylinConfig.getInstanceFromEnv().exportToProperties(); + BackwardCompatibilityConfig bcc = new BackwardCompatibilityConfig(); String key = bcc.check(args[0].trim()); if (!key.endsWith(".")) {
