Repository: kylin Updated Branches: refs/heads/KYLIN-2374 24cae5c73 -> 03e3dc928
Make some properties configable Signed-off-by: shaofengshi <shaofeng...@apache.org> Project: http://git-wip-us.apache.org/repos/asf/kylin/repo Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/03e3dc92 Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/03e3dc92 Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/03e3dc92 Branch: refs/heads/KYLIN-2374 Commit: 03e3dc92862ba846b8098e83c79d8f7f32f774a9 Parents: 24cae5c Author: xiefan46 <958034...@qq.com> Authored: Fri Jan 20 18:15:56 2017 +0800 Committer: shaofengshi <shaofeng...@apache.org> Committed: Fri Jan 20 18:19:11 2017 +0800 ---------------------------------------------------------------------- .../apache/kylin/common/KylinConfigBase.java | 10 +++++++- .../test_case_data/sandbox/kylin.properties | 4 +++ .../storage/hdfs/ITHDFSResourceStoreTest.java | 1 - .../kylin/storage/hdfs/ITLockManagerTest.java | 1 - .../kylin/storage/hdfs/HDFSResourceStore.java | 21 ++++++++------- .../apache/kylin/storage/hdfs/LockManager.java | 19 +++++++------- .../kylin/storage/hdfs/ZookeeperConfig.java | 27 -------------------- 7 files changed, 34 insertions(+), 49 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/kylin/blob/03e3dc92/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java ---------------------------------------------------------------------- diff --git a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java index 0602e9d..a2038aa 100644 --- a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java +++ b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java @@ -775,7 +775,6 @@ abstract public class KylinConfigBase implements Serializable { return Float.valueOf(getOptional("kylin.engine.spark.rdd-partition-cut-mb", "10.0")); } - public int getSparkMinPartition() { return Integer.valueOf(getOptional("kylin.engine.spark.min-partition", "1")); } @@ -925,4 +924,13 @@ abstract public class KylinConfigBase implements Serializable { return Boolean.parseBoolean(getOptional("kylin.web.cross-domain-enabled", "true")); } + //zoo keeper + public String getZooKeeperHost() { + return getOptional("kylin.storage-zookeeper.host", "localhost"); + } + + public String getZooKeeperPort() { + return getOptional("kylin.storage-zookeeper.port", "2181"); + } + } http://git-wip-us.apache.org/repos/asf/kylin/blob/03e3dc92/examples/test_case_data/sandbox/kylin.properties ---------------------------------------------------------------------- diff --git a/examples/test_case_data/sandbox/kylin.properties b/examples/test_case_data/sandbox/kylin.properties index 2d3cd79..cc28541 100644 --- a/examples/test_case_data/sandbox/kylin.properties +++ b/examples/test_case_data/sandbox/kylin.properties @@ -177,3 +177,7 @@ kylin.engine.spark-conf.spark.eventLog.dir=hdfs\:///kylin/spark-history #kylin.engine.spark-conf.spark.yarn.queue=default #kylin.engine.spark-conf.spark.yarn.jar=hdfs://sandbox.hortonworks.com:8020/kylin/spark/spark-assembly-1.6.3-hadoop2.6.0.jar #kylin.engine.spark-conf.spark.io.compression.codec=org.apache.spark.io.SnappyCompressionCodec + + +#zoo keeper +kylin.storage-zookeeper.host=sandbox \ No newline at end of file http://git-wip-us.apache.org/repos/asf/kylin/blob/03e3dc92/kylin-it/src/test/java/org/apache/kylin/storage/hdfs/ITHDFSResourceStoreTest.java ---------------------------------------------------------------------- diff --git a/kylin-it/src/test/java/org/apache/kylin/storage/hdfs/ITHDFSResourceStoreTest.java b/kylin-it/src/test/java/org/apache/kylin/storage/hdfs/ITHDFSResourceStoreTest.java index 41bbcc0..27d8a3c 100644 --- a/kylin-it/src/test/java/org/apache/kylin/storage/hdfs/ITHDFSResourceStoreTest.java +++ b/kylin-it/src/test/java/org/apache/kylin/storage/hdfs/ITHDFSResourceStoreTest.java @@ -24,7 +24,6 @@ import org.apache.kylin.common.persistence.ResourceStoreTest; import org.apache.kylin.common.util.HBaseMetadataTestCase; import org.junit.After; import org.junit.Before; -import org.junit.Ignore; import org.junit.Test; http://git-wip-us.apache.org/repos/asf/kylin/blob/03e3dc92/kylin-it/src/test/java/org/apache/kylin/storage/hdfs/ITLockManagerTest.java ---------------------------------------------------------------------- diff --git a/kylin-it/src/test/java/org/apache/kylin/storage/hdfs/ITLockManagerTest.java b/kylin-it/src/test/java/org/apache/kylin/storage/hdfs/ITLockManagerTest.java index 56347e4..2b58d30 100644 --- a/kylin-it/src/test/java/org/apache/kylin/storage/hdfs/ITLockManagerTest.java +++ b/kylin-it/src/test/java/org/apache/kylin/storage/hdfs/ITLockManagerTest.java @@ -26,7 +26,6 @@ import org.apache.kylin.common.KylinConfig; import org.apache.kylin.common.util.HBaseMetadataTestCase; import org.junit.After; import org.junit.Before; -import org.junit.Ignore; import org.junit.Test; import java.io.Closeable; http://git-wip-us.apache.org/repos/asf/kylin/blob/03e3dc92/storage-hbase/src/main/java/org/apache/kylin/storage/hdfs/HDFSResourceStore.java ---------------------------------------------------------------------- diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hdfs/HDFSResourceStore.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hdfs/HDFSResourceStore.java index 8cb9f0d..a746a97 100644 --- a/storage-hbase/src/main/java/org/apache/kylin/storage/hdfs/HDFSResourceStore.java +++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hdfs/HDFSResourceStore.java @@ -43,14 +43,18 @@ import java.util.concurrent.TimeUnit; public class HDFSResourceStore extends ResourceStore { + private static final Logger logger = LoggerFactory.getLogger(HDFSResourceStore.class); + + private static final long DEFAULT_ACQUIRE_LOCK_TIMEOUT = 10; + private static final String DEFAULT_FOLDER_NAME = "kylin_default_instance"; + private static final String DEFAULT_METADATA_FOLDER_NAME = "hdfs_metadata"; + private Path hdfsMetaPath; private FileSystem fs; - private static final Logger logger = LoggerFactory.getLogger(HDFSResourceStore.class); - private LockManager lockManager; //public for test. Normal should be protected @@ -59,7 +63,7 @@ public class HDFSResourceStore extends ResourceStore { String metadataUrl = kylinConfig.getMetadataUrl(); int cut = metadataUrl.indexOf('@'); String metaDirName = cut < 0 ? DEFAULT_FOLDER_NAME : metadataUrl.substring(0, cut); - metaDirName += "/hdfs_metadata"; + metaDirName += "/" + DEFAULT_METADATA_FOLDER_NAME; logger.info("meta dir name :" + metaDirName); createMetaFolder(metaDirName, kylinConfig); } @@ -73,13 +77,13 @@ public class HDFSResourceStore extends ResourceStore { throw new IOException("HDFS working dir not exist"); } //creat lock manager - this.lockManager = new LockManager(kylinConfig,kylinConfig.getRawHdfsWorkingDirectory() + metaDirName); + this.lockManager = new LockManager(kylinConfig, kylinConfig.getRawHdfsWorkingDirectory() + metaDirName); //create hdfs meta path hdfsMetaPath = new Path(hdfsWorkingPath, metaDirName); if (!fs.exists(hdfsMetaPath)) { ResourceLock lock = lockManager.getLock(lockManager.getLockPath("/")); try { - if (lock.acquire(ZookeeperConfig.DEFAULT_TIMEOUT, TimeUnit.MINUTES)) { + if (lock.acquire(DEFAULT_ACQUIRE_LOCK_TIMEOUT, TimeUnit.MINUTES)) { logger.info("get root lock successfully"); if (!fs.exists(hdfsMetaPath)) { fs.mkdirs(hdfsMetaPath); @@ -163,7 +167,7 @@ public class HDFSResourceStore extends ResourceStore { ResourceLock lock = null; try { lock = lockManager.getLock(resPath); - lock.acquire(ZookeeperConfig.DEFAULT_TIMEOUT, TimeUnit.MINUTES); + lock.acquire(DEFAULT_ACQUIRE_LOCK_TIMEOUT, TimeUnit.MINUTES); in = fs.open(p); long t = in.readLong(); return t; @@ -185,7 +189,7 @@ public class HDFSResourceStore extends ResourceStore { ResourceLock lock = null; try { lock = lockManager.getLock(resPath); - lock.acquire(ZookeeperConfig.DEFAULT_TIMEOUT, TimeUnit.MINUTES); + lock.acquire(DEFAULT_ACQUIRE_LOCK_TIMEOUT, TimeUnit.MINUTES); out = fs.create(p, true); out.writeLong(ts); IOUtils.copy(content, out); @@ -221,7 +225,7 @@ public class HDFSResourceStore extends ResourceStore { ResourceLock lock = null; try { lock = lockManager.getLock(resPath); - lock.acquire(ZookeeperConfig.DEFAULT_TIMEOUT, TimeUnit.MINUTES); + lock.acquire(DEFAULT_ACQUIRE_LOCK_TIMEOUT, TimeUnit.MINUTES); Path p = getRealHDFSPath(resPath); if (fs.exists(p)) { fs.delete(p, true); @@ -244,5 +248,4 @@ public class HDFSResourceStore extends ResourceStore { return new Path(this.hdfsMetaPath, resourcePath); } - } http://git-wip-us.apache.org/repos/asf/kylin/blob/03e3dc92/storage-hbase/src/main/java/org/apache/kylin/storage/hdfs/LockManager.java ---------------------------------------------------------------------- diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hdfs/LockManager.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hdfs/LockManager.java index a4d0080..4959718 100644 --- a/storage-hbase/src/main/java/org/apache/kylin/storage/hdfs/LockManager.java +++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hdfs/LockManager.java @@ -27,7 +27,6 @@ import org.apache.curator.framework.recipes.locks.InterProcessMutex; import org.apache.curator.retry.ExponentialBackoffRetry; import org.apache.kylin.common.KylinConfig; import org.apache.kylin.storage.hbase.util.ZookeeperDistributedJobLock; -import org.apache.zookeeper.CreateMode; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -44,15 +43,15 @@ public class LockManager { private String lockRootPath; - public LockManager(String lockRootPath) throws Exception{ + public LockManager(String lockRootPath) throws Exception { - this(KylinConfig.getInstanceFromEnv(),lockRootPath); + this(KylinConfig.getInstanceFromEnv(), lockRootPath); } - public LockManager(KylinConfig config,String lockRootPath) throws Exception{ + public LockManager(KylinConfig config, String lockRootPath) throws Exception { this.config = config; this.lockRootPath = lockRootPath; - String zkConnectString = getZKConnectString(); + String zkConnectString = getZKConnectString(config); logger.info("zk connection string:" + zkConnectString); if (StringUtils.isEmpty(zkConnectString)) { throw new IllegalArgumentException("ZOOKEEPER_QUORUM is empty!"); @@ -60,7 +59,7 @@ public class LockManager { RetryPolicy retryPolicy = new ExponentialBackoffRetry(1000, 3); zkClient = CuratorFrameworkFactory.newClient(zkConnectString, retryPolicy); zkClient.start(); - if(zkClient.checkExists().forPath(lockRootPath) == null) + if (zkClient.checkExists().forPath(lockRootPath) == null) zkClient.create().creatingParentsIfNeeded().forPath(lockRootPath); Runtime.getRuntime().addShutdownHook(new Thread(new Runnable() { @Override @@ -86,10 +85,10 @@ public class LockManager { } } - private static String getZKConnectString() { - final String serverList = ZookeeperConfig.DEFAULT_ZK_HOST; - final String port = ZookeeperConfig.DEFAULT_ZK_PORT; - return StringUtils.join(Iterables.transform(Arrays.asList(serverList.split(",")), new Function<String, String>() { + private static String getZKConnectString(KylinConfig kylinConfig) { + final String host = kylinConfig.getZooKeeperHost(); + final String port = kylinConfig.getZooKeeperPort(); + return StringUtils.join(Iterables.transform(Arrays.asList(host.split(",")), new Function<String, String>() { @Nullable @Override public String apply(String input) { http://git-wip-us.apache.org/repos/asf/kylin/blob/03e3dc92/storage-hbase/src/main/java/org/apache/kylin/storage/hdfs/ZookeeperConfig.java ---------------------------------------------------------------------- diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hdfs/ZookeeperConfig.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hdfs/ZookeeperConfig.java deleted file mode 100644 index 3f67e8d..0000000 --- a/storage-hbase/src/main/java/org/apache/kylin/storage/hdfs/ZookeeperConfig.java +++ /dev/null @@ -1,27 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. -*/ -package org.apache.kylin.storage.hdfs; - - -public interface ZookeeperConfig { - String DEFAULT_ZK_HOST = "sandbox"; - - String DEFAULT_ZK_PORT = "2181"; - - long DEFAULT_TIMEOUT = 10; -}
