Repository: kylin Updated Branches: refs/heads/2.0.x ad67fd763 -> eb30e3ca4
Revert "KYLIN-2624 introduce StorageURL, add timeout to HBaseResourceStore" This reverts commit 79db2768bc517d9a8e64f657baff10e1914caa7c. Project: http://git-wip-us.apache.org/repos/asf/kylin/repo Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/0a2ee4e2 Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/0a2ee4e2 Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/0a2ee4e2 Branch: refs/heads/2.0.x Commit: 0a2ee4e28ed6fe92a3f1874ade6ecbfc1e0aecaa Parents: ad67fd7 Author: Hongbin Ma <mahong...@apache.org> Authored: Sat May 20 16:36:54 2017 +0800 Committer: Hongbin Ma <mahong...@apache.org> Committed: Sat May 20 16:36:54 2017 +0800 ---------------------------------------------------------------------- .../apache/kylin/common/KylinConfigBase.java | 30 +-- .../org/apache/kylin/common/StorageURL.java | 183 ------------------- .../common/persistence/FileResourceStore.java | 2 +- .../apache/kylin/common/KylinConfigTest.java | 9 +- .../org/apache/kylin/common/StorageURLTest.java | 103 ----------- .../persistence/LocalFileResourceStoreTest.java | 3 +- .../common/persistence/ResourceStoreTest.java | 2 +- .../apache/kylin/dict/DictionaryManager.java | 3 +- .../apache/kylin/storage/StorageContext.java | 7 +- .../kylin/engine/mr/steps/CubeReducerTest.java | 2 +- .../engine/mr/steps/NDCuboidMapperTest.java | 2 +- examples/test_case_data/localmeta/UUID | Bin 38 -> 0 bytes .../kylin/job/BaseTestDistributedScheduler.java | 8 +- .../kylin/storage/hbase/ITStorageTest.java | 3 +- .../apache/kylin/query/schema/OLAPSchema.java | 5 +- .../rest/security/MockAclHBaseStorage.java | 5 +- .../rest/security/RealAclHBaseStorage.java | 9 +- .../apache/kylin/rest/service/QueryService.java | 11 +- .../kylin/storage/hbase/HBaseConnection.java | 26 +-- .../kylin/storage/hbase/HBaseResourceStore.java | 36 ++-- .../storage/hbase/util/CubeMigrationCLI.java | 20 +- .../hbase/util/GridTableHBaseBenchmark.java | 3 +- .../hbase/util/ZookeeperDistributedJobLock.java | 14 +- .../kylin/storage/hdfs/HDFSResourceStore.java | 33 +++- .../hbase/steps/CubeHFileMapper2Test.java | 4 +- .../storage/hbase/steps/MockupMapContext.java | 2 +- .../org/apache/kylin/tool/CubeMigrationCLI.java | 20 +- 27 files changed, 135 insertions(+), 410 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/kylin/blob/0a2ee4e2/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java ---------------------------------------------------------------------- diff --git a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java index 9c698ac..09bbb72 100644 --- a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java +++ b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java @@ -167,7 +167,7 @@ abstract public class KylinConfigBase implements Serializable { } public String toString() { - return getMetadataUrl().toString(); + return getMetadataUrl(); } // ============================================================================ @@ -203,8 +203,8 @@ abstract public class KylinConfigBase implements Serializable { // METADATA // ============================================================================ - public StorageURL getMetadataUrl() { - return StorageURL.valueOf(getOptional("kylin.metadata.url", "")); + public String getMetadataUrl() { + return getOptional("kylin.metadata.url"); } // for test only @@ -213,7 +213,14 @@ abstract public class KylinConfigBase implements Serializable { } public String getMetadataUrlPrefix() { - return getMetadataUrl().getIdentifier(); + String metadataUrl = getMetadataUrl(); + String defaultPrefix = "kylin_metadata"; + + int cut = metadataUrl.lastIndexOf('@'); + if (cut > 0) { + return metadataUrl.substring(0, cut); + } + return defaultPrefix; } public String[] getRealizationProviders() { @@ -536,14 +543,8 @@ abstract public class KylinConfigBase implements Serializable { return Integer.parseInt(getOptional("kylin.storage.default", "2")); } - public StorageURL getStorageUrl() { - String url = getOptional("kylin.storage.url", "default@hbase"); - - // for backward compatibility - if ("hbase".equals(url)) - url = "default@hbase"; - - return StorageURL.valueOf(url); + public String getStorageUrl() { + return getOptional("kylin.storage.url"); } public String getHBaseClusterFs() { @@ -971,7 +972,10 @@ abstract public class KylinConfigBase implements Serializable { } public String getResourceStoreImpl() { - return getResourceStoreImpls().get(getMetadataUrl().getScheme()); + String metadataUrl = KylinConfig.getInstanceFromEnv().getMetadataUrl(); + int cut = metadataUrl.indexOf('@'); + String key = cut < 0 ? "" : metadataUrl.substring(cut + 1); + return getResourceStoreImpls().get(key); } public String getJobTrackingURLPattern() { http://git-wip-us.apache.org/repos/asf/kylin/blob/0a2ee4e2/core-common/src/main/java/org/apache/kylin/common/StorageURL.java ---------------------------------------------------------------------- diff --git a/core-common/src/main/java/org/apache/kylin/common/StorageURL.java b/core-common/src/main/java/org/apache/kylin/common/StorageURL.java deleted file mode 100644 index 04540e3..0000000 --- a/core-common/src/main/java/org/apache/kylin/common/StorageURL.java +++ /dev/null @@ -1,183 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. -*/ - -package org.apache.kylin.common; - -import java.util.LinkedHashMap; -import java.util.Map; -import java.util.Map.Entry; -import java.util.concurrent.ExecutionException; - -import com.google.common.cache.CacheBuilder; -import com.google.common.cache.CacheLoader; -import com.google.common.cache.LoadingCache; -import com.google.common.collect.ImmutableMap; - -/** - * The object form of metadata/storage URL: IDENTIFIER@SCHEME[,PARAM=VALUE,PARAM=VALUE...] - * - * It is not standard URL, but a string of specific format that shares some similar parts with URL. - * - * Immutable by design. - */ -public class StorageURL { - - private static final LoadingCache<String, StorageURL> cache = CacheBuilder.newBuilder()// - .maximumSize(100)// - .build(new CacheLoader<String, StorageURL>() { - @Override - public StorageURL load(String metadataUrl) throws Exception { - return new StorageURL(metadataUrl); - } - }); - - public static StorageURL valueOf(String metadataUrl) { - try { - return cache.get(metadataUrl); - } catch (ExecutionException e) { - throw new RuntimeException(e); - } - } - - // ============================================================================ - - final String identifier; - final String scheme; - final Map<String, String> params; - - // package private for test - StorageURL(String metadataUrl) { - boolean first = true; - String n = null; - String s = null; - Map<String, String> m = new LinkedHashMap<>(); - - for (String split : metadataUrl.split(",")) { - if (first) { - // identifier @ scheme - int cut = split.lastIndexOf('@'); - if (cut < 0) { - n = split.trim(); - s = ""; - } else { - n = split.substring(0, cut).trim(); - s = split.substring(cut + 1).trim(); - } - first = false; - } else { - // param = value - int cut = split.indexOf('='); - String k, v; - if (cut < 0) { - k = split.trim(); - v = ""; - } else { - k = split.substring(0, cut).trim(); - v = split.substring(cut + 1).trim(); - } - m.put(k, v); - } - } - - this.identifier = n.isEmpty() ? "kylin_metadata" : n; - this.scheme = s; - this.params = ImmutableMap.copyOf(m); - } - - private StorageURL(String identifier, String scheme, Map<String, String> params) { - this.identifier = identifier; - this.scheme = scheme; - this.params = ImmutableMap.copyOf(params); - } - - public String getIdentifier() { - return identifier; - } - - public String getScheme() { - return scheme; - } - - public boolean containsParameter(String k) { - return params.containsKey(k); - } - - public String getParameter(String k) { - return params.get(k); - } - - public Map<String, String> getAllParameters() { - return params; - } - - public StorageURL copy(Map<String, String> params) { - return new StorageURL(identifier, scheme, params); - } - - @Override - public String toString() { - String str = identifier; - if (!scheme.isEmpty()) - str += "@" + scheme; - - for (Entry<String, String> kv : params.entrySet()) { - str += "," + kv.getKey(); - if (!kv.getValue().isEmpty()) - str += "=" + kv.getValue(); - } - return str; - } - - @Override - public int hashCode() { - final int prime = 31; - int result = 1; - result = prime * result + ((identifier == null) ? 0 : identifier.hashCode()); - result = prime * result + ((params == null) ? 0 : params.hashCode()); - result = prime * result + ((scheme == null) ? 0 : scheme.hashCode()); - return result; - } - - @Override - public boolean equals(Object obj) { - if (this == obj) - return true; - if (obj == null) - return false; - if (getClass() != obj.getClass()) - return false; - StorageURL other = (StorageURL) obj; - if (identifier == null) { - if (other.identifier != null) - return false; - } else if (!identifier.equals(other.identifier)) - return false; - if (params == null) { - if (other.params != null) - return false; - } else if (!params.equals(other.params)) - return false; - if (scheme == null) { - if (other.scheme != null) - return false; - } else if (!scheme.equals(other.scheme)) - return false; - return true; - } - -} http://git-wip-us.apache.org/repos/asf/kylin/blob/0a2ee4e2/core-common/src/main/java/org/apache/kylin/common/persistence/FileResourceStore.java ---------------------------------------------------------------------- diff --git a/core-common/src/main/java/org/apache/kylin/common/persistence/FileResourceStore.java b/core-common/src/main/java/org/apache/kylin/common/persistence/FileResourceStore.java index 82cf451..d84e587 100644 --- a/core-common/src/main/java/org/apache/kylin/common/persistence/FileResourceStore.java +++ b/core-common/src/main/java/org/apache/kylin/common/persistence/FileResourceStore.java @@ -44,7 +44,7 @@ public class FileResourceStore extends ResourceStore { public FileResourceStore(KylinConfig kylinConfig) { super(kylinConfig); - root = new File(kylinConfig.getMetadataUrl().getIdentifier()).getAbsoluteFile(); + root = new File(kylinConfig.getMetadataUrl()).getAbsoluteFile(); if (root.exists() == false) throw new IllegalArgumentException("File not exist by '" + kylinConfig.getMetadataUrl() + "': " + root.getAbsolutePath()); } http://git-wip-us.apache.org/repos/asf/kylin/blob/0a2ee4e2/core-common/src/test/java/org/apache/kylin/common/KylinConfigTest.java ---------------------------------------------------------------------- diff --git a/core-common/src/test/java/org/apache/kylin/common/KylinConfigTest.java b/core-common/src/test/java/org/apache/kylin/common/KylinConfigTest.java index 6027af3..3976c6c 100644 --- a/core-common/src/test/java/org/apache/kylin/common/KylinConfigTest.java +++ b/core-common/src/test/java/org/apache/kylin/common/KylinConfigTest.java @@ -86,6 +86,7 @@ public class KylinConfigTest extends HotLoadKylinPropertiesTestCase { @Test public void testGetMetadataUrlPrefix() { KylinConfig config = KylinConfig.getInstanceFromEnv(); + final String default_metadata_prefix = "kylin_metadata"; config.setMetadataUrl("testMetaPrefix@hbase"); assertEquals("testMetaPrefix", config.getMetadataUrlPrefix()); @@ -94,7 +95,7 @@ public class KylinConfigTest extends HotLoadKylinPropertiesTestCase { assertEquals("testMetaPrefix", config.getMetadataUrlPrefix()); config.setMetadataUrl("/kylin/temp"); - assertEquals("/kylin/temp", config.getMetadataUrlPrefix()); + assertEquals(default_metadata_prefix, config.getMetadataUrlPrefix()); } @Test @@ -106,21 +107,21 @@ public class KylinConfigTest extends HotLoadKylinPropertiesTestCase { KylinConfig sysConfig = KylinConfig.getInstanceFromEnv(); sysConfig.setMetadataUrl(metadata1); - assertEquals(metadata1, KylinConfig.getInstanceFromEnv().getMetadataUrl().toString()); + assertEquals(metadata1, KylinConfig.getInstanceFromEnv().getMetadataUrl()); // test thread-local override KylinConfig threadConfig = KylinConfig.createKylinConfig(new Properties()); threadConfig.setMetadataUrl(metadata2); KylinConfig.setKylinConfigThreadLocal(threadConfig); - assertEquals(metadata2, KylinConfig.getInstanceFromEnv().getMetadataUrl().toString()); + assertEquals(metadata2, KylinConfig.getInstanceFromEnv().getMetadataUrl()); // other threads still use system KylinConfig new Thread(new Runnable() { @Override public void run() { System.out.println("Started new thread."); - assertEquals(metadata1, KylinConfig.getInstanceFromEnv().getMetadataUrl().toString()); + assertEquals(metadata1, KylinConfig.getInstanceFromEnv().getMetadataUrl()); } }).start(); } http://git-wip-us.apache.org/repos/asf/kylin/blob/0a2ee4e2/core-common/src/test/java/org/apache/kylin/common/StorageURLTest.java ---------------------------------------------------------------------- diff --git a/core-common/src/test/java/org/apache/kylin/common/StorageURLTest.java b/core-common/src/test/java/org/apache/kylin/common/StorageURLTest.java deleted file mode 100644 index eaa7548..0000000 --- a/core-common/src/test/java/org/apache/kylin/common/StorageURLTest.java +++ /dev/null @@ -1,103 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. -*/ - -package org.apache.kylin.common; - -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertTrue; - -import org.junit.Test; - -public class StorageURLTest { - - @Test - public void testBasic() { - { - StorageURL id = new StorageURL("hello@hbase"); - assertEquals("hello", id.getIdentifier()); - assertEquals("hbase", id.getScheme()); - assertEquals(0, id.getAllParameters().size()); - assertEquals("hello@hbase", id.toString()); - } - { - StorageURL id = new StorageURL("hello@hbase,a=b,c=d"); - assertEquals("hello", id.getIdentifier()); - assertEquals("hbase", id.getScheme()); - assertEquals(2, id.getAllParameters().size()); - assertEquals("b", id.getParameter("a")); - assertEquals("d", id.getParameter("c")); - assertEquals("hello@hbase,a=b,c=d", id.toString()); - } - { - StorageURL o = new StorageURL("hello@hbase,c=d"); - StorageURL o2 = new StorageURL("hello@hbase,a=b"); - StorageURL id = o.copy(o2.getAllParameters()); - assertEquals("hello", id.getIdentifier()); - assertEquals("hbase", id.getScheme()); - assertEquals(1, id.getAllParameters().size()); - assertEquals("b", id.getParameter("a")); - assertEquals("hello@hbase,a=b", id.toString()); - assertEquals("hello@hbase,c=d", o.toString()); - assertEquals("hello@hbase,a=b", o2.toString()); - } - } - - @Test(expected = NullPointerException.class) - public void testNullInput() { - new StorageURL(null); - } - - @Test - public void testEdgeCases() { - { - StorageURL id = new StorageURL(""); - assertEquals("kylin_metadata", id.getIdentifier()); - assertEquals("", id.getScheme()); - assertEquals(0, id.getAllParameters().size()); - assertEquals("kylin_metadata", id.toString()); - } - { - StorageURL id = new StorageURL("hello@"); - assertEquals("hello", id.getIdentifier()); - assertEquals("", id.getScheme()); - assertEquals(0, id.getAllParameters().size()); - assertEquals("hello", id.toString()); - } - { - StorageURL id = new StorageURL("hello@hbase,a"); - assertEquals("hello", id.getIdentifier()); - assertEquals("hbase", id.getScheme()); - assertEquals(1, id.getAllParameters().size()); - assertEquals("", id.getParameter("a")); - assertEquals("hello@hbase,a", id.toString()); - } - } - - @Test - public void testValueOfCache() { - StorageURL id1 = StorageURL.valueOf("hello@hbase"); - StorageURL id2 = StorageURL.valueOf("hello@hbase"); - StorageURL id3 = StorageURL.valueOf("hello @ hbase"); - StorageURL id4 = StorageURL.valueOf("hello@hbase,a=b"); - assertTrue(id1 == id2); - assertTrue(id1 != id3); - assertTrue(id1.equals(id3)); - assertTrue(id2 != id4); - assertTrue(!id2.equals(id4)); - } -} http://git-wip-us.apache.org/repos/asf/kylin/blob/0a2ee4e2/core-common/src/test/java/org/apache/kylin/common/persistence/LocalFileResourceStoreTest.java ---------------------------------------------------------------------- diff --git a/core-common/src/test/java/org/apache/kylin/common/persistence/LocalFileResourceStoreTest.java b/core-common/src/test/java/org/apache/kylin/common/persistence/LocalFileResourceStoreTest.java index ebef377..17b608d 100644 --- a/core-common/src/test/java/org/apache/kylin/common/persistence/LocalFileResourceStoreTest.java +++ b/core-common/src/test/java/org/apache/kylin/common/persistence/LocalFileResourceStoreTest.java @@ -38,8 +38,7 @@ public class LocalFileResourceStoreTest extends LocalFileMetadataTestCase { @Test public void testFileStore() throws Exception { - KylinConfig config = KylinConfig.getInstanceFromEnv(); - ResourceStoreTest.testAStore(config.getMetadataUrl().toString(), config); + ResourceStoreTest.testAStore("", KylinConfig.getInstanceFromEnv()); } } http://git-wip-us.apache.org/repos/asf/kylin/blob/0a2ee4e2/core-common/src/test/java/org/apache/kylin/common/persistence/ResourceStoreTest.java ---------------------------------------------------------------------- diff --git a/core-common/src/test/java/org/apache/kylin/common/persistence/ResourceStoreTest.java b/core-common/src/test/java/org/apache/kylin/common/persistence/ResourceStoreTest.java index 9686ee9..76ab029 100644 --- a/core-common/src/test/java/org/apache/kylin/common/persistence/ResourceStoreTest.java +++ b/core-common/src/test/java/org/apache/kylin/common/persistence/ResourceStoreTest.java @@ -248,7 +248,7 @@ public class ResourceStoreTest { } public static String replaceMetadataUrl(KylinConfig kylinConfig, String newUrl) { - String oldUrl = kylinConfig.getMetadataUrl().toString(); + String oldUrl = kylinConfig.getMetadataUrl(); kylinConfig.setProperty("kylin.metadata.url", newUrl); return oldUrl; } http://git-wip-us.apache.org/repos/asf/kylin/blob/0a2ee4e2/core-dictionary/src/main/java/org/apache/kylin/dict/DictionaryManager.java ---------------------------------------------------------------------- diff --git a/core-dictionary/src/main/java/org/apache/kylin/dict/DictionaryManager.java b/core-dictionary/src/main/java/org/apache/kylin/dict/DictionaryManager.java index b10bcd3..986f9c3 100644 --- a/core-dictionary/src/main/java/org/apache/kylin/dict/DictionaryManager.java +++ b/core-dictionary/src/main/java/org/apache/kylin/dict/DictionaryManager.java @@ -99,8 +99,7 @@ public class DictionaryManager { } })// .maximumSize(config.getCachedDictMaxEntrySize())// - .expireAfterWrite(1, TimeUnit.DAYS)// - .build(new CacheLoader<String, DictionaryInfo>() { + .expireAfterWrite(1, TimeUnit.DAYS).build(new CacheLoader<String, DictionaryInfo>() { @Override public DictionaryInfo load(String key) throws Exception { DictionaryInfo dictInfo = DictionaryManager.this.load(key, true); http://git-wip-us.apache.org/repos/asf/kylin/blob/0a2ee4e2/core-storage/src/main/java/org/apache/kylin/storage/StorageContext.java ---------------------------------------------------------------------- diff --git a/core-storage/src/main/java/org/apache/kylin/storage/StorageContext.java b/core-storage/src/main/java/org/apache/kylin/storage/StorageContext.java index 78cf97c..4522261 100644 --- a/core-storage/src/main/java/org/apache/kylin/storage/StorageContext.java +++ b/core-storage/src/main/java/org/apache/kylin/storage/StorageContext.java @@ -20,7 +20,6 @@ package org.apache.kylin.storage; import java.util.concurrent.atomic.AtomicLong; -import org.apache.kylin.common.StorageURL; import org.apache.kylin.common.debug.BackdoorToggles; import org.apache.kylin.cube.cuboid.Cuboid; import org.apache.kylin.metadata.realization.IRealization; @@ -36,7 +35,7 @@ import com.google.common.collect.Range; public class StorageContext { private static final Logger logger = LoggerFactory.getLogger(StorageContext.class); - private StorageURL connUrl; + private String connUrl; private int limit = Integer.MAX_VALUE; private boolean overlookOuterLimit = false; private int offset = 0; @@ -57,11 +56,11 @@ public class StorageContext { private Range<Long> reusedPeriod; - public StorageURL getConnUrl() { + public String getConnUrl() { return connUrl; } - public void setConnUrl(StorageURL connUrl) { + public void setConnUrl(String connUrl) { this.connUrl = connUrl; } http://git-wip-us.apache.org/repos/asf/kylin/blob/0a2ee4e2/engine-mr/src/test/java/org/apache/kylin/engine/mr/steps/CubeReducerTest.java ---------------------------------------------------------------------- diff --git a/engine-mr/src/test/java/org/apache/kylin/engine/mr/steps/CubeReducerTest.java b/engine-mr/src/test/java/org/apache/kylin/engine/mr/steps/CubeReducerTest.java index 7616df2..29ca9b8 100644 --- a/engine-mr/src/test/java/org/apache/kylin/engine/mr/steps/CubeReducerTest.java +++ b/engine-mr/src/test/java/org/apache/kylin/engine/mr/steps/CubeReducerTest.java @@ -60,7 +60,7 @@ public class CubeReducerTest extends LocalFileMetadataTestCase { // hack for distributed cache FileUtils.deleteDirectory(new File("../job/meta")); - FileUtils.copyDirectory(new File(getTestConfig().getMetadataUrl().toString()), new File("../job/meta")); + FileUtils.copyDirectory(new File(getTestConfig().getMetadataUrl()), new File("../job/meta")); CuboidReducer reducer = new CuboidReducer(); reduceDriver = ReduceDriver.newReduceDriver(reducer); http://git-wip-us.apache.org/repos/asf/kylin/blob/0a2ee4e2/engine-mr/src/test/java/org/apache/kylin/engine/mr/steps/NDCuboidMapperTest.java ---------------------------------------------------------------------- diff --git a/engine-mr/src/test/java/org/apache/kylin/engine/mr/steps/NDCuboidMapperTest.java b/engine-mr/src/test/java/org/apache/kylin/engine/mr/steps/NDCuboidMapperTest.java index c0ce1a4..815d9b3 100644 --- a/engine-mr/src/test/java/org/apache/kylin/engine/mr/steps/NDCuboidMapperTest.java +++ b/engine-mr/src/test/java/org/apache/kylin/engine/mr/steps/NDCuboidMapperTest.java @@ -54,7 +54,7 @@ public class NDCuboidMapperTest extends LocalFileMetadataTestCase { // hack for distributed cache FileUtils.deleteDirectory(new File("./meta")); - FileUtils.copyDirectory(new File(getTestConfig().getMetadataUrl().toString()), new File("./meta")); + FileUtils.copyDirectory(new File(getTestConfig().getMetadataUrl()), new File("./meta")); NDCuboidMapper mapper = new NDCuboidMapper(); CuboidReducer reducer = new CuboidReducer(); http://git-wip-us.apache.org/repos/asf/kylin/blob/0a2ee4e2/examples/test_case_data/localmeta/UUID ---------------------------------------------------------------------- diff --git a/examples/test_case_data/localmeta/UUID b/examples/test_case_data/localmeta/UUID deleted file mode 100644 index 5d7b2aa..0000000 Binary files a/examples/test_case_data/localmeta/UUID and /dev/null differ http://git-wip-us.apache.org/repos/asf/kylin/blob/0a2ee4e2/kylin-it/src/test/java/org/apache/kylin/job/BaseTestDistributedScheduler.java ---------------------------------------------------------------------- diff --git a/kylin-it/src/test/java/org/apache/kylin/job/BaseTestDistributedScheduler.java b/kylin-it/src/test/java/org/apache/kylin/job/BaseTestDistributedScheduler.java index 5f1b8cf..2d79970 100644 --- a/kylin-it/src/test/java/org/apache/kylin/job/BaseTestDistributedScheduler.java +++ b/kylin-it/src/test/java/org/apache/kylin/job/BaseTestDistributedScheduler.java @@ -75,7 +75,7 @@ public class BaseTestDistributedScheduler extends HBaseMetadataTestCase { KylinConfig srcConfig = KylinConfig.getInstanceFromEnv(); localMetaDir = Files.createTempDir(); - String backup = srcConfig.getMetadataUrl().toString(); + String backup = srcConfig.getMetadataUrl(); srcConfig.setProperty("kylin.metadata.url", localMetaDir.getAbsolutePath()); srcConfig.writeProperties(new File(confDstPath1)); srcConfig.writeProperties(new File(confDstPath2)); @@ -197,10 +197,6 @@ public class BaseTestDistributedScheduler extends HBaseMetadataTestCase { } private String getLockPath(String pathName) { - String metadataUrlPrefix = kylinConfig1.getMetadataUrlPrefix(); - String path = ZookeeperDistributedJobLock.dropDoubleSlash(// - ZookeeperDistributedJobLock.ZOOKEEPER_LOCK_PATH + "/" + metadataUrlPrefix + "/" + pathName); - logger.debug("Building lock path " + path); - return path; + return ZookeeperDistributedJobLock.ZOOKEEPER_LOCK_PATH + "/" + kylinConfig1.getMetadataUrlPrefix() + "/" + pathName; } } http://git-wip-us.apache.org/repos/asf/kylin/blob/0a2ee4e2/kylin-it/src/test/java/org/apache/kylin/storage/hbase/ITStorageTest.java ---------------------------------------------------------------------- diff --git a/kylin-it/src/test/java/org/apache/kylin/storage/hbase/ITStorageTest.java b/kylin-it/src/test/java/org/apache/kylin/storage/hbase/ITStorageTest.java index 24589a8..c2f68ff 100644 --- a/kylin-it/src/test/java/org/apache/kylin/storage/hbase/ITStorageTest.java +++ b/kylin-it/src/test/java/org/apache/kylin/storage/hbase/ITStorageTest.java @@ -71,8 +71,9 @@ public class ITStorageTest extends HBaseMetadataTestCase { cube = cubeMgr.getCube("test_kylin_cube_without_slr_left_join_empty"); Assert.assertNotNull(cube); storageEngine = StorageFactory.createQuery(cube); + String url = KylinConfig.getInstanceFromEnv().getStorageUrl(); context = new StorageContext(); - context.setConnUrl(KylinConfig.getInstanceFromEnv().getStorageUrl()); + context.setConnUrl(url); mockup = new StorageMockUtils(cube.getModel()); } http://git-wip-us.apache.org/repos/asf/kylin/blob/0a2ee4e2/query/src/main/java/org/apache/kylin/query/schema/OLAPSchema.java ---------------------------------------------------------------------- diff --git a/query/src/main/java/org/apache/kylin/query/schema/OLAPSchema.java b/query/src/main/java/org/apache/kylin/query/schema/OLAPSchema.java index f04acd2..27d456f 100644 --- a/query/src/main/java/org/apache/kylin/query/schema/OLAPSchema.java +++ b/query/src/main/java/org/apache/kylin/query/schema/OLAPSchema.java @@ -25,7 +25,6 @@ import java.util.Set; import org.apache.calcite.schema.Table; import org.apache.calcite.schema.impl.AbstractSchema; import org.apache.kylin.common.KylinConfig; -import org.apache.kylin.common.StorageURL; import org.apache.kylin.cube.CubeManager; import org.apache.kylin.metadata.MetadataManager; import org.apache.kylin.metadata.model.TableDesc; @@ -41,7 +40,7 @@ public class OLAPSchema extends AbstractSchema { private KylinConfig config; private String projectName; private String schemaName; - private StorageURL storageUrl; + private String storageUrl; private String starSchemaUrl; private String starSchemaUser; private String starSchemaPassword; @@ -90,7 +89,7 @@ public class OLAPSchema extends AbstractSchema { return schemaName; } - public StorageURL getStorageUrl() { + public String getStorageUrl() { return storageUrl; } http://git-wip-us.apache.org/repos/asf/kylin/blob/0a2ee4e2/server-base/src/main/java/org/apache/kylin/rest/security/MockAclHBaseStorage.java ---------------------------------------------------------------------- diff --git a/server-base/src/main/java/org/apache/kylin/rest/security/MockAclHBaseStorage.java b/server-base/src/main/java/org/apache/kylin/rest/security/MockAclHBaseStorage.java index e7df63b..cc76b87 100644 --- a/server-base/src/main/java/org/apache/kylin/rest/security/MockAclHBaseStorage.java +++ b/server-base/src/main/java/org/apache/kylin/rest/security/MockAclHBaseStorage.java @@ -23,7 +23,6 @@ import java.io.IOException; import org.apache.commons.lang.StringUtils; import org.apache.hadoop.hbase.client.Table; import org.apache.kylin.common.KylinConfig; -import org.apache.kylin.common.StorageURL; import org.apache.kylin.rest.service.AclService; import org.apache.kylin.rest.service.QueryService; import org.apache.kylin.rest.service.UserService; @@ -40,8 +39,8 @@ public class MockAclHBaseStorage implements AclHBaseStorage { private RealAclHBaseStorage realAcl; public MockAclHBaseStorage() { - StorageURL metadataUrl = KylinConfig.getInstanceFromEnv().getMetadataUrl(); - if (metadataUrl.getScheme().endsWith("hbase")) { + String metadataUrl = KylinConfig.getInstanceFromEnv().getMetadataUrl(); + if (metadataUrl != null && metadataUrl.endsWith("hbase")) { // hbase must be available since metadata is on it // in this case, let us use a real ACL instead of mockup realAcl = new RealAclHBaseStorage(); http://git-wip-us.apache.org/repos/asf/kylin/blob/0a2ee4e2/server-base/src/main/java/org/apache/kylin/rest/security/RealAclHBaseStorage.java ---------------------------------------------------------------------- diff --git a/server-base/src/main/java/org/apache/kylin/rest/security/RealAclHBaseStorage.java b/server-base/src/main/java/org/apache/kylin/rest/security/RealAclHBaseStorage.java index d950437..d1a1384 100644 --- a/server-base/src/main/java/org/apache/kylin/rest/security/RealAclHBaseStorage.java +++ b/server-base/src/main/java/org/apache/kylin/rest/security/RealAclHBaseStorage.java @@ -24,7 +24,6 @@ import org.apache.commons.lang.StringUtils; import org.apache.hadoop.hbase.TableName; import org.apache.hadoop.hbase.client.Table; import org.apache.kylin.common.KylinConfig; -import org.apache.kylin.common.StorageURL; import org.apache.kylin.rest.service.AclService; import org.apache.kylin.rest.service.QueryService; import org.apache.kylin.rest.service.UserService; @@ -34,15 +33,17 @@ import org.apache.kylin.storage.hbase.HBaseConnection; */ public class RealAclHBaseStorage implements AclHBaseStorage { - private StorageURL hbaseUrl; + private String hbaseUrl; private String aclTableName; private String userTableName; @Override public String prepareHBaseTable(Class<?> clazz) throws IOException { KylinConfig kylinConfig = KylinConfig.getInstanceFromEnv(); - hbaseUrl = kylinConfig.getMetadataUrl(); - String tableNameBase = hbaseUrl.getIdentifier(); + String metadataUrl = kylinConfig.getMetadataUrl(); + int cut = metadataUrl.indexOf('@'); + hbaseUrl = cut < 0 ? metadataUrl : metadataUrl.substring(cut + 1); + String tableNameBase = kylinConfig.getMetadataUrlPrefix(); if (clazz == AclService.class) { aclTableName = tableNameBase + ACL_TABLE_NAME; http://git-wip-us.apache.org/repos/asf/kylin/blob/0a2ee4e2/server-base/src/main/java/org/apache/kylin/rest/service/QueryService.java ---------------------------------------------------------------------- diff --git a/server-base/src/main/java/org/apache/kylin/rest/service/QueryService.java b/server-base/src/main/java/org/apache/kylin/rest/service/QueryService.java index 9ba900f..6a38638 100644 --- a/server-base/src/main/java/org/apache/kylin/rest/service/QueryService.java +++ b/server-base/src/main/java/org/apache/kylin/rest/service/QueryService.java @@ -55,7 +55,6 @@ import org.apache.hadoop.hbase.client.Result; import org.apache.hadoop.hbase.client.Table; import org.apache.kylin.common.KylinConfig; import org.apache.kylin.common.QueryContext; -import org.apache.kylin.common.StorageURL; import org.apache.kylin.common.debug.BackdoorToggles; import org.apache.kylin.common.exceptions.ResourceLimitExceededException; import org.apache.kylin.common.util.Bytes; @@ -116,7 +115,7 @@ public class QueryService extends BasicService { private final Serializer<Query[]> querySerializer = new Serializer<Query[]>(Query[].class); private final BadQueryDetector badQueryDetector = new BadQueryDetector(); - private final StorageURL hbaseUrl; + private final String hbaseUrl; private final String userTableName; @Autowired @@ -132,8 +131,12 @@ public class QueryService extends BasicService { public QueryService() { KylinConfig kylinConfig = KylinConfig.getInstanceFromEnv(); - hbaseUrl = kylinConfig.getMetadataUrl(); - userTableName = hbaseUrl.getIdentifier() + USER_TABLE_NAME; + String metadataUrl = kylinConfig.getMetadataUrl(); + // split TABLE@HBASE_URL + int cut = metadataUrl.indexOf('@'); + hbaseUrl = cut < 0 ? metadataUrl : metadataUrl.substring(cut + 1); + String tableNameBase = kylinConfig.getMetadataUrlPrefix(); + userTableName = tableNameBase + USER_TABLE_NAME; badQueryDetector.start(); } http://git-wip-us.apache.org/repos/asf/kylin/blob/0a2ee4e2/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/HBaseConnection.java ---------------------------------------------------------------------- diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/HBaseConnection.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/HBaseConnection.java index 33662b0..7e2cefc 100644 --- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/HBaseConnection.java +++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/HBaseConnection.java @@ -23,7 +23,6 @@ import java.io.UnsupportedEncodingException; import java.util.Collection; import java.util.HashSet; import java.util.Map; -import java.util.Map.Entry; import java.util.Set; import java.util.UUID; import java.util.concurrent.ConcurrentHashMap; @@ -46,7 +45,6 @@ import org.apache.hadoop.hbase.client.ConnectionFactory; import org.apache.hadoop.hbase.util.Threads; import org.apache.hadoop.hdfs.DFSConfigKeys; import org.apache.kylin.common.KylinConfig; -import org.apache.kylin.common.StorageURL; import org.apache.kylin.common.persistence.StorageException; import org.apache.kylin.common.util.HadoopUtil; import org.slf4j.Logger; @@ -64,8 +62,8 @@ public class HBaseConnection { private static final Logger logger = LoggerFactory.getLogger(HBaseConnection.class); - private static final Map<StorageURL, Configuration> configCache = new ConcurrentHashMap<StorageURL, Configuration>(); - private static final Map<StorageURL, Connection> connPool = new ConcurrentHashMap<StorageURL, Connection>(); + private static final Map<String, Configuration> configCache = new ConcurrentHashMap<String, Configuration>(); + private static final Map<String, Connection> connPool = new ConcurrentHashMap<String, Connection>(); private static final ThreadLocal<Configuration> configThreadLocal = new ThreadLocal<>(); private static ExecutorService coprocessorPool = null; @@ -80,7 +78,7 @@ public class HBaseConnection { try { conn.close(); } catch (IOException e) { - logger.error("error closing hbase connection " + conn, e); + e.printStackTrace(); } } } @@ -135,15 +133,15 @@ public class HBaseConnection { public static Configuration getCurrentHBaseConfiguration() { if (configThreadLocal.get() == null) { - StorageURL storageUrl = KylinConfig.getInstanceFromEnv().getStorageUrl(); + String storageUrl = KylinConfig.getInstanceFromEnv().getStorageUrl(); configThreadLocal.set(newHBaseConfiguration(storageUrl)); } return configThreadLocal.get(); } - private static Configuration newHBaseConfiguration(StorageURL url) { + private static Configuration newHBaseConfiguration(String url) { // using a hbase:xxx URL is deprecated, instead hbase config is always loaded from hbase-site.xml in classpath - if (!"hbase".equals(url.getScheme())) + if (!(StringUtils.isEmpty(url) || "hbase".equals(url))) throw new IllegalArgumentException("to use hbase storage, pls set 'kylin.storage.url=hbase' in kylin.properties"); Configuration conf = HBaseConfiguration.create(HadoopUtil.getCurrentConfiguration()); @@ -163,10 +161,6 @@ public class HBaseConnection { if (StringUtils.isBlank(conf.get("hbase.fs.tmp.dir"))) { conf.set("hbase.fs.tmp.dir", "/tmp"); } - - for (Entry<String, String> entry : url.getAllParameters().entrySet()) { - conf.set(entry.getKey(), entry.getValue()); - } return conf; } @@ -218,7 +212,7 @@ public class HBaseConnection { // returned Connection can be shared by multiple threads and does not require close() @SuppressWarnings("resource") - public static Connection get(StorageURL url) { + public static Connection get(String url) { // find configuration Configuration conf = configCache.get(url); if (conf == null) { @@ -260,15 +254,15 @@ public class HBaseConnection { } } - public static boolean tableExists(StorageURL hbaseUrl, String tableName) throws IOException { + public static boolean tableExists(String hbaseUrl, String tableName) throws IOException { return tableExists(HBaseConnection.get(hbaseUrl), tableName); } - public static void createHTableIfNeeded(StorageURL hbaseUrl, String tableName, String... families) throws IOException { + public static void createHTableIfNeeded(String hbaseUrl, String tableName, String... families) throws IOException { createHTableIfNeeded(HBaseConnection.get(hbaseUrl), tableName, families); } - public static void deleteTable(StorageURL hbaseUrl, String tableName) throws IOException { + public static void deleteTable(String hbaseUrl, String tableName) throws IOException { deleteTable(HBaseConnection.get(hbaseUrl), tableName); } http://git-wip-us.apache.org/repos/asf/kylin/blob/0a2ee4e2/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/HBaseResourceStore.java ---------------------------------------------------------------------- diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/HBaseResourceStore.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/HBaseResourceStore.java index 7674bf4..74ab017 100644 --- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/HBaseResourceStore.java +++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/HBaseResourceStore.java @@ -22,9 +22,7 @@ import java.io.ByteArrayInputStream; import java.io.ByteArrayOutputStream; import java.io.IOException; import java.io.InputStream; -import java.util.LinkedHashMap; import java.util.List; -import java.util.Map; import java.util.NavigableSet; import java.util.TreeSet; @@ -47,7 +45,6 @@ import org.apache.hadoop.hbase.filter.FilterList; import org.apache.hadoop.hbase.filter.KeyOnlyFilter; import org.apache.hadoop.hbase.filter.SingleColumnValueFilter; import org.apache.kylin.common.KylinConfig; -import org.apache.kylin.common.StorageURL; import org.apache.kylin.common.persistence.RawResource; import org.apache.kylin.common.persistence.ResourceStore; import org.apache.kylin.common.util.Bytes; @@ -69,42 +66,32 @@ public class HBaseResourceStore extends ResourceStore { private static final String COLUMN_TS = "t"; private static final byte[] B_COLUMN_TS = Bytes.toBytes(COLUMN_TS); - final String tableName; - final StorageURL metadataUrl; + final String tableNameBase; + final String hbaseUrl; Connection getConnection() throws IOException { - return HBaseConnection.get(metadataUrl); + return HBaseConnection.get(hbaseUrl); } public HBaseResourceStore(KylinConfig kylinConfig) throws IOException { super(kylinConfig); - metadataUrl = buildMetadataUrl(kylinConfig); - tableName = metadataUrl.getIdentifier(); + String metadataUrl = kylinConfig.getMetadataUrl(); + // split TABLE@HBASE_URL + int cut = metadataUrl.indexOf('@'); + tableNameBase = cut < 0 ? DEFAULT_STORE_NAME : metadataUrl.substring(0, cut); + hbaseUrl = cut < 0 ? metadataUrl : metadataUrl.substring(cut + 1); + if (!hbaseUrl.equals("hbase")) + throw new IOException("Can not create HBaseResourceStore. Url not match. Url:" + hbaseUrl); createHTableIfNeeded(getAllInOneTableName()); } - private StorageURL buildMetadataUrl(KylinConfig kylinConfig) throws IOException { - StorageURL url = kylinConfig.getMetadataUrl(); - if (!url.getScheme().equals("hbase")) - throw new IOException("Cannot create HBaseResourceStore. Url not match. Url: " + url); - - // control timeout for prompt error report - Map<String, String> newParams = new LinkedHashMap<>(); - newParams.put("hbase.client.scanner.timeout.period", "10000"); - newParams.put("hbase.rpc.timeout", "5000"); - newParams.put("hbase.client.retries.number", "1"); - newParams.putAll(url.getAllParameters()); - - return url.copy(newParams); - } - private void createHTableIfNeeded(String tableName) throws IOException { HBaseConnection.createHTableIfNeeded(getConnection(), tableName, FAMILY); } private String getAllInOneTableName() { - return tableName; + return tableNameBase; } @Override @@ -173,7 +160,6 @@ public class HBaseResourceStore extends ResourceStore { void visit(String childPath, String fullPath, Result hbaseResult) throws IOException; } - /* override get meta store uuid method for backward compatibility */ @Override protected List<RawResource> getAllResourcesImpl(String folderPath, long timeStart, long timeEndExclusive) throws IOException { FilterList filter = generateTimeFilterList(timeStart, timeEndExclusive); http://git-wip-us.apache.org/repos/asf/kylin/blob/0a2ee4e2/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CubeMigrationCLI.java ---------------------------------------------------------------------- diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CubeMigrationCLI.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CubeMigrationCLI.java index 2154ed1..581de38 100644 --- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CubeMigrationCLI.java +++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/CubeMigrationCLI.java @@ -40,7 +40,6 @@ import org.apache.hadoop.hbase.client.Put; import org.apache.hadoop.hbase.client.Result; import org.apache.hadoop.hbase.client.Table; import org.apache.kylin.common.KylinConfig; -import org.apache.kylin.common.StorageURL; import org.apache.kylin.common.persistence.JsonSerializer; import org.apache.kylin.common.persistence.RawResource; import org.apache.kylin.common.persistence.ResourceStore; @@ -173,15 +172,26 @@ public class CubeMigrationCLI { checkCLI.execute(cubeName); } - private static void checkAndGetHbaseUrl() { - StorageURL srcMetadataUrl = srcConfig.getMetadataUrl(); - StorageURL dstMetadataUrl = dstConfig.getMetadataUrl(); + private static String checkAndGetHbaseUrl() { + String srcMetadataUrl = srcConfig.getMetadataUrl(); + String dstMetadataUrl = dstConfig.getMetadataUrl(); logger.info("src metadata url is " + srcMetadataUrl); logger.info("dst metadata url is " + dstMetadataUrl); - if (!"hbase".equals(srcMetadataUrl.getScheme()) || !"hbase".equals(dstMetadataUrl.getScheme())) + int srcIndex = srcMetadataUrl.toLowerCase().indexOf("hbase"); + int dstIndex = dstMetadataUrl.toLowerCase().indexOf("hbase"); + if (srcIndex < 0 || dstIndex < 0) throw new IllegalStateException("Both metadata urls should be hbase metadata url"); + + String srcHbaseUrl = srcMetadataUrl.substring(srcIndex).trim(); + String dstHbaseUrl = dstMetadataUrl.substring(dstIndex).trim(); + if (!srcHbaseUrl.equalsIgnoreCase(dstHbaseUrl)) { + throw new IllegalStateException("hbase url not equal! "); + } + + logger.info("hbase url is " + srcHbaseUrl.trim()); + return srcHbaseUrl.trim(); } private static void renameFoldersInHdfs(CubeInstance cube) { http://git-wip-us.apache.org/repos/asf/kylin/blob/0a2ee4e2/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/GridTableHBaseBenchmark.java ---------------------------------------------------------------------- diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/GridTableHBaseBenchmark.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/GridTableHBaseBenchmark.java index a317110..dd5f8fa 100644 --- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/GridTableHBaseBenchmark.java +++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/GridTableHBaseBenchmark.java @@ -36,7 +36,6 @@ import org.apache.hadoop.hbase.client.ResultScanner; import org.apache.hadoop.hbase.client.Scan; import org.apache.hadoop.hbase.client.Table; import org.apache.hadoop.hbase.filter.KeyOnlyFilter; -import org.apache.kylin.common.StorageURL; import org.apache.kylin.common.util.Bytes; import org.apache.kylin.common.util.Pair; import org.apache.kylin.storage.hbase.HBaseConnection; @@ -74,7 +73,7 @@ public class GridTableHBaseBenchmark { public static void testGridTable(double hitRatio, double indexRatio) throws IOException { System.out.println("Testing grid table scanning, hit ratio " + hitRatio + ", index ratio " + indexRatio); - StorageURL hbaseUrl = StorageURL.valueOf("default@hbase"); // use hbase-site.xml on classpath + String hbaseUrl = "hbase"; // use hbase-site.xml on classpath Connection conn = HBaseConnection.get(hbaseUrl); createHTableIfNeeded(conn, TEST_TABLE); http://git-wip-us.apache.org/repos/asf/kylin/blob/0a2ee4e2/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/ZookeeperDistributedJobLock.java ---------------------------------------------------------------------- diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/ZookeeperDistributedJobLock.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/ZookeeperDistributedJobLock.java index 052563f..983bfd9 100644 --- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/ZookeeperDistributedJobLock.java +++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/ZookeeperDistributedJobLock.java @@ -235,21 +235,11 @@ public class ZookeeperDistributedJobLock implements DistributedJobLock { } private String getLockPath(String pathName) { - String metadataUrlPrefix = config.getMetadataUrlPrefix(); - return dropDoubleSlash(ZOOKEEPER_LOCK_PATH + "/" + metadataUrlPrefix + "/" + pathName); + return ZOOKEEPER_LOCK_PATH + "/" + config.getMetadataUrlPrefix() + "/" + pathName; } private String getWatchPath() { - String metadataUrlPrefix = config.getMetadataUrlPrefix(); - return dropDoubleSlash(ZOOKEEPER_LOCK_PATH + "/" + metadataUrlPrefix); - } - - public static String dropDoubleSlash(String path) { - for (int n = Integer.MAX_VALUE; n > path.length();) { - n = path.length(); - path = path.replace("//", "/"); - } - return path; + return ZOOKEEPER_LOCK_PATH + "/" + config.getMetadataUrlPrefix(); } @Override http://git-wip-us.apache.org/repos/asf/kylin/blob/0a2ee4e2/storage-hbase/src/main/java/org/apache/kylin/storage/hdfs/HDFSResourceStore.java ---------------------------------------------------------------------- diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hdfs/HDFSResourceStore.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hdfs/HDFSResourceStore.java index fe1ad4e..6744805 100644 --- a/storage-hbase/src/main/java/org/apache/kylin/storage/hdfs/HDFSResourceStore.java +++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hdfs/HDFSResourceStore.java @@ -33,7 +33,6 @@ import org.apache.hadoop.fs.FileStatus; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; import org.apache.kylin.common.KylinConfig; -import org.apache.kylin.common.StorageURL; import org.apache.kylin.common.persistence.RawResource; import org.apache.kylin.common.persistence.ResourceStore; import org.apache.kylin.common.util.HadoopUtil; @@ -52,12 +51,16 @@ public class HDFSResourceStore extends ResourceStore { public HDFSResourceStore(KylinConfig kylinConfig) throws Exception { super(kylinConfig); - StorageURL metadataUrl = kylinConfig.getMetadataUrl(); - - if (!metadataUrl.getScheme().equals("hdfs")) - throw new IOException("kylin.metadata.url not recognized for HDFSResourceStore:" + metadataUrl); + String metadataUrl = kylinConfig.getMetadataUrl(); + int cut = metadataUrl.indexOf('@'); + if (cut < 0) { + throw new IOException("kylin.metadata.url not recognized for HDFSResourceStore: " + metadataUrl); + } + String suffix = metadataUrl.substring(cut + 1); + if (!suffix.equals("hdfs")) + throw new IOException("kylin.metadata.url not recognized for HDFSResourceStore:" + suffix); - String path = metadataUrl.getIdentifier(); + String path = metadataUrl.substring(0, cut); fs = HadoopUtil.getFileSystem(path); Path metadataPath = new Path(path); if (fs.exists(metadataPath) == false) { @@ -218,4 +221,22 @@ public class HDFSResourceStore extends ResourceStore { resourcePath = resourcePath.substring(1, resourcePath.length()); return new Path(this.hdfsMetaPath, resourcePath); } + + private static String getRelativePath(Path hdfsPath) { + String path = hdfsPath.toString(); + int index = path.indexOf("://"); + if (index > 0) { + path = path.substring(index + 3); + } + + if (path.startsWith("/") == false) { + if (path.indexOf("/") > 0) { + path = path.substring(path.indexOf("/")); + } else { + path = "/" + path; + } + } + return path; + } + } http://git-wip-us.apache.org/repos/asf/kylin/blob/0a2ee4e2/storage-hbase/src/test/java/org/apache/kylin/storage/hbase/steps/CubeHFileMapper2Test.java ---------------------------------------------------------------------- diff --git a/storage-hbase/src/test/java/org/apache/kylin/storage/hbase/steps/CubeHFileMapper2Test.java b/storage-hbase/src/test/java/org/apache/kylin/storage/hbase/steps/CubeHFileMapper2Test.java index d47f393..f36a62c 100644 --- a/storage-hbase/src/test/java/org/apache/kylin/storage/hbase/steps/CubeHFileMapper2Test.java +++ b/storage-hbase/src/test/java/org/apache/kylin/storage/hbase/steps/CubeHFileMapper2Test.java @@ -55,7 +55,7 @@ public class CubeHFileMapper2Test extends LocalFileMetadataTestCase { this.createTestMetadata(); // hack for distributed cache FileUtils.deleteDirectory(new File("../job/meta")); - FileUtils.copyDirectory(new File(getTestConfig().getMetadataUrl().toString()), new File("../job/meta")); + FileUtils.copyDirectory(new File(getTestConfig().getMetadataUrl()), new File("../job/meta")); CubeDesc desc = CubeManager.getInstance(getTestConfig()).getCube(cubeName).getDescriptor(); codec = new MeasureCodec(desc.getMeasures()); } @@ -70,7 +70,7 @@ public class CubeHFileMapper2Test extends LocalFileMetadataTestCase { public void testBasic() throws Exception { Configuration hconf = HadoopUtil.getCurrentConfiguration(); - Context context = MockupMapContext.create(hconf, cubeName, outKV); + Context context = MockupMapContext.create(hconf, getTestConfig().getMetadataUrl(), cubeName, outKV); CubeHFileMapper mapper = new CubeHFileMapper(); mapper.setup(context); http://git-wip-us.apache.org/repos/asf/kylin/blob/0a2ee4e2/storage-hbase/src/test/java/org/apache/kylin/storage/hbase/steps/MockupMapContext.java ---------------------------------------------------------------------- diff --git a/storage-hbase/src/test/java/org/apache/kylin/storage/hbase/steps/MockupMapContext.java b/storage-hbase/src/test/java/org/apache/kylin/storage/hbase/steps/MockupMapContext.java index 61170e8..536634d 100644 --- a/storage-hbase/src/test/java/org/apache/kylin/storage/hbase/steps/MockupMapContext.java +++ b/storage-hbase/src/test/java/org/apache/kylin/storage/hbase/steps/MockupMapContext.java @@ -53,7 +53,7 @@ public class MockupMapContext implements MapContext { private Object[] outKV; - public static Context create(final Configuration hconf, String cubeName, final Object[] outKV) { + public static Context create(final Configuration hconf, String metadataUrl, String cubeName, final Object[] outKV) { hconf.set(BatchConstants.CFG_CUBE_NAME, cubeName); http://git-wip-us.apache.org/repos/asf/kylin/blob/0a2ee4e2/tool/src/main/java/org/apache/kylin/tool/CubeMigrationCLI.java ---------------------------------------------------------------------- diff --git a/tool/src/main/java/org/apache/kylin/tool/CubeMigrationCLI.java b/tool/src/main/java/org/apache/kylin/tool/CubeMigrationCLI.java index 08d4292..c162a76 100644 --- a/tool/src/main/java/org/apache/kylin/tool/CubeMigrationCLI.java +++ b/tool/src/main/java/org/apache/kylin/tool/CubeMigrationCLI.java @@ -40,7 +40,6 @@ import org.apache.hadoop.hbase.client.Put; import org.apache.hadoop.hbase.client.Result; import org.apache.hadoop.hbase.client.Table; import org.apache.kylin.common.KylinConfig; -import org.apache.kylin.common.StorageURL; import org.apache.kylin.common.persistence.JsonSerializer; import org.apache.kylin.common.persistence.RawResource; import org.apache.kylin.common.persistence.ResourceStore; @@ -173,15 +172,26 @@ public class CubeMigrationCLI { checkCLI.execute(cubeName); } - private void checkAndGetHbaseUrl() { - StorageURL srcMetadataUrl = srcConfig.getMetadataUrl(); - StorageURL dstMetadataUrl = dstConfig.getMetadataUrl(); + private String checkAndGetHbaseUrl() { + String srcMetadataUrl = srcConfig.getMetadataUrl(); + String dstMetadataUrl = dstConfig.getMetadataUrl(); logger.info("src metadata url is " + srcMetadataUrl); logger.info("dst metadata url is " + dstMetadataUrl); - if (!"hbase".equals(srcMetadataUrl.getScheme()) || !"hbase".equals(dstMetadataUrl.getScheme())) + int srcIndex = srcMetadataUrl.toLowerCase().indexOf("hbase"); + int dstIndex = dstMetadataUrl.toLowerCase().indexOf("hbase"); + if (srcIndex < 0 || dstIndex < 0) throw new IllegalStateException("Both metadata urls should be hbase metadata url"); + + String srcHbaseUrl = srcMetadataUrl.substring(srcIndex).trim(); + String dstHbaseUrl = dstMetadataUrl.substring(dstIndex).trim(); + if (!srcHbaseUrl.equalsIgnoreCase(dstHbaseUrl)) { + throw new IllegalStateException("hbase url not equal! "); + } + + logger.info("hbase url is " + srcHbaseUrl.trim()); + return srcHbaseUrl.trim(); } protected void renameFoldersInHdfs(CubeInstance cube) throws IOException {
