Merge commit '87fedf75e5bbf61a33cf1b31720ad9ac55de6287'
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/f36372bc Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/f36372bc Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/f36372bc Branch: refs/heads/KYLIN-2624 Commit: f36372bc657b1edb312b36a071b077cde184eab8 Parents: 8ce972f 87fedf7 Author: Hongbin Ma <mahong...@apache.org> Authored: Fri May 19 17:55:26 2017 +0800 Committer: Hongbin Ma <mahong...@apache.org> Committed: Fri May 19 17:55:26 2017 +0800 ---------------------------------------------------------------------- .../rel/rules/OLAPJoinPushThroughJoinRule.java | 172 ++++++++ .../rel/rules/OLAPJoinPushThroughJoinRule2.java | 205 +++++++++ .../java/org/apache/calcite/tools/Programs.java | 438 +++++++++++++++++++ build/bin/find-hive-dependency.sh | 4 +- build/bin/kylin.sh | 41 +- build/bin/sample.sh | 24 +- .../apache/kylin/common/KylinConfigBase.java | 8 +- .../kylin/common/lock/DistributedLock.java | 63 ++- .../common/lock/DistributedLockFactory.java | 43 ++ .../kylin/common/restclient/RestClient.java | 16 +- .../org/apache/kylin/common/util/CheckUtil.java | 50 ++- .../apache/kylin/cube/model/DimensionDesc.java | 3 +- .../kylin/dict/BuiltInFunctionTransformer.java | 8 +- .../kylin/dict/GlobalDictionaryBuilder.java | 71 +-- .../impl/threadpool/DistributedScheduler.java | 55 +-- .../kylin/job/lock/DistributedJobLock.java | 24 - .../kylin/metadata/cachesync/Broadcaster.java | 66 ++- .../metadata/filter/function/BuiltInMethod.java | 39 +- .../metadata/filter/function/LikeMatchers.java | 141 ++++++ examples/sample_cube/create_sample_tables.sql | 10 +- .../kylin/job/BaseTestDistributedScheduler.java | 35 +- .../job/ITDistributedSchedulerBaseTest.java | 6 +- .../job/ITDistributedSchedulerTakeOverTest.java | 2 +- .../resources/query/sql_subquery/query17.sql | 17 + .../resources/query/sql_subquery/query18.sql | 23 + .../resources/query/sql_subquery/query19.sql | 20 + .../resources/query/sql_subquery/query21.sql | 26 ++ .../resources/query/sql_subquery/query22.sql | 25 ++ .../resources/query/sql_subquery/query23.sql | 26 ++ .../resources/query/sql_subquery/query24.sql | 29 ++ .../resources/query/sql_subquery/query25.sql | 22 + .../resources/query/sql_subquery/query26.sql | 16 + .../resources/query/sql_subquery/query27.sql | 20 + .../resources/query/sql_subquery/query28.sql | 21 + .../query/sql_subquery/query29.sql.todo | 22 + .../kylin/query/relnode/OLAPAggregateRel.java | 4 +- .../apache/kylin/query/relnode/OLAPContext.java | 10 + .../kylin/query/relnode/OLAPFilterRel.java | 2 +- .../apache/kylin/query/relnode/OLAPJoinRel.java | 12 +- .../kylin/query/relnode/OLAPProjectRel.java | 15 +- .../rest/security/AuthoritiesPopulator.java | 20 +- .../apache/kylin/rest/service/AclService.java | 1 - .../kylin/rest/service/CacheServiceTest.java | 10 +- .../kylin/storage/hbase/HBaseConnection.java | 30 +- .../hbase/util/ZookeeperDistributedJobLock.java | 257 ----------- .../hbase/util/ZookeeperDistributedLock.java | 307 +++++++++++++ .../storage/hbase/util/ZookeeperJobLock.java | 144 ++---- .../apache/kylin/storage/hdfs/LockManager.java | 8 +- .../util/ITZookeeperDistributedLockTest.java | 254 +++++++++++ .../apache/kylin/tool/MetadataCleanupJob.java | 14 +- webapp/app/js/controllers/access.js | 1 + webapp/app/js/controllers/admin.js | 12 +- webapp/app/js/controllers/cubeMeasures.js | 2 +- webapp/app/js/controllers/query.js | 4 +- 54 files changed, 2273 insertions(+), 625 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/kylin/blob/f36372bc/build/bin/find-hive-dependency.sh ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/kylin/blob/f36372bc/build/bin/sample.sh ---------------------------------------------------------------------- diff --cc build/bin/sample.sh index 2e13805,10900bb..b9c4ea3 --- a/build/bin/sample.sh +++ b/build/bin/sample.sh @@@ -52,10 -62,10 +63,11 @@@ the else beeline_params=${beeline_params/${hive2_url}/${hive2_url}${sample_database}} fi - beeline ${hive_conf_properties} ${beeline_params} -f ${KYLIN_HOME}/sample_cube/create_sample_tables.sql || { exit 1; } - beeline ${beeline_params} --hivevar hdfs_tmp_dir=${hdfs_tmp_dir} -f ${KYLIN_HOME}/sample_cube/create_sample_tables.sql || { exit 1; } ++ ++ beeline ${hive_conf_properties} --hivevar hdfs_tmp_dir=${hdfs_tmp_dir} ${beeline_params} -f ${KYLIN_HOME}/sample_cube/create_sample_tables.sql || { exit 1; } else - hive -e "CREATE DATABASE IF NOT EXISTS "$sample_database - hive --database $sample_database --hivevar hdfs_tmp_dir=${hdfs_tmp_dir} -f ${KYLIN_HOME}/sample_cube/create_sample_tables.sql || { exit 1; } + hive ${hive_conf_properties} -e "CREATE DATABASE IF NOT EXISTS "$sample_database - hive ${hive_conf_properties} --database $sample_database -f ${KYLIN_HOME}/sample_cube/create_sample_tables.sql || { exit 1; } ++ hive ${hive_conf_properties} --hivevar hdfs_tmp_dir=${hdfs_tmp_dir} --database $sample_database -f ${KYLIN_HOME}/sample_cube/create_sample_tables.sql || { exit 1; } fi echo "Sample hive tables are created successfully; Going to create sample cube..." http://git-wip-us.apache.org/repos/asf/kylin/blob/f36372bc/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/kylin/blob/f36372bc/core-dictionary/src/main/java/org/apache/kylin/dict/BuiltInFunctionTransformer.java ---------------------------------------------------------------------- diff --cc core-dictionary/src/main/java/org/apache/kylin/dict/BuiltInFunctionTransformer.java index 4d7b15d,5196593..f5cbe68 --- a/core-dictionary/src/main/java/org/apache/kylin/dict/BuiltInFunctionTransformer.java +++ b/core-dictionary/src/main/java/org/apache/kylin/dict/BuiltInFunctionTransformer.java @@@ -100,13 -97,9 +100,13 @@@ public class BuiltInFunctionTransforme Object dictVal = dict.getValueFromId(i); if ((Boolean) builtInFunctionTupleFilter.invokeFunction(dictVal)) { translated.addChild(new ConstantTupleFilter(dictVal)); + + if (translated.getChildren().size() > translatedInClauseMaxSize) { + return null; + } } } - + logger.debug("getting a in clause with {} children", translated.getChildren().size()); } catch (Exception e) { logger.debug(e.getMessage()); return null; http://git-wip-us.apache.org/repos/asf/kylin/blob/f36372bc/core-job/src/main/java/org/apache/kylin/job/impl/threadpool/DistributedScheduler.java ---------------------------------------------------------------------- diff --cc core-job/src/main/java/org/apache/kylin/job/impl/threadpool/DistributedScheduler.java index fd7b43a,6d12b8b..d01cb5a --- a/core-job/src/main/java/org/apache/kylin/job/impl/threadpool/DistributedScheduler.java +++ b/core-job/src/main/java/org/apache/kylin/job/impl/threadpool/DistributedScheduler.java @@@ -319,21 -304,14 +304,21 @@@ public class DistributedScheduler imple } } - public String getLockPath(String pathName) { - return dropDoubleSlash(ZOOKEEPER_LOCK_PATH + "/" + KylinConfig.getInstanceFromEnv().getMetadataUrlPrefix() + "/" + pathName); + public static String getLockPath(String pathName) { - return ZOOKEEPER_LOCK_PATH + "/" + pathName; ++ return dropDoubleSlash(ZOOKEEPER_LOCK_PATH + "/" + pathName); } - private String getWatchPath() { - return dropDoubleSlash(ZOOKEEPER_LOCK_PATH + "/" + KylinConfig.getInstanceFromEnv().getMetadataUrlPrefix()); + private static String getWatchPath() { - return ZOOKEEPER_LOCK_PATH; ++ return dropDoubleSlash(ZOOKEEPER_LOCK_PATH); } + public static String dropDoubleSlash(String path) { + for (int n = Integer.MAX_VALUE; n > path.length();) { + n = path.length(); + path = path.replace("//", "/"); + } + return path; + } @Override public void shutdown() throws SchedulerException { logger.info("Will shut down Job Engine ...."); http://git-wip-us.apache.org/repos/asf/kylin/blob/f36372bc/kylin-it/src/test/java/org/apache/kylin/job/BaseTestDistributedScheduler.java ---------------------------------------------------------------------- diff --cc kylin-it/src/test/java/org/apache/kylin/job/BaseTestDistributedScheduler.java index f410f59,48ef923..6343dfa --- a/kylin-it/src/test/java/org/apache/kylin/job/BaseTestDistributedScheduler.java +++ b/kylin-it/src/test/java/org/apache/kylin/job/BaseTestDistributedScheduler.java @@@ -197,8 -193,7 +193,7 @@@ public class BaseTestDistributedSchedul return serverName; } - private String getLockPath(String pathName) { - return DistributedScheduler.dropDoubleSlash(// - DistributedScheduler.ZOOKEEPER_LOCK_PATH + "/" + kylinConfig1.getMetadataUrlPrefix() + "/" + pathName); + private String getFullLockPath(String segName) { - return "/kylin/" + kylinConfig1.getMetadataUrlPrefix() + DistributedScheduler.getLockPath(segName); ++ return DistributedScheduler.dropDoubleSlash("/kylin/" + kylinConfig1.getMetadataUrlPrefix() + DistributedScheduler.getLockPath(segName)); } } http://git-wip-us.apache.org/repos/asf/kylin/blob/f36372bc/query/src/main/java/org/apache/kylin/query/relnode/OLAPFilterRel.java ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/kylin/blob/f36372bc/server-base/src/main/java/org/apache/kylin/rest/service/AclService.java ---------------------------------------------------------------------- diff --cc server-base/src/main/java/org/apache/kylin/rest/service/AclService.java index 097dd92,63b155b..dcfc867 --- a/server-base/src/main/java/org/apache/kylin/rest/service/AclService.java +++ b/server-base/src/main/java/org/apache/kylin/rest/service/AclService.java @@@ -408,4 -408,99 +408,3 @@@ class AclRecord extends RootPersistentE } } - -class DomainObjectInfo { - private String id; - private String type; - - public DomainObjectInfo() { - } - - public DomainObjectInfo(ObjectIdentity oid) { - super(); - this.id = (String) oid.getIdentifier(); - this.type = oid.getType(); - } - - public String getId() { - return id; - } - - public void setId(String id) { - this.id = id; - } - - public String getType() { - return type; - } - - public void setType(String type) { - this.type = type; - } -} - -class SidInfo { - private String sid; - private boolean isPrincipal; - - public SidInfo() { - } - - public SidInfo(Sid sid) { - if (sid instanceof PrincipalSid) { - this.sid = ((PrincipalSid) sid).getPrincipal(); - this.isPrincipal = true; - } else if (sid instanceof GrantedAuthoritySid) { - this.sid = ((GrantedAuthoritySid) sid).getGrantedAuthority(); - this.isPrincipal = false; - } - } - - public String getSid() { - return sid; - } - - public void setSid(String sid) { - this.sid = sid; - } - - public boolean isPrincipal() { - return isPrincipal; - } - - public void setPrincipal(boolean isPrincipal) { - this.isPrincipal = isPrincipal; - } - -} - -class AceInfo { - private SidInfo sidInfo; - private int permissionMask; - - public AceInfo() { - } - - public AceInfo(AccessControlEntry ace) { - super(); - this.sidInfo = new SidInfo(ace.getSid()); - this.permissionMask = ace.getPermission().getMask(); - } - - public SidInfo getSidInfo() { - return sidInfo; - } - - public void setSidInfo(SidInfo sidInfo) { - this.sidInfo = sidInfo; - } - - public int getPermissionMask() { - return permissionMask; - } - - public void setPermissionMask(int permissionMask) { - this.permissionMask = permissionMask; - } -- -} http://git-wip-us.apache.org/repos/asf/kylin/blob/f36372bc/server/src/test/java/org/apache/kylin/rest/service/CacheServiceTest.java ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/kylin/blob/f36372bc/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/HBaseConnection.java ---------------------------------------------------------------------- diff --cc storage-hbase/src/main/java/org/apache/kylin/storage/hbase/HBaseConnection.java index 2bbc2bc,5fafa2b..6580107 --- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/HBaseConnection.java +++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/HBaseConnection.java @@@ -45,7 -44,7 +45,8 @@@ import org.apache.hadoop.hbase.client.C import org.apache.hadoop.hbase.util.Threads; import org.apache.hadoop.hdfs.DFSConfigKeys; import org.apache.kylin.common.KylinConfig; +import org.apache.kylin.common.StorageURL; + import org.apache.kylin.common.lock.DistributedLock; import org.apache.kylin.common.persistence.StorageException; import org.apache.kylin.common.util.HadoopUtil; import org.slf4j.Logger; @@@ -357,4 -365,8 +371,8 @@@ public class HBaseConnection } } + private static String getLockPath(String table) { + return "/create_htable/" + table + "/lock"; + } + -} +}
