KYLIN-1875 A good snowflake sample data, and rename 'kylin_metadata' to 'kylin_default_instance'
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/a1a2a4a6 Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/a1a2a4a6 Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/a1a2a4a6 Branch: refs/heads/master Commit: a1a2a4a643d013471079ac0d353bbf179fd0d85c Parents: c3d62e9 Author: Yang Li <liy...@apache.org> Authored: Sun Nov 27 18:26:33 2016 +0800 Committer: Yang Li <liy...@apache.org> Committed: Tue Nov 29 07:25:25 2016 +0800 ---------------------------------------------------------------------- build/conf/kylin.properties | 2 +- .../apache/kylin/common/KylinConfigBase.java | 11 +- .../kylin/metadata/model/DataModelDesc.java | 3 + .../kylin/metadata/model/MeasureDesc.java | 5 + .../realization/IRealizationConstants.java | 2 +- examples/sample_cube/create_sample_tables.sql | 21 +- .../sample_cube/data/DEFAULT.KYLIN_ACCOUNT.csv | 10000 +++++++++ .../sample_cube/data/DEFAULT.KYLIN_SALES.csv | 20000 ++++++++--------- .../template/cube_desc/kylin_sales_cube.json | 157 +- .../template/model_desc/kylin_sales_model.json | 50 +- .../template/table/DEFAULT.KYLIN_ACCOUNT.json | 28 + .../template/table/DEFAULT.KYLIN_SALES.json | 12 +- .../test_case_data/sandbox/kylin.properties | 2 +- .../apache/kylin/query/relnode/OLAPJoinRel.java | 25 +- .../kylin/rest/security/AclHBaseStorage.java | 1 - .../rest/security/RealAclHBaseStorage.java | 5 +- .../apache/kylin/rest/service/QueryService.java | 6 +- .../kylin/storage/hbase/HBaseResourceStore.java | 2 +- .../storage/hbase/util/StorageCleanupJob.java | 2 +- .../apache/kylin/tool/StorageCleanupJob.java | 2 +- 20 files changed, 20246 insertions(+), 10090 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/kylin/blob/a1a2a4a6/build/conf/kylin.properties ---------------------------------------------------------------------- diff --git a/build/conf/kylin.properties b/build/conf/kylin.properties index 1409394..131a725 100644 --- a/build/conf/kylin.properties +++ b/build/conf/kylin.properties @@ -18,7 +18,7 @@ ### METADATA | ENV ### # The metadata store in hbase -kylin.metadata.url=kylin_metadata@hbase +kylin.metadata.url=kylin_default_instance@hbase # Working folder in HDFS, make sure user has the right access to the hdfs directory kylin.env.hdfs-working-dir=/kylin http://git-wip-us.apache.org/repos/asf/kylin/blob/a1a2a4a6/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java ---------------------------------------------------------------------- diff --git a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java index aa5e6e1..7dcc771 100644 --- a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java +++ b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java @@ -188,13 +188,12 @@ abstract public class KylinConfigBase implements Serializable { } public String getMetadataUrlPrefix() { - String hbaseMetadataUrl = getMetadataUrl(); - String defaultPrefix = "kylin_metadata"; + String metadataUrl = getMetadataUrl(); + String defaultPrefix = "kylin_default_instance"; - if (org.apache.commons.lang3.StringUtils.containsIgnoreCase(hbaseMetadataUrl, "@hbase")) { - int cut = hbaseMetadataUrl.indexOf('@'); - String tmp = cut < 0 ? defaultPrefix : hbaseMetadataUrl.substring(0, cut); - return tmp; + if (metadataUrl.endsWith("@hbase")) { + int cut = metadataUrl.lastIndexOf('@'); + return metadataUrl.substring(0, cut); } else { return defaultPrefix; } http://git-wip-us.apache.org/repos/asf/kylin/blob/a1a2a4a6/core-metadata/src/main/java/org/apache/kylin/metadata/model/DataModelDesc.java ---------------------------------------------------------------------- diff --git a/core-metadata/src/main/java/org/apache/kylin/metadata/model/DataModelDesc.java b/core-metadata/src/main/java/org/apache/kylin/metadata/model/DataModelDesc.java index c2de5d6..ff92def 100644 --- a/core-metadata/src/main/java/org/apache/kylin/metadata/model/DataModelDesc.java +++ b/core-metadata/src/main/java/org/apache/kylin/metadata/model/DataModelDesc.java @@ -378,6 +378,9 @@ public class DataModelDesc extends RootPersistentEntity { for (int i = 0; i < pks.length; i++) { TblColRef col = dimTable.getColumn(pks[i]); if (col == null) { + col = findColumn(pks[i]); + } + if (col == null || col.getTableRef().equals(dimTable) == false) { throw new IllegalStateException("Can't find column " + pks[i] + " in table " + dimTable.getTableIdentity()); } pkCols[i] = col; http://git-wip-us.apache.org/repos/asf/kylin/blob/a1a2a4a6/core-metadata/src/main/java/org/apache/kylin/metadata/model/MeasureDesc.java ---------------------------------------------------------------------- diff --git a/core-metadata/src/main/java/org/apache/kylin/metadata/model/MeasureDesc.java b/core-metadata/src/main/java/org/apache/kylin/metadata/model/MeasureDesc.java index dcf6350..253b06b 100644 --- a/core-metadata/src/main/java/org/apache/kylin/metadata/model/MeasureDesc.java +++ b/core-metadata/src/main/java/org/apache/kylin/metadata/model/MeasureDesc.java @@ -21,6 +21,7 @@ package org.apache.kylin.metadata.model; import java.util.Objects; import com.fasterxml.jackson.annotation.JsonAutoDetect; +import com.fasterxml.jackson.annotation.JsonInclude; import com.fasterxml.jackson.annotation.JsonAutoDetect.Visibility; import com.fasterxml.jackson.annotation.JsonProperty; @@ -35,6 +36,8 @@ public class MeasureDesc { @JsonProperty("function") private FunctionDesc function; @JsonProperty("dependent_measure_ref") + @JsonInclude(JsonInclude.Include.NON_NULL) + @Deprecated private String dependentMeasureRef; public String getName() { @@ -53,10 +56,12 @@ public class MeasureDesc { this.function = function; } + @Deprecated public String getDependentMeasureRef() { return dependentMeasureRef; } + @Deprecated public void setDependentMeasureRef(String dependentMeasureRef) { this.dependentMeasureRef = dependentMeasureRef; } http://git-wip-us.apache.org/repos/asf/kylin/blob/a1a2a4a6/core-metadata/src/main/java/org/apache/kylin/metadata/realization/IRealizationConstants.java ---------------------------------------------------------------------- diff --git a/core-metadata/src/main/java/org/apache/kylin/metadata/realization/IRealizationConstants.java b/core-metadata/src/main/java/org/apache/kylin/metadata/realization/IRealizationConstants.java index 2a3b01e..adc53a1 100644 --- a/core-metadata/src/main/java/org/apache/kylin/metadata/realization/IRealizationConstants.java +++ b/core-metadata/src/main/java/org/apache/kylin/metadata/realization/IRealizationConstants.java @@ -28,7 +28,7 @@ public class IRealizationConstants { /** * For each cube htable, we leverage htable's metadata to keep track of - * which kylin server(represented by its kylin_metadata prefix) owns this htable + * which kylin server(represented by its kylin_default_instance prefix) owns this htable */ public final static String HTableTag = "KYLIN_HOST"; http://git-wip-us.apache.org/repos/asf/kylin/blob/a1a2a4a6/examples/sample_cube/create_sample_tables.sql ---------------------------------------------------------------------- diff --git a/examples/sample_cube/create_sample_tables.sql b/examples/sample_cube/create_sample_tables.sql index 3189b58..bdfc4d7 100644 --- a/examples/sample_cube/create_sample_tables.sql +++ b/examples/sample_cube/create_sample_tables.sql @@ -182,6 +182,19 @@ COUNTRY string ROW FORMAT DELIMITED FIELDS TERMINATED BY ',' STORED AS TEXTFILE; +DROP TABLE IF EXISTS DEFAULT.KYLIN_ACCOUNT; + +CREATE TABLE DEFAULT.KYLIN_ACCOUNT +( +ACCOUNT_ID bigint +,ACCOUNT_BUYER_LEVEL int COMMENT 'Account Buyer Level' +,ACCOUNT_SELLER_LEVEL int COMMENT 'Account Seller Level' +,ACCOUNT_COUNTRY string COMMENT 'Account Country' +,ACCOUNT_CONTACT string COMMENT 'Account Contact Info' +) +ROW FORMAT DELIMITED FIELDS TERMINATED BY ',' +STORED AS TEXTFILE; + DROP TABLE IF EXISTS DEFAULT.KYLIN_SALES; CREATE TABLE DEFAULT.KYLIN_SALES @@ -195,16 +208,16 @@ TRANS_ID bigint ,PRICE decimal(19,4) COMMENT 'Order Price' ,ITEM_COUNT bigint COMMENT 'Number of Purchased Goods' ,SELLER_ID bigint COMMENT 'Seller ID' -,BUYER_COUNTRY string COMMENT 'Buyer Country' -,SELLER_COUNTRY string COMMENT 'Seller Country' -,USER_ID string COMMENT 'System User ID' -,REGION string COMMENT 'System User Region' +,BUYER_ID bigint COMMENT 'Buyer ID' +,OPS_USER_ID string COMMENT 'System User ID' +,OPS_REGION string COMMENT 'System User Region' ) COMMENT 'Sales order table, fact table' ROW FORMAT DELIMITED FIELDS TERMINATED BY ',' STORED AS TEXTFILE; LOAD DATA INPATH '/tmp/kylin/sample_cube/data/DEFAULT.KYLIN_SALES.csv' OVERWRITE INTO TABLE DEFAULT.KYLIN_SALES; +LOAD DATA INPATH '/tmp/kylin/sample_cube/data/DEFAULT.KYLIN_ACCOUNT.csv' OVERWRITE INTO TABLE DEFAULT.KYLIN_ACCOUNT; LOAD DATA INPATH '/tmp/kylin/sample_cube/data/DEFAULT.KYLIN_COUNTRY.csv' OVERWRITE INTO TABLE DEFAULT.KYLIN_COUNTRY; LOAD DATA INPATH '/tmp/kylin/sample_cube/data/DEFAULT.KYLIN_CAL_DT.csv' OVERWRITE INTO TABLE DEFAULT.KYLIN_CAL_DT; LOAD DATA INPATH '/tmp/kylin/sample_cube/data/DEFAULT.KYLIN_CATEGORY_GROUPINGS.csv' OVERWRITE INTO TABLE DEFAULT.KYLIN_CATEGORY_GROUPINGS;
