KYLIN-2424 Optimize the integration test's performance
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/5d83c80f Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/5d83c80f Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/5d83c80f Branch: refs/heads/master-hbase0.98 Commit: 5d83c80fbc0d6c3db434368cc8c29d786eebcd94 Parents: 6f0bc1c Author: shaofengshi <shaofeng...@apache.org> Authored: Sat Feb 4 21:37:05 2017 +0800 Committer: shaofengshi <shaofeng...@apache.org> Committed: Sat Feb 4 21:37:05 2017 +0800 ---------------------------------------------------------------------- .../apache/kylin/metadata/model/ColumnDesc.java | 8 +++++++ .../spark/SparkBatchCubingJobBuilder2.java | 2 +- .../table/DEFAULT.STREAMING_TABLE.json | 9 +++++--- .../localmeta/table/DEFAULT.TEST_ACCOUNT.json | 6 +++-- .../table/DEFAULT.TEST_CATEGORY_GROUPINGS.json | 9 +++++--- .../localmeta/table/DEFAULT.TEST_COUNTRY.json | 6 +++-- .../table/DEFAULT.TEST_KYLIN_FACT.json | 21 ++++++++++++------ .../localmeta/table/DEFAULT.TEST_ORDER.json | 6 +++-- .../localmeta/table/EDW.TEST_CAL_DT.json | 15 ++++++++----- .../table/EDW.TEST_SELLER_TYPE_DIM.json | 6 +++-- .../table/EDW.TEST_SELLER_TYPE_DIM_TABLE.json | 6 +++-- .../localmeta/table/EDW.TEST_SITES.json | 9 +++++--- .../java/org/apache/kylin/query/H2Database.java | 23 ++++++++++++++++++++ .../org/apache/kylin/query/KylinTestBase.java | 4 +++- 14 files changed, 97 insertions(+), 33 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/kylin/blob/5d83c80f/core-metadata/src/main/java/org/apache/kylin/metadata/model/ColumnDesc.java ---------------------------------------------------------------------- diff --git a/core-metadata/src/main/java/org/apache/kylin/metadata/model/ColumnDesc.java b/core-metadata/src/main/java/org/apache/kylin/metadata/model/ColumnDesc.java index 7105ede..e5b51e4 100644 --- a/core-metadata/src/main/java/org/apache/kylin/metadata/model/ColumnDesc.java +++ b/core-metadata/src/main/java/org/apache/kylin/metadata/model/ColumnDesc.java @@ -49,6 +49,10 @@ public class ColumnDesc implements Serializable { @JsonInclude(JsonInclude.Include.NON_NULL) private String dataGen; + @JsonProperty("index") + @JsonInclude(JsonInclude.Include.NON_NULL) + private String index; + // parsed from data type private DataType type; private DataType upgradedType; @@ -157,6 +161,10 @@ public class ColumnDesc implements Serializable { return dataGen; } + public String getIndex() { + return index; + } + public void init(TableDesc table) { this.table = table; http://git-wip-us.apache.org/repos/asf/kylin/blob/5d83c80f/engine-spark/src/main/java/org/apache/kylin/engine/spark/SparkBatchCubingJobBuilder2.java ---------------------------------------------------------------------- diff --git a/engine-spark/src/main/java/org/apache/kylin/engine/spark/SparkBatchCubingJobBuilder2.java b/engine-spark/src/main/java/org/apache/kylin/engine/spark/SparkBatchCubingJobBuilder2.java index 76b73b6..327d215 100644 --- a/engine-spark/src/main/java/org/apache/kylin/engine/spark/SparkBatchCubingJobBuilder2.java +++ b/engine-spark/src/main/java/org/apache/kylin/engine/spark/SparkBatchCubingJobBuilder2.java @@ -77,7 +77,7 @@ public class SparkBatchCubingJobBuilder2 extends BatchCubingJobBuilder2 { try { return ClassUtil.findContainingJar(Class.forName(className)); } catch (ClassNotFoundException e) { - logger.error("failed to locate jar for class " + className, e); + logger.warn("failed to locate jar for class " + className + ", ignore it", e); } return ""; http://git-wip-us.apache.org/repos/asf/kylin/blob/5d83c80f/examples/test_case_data/localmeta/table/DEFAULT.STREAMING_TABLE.json ---------------------------------------------------------------------- diff --git a/examples/test_case_data/localmeta/table/DEFAULT.STREAMING_TABLE.json b/examples/test_case_data/localmeta/table/DEFAULT.STREAMING_TABLE.json index f28683f..d67cbe5 100644 --- a/examples/test_case_data/localmeta/table/DEFAULT.STREAMING_TABLE.json +++ b/examples/test_case_data/localmeta/table/DEFAULT.STREAMING_TABLE.json @@ -5,17 +5,20 @@ { "id": "1", "name": "minute_start", - "datatype": "timestamp" + "datatype": "timestamp", + "index": "T" }, { "id": "2", "name": "hour_start", - "datatype": "timestamp" + "datatype": "timestamp", + "index": "T" }, { "id": "3", "name": "day_start", - "datatype": "date" + "datatype": "date", + "index": "T" }, { "id": "4", http://git-wip-us.apache.org/repos/asf/kylin/blob/5d83c80f/examples/test_case_data/localmeta/table/DEFAULT.TEST_ACCOUNT.json ---------------------------------------------------------------------- diff --git a/examples/test_case_data/localmeta/table/DEFAULT.TEST_ACCOUNT.json b/examples/test_case_data/localmeta/table/DEFAULT.TEST_ACCOUNT.json index 89e265b..5945713 100644 --- a/examples/test_case_data/localmeta/table/DEFAULT.TEST_ACCOUNT.json +++ b/examples/test_case_data/localmeta/table/DEFAULT.TEST_ACCOUNT.json @@ -8,7 +8,8 @@ "id" : "1", "name" : "ACCOUNT_ID", "datatype" : "bigint", - "data_gen" : "ID|10000000" + "data_gen" : "ID|10000000", + "index": "T" }, { "id" : "2", "name" : "ACCOUNT_BUYER_LEVEL", @@ -23,7 +24,8 @@ "id" : "4", "name" : "ACCOUNT_COUNTRY", "datatype" : "string", - "data_gen" : "CN|FR|GB|GE|JP|IT|RU|US" + "data_gen" : "CN|FR|GB|GE|JP|IT|RU|US", + "index": "T" }, { "id" : "5", "name" : "ACCOUNT_CONTACT", http://git-wip-us.apache.org/repos/asf/kylin/blob/5d83c80f/examples/test_case_data/localmeta/table/DEFAULT.TEST_CATEGORY_GROUPINGS.json ---------------------------------------------------------------------- diff --git a/examples/test_case_data/localmeta/table/DEFAULT.TEST_CATEGORY_GROUPINGS.json b/examples/test_case_data/localmeta/table/DEFAULT.TEST_CATEGORY_GROUPINGS.json index d7ab580..adaf987 100644 --- a/examples/test_case_data/localmeta/table/DEFAULT.TEST_CATEGORY_GROUPINGS.json +++ b/examples/test_case_data/localmeta/table/DEFAULT.TEST_CATEGORY_GROUPINGS.json @@ -5,15 +5,18 @@ "columns" : [ { "id" : "1", "name" : "LEAF_CATEG_ID", - "datatype" : "bigint" + "datatype" : "bigint", + "index": "T" }, { "id" : "2", "name" : "LEAF_CATEG_NAME", - "datatype" : "string" + "datatype" : "string", + "index": "T" }, { "id" : "3", "name" : "SITE_ID", - "datatype" : "int" + "datatype" : "int", + "index": "T" }, { "id" : "4", "name" : "CATEG_BUSN_MGR", http://git-wip-us.apache.org/repos/asf/kylin/blob/5d83c80f/examples/test_case_data/localmeta/table/DEFAULT.TEST_COUNTRY.json ---------------------------------------------------------------------- diff --git a/examples/test_case_data/localmeta/table/DEFAULT.TEST_COUNTRY.json b/examples/test_case_data/localmeta/table/DEFAULT.TEST_COUNTRY.json index 202182b..870cb2d 100644 --- a/examples/test_case_data/localmeta/table/DEFAULT.TEST_COUNTRY.json +++ b/examples/test_case_data/localmeta/table/DEFAULT.TEST_COUNTRY.json @@ -5,7 +5,8 @@ "columns" : [ { "id" : "1", "name" : "COUNTRY", - "datatype" : "string" + "datatype" : "string", + "index": "T" }, { "id" : "2", "name" : "LATITUDE", @@ -17,7 +18,8 @@ }, { "id" : "4", "name" : "NAME", - "datatype" : "string" + "datatype" : "string", + "index": "T" } ], "database" : "DEFAULT", "last_modified" : 0 http://git-wip-us.apache.org/repos/asf/kylin/blob/5d83c80f/examples/test_case_data/localmeta/table/DEFAULT.TEST_KYLIN_FACT.json ---------------------------------------------------------------------- diff --git a/examples/test_case_data/localmeta/table/DEFAULT.TEST_KYLIN_FACT.json b/examples/test_case_data/localmeta/table/DEFAULT.TEST_KYLIN_FACT.json index 2b33e66..e82d535 100644 --- a/examples/test_case_data/localmeta/table/DEFAULT.TEST_KYLIN_FACT.json +++ b/examples/test_case_data/localmeta/table/DEFAULT.TEST_KYLIN_FACT.json @@ -10,36 +10,43 @@ }, { "id" : "2", "name" : "ORDER_ID", - "datatype" : "bigint" + "datatype" : "bigint", + "index": "T" }, { "id" : "3", "name" : "CAL_DT", "datatype" : "date", - "data_gen" : "FK,order" + "data_gen" : "FK,order", + "index": "T" }, { "id" : "4", "name" : "LSTG_FORMAT_NAME", "datatype" : "string", - "data_gen" : "FP-GTC|FP-non GTC|ABIN|Auction|Others" + "data_gen" : "FP-GTC|FP-non GTC|ABIN|Auction|Others", + "index": "T" }, { "id" : "5", "name" : "LEAF_CATEG_ID", "datatype" : "bigint", - "data_gen" : "FK,null,nullstr=0" + "data_gen" : "FK,null,nullstr=0", + "index": "T" }, { "id" : "6", "name" : "LSTG_SITE_ID", - "datatype" : "int" + "datatype" : "int", + "index": "T" }, { "id" : "7", "name" : "SLR_SEGMENT_CD", "datatype" : "smallint", - "data_gen" : "FK,pk=EDW.TEST_SELLER_TYPE_DIM_TABLE.SELLER_TYPE_CD" + "data_gen" : "FK,pk=EDW.TEST_SELLER_TYPE_DIM_TABLE.SELLER_TYPE_CD", + "index": "T" }, { "id" : "8", "name" : "SELLER_ID", "datatype" : "bigint", - "data_gen" : "RAND||10000000|10001000" + "data_gen" : "RAND||10000000|10001000", + "index": "T" }, { "id" : "9", "name" : "PRICE", http://git-wip-us.apache.org/repos/asf/kylin/blob/5d83c80f/examples/test_case_data/localmeta/table/DEFAULT.TEST_ORDER.json ---------------------------------------------------------------------- diff --git a/examples/test_case_data/localmeta/table/DEFAULT.TEST_ORDER.json b/examples/test_case_data/localmeta/table/DEFAULT.TEST_ORDER.json index f157846..7232492 100644 --- a/examples/test_case_data/localmeta/table/DEFAULT.TEST_ORDER.json +++ b/examples/test_case_data/localmeta/table/DEFAULT.TEST_ORDER.json @@ -6,12 +6,14 @@ "id" : "1", "name" : "ORDER_ID", "datatype" : "bigint", - "data_gen" : "ID" + "data_gen" : "ID", + "index": "T" }, { "id" : "2", "name" : "BUYER_ID", "datatype" : "bigint", - "data_gen" : "RAND||10000500|10001500" + "data_gen" : "RAND||10000500|10001500", + "index": "T" }, { "id" : "3", "name" : "TEST_DATE_ENC", http://git-wip-us.apache.org/repos/asf/kylin/blob/5d83c80f/examples/test_case_data/localmeta/table/EDW.TEST_CAL_DT.json ---------------------------------------------------------------------- diff --git a/examples/test_case_data/localmeta/table/EDW.TEST_CAL_DT.json b/examples/test_case_data/localmeta/table/EDW.TEST_CAL_DT.json index 02299d9..1230649 100644 --- a/examples/test_case_data/localmeta/table/EDW.TEST_CAL_DT.json +++ b/examples/test_case_data/localmeta/table/EDW.TEST_CAL_DT.json @@ -5,23 +5,28 @@ "columns" : [ { "id" : "1", "name" : "CAL_DT", - "datatype" : "date" + "datatype" : "date", + "index": "T" }, { "id" : "2", "name" : "YEAR_BEG_DT", - "datatype" : "date" + "datatype" : "date", + "index": "T" }, { "id" : "3", "name" : "QTR_BEG_DT", - "datatype" : "date" + "datatype" : "date", + "index": "T" }, { "id" : "4", "name" : "MONTH_BEG_DT", - "datatype" : "date" + "datatype" : "date", + "index": "T" }, { "id" : "5", "name" : "WEEK_BEG_DT", - "datatype" : "date" + "datatype" : "date", + "index": "T" }, { "id" : "6", "name" : "AGE_FOR_YEAR_ID", http://git-wip-us.apache.org/repos/asf/kylin/blob/5d83c80f/examples/test_case_data/localmeta/table/EDW.TEST_SELLER_TYPE_DIM.json ---------------------------------------------------------------------- diff --git a/examples/test_case_data/localmeta/table/EDW.TEST_SELLER_TYPE_DIM.json b/examples/test_case_data/localmeta/table/EDW.TEST_SELLER_TYPE_DIM.json index 2bfbb70..136ae57 100644 --- a/examples/test_case_data/localmeta/table/EDW.TEST_SELLER_TYPE_DIM.json +++ b/examples/test_case_data/localmeta/table/EDW.TEST_SELLER_TYPE_DIM.json @@ -4,7 +4,8 @@ "columns" : [ { "id" : "1", "name" : "SELLER_TYPE_CD", - "datatype" : "smallint" + "datatype" : "smallint", + "index": "T" }, { "id" : "2", "name" : "SELLER_TYPE_DESC", @@ -16,7 +17,8 @@ }, { "id" : "4", "name" : "SELLER_GROUP_CD", - "datatype" : "tinyint" + "datatype" : "tinyint", + "index": "T" }, { "id" : "5", "name" : "SELLER_GROUP_DESC", http://git-wip-us.apache.org/repos/asf/kylin/blob/5d83c80f/examples/test_case_data/localmeta/table/EDW.TEST_SELLER_TYPE_DIM_TABLE.json ---------------------------------------------------------------------- diff --git a/examples/test_case_data/localmeta/table/EDW.TEST_SELLER_TYPE_DIM_TABLE.json b/examples/test_case_data/localmeta/table/EDW.TEST_SELLER_TYPE_DIM_TABLE.json index 3c5749b..794810d 100644 --- a/examples/test_case_data/localmeta/table/EDW.TEST_SELLER_TYPE_DIM_TABLE.json +++ b/examples/test_case_data/localmeta/table/EDW.TEST_SELLER_TYPE_DIM_TABLE.json @@ -4,7 +4,8 @@ "columns" : [ { "id" : "1", "name" : "SELLER_TYPE_CD", - "datatype" : "smallint" + "datatype" : "smallint", + "index": "T" }, { "id" : "2", "name" : "SELLER_TYPE_DESC", @@ -16,7 +17,8 @@ }, { "id" : "4", "name" : "SELLER_GROUP_CD", - "datatype" : "tinyint" + "datatype" : "tinyint", + "index": "T" }, { "id" : "5", "name" : "SELLER_GROUP_DESC", http://git-wip-us.apache.org/repos/asf/kylin/blob/5d83c80f/examples/test_case_data/localmeta/table/EDW.TEST_SITES.json ---------------------------------------------------------------------- diff --git a/examples/test_case_data/localmeta/table/EDW.TEST_SITES.json b/examples/test_case_data/localmeta/table/EDW.TEST_SITES.json index cc801af..4c2d727 100644 --- a/examples/test_case_data/localmeta/table/EDW.TEST_SITES.json +++ b/examples/test_case_data/localmeta/table/EDW.TEST_SITES.json @@ -5,11 +5,13 @@ "columns" : [ { "id" : "1", "name" : "SITE_ID", - "datatype" : "int" + "datatype" : "int", + "index": "T" }, { "id" : "2", "name" : "SITE_NAME", - "datatype" : "string" + "datatype" : "string", + "index": "T" }, { "id" : "3", "name" : "SITE_DOMAIN_CODE", @@ -25,7 +27,8 @@ }, { "id" : "6", "name" : "SITE_CNTRY_ID", - "datatype" : "int" + "datatype" : "int", + "index": "T" }, { "id" : "7", "name" : "CRE_DATE", http://git-wip-us.apache.org/repos/asf/kylin/blob/5d83c80f/kylin-it/src/test/java/org/apache/kylin/query/H2Database.java ---------------------------------------------------------------------- diff --git a/kylin-it/src/test/java/org/apache/kylin/query/H2Database.java b/kylin-it/src/test/java/org/apache/kylin/query/H2Database.java index c449837..8b60c49 100644 --- a/kylin-it/src/test/java/org/apache/kylin/query/H2Database.java +++ b/kylin-it/src/test/java/org/apache/kylin/query/H2Database.java @@ -26,8 +26,10 @@ import java.sql.Connection; import java.sql.SQLException; import java.sql.Statement; import java.util.HashMap; +import java.util.List; import java.util.Map; +import com.clearspring.analytics.util.Lists; import org.apache.commons.io.IOUtils; import org.apache.kylin.common.KylinConfig; import org.apache.kylin.metadata.MetadataManager; @@ -103,6 +105,11 @@ public class H2Database { String sql = generateCreateH2TableSql(tableDesc, cvsFilePath); stmt.executeUpdate(sql); + List<String> createIndexStatements = generateCreateH2IndexSql(tableDesc); + for (String indexSql : createIndexStatements) { + stmt.executeUpdate(indexSql); + } + if (tempFile != null) tempFile.delete(); } @@ -136,6 +143,22 @@ public class H2Database { return ddl.toString(); } + private List<String> generateCreateH2IndexSql(TableDesc tableDesc) { + List<String> result = Lists.newArrayList(); + int x = 0; + for (ColumnDesc col : tableDesc.getColumns()) { + if ("T".equalsIgnoreCase(col.getIndex())) { + StringBuilder ddl = new StringBuilder(); + ddl.append("CREATE INDEX IDX_" + tableDesc.getName() + "_" + x + " ON " + tableDesc.getIdentity() + "(" + col.getName() + ")"); + ddl.append("\n"); + result.add(ddl.toString()); + x++; + } + } + + return result; + } + private static String getH2DataType(String javaDataType) { String hiveDataType = javaToH2DataTypeMapping.get(javaDataType.toLowerCase()); if (hiveDataType == null) { http://git-wip-us.apache.org/repos/asf/kylin/blob/5d83c80f/kylin-it/src/test/java/org/apache/kylin/query/KylinTestBase.java ---------------------------------------------------------------------- diff --git a/kylin-it/src/test/java/org/apache/kylin/query/KylinTestBase.java b/kylin-it/src/test/java/org/apache/kylin/query/KylinTestBase.java index deb1104..1cc5c76 100644 --- a/kylin-it/src/test/java/org/apache/kylin/query/KylinTestBase.java +++ b/kylin-it/src/test/java/org/apache/kylin/query/KylinTestBase.java @@ -499,7 +499,9 @@ public class KylinTestBase { // execute H2 printInfo("Query Result from H2 - " + queryName); + long currentTime = System.currentTimeMillis(); ITable h2Table = executeQuery(newH2Connection(), queryName, sql, needSort); + printInfo("H2 spent " + (System.currentTimeMillis() - currentTime) + " mili-seconds."); try { // compare the result @@ -644,7 +646,7 @@ public class KylinTestBase { cubeConnection = DriverManager.getConnection("jdbc:calcite:model=" + olapTmp.getAbsolutePath(), props); //setup h2 - h2Connection = DriverManager.getConnection("jdbc:h2:mem:db" + (h2InstanceCount++), "sa", ""); + h2Connection = DriverManager.getConnection("jdbc:h2:mem:db" + (h2InstanceCount++) + ";CACHE_SIZE=32072;DB_CLOSE_DELAY=-1", "sa", ""); // Load H2 Tables (inner join) H2Database h2DB = new H2Database(h2Connection, config); h2DB.loadAllTables();
