KYLIN-1971 bug fixes after manual test
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/d3ac3542 Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/d3ac3542 Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/d3ac3542 Branch: refs/heads/KYLIN-1971 Commit: d3ac35420c641257d1644c27cecb97b34a21ace7 Parents: 13eeb68 Author: Yang Li <liy...@apache.org> Authored: Sun Nov 6 17:30:10 2016 +0800 Committer: Yang Li <liy...@apache.org> Committed: Sun Nov 6 17:30:10 2016 +0800 ---------------------------------------------------------------------- .../kylin/job/dataGen/FactTableGenerator.java | 4 +- .../java/org/apache/kylin/cube/CubeManager.java | 2 +- .../kylin/cube/cli/DictionaryGeneratorCLI.java | 25 ++-- .../org/apache/kylin/cube/model/CubeDesc.java | 2 +- .../org/apache/kylin/cube/CubeManagerTest.java | 4 +- .../org/apache/kylin/job/JoinedFlatTable.java | 135 ++++++------------- .../kylin/metadata/model/PartitionDesc.java | 102 +++++++------- .../apache/kylin/metadata/model/TblColRef.java | 2 +- .../DefaultPartitionConditionBuilderTest.java | 44 +++--- .../apache/kylin/query/relnode/OLAPJoinRel.java | 3 +- .../org/apache/kylin/query/relnode/OLAPRel.java | 37 ++--- .../kylin/query/relnode/OLAPUnionRel.java | 6 +- .../kylin/rest/controller/CubeController.java | 6 +- 13 files changed, 156 insertions(+), 216 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/kylin/blob/d3ac3542/assembly/src/test/java/org/apache/kylin/job/dataGen/FactTableGenerator.java ---------------------------------------------------------------------- diff --git a/assembly/src/test/java/org/apache/kylin/job/dataGen/FactTableGenerator.java b/assembly/src/test/java/org/apache/kylin/job/dataGen/FactTableGenerator.java index ce5d8c4..84a642d 100644 --- a/assembly/src/test/java/org/apache/kylin/job/dataGen/FactTableGenerator.java +++ b/assembly/src/test/java/org/apache/kylin/job/dataGen/FactTableGenerator.java @@ -243,7 +243,7 @@ public class FactTableGenerator { JoinDesc join = dim.getJoin(); if (join != null) { - String lookupTable = dim.getTable(); + String lookupTable = dim.getTableRef().getTableIdentity(); for (String column : join.getPrimaryKey()) { if (!lookupTableKeys.containsKey(lookupTable)) { lookupTableKeys.put(lookupTable, new LinkedList<String>()); @@ -301,7 +301,7 @@ public class FactTableGenerator { String[] pks = jDesc.getPrimaryKey(); int num = fks.length; for (int i = 0; i < num; ++i) { - String value = dim.getTable() + "/" + pks[i]; + String value = dim.getTableRef().getTableIdentity() + "/" + pks[i]; lookupCol2factTableCol.put(value, fks[i]); http://git-wip-us.apache.org/repos/asf/kylin/blob/d3ac3542/core-cube/src/main/java/org/apache/kylin/cube/CubeManager.java ---------------------------------------------------------------------- diff --git a/core-cube/src/main/java/org/apache/kylin/cube/CubeManager.java b/core-cube/src/main/java/org/apache/kylin/cube/CubeManager.java index b7ea7b8..87bb93d 100644 --- a/core-cube/src/main/java/org/apache/kylin/cube/CubeManager.java +++ b/core-cube/src/main/java/org/apache/kylin/cube/CubeManager.java @@ -662,7 +662,7 @@ public class CubeManager implements IRealizationProvider { public LookupStringTable getLookupTable(CubeSegment cubeSegment, DimensionDesc dim) { - String tableName = dim.getTable(); + String tableName = dim.getTableRef().getTableIdentity(); String[] pkCols = dim.getJoin().getPrimaryKey(); String snapshotResPath = cubeSegment.getSnapshotResPath(tableName); if (snapshotResPath == null) http://git-wip-us.apache.org/repos/asf/kylin/blob/d3ac3542/core-cube/src/main/java/org/apache/kylin/cube/cli/DictionaryGeneratorCLI.java ---------------------------------------------------------------------- diff --git a/core-cube/src/main/java/org/apache/kylin/cube/cli/DictionaryGeneratorCLI.java b/core-cube/src/main/java/org/apache/kylin/cube/cli/DictionaryGeneratorCLI.java index d3b0782..fcd68ba 100644 --- a/core-cube/src/main/java/org/apache/kylin/cube/cli/DictionaryGeneratorCLI.java +++ b/core-cube/src/main/java/org/apache/kylin/cube/cli/DictionaryGeneratorCLI.java @@ -19,6 +19,7 @@ package org.apache.kylin.cube.cli; import java.io.IOException; +import java.util.Set; import org.apache.kylin.common.KylinConfig; import org.apache.kylin.cube.CubeInstance; @@ -30,6 +31,8 @@ import org.apache.kylin.metadata.model.TblColRef; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import com.google.common.collect.Sets; + public class DictionaryGeneratorCLI { private static final Logger logger = LoggerFactory.getLogger(DictionaryGeneratorCLI.class); @@ -50,16 +53,20 @@ public class DictionaryGeneratorCLI { cubeMgr.buildDictionary(cubeSeg, col, factTableValueProvider); } + // snapshot + Set<String> toSnapshot = Sets.newHashSet(); for (DimensionDesc dim : cubeSeg.getCubeDesc().getDimensions()) { - // build snapshot - if (dim.getTable() != null && !dim.getTable().equalsIgnoreCase(cubeSeg.getCubeDesc().getFactTable())) { - // CubeSegment seg = cube.getTheOnlySegment(); - logger.info("Building snapshot of " + dim.getTable()); - cubeMgr.buildSnapshotTable(cubeSeg, dim.getTable()); - logger.info("Checking snapshot of " + dim.getTable()); - cubeMgr.getLookupTable(cubeSeg, dim); // load the table for sanity check - } + if (dim.getTableRef() == null) + continue; + + String lookupTable = dim.getTableRef().getTableIdentity(); + toSnapshot.add(lookupTable); + } + toSnapshot.remove(cubeSeg.getCubeDesc().getFactTable()); + + for (String tableIdentity : toSnapshot) { + logger.info("Building snapshot of " + tableIdentity); + cubeMgr.buildSnapshotTable(cubeSeg, tableIdentity); } } - } http://git-wip-us.apache.org/repos/asf/kylin/blob/d3ac3542/core-cube/src/main/java/org/apache/kylin/cube/model/CubeDesc.java ---------------------------------------------------------------------- diff --git a/core-cube/src/main/java/org/apache/kylin/cube/model/CubeDesc.java b/core-cube/src/main/java/org/apache/kylin/cube/model/CubeDesc.java index 159042d..34650f4 100644 --- a/core-cube/src/main/java/org/apache/kylin/cube/model/CubeDesc.java +++ b/core-cube/src/main/java/org/apache/kylin/cube/model/CubeDesc.java @@ -242,7 +242,7 @@ public class CubeDesc extends RootPersistentEntity implements IEngineAware { public DimensionDesc findDimensionByTable(String lookupTableName) { lookupTableName = lookupTableName.toUpperCase(); for (DimensionDesc dim : dimensions) - if (dim.getTable() != null && dim.getTable().equals(lookupTableName)) + if (dim.getTableRef() != null && dim.getTableRef().getTableIdentity().equals(lookupTableName)) return dim; return null; } http://git-wip-us.apache.org/repos/asf/kylin/blob/d3ac3542/core-cube/src/test/java/org/apache/kylin/cube/CubeManagerTest.java ---------------------------------------------------------------------- diff --git a/core-cube/src/test/java/org/apache/kylin/cube/CubeManagerTest.java b/core-cube/src/test/java/org/apache/kylin/cube/CubeManagerTest.java index 2904eb2..0266067 100644 --- a/core-cube/src/test/java/org/apache/kylin/cube/CubeManagerTest.java +++ b/core-cube/src/test/java/org/apache/kylin/cube/CubeManagerTest.java @@ -19,7 +19,6 @@ package org.apache.kylin.cube; import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertNotNull; import static org.junit.Assert.assertNull; import static org.junit.Assert.assertTrue; @@ -27,7 +26,6 @@ import java.util.List; import java.util.Map; import java.util.NavigableSet; -import com.google.common.collect.Maps; import org.apache.kylin.common.KylinConfig; import org.apache.kylin.common.persistence.ResourceStore; import org.apache.kylin.common.util.JsonUtil; @@ -41,6 +39,8 @@ import org.junit.After; import org.junit.Before; import org.junit.Test; +import com.google.common.collect.Maps; + /** * @author yangli9 */ http://git-wip-us.apache.org/repos/asf/kylin/blob/d3ac3542/core-job/src/main/java/org/apache/kylin/job/JoinedFlatTable.java ---------------------------------------------------------------------- diff --git a/core-job/src/main/java/org/apache/kylin/job/JoinedFlatTable.java b/core-job/src/main/java/org/apache/kylin/job/JoinedFlatTable.java index e43bca3..af9c948 100644 --- a/core-job/src/main/java/org/apache/kylin/job/JoinedFlatTable.java +++ b/core-job/src/main/java/org/apache/kylin/job/JoinedFlatTable.java @@ -19,10 +19,7 @@ package org.apache.kylin.job; import java.io.File; -import java.util.HashMap; import java.util.HashSet; -import java.util.List; -import java.util.Map; import java.util.Set; import javax.xml.parsers.DocumentBuilder; @@ -34,12 +31,11 @@ import org.apache.kylin.metadata.model.IJoinedFlatTableDesc; import org.apache.kylin.metadata.model.JoinDesc; import org.apache.kylin.metadata.model.LookupDesc; import org.apache.kylin.metadata.model.PartitionDesc; +import org.apache.kylin.metadata.model.TableRef; import org.apache.kylin.metadata.model.TblColRef; import org.w3c.dom.Document; import org.w3c.dom.NodeList; -import com.google.common.collect.Lists; - /** * */ @@ -88,13 +84,11 @@ public class JoinedFlatTable { if (i > 0) { ddl.append(","); } - ddl.append(colName(col.getCanonicalName()) + " " + getHiveDataType(col.getDatatype()) + "\n"); + ddl.append(colName(col) + " " + getHiveDataType(col.getDatatype()) + "\n"); } ddl.append(")" + "\n"); ddl.append("STORED AS SEQUENCEFILE" + "\n"); ddl.append("LOCATION '" + getTableDir(flatDesc, storageDfsDir) + "';").append("\n"); - // ddl.append("TBLPROPERTIES ('serialization.null.format'='\\\\N')" + - // ";\n"); return ddl.toString(); } @@ -114,125 +108,74 @@ public class JoinedFlatTable { public static String generateSelectDataStatement(IJoinedFlatTableDesc flatDesc, boolean redistribute) { StringBuilder sql = new StringBuilder(); sql.append("SELECT" + "\n"); - String tableAlias; - Map<String, String> tableAliasMap = buildTableAliasMap(flatDesc.getDataModel()); for (int i = 0; i < flatDesc.getAllColumns().size(); i++) { TblColRef col = flatDesc.getAllColumns().get(i); if (i > 0) { sql.append(","); } - tableAlias = tableAliasMap.get(col.getTable()); - sql.append(tableAlias + "." + col.getName() + "\n"); + sql.append(col.getTableAlias() + "." + col.getName() + "\n"); } - appendJoinStatement(flatDesc, sql, tableAliasMap); - appendWhereStatement(flatDesc, sql, tableAliasMap); + appendJoinStatement(flatDesc, sql); + appendWhereStatement(flatDesc, sql); if (redistribute == true) { - String redistributeCol = null; - TblColRef distDcol = flatDesc.getDistributedBy(); - if (distDcol != null) { - String tblAlias = tableAliasMap.get(distDcol.getTable()); - redistributeCol = tblAlias + "." + distDcol.getName(); - } - appendDistributeStatement(sql, redistributeCol); + TblColRef distCol = flatDesc.getDistributedBy(); + appendDistributeStatement(sql, distCol); } return sql.toString(); } public static String generateCountDataStatement(IJoinedFlatTableDesc flatDesc, final String outputDir) { - final Map<String, String> tableAliasMap = buildTableAliasMap(flatDesc.getDataModel()); final StringBuilder sql = new StringBuilder(); - final String factTbl = flatDesc.getDataModel().getFactTable(); + final TableRef factTbl = flatDesc.getDataModel().getFactTableRef(); sql.append("dfs -mkdir -p " + outputDir + ";\n"); - sql.append("INSERT OVERWRITE DIRECTORY '" + outputDir + "' SELECT count(*) FROM " + factTbl + " " + tableAliasMap.get(factTbl) + "\n"); - appendWhereStatement(flatDesc, sql, tableAliasMap); + sql.append("INSERT OVERWRITE DIRECTORY '" + outputDir + "' SELECT count(*) FROM " + factTbl.getTableIdentity() + " " + factTbl.getAlias() + "\n"); + appendWhereStatement(flatDesc, sql); return sql.toString(); } - private static Map<String, String> buildTableAliasMap(DataModelDesc dataModelDesc) { - Map<String, String> tableAliasMap = new HashMap<String, String>(); - - addTableAlias(dataModelDesc.getFactTable(), tableAliasMap); - - for (LookupDesc lookupDesc : dataModelDesc.getLookups()) { - JoinDesc join = lookupDesc.getJoin(); - if (join != null) { - addTableAlias(lookupDesc.getTable(), tableAliasMap); - } - } - return tableAliasMap; - } - - // The table alias used to be "FACT_TABLE" and "LOOKUP_#", but that's too unpredictable - // for those who want to write a filter. (KYLIN-900) - // Also yet don't support joining the same table more than once, since table name is the map key. - private static void addTableAlias(String table, Map<String, String> tableAliasMap) { - String alias; - int cut = table.lastIndexOf('.'); - if (cut < 0) - alias = table; - else - alias = table.substring(cut + 1); - - tableAliasMap.put(table, alias); - } - - private static void appendJoinStatement(IJoinedFlatTableDesc flatDesc, StringBuilder sql, Map<String, String> tableAliasMap) { - List<JoinDesc> cubeJoins = getUsedJoinsSet(flatDesc); - - Set<String> dimTableCache = new HashSet<String>(); + private static void appendJoinStatement(IJoinedFlatTableDesc flatDesc, StringBuilder sql) { + Set<TableRef> dimTableCache = new HashSet<>(); DataModelDesc dataModelDesc = flatDesc.getDataModel(); - String factTableName = dataModelDesc.getFactTable(); - String factTableAlias = tableAliasMap.get(factTableName); - sql.append("FROM " + factTableName + " as " + factTableAlias + " \n"); + TableRef factTable = dataModelDesc.getFactTableRef(); + sql.append("FROM " + factTable.getTableIdentity() + " as " + factTable.getAlias() + " \n"); for (LookupDesc lookupDesc : dataModelDesc.getLookups()) { JoinDesc join = lookupDesc.getJoin(); - if (!cubeJoins.contains(join)) { - continue; - } if (join != null && join.getType().equals("") == false) { String joinType = join.getType().toUpperCase(); - String dimTableName = lookupDesc.getTable(); - if (!dimTableCache.contains(dimTableName)) { + TableRef dimTable = lookupDesc.getTableRef(); + if (!dimTableCache.contains(dimTable)) { TblColRef[] pk = join.getPrimaryKeyColumns(); TblColRef[] fk = join.getForeignKeyColumns(); if (pk.length != fk.length) { throw new RuntimeException("Invalid join condition of lookup table:" + lookupDesc); } - sql.append(joinType + " JOIN " + dimTableName + " as " + tableAliasMap.get(dimTableName) + "\n"); + sql.append(joinType + " JOIN " + dimTable.getTableIdentity() + " as " + dimTable.getAlias() + "\n"); sql.append("ON "); for (int i = 0; i < pk.length; i++) { if (i > 0) { sql.append(" AND "); } - sql.append(factTableAlias + "." + fk[i].getName() + " = " + tableAliasMap.get(dimTableName) + "." + pk[i].getName()); + sql.append(fk[i].getTableAlias() + "." + fk[i].getName() + " = " + pk[i].getTableAlias() + "." + pk[i].getName()); } sql.append("\n"); - dimTableCache.add(dimTableName); + dimTableCache.add(dimTable); } } } } - private static List<JoinDesc> getUsedJoinsSet(IJoinedFlatTableDesc flatDesc) { - List<JoinDesc> result = Lists.newArrayList(); - for (LookupDesc lookup : flatDesc.getDataModel().getLookups()) { - result.add(lookup.getJoin()); - } - return result; - } - - private static void appendDistributeStatement(StringBuilder sql, String redistributeCol) { - if (redistributeCol != null) { - sql.append(" DISTRIBUTE BY ").append(redistributeCol).append(";\n"); + private static void appendDistributeStatement(StringBuilder sql, TblColRef redistCol) { + if (redistCol != null) { + sql.append(" DISTRIBUTE BY ").append(redistCol.getTableAlias() + "." + redistCol.getName()).append(";\n"); } else { sql.append(" DISTRIBUTE BY RAND()").append(";\n"); } } - private static void appendWhereStatement(IJoinedFlatTableDesc flatDesc, StringBuilder sql, Map<String, String> tableAliasMap) { + private static void appendWhereStatement(IJoinedFlatTableDesc flatDesc, StringBuilder sql) { boolean hasCondition = false; StringBuilder whereBuilder = new StringBuilder(); whereBuilder.append("WHERE"); @@ -244,16 +187,18 @@ public class JoinedFlatTable { hasCondition = true; } - PartitionDesc partDesc = model.getPartitionDesc(); - if (partDesc != null && partDesc.getPartitionDateColumn() != null) { - long dateStart = flatDesc.getSourceOffsetStart(); - long dateEnd = flatDesc.getSourceOffsetEnd(); - - if (!(dateStart == 0 && dateEnd == Long.MAX_VALUE)) { - whereBuilder.append(hasCondition ? " AND (" : " ("); - whereBuilder.append(partDesc.getPartitionConditionBuilder().buildDateRangeCondition(partDesc, dateStart, dateEnd, tableAliasMap)); - whereBuilder.append(")\n"); - hasCondition = true; + if (flatDesc.getSegment() != null) { + PartitionDesc partDesc = model.getPartitionDesc(); + if (partDesc != null && partDesc.getPartitionDateColumn() != null) { + long dateStart = flatDesc.getSourceOffsetStart(); + long dateEnd = flatDesc.getSourceOffsetEnd(); + + if (!(dateStart == 0 && dateEnd == Long.MAX_VALUE)) { + whereBuilder.append(hasCondition ? " AND (" : " ("); + whereBuilder.append(partDesc.getPartitionConditionBuilder().buildDateRangeCondition(partDesc, dateStart, dateEnd)); + whereBuilder.append(")\n"); + hasCondition = true; + } } } @@ -262,8 +207,8 @@ public class JoinedFlatTable { } } - private static String colName(String canonicalColName) { - return canonicalColName.replace(".", "_"); + private static String colName(TblColRef col) { + return col.getTableAlias() + "_" + col.getName(); } private static String getHiveDataType(String javaDataType) { @@ -285,12 +230,8 @@ public class JoinedFlatTable { StringBuilder sql = new StringBuilder(); sql.append("INSERT OVERWRITE TABLE " + tableName + " SELECT * FROM " + tableName); - String redistributeCol = null; TblColRef distDcol = flatDesc.getDistributedBy(); - if (distDcol != null) { - redistributeCol = colName(distDcol.getCanonicalName()); - } - appendDistributeStatement(sql, redistributeCol); + appendDistributeStatement(sql, distDcol); return sql.toString(); } http://git-wip-us.apache.org/repos/asf/kylin/blob/d3ac3542/core-metadata/src/main/java/org/apache/kylin/metadata/model/PartitionDesc.java ---------------------------------------------------------------------- diff --git a/core-metadata/src/main/java/org/apache/kylin/metadata/model/PartitionDesc.java b/core-metadata/src/main/java/org/apache/kylin/metadata/model/PartitionDesc.java index 127d5e1..d6f18e5 100644 --- a/core-metadata/src/main/java/org/apache/kylin/metadata/model/PartitionDesc.java +++ b/core-metadata/src/main/java/org/apache/kylin/metadata/model/PartitionDesc.java @@ -18,8 +18,6 @@ package org.apache.kylin.metadata.model; -import java.util.Map; - import org.apache.commons.lang3.StringUtils; import org.apache.kylin.common.util.ClassUtil; import org.apache.kylin.common.util.DateFormat; @@ -61,6 +59,7 @@ public class PartitionDesc { private String partitionConditionBuilderClz = DefaultPartitionConditionBuilder.class.getName(); private TblColRef partitionDateColumnRef; + private TblColRef partitionTimeColumnRef; private IPartitionConditionBuilder partitionConditionBuilder; public void init(DataModelDesc model) { @@ -69,6 +68,10 @@ public class PartitionDesc { partitionDateColumn = partitionDateColumn.toUpperCase(); partitionDateColumnRef = model.findColumn(partitionDateColumn); + if (StringUtils.isBlank(partitionTimeColumn) == false) { + partitionTimeColumn = partitionTimeColumn.toUpperCase(); + partitionTimeColumnRef = model.findColumn(partitionTimeColumn); + } partitionConditionBuilder = (IPartitionConditionBuilder) ClassUtil.newInstance(partitionConditionBuilderClz); } @@ -96,18 +99,30 @@ public class PartitionDesc { return partitionDateColumn; } + // for test public void setPartitionDateColumn(String partitionDateColumn) { this.partitionDateColumn = partitionDateColumn; } - + + // for test + void setPartitionDateColumnRef(TblColRef partitionDateColumnRef) { + this.partitionDateColumnRef = partitionDateColumnRef; + } + public String getPartitionTimeColumn() { return partitionTimeColumn; } - public void setPartitionTimeColumn(String partitionTimeColumn) { + // for test + void setPartitionTimeColumn(String partitionTimeColumn) { this.partitionTimeColumn = partitionTimeColumn; } + // for test + void setPartitionTimeColumnRef(TblColRef partitionTimeColumnRef) { + this.partitionTimeColumnRef = partitionTimeColumnRef; + } + @Deprecated public long getPartitionDateStart() { return partitionDateStart; @@ -150,50 +165,41 @@ public class PartitionDesc { return partitionDateColumnRef; } + public TblColRef getPartitionTimeColumnRef() { + return partitionTimeColumnRef; + } + // ============================================================================ public static interface IPartitionConditionBuilder { - String buildDateRangeCondition(PartitionDesc partDesc, long startInclusive, long endExclusive, Map<String, String> tableAlias); + String buildDateRangeCondition(PartitionDesc partDesc, long startInclusive, long endExclusive); } public static class DefaultPartitionConditionBuilder implements IPartitionConditionBuilder { @Override - public String buildDateRangeCondition(PartitionDesc partDesc, long startInclusive, long endExclusive, Map<String, String> tableAlias) { + public String buildDateRangeCondition(PartitionDesc partDesc, long startInclusive, long endExclusive) { StringBuilder builder = new StringBuilder(); - String partitionDateColumnName = partDesc.getPartitionDateColumn(); - String partitionTimeColumnName = partDesc.getPartitionTimeColumn(); + TblColRef partitionDateColumn = partDesc.getPartitionDateColumnRef(); + TblColRef partitionTimeColumn = partDesc.getPartitionTimeColumnRef(); if (partDesc.partitionColumnIsYmdInt()) { - buildSingleColumnRangeCondAsYmdInt(builder, partitionDateColumnName, startInclusive, endExclusive, tableAlias); + buildSingleColumnRangeCondAsYmdInt(builder, partitionDateColumn, startInclusive, endExclusive); } else if (partDesc.partitionColumnIsTimeMillis()) { - buildSingleColumnRangeCondAsTimeMillis(builder, partitionDateColumnName, startInclusive, endExclusive, tableAlias); - } else if (partitionDateColumnName != null && partitionTimeColumnName == null) { - buildSingleColumnRangeCondition(builder, partitionDateColumnName, startInclusive, endExclusive, partDesc.getPartitionDateFormat(), tableAlias); - } else if (partitionDateColumnName == null && partitionTimeColumnName != null) { - buildSingleColumnRangeCondition(builder, partitionTimeColumnName, startInclusive, endExclusive, partDesc.getPartitionTimeFormat(), tableAlias); - } else if (partitionDateColumnName != null && partitionTimeColumnName != null) { - buildMultipleColumnRangeCondition(builder, partitionDateColumnName, partitionTimeColumnName, startInclusive, endExclusive, partDesc.getPartitionDateFormat(), partDesc.getPartitionTimeFormat(), tableAlias); + buildSingleColumnRangeCondAsTimeMillis(builder, partitionDateColumn, startInclusive, endExclusive); + } else if (partitionDateColumn != null && partitionTimeColumn == null) { + buildSingleColumnRangeCondition(builder, partitionDateColumn, startInclusive, endExclusive, partDesc.getPartitionDateFormat()); + } else if (partitionDateColumn == null && partitionTimeColumn != null) { + buildSingleColumnRangeCondition(builder, partitionTimeColumn, startInclusive, endExclusive, partDesc.getPartitionTimeFormat()); + } else if (partitionDateColumn != null && partitionTimeColumn != null) { + buildMultipleColumnRangeCondition(builder, partitionDateColumn, partitionTimeColumn, startInclusive, endExclusive, partDesc.getPartitionDateFormat(), partDesc.getPartitionTimeFormat()); } return builder.toString(); } - /** - * Convert to use table alias - */ - private static String replaceColumnNameWithAlias(String columnName, Map<String, String> tableAlias) { - int indexOfDot = columnName.lastIndexOf("."); - if (indexOfDot > 0) { - String partitionTableName = columnName.substring(0, indexOfDot); - if (tableAlias != null && tableAlias.containsKey(partitionTableName)) - columnName = tableAlias.get(partitionTableName) + columnName.substring(indexOfDot); - } - return columnName; - } - - private static void buildSingleColumnRangeCondAsTimeMillis(StringBuilder builder, String partitionColumnName, long startInclusive, long endExclusive, Map<String, String> tableAlias) { - partitionColumnName = replaceColumnNameWithAlias(partitionColumnName, tableAlias); + private static void buildSingleColumnRangeCondAsTimeMillis(StringBuilder builder, TblColRef partitionColumn, long startInclusive, long endExclusive) { + String partitionColumnName = partitionColumn.getTableAlias() + "." + partitionColumn.getName(); if (startInclusive > 0) { builder.append(partitionColumnName + " >= " + startInclusive); builder.append(" AND "); @@ -201,8 +207,8 @@ public class PartitionDesc { builder.append(partitionColumnName + " < " + endExclusive); } - private static void buildSingleColumnRangeCondAsYmdInt(StringBuilder builder, String partitionColumnName, long startInclusive, long endExclusive, Map<String, String> tableAlias) { - partitionColumnName = replaceColumnNameWithAlias(partitionColumnName, tableAlias); + private static void buildSingleColumnRangeCondAsYmdInt(StringBuilder builder, TblColRef partitionColumn, long startInclusive, long endExclusive) { + String partitionColumnName = partitionColumn.getTableAlias() + "." + partitionColumn.getName(); if (startInclusive > 0) { builder.append(partitionColumnName + " >= " + DateFormat.formatToDateStr(startInclusive, DateFormat.COMPACT_DATE_PATTERN)); builder.append(" AND "); @@ -210,8 +216,8 @@ public class PartitionDesc { builder.append(partitionColumnName + " < " + DateFormat.formatToDateStr(endExclusive, DateFormat.COMPACT_DATE_PATTERN)); } - private static void buildSingleColumnRangeCondition(StringBuilder builder, String partitionColumnName, long startInclusive, long endExclusive, String partitionColumnDateFormat, Map<String, String> tableAlias) { - partitionColumnName = replaceColumnNameWithAlias(partitionColumnName, tableAlias); + private static void buildSingleColumnRangeCondition(StringBuilder builder, TblColRef partitionColumn, long startInclusive, long endExclusive, String partitionColumnDateFormat) { + String partitionColumnName = partitionColumn.getTableAlias() + "." + partitionColumn.getName(); if (startInclusive > 0) { builder.append(partitionColumnName + " >= '" + DateFormat.formatToDateStr(startInclusive, partitionColumnDateFormat) + "'"); builder.append(" AND "); @@ -219,9 +225,9 @@ public class PartitionDesc { builder.append(partitionColumnName + " < '" + DateFormat.formatToDateStr(endExclusive, partitionColumnDateFormat) + "'"); } - private static void buildMultipleColumnRangeCondition(StringBuilder builder, String partitionDateColumnName, String partitionTimeColumnName, long startInclusive, long endExclusive, String partitionColumnDateFormat, String partitionColumnTimeFormat, Map<String, String> tableAlias) { - partitionDateColumnName = replaceColumnNameWithAlias(partitionDateColumnName, tableAlias); - partitionTimeColumnName = replaceColumnNameWithAlias(partitionTimeColumnName, tableAlias); + private static void buildMultipleColumnRangeCondition(StringBuilder builder, TblColRef partitionDateColumn, TblColRef partitionTimeColumn, long startInclusive, long endExclusive, String partitionColumnDateFormat, String partitionColumnTimeFormat) { + String partitionDateColumnName = partitionDateColumn.getTableAlias() + "." + partitionDateColumn.getName(); + String partitionTimeColumnName = partitionTimeColumn.getTableAlias() + "." + partitionTimeColumn.getName(); if (startInclusive > 0) { builder.append("("); builder.append("("); @@ -254,24 +260,12 @@ public class PartitionDesc { public static class YearMonthDayPartitionConditionBuilder implements PartitionDesc.IPartitionConditionBuilder { @Override - public String buildDateRangeCondition(PartitionDesc partDesc, long startInclusive, long endExclusive, Map<String, String> tableAlias) { - - String partitionColumnName = partDesc.getPartitionDateColumn(); - String partitionTableName; + public String buildDateRangeCondition(PartitionDesc partDesc, long startInclusive, long endExclusive) { - // convert to use table alias - int indexOfDot = partitionColumnName.lastIndexOf("."); - if (indexOfDot > 0) { - partitionTableName = partitionColumnName.substring(0, indexOfDot).toUpperCase(); - } else { - throw new IllegalStateException("The partitionColumnName is invalid: " + partitionColumnName); - } - - if (tableAlias.containsKey(partitionTableName)) { - partitionTableName = tableAlias.get(partitionTableName); - } + TblColRef partitionColumn = partDesc.getPartitionDateColumnRef(); + String tableAlias = partitionColumn.getTableAlias(); - String concatField = String.format("CONCAT(%s.YEAR,'-',%s.MONTH,'-',%s.DAY)", partitionTableName, partitionTableName, partitionTableName); + String concatField = String.format("CONCAT(%s.YEAR,'-',%s.MONTH,'-',%s.DAY)", tableAlias, tableAlias, tableAlias); StringBuilder builder = new StringBuilder(); if (startInclusive > 0) { http://git-wip-us.apache.org/repos/asf/kylin/blob/d3ac3542/core-metadata/src/main/java/org/apache/kylin/metadata/model/TblColRef.java ---------------------------------------------------------------------- diff --git a/core-metadata/src/main/java/org/apache/kylin/metadata/model/TblColRef.java b/core-metadata/src/main/java/org/apache/kylin/metadata/model/TblColRef.java index f44972c..9b71524 100644 --- a/core-metadata/src/main/java/org/apache/kylin/metadata/model/TblColRef.java +++ b/core-metadata/src/main/java/org/apache/kylin/metadata/model/TblColRef.java @@ -123,7 +123,7 @@ public class TblColRef implements Serializable { } public String getTableAlias() { - return table.getAlias(); + return table != null ? table.getAlias() : "UNKNOWN_ALIAS"; } public String getTable() { http://git-wip-us.apache.org/repos/asf/kylin/blob/d3ac3542/core-metadata/src/test/java/org/apache/kylin/metadata/model/DefaultPartitionConditionBuilderTest.java ---------------------------------------------------------------------- diff --git a/core-metadata/src/test/java/org/apache/kylin/metadata/model/DefaultPartitionConditionBuilderTest.java b/core-metadata/src/test/java/org/apache/kylin/metadata/model/DefaultPartitionConditionBuilderTest.java index b188668..4e56a71 100644 --- a/core-metadata/src/test/java/org/apache/kylin/metadata/model/DefaultPartitionConditionBuilderTest.java +++ b/core-metadata/src/test/java/org/apache/kylin/metadata/model/DefaultPartitionConditionBuilderTest.java @@ -16,8 +16,6 @@ */ package org.apache.kylin.metadata.model; -import java.util.HashMap; - import org.apache.kylin.common.util.DateFormat; import org.junit.Assert; import org.junit.Before; @@ -34,44 +32,38 @@ public class DefaultPartitionConditionBuilderTest { @Test public void testDatePartition() { PartitionDesc partitionDesc = new PartitionDesc(); - partitionDesc.setPartitionDateColumn("DEFAULT.TABLE_NAME.DATE_COLUMN"); + TblColRef col = TblColRef.mockup(TableDesc.mockup("DEFAULT.TABLE_NAME"), 1, "DATE_COLUMN", "string"); + partitionDesc.setPartitionDateColumnRef(col); + partitionDesc.setPartitionDateColumn(col.getCanonicalName()); partitionDesc.setPartitionDateFormat("yyyy-MM-dd"); - String condition = partitionConditionBuilder.buildDateRangeCondition(partitionDesc, DateFormat.stringToMillis("2016-02-22"), DateFormat.stringToMillis("2016-02-23"), new HashMap<String, String>()); - Assert.assertEquals("DEFAULT.TABLE_NAME.DATE_COLUMN >= '2016-02-22' AND DEFAULT.TABLE_NAME.DATE_COLUMN < '2016-02-23'", condition); + String condition = partitionConditionBuilder.buildDateRangeCondition(partitionDesc, DateFormat.stringToMillis("2016-02-22"), DateFormat.stringToMillis("2016-02-23")); + Assert.assertEquals("UNKNOWN_ALIAS.DATE_COLUMN >= '2016-02-22' AND UNKNOWN_ALIAS.DATE_COLUMN < '2016-02-23'", condition); } @Test public void testTimePartition() { PartitionDesc partitionDesc = new PartitionDesc(); - partitionDesc.setPartitionTimeColumn("DEFAULT.TABLE_NAME.HOUR_COLUMN"); + TblColRef col = TblColRef.mockup(TableDesc.mockup("DEFAULT.TABLE_NAME"), 2, "HOUR_COLUMN", "string"); + partitionDesc.setPartitionTimeColumnRef(col); + partitionDesc.setPartitionTimeColumn(col.getCanonicalName()); partitionDesc.setPartitionTimeFormat("HH"); - String condition = partitionConditionBuilder.buildDateRangeCondition(partitionDesc, DateFormat.stringToMillis("2016-02-22 00:00:00"), DateFormat.stringToMillis("2016-02-23 01:00:00"), new HashMap<String, String>()); - Assert.assertEquals("DEFAULT.TABLE_NAME.HOUR_COLUMN >= '00' AND DEFAULT.TABLE_NAME.HOUR_COLUMN < '01'", condition); + String condition = partitionConditionBuilder.buildDateRangeCondition(partitionDesc, DateFormat.stringToMillis("2016-02-22 00:00:00"), DateFormat.stringToMillis("2016-02-23 01:00:00")); + Assert.assertEquals("UNKNOWN_ALIAS.HOUR_COLUMN >= '00' AND UNKNOWN_ALIAS.HOUR_COLUMN < '01'", condition); } @Test public void testDateAndTimePartition() { PartitionDesc partitionDesc = new PartitionDesc(); - partitionDesc.setPartitionDateColumn("DEFAULT.TABLE_NAME.DATE_COLUMN"); + TblColRef col1 = TblColRef.mockup(TableDesc.mockup("DEFAULT.TABLE_NAME"), 1, "DATE_COLUMN", "string"); + partitionDesc.setPartitionDateColumnRef(col1); + partitionDesc.setPartitionDateColumn(col1.getCanonicalName()); partitionDesc.setPartitionDateFormat("yyyy-MM-dd"); - partitionDesc.setPartitionTimeColumn("DEFAULT.TABLE_NAME.HOUR_COLUMN"); + TblColRef col2 = TblColRef.mockup(TableDesc.mockup("DEFAULT.TABLE_NAME"), 2, "HOUR_COLUMN", "string"); + partitionDesc.setPartitionTimeColumnRef(col2); + partitionDesc.setPartitionTimeColumn(col2.getCanonicalName()); partitionDesc.setPartitionTimeFormat("H"); - String condition = partitionConditionBuilder.buildDateRangeCondition(partitionDesc, DateFormat.stringToMillis("2016-02-22 00:00:00"), DateFormat.stringToMillis("2016-02-23 01:00:00"), new HashMap<String, String>()); - Assert.assertEquals("((DEFAULT.TABLE_NAME.DATE_COLUMN = '2016-02-22' AND DEFAULT.TABLE_NAME.HOUR_COLUMN >= '0') OR (DEFAULT.TABLE_NAME.DATE_COLUMN > '2016-02-22')) AND ((DEFAULT.TABLE_NAME.DATE_COLUMN = '2016-02-23' AND DEFAULT.TABLE_NAME.HOUR_COLUMN < '1') OR (DEFAULT.TABLE_NAME.DATE_COLUMN < '2016-02-23'))", condition); + String condition = partitionConditionBuilder.buildDateRangeCondition(partitionDesc, DateFormat.stringToMillis("2016-02-22 00:00:00"), DateFormat.stringToMillis("2016-02-23 01:00:00")); + Assert.assertEquals("((UNKNOWN_ALIAS.DATE_COLUMN = '2016-02-22' AND UNKNOWN_ALIAS.HOUR_COLUMN >= '0') OR (UNKNOWN_ALIAS.DATE_COLUMN > '2016-02-22')) AND ((UNKNOWN_ALIAS.DATE_COLUMN = '2016-02-23' AND UNKNOWN_ALIAS.HOUR_COLUMN < '1') OR (UNKNOWN_ALIAS.DATE_COLUMN < '2016-02-23'))", condition); } - @Test - public void testDateAndTimePartitionWithAlias() { - PartitionDesc partitionDesc = new PartitionDesc(); - partitionDesc.setPartitionDateColumn("TABLE_ALIAS.DATE_COLUMN"); - partitionDesc.setPartitionDateFormat("yyyy-MM-dd"); - partitionDesc.setPartitionTimeColumn("TABLE_ALIAS.HOUR_COLUMN"); - partitionDesc.setPartitionTimeFormat("H"); - String condition = partitionConditionBuilder.buildDateRangeCondition(partitionDesc, DateFormat.stringToMillis("2016-02-22 00:00:00"), DateFormat.stringToMillis("2016-02-23 01:00:00"), new HashMap<String, String>() { - { - put("TABLE_ALIAS", "DEFAULT.TABLE_NAME"); - } - }); - Assert.assertEquals("((DEFAULT.TABLE_NAME.DATE_COLUMN = '2016-02-22' AND DEFAULT.TABLE_NAME.HOUR_COLUMN >= '0') OR (DEFAULT.TABLE_NAME.DATE_COLUMN > '2016-02-22')) AND ((DEFAULT.TABLE_NAME.DATE_COLUMN = '2016-02-23' AND DEFAULT.TABLE_NAME.HOUR_COLUMN < '1') OR (DEFAULT.TABLE_NAME.DATE_COLUMN < '2016-02-23'))", condition); - } } \ No newline at end of file http://git-wip-us.apache.org/repos/asf/kylin/blob/d3ac3542/query/src/main/java/org/apache/kylin/query/relnode/OLAPJoinRel.java ---------------------------------------------------------------------- diff --git a/query/src/main/java/org/apache/kylin/query/relnode/OLAPJoinRel.java b/query/src/main/java/org/apache/kylin/query/relnode/OLAPJoinRel.java index be7d263..e75d747 100644 --- a/query/src/main/java/org/apache/kylin/query/relnode/OLAPJoinRel.java +++ b/query/src/main/java/org/apache/kylin/query/relnode/OLAPJoinRel.java @@ -121,7 +121,7 @@ public class OLAPJoinRel extends EnumerableJoin implements OLAPRel { boolean rightHasSubquery = false; // as we keep the first table as fact table, we need to visit from left to right - implementor.fixSharedOlapTableScan(this); + implementor.fixSharedOlapTableScanOnTheLeft(this); implementor.visitChild(this.left, this); if (this.context != implementor.getContext() || ((OLAPRel) this.left).hasSubQuery()) { this.hasSubQuery = true; @@ -131,6 +131,7 @@ public class OLAPJoinRel extends EnumerableJoin implements OLAPRel { implementor.freeContext(); } } + implementor.fixSharedOlapTableScanOnTheRight(this); implementor.visitChild(this.right, this); if (this.context != implementor.getContext() || ((OLAPRel) this.right).hasSubQuery()) { this.hasSubQuery = true; http://git-wip-us.apache.org/repos/asf/kylin/blob/d3ac3542/query/src/main/java/org/apache/kylin/query/relnode/OLAPRel.java ---------------------------------------------------------------------- diff --git a/query/src/main/java/org/apache/kylin/query/relnode/OLAPRel.java b/query/src/main/java/org/apache/kylin/query/relnode/OLAPRel.java index a24b447..941ef6a 100644 --- a/query/src/main/java/org/apache/kylin/query/relnode/OLAPRel.java +++ b/query/src/main/java/org/apache/kylin/query/relnode/OLAPRel.java @@ -104,23 +104,30 @@ public interface OLAPRel extends RelNode { OLAPContext.registerContext(context); } - public void fixSharedOlapTableScan(OLAPRel parent) { - if (parent instanceof SingleRel) { - SingleRel single = (SingleRel) parent; - OLAPTableScan copy = copyTableScanIfNeeded(single.getInput()); - if (copy != null) - single.replaceInput(0, copy); - } else if (parent instanceof BiRel) { - BiRel bi = (BiRel) parent; - OLAPTableScan copyLeft = copyTableScanIfNeeded(bi.getLeft()); - if (copyLeft != null) - bi.replaceInput(0, copyLeft); - OLAPTableScan copyRight = copyTableScanIfNeeded(bi.getRight()); - if (copyRight != null) - bi.replaceInput(1, copyRight); - } + public void fixSharedOlapTableScan(SingleRel parent) { + OLAPTableScan copy = copyTableScanIfNeeded(parent.getInput()); + if (copy != null) + parent.replaceInput(0, copy); } + public void fixSharedOlapTableScanOnTheLeft(BiRel parent) { + OLAPTableScan copy = copyTableScanIfNeeded(parent.getLeft()); + if (copy != null) + parent.replaceInput(0, copy); + } + + public void fixSharedOlapTableScanOnTheRight(BiRel parent) { + OLAPTableScan copy = copyTableScanIfNeeded(parent.getRight()); + if (copy != null) + parent.replaceInput(1, copy); + } + + public void fixSharedOlapTableScanAt(RelNode parent, int ordinalInParent) { + OLAPTableScan copy = copyTableScanIfNeeded(parent.getInputs().get(ordinalInParent)); + if (copy != null) + parent.replaceInput(ordinalInParent, copy); + } + private OLAPTableScan copyTableScanIfNeeded(RelNode input) { if (input instanceof OLAPTableScan) { OLAPTableScan tableScan = (OLAPTableScan) input; http://git-wip-us.apache.org/repos/asf/kylin/blob/d3ac3542/query/src/main/java/org/apache/kylin/query/relnode/OLAPUnionRel.java ---------------------------------------------------------------------- diff --git a/query/src/main/java/org/apache/kylin/query/relnode/OLAPUnionRel.java b/query/src/main/java/org/apache/kylin/query/relnode/OLAPUnionRel.java index adcf27f..e04ba6f 100644 --- a/query/src/main/java/org/apache/kylin/query/relnode/OLAPUnionRel.java +++ b/query/src/main/java/org/apache/kylin/query/relnode/OLAPUnionRel.java @@ -71,9 +71,9 @@ public class OLAPUnionRel extends Union implements OLAPRel { @Override public void implementOLAP(OLAPImplementor implementor) { - implementor.fixSharedOlapTableScan(this); - for (RelNode child : getInputs()) { - implementor.visitChild(child, this); + for (int i = 0, n = getInputs().size(); i < n; i++) { + implementor.fixSharedOlapTableScanAt(this, i); + implementor.visitChild(getInputs().get(i), this); } this.columnRowType = buildColumnRowType(); http://git-wip-us.apache.org/repos/asf/kylin/blob/d3ac3542/server-base/src/main/java/org/apache/kylin/rest/controller/CubeController.java ---------------------------------------------------------------------- diff --git a/server-base/src/main/java/org/apache/kylin/rest/controller/CubeController.java b/server-base/src/main/java/org/apache/kylin/rest/controller/CubeController.java index 10cd1f2..79739c2 100644 --- a/server-base/src/main/java/org/apache/kylin/rest/controller/CubeController.java +++ b/server-base/src/main/java/org/apache/kylin/rest/controller/CubeController.java @@ -27,7 +27,6 @@ import java.util.Map; import java.util.Set; import java.util.UUID; -import com.google.common.collect.Lists; import org.apache.commons.lang.StringUtils; import org.apache.kylin.common.KylinConfig; import org.apache.kylin.common.util.JsonUtil; @@ -42,7 +41,6 @@ import org.apache.kylin.job.JobInstance; import org.apache.kylin.job.JoinedFlatTable; import org.apache.kylin.metadata.model.IJoinedFlatTableDesc; import org.apache.kylin.metadata.model.ISourceAware; -import org.apache.kylin.metadata.model.SegmentStatusEnum; import org.apache.kylin.metadata.project.ProjectInstance; import org.apache.kylin.metadata.realization.RealizationStatusEnum; import org.apache.kylin.rest.exception.BadRequestException; @@ -75,6 +73,7 @@ import com.fasterxml.jackson.core.JsonParseException; import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.databind.JsonMappingException; import com.google.common.base.Joiner; +import com.google.common.collect.Lists; import com.google.common.collect.Sets; /** @@ -145,8 +144,7 @@ public class CubeController extends BasicController { @ResponseBody public GeneralResponse getSql(@PathVariable String cubeName, @PathVariable String segmentName) { CubeInstance cube = cubeService.getCubeManager().getCube(cubeName); - CubeSegment cubeSegment = cube.getSegment(segmentName, SegmentStatusEnum.READY); - IJoinedFlatTableDesc flatTableDesc = EngineFactory.getJoinedFlatTableDesc(cubeSegment); + IJoinedFlatTableDesc flatTableDesc = EngineFactory.getJoinedFlatTableDesc(cube.getDescriptor()); String sql = JoinedFlatTable.generateSelectDataStatement(flatTableDesc, false); GeneralResponse repsonse = new GeneralResponse();
