Repository: kylin Updated Branches: refs/heads/master 634aeaa94 -> 1d4f57af7
KYLIN-1971 bug fix, model matching must match joining tables Project: http://git-wip-us.apache.org/repos/asf/kylin/repo Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/1d4f57af Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/1d4f57af Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/1d4f57af Branch: refs/heads/master Commit: 1d4f57af7cf0ab89a1e1acd10bb4d1aaec1012cf Parents: 634aeaa Author: Yang Li <liy...@apache.org> Authored: Sat Nov 19 21:58:21 2016 +0800 Committer: Yang Li <liy...@apache.org> Committed: Sun Nov 20 07:48:48 2016 +0800 ---------------------------------------------------------------------- .../kylin/dict/NumberDictionaryBuilder.java | 3 + .../apache/kylin/metadata/model/ColumnDesc.java | 6 ++ .../kylin/metadata/model/DataModelDesc.java | 2 + .../apache/kylin/metadata/model/JoinDesc.java | 63 ++++++++++++------ .../apache/kylin/query/relnode/OLAPJoinRel.java | 1 + .../kylin/query/relnode/OLAPTableScan.java | 1 + .../v1/coprocessor/observer/RowTypeTest.java | 69 -------------------- 7 files changed, 56 insertions(+), 89 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/kylin/blob/1d4f57af/core-dictionary/src/main/java/org/apache/kylin/dict/NumberDictionaryBuilder.java ---------------------------------------------------------------------- diff --git a/core-dictionary/src/main/java/org/apache/kylin/dict/NumberDictionaryBuilder.java b/core-dictionary/src/main/java/org/apache/kylin/dict/NumberDictionaryBuilder.java index 0655ce1..68a05d4 100644 --- a/core-dictionary/src/main/java/org/apache/kylin/dict/NumberDictionaryBuilder.java +++ b/core-dictionary/src/main/java/org/apache/kylin/dict/NumberDictionaryBuilder.java @@ -21,8 +21,11 @@ package org.apache.kylin.dict; import org.apache.kylin.common.util.Bytes; /** + * Use <code>NumberDictionaryForestBuilder</code> instead. + * * @author yangli9 */ +@Deprecated public class NumberDictionaryBuilder<T> extends TrieDictionaryBuilder<T> { NumberDictionary.NumberBytesCodec codec = new NumberDictionary.NumberBytesCodec(NumberDictionary.MAX_DIGITS_BEFORE_DECIMAL_POINT); http://git-wip-us.apache.org/repos/asf/kylin/blob/1d4f57af/core-metadata/src/main/java/org/apache/kylin/metadata/model/ColumnDesc.java ---------------------------------------------------------------------- diff --git a/core-metadata/src/main/java/org/apache/kylin/metadata/model/ColumnDesc.java b/core-metadata/src/main/java/org/apache/kylin/metadata/model/ColumnDesc.java index e0184b4..7d9133d 100644 --- a/core-metadata/src/main/java/org/apache/kylin/metadata/model/ColumnDesc.java +++ b/core-metadata/src/main/java/org/apache/kylin/metadata/model/ColumnDesc.java @@ -191,6 +191,12 @@ public class ColumnDesc implements Serializable { } else if (!name.equals(other.name)) return false; + if (table == null) { + if (other.table != null) + return false; + } else if (!table.equals(other.table)) + return false; + if (datatype == null) { if (other.datatype != null) return false; http://git-wip-us.apache.org/repos/asf/kylin/blob/1d4f57af/core-metadata/src/main/java/org/apache/kylin/metadata/model/DataModelDesc.java ---------------------------------------------------------------------- diff --git a/core-metadata/src/main/java/org/apache/kylin/metadata/model/DataModelDesc.java b/core-metadata/src/main/java/org/apache/kylin/metadata/model/DataModelDesc.java index 7c39a25..139fcbe 100644 --- a/core-metadata/src/main/java/org/apache/kylin/metadata/model/DataModelDesc.java +++ b/core-metadata/src/main/java/org/apache/kylin/metadata/model/DataModelDesc.java @@ -369,6 +369,8 @@ public class DataModelDesc extends RootPersistentEntity { fkCols[i] = col; } join.setForeignKeyColumns(fkCols); + + join.sortByFK(); // Validate join in dimension if (pkCols.length != fkCols.length) { http://git-wip-us.apache.org/repos/asf/kylin/blob/1d4f57af/core-metadata/src/main/java/org/apache/kylin/metadata/model/JoinDesc.java ---------------------------------------------------------------------- diff --git a/core-metadata/src/main/java/org/apache/kylin/metadata/model/JoinDesc.java b/core-metadata/src/main/java/org/apache/kylin/metadata/model/JoinDesc.java index 5beda0a..1dd8725 100644 --- a/core-metadata/src/main/java/org/apache/kylin/metadata/model/JoinDesc.java +++ b/core-metadata/src/main/java/org/apache/kylin/metadata/model/JoinDesc.java @@ -23,6 +23,7 @@ import java.util.Arrays; import com.fasterxml.jackson.annotation.JsonAutoDetect; import com.fasterxml.jackson.annotation.JsonAutoDetect.Visibility; import com.fasterxml.jackson.annotation.JsonProperty; +import com.google.common.base.Preconditions; /** */ @@ -98,6 +99,37 @@ public class JoinDesc { this.foreignKeyColumns = foreignKeyColumns; } + public void sortByFK() { + Preconditions.checkState(primaryKey.length == foreignKey.length && primaryKey.length == primaryKeyColumns.length && foreignKey.length == foreignKeyColumns.length); + boolean cont = true; + int n = foreignKey.length; + for (int i = 0; i < n - 1 && cont; i++) { + cont = false; + for (int j = i; j < n - 1; j++) { + int jj = j + 1; + if (foreignKey[j].compareTo(foreignKey[jj]) > 0) { + swap(foreignKey, j, jj); + swap(primaryKey, j, jj); + swap(foreignKeyColumns, j, jj); + swap(primaryKeyColumns, j, jj); + cont = true; + } + } + } + } + + private void swap(String[] arr, int j, int jj) { + String tmp = arr[j]; + arr[j] = arr[jj]; + arr[jj] = tmp; + } + + private void swap(TblColRef[] arr, int j, int jj) { + TblColRef tmp = arr[j]; + arr[j] = arr[jj]; + arr[jj] = tmp; + } + @Override public int hashCode() { final int prime = 31; @@ -118,9 +150,10 @@ public class JoinDesc { return false; JoinDesc other = (JoinDesc) obj; - if (!this.colRefsEqualIgnoringOrder(foreignKeyColumns, other.foreignKeyColumns)) + // note pk/fk are sorted, sortByFK() + if (!Arrays.equals(foreignKeyColumns, other.foreignKeyColumns)) return false; - if (!this.colRefsEqualIgnoringOrder(primaryKeyColumns, other.primaryKeyColumns)) + if (!Arrays.equals(primaryKeyColumns, other.primaryKeyColumns)) return false; if (!this.type.equalsIgnoreCase(other.getType())) @@ -128,39 +161,29 @@ public class JoinDesc { return true; } - private boolean colRefsEqualIgnoringOrder(TblColRef[] a, TblColRef[] b) { - if (a.length != b.length) - return false; - - return Arrays.asList(a).containsAll(Arrays.asList(b)); - } - // equals() without alias public boolean matches(JoinDesc other) { if (!this.type.equalsIgnoreCase(other.getType())) return false; - if (!this.columnsEqualIgnoringOrder(foreignKeyColumns, other.foreignKeyColumns)) + + // note pk/fk are sorted, sortByFK() + if (!this.columnDescEquals(foreignKeyColumns, other.foreignKeyColumns)) return false; - if (!this.columnsEqualIgnoringOrder(primaryKeyColumns, other.primaryKeyColumns)) + if (!this.columnDescEquals(primaryKeyColumns, other.primaryKeyColumns)) return false; return true; } - private boolean columnsEqualIgnoringOrder(TblColRef[] a, TblColRef[] b) { + private boolean columnDescEquals(TblColRef[] a, TblColRef[] b) { if (a.length != b.length) return false; - int match = 0; for (int i = 0; i < a.length; i++) { - for (int j = 0; j < b.length; j++) { - if (a[i].getColumnDesc().equals(b[j].getColumnDesc())) { - match++; - break; - } - } + if (a[i].getColumnDesc().equals(b[i].getColumnDesc()) == false) + return false; } - return match == a.length; + return true; } @Override http://git-wip-us.apache.org/repos/asf/kylin/blob/1d4f57af/query/src/main/java/org/apache/kylin/query/relnode/OLAPJoinRel.java ---------------------------------------------------------------------- diff --git a/query/src/main/java/org/apache/kylin/query/relnode/OLAPJoinRel.java b/query/src/main/java/org/apache/kylin/query/relnode/OLAPJoinRel.java index 265d65d..4c0a1dc 100644 --- a/query/src/main/java/org/apache/kylin/query/relnode/OLAPJoinRel.java +++ b/query/src/main/java/org/apache/kylin/query/relnode/OLAPJoinRel.java @@ -220,6 +220,7 @@ public class OLAPJoinRel extends EnumerableJoin implements OLAPRel { join.setForeignKeyColumns(fkCols.toArray(new TblColRef[fkCols.size()])); join.setPrimaryKey(pks.toArray(COLUMN_ARRAY_MARKER)); join.setPrimaryKeyColumns(pkCols.toArray(new TblColRef[pkCols.size()])); + join.sortByFK(); return join; } http://git-wip-us.apache.org/repos/asf/kylin/blob/1d4f57af/query/src/main/java/org/apache/kylin/query/relnode/OLAPTableScan.java ---------------------------------------------------------------------- diff --git a/query/src/main/java/org/apache/kylin/query/relnode/OLAPTableScan.java b/query/src/main/java/org/apache/kylin/query/relnode/OLAPTableScan.java index 8b5ad78..e63a978 100644 --- a/query/src/main/java/org/apache/kylin/query/relnode/OLAPTableScan.java +++ b/query/src/main/java/org/apache/kylin/query/relnode/OLAPTableScan.java @@ -239,6 +239,7 @@ public class OLAPTableScan extends TableScan implements OLAPRel, EnumerableRel { return new ColumnRowType(columns); } + @SuppressWarnings("deprecation") public TblColRef makeRewriteColumn(String name) { TableRef tableRef = columnRowType.getColumnByIndex(0).getTableRef(); return tableRef.makeFakeColumn(name); http://git-wip-us.apache.org/repos/asf/kylin/blob/1d4f57af/storage-hbase/src/test/java/org/apache/kylin/storage/hbase/cube/v1/coprocessor/observer/RowTypeTest.java ---------------------------------------------------------------------- diff --git a/storage-hbase/src/test/java/org/apache/kylin/storage/hbase/cube/v1/coprocessor/observer/RowTypeTest.java b/storage-hbase/src/test/java/org/apache/kylin/storage/hbase/cube/v1/coprocessor/observer/RowTypeTest.java deleted file mode 100644 index 364be9a..0000000 --- a/storage-hbase/src/test/java/org/apache/kylin/storage/hbase/cube/v1/coprocessor/observer/RowTypeTest.java +++ /dev/null @@ -1,69 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. -*/ - -package org.apache.kylin.storage.hbase.cube.v1.coprocessor.observer; - -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertTrue; - -import java.util.Arrays; - -import org.apache.kylin.common.util.LocalFileMetadataTestCase; -import org.apache.kylin.cube.CubeInstance; -import org.apache.kylin.cube.CubeManager; -import org.apache.kylin.cube.cuboid.Cuboid; -import org.apache.kylin.cube.model.CubeDesc; -import org.apache.kylin.storage.hbase.common.coprocessor.CoprocessorRowType; -import org.junit.After; -import org.junit.Before; -import org.junit.Test; - -/** - * @author yangli9 - * - */ -public class RowTypeTest extends LocalFileMetadataTestCase { - - @Before - public void setUp() throws Exception { - this.createTestMetadata(); - } - - @After - public void after() throws Exception { - this.cleanupTestMetadata(); - } - - @Test - public void testSerialize() { - - CubeInstance cube = CubeManager.getInstance(getTestConfig()).getCube("test_kylin_cube_without_slr_ready"); - CubeDesc cubeDesc = cube.getDescriptor(); - long baseCuboidId = Cuboid.getBaseCuboidId(cubeDesc); - Cuboid cuboid = Cuboid.findById(cubeDesc, baseCuboidId); - - CoprocessorRowType rowType = CoprocessorRowType.fromCuboid(cube.getLatestReadySegment(), cuboid); - byte[] bytes = CoprocessorRowType.serialize(rowType); - CoprocessorRowType copy = CoprocessorRowType.deserialize(bytes); - - assertTrue(Arrays.equals(rowType.columnSizes, copy.columnSizes)); - for (int i = 0; i < rowType.columns.length; i++) { - assertEquals(rowType.columns[i].getColumnDesc(), copy.columns[i].getColumnDesc()); - } - } -}
