KYLIN-2380 refactor DbUnit assertion
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/2e088159 Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/2e088159 Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/2e088159 Branch: refs/heads/yang22-cdh5.7 Commit: 2e0881596e251c7e517748c1468d459ec86210d3 Parents: 3a3ee3c Author: Li Yang <liy...@apache.org> Authored: Wed Jan 11 12:10:36 2017 +0800 Committer: Li Yang <liy...@apache.org> Committed: Wed Jan 11 12:10:36 2017 +0800 ---------------------------------------------------------------------- .../apache/kylin/query/HackedDbUnitAssert.java | 125 ++++++++++++++++--- .../apache/kylin/query/ITMassInQueryTest.java | 9 +- .../org/apache/kylin/query/KylinTestBase.java | 60 +++++---- 3 files changed, 148 insertions(+), 46 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/kylin/blob/2e088159/kylin-it/src/test/java/org/apache/kylin/query/HackedDbUnitAssert.java ---------------------------------------------------------------------- diff --git a/kylin-it/src/test/java/org/apache/kylin/query/HackedDbUnitAssert.java b/kylin-it/src/test/java/org/apache/kylin/query/HackedDbUnitAssert.java index 338f698..3a21570 100644 --- a/kylin-it/src/test/java/org/apache/kylin/query/HackedDbUnitAssert.java +++ b/kylin-it/src/test/java/org/apache/kylin/query/HackedDbUnitAssert.java @@ -26,16 +26,34 @@ import org.dbunit.dataset.Columns; import org.dbunit.dataset.DataSetException; import org.dbunit.dataset.ITable; import org.dbunit.dataset.ITableMetaData; +import org.dbunit.dataset.datatype.BigIntegerDataType; import org.dbunit.dataset.datatype.DataType; +import org.dbunit.dataset.datatype.IntegerDataType; +import org.dbunit.dataset.datatype.UnknownDataType; import org.slf4j.Logger; import org.slf4j.LoggerFactory; /** - * dirty hack to support checking result of SQL with limit + * A copy of DbUnitAssert, in which a few hacks applied + * + * - tolerate some column type difference, like INT vs. BIGINT + * - check expected table contains actual table (instead of equals), for sql with limit */ public class HackedDbUnitAssert extends DbUnitAssert { private static final Logger logger = LoggerFactory.getLogger(HackedDbUnitAssert.class); + private boolean hackCheckContains; + private boolean hackIgnoreIntBigIntMismatch; + + public void hackCheckContains() { + hackCheckContains = true; + } + + public void hackIgnoreIntBigIntMismatch() { + hackIgnoreIntBigIntMismatch = true; + } + + // THIS METHOD IS MOSTLY COPIED FROM DbUnitAssert. CHANGES ARE LEAD BY hackXXX CONDITION CHECKS. public void assertEquals(ITable expectedTable, ITable actualTable, FailureHandler failureHandler) throws DatabaseUnitException { logger.trace("assertEquals(expectedTable, actualTable, failureHandler) - start"); logger.debug("assertEquals: expectedTable={}", expectedTable); @@ -57,22 +75,21 @@ public class HackedDbUnitAssert extends DbUnitAssert { ITableMetaData actualMetaData = actualTable.getTableMetaData(); String expectedTableName = expectedMetaData.getTableName(); - // // Verify row count - // int expectedRowsCount = expectedTable.getRowCount(); - // int actualRowsCount = actualTable.getRowCount(); - // if (expectedRowsCount != actualRowsCount) { - // String msg = "row count (table=" + expectedTableName + ")"; - // Error error = - // failureHandler.createFailure(msg, String - // .valueOf(expectedRowsCount), String - // .valueOf(actualRowsCount)); - // logger.error(error.toString()); - // throw error; - // } + // Verify row count + int expectedRowsCount = expectedTable.getRowCount(); + int actualRowsCount = actualTable.getRowCount(); + if (!hackCheckContains) { + if (expectedRowsCount != actualRowsCount) { + String msg = "row count (table=" + expectedTableName + ")"; + Error error = failureHandler.createFailure(msg, String.valueOf(expectedRowsCount), String.valueOf(actualRowsCount)); + logger.error(error.toString()); + throw error; + } + } // if both tables are empty, it is not necessary to compare columns, as such comparison // can fail if column metadata is different (which could occurs when comparing empty tables) - if (expectedTable.getRowCount() == 0 && actualTable.getRowCount() == 0) { + if (expectedRowsCount == 0 && actualRowsCount == 0) { logger.debug("Tables are empty, hence equals."); return; } @@ -94,10 +111,86 @@ public class HackedDbUnitAssert extends DbUnitAssert { ComparisonColumn[] comparisonCols = getComparisonColumns(expectedTableName, expectedColumns, actualColumns, failureHandler); // Finally compare the data - compareData(expectedTable, actualTable, comparisonCols, failureHandler); + if (hackCheckContains) + compareDataContains(expectedTable, actualTable, comparisonCols, failureHandler); + else + compareData(expectedTable, actualTable, comparisonCols, failureHandler); + } + + // THIS METHOD IS COPIED FROM SUPER CLASS TO CHANGE ComparisonColumn TO OUR OWN. + @Override + protected ComparisonColumn[] getComparisonColumns(String expectedTableName, Column[] expectedColumns, Column[] actualColumns, FailureHandler failureHandler) { + ComparisonColumn[] result = new ComparisonColumn[expectedColumns.length]; + + for (int j = 0; j < expectedColumns.length; j++) { + Column expectedColumn = expectedColumns[j]; + Column actualColumn = actualColumns[j]; + result[j] = new HackedComparisonColumn(expectedTableName, expectedColumn, actualColumn, failureHandler); + } + return result; + } + + // MOSTLY COPIED FROM SUPER CLASS + private class HackedComparisonColumn extends ComparisonColumn { + private String columnName; + private DataType dataType; + + public HackedComparisonColumn(String tableName, Column expectedColumn, Column actualColumn, FailureHandler failureHandler) { + + // super class is actually useless, all public methods are overridden below + super(tableName, expectedColumn, expectedColumn, failureHandler); + + this.columnName = expectedColumn.getColumnName(); + this.dataType = getComparisonDataType(tableName, expectedColumn, actualColumn, failureHandler); + } + + @Override + public String getColumnName() { + return this.columnName; + } + + @Override + public DataType getDataType() { + return this.dataType; + } + + // COPIED FROM SUPER CLASS, CHANGES ARE LEAD BY hackXXX CONDITION CHECKS. + private DataType getComparisonDataType(String tableName, Column expectedColumn, Column actualColumn, FailureHandler failureHandler) { + if (logger.isDebugEnabled()) + logger.debug("getComparisonDataType(tableName={}, expectedColumn={}, actualColumn={}, failureHandler={}) - start", new Object[] { tableName, expectedColumn, actualColumn, failureHandler }); + + DataType expectedDataType = expectedColumn.getDataType(); + DataType actualDataType = actualColumn.getDataType(); + + // The two columns have different data type + if (!expectedDataType.getClass().isInstance(actualDataType)) { + // Expected column data type is unknown, use actual column data type + if (expectedDataType instanceof UnknownDataType) { + return actualDataType; + } + + // Actual column data type is unknown, use expected column data type + if (actualDataType instanceof UnknownDataType) { + return expectedDataType; + } + + if (hackIgnoreIntBigIntMismatch) { + if (expectedDataType instanceof IntegerDataType && actualDataType instanceof BigIntegerDataType) + return actualDataType; + } + + // Impossible to determine which data type to use + String msg = "Incompatible data types: (table=" + tableName + ", col=" + expectedColumn.getColumnName() + ")"; + throw failureHandler.createFailure(msg, String.valueOf(expectedDataType), String.valueOf(actualDataType)); + } + + // Both columns have same data type, return any one of them + return expectedDataType; + } + } - protected void compareData(ITable expectedTable, ITable actualTable, ComparisonColumn[] comparisonCols, FailureHandler failureHandler) throws DataSetException { + private void compareDataContains(ITable expectedTable, ITable actualTable, ComparisonColumn[] comparisonCols, FailureHandler failureHandler) throws DataSetException { logger.debug("compareData(expectedTable={}, actualTable={}, " + "comparisonCols={}, failureHandler={}) - start", new Object[] { expectedTable, actualTable, comparisonCols, failureHandler }); if (expectedTable == null) { http://git-wip-us.apache.org/repos/asf/kylin/blob/2e088159/kylin-it/src/test/java/org/apache/kylin/query/ITMassInQueryTest.java ---------------------------------------------------------------------- diff --git a/kylin-it/src/test/java/org/apache/kylin/query/ITMassInQueryTest.java b/kylin-it/src/test/java/org/apache/kylin/query/ITMassInQueryTest.java index fd3abfb..28cdb67 100644 --- a/kylin-it/src/test/java/org/apache/kylin/query/ITMassInQueryTest.java +++ b/kylin-it/src/test/java/org/apache/kylin/query/ITMassInQueryTest.java @@ -30,12 +30,9 @@ import org.apache.hadoop.fs.FSDataOutputStream; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; import org.apache.kylin.engine.mr.HadoopUtil; -import org.dbunit.Assertion; -import org.dbunit.database.DatabaseConfig; import org.dbunit.database.DatabaseConnection; import org.dbunit.database.IDatabaseConnection; import org.dbunit.dataset.ITable; -import org.dbunit.ext.h2.H2Connection; import org.junit.After; import org.junit.AfterClass; import org.junit.Before; @@ -144,13 +141,11 @@ public class ITMassInQueryTest extends KylinTestBase { sql = sql.replace("massin(test_kylin_fact.SELLER_ID,'vip_customers')", "test_kylin_fact.SELLER_ID in ( " + org.apache.commons.lang.StringUtils.join(vipSellers, ",") + ")"); printInfo("Query Result from H2 - " + queryName); printInfo("Query for H2 - " + sql); - H2Connection h2Conn = new H2Connection(h2Connection, null); - h2Conn.getConfig().setProperty(DatabaseConfig.PROPERTY_DATATYPE_FACTORY, new TestH2DataTypeFactory()); - ITable h2Table = executeQuery(h2Conn, queryName, sql, needSort); + ITable h2Table = executeQuery(newH2Connection(), queryName, sql, needSort); try { // compare the result - Assertion.assertEquals(h2Table, kylinTable); + assertTableEquals(h2Table, kylinTable); } catch (Throwable t) { printInfo("execAndCompQuery failed on: " + sqlFile.getAbsolutePath()); throw t; http://git-wip-us.apache.org/repos/asf/kylin/blob/2e088159/kylin-it/src/test/java/org/apache/kylin/query/KylinTestBase.java ---------------------------------------------------------------------- diff --git a/kylin-it/src/test/java/org/apache/kylin/query/KylinTestBase.java b/kylin-it/src/test/java/org/apache/kylin/query/KylinTestBase.java index b8a48ef..a948dac 100644 --- a/kylin-it/src/test/java/org/apache/kylin/query/KylinTestBase.java +++ b/kylin-it/src/test/java/org/apache/kylin/query/KylinTestBase.java @@ -53,7 +53,7 @@ import org.apache.kylin.query.relnode.OLAPContext; import org.apache.kylin.query.routing.rules.RemoveBlackoutRealizationsRule; import org.apache.kylin.query.schema.OLAPSchemaFactory; import org.apache.kylin.storage.hbase.cube.v1.coprocessor.observer.ObserverEnabler; -import org.dbunit.Assertion; +import org.dbunit.DatabaseUnitException; import org.dbunit.database.DatabaseConfig; import org.dbunit.database.DatabaseConnection; import org.dbunit.database.IDatabaseConnection; @@ -231,11 +231,13 @@ public class KylinTestBase { sql = changeJoinType(sql, joinType); ITable queryTable = dbConn.createQueryTable(resultTableName + queryName, sql); - String[] columnNames = new String[queryTable.getTableMetaData().getColumns().length]; - for (int i = 0; i < columnNames.length; i++) { - columnNames[i] = queryTable.getTableMetaData().getColumns()[i].getColumnName(); - } if (needSort) { + String[] columnNames = new String[queryTable.getTableMetaData().getColumns().length]; + for (int i = 0; i < columnNames.length; i++) { + columnNames[i] = queryTable.getTableMetaData().getColumns()[i].getColumnName(); + } + Arrays.sort(columnNames); + queryTable = new SortedTable(queryTable, columnNames); } if (PRINT_RESULT) @@ -359,7 +361,7 @@ public class KylinTestBase { // compare the result Assert.assertEquals(queryName, expectRowCount, kylinTable.getRowCount()); - // Assertion.assertEquals(expectRowCount, kylinTable.getRowCount()); + // assertTableEquals(expectRowCount, kylinTable.getRowCount()); } } @@ -382,7 +384,7 @@ public class KylinTestBase { ITable kylinTable = executeQuery(kylinConn, queryName, sql, false); // compare the result - Assertion.assertEquals(expectTable, kylinTable); + assertTableEquals(expectTable, kylinTable); } } @@ -405,9 +407,7 @@ public class KylinTestBase { // execute H2 printInfo("Query Result from H2 - " + queryName); - H2Connection h2Conn = new H2Connection(h2Connection, null); - h2Conn.getConfig().setProperty(DatabaseConfig.PROPERTY_DATATYPE_FACTORY, new TestH2DataTypeFactory()); - ITable h2Table = executeQuery(h2Conn, queryName, sql, needSort); + ITable h2Table = executeQuery(newH2Connection(), queryName, sql, needSort); try { // compare the result @@ -468,13 +468,10 @@ public class KylinTestBase { // execute H2 printInfo("Query Result from H2 - " + queryName); - H2Connection h2Conn = new H2Connection(h2Connection, null); - h2Conn.getConfig().setProperty(DatabaseConfig.PROPERTY_DATATYPE_FACTORY, new TestH2DataTypeFactory()); - ITable h2Table = executeQuery(h2Conn, queryName, sql, false); + ITable h2Table = executeQuery(newH2Connection(), queryName, sql, false); try { - HackedDbUnitAssert hackedDbUnitAssert = new HackedDbUnitAssert(); - hackedDbUnitAssert.assertEquals(h2Table, kylinTable); + assertTableContains(h2Table, kylinTable); } catch (Throwable t) { printInfo("execAndCompQuery failed on: " + sqlFile.getAbsolutePath()); throw t; @@ -507,13 +504,11 @@ public class KylinTestBase { // execute H2 printInfo("Query Result from H2 - " + queryName); - H2Connection h2Conn = new H2Connection(h2Connection, null); - h2Conn.getConfig().setProperty(DatabaseConfig.PROPERTY_DATATYPE_FACTORY, new TestH2DataTypeFactory()); - ITable h2Table = executeQuery(h2Conn, queryName, sql, needSort); + ITable h2Table = executeQuery(newH2Connection(), queryName, sql, needSort); try { // compare the result - Assertion.assertEquals(h2Table, kylinTable); + assertTableEquals(h2Table, kylinTable); } catch (Throwable t) { printInfo("execAndCompQuery failed on: " + sqlFile.getAbsolutePath()); throw t; @@ -546,14 +541,33 @@ public class KylinTestBase { // execute H2 printInfo("Query Result from H2 - " + queryName); - IDatabaseConnection h2Conn = new DatabaseConnection(h2Connection); - h2Conn.getConfig().setProperty(DatabaseConfig.PROPERTY_DATATYPE_FACTORY, new TestH2DataTypeFactory()); - ITable h2Table = executeDynamicQuery(h2Conn, queryName, sql, parameters, needSort); + ITable h2Table = executeDynamicQuery(newH2Connection(), queryName, sql, parameters, needSort); // compare the result - Assertion.assertEquals(h2Table, kylinTable); + assertTableEquals(h2Table, kylinTable); } } + + protected void assertTableEquals(ITable h2Table, ITable kylinTable) throws DatabaseUnitException { + HackedDbUnitAssert dbUnit = new HackedDbUnitAssert(); + dbUnit.hackIgnoreIntBigIntMismatch(); + dbUnit.assertEquals(h2Table, kylinTable); + } + + protected void assertTableContains(ITable h2Table, ITable kylinTable) throws DatabaseUnitException { + HackedDbUnitAssert dbUnit = new HackedDbUnitAssert(); + dbUnit.hackIgnoreIntBigIntMismatch(); + dbUnit.hackCheckContains(); + dbUnit.assertEquals(h2Table, kylinTable); + } + + @SuppressWarnings("deprecation") + protected static H2Connection newH2Connection() throws DatabaseUnitException { + H2Connection h2Conn = new H2Connection(h2Connection, null); + h2Conn.getConfig().setProperty(DatabaseConfig.PROPERTY_DATATYPE_FACTORY, new TestH2DataTypeFactory()); + h2Conn.getConfig().setFeature(DatabaseConfig.FEATURE_DATATYPE_WARNING, false); + return h2Conn; + } protected int runSqlFile(String file) throws Exception { return runSQL(new File(file), true, false);
