KYLIN-1351 Add unit tests for JDBC source Signed-off-by: Li Yang <liy...@apache.org>
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/ad47dc00 Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/ad47dc00 Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/ad47dc00 Branch: refs/heads/KYLIN-2606 Commit: ad47dc00e672e8e967d8605d2c8d607ce28225d5 Parents: 926a17a Author: auphyroc99 <454530...@qq.com> Authored: Wed Jul 5 11:03:00 2017 +0800 Committer: Li Yang <liy...@apache.org> Committed: Wed Jul 5 17:02:56 2017 +0800 ---------------------------------------------------------------------- .../apache/kylin/common/KylinConfigBase.java | 1 + .../jdbc/ITJdbcSourceTableLoaderTest.java | 121 +++++++++++++++++++ .../source/jdbc/ITJdbcTableReaderTest.java | 116 ++++++++++++++++++ source-hive/pom.xml | 5 + 4 files changed, 243 insertions(+) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/kylin/blob/ad47dc00/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java ---------------------------------------------------------------------- diff --git a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java index 179d61f..f9c3adb 100644 --- a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java +++ b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java @@ -532,6 +532,7 @@ abstract public class KylinConfigBase implements Serializable { // ref constants in ISourceAware r.put(0, "org.apache.kylin.source.hive.HiveSource"); r.put(1, "org.apache.kylin.source.kafka.KafkaSource"); + r.put(8, "org.apache.kylin.source.jdbc.JdbcSource"); r.putAll(convertKeyToInteger(getPropertiesByPrefix("kylin.source.provider."))); return r; } http://git-wip-us.apache.org/repos/asf/kylin/blob/ad47dc00/kylin-it/src/test/java/org/apache/kylin/source/jdbc/ITJdbcSourceTableLoaderTest.java ---------------------------------------------------------------------- diff --git a/kylin-it/src/test/java/org/apache/kylin/source/jdbc/ITJdbcSourceTableLoaderTest.java b/kylin-it/src/test/java/org/apache/kylin/source/jdbc/ITJdbcSourceTableLoaderTest.java new file mode 100644 index 0000000..a84dca9 --- /dev/null +++ b/kylin-it/src/test/java/org/apache/kylin/source/jdbc/ITJdbcSourceTableLoaderTest.java @@ -0,0 +1,121 @@ +package org.apache.kylin.source.jdbc; + +import static org.junit.Assert.assertTrue; + +import java.sql.Connection; +import java.sql.DriverManager; +import java.sql.SQLException; +import java.sql.Types; + +import org.apache.kylin.common.KylinConfig; +import org.apache.kylin.common.util.LocalFileMetadataTestCase; +import org.apache.kylin.common.util.Pair; +import org.apache.kylin.metadata.MetadataManager; +import org.apache.kylin.metadata.model.ISourceAware; +import org.apache.kylin.metadata.model.TableDesc; +import org.apache.kylin.metadata.model.TableExtDesc; +import org.apache.kylin.query.H2Database; +import org.apache.kylin.source.ISource; +import org.apache.kylin.source.ISourceMetadataExplorer; +import org.apache.kylin.source.SourceFactory; +import org.apache.kylin.source.datagen.ModelDataGenerator; +import org.dbunit.DatabaseUnitException; +import org.dbunit.database.DatabaseConfig; +import org.dbunit.dataset.datatype.DataType; +import org.dbunit.dataset.datatype.DataTypeException; +import org.dbunit.ext.h2.H2Connection; +import org.dbunit.ext.h2.H2DataTypeFactory; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; + +public class ITJdbcSourceTableLoaderTest extends LocalFileMetadataTestCase implements ISourceAware { + + protected KylinConfig config = null; + protected static Connection h2Connection = null; + + @Before + public void setup() throws Exception { + + super.createTestMetadata(); + + System.setProperty("kylin.source.jdbc.connection-url", "jdbc:h2:mem:db"); + System.setProperty("kylin.source.jdbc.driver", "org.h2.Driver"); + System.setProperty("kylin.source.jdbc.user", "sa"); + System.setProperty("kylin.source.jdbc.pass", ""); + + config = KylinConfig.getInstanceFromEnv(); + + h2Connection = DriverManager.getConnection("jdbc:h2:mem:db", "sa", ""); + + H2Database h2DB = new H2Database(h2Connection, config); + + MetadataManager mgr = MetadataManager.getInstance(KylinConfig.getInstanceFromEnv()); + ModelDataGenerator gen = new ModelDataGenerator(mgr.getDataModelDesc("ci_left_join_model"), 10000); + gen.generate(); + + h2DB.loadAllTables(); + + } + + @After + public void after() throws Exception { + + super.cleanupTestMetadata(); + + if (h2Connection != null) { + try { + h2Connection.close(); + } catch (SQLException e) { + e.printStackTrace(); + } + } + + System.clearProperty("kylin.source.jdbc.connection-url"); + System.clearProperty("kylin.source.jdbc.driver"); + System.clearProperty("kylin.source.jdbc.user"); + System.clearProperty("kylin.source.jdbc.pass"); + + } + + @Test + public void test() throws Exception { + + ISource source = SourceFactory.getSource(new ITJdbcSourceTableLoaderTest()); + ISourceMetadataExplorer explr = source.getSourceMetadataExplorer(); + Pair<TableDesc, TableExtDesc> pair; + + pair = explr.loadTableMetadata("DEFAULT", "TEST_KYLIN_FACT"); + assertTrue(pair.getFirst().getIdentity().equals("DEFAULT.TEST_KYLIN_FACT")); + + pair = explr.loadTableMetadata("EDW", "TEST_CAL_DT"); + assertTrue(pair.getFirst().getIdentity().equals("EDW.TEST_CAL_DT")); + + } + + @Override + public int getSourceType() { + return ISourceAware.ID_JDBC; + } + + @SuppressWarnings("deprecation") + protected static H2Connection newH2Connection() throws DatabaseUnitException { + H2Connection h2Conn = new H2Connection(h2Connection, null); + h2Conn.getConfig().setProperty(DatabaseConfig.PROPERTY_DATATYPE_FACTORY, new TestH2DataTypeFactory()); + h2Conn.getConfig().setFeature(DatabaseConfig.FEATURE_DATATYPE_WARNING, false); + return h2Conn; + } + + public static class TestH2DataTypeFactory extends H2DataTypeFactory { + @Override + public DataType createDataType(int sqlType, String sqlTypeName, String tableName, String columnName) + throws DataTypeException { + + if ((columnName.startsWith("COL") || columnName.startsWith("col")) && sqlType == Types.BIGINT) { + return DataType.INTEGER; + } + return super.createDataType(sqlType, sqlTypeName); + } + } + +} http://git-wip-us.apache.org/repos/asf/kylin/blob/ad47dc00/kylin-it/src/test/java/org/apache/kylin/source/jdbc/ITJdbcTableReaderTest.java ---------------------------------------------------------------------- diff --git a/kylin-it/src/test/java/org/apache/kylin/source/jdbc/ITJdbcTableReaderTest.java b/kylin-it/src/test/java/org/apache/kylin/source/jdbc/ITJdbcTableReaderTest.java new file mode 100644 index 0000000..7850952 --- /dev/null +++ b/kylin-it/src/test/java/org/apache/kylin/source/jdbc/ITJdbcTableReaderTest.java @@ -0,0 +1,116 @@ +package org.apache.kylin.source.jdbc; + +import java.sql.Connection; +import java.sql.DriverManager; +import java.sql.SQLException; +import java.sql.Types; + +import org.apache.kylin.common.KylinConfig; +import org.apache.kylin.common.util.LocalFileMetadataTestCase; +import org.apache.kylin.metadata.MetadataManager; +import org.apache.kylin.metadata.model.ISourceAware; +import org.apache.kylin.query.H2Database; +import org.apache.kylin.source.datagen.ModelDataGenerator; +import org.dbunit.DatabaseUnitException; +import org.dbunit.database.DatabaseConfig; +import org.dbunit.dataset.datatype.DataType; +import org.dbunit.dataset.datatype.DataTypeException; +import org.dbunit.ext.h2.H2Connection; +import org.dbunit.ext.h2.H2DataTypeFactory; +import org.junit.After; +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; + +public class ITJdbcTableReaderTest extends LocalFileMetadataTestCase implements ISourceAware { + + protected KylinConfig config = null; + protected static Connection h2Connection = null; + + @Before + public void setup() throws Exception { + + super.createTestMetadata(); + + System.setProperty("kylin.source.jdbc.connection-url", "jdbc:h2:mem:db"); + System.setProperty("kylin.source.jdbc.driver", "org.h2.Driver"); + System.setProperty("kylin.source.jdbc.user", "sa"); + System.setProperty("kylin.source.jdbc.pass", ""); + + config = KylinConfig.getInstanceFromEnv(); + + h2Connection = DriverManager.getConnection("jdbc:h2:mem:db", "sa", ""); + + H2Database h2DB = new H2Database(h2Connection, config); + + MetadataManager mgr = MetadataManager.getInstance(KylinConfig.getInstanceFromEnv()); + ModelDataGenerator gen = new ModelDataGenerator(mgr.getDataModelDesc("ci_left_join_model"), 10000); + gen.generate(); + + h2DB.loadAllTables(); + + } + + @After + public void after() throws Exception { + + super.cleanupTestMetadata(); + + if (h2Connection != null) { + try { + h2Connection.close(); + } catch (SQLException e) { + e.printStackTrace(); + } + } + + System.clearProperty("kylin.source.jdbc.connection-url"); + System.clearProperty("kylin.source.jdbc.driver"); + System.clearProperty("kylin.source.jdbc.user"); + System.clearProperty("kylin.source.jdbc.pass"); + + } + + @Test + public void test() throws Exception { + + JdbcTableReader reader = new JdbcTableReader("default", "test_kylin_fact"); + int rowNumber = 0; + while (reader.next()) { + String[] row = reader.getRow(); + Assert.assertEquals(11, row.length); + + rowNumber++; + } + + reader.close(); + Assert.assertEquals(10000, rowNumber); + + } + + @Override + public int getSourceType() { + return ISourceAware.ID_JDBC; + } + + @SuppressWarnings("deprecation") + protected static H2Connection newH2Connection() throws DatabaseUnitException { + H2Connection h2Conn = new H2Connection(h2Connection, null); + h2Conn.getConfig().setProperty(DatabaseConfig.PROPERTY_DATATYPE_FACTORY, new TestH2DataTypeFactory()); + h2Conn.getConfig().setFeature(DatabaseConfig.FEATURE_DATATYPE_WARNING, false); + return h2Conn; + } + + public static class TestH2DataTypeFactory extends H2DataTypeFactory { + @Override + public DataType createDataType(int sqlType, String sqlTypeName, String tableName, String columnName) + throws DataTypeException { + + if ((columnName.startsWith("COL") || columnName.startsWith("col")) && sqlType == Types.BIGINT) { + return DataType.INTEGER; + } + return super.createDataType(sqlType, sqlTypeName); + } + } + +} http://git-wip-us.apache.org/repos/asf/kylin/blob/ad47dc00/source-hive/pom.xml ---------------------------------------------------------------------- diff --git a/source-hive/pom.xml b/source-hive/pom.xml index abe8af5..b58f25e 100644 --- a/source-hive/pom.xml +++ b/source-hive/pom.xml @@ -41,6 +41,11 @@ <!-- Env & Test --> <dependency> + <groupId>com.h2database</groupId> + <artifactId>h2</artifactId> + <scope>test</scope> + </dependency> + <dependency> <groupId>org.apache.kylin</groupId> <artifactId>kylin-core-common</artifactId> <type>test-jar</type>
