This is an automated email from the ASF dual-hosted git repository.
xiangfu pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/pinot.git
The following commit(s) were added to refs/heads/master by this push:
new 316842f66e [multistage] Support array type for select query (#10434)
316842f66e is described below
commit 316842f66e8470ba03042bd5e4d41e09e1d73492
Author: Xiang Fu <xiangfu.1...@gmail.com>
AuthorDate: Fri Mar 24 02:29:12 2023 -0700
[multistage] Support array type for select query (#10434)
* [multistage] Support array type for select query
* Adding more tests
* upgrade h2 version
---
.github/workflows/pinot_compatibility_tests.yml | 2 +-
.github/workflows/pinot_tests.yml | 10 +-
...ultiStageEngineCustomTenantIntegrationTest.java | 2 +-
.../tests/MultiStageEngineIntegrationTest.java | 15 ++-
.../query/planner/logical/RelToStageConverter.java | 24 ++--
.../org/apache/pinot/query/type/TypeFactory.java | 27 +++--
.../planner/logical/RelToStageConverterTest.java | 133 +++++++++++++++++++++
.../apache/pinot/query/type/TypeFactoryTest.java | 105 ++++++++++++++++
.../pinot/query/runtime/QueryRunnerTestBase.java | 114 ++++++++++++------
.../src/test/resources/queries/Aggregates.json | 3 +-
.../src/test/resources/queries/BooleanLogic.json | 9 +-
.../test/resources/queries/SelectExpressions.json | 12 +-
12 files changed, 387 insertions(+), 69 deletions(-)
diff --git a/.github/workflows/pinot_compatibility_tests.yml
b/.github/workflows/pinot_compatibility_tests.yml
index f02304dc85..4059852163 100644
--- a/.github/workflows/pinot_compatibility_tests.yml
+++ b/.github/workflows/pinot_compatibility_tests.yml
@@ -36,7 +36,7 @@ jobs:
test_suite: [ "compatibility-verifier/sample-test-suite" ]
name: Pinot Compatibility Regression Testing against ${{
github.event.inputs.oldCommit }} and ${{ github.event.inputs.newCommit }} on
${{ matrix.test_suite }}
steps:
- - uses: actions/checkout@v2
+ - uses: actions/checkout@v3
- name: Set up JDK 11
uses: actions/setup-java@v3
with:
diff --git a/.github/workflows/pinot_tests.yml
b/.github/workflows/pinot_tests.yml
index 97d917b168..f817c3b854 100644
--- a/.github/workflows/pinot_tests.yml
+++ b/.github/workflows/pinot_tests.yml
@@ -49,7 +49,7 @@ jobs:
runs-on: ubuntu-latest
name: Pinot Linter Test Set
steps:
- - uses: actions/checkout@v2
+ - uses: actions/checkout@v3
- name: Set up JDK 11
uses: actions/setup-java@v3
with:
@@ -76,7 +76,7 @@ jobs:
testset: [ 1, 2 ]
name: Pinot Unit Test Set ${{ matrix.testset }}
steps:
- - uses: actions/checkout@v2
+ - uses: actions/checkout@v3
- name: Set up JDK 11
uses: actions/setup-java@v3
with:
@@ -122,7 +122,7 @@ jobs:
testset: [ 1, 2 ]
name: Pinot Integration Test Set ${{ matrix.testset }}
steps:
- - uses: actions/checkout@v2
+ - uses: actions/checkout@v3
- name: Set up JDK 11
uses: actions/setup-java@v3
with:
@@ -223,7 +223,7 @@ jobs:
java: [ 8, 11, 15 ]
name: Pinot Quickstart on JDK ${{ matrix.java }}
steps:
- - uses: actions/checkout@v2
+ - uses: actions/checkout@v3
- name: Set up JDK ${{ matrix.java }}
uses: actions/setup-java@v3
with:
@@ -247,7 +247,7 @@ jobs:
runs-on: ubuntu-latest
name: Build Presto Pinot Driver
steps:
- - uses: actions/checkout@v2
+ - uses: actions/checkout@v3
- name: Set up JDK 11
uses: actions/setup-java@v3
with:
diff --git
a/pinot-integration-tests/src/test/java/org/apache/pinot/integration/tests/MultiStageEngineCustomTenantIntegrationTest.java
b/pinot-integration-tests/src/test/java/org/apache/pinot/integration/tests/MultiStageEngineCustomTenantIntegrationTest.java
index b2336b3c80..c77d6c4cd1 100644
---
a/pinot-integration-tests/src/test/java/org/apache/pinot/integration/tests/MultiStageEngineCustomTenantIntegrationTest.java
+++
b/pinot-integration-tests/src/test/java/org/apache/pinot/integration/tests/MultiStageEngineCustomTenantIntegrationTest.java
@@ -42,7 +42,7 @@ import org.testng.annotations.Test;
public class MultiStageEngineCustomTenantIntegrationTest extends
MultiStageEngineIntegrationTest {
private static final String SCHEMA_FILE_NAME =
-
"On_Time_On_Time_Performance_2014_100k_subset_nonulls_single_value_columns.schema";
+ "On_Time_On_Time_Performance_2014_100k_subset_nonulls.schema";
private static final String TEST_TENANT = "TestTenant";
@Override
diff --git
a/pinot-integration-tests/src/test/java/org/apache/pinot/integration/tests/MultiStageEngineIntegrationTest.java
b/pinot-integration-tests/src/test/java/org/apache/pinot/integration/tests/MultiStageEngineIntegrationTest.java
index 651a8da1bd..6dabf814d8 100644
---
a/pinot-integration-tests/src/test/java/org/apache/pinot/integration/tests/MultiStageEngineIntegrationTest.java
+++
b/pinot-integration-tests/src/test/java/org/apache/pinot/integration/tests/MultiStageEngineIntegrationTest.java
@@ -39,8 +39,7 @@ import org.testng.annotations.Test;
public class MultiStageEngineIntegrationTest extends
BaseClusterIntegrationTestSet {
- private static final String SCHEMA_FILE_NAME =
-
"On_Time_On_Time_Performance_2014_100k_subset_nonulls_single_value_columns.schema";
+ private static final String SCHEMA_FILE_NAME =
"On_Time_On_Time_Performance_2014_100k_subset_nonulls.schema";
@Override
protected String getSchemaFileName() {
@@ -108,6 +107,18 @@ public class MultiStageEngineIntegrationTest extends
BaseClusterIntegrationTestS
h2Query, getH2Connection(), null, ImmutableMap.of("queryOptions",
"useMultistageEngine=true"));
}
+ @Test
+ public void testMultiValueColumnSelectionQuery()
+ throws Exception {
+ String pinotQuery =
+ "SELECT DivAirportIDs, DivAirports FROM mytable WHERE
DATE_TIME_CONVERT(DaysSinceEpoch, '1:DAYS:EPOCH', "
+ + "'1:DAYS:SIMPLE_DATE_FORMAT:yyyy-MM-dd''T''HH:mm:ss.SSS''Z''',
'1:DAYS') = '2014-09-05T00:00:00.000Z'";
+ String h2Query =
+ "SELECT DivAirportIDs__MV0, DivAirports__MV0 FROM mytable WHERE
DaysSinceEpoch = 16318 LIMIT 10000";
+ ClusterIntegrationTestUtils.testQueryWithMatchingRowCount(pinotQuery,
_brokerBaseApiUrl, getPinotConnection(),
+ h2Query, getH2Connection(), null, ImmutableMap.of("queryOptions",
"useMultistageEngine=true"));
+ }
+
@Override
protected Connection getPinotConnection() {
Properties properties = new Properties();
diff --git
a/pinot-query-planner/src/main/java/org/apache/pinot/query/planner/logical/RelToStageConverter.java
b/pinot-query-planner/src/main/java/org/apache/pinot/query/planner/logical/RelToStageConverter.java
index 2bc1ebdae0..4b8c07ed1a 100644
---
a/pinot-query-planner/src/main/java/org/apache/pinot/query/planner/logical/RelToStageConverter.java
+++
b/pinot-query-planner/src/main/java/org/apache/pinot/query/planner/logical/RelToStageConverter.java
@@ -34,6 +34,7 @@ import org.apache.calcite.rel.logical.LogicalWindow;
import org.apache.calcite.rel.type.RelDataType;
import org.apache.calcite.rel.type.RelDataTypeField;
import org.apache.calcite.rel.type.RelRecordType;
+import org.apache.calcite.sql.type.SqlTypeName;
import org.apache.pinot.common.utils.DataSchema;
import org.apache.pinot.common.utils.PinotDataType;
import org.apache.pinot.query.planner.partitioning.FieldSelectionKeySelector;
@@ -151,34 +152,39 @@ public final class RelToStageConverter {
}
public static DataSchema.ColumnDataType convertToColumnDataType(RelDataType
relDataType) {
- switch (relDataType.getSqlTypeName()) {
+ SqlTypeName sqlTypeName = relDataType.getSqlTypeName();
+ boolean isArray = sqlTypeName == SqlTypeName.ARRAY;
+ if (isArray) {
+ sqlTypeName = relDataType.getComponentType().getSqlTypeName();
+ }
+ switch (sqlTypeName) {
case BOOLEAN:
- return DataSchema.ColumnDataType.BOOLEAN;
+ return isArray ? DataSchema.ColumnDataType.BOOLEAN_ARRAY :
DataSchema.ColumnDataType.BOOLEAN;
case TINYINT:
case SMALLINT:
case INTEGER:
- return DataSchema.ColumnDataType.INT;
+ return isArray ? DataSchema.ColumnDataType.INT_ARRAY :
DataSchema.ColumnDataType.INT;
case BIGINT:
- return DataSchema.ColumnDataType.LONG;
+ return isArray ? DataSchema.ColumnDataType.LONG_ARRAY :
DataSchema.ColumnDataType.LONG;
case DECIMAL:
return resolveDecimal(relDataType);
case FLOAT:
- return DataSchema.ColumnDataType.FLOAT;
+ return isArray ? DataSchema.ColumnDataType.FLOAT_ARRAY :
DataSchema.ColumnDataType.FLOAT;
case REAL:
case DOUBLE:
- return DataSchema.ColumnDataType.DOUBLE;
+ return isArray ? DataSchema.ColumnDataType.DOUBLE_ARRAY :
DataSchema.ColumnDataType.DOUBLE;
case DATE:
case TIME:
case TIMESTAMP:
- return DataSchema.ColumnDataType.TIMESTAMP;
+ return isArray ? DataSchema.ColumnDataType.TIMESTAMP_ARRAY :
DataSchema.ColumnDataType.TIMESTAMP;
case CHAR:
case VARCHAR:
- return DataSchema.ColumnDataType.STRING;
+ return isArray ? DataSchema.ColumnDataType.STRING_ARRAY :
DataSchema.ColumnDataType.STRING;
case OTHER:
return DataSchema.ColumnDataType.OBJECT;
case BINARY:
case VARBINARY:
- return DataSchema.ColumnDataType.BYTES;
+ return isArray ? DataSchema.ColumnDataType.BYTES_ARRAY :
DataSchema.ColumnDataType.BYTES;
default:
return DataSchema.ColumnDataType.BYTES;
}
diff --git
a/pinot-query-planner/src/main/java/org/apache/pinot/query/type/TypeFactory.java
b/pinot-query-planner/src/main/java/org/apache/pinot/query/type/TypeFactory.java
index 0e1814e8c9..1877f7ec9c 100644
---
a/pinot-query-planner/src/main/java/org/apache/pinot/query/type/TypeFactory.java
+++
b/pinot-query-planner/src/main/java/org/apache/pinot/query/type/TypeFactory.java
@@ -54,23 +54,32 @@ public class TypeFactory extends JavaTypeFactoryImpl {
private RelDataType toRelDataType(FieldSpec fieldSpec) {
switch (fieldSpec.getDataType()) {
case INT:
- return createSqlType(SqlTypeName.INTEGER);
+ return fieldSpec.isSingleValueField() ?
createSqlType(SqlTypeName.INTEGER)
+ : createArrayType(createSqlType(SqlTypeName.INTEGER), -1);
case LONG:
- return createSqlType(SqlTypeName.BIGINT);
+ return fieldSpec.isSingleValueField() ?
createSqlType(SqlTypeName.BIGINT)
+ : createArrayType(createSqlType(SqlTypeName.BIGINT), -1);
case FLOAT:
- return createSqlType(SqlTypeName.FLOAT);
+ return fieldSpec.isSingleValueField() ?
createSqlType(SqlTypeName.FLOAT)
+ : createArrayType(createSqlType(SqlTypeName.FLOAT), -1);
case DOUBLE:
- return createSqlType(SqlTypeName.DOUBLE);
+ return fieldSpec.isSingleValueField() ?
createSqlType(SqlTypeName.DOUBLE)
+ : createArrayType(createSqlType(SqlTypeName.DOUBLE), -1);
case BOOLEAN:
- return createSqlType(SqlTypeName.BOOLEAN);
+ return fieldSpec.isSingleValueField() ?
createSqlType(SqlTypeName.BOOLEAN)
+ : createArrayType(createSqlType(SqlTypeName.BOOLEAN), -1);
case TIMESTAMP:
- return createSqlType(SqlTypeName.TIMESTAMP);
+ return fieldSpec.isSingleValueField() ?
createSqlType(SqlTypeName.TIMESTAMP)
+ : createArrayType(createSqlType(SqlTypeName.TIMESTAMP), -1);
case STRING:
- return createSqlType(SqlTypeName.VARCHAR);
+ return fieldSpec.isSingleValueField() ?
createSqlType(SqlTypeName.VARCHAR)
+ : createArrayType(createSqlType(SqlTypeName.VARCHAR), -1);
case BYTES:
- return createSqlType(SqlTypeName.VARBINARY);
+ return fieldSpec.isSingleValueField() ?
createSqlType(SqlTypeName.VARBINARY)
+ : createArrayType(createSqlType(SqlTypeName.VARBINARY), -1);
case BIG_DECIMAL:
- return createSqlType(SqlTypeName.DECIMAL);
+ return fieldSpec.isSingleValueField() ?
createSqlType(SqlTypeName.DECIMAL)
+ : createArrayType(createSqlType(SqlTypeName.DECIMAL), -1);
case JSON:
// TODO: support JSON, JSON should be supported using a special
RelDataType as it is not a simple String,
// nor can it be easily parsed as a STRUCT.
diff --git
a/pinot-query-planner/src/test/java/org/apache/pinot/query/planner/logical/RelToStageConverterTest.java
b/pinot-query-planner/src/test/java/org/apache/pinot/query/planner/logical/RelToStageConverterTest.java
new file mode 100644
index 0000000000..32a27c979b
--- /dev/null
+++
b/pinot-query-planner/src/test/java/org/apache/pinot/query/planner/logical/RelToStageConverterTest.java
@@ -0,0 +1,133 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.pinot.query.planner.logical;
+
+import org.apache.calcite.rel.type.RelDataTypeSystem;
+import org.apache.calcite.sql.SqlIdentifier;
+import org.apache.calcite.sql.type.ArraySqlType;
+import org.apache.calcite.sql.type.BasicSqlType;
+import org.apache.calcite.sql.type.ObjectSqlType;
+import org.apache.calcite.sql.type.SqlTypeName;
+import org.apache.pinot.common.utils.DataSchema;
+import org.testng.Assert;
+import org.testng.annotations.Test;
+
+
+public class RelToStageConverterTest {
+
+ @Test
+ public void testConvertToColumnDataTypeForObjectTypes() {
+ Assert.assertEquals(RelToStageConverter.convertToColumnDataType(
+ new ObjectSqlType(SqlTypeName.BOOLEAN, SqlIdentifier.STAR, true,
null, null)),
+ DataSchema.ColumnDataType.BOOLEAN);
+ Assert.assertEquals(RelToStageConverter.convertToColumnDataType(
+ new ObjectSqlType(SqlTypeName.TINYINT, SqlIdentifier.STAR, true,
null, null)),
+ DataSchema.ColumnDataType.INT);
+ Assert.assertEquals(RelToStageConverter.convertToColumnDataType(
+ new ObjectSqlType(SqlTypeName.SMALLINT, SqlIdentifier.STAR, true,
null, null)),
+ DataSchema.ColumnDataType.INT);
+ Assert.assertEquals(RelToStageConverter.convertToColumnDataType(
+ new ObjectSqlType(SqlTypeName.INTEGER, SqlIdentifier.STAR, true,
null, null)),
+ DataSchema.ColumnDataType.INT);
+ Assert.assertEquals(RelToStageConverter.convertToColumnDataType(
+ new ObjectSqlType(SqlTypeName.BIGINT, SqlIdentifier.STAR, true,
null, null)),
+ DataSchema.ColumnDataType.LONG);
+ Assert.assertEquals(RelToStageConverter.convertToColumnDataType(
+ new ObjectSqlType(SqlTypeName.FLOAT, SqlIdentifier.STAR, true,
null, null)),
+ DataSchema.ColumnDataType.FLOAT);
+ Assert.assertEquals(RelToStageConverter.convertToColumnDataType(
+ new ObjectSqlType(SqlTypeName.DOUBLE, SqlIdentifier.STAR, true,
null, null)),
+ DataSchema.ColumnDataType.DOUBLE);
+ Assert.assertEquals(RelToStageConverter.convertToColumnDataType(
+ new ObjectSqlType(SqlTypeName.TIMESTAMP, SqlIdentifier.STAR, true,
null, null)),
+ DataSchema.ColumnDataType.TIMESTAMP);
+ Assert.assertEquals(RelToStageConverter.convertToColumnDataType(
+ new ObjectSqlType(SqlTypeName.CHAR, SqlIdentifier.STAR, true,
null, null)),
+ DataSchema.ColumnDataType.STRING);
+ Assert.assertEquals(RelToStageConverter.convertToColumnDataType(
+ new ObjectSqlType(SqlTypeName.VARCHAR, SqlIdentifier.STAR, true,
null, null)),
+ DataSchema.ColumnDataType.STRING);
+ Assert.assertEquals(RelToStageConverter.convertToColumnDataType(
+ new ObjectSqlType(SqlTypeName.VARBINARY, SqlIdentifier.STAR, true,
null, null)),
+ DataSchema.ColumnDataType.BYTES);
+ Assert.assertEquals(RelToStageConverter.convertToColumnDataType(
+ new ObjectSqlType(SqlTypeName.OTHER, SqlIdentifier.STAR, true,
null, null)),
+ DataSchema.ColumnDataType.OBJECT);
+ }
+
+ @Test
+ public void testBigDecimal() {
+ Assert.assertEquals(RelToStageConverter.convertToColumnDataType(
+ new BasicSqlType(RelDataTypeSystem.DEFAULT, SqlTypeName.DECIMAL,
10)),
+ DataSchema.ColumnDataType.INT);
+ Assert.assertEquals(RelToStageConverter.convertToColumnDataType(
+ new BasicSqlType(RelDataTypeSystem.DEFAULT, SqlTypeName.DECIMAL,
38)),
+ DataSchema.ColumnDataType.LONG);
+ Assert.assertEquals(RelToStageConverter.convertToColumnDataType(
+ new BasicSqlType(RelDataTypeSystem.DEFAULT, SqlTypeName.DECIMAL,
39)),
+ DataSchema.ColumnDataType.BIG_DECIMAL);
+
+ Assert.assertEquals(RelToStageConverter.convertToColumnDataType(
+ new BasicSqlType(RelDataTypeSystem.DEFAULT, SqlTypeName.DECIMAL,
14, 10)),
+ DataSchema.ColumnDataType.FLOAT);
+ Assert.assertEquals(RelToStageConverter.convertToColumnDataType(
+ new BasicSqlType(RelDataTypeSystem.DEFAULT, SqlTypeName.DECIMAL,
30, 10)),
+ DataSchema.ColumnDataType.DOUBLE);
+ Assert.assertEquals(RelToStageConverter.convertToColumnDataType(
+ new BasicSqlType(RelDataTypeSystem.DEFAULT, SqlTypeName.DECIMAL,
31, 10)),
+ DataSchema.ColumnDataType.BIG_DECIMAL);
+ }
+
+ @Test
+ public void testConvertToColumnDataTypeForArray() {
+ Assert.assertEquals(RelToStageConverter.convertToColumnDataType(
+ new ArraySqlType(new ObjectSqlType(SqlTypeName.BOOLEAN,
SqlIdentifier.STAR, true, null, null), true)),
+ DataSchema.ColumnDataType.BOOLEAN_ARRAY);
+ Assert.assertEquals(RelToStageConverter.convertToColumnDataType(
+ new ArraySqlType(new ObjectSqlType(SqlTypeName.TINYINT,
SqlIdentifier.STAR, true, null, null), true)),
+ DataSchema.ColumnDataType.INT_ARRAY);
+ Assert.assertEquals(RelToStageConverter.convertToColumnDataType(
+ new ArraySqlType(new ObjectSqlType(SqlTypeName.SMALLINT,
SqlIdentifier.STAR, true, null, null), true)),
+ DataSchema.ColumnDataType.INT_ARRAY);
+ Assert.assertEquals(RelToStageConverter.convertToColumnDataType(
+ new ArraySqlType(new ObjectSqlType(SqlTypeName.INTEGER,
SqlIdentifier.STAR, true, null, null), true)),
+ DataSchema.ColumnDataType.INT_ARRAY);
+ Assert.assertEquals(RelToStageConverter.convertToColumnDataType(
+ new ArraySqlType(new ObjectSqlType(SqlTypeName.BIGINT,
SqlIdentifier.STAR, true, null, null), true)),
+ DataSchema.ColumnDataType.LONG_ARRAY);
+ Assert.assertEquals(RelToStageConverter.convertToColumnDataType(
+ new ArraySqlType(new ObjectSqlType(SqlTypeName.FLOAT,
SqlIdentifier.STAR, true, null, null), true)),
+ DataSchema.ColumnDataType.FLOAT_ARRAY);
+ Assert.assertEquals(RelToStageConverter.convertToColumnDataType(
+ new ArraySqlType(new ObjectSqlType(SqlTypeName.DOUBLE,
SqlIdentifier.STAR, true, null, null), true)),
+ DataSchema.ColumnDataType.DOUBLE_ARRAY);
+ Assert.assertEquals(RelToStageConverter.convertToColumnDataType(
+ new ArraySqlType(new ObjectSqlType(SqlTypeName.TIMESTAMP,
SqlIdentifier.STAR, true, null, null), true)),
+ DataSchema.ColumnDataType.TIMESTAMP_ARRAY);
+ Assert.assertEquals(RelToStageConverter.convertToColumnDataType(
+ new ArraySqlType(new ObjectSqlType(SqlTypeName.CHAR,
SqlIdentifier.STAR, true, null, null), true)),
+ DataSchema.ColumnDataType.STRING_ARRAY);
+ Assert.assertEquals(RelToStageConverter.convertToColumnDataType(
+ new ArraySqlType(new ObjectSqlType(SqlTypeName.VARCHAR,
SqlIdentifier.STAR, true, null, null), true)),
+ DataSchema.ColumnDataType.STRING_ARRAY);
+ Assert.assertEquals(RelToStageConverter.convertToColumnDataType(
+ new ArraySqlType(new ObjectSqlType(SqlTypeName.VARBINARY,
SqlIdentifier.STAR, true, null, null), true)),
+ DataSchema.ColumnDataType.BYTES_ARRAY);
+ }
+}
diff --git
a/pinot-query-planner/src/test/java/org/apache/pinot/query/type/TypeFactoryTest.java
b/pinot-query-planner/src/test/java/org/apache/pinot/query/type/TypeFactoryTest.java
new file mode 100644
index 0000000000..0e1d55d646
--- /dev/null
+++
b/pinot-query-planner/src/test/java/org/apache/pinot/query/type/TypeFactoryTest.java
@@ -0,0 +1,105 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.pinot.query.type;
+
+import java.util.List;
+import org.apache.calcite.rel.type.RelDataType;
+import org.apache.calcite.rel.type.RelDataTypeField;
+import org.apache.calcite.sql.type.ArraySqlType;
+import org.apache.calcite.sql.type.BasicSqlType;
+import org.apache.calcite.sql.type.SqlTypeName;
+import org.apache.pinot.spi.data.FieldSpec;
+import org.apache.pinot.spi.data.Schema;
+import org.testng.Assert;
+import org.testng.annotations.Test;
+
+
+public class TypeFactoryTest {
+ private static final TypeSystem TYPE_SYSTEM = new TypeSystem() {
+ };
+
+ @Test
+ public void testRelDataTypeConversion() {
+ TypeFactory typeFactory = new TypeFactory(TYPE_SYSTEM);
+ Schema testSchema = new
Schema.SchemaBuilder().addSingleValueDimension("INT_COL",
FieldSpec.DataType.INT)
+ .addSingleValueDimension("LONG_COL", FieldSpec.DataType.LONG)
+ .addSingleValueDimension("FLOAT_COL", FieldSpec.DataType.FLOAT)
+ .addSingleValueDimension("DOUBLE_COL", FieldSpec.DataType.DOUBLE)
+ .addSingleValueDimension("STRING_COL", FieldSpec.DataType.STRING)
+ .addSingleValueDimension("BYTES_COL", FieldSpec.DataType.BYTES)
+ .addMultiValueDimension("INT_ARRAY_COL", FieldSpec.DataType.INT)
+ .addMultiValueDimension("LONG_ARRAY_COL", FieldSpec.DataType.LONG)
+ .addMultiValueDimension("FLOAT_ARRAY_COL", FieldSpec.DataType.FLOAT)
+ .addMultiValueDimension("DOUBLE_ARRAY_COL", FieldSpec.DataType.DOUBLE)
+ .addMultiValueDimension("STRING_ARRAY_COL", FieldSpec.DataType.STRING)
+ .addMultiValueDimension("BYTES_ARRAY_COL", FieldSpec.DataType.BYTES)
+ .build();
+ RelDataType relDataTypeFromSchema =
typeFactory.createRelDataTypeFromSchema(testSchema);
+ List<RelDataTypeField> fieldList = relDataTypeFromSchema.getFieldList();
+ for (RelDataTypeField field : fieldList) {
+ switch (field.getName()) {
+ case "INT_COL":
+ Assert.assertEquals(field.getType(), new BasicSqlType(TYPE_SYSTEM,
SqlTypeName.INTEGER));
+ break;
+ case "LONG_COL":
+ Assert.assertEquals(field.getType(), new BasicSqlType(TYPE_SYSTEM,
SqlTypeName.BIGINT));
+ break;
+ case "FLOAT_COL":
+ Assert.assertEquals(field.getType(), new BasicSqlType(TYPE_SYSTEM,
SqlTypeName.FLOAT));
+ break;
+ case "DOUBLE_COL":
+ Assert.assertEquals(field.getType(), new BasicSqlType(TYPE_SYSTEM,
SqlTypeName.DOUBLE));
+ break;
+ case "STRING_COL":
+ Assert.assertEquals(field.getType(), new BasicSqlType(TYPE_SYSTEM,
SqlTypeName.VARCHAR));
+ break;
+ case "BYTES_COL":
+ Assert.assertEquals(field.getType(), new BasicSqlType(TYPE_SYSTEM,
SqlTypeName.VARBINARY));
+ break;
+ case "INT_ARRAY_COL":
+ Assert.assertEquals(field.getType(),
+ new ArraySqlType(new BasicSqlType(TYPE_SYSTEM,
SqlTypeName.INTEGER), false));
+ break;
+ case "LONG_ARRAY_COL":
+ Assert.assertEquals(field.getType(),
+ new ArraySqlType(new BasicSqlType(TYPE_SYSTEM,
SqlTypeName.BIGINT), false));
+ break;
+ case "FLOAT_ARRAY_COL":
+ Assert.assertEquals(field.getType(),
+ new ArraySqlType(new BasicSqlType(TYPE_SYSTEM,
SqlTypeName.FLOAT), false));
+ break;
+ case "DOUBLE_ARRAY_COL":
+ Assert.assertEquals(field.getType(),
+ new ArraySqlType(new BasicSqlType(TYPE_SYSTEM,
SqlTypeName.DOUBLE), false));
+ break;
+ case "STRING_ARRAY_COL":
+ Assert.assertEquals(field.getType(),
+ new ArraySqlType(new BasicSqlType(TYPE_SYSTEM,
SqlTypeName.VARCHAR), false));
+ break;
+ case "BYTES_ARRAY_COL":
+ Assert.assertEquals(field.getType(),
+ new ArraySqlType(new BasicSqlType(TYPE_SYSTEM,
SqlTypeName.VARBINARY), false));
+ break;
+ default:
+ Assert.fail("Unexpected column name: " + field.getName());
+ break;
+ }
+ }
+ }
+}
diff --git
a/pinot-query-runtime/src/test/java/org/apache/pinot/query/runtime/QueryRunnerTestBase.java
b/pinot-query-runtime/src/test/java/org/apache/pinot/query/runtime/QueryRunnerTestBase.java
index e083948758..06379ac21d 100644
---
a/pinot-query-runtime/src/test/java/org/apache/pinot/query/runtime/QueryRunnerTestBase.java
+++
b/pinot-query-runtime/src/test/java/org/apache/pinot/query/runtime/QueryRunnerTestBase.java
@@ -61,6 +61,7 @@ import org.apache.pinot.spi.data.readers.GenericRow;
import org.apache.pinot.spi.utils.ByteArray;
import org.apache.pinot.spi.utils.CommonConstants;
import org.apache.pinot.spi.utils.StringUtil;
+import org.h2.jdbc.JdbcArray;
import org.testng.Assert;
@@ -193,6 +194,35 @@ public abstract class QueryRunnerTestBase extends
QueryTestSet {
return ((ByteArray) l).compareTo(new ByteArray((byte[]) r));
} else if (l instanceof Timestamp) {
return ((Timestamp) l).compareTo((Timestamp) r);
+ } else if (l instanceof int[]) {
+ int[] larray = (int[]) l;
+ Object[] rarray;
+ try {
+ rarray = (Object[]) ((JdbcArray) r).getArray();
+ } catch (SQLException e) {
+ throw new RuntimeException(e);
+ }
+ for (int idx = 0; idx < larray.length; idx++) {
+ Number relement = (Number) rarray[idx];
+ if (larray[idx] != relement.intValue()) {
+ return -1;
+ }
+ }
+ return 0;
+ } else if (l instanceof String[]) {
+ String[] larray = (String[]) l;
+ Object[] rarray;
+ try {
+ rarray = (Object[]) ((JdbcArray) r).getArray();
+ } catch (SQLException e) {
+ throw new RuntimeException(e);
+ }
+ for (int idx = 0; idx < larray.length; idx++) {
+ if (!larray[idx].equals(rarray[idx])) {
+ return -1;
+ }
+ }
+ return 0;
} else {
throw new RuntimeException("non supported type " + l.getClass());
}
@@ -231,7 +261,11 @@ public abstract class QueryRunnerTestBase extends
QueryTestSet {
protected Schema constructSchema(String schemaName,
List<QueryTestCase.ColumnAndType> columnAndTypes) {
Schema.SchemaBuilder builder = new Schema.SchemaBuilder();
for (QueryTestCase.ColumnAndType columnAndType : columnAndTypes) {
- builder.addSingleValueDimension(columnAndType._name,
FieldSpec.DataType.valueOf(columnAndType._type));
+ if (columnAndType._isSingleValue) {
+ builder.addSingleValueDimension(columnAndType._name,
FieldSpec.DataType.valueOf(columnAndType._type));
+ } else {
+ builder.addMultiValueDimension(columnAndType._name,
FieldSpec.DataType.valueOf(columnAndType._type));
+ }
}
// TODO: ts is built-in, but we should allow user overwrite
builder.addDateTime("ts", FieldSpec.DataType.LONG, "1:MILLISECONDS:EPOCH",
"1:SECONDS");
@@ -299,13 +333,19 @@ public abstract class QueryRunnerTestBase extends
QueryTestSet {
int h2Index = 1;
for (String fieldName : schema.getColumnNames()) {
Object value = row.getValue(fieldName);
- switch (schema.getFieldSpecFor(fieldName).getDataType()) {
- case BYTES:
- h2Statement.setBytes(h2Index++, Hex.decodeHex((String) value));
- break;
- default:
- h2Statement.setObject(h2Index++, value);
- break;
+ if (value instanceof List) {
+ h2Statement.setArray(h2Index++,
+
_h2Connection.createArrayOf(getH2FieldType(schema.getFieldSpecFor(fieldName).getDataType()),
+ ((List) value).toArray()));
+ } else {
+ switch (schema.getFieldSpecFor(fieldName).getDataType()) {
+ case BYTES:
+ h2Statement.setBytes(h2Index++, Hex.decodeHex((String) value));
+ break;
+ default:
+ h2Statement.setObject(h2Index++, value);
+ break;
+ }
}
}
h2Statement.execute();
@@ -317,41 +357,39 @@ public abstract class QueryRunnerTestBase extends
QueryTestSet {
List<String> fieldNamesAndTypes = new ArrayList<>(pinotSchema.size());
for (String fieldName : pinotSchema.getColumnNames()) {
FieldSpec.DataType dataType =
pinotSchema.getFieldSpecFor(fieldName).getDataType();
- String fieldType;
- switch (dataType) {
- case INT:
- case LONG:
- fieldType = "bigint";
- break;
- case STRING:
- fieldType = "varchar(128)";
- break;
- case FLOAT:
- fieldType = "real";
- break;
- case DOUBLE:
- fieldType = "double";
- break;
- case BOOLEAN:
- fieldType = "BOOLEAN";
- break;
- case BIG_DECIMAL:
- fieldType = "NUMERIC(65535, 32767)";
- break;
- case BYTES:
- fieldType = "BYTEA";
- break;
- case TIMESTAMP:
- fieldType = "TIMESTAMP";
- break;
- default:
- throw new UnsupportedOperationException("Unsupported type conversion
to h2 type: " + dataType);
+ String fieldType = getH2FieldType(dataType);
+ if (!pinotSchema.getFieldSpecFor(fieldName).isSingleValueField()) {
+ fieldType += " ARRAY";
}
fieldNamesAndTypes.add(fieldName + " " + fieldType);
}
return fieldNamesAndTypes;
}
+ private static String getH2FieldType(FieldSpec.DataType dataType) {
+ switch (dataType) {
+ case INT:
+ case LONG:
+ return "bigint";
+ case STRING:
+ return "varchar(128)";
+ case FLOAT:
+ return "real";
+ case DOUBLE:
+ return "double";
+ case BOOLEAN:
+ return "BOOLEAN";
+ case BIG_DECIMAL:
+ return "NUMERIC(65535, 32767)";
+ case BYTES:
+ return "BYTEA";
+ case TIMESTAMP:
+ return "TIMESTAMP";
+ default:
+ throw new UnsupportedOperationException("Unsupported type conversion
to h2 type: " + dataType);
+ }
+ }
+
@JsonIgnoreProperties(ignoreUnknown = true)
public static class QueryTestCase {
public static final String BLOCK_SIZE_KEY = "blockSize";
@@ -400,6 +438,8 @@ public abstract class QueryRunnerTestBase extends
QueryTestSet {
String _name;
@JsonProperty("type")
String _type;
+ @JsonProperty("isSingleValue")
+ boolean _isSingleValue = true;
}
}
}
diff --git a/pinot-query-runtime/src/test/resources/queries/Aggregates.json
b/pinot-query-runtime/src/test/resources/queries/Aggregates.json
index ade91b5ee1..aaafa39651 100644
--- a/pinot-query-runtime/src/test/resources/queries/Aggregates.json
+++ b/pinot-query-runtime/src/test/resources/queries/Aggregates.json
@@ -195,7 +195,8 @@
{
"psql": "4.2.7",
"description": "aggregate int column and filter by int column",
- "sql": "SELECT sum(1 / int_col) FROM {tbl} WHERE int_col > 0"
+ "sql": "SELECT sum(1 / int_col) FROM {tbl} WHERE int_col > 0",
+ "h2Sql": "SELECT sum(1.0 / int_col) FROM {tbl} WHERE int_col > 0"
},
{
"psql": "4.2.7",
diff --git a/pinot-query-runtime/src/test/resources/queries/BooleanLogic.json
b/pinot-query-runtime/src/test/resources/queries/BooleanLogic.json
index 0a599e6caa..4473c4704a 100644
--- a/pinot-query-runtime/src/test/resources/queries/BooleanLogic.json
+++ b/pinot-query-runtime/src/test/resources/queries/BooleanLogic.json
@@ -58,13 +58,15 @@
"ignored": true,
"note": "H2 doesn't support this",
"description": "check implicit cast between boolean and int",
- "sql": "select b = i FROM {tbl}"
+ "sql": "select b = i FROM {tbl}",
+ "expectedException": "Values of types .* and .* are not comparable"
},
{
"ignored": true,
"comment": "H2 doesn't support this",
"description": "check implicit cast between boolean and double",
- "sql": "select b = d FROM {tbl}"
+ "sql": "select b = d FROM {tbl}",
+ "expectedException": "Values of types .* and .* are not comparable"
},
{
"ignored": true,
@@ -76,7 +78,8 @@
"ignored": true,
"comment": "H2 doesn't support this",
"description": "check implicit cast between boolean and numeric",
- "sql": "select b = n FROM {tbl}"
+ "sql": "select b = n FROM {tbl}",
+ "expectedException": "Values of types .* and .* are not comparable"
},
{
"description": "implicit cast should fail between boolean and
timestamp",
diff --git
a/pinot-query-runtime/src/test/resources/queries/SelectExpressions.json
b/pinot-query-runtime/src/test/resources/queries/SelectExpressions.json
index 7445bcf982..e0e4fb8b23 100644
--- a/pinot-query-runtime/src/test/resources/queries/SelectExpressions.json
+++ b/pinot-query-runtime/src/test/resources/queries/SelectExpressions.json
@@ -27,6 +27,15 @@
[1, "foo"],
[2, "bar"]
]
+ },
+ "tbl3": {
+ "schema":[
+ {"name": "intArrayCol", "type": "INT", "isSingleValue": false},
+ {"name": "strArrayCol", "type": "STRING", "isSingleValue": false}
+ ],
+ "inputs": [
+ [[1, 10], ["foo1", "foo2"]]
+ ]
}
},
"queries": [
@@ -53,7 +62,8 @@
{ "sql": "SELECT intCol, intCol, doubleCol, strCol, strCol FROM {tbl1}"
},
{ "sql": "SELECT {tbl1}.intCol, {tbl1}.intCol, {tbl1}.doubleCol,
{tbl2}.strCol, {tbl2}.strCol FROM {tbl1}, {tbl2} WHERE {tbl1}.intCol =
{tbl2}.intCol" },
{ "sql": "SELECT {tbl2}.intCol, {tbl2}.intCol FROM {tbl1}, {tbl2} WHERE
{tbl1}.intCol = {tbl2}.intCol AND {tbl1}.intCol < 100 ORDER BY
{tbl1}.doubleCol" },
- { "sql": "SELECT intCol, intCol FROM {tbl1} WHERE intCol < 100"}
+ { "sql": "SELECT intCol, intCol FROM {tbl1} WHERE intCol < 100"},
+ { "sql": "SELECT intArrayCol, strArrayCol FROM {tbl3}"}
]
}
}
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscr...@pinot.apache.org
For additional commands, e-mail: commits-h...@pinot.apache.org
