This is an automated email from the ASF dual-hosted git repository.
xxyu pushed a commit to branch kylin5
in repository https://gitbox.apache.org/repos/asf/kylin.git
commit 85590405dd674305a87cdb87c656f8f7f1d315b4
Author: Pengfei Zhan <dethr...@gmail.com>
AuthorDate: Wed Mar 22 23:29:59 2023 +0800
KYLIN-5581 Query min and max aggregations using metadata for all data types.
---
.../kylin/query/engine/QueryExecWithMetaTest.java | 13 ++
.../test/resources/query/sql_min_max/query08.sql | 37 +++++
.../c19de21a-c43a-ad7b-9eec-3928da617025.json | 96 +++++++++++
.../67ffb57f-8231-0534-93b1-a8239ffd26ab.json | 39 +++++
.../c19de21a-c43a-ad7b-9eec-3928da617025.json | 63 +++++++
.../c19de21a-c43a-ad7b-9eec-3928da617025.json | 181 +++++++++++++++++++++
.../metadata/min_max/table/TDVT.TEST_MEASURE.json | 143 ++++++++++++++++
.../apache/kylin/query/relnode/OLAPContext.java | 10 +-
.../kylin/query/runtime/plan/TableScanPlan.scala | 12 +-
.../org/apache/spark/sql/SparderTypeUtil.scala | 6 +-
10 files changed, 592 insertions(+), 8 deletions(-)
diff --git
a/src/kylin-it/src/test/java/org/apache/kylin/query/engine/QueryExecWithMetaTest.java
b/src/kylin-it/src/test/java/org/apache/kylin/query/engine/QueryExecWithMetaTest.java
index 906128c97d..ac84de6f06 100644
---
a/src/kylin-it/src/test/java/org/apache/kylin/query/engine/QueryExecWithMetaTest.java
+++
b/src/kylin-it/src/test/java/org/apache/kylin/query/engine/QueryExecWithMetaTest.java
@@ -221,6 +221,19 @@ public class QueryExecWithMetaTest extends
NLocalWithSparkSessionTest {
Assert.assertEquals("1001", thirdRow.get(2));
}
+ @Test
+ public void testAllType() throws IOException, SQLException {
+ String sql = getSql("/query/sql_min_max/query08.sql");
+ QueryResult queryResult = queryExec.executeQuery(sql);
+ Assert.assertEquals(1, queryResult.getSize());
+ Iterator<List<String>> iterator =
queryResult.getRowsIterable().iterator();
+ List<String> result = iterator.next();
+
Assert.assertEquals("2147483648,21474836483289,2132,2147483647,-128,127,0,9,0.0,10000.0,"
+ +
"0.3255242,85208.3241,10.0000,201.3235,abc,xyz,aaaaaaaa,xxxxxxxxxxxxxxxxxxxxx,abcd,zzzz,"
+ + "2000-12-31,2004-04-16,2004-04-01 00:00:00,2004-04-17
00:32:23.032,false,true,"
+ +
"null,null,null,null,null,null,null,null,null,null,null,null", String.join(",",
result));
+ }
+
private String getSql(String path) throws IOException {
String sql = CharStreams.toString(
new
InputStreamReader(Objects.requireNonNull(getClass().getResourceAsStream(path)),
Charsets.UTF_8));
diff --git a/src/kylin-it/src/test/resources/query/sql_min_max/query08.sql
b/src/kylin-it/src/test/resources/query/sql_min_max/query08.sql
new file mode 100644
index 0000000000..e0ee90e3e0
--- /dev/null
+++ b/src/kylin-it/src/test/resources/query/sql_min_max/query08.sql
@@ -0,0 +1,37 @@
+--
+-- Licensed to the Apache Software Foundation (ASF) under one
+-- or more contributor license agreements. See the NOTICE file
+-- distributed with this work for additional information
+-- regarding copyright ownership. The ASF licenses this file
+-- to you under the Apache License, Version 2.0 (the
+-- "License"); you may not use this file except in compliance
+-- with the License. You may obtain a copy of the License at
+--
+-- http://www.apache.org/licenses/LICENSE-2.0
+--
+-- Unless required by applicable law or agreed to in writing, software
+-- distributed under the License is distributed on an "AS IS" BASIS,
+-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+-- See the License for the specific language governing permissions and
+-- limitations under the License.
+
+SELECT min(id1), max(id1),
+ min(id2), max(id2),
+ min(id3), max(id3),
+ min(id4), max(id4),
+ min(price1), max(price1),
+ min(price2), max(price2),
+ min(price3), max(price3),
+ min(name1), max(name1),
+ min(name2), max(name2),
+ min(name3), max(name3),
+ min(date1), max(date1),
+ min(time1), max(time1),
+ min(flag), max(flag),
+ min(id_null), max(id_null),
+ min(date_null), max(date_null),
+ min(timestamp_null), max(timestamp_null),
+ min(string_null), max(string_null),
+ min(flag_null), max(flag_null),
+ min(double_null), max(double_null)
+FROM tdvt.test_measure
diff --git
a/src/kylin-it/src/test/resources/ut_meta/query_min_max_with_meta/metadata/min_max/dataflow/c19de21a-c43a-ad7b-9eec-3928da617025.json
b/src/kylin-it/src/test/resources/ut_meta/query_min_max_with_meta/metadata/min_max/dataflow/c19de21a-c43a-ad7b-9eec-3928da617025.json
new file mode 100644
index 0000000000..994e2bec66
--- /dev/null
+++
b/src/kylin-it/src/test/resources/ut_meta/query_min_max_with_meta/metadata/min_max/dataflow/c19de21a-c43a-ad7b-9eec-3928da617025.json
@@ -0,0 +1,96 @@
+{
+ "uuid": "c19de21a-c43a-ad7b-9eec-3928da617025",
+ "last_modified": 1679548434834,
+ "create_time": 1679548043512,
+ "version": "4.0.0.0",
+ "status": "ONLINE",
+ "last_status": null,
+ "cost": 50,
+ "query_hit_count": 0,
+ "last_query_time": 0,
+ "layout_query_hit_count": {},
+ "segments": [
+ {
+ "id": "67ffb57f-8231-0534-93b1-a8239ffd26ab",
+ "name": "FULL_BUILD",
+ "create_time_utc": 1679548043516,
+ "status": "READY",
+ "segRange": {
+ "@class":
"org.apache.kylin.metadata.model.SegmentRange$TimePartitionedSegmentRange",
+ "date_range_start": 0,
+ "date_range_end": 9223372036854775807
+ },
+ "timeRange": null,
+ "dimension_range_info_map": {
+ "0": {
+ "min": "0.0",
+ "max": "10000"
+ },
+ "1": {
+ "min": "10.00002",
+ "max": "201.323523"
+ },
+ "2": {
+ "min": "0.3255242",
+ "max": "085208.3241"
+ },
+ "3": {
+ "min": "2001-01-01",
+ "max": "2004-04-17"
+ },
+ "4": {
+ "min": "2004-04-01 00:00:00.0",
+ "max": "2004-04-17 00:32:23.32"
+ },
+ "5": {
+ "min": "0",
+ "max": "9"
+ },
+ "6": {
+ "min": "-128",
+ "max": "127"
+ },
+ "7": {
+ "min": "abcd",
+ "max": "zzzz"
+ },
+ "8": {
+ "min": "false",
+ "max": "true"
+ },
+ "9": {
+ "min": "2132",
+ "max": "2147483647"
+ },
+ "10": {
+ "min": "aaaaaaaa",
+ "max": "xxxxxxxxxxxxxxxxxxxxx"
+ },
+ "11": {
+ "min": "2147483648",
+ "max": "21474836483289"
+ },
+ "12": {
+ "min": "abc",
+ "max": "xyz"
+ }
+ },
+ "parameters": null,
+ "dictionaries": null,
+ "snapshots": null,
+ "last_build_time": 1679548122080,
+ "source_count": 0,
+ "source_bytes_size": 0,
+ "column_source_bytes": {},
+ "ori_snapshot_size": {},
+ "additionalInfo": {},
+ "is_realtime_segment": false,
+ "is_snapshot_ready": false,
+ "is_dict_ready": false,
+ "is_flat_table_ready": true,
+ "is_fact_view_ready": false,
+ "multi_partitions": [],
+ "max_bucket_id": -1
+ }
+ ]
+}
\ No newline at end of file
diff --git
a/src/kylin-it/src/test/resources/ut_meta/query_min_max_with_meta/metadata/min_max/dataflow_details/c19de21a-c43a-ad7b-9eec-3928da617025/67ffb57f-8231-0534-93b1-a8239ffd26ab.json
b/src/kylin-it/src/test/resources/ut_meta/query_min_max_with_meta/metadata/min_max/dataflow_details/c19de21a-c43a-ad7b-9eec-3928da617025/67ffb57f-8231-0534-93b1-a8239ffd26ab.json
new file mode 100644
index 0000000000..f5b2868786
--- /dev/null
+++
b/src/kylin-it/src/test/resources/ut_meta/query_min_max_with_meta/metadata/min_max/dataflow_details/c19de21a-c43a-ad7b-9eec-3928da617025/67ffb57f-8231-0534-93b1-a8239ffd26ab.json
@@ -0,0 +1,39 @@
+{
+ "uuid": "67ffb57f-8231-0534-93b1-a8239ffd26ab",
+ "last_modified": 1679548043529,
+ "create_time": 1679548043527,
+ "version": "4.0.0.0",
+ "dataflow": "c19de21a-c43a-ad7b-9eec-3928da617025",
+ "layout_instances": [
+ {
+ "layout_id": 1,
+ "build_job_id":
"150c47f3-caa2-875b-f82a-393f311e9f91-c19de21a-c43a-ad7b-9eec-3928da617025",
+ "rows": 0,
+ "byte_size": 1446,
+ "file_count": 1,
+ "source_rows": 100000,
+ "source_byte_size": 23245413,
+ "partition_num": 1,
+ "partition_values": [],
+ "is_ready": false,
+ "create_time": 1679548393882,
+ "multi_partition": [],
+ "abnormal_type": null
+ },
+ {
+ "layout_id": 20000000001,
+ "build_job_id":
"150c47f3-caa2-875b-f82a-393f311e9f91-c19de21a-c43a-ad7b-9eec-3928da617025",
+ "rows": 0,
+ "byte_size": 1366,
+ "file_count": 1,
+ "source_rows": 100000,
+ "source_byte_size": 23245413,
+ "partition_num": 1,
+ "partition_values": [],
+ "is_ready": false,
+ "create_time": 1679548393883,
+ "multi_partition": [],
+ "abnormal_type": null
+ }
+ ]
+}
\ No newline at end of file
diff --git
a/src/kylin-it/src/test/resources/ut_meta/query_min_max_with_meta/metadata/min_max/index_plan/c19de21a-c43a-ad7b-9eec-3928da617025.json
b/src/kylin-it/src/test/resources/ut_meta/query_min_max_with_meta/metadata/min_max/index_plan/c19de21a-c43a-ad7b-9eec-3928da617025.json
new file mode 100644
index 0000000000..a483faedd8
--- /dev/null
+++
b/src/kylin-it/src/test/resources/ut_meta/query_min_max_with_meta/metadata/min_max/index_plan/c19de21a-c43a-ad7b-9eec-3928da617025.json
@@ -0,0 +1,63 @@
+{
+ "uuid" : "c19de21a-c43a-ad7b-9eec-3928da617025",
+ "last_modified" : 1679548043511,
+ "create_time" : 1679548043386,
+ "version" : "4.0.0.0",
+ "description" : null,
+ "rule_based_index" : null,
+ "indexes" : [ {
+ "id" : 0,
+ "dimensions" : [ 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14,15, 16,
17, 18 ],
+ "measures" : [ 100000 ],
+ "layouts" : [ {
+ "id" : 1,
+ "name" : null,
+ "owner" : null,
+ "col_order" : [ 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15,
16, 17, 18, 100000 ],
+ "shard_by_columns" : [ ],
+ "partition_by_columns" : [ ],
+ "sort_by_columns" : [ ],
+ "storage_type" : 20,
+ "update_time" : 1679548043409,
+ "manual" : false,
+ "auto" : false,
+ "base" : true,
+ "draft_version" : null,
+ "index_range" : null
+ } ],
+ "next_layout_offset" : 2
+ }, {
+ "id" : 20000000000,
+ "dimensions" : [ 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16,
17, 18],
+ "measures" : [ ],
+ "layouts" : [ {
+ "id" : 20000000001,
+ "name" : null,
+ "owner" : null,
+ "col_order" : [ 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15,
16, 17, 18 ],
+ "shard_by_columns" : [ ],
+ "partition_by_columns" : [ ],
+ "sort_by_columns" : [ ],
+ "storage_type" : 20,
+ "update_time" : 1679548043411,
+ "manual" : false,
+ "auto" : false,
+ "base" : true,
+ "draft_version" : null,
+ "index_range" : null
+ } ],
+ "next_layout_offset" : 2
+ } ],
+ "override_properties" : { },
+ "to_be_deleted_indexes" : [ ],
+ "auto_merge_time_ranges" : null,
+ "retention_range" : 0,
+ "engine_type" : 80,
+ "next_aggregation_index_id" : 10000,
+ "next_table_index_id" : 20000010000,
+ "agg_shard_by_columns" : [ ],
+ "extend_partition_columns" : [ ],
+ "layout_bucket_num" : { },
+ "approved_additional_recs" : 0,
+ "approved_removal_recs" : 0
+}
\ No newline at end of file
diff --git
a/src/kylin-it/src/test/resources/ut_meta/query_min_max_with_meta/metadata/min_max/model_desc/c19de21a-c43a-ad7b-9eec-3928da617025.json
b/src/kylin-it/src/test/resources/ut_meta/query_min_max_with_meta/metadata/min_max/model_desc/c19de21a-c43a-ad7b-9eec-3928da617025.json
new file mode 100644
index 0000000000..9850e032de
--- /dev/null
+++
b/src/kylin-it/src/test/resources/ut_meta/query_min_max_with_meta/metadata/min_max/model_desc/c19de21a-c43a-ad7b-9eec-3928da617025.json
@@ -0,0 +1,181 @@
+{
+ "uuid": "c19de21a-c43a-ad7b-9eec-3928da617025",
+ "last_modified": 1679548043383,
+ "create_time": 1679548030348,
+ "version": "4.0.0.0",
+ "alias": "test_measure",
+ "owner": "ADMIN",
+ "config_last_modifier": null,
+ "config_last_modified": 0,
+ "description": "",
+ "fact_table": "TDVT.TEST_MEASURE",
+ "fact_table_alias": null,
+ "management_type": "MODEL_BASED",
+ "join_tables": [],
+ "filter_condition": "",
+ "partition_desc": null,
+ "capacity": "MEDIUM",
+ "segment_config": {
+ "auto_merge_enabled": null,
+ "auto_merge_time_ranges": null,
+ "volatile_range": null,
+ "retention_range": null,
+ "create_empty_segment_enabled": false
+ },
+ "data_check_desc": null,
+ "semantic_version": 0,
+ "storage_type": 0,
+ "model_type": "BATCH",
+ "all_named_columns": [
+ {
+ "id": 0,
+ "name": "PRICE1",
+ "column": "TEST_MEASURE.PRICE1",
+ "status": "DIMENSION"
+ },
+ {
+ "id": 1,
+ "name": "PRICE3",
+ "column": "TEST_MEASURE.PRICE3",
+ "status": "DIMENSION"
+ },
+ {
+ "id": 2,
+ "name": "PRICE2",
+ "column": "TEST_MEASURE.PRICE2",
+ "status": "DIMENSION"
+ },
+ {
+ "id": 3,
+ "name": "DATE1",
+ "column": "TEST_MEASURE.DATE1",
+ "status": "DIMENSION"
+ },
+ {
+ "id": 4,
+ "name": "TIME1",
+ "column": "TEST_MEASURE.TIME1",
+ "status": "DIMENSION"
+ },
+ {
+ "id": 5,
+ "name": "ID4",
+ "column": "TEST_MEASURE.ID4",
+ "status": "DIMENSION"
+ },
+ {
+ "id": 6,
+ "name": "ID3",
+ "column": "TEST_MEASURE.ID3",
+ "status": "DIMENSION"
+ },
+ {
+ "id": 7,
+ "name": "NAME3",
+ "column": "TEST_MEASURE.NAME3",
+ "status": "DIMENSION"
+ },
+ {
+ "id": 8,
+ "name": "FLAG",
+ "column": "TEST_MEASURE.FLAG",
+ "status": "DIMENSION"
+ },
+ {
+ "id": 9,
+ "name": "ID2",
+ "column": "TEST_MEASURE.ID2",
+ "status": "DIMENSION"
+ },
+ {
+ "id": 10,
+ "name": "NAME2",
+ "column": "TEST_MEASURE.NAME2",
+ "status": "DIMENSION"
+ },
+ {
+ "id": 11,
+ "name": "ID1",
+ "column": "TEST_MEASURE.ID1",
+ "status": "DIMENSION"
+ },
+ {
+ "id": 12,
+ "name": "NAME1",
+ "column": "TEST_MEASURE.NAME1",
+ "status": "DIMENSION"
+ },
+ {
+ "id": 13,
+ "name": "ID_NULL",
+ "column": "TEST_MEASURE.ID_NULL",
+ "status": "DIMENSION"
+ },
+ {
+ "id": 14,
+ "name": "DATE_NULL",
+ "column": "TEST_MEASURE.DATE_NULL",
+ "status": "DIMENSION"
+ },
+ {
+ "id": 15,
+ "name": "TIMESTAMP_NULL",
+ "column": "TEST_MEASURE.TIMESTAMP_NULL",
+ "status": "DIMENSION"
+ },
+ {
+ "id": 16,
+ "name": "STRING_NULL",
+ "column": "TEST_MEASURE.STRING_NULL",
+ "status": "DIMENSION"
+ },
+ {
+ "id": 17,
+ "name": "FLAG_NULL",
+ "column": "TEST_MEASURE.FLAG_NULL",
+ "status": "DIMENSION"
+ },
+ {
+ "id": 18,
+ "name": "DOUBLE_NULL",
+ "column": "TEST_MEASURE.DOUBLE_NULL",
+ "status": "DIMENSION"
+ }
+ ],
+ "all_measures": [
+ {
+ "name": "COUNT_ALL",
+ "function": {
+ "expression": "COUNT",
+ "parameters": [
+ {
+ "type": "constant",
+ "value": "1"
+ }
+ ],
+ "returntype": "bigint"
+ },
+ "column": null,
+ "comment": null,
+ "id": 100000,
+ "type": "NORMAL",
+ "internal_ids": []
+ }
+ ],
+ "recommendations_count": 0,
+ "computed_columns": [],
+ "canvas": {
+ "coordinate": {
+ "TEST_MEASURE": {
+ "x": 391.33327907986114,
+ "y": 238.99997287326389,
+ "width": 200.0,
+ "height": 230.0
+ }
+ },
+ "zoom": 9.0
+ },
+ "multi_partition_desc": null,
+ "multi_partition_key_mapping": null,
+ "fusion_id": null
+}
\ No newline at end of file
diff --git
a/src/kylin-it/src/test/resources/ut_meta/query_min_max_with_meta/metadata/min_max/table/TDVT.TEST_MEASURE.json
b/src/kylin-it/src/test/resources/ut_meta/query_min_max_with_meta/metadata/min_max/table/TDVT.TEST_MEASURE.json
new file mode 100644
index 0000000000..6a8dde60d0
--- /dev/null
+++
b/src/kylin-it/src/test/resources/ut_meta/query_min_max_with_meta/metadata/min_max/table/TDVT.TEST_MEASURE.json
@@ -0,0 +1,143 @@
+{
+ "uuid": "51f40fcc-ab67-8654-0ff1-3619b5072d78",
+ "last_modified": 0,
+ "create_time": 1679547981840,
+ "version": "4.0.0.0",
+ "name": "TEST_MEASURE",
+ "columns": [
+ {
+ "id": "1",
+ "name": "ID1",
+ "datatype": "bigint",
+ "case_sensitive_name": "id1"
+ },
+ {
+ "id": "2",
+ "name": "ID2",
+ "datatype": "integer",
+ "case_sensitive_name": "id2"
+ },
+ {
+ "id": "3",
+ "name": "ID3",
+ "datatype": "smallint",
+ "case_sensitive_name": "id3"
+ },
+ {
+ "id": "4",
+ "name": "ID4",
+ "datatype": "tinyint",
+ "case_sensitive_name": "id4"
+ },
+ {
+ "id": "5",
+ "name": "PRICE1",
+ "datatype": "double",
+ "case_sensitive_name": "price1"
+ },
+ {
+ "id": "6",
+ "name": "PRICE2",
+ "datatype": "double",
+ "case_sensitive_name": "price2"
+ },
+ {
+ "id": "7",
+ "name": "PRICE3",
+ "datatype": "decimal(19,4)",
+ "case_sensitive_name": "price3"
+ },
+ {
+ "id": "8",
+ "name": "NAME1",
+ "datatype": "varchar(4096)",
+ "case_sensitive_name": "name1"
+ },
+ {
+ "id": "9",
+ "name": "NAME2",
+ "datatype": "varchar(256)",
+ "case_sensitive_name": "name2"
+ },
+ {
+ "id": "10",
+ "name": "NAME3",
+ "datatype": "char(10)",
+ "case_sensitive_name": "name3"
+ },
+ {
+ "id": "11",
+ "name": "DATE1",
+ "datatype": "date",
+ "case_sensitive_name": "date1"
+ },
+ {
+ "id": "12",
+ "name": "TIME1",
+ "datatype": "timestamp",
+ "case_sensitive_name": "time1"
+ },
+ {
+ "id": "13",
+ "name": "FLAG",
+ "datatype": "boolean",
+ "case_sensitive_name": "flag"
+ },
+ {
+ "id": "14",
+ "name": "ID_NULL",
+ "datatype": "int",
+ "case_sensitive_name": "id_null"
+ },
+ {
+ "id": "15",
+ "name": "DATE_NULL",
+ "datatype": "date",
+ "case_sensitive_name": "date_null"
+ },
+ {
+ "id": "16",
+ "name": "TIMESTAMP_NULL",
+ "datatype": "timestamp",
+ "case_sensitive_name": "timestamp_null"
+ },
+ {
+ "id": "17",
+ "name": "STRING_NULL",
+ "datatype": "string",
+ "case_sensitive_name": "string_null"
+ },
+ {
+ "id": "18",
+ "name": "FLAG_NULL",
+ "datatype": "boolean",
+ "case_sensitive_name": "flag_null"
+ },
+ {
+ "id": "19",
+ "name": "DOUBLE_NULL",
+ "datatype": "double",
+ "case_sensitive_name": "double_null"
+ }
+ ],
+ "source_type": 9,
+ "table_type": "MANAGED",
+ "top": false,
+ "increment_loading": false,
+ "last_snapshot_path": null,
+ "last_snapshot_size": 0,
+ "snapshot_last_modified": 0,
+ "query_hit_count": 0,
+ "partition_column": null,
+ "snapshot_partitions": {},
+ "snapshot_partitions_info": {},
+ "snapshot_total_rows": 0,
+ "snapshot_partition_col": null,
+ "selected_snapshot_partition_col": null,
+ "temp_snapshot_path": null,
+ "snapshot_has_broken": false,
+ "database": "TDVT",
+ "transactional": false,
+ "rangePartition": false,
+ "partition_desc": null
+}
\ No newline at end of file
diff --git
a/src/query-common/src/main/java/org/apache/kylin/query/relnode/OLAPContext.java
b/src/query-common/src/main/java/org/apache/kylin/query/relnode/OLAPContext.java
index 991a576fa0..2c13fc141b 100644
---
a/src/query-common/src/main/java/org/apache/kylin/query/relnode/OLAPContext.java
+++
b/src/query-common/src/main/java/org/apache/kylin/query/relnode/OLAPContext.java
@@ -653,8 +653,14 @@ public class OLAPContext {
String dataType =
col.getColumnDesc().getUpgradedType().getName();
int colId = allFields.indexOf(col);
String tblColRefIndex = getTblColRefIndex(col, realization);
- minList[colId] =
Tuple.convertOptiqCellValue(infoMap.get(tblColRefIndex).getMin(), dataType);
- maxList[colId] =
Tuple.convertOptiqCellValue(infoMap.get(tblColRefIndex).getMax(), dataType);
+ DimensionRangeInfo dimensionRangeInfo =
infoMap.get(tblColRefIndex);
+ if (dimensionRangeInfo == null) {
+ minList[colId] = null;
+ maxList[colId] = null;
+ } else {
+ minList[colId] =
Tuple.convertOptiqCellValue(dimensionRangeInfo.getMin(), dataType);
+ maxList[colId] =
Tuple.convertOptiqCellValue(dimensionRangeInfo.getMax(), dataType);
+ }
}
result.add(minList);
diff --git
a/src/spark-project/sparder/src/main/scala/org/apache/kylin/query/runtime/plan/TableScanPlan.scala
b/src/spark-project/sparder/src/main/scala/org/apache/kylin/query/runtime/plan/TableScanPlan.scala
index 6d30326e70..544f20c2c2 100644
---
a/src/spark-project/sparder/src/main/scala/org/apache/kylin/query/runtime/plan/TableScanPlan.scala
+++
b/src/spark-project/sparder/src/main/scala/org/apache/kylin/query/runtime/plan/TableScanPlan.scala
@@ -31,12 +31,12 @@ import org.apache.kylin.metadata.tuple.TupleInfo
import org.apache.kylin.query.implicits.sessionToQueryContext
import org.apache.kylin.query.relnode.{KapRel, OLAPContext}
import org.apache.kylin.query.util.{RuntimeHelper, SparderDerivedUtil}
+import org.apache.spark.sql.{Column, DataFrame, Row, SparderEnv,
SparkOperation, SparkSession}
import org.apache.spark.sql.execution.utils.SchemaProcessor
import org.apache.spark.sql.functions.col
import org.apache.spark.sql.manager.SparderLookupManager
-import org.apache.spark.sql.types._
+import org.apache.spark.sql.types.{ArrayType, DataTypes, DoubleType,
StringType, StructField, StructType}
import org.apache.spark.sql.util.SparderTypeUtil
-import org.apache.spark.sql._
import java.util.concurrent.ConcurrentHashMap
import java.{lang, util}
@@ -103,7 +103,13 @@ object TableScanPlan extends LogEx {
allFields.forEach(col => {
try {
val dataType = col.getColumnDesc.getUpgradedType
- val spaType = SparderTypeUtil.kylinTypeToSparkResultType(dataType)
+ val spaType = if (dataType.isDate) {
+ DataTypes.DateType
+ } else if (dataType.isDateTimeFamily) {
+ DataTypes.TimestampType
+ } else {
+ SparderTypeUtil.kylinTypeToSparkResultType(dataType)
+ }
structTypes.add(StructField(col.getIdentity.replace(".", "_"),
spaType))
} catch {
// some dataTypes are not support in sparder, such as 'any',
diff --git
a/src/spark-project/spark-common/src/main/scala/org/apache/spark/sql/SparderTypeUtil.scala
b/src/spark-project/spark-common/src/main/scala/org/apache/spark/sql/SparderTypeUtil.scala
index 31022d8a77..90981d3d64 100644
---
a/src/spark-project/spark-common/src/main/scala/org/apache/spark/sql/SparderTypeUtil.scala
+++
b/src/spark-project/spark-common/src/main/scala/org/apache/spark/sql/SparderTypeUtil.scala
@@ -129,9 +129,9 @@ object SparderTypeUtil extends Logging {
dataTp.getName match {
case "decimal" => new java.math.BigDecimal(s.toString)
case "date" => new java.sql.Date(DateFormat.stringToMillis(s.toString))
- case "time" => new Timestamp(DateFormat.stringToMillis(s.toString))
- case "timestamp" => new
Timestamp(DateFormat.stringToMillis(s.toString))
- case "datetime" => new
java.sql.Date(DateFormat.stringToMillis(s.toString))
+ case "time" | "timestamp" | "datetime" =>
+ val l = java.lang.Long.parseLong(s.toString)
+ Timestamp.valueOf(DateFormat.castTimestampToString(l))
case "tinyint" => s.toString.toByte
case "smallint" => s.toString.toShort
case "integer" => s.toString.toInt
