morningman commented on code in PR #17372: URL: https://github.com/apache/doris/pull/17372#discussion_r1126217232
########## regression-test/suites/external_catalog_p0/hive/test_different_parquet_types.groovy: ########## @@ -0,0 +1,230 @@ +package suites.external_catalog_p0.hive +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +suite("test_different_parquet_types", "p0") { + + String hms_port = context.config.otherConfigs.get("hms_port") + String hdfs_port = context.config.otherConfigs.get("hdfs_port") + + // 发现问题 1 ： hive执行select * from delta_byte_array limit 10可以查出数据，但是doris无法查出数据 + def q01 = { + def res1_1 = sql """ + select * from delta_byte_array limit 10 + """ + logger.info("record res" + res1_1.toString()) + + def res1_2 = sql """ + select count(*) from delta_byte_array + """ + logger.info("record res" + res1_2.toString()) + + def res1_3 = sql """ + select * from hdfs(\"uri" = \"hdfs://127.0.0.1:${hdfs_port}/user/doris/preinstalled_data/different_types_parquet/delta_byte_array/delta_byte_array.parquet\",\"fs.defaultFS\" = \"hdfs://127.0.0.1:${hdfs_port}\",\"format\" = \"parquet\") limit 10 + """ + logger.info("record res" + res1_3.toString()) + } + + // hive加载数据报错 此用例无效 + // def q02 = { + // def res2_1 = sql """ + // select * from delta_length_byte_array limit 10; + // """ + // logger.info("record res" + res2_1.toString()) + + // def res2_2 = sql """ + // select count(*) from delta_length_byte_array; + // """ + // logger.info("record res" + res2_2.toString()) + + // def res2_3 = sql """ + // select * from hdfs(\"uri" = \"hdfs://127.0.0.1:${hdfs_port}/user/doris/preinstalled_data/different_types_parquet/delta_length_byte_array/delta_length_byte_array.parquet\",\"fs.defaultFS\" = \"hdfs://127.0.0.1:${hdfs_port}\",\"format\" = \"parquet\") limit 10 + // """ + // logger.info("record res" + res2_3.toString()) + // } + + + // 发现问题二： hive查询出的数据全为空，doris catalog查询报错 tvf不报错 返回为空 + def q03 = { //hive查询出的数据全为空 + + //查询报错 [INTERNAL_ERROR]Only support csv data in utf8 codec + + def res3_1 = sql """ + select * from delta_binary_packed limit 10; + """ + logger.info("record res" + res3_1.toString()) + + //查询报错 [INTERNAL_ERROR]Only support csv data in utf8 codec + + def res3_2 = sql """ + select count(*) from delta_binary_packed; + """ + logger.info("record res" + res3_1.toString()) + + //查询返回数据为空 不报错 + def res3_3 = sql """ + select * from hdfs(\"uri" = \"hdfs://127.0.0.1:${hdfs_port}/user/doris/preinstalled_data/different_types_parquet/delta_binary_packed/delta_binary_packed.parquet\",\"fs.defaultFS\" = \"hdfs://127.0.0.1:${hdfs_port}\",\"format\" = \"parquet\") limit 10 + """ + logger.info("record res" + res3_3.toString()) + } + + def q04 = { + def res4_1 = sql """ + select * from delta_encoding_required_column limit 10; + """ + logger.info("record res" + res4_1.toString()) + + def res4_2 = sql """ + select count(*) from delta_encoding_required_column; + """ + logger.info("record res" + res4_2.toString()) + + //发现问题 Can not get first file, please check uri. + def res4_3 = sql """ + select * from hdfs(\"uri" = \"hdfs://127.0.0.1:${hdfs_port}/user/doris/preinstalled_data/different_types_parquet/delta_encoding_required_column/delta_encoding_required_column.parquet\",\"fs.defaultFS\" = \"hdfs://127.0.0.1:${hdfs_port}\",\"format\" = \"parquet\") limit 10 + """ + logger.info("record res" + res4_3.toString()) + } + + // 发现问题三： hive查询报错 doris查询全部为空 + def q05 = { + def res5_1 = sql """ + select * from delta_encoding_optional_column limit 10; + """ + logger.info("record res" + res5_1.toString()) + + + def res5_2 = sql """ + select count(*) from delta_encoding_optional_column; + """ + logger.info("record res" + res5_2.toString()) + + def res5_3 = sql """ + select * from hdfs(\"uri" = \"hdfs://127.0.0.1:${hdfs_port}/user/doris/preinstalled_data/different_types_parquet/delta_encoding_optional_column/delta_encoding_optional_column.parquet\",\"fs.defaultFS\" = \"hdfs://127.0.0.1:${hdfs_port}\",\"format\" = \"parquet\") limit 10 + """ + logger.info("record res" + res5_3.toString()) + } + + + // 发现问题四：tvf查询报错 Can not get first file, please check uri. + def q06 = { + def res6_1 = sql """ + select * from datapage_v1_snappy_compressed_checksum limit 10; + """ + logger.info("record res" + res6_1.toString()) + + def res6_2 = sql """ + select count(*) from datapage_v1_snappy_compressed_checksum; + """ + logger.info("record res" + res6_2.toString()) + + // Can not get first file, please check uri. + def res6_3 = sql """ + select * from hdfs(\"uri" = \"hdfs://127.0.0.1:${hdfs_port}/user/doris/preinstalled_data/different_types_parquet/datapage_v1_snappy_compressed_checksum/datapage_v1_snappy_compressed_checksum.parquet\",\"fs.defaultFS\" = \"hdfs://127.0.0.1:${hdfs_port}\",\"format\" = \"parquet\") limit 10 + """ + logger.info("record res" + res6_3.toString()) + + } + + //通过 + def q07 = { + def res7_1 = sql """ + select * from overflow_i16_page_cnt limit 10; + """ + logger.info("record res" + res7_1.toString()) + + def res7_2 = sql """ + select count(*) from overflow_i16_page_cnt; + """ + logger.info("record res" + res7_2.toString()) + + def res7_3 = sql """ + select * from hdfs(\"uri" = \"hdfs://127.0.0.1:${hdfs_port}/user/doris/preinstalled_data/different_types_parquet/overflow_i16_page_cnt/overflow_i16_page_cnt.parquet\",\"fs.defaultFS\" = \"hdfs://127.0.0.1:${hdfs_port}\",\"format\" = \"parquet\") limit 10 + """ + logger.info("record res" + res7_3.toString()) + } + + //通过 + def q08 = { + def res8_1 = sql """ + select * from alltypes_tiny_pages limit 10; + """ + logger.info("record res" + res8_1.toString()) + + + def res8_2 = sql """ + select count(*) from alltypes_tiny_pages limit 10; + """ + logger.info("record res" + res8_2.toString()) + + def res8_3 = sql """ + select * from hdfs(\"uri" = \"hdfs://127.0.0.1:${hdfs_port}/user/doris/preinstalled_data/different_types_parquet/alltypes_tiny_pages/alltypes_tiny_pages.parquet\",\"fs.defaultFS\" = \"hdfs://127.0.0.1:${hdfs_port}\",\"format\" = \"parquet\") limit 10 + """ + logger.info("record res" + res8_3.toString()) + } + //通过 Review Comment: Use English please -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org For queries about this service, please contact Infrastructure at: us...@infra.apache.org --------------------------------------------------------------------- To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org For additional commands, e-mail: commits-h...@doris.apache.org
