zhangstar333 commented on code in PR #59720: URL: https://github.com/apache/doris/pull/59720#discussion_r2697603796
########## regression-test/suites/external_table_p0/iceberg/test_iceberg_export_timestamp_tz.groovy: ########## @@ -0,0 +1,168 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +import org.codehaus.groovy.runtime.IOGroovyMethods + +import java.nio.charset.StandardCharsets +import java.nio.file.Files +import java.nio.file.Paths + +suite("test_iceberg_export_timestamp_tz", "external,hive,external_docker") { + + String enabled = context.config.otherConfigs.get("enableHiveTest") + if (enabled == null || !enabled.equalsIgnoreCase("true")) { + logger.info("diable Hive test.") + return; + } + + for (String hivePrefix : ["hive2"]) { + setHivePrefix(hivePrefix) + String hms_port = context.config.otherConfigs.get(hivePrefix + "HmsPort") + String hdfs_port = context.config.otherConfigs.get(hivePrefix + "HdfsPort") + String externalEnvIp = context.config.otherConfigs.get("externalEnvIp") + + // It's okay to use random `hdfsUser`, but can not be empty. + def hdfsUserName = "doris" + def defaultFS = "hdfs://${externalEnvIp}:${hdfs_port}" + def outfile_path = "/user/doris/tmp_data" + def uri = "${defaultFS}" + "${outfile_path}/exp_" + + def outfile_to_HDFS = {format,export_table_name -> + // select ... into outfile ... + def uuid = UUID.randomUUID().toString() + outfile_path = "/user/doris/tmp_data/${uuid}" + uri = "${defaultFS}" + "${outfile_path}/exp_" + + def res = sql """ + SELECT * FROM ${export_table_name} t ORDER BY id + INTO OUTFILE "${uri}" + FORMAT AS ${format} + PROPERTIES ( + "fs.defaultFS"="${defaultFS}", + "hadoop.username" = "${hdfsUserName}" + ); + """ + logger.info("outfile success path: " + res[0][3]); + return res[0][3] + } + + try { + String catalog_name_with_export = "test_iceberg_timestamp_tz_with_mapping_export" + String db_name = "test_timestamp_tz" + String rest_port = context.config.otherConfigs.get("iceberg_rest_uri_port") + String minio_port = context.config.otherConfigs.get("iceberg_minio_port") + sql """set time_zone = 'Asia/Shanghai';""" + + sql """drop catalog if exists ${catalog_name_with_export}""" + sql """ + CREATE CATALOG ${catalog_name_with_export} PROPERTIES ( + 'type'='iceberg', + 'iceberg.catalog.type'='rest', + 'uri' = 'http://${externalEnvIp}:${rest_port}', + "s3.access_key" = "admin", + "s3.secret_key" = "password", + "s3.endpoint" = "http://${externalEnvIp}:${minio_port}";, + "s3.region" = "us-east-1", + "s3.path.style.access" = "true", + "s3.connection.ssl.enabled" = "false", + "enable.mapping.varbinary"="true", + "enable.mapping.timestamp_tz"="true" + );""" + + + sql """switch ${catalog_name_with_export}""" + sql """use ${db_name}""" + order_qt_select_desc_orc """ desc test_ice_timestamp_tz_orc; """ + order_qt_select_desc_parquet """ desc test_ice_timestamp_tz_parquet; """ + // TODO: seems write to parquet with timestamp_tz has some problem Review Comment: parquet maybe write int96 or int64 have find the solution by add "enable_int96_timestamps" -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: [email protected] For queries about this service, please contact Infrastructure at: [email protected] --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
