This is an automated email from the ASF dual-hosted git repository.
morrysnow pushed a commit to branch branch-3.1
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/branch-3.1 by this push:
new b14cf27cb48 branch-3.1: [opt](catalog) add error msg for catalog init
and timezone info of column #54825 #55233 (#55231)
b14cf27cb48 is described below
commit b14cf27cb48e266dc8a0b914a2169d2889abbddf
Author: Mingyu Chen (Rayner) <[email protected]>
AuthorDate: Sun Aug 24 20:02:06 2025 -0700
branch-3.1: [opt](catalog) add error msg for catalog init and timezone info
of column #54825 #55233 (#55231)
bp #54825 #55233
---
.../create_preinstalled_scripts/paimon/run08.sql | 5 +-
.../org/apache/doris/analysis/ShowCatalogStmt.java | 1 +
.../main/java/org/apache/doris/catalog/Column.java | 12 +++++
.../doris/common/proc/IndexSchemaProcNode.java | 4 ++
.../org/apache/doris/datasource/CatalogIf.java | 4 ++
.../org/apache/doris/datasource/CatalogMgr.java | 6 +--
.../apache/doris/datasource/ExternalCatalog.java | 16 ++++++
.../doris/datasource/iceberg/IcebergUtils.java | 6 +++
.../datasource/paimon/PaimonExternalTable.java | 4 ++
.../tablefunction/PaimonTableValuedFunction.java | 1 -
.../apache/doris/analysis/ShowCatalogStmtTest.java | 4 +-
.../write/test_iceberg_write_timestamp_ntz.out | Bin 257 -> 358 bytes
.../test_paimon_timestamp_with_time_zone.out | Bin 483 -> 595 bytes
.../write/test_iceberg_write_timestamp_ntz.groovy | 3 ++
.../test_paimon_timestamp_with_time_zone.groovy | 2 +
....groovy => test_show_catalogs_error_msg.groovy} | 55 +++++++++++++--------
16 files changed, 95 insertions(+), 28 deletions(-)
diff --git
a/docker/thirdparties/docker-compose/iceberg/scripts/create_preinstalled_scripts/paimon/run08.sql
b/docker/thirdparties/docker-compose/iceberg/scripts/create_preinstalled_scripts/paimon/run08.sql
index 073d26548a0..06bc3fc91ae 100644
---
a/docker/thirdparties/docker-compose/iceberg/scripts/create_preinstalled_scripts/paimon/run08.sql
+++
b/docker/thirdparties/docker-compose/iceberg/scripts/create_preinstalled_scripts/paimon/run08.sql
@@ -4,7 +4,8 @@ use test_paimon_spark;
SET TIME ZONE '+08:00';
-CREATE TABLE IF NOT EXISTS t_ts_ntz (
+DROP TABLE IF EXISTS t_ts_ntz;
+CREATE TABLE t_ts_ntz (
id INT,
ts TIMESTAMP,
ts_ntz TIMESTAMP_NTZ
@@ -12,4 +13,4 @@ CREATE TABLE IF NOT EXISTS t_ts_ntz (
INSERT INTO t_ts_ntz VALUES
(1, CAST('2025-08-12 06:00:00+00:00' AS TIMESTAMP), CAST('2025-08-12
06:00:00' AS TIMESTAMP_NTZ)),
- (2, CAST('2025-08-12 14:00:00+08:00' AS TIMESTAMP), CAST('2025-08-12
14:00:00' AS TIMESTAMP_NTZ));
\ No newline at end of file
+ (2, CAST('2025-08-12 14:00:00+08:00' AS TIMESTAMP), CAST('2025-08-12
14:00:00' AS TIMESTAMP_NTZ));
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/analysis/ShowCatalogStmt.java
b/fe/fe-core/src/main/java/org/apache/doris/analysis/ShowCatalogStmt.java
index f1770859002..ef8289745db 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/analysis/ShowCatalogStmt.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/ShowCatalogStmt.java
@@ -34,6 +34,7 @@ public class ShowCatalogStmt extends ShowStmt implements
NotFallbackInParser {
.addColumn(new Column("CreateTime",
ScalarType.createStringType()))
.addColumn(new Column("LastUpdateTime",
ScalarType.createStringType()))
.addColumn(new Column("Comment",
ScalarType.createStringType()))
+ .addColumn(new Column("ErrorMsg",
ScalarType.createStringType()))
.build();
private static final ShowResultSetMetaData META_DATA_SPECIFIC =
diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/Column.java
b/fe/fe-core/src/main/java/org/apache/doris/catalog/Column.java
index b92114369f3..47cca3b90bb 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/catalog/Column.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/Column.java
@@ -162,6 +162,11 @@ public class Column implements GsonPostProcessable {
@SerializedName(value = "fpt")
private TPatternType fieldPatternType;
+ // used for saving some extra information, such as timezone info of
datetime column
+ // Maybe deprecated if we implement real timestamp with timezone type.
+ @SerializedName(value = "ei")
+ private String extraInfo;
+
public Column() {
this.name = "";
this.type = Type.NULL;
@@ -1213,6 +1218,10 @@ public class Column implements GsonPostProcessable {
this.uniqueId = colUniqueId;
}
+ public void setWithTZExtraInfo() {
+ this.extraInfo = Strings.isNullOrEmpty(extraInfo) ? "WITH_TIMEZONE" :
extraInfo + ", WITH_TIMEZONE";
+ }
+
public int getUniqueId() {
return this.uniqueId;
}
@@ -1280,4 +1289,7 @@ public class Column implements GsonPostProcessable {
this.realDefaultValue = refColumn.realDefaultValue;
}
+ public String getExtraInfo() {
+ return extraInfo;
+ }
}
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/common/proc/IndexSchemaProcNode.java
b/fe/fe-core/src/main/java/org/apache/doris/common/proc/IndexSchemaProcNode.java
index 31373ac781c..3630dac0c57 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/common/proc/IndexSchemaProcNode.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/common/proc/IndexSchemaProcNode.java
@@ -23,6 +23,7 @@ import org.apache.doris.common.FeConstants;
import org.apache.doris.qe.ConnectContext;
import com.google.common.base.Preconditions;
+import com.google.common.base.Strings;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.Lists;
import org.apache.commons.lang3.StringUtils;
@@ -80,6 +81,9 @@ public class IndexSchemaProcNode implements ProcNodeInterface
{
if (column.getGeneratedColumnInfo() != null) {
extras.add("STORED GENERATED");
}
+ if (!Strings.isNullOrEmpty(column.getExtraInfo())) {
+ extras.add(column.getExtraInfo());
+ }
String extraStr = StringUtils.join(extras, ",");
String comment = column.getComment();
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/datasource/CatalogIf.java
b/fe/fe-core/src/main/java/org/apache/doris/datasource/CatalogIf.java
index 646510e01d0..e9a346a9102 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/datasource/CatalogIf.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/CatalogIf.java
@@ -62,6 +62,10 @@ public interface CatalogIf<T extends DatabaseIf> {
List<String> getDbNames();
+ default String getErrorMsg() {
+ return "";
+ }
+
default boolean isInternalCatalog() {
return this instanceof InternalCatalog;
}
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/datasource/CatalogMgr.java
b/fe/fe-core/src/main/java/org/apache/doris/datasource/CatalogMgr.java
index 7ee865576b9..2b863b7e113 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/datasource/CatalogMgr.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/CatalogMgr.java
@@ -66,6 +66,7 @@ import java.io.DataInput;
import java.io.DataOutput;
import java.io.IOException;
import java.util.Arrays;
+import java.util.Comparator;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
@@ -386,12 +387,11 @@ public class CatalogMgr implements Writable,
GsonPostProcessable {
row.add(createTime);
row.add(TimeUtils.longToTimeString(catalog.getLastUpdateTime()));
row.add(catalog.getComment());
+ row.add(Strings.nullToEmpty(catalog.getErrorMsg()));
rows.add(row);
// sort by catalog name
- rows.sort((x, y) -> {
- return x.get(1).compareTo(y.get(1));
- });
+ rows.sort(Comparator.comparing(x -> x.get(1)));
}
} else {
if (!nameToCatalog.containsKey(showStmt.getCatalogName())) {
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalCatalog.java
b/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalCatalog.java
index abee493b7f4..156deefca72 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalCatalog.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalCatalog.java
@@ -82,6 +82,7 @@ import com.google.common.collect.Sets;
import com.google.gson.annotations.SerializedName;
import org.apache.commons.lang3.NotImplementedException;
import org.apache.commons.lang3.StringUtils;
+import org.apache.commons.lang3.exception.ExceptionUtils;
import org.apache.commons.lang3.math.NumberUtils;
import org.apache.hadoop.conf.Configuration;
import org.apache.logging.log4j.LogManager;
@@ -161,6 +162,11 @@ public abstract class ExternalCatalog
@SerializedName(value = "comment")
private String comment;
+ // Save the error info if initialization fails.
+ // can be seen in `show catalogs` result.
+ // no need to persist this field.
+ private String errorMsg = "";
+
// db name does not contains "default_cluster"
protected Map<String, Long> dbNameToId = Maps.newConcurrentMap();
private boolean objectCreated = false;
@@ -329,7 +335,12 @@ public abstract class ExternalCatalog
init();
}
initialized = true;
+ this.errorMsg = "";
}
+ } catch (Exception e) {
+ LOG.warn("failed to init catalog {}:{}", name, id, e);
+ this.errorMsg = ExceptionUtils.getRootCauseMessage(e);
+ throw new RuntimeException("Failed to init catalog: " + name + ",
error: " + this.errorMsg, e);
} finally {
isInitializing = false;
}
@@ -631,6 +642,11 @@ public abstract class ExternalCatalog
this.comment = comment;
}
+ @Override
+ public String getErrorMsg() {
+ return errorMsg;
+ }
+
/**
* Different from 'listDatabases()', this method will return dbnames from
cache.
* while 'listDatabases()' will return dbnames from remote datasource.
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergUtils.java
b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergUtils.java
index f14d5c69b3e..0bba739c179 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergUtils.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergUtils.java
@@ -748,6 +748,12 @@ public class IcebergUtils {
IcebergUtils.icebergTypeToDorisType(field.type()),
true, null,
true, field.doc(), true, -1);
updateIcebergColumnUniqueId(column, field);
+ if (field.type().isPrimitiveType() && field.type().typeId() ==
TypeID.TIMESTAMP) {
+ Types.TimestampType timestampType = (Types.TimestampType)
field.type();
+ if (timestampType.shouldAdjustToUTC()) {
+ column.setWithTZExtraInfo();
+ }
+ }
resSchema.add(column);
}
return resSchema;
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonExternalTable.java
b/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonExternalTable.java
index 9f492123ea1..bdb45a0eb9f 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonExternalTable.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonExternalTable.java
@@ -60,6 +60,7 @@ import org.apache.paimon.table.DataTable;
import org.apache.paimon.table.Table;
import org.apache.paimon.table.source.Split;
import org.apache.paimon.types.DataField;
+import org.apache.paimon.types.DataTypeRoot;
import java.util.Collections;
import java.util.HashMap;
@@ -249,6 +250,9 @@ public class PaimonExternalTable extends ExternalTable
implements MTMVRelatedTab
PaimonUtil.paimonTypeToDorisType(field.type()), true,
null, true, field.description(), true,
-1);
PaimonUtil.updatePaimonColumnUniqueId(column, field);
+ if (field.type().getTypeRoot() ==
DataTypeRoot.TIMESTAMP_WITH_LOCAL_TIME_ZONE) {
+ column.setWithTZExtraInfo();
+ }
dorisColumns.add(column);
if (partitionColumnNames.contains(field.name())) {
partitionColumns.add(column);
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/tablefunction/PaimonTableValuedFunction.java
b/fe/fe-core/src/main/java/org/apache/doris/tablefunction/PaimonTableValuedFunction.java
index 5fdd3b1846b..0827374e92e 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/tablefunction/PaimonTableValuedFunction.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/tablefunction/PaimonTableValuedFunction.java
@@ -97,7 +97,6 @@ public class PaimonTableValuedFunction extends
MetadataTableValuedFunction {
this.paimonSysTable =
paimonExternalCatalog.getPaimonTable(buildNameMapping,
"main", queryType);
this.schema = PaimonUtil.parseSchema(paimonSysTable);
-
}
public static PaimonTableValuedFunction create(Map<String, String> params)
throws AnalysisException {
diff --git
a/fe/fe-core/src/test/java/org/apache/doris/analysis/ShowCatalogStmtTest.java
b/fe/fe-core/src/test/java/org/apache/doris/analysis/ShowCatalogStmtTest.java
index 56e33f1b1f9..0fc763ca20a 100644
---
a/fe/fe-core/src/test/java/org/apache/doris/analysis/ShowCatalogStmtTest.java
+++
b/fe/fe-core/src/test/java/org/apache/doris/analysis/ShowCatalogStmtTest.java
@@ -30,14 +30,14 @@ public class ShowCatalogStmtTest {
ShowCatalogStmt stmt = new ShowCatalogStmt();
stmt.analyze(analyzer);
Assert.assertNull(stmt.getCatalogName());
- Assert.assertEquals(7, stmt.getMetaData().getColumnCount());
+ Assert.assertEquals(8, stmt.getMetaData().getColumnCount());
Assert.assertEquals("SHOW CATALOGS", stmt.toSql());
stmt = new ShowCatalogStmt(null, "%hive%");
stmt.analyze(analyzer);
Assert.assertNull(stmt.getCatalogName());
Assert.assertNotNull(stmt.getPattern());
- Assert.assertEquals(7, stmt.getMetaData().getColumnCount());
+ Assert.assertEquals(8, stmt.getMetaData().getColumnCount());
Assert.assertEquals("SHOW CATALOGS LIKE '%hive%'", stmt.toSql());
stmt = new ShowCatalogStmt("testCatalog", null);
diff --git
a/regression-test/data/external_table_p0/iceberg/write/test_iceberg_write_timestamp_ntz.out
b/regression-test/data/external_table_p0/iceberg/write/test_iceberg_write_timestamp_ntz.out
index 1980a167f36..7276e6236df 100644
Binary files
a/regression-test/data/external_table_p0/iceberg/write/test_iceberg_write_timestamp_ntz.out
and
b/regression-test/data/external_table_p0/iceberg/write/test_iceberg_write_timestamp_ntz.out
differ
diff --git
a/regression-test/data/external_table_p0/paimon/test_paimon_timestamp_with_time_zone.out
b/regression-test/data/external_table_p0/paimon/test_paimon_timestamp_with_time_zone.out
index 6c4acf47ef8..020bdd5f8f3 100644
Binary files
a/regression-test/data/external_table_p0/paimon/test_paimon_timestamp_with_time_zone.out
and
b/regression-test/data/external_table_p0/paimon/test_paimon_timestamp_with_time_zone.out
differ
diff --git
a/regression-test/suites/external_table_p0/iceberg/write/test_iceberg_write_timestamp_ntz.groovy
b/regression-test/suites/external_table_p0/iceberg/write/test_iceberg_write_timestamp_ntz.groovy
index 33418c0e247..2a4a04e3956 100644
---
a/regression-test/suites/external_table_p0/iceberg/write/test_iceberg_write_timestamp_ntz.groovy
+++
b/regression-test/suites/external_table_p0/iceberg/write/test_iceberg_write_timestamp_ntz.groovy
@@ -55,6 +55,9 @@ suite("test_iceberg_write_timestamp_ntz",
"p0,external,iceberg,external_docker,e
sql "set time_zone = 'Asia/Shanghai'"
qt_timestamp_ntz """select * from t_ntz_doris;"""
qt_timestamp_tz """select * from t_tz_doris;"""
+ // test Extra column in desc result
+ qt_desc01 """desc t_ntz_doris"""
+ qt_desc02 """desc t_tz_doris"""
sql "set time_zone = 'Europe/Tirane'"
qt_timestamp_ntz2 """select * from t_ntz_doris;"""
diff --git
a/regression-test/suites/external_table_p0/paimon/test_paimon_timestamp_with_time_zone.groovy
b/regression-test/suites/external_table_p0/paimon/test_paimon_timestamp_with_time_zone.groovy
index 68a9a06522c..bcc5f930b32 100644
---
a/regression-test/suites/external_table_p0/paimon/test_paimon_timestamp_with_time_zone.groovy
+++
b/regression-test/suites/external_table_p0/paimon/test_paimon_timestamp_with_time_zone.groovy
@@ -37,6 +37,8 @@ suite("test_paimon_timestamp_with_time_zone",
"p0,external,doris,external_docker
);
"""
sql """use `${catalog_name}`.`${db_name}`;"""
+ // test Extra column in desc result
+ qt_desc_table """desc t_ts_ntz"""
def test_select_timestamp = {
qt_select_timestamp """ select * from t_ts_ntz order by id; """
diff --git
a/regression-test/suites/external_table_p0/iceberg/write/test_iceberg_write_timestamp_ntz.groovy
b/regression-test/suites/external_table_p0/test_show_catalogs_error_msg.groovy
similarity index 51%
copy from
regression-test/suites/external_table_p0/iceberg/write/test_iceberg_write_timestamp_ntz.groovy
copy to
regression-test/suites/external_table_p0/test_show_catalogs_error_msg.groovy
index 33418c0e247..e37bd446590 100644
---
a/regression-test/suites/external_table_p0/iceberg/write/test_iceberg_write_timestamp_ntz.groovy
+++
b/regression-test/suites/external_table_p0/test_show_catalogs_error_msg.groovy
@@ -15,7 +15,7 @@
// specific language governing permissions and limitations
// under the License.
-suite("test_iceberg_write_timestamp_ntz",
"p0,external,iceberg,external_docker,external_docker_iceberg") {
+suite("test_show_catalogs_error_msg",
"p0,external,iceberg,external_docker,external_docker_iceberg") {
String enabled = context.config.otherConfigs.get("enableIcebergTest")
if (enabled == null || !enabled.equalsIgnoreCase("true")) {
@@ -23,43 +23,58 @@ suite("test_iceberg_write_timestamp_ntz",
"p0,external,iceberg,external_docker,e
return
}
-
try {
-
- String rest_port =
context.config.otherConfigs.get("iceberg_rest_uri_port")
+ String rest_port =
context.config.otherConfigs.get("iceberg_rest_uri_port");
String minio_port =
context.config.otherConfigs.get("iceberg_minio_port")
String externalEnvIp = context.config.otherConfigs.get("externalEnvIp")
- String catalog_name = "iceberg_timestamp_ntz_test"
+ String catalog_name = "test_show_catalogs_error_msg"
sql """drop catalog if exists ${catalog_name}"""
+ // use wrong port 181812
sql """
CREATE CATALOG ${catalog_name} PROPERTIES (
'type'='iceberg',
'iceberg.catalog.type'='rest',
- 'uri' = 'http://${externalEnvIp}:${rest_port}',
+ 'uri' = 'http://${externalEnvIp}:181812',
"s3.access_key" = "admin",
"s3.secret_key" = "password",
"s3.endpoint" = "http://${externalEnvIp}:${minio_port}";,
"s3.region" = "us-east-1"
);"""
- logger.info("catalog " + catalog_name + " created")
- sql """switch ${catalog_name};"""
- logger.info("switched to catalog " + catalog_name)
- sql """ use test_db;"""
+ test {
+ sql """show databases from ${catalog_name}"""
+ exception "is out of range"
+ }
- sql """INSERT INTO t_ntz_doris VALUES ('2025-02-07 20:12:00');"""
- sql """INSERT INTO t_tz_doris VALUES ('2025-02-07 20:12:01');"""
+ boolean found = false;
+ List<List<Object>> res = sql """show catalogs"""
+ for (List<Object> line : res) {
+ logger.info("get show catalogs line: " + line + ", name: " +
line[1] + ", msg: " + line[7]);
+ if (line[1].equals("test_show_catalogs_error_msg")) {
+ if (line[7].contains("181812 is out of range")) {
+ found = true;
+ break;
+ }
+ }
+ }
+ assertTrue(found, "failed to find invalid catalog")
-
- sql "set time_zone = 'Asia/Shanghai'"
- qt_timestamp_ntz """select * from t_ntz_doris;"""
- qt_timestamp_tz """select * from t_tz_doris;"""
+ // change to right port, the error msg will be removed
+ sql """alter catalog test_show_catalogs_error_msg set properties('uri'
= 'http://${externalEnvIp}:${rest_port}');"""
+ sql """show databases from test_show_catalogs_error_msg"""
+ res = sql """show catalogs"""
+ for (List<Object> line : res) {
+ logger.info("get show catalogs line: " + line + ", name: " +
line[1] + ", msg: " + line[7]);
+ if (line[1].equals("test_show_catalogs_error_msg")) {
+ if (line[7].isEmpty()) {
+ found = true;
+ break;
+ }
+ }
+ }
+ assertTrue(found, "failed to find valid catalog")
- sql "set time_zone = 'Europe/Tirane'"
- qt_timestamp_ntz2 """select * from t_ntz_doris;"""
- qt_timestamp_tz2 """select * from t_tz_doris;"""
-
// sql """drop catalog if exists ${catalog_name}"""
} finally {
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
