This is an automated email from the ASF dual-hosted git repository.
nextdreamblue pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/master by this push:
new f7d00d467a [fix](multicatlog) fix read hive/iceberg catalog on cosn &
fix read data via broker (#22087)
f7d00d467a is described below
commit f7d00d467a33d5e317d6dab318cd135f7bad3d73
Author: Yulei-Yang <yulei.yang0...@gmail.com>
AuthorDate: Thu Aug 10 14:44:53 2023 +0800
[fix](multicatlog) fix read hive/iceberg catalog on cosn & fix read data
via broker (#22087)
* [fix](multicatlog) fix read hive/iceberg catalog on cosn & fix read data
via broker
* Update FileSystemFactory.java
---
.../main/java/org/apache/doris/common/FeConstants.java | 1 +
.../main/java/org/apache/doris/common/util/S3Util.java | 7 ++++---
.../main/java/org/apache/doris/fs/FileSystemFactory.java | 3 ++-
.../apache/doris/planner/external/FileQueryScanNode.java | 16 +++++++++++-----
4 files changed, 18 insertions(+), 9 deletions(-)
diff --git a/fe/fe-core/src/main/java/org/apache/doris/common/FeConstants.java
b/fe/fe-core/src/main/java/org/apache/doris/common/FeConstants.java
index 28241da750..59f3efa32f 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/common/FeConstants.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/common/FeConstants.java
@@ -80,6 +80,7 @@ public class FeConstants {
public static String FS_PREFIX_GCS = "gs";
public static String FS_PREFIX_BOS = "bos";
public static String FS_PREFIX_COS = "cos";
+ public static String FS_PREFIX_COSN = "cosn";
public static String FS_PREFIX_OBS = "obs";
public static String FS_PREFIX_OFS = "ofs";
public static String FS_PREFIX_GFS = "gfs";
diff --git a/fe/fe-core/src/main/java/org/apache/doris/common/util/S3Util.java
b/fe/fe-core/src/main/java/org/apache/doris/common/util/S3Util.java
index fe59dcee33..6d433d5435 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/common/util/S3Util.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/common/util/S3Util.java
@@ -53,9 +53,10 @@ public class S3Util {
public static boolean isObjStorage(String location) {
return isObjStorageUseS3Client(location)
- || location.startsWith(FeConstants.FS_PREFIX_COS)
- || location.startsWith(FeConstants.FS_PREFIX_OSS)
- || location.startsWith(FeConstants.FS_PREFIX_OBS);
+ // if treat cosn(tencent hadoop-cos) as a s3 file system, may
bring incompatible issues
+ || (location.startsWith(FeConstants.FS_PREFIX_COS) &&
!location.startsWith(FeConstants.FS_PREFIX_COSN))
+ || location.startsWith(FeConstants.FS_PREFIX_OSS)
+ || location.startsWith(FeConstants.FS_PREFIX_OBS);
}
private static boolean isObjStorageUseS3Client(String location) {
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/fs/FileSystemFactory.java
b/fe/fe-core/src/main/java/org/apache/doris/fs/FileSystemFactory.java
index 1c6217ff4a..eca8384cfa 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/fs/FileSystemFactory.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/fs/FileSystemFactory.java
@@ -66,7 +66,8 @@ public class FileSystemFactory {
fsType = FileSystemType.S3;
} else if (location.startsWith(FeConstants.FS_PREFIX_HDFS) ||
location.startsWith(FeConstants.FS_PREFIX_GFS)) {
fsType = FileSystemType.DFS;
- } else if (location.startsWith(FeConstants.FS_PREFIX_OFS)) {
+ } else if (location.startsWith(FeConstants.FS_PREFIX_OFS) ||
location.startsWith(FeConstants.FS_PREFIX_COSN)) {
+ // ofs:// and cosn:// use the same underlying file system: Tencent
Cloud HDFS, aka CHDFS)) {
fsType = FileSystemType.OFS;
} else if (location.startsWith(FeConstants.FS_PREFIX_JFS)) {
fsType = FileSystemType.JFS;
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/planner/external/FileQueryScanNode.java
b/fe/fe-core/src/main/java/org/apache/doris/planner/external/FileQueryScanNode.java
index 90f84deb94..586baa58d0 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/planner/external/FileQueryScanNode.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/planner/external/FileQueryScanNode.java
@@ -353,12 +353,16 @@ public abstract class FileQueryScanNode extends
FileScanNode {
params.setHdfsParams(tHdfsParams);
}
- if (locationType == TFileType.FILE_BROKER &&
!params.isSetBrokerAddresses()) {
- FsBroker broker =
Env.getCurrentEnv().getBrokerMgr().getAnyAliveBroker();
- if (broker == null) {
- throw new UserException("No alive broker.");
+ if (locationType == TFileType.FILE_BROKER) {
+ params.setProperties(locationProperties);
+
+ if (!params.isSetBrokerAddresses()) {
+ FsBroker broker =
Env.getCurrentEnv().getBrokerMgr().getAnyAliveBroker();
+ if (broker == null) {
+ throw new UserException("No alive broker.");
+ }
+ params.addToBrokerAddresses(new
TNetworkAddress(broker.host, broker.port));
}
- params.addToBrokerAddresses(new TNetworkAddress(broker.host,
broker.port));
}
} else if (locationType == TFileType.FILE_S3 &&
!params.isSetProperties()) {
params.setProperties(locationProperties);
@@ -450,6 +454,8 @@ public abstract class FileQueryScanNode extends
FileScanNode {
return Optional.of(TFileType.FILE_LOCAL);
} else if (location.startsWith(FeConstants.FS_PREFIX_OFS)) {
return Optional.of(TFileType.FILE_BROKER);
+ } else if (location.startsWith(FeConstants.FS_PREFIX_COSN)) {
+ return Optional.of(TFileType.FILE_BROKER);
} else if (location.startsWith(FeConstants.FS_PREFIX_GFS)) {
return Optional.of(TFileType.FILE_BROKER);
} else if (location.startsWith(FeConstants.FS_PREFIX_JFS)) {
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org
For additional commands, e-mail: commits-h...@doris.apache.org
