This is an automated email from the ASF dual-hosted git repository.
xuyang pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/master by this push:
new 33849e00c00 [fix](multicatalog) fix no data error when read hive table
on cosn (#32815)
33849e00c00 is described below
commit 33849e00c007e4ed07387a95aceba592532f8a47
Author: Yulei-Yang <yulei.yang0...@gmail.com>
AuthorDate: Wed Mar 27 18:45:58 2024 +0800
[fix](multicatalog) fix no data error when read hive table on cosn (#32815)
Currently, when reading a hive on cosn table, doris return empty result,
but the table has data.
iceberg on cosn is ok.
The reason is misuse of cosn's file sytem. according to cosn's doc, its
fs.cosn.impl should be org.apache.hadoop.fs.CosFileSystem
---
conf/fe.conf | 2 +-
fe/fe-core/pom.xml | 10 ++++++++--
.../apache/doris/datasource/property/PropertyConverter.java | 10 +++++-----
3 files changed, 14 insertions(+), 8 deletions(-)
diff --git a/conf/fe.conf b/conf/fe.conf
index 8b8c3be747d..91f7d048cb1 100644
--- a/conf/fe.conf
+++ b/conf/fe.conf
@@ -30,7 +30,7 @@ LOG_DIR = ${DORIS_HOME}/log
JAVA_OPTS="-Djavax.security.auth.useSubjectCredsOnly=false -Xss4m -Xmx8192m
-XX:+UnlockExperimentalVMOptions -XX:+UseG1GC -XX:MaxGCPauseMillis=200
-XX:+PrintGCDateStamps -XX:+PrintGCDetails
-Xloggc:$DORIS_HOME/log/fe.gc.log.$CUR_DATE -XX:+UseGCLogFileRotation
-XX:NumberOfGCLogFiles=10 -XX:GCLogFileSize=50M
-Dlog4j2.formatMsgNoLookups=true"
# For jdk 17, this JAVA_OPTS will be used as default JVM options
-JAVA_OPTS_FOR_JDK_17="-Djavax.security.auth.useSubjectCredsOnly=false
-Xmx8192m -Xms8192m -XX:+HeapDumpOnOutOfMemoryError
-XX:HeapDumpPath=$DORIS_HOME/log/
-Xlog:gc*:$DORIS_HOME/log/fe.gc.log.$CUR_DATE:time,uptime:filecount=10,filesize=50M"
+JAVA_OPTS_FOR_JDK_17="-Djavax.security.auth.useSubjectCredsOnly=false
-Xmx8192m -Xms8192m -XX:+HeapDumpOnOutOfMemoryError
-XX:HeapDumpPath=$DORIS_HOME/log/
-Xlog:gc*:$DORIS_HOME/log/fe.gc.log.$CUR_DATE:time,uptime:filecount=10,filesize=50M
--add-opens=java.base/java.nio=ALL-UNNAMED --add-opens
java.base/jdk.internal.ref=ALL-UNNAMED"
# Set your own JAVA_HOME
# JAVA_HOME=/path/to/jdk/
diff --git a/fe/fe-core/pom.xml b/fe/fe-core/pom.xml
index 21dd17d60b4..307f9046cc6 100644
--- a/fe/fe-core/pom.xml
+++ b/fe/fe-core/pom.xml
@@ -35,7 +35,7 @@ under the License.
<antlr4.version>4.13.1</antlr4.version>
<awssdk.version>2.20.131</awssdk.version>
<huaweiobs.version>3.1.1-hw-46</huaweiobs.version>
- <tencentcos.version>3.3.5</tencentcos.version>
+ <tencentcos.version>8.2.7</tencentcos.version>
</properties>
<profiles>
<profile>
@@ -418,9 +418,15 @@ under the License.
<artifactId>hadoop-aliyun</artifactId>
</dependency>
<dependency>
- <groupId>org.apache.hadoop</groupId>
+ <groupId>com.qcloud.cos</groupId>
<artifactId>hadoop-cos</artifactId>
<version>${tencentcos.version}</version>
+ <exclusions>
+ <exclusion>
+ <groupId>org.json</groupId>
+ <artifactId>json</artifactId>
+ </exclusion>
+ </exclusions>
</dependency>
<dependency>
<groupId>com.aliyun.odps</groupId>
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/datasource/property/PropertyConverter.java
b/fe/fe-core/src/main/java/org/apache/doris/datasource/property/PropertyConverter.java
index 24457abc748..b385fb838b3 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/datasource/property/PropertyConverter.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/datasource/property/PropertyConverter.java
@@ -39,9 +39,9 @@ import com.aliyun.datalake.metastore.common.DataLakeConfig;
import com.amazonaws.glue.catalog.util.AWSGlueConfig;
import com.google.common.base.Strings;
import com.google.common.collect.Maps;
+import org.apache.hadoop.fs.CosFileSystem;
+import org.apache.hadoop.fs.CosNConfigKeys;
import org.apache.hadoop.fs.aliyun.oss.AliyunOSSFileSystem;
-import org.apache.hadoop.fs.cosn.CosNConfigKeys;
-import org.apache.hadoop.fs.cosn.CosNFileSystem;
import org.apache.hadoop.fs.obs.OBSConstants;
import org.apache.hadoop.fs.obs.OBSFileSystem;
import org.apache.hadoop.fs.s3a.Constants;
@@ -186,7 +186,7 @@ public class PropertyConverter {
} else if (fsScheme.equalsIgnoreCase("oss")) {
return AliyunOSSFileSystem.class.getName();
} else if (fsScheme.equalsIgnoreCase("cosn")) {
- return CosNFileSystem.class.getName();
+ return CosFileSystem.class.getName();
} else {
return S3AFileSystem.class.getName();
}
@@ -352,8 +352,8 @@ public class PropertyConverter {
cosProperties.put("fs.cosn.impl.disable.cache", "true");
cosProperties.put("fs.cosn.impl", getHadoopFSImplByScheme("cosn"));
if (credential.isWhole()) {
- cosProperties.put(CosNConfigKeys.COSN_SECRET_ID_KEY,
credential.getAccessKey());
- cosProperties.put(CosNConfigKeys.COSN_SECRET_KEY_KEY,
credential.getSecretKey());
+ cosProperties.put(CosNConfigKeys.COSN_USERINFO_SECRET_ID_KEY,
credential.getAccessKey());
+ cosProperties.put(CosNConfigKeys.COSN_USERINFO_SECRET_KEY_KEY,
credential.getSecretKey());
}
// session token is unsupported
for (Map.Entry<String, String> entry : props.entrySet()) {
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org
For additional commands, e-mail: commits-h...@doris.apache.org
