This is an automated email from the ASF dual-hosted git repository.
kirs pushed a commit to branch branch-2.1
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/branch-2.1 by this push:
new 9eb07e578ea branch-2.1 [Fix](catalog)Remove the fs.disable.cache
parameter to prevent excessive FS-associated objects and memory leaks (#46184)
(#46191)
9eb07e578ea is described below
commit 9eb07e578ea62e5c29f3875c5789b1f8f7170df6
Author: Calvin Kirs <guoqi...@selectdb.com>
AuthorDate: Tue Dec 31 16:54:10 2024 +0800
branch-2.1 [Fix](catalog)Remove the fs.disable.cache parameter to prevent
excessive FS-associated objects and memory leaks (#46184) (#46191)
…
#46184
(cherry picked from commit 17667aeb92efc5580b8615e53a3ef4f514a27204)
---
.../doris/datasource/hive/HiveMetaStoreCache.java | 3 ---
.../doris/datasource/property/PropertyConverter.java | 5 -----
.../datasource/property/PropertyConverterTest.java | 18 +++++++++++-------
3 files changed, 11 insertions(+), 15 deletions(-)
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HiveMetaStoreCache.java
b/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HiveMetaStoreCache.java
index d66d22f2bb2..48a625c35a7 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HiveMetaStoreCache.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HiveMetaStoreCache.java
@@ -395,7 +395,6 @@ public class HiveMetaStoreCache {
URI uri = finalLocation.getPath().toUri();
if (uri.getScheme() != null) {
String scheme = uri.getScheme();
- updateJobConf("fs." + scheme + ".impl.disable.cache",
"true");
if (jobConf.get("fs." + scheme + ".impl") == null) {
if (!scheme.equals("hdfs") &&
!scheme.equals("viewfs")) {
updateJobConf("fs." + scheme + ".impl",
PropertyConverter.getHadoopFSImplByScheme(scheme));
@@ -444,8 +443,6 @@ public class HiveMetaStoreCache {
//
https://blog.actorsfit.com/a?ID=00550-ce56ec63-1bff-4b0c-a6f7-447b93efaa31
jobConf.set("mapreduce.input.fileinputformat.input.dir.recursive",
"true");
// disable FileSystem's cache
- jobConf.set("fs.hdfs.impl.disable.cache", "true");
- jobConf.set("fs.file.impl.disable.cache", "true");
}
private synchronized void updateJobConf(String key, String value) {
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/datasource/property/PropertyConverter.java
b/fe/fe-core/src/main/java/org/apache/doris/datasource/property/PropertyConverter.java
index 5a378336950..a236f123b77 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/datasource/property/PropertyConverter.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/datasource/property/PropertyConverter.java
@@ -163,7 +163,6 @@ public class PropertyConverter {
CloudCredential
credential) {
Map<String, String> obsProperties = Maps.newHashMap();
obsProperties.put(ObsProperties.HadoopFsObsConstants.ENDPOINT,
props.get(ObsProperties.ENDPOINT));
- obsProperties.put(ObsProperties.FS.IMPL_DISABLE_CACHE, "true");
obsProperties.put("fs.obs.impl", getHadoopFSImplByScheme("obs"));
if (credential.isWhole()) {
obsProperties.put(ObsProperties.HadoopFsObsConstants.ACCESS_KEY,
credential.getAccessKey());
@@ -264,7 +263,6 @@ public class PropertyConverter {
private static void setS3FsAccess(Map<String, String> s3Properties,
Map<String, String> properties,
CloudCredential credential) {
s3Properties.put(Constants.MAX_ERROR_RETRIES, "2");
- s3Properties.put("fs.s3.impl.disable.cache", "true");
s3Properties.putIfAbsent("fs.s3.impl", S3AFileSystem.class.getName());
String credentialsProviders = getAWSCredentialsProviders(properties);
s3Properties.put(Constants.AWS_CREDENTIALS_PROVIDER,
credentialsProviders);
@@ -275,8 +273,6 @@ public class PropertyConverter {
if (credential.isTemporary()) {
s3Properties.put(Constants.SESSION_TOKEN,
credential.getSessionToken());
s3Properties.put(Constants.AWS_CREDENTIALS_PROVIDER,
TemporaryAWSCredentialsProvider.class.getName());
- s3Properties.put("fs.s3.impl.disable.cache", "true");
- s3Properties.put("fs.s3a.impl.disable.cache", "true");
}
s3Properties.put(Constants.PATH_STYLE_ACCESS,
properties.getOrDefault(USE_PATH_STYLE, "false"));
for (Map.Entry<String, String> entry : properties.entrySet()) {
@@ -352,7 +348,6 @@ public class PropertyConverter {
Map<String, String> cosProperties = Maps.newHashMap();
cosProperties.put(CosProperties.HadoopFsCosConstants.COSN_ENDPOINT_SUFFIX_KEY,
props.get(CosProperties.ENDPOINT));
- cosProperties.put("fs.cosn.impl.disable.cache", "true");
cosProperties.put("fs.cosn.impl", getHadoopFSImplByScheme("cosn"));
cosProperties.put("fs.lakefs.impl", getHadoopFSImplByScheme("lakefs"));
if (credential.isWhole()) {
diff --git
a/fe/fe-core/src/test/java/org/apache/doris/datasource/property/PropertyConverterTest.java
b/fe/fe-core/src/test/java/org/apache/doris/datasource/property/PropertyConverterTest.java
index d034440bb6d..8967ca5fae0 100644
---
a/fe/fe-core/src/test/java/org/apache/doris/datasource/property/PropertyConverterTest.java
+++
b/fe/fe-core/src/test/java/org/apache/doris/datasource/property/PropertyConverterTest.java
@@ -273,7 +273,7 @@ public class PropertyConverterTest extends
TestWithFeService {
Assertions.assertEquals(13, properties.size());
Map<String, String> hdProps =
catalog.getCatalogProperty().getHadoopProperties();
- Assertions.assertEquals(22, hdProps.size());
+ Assertions.assertEquals(21, hdProps.size());
}
@Test
@@ -291,7 +291,8 @@ public class PropertyConverterTest extends
TestWithFeService {
Assertions.assertEquals(12, properties.size());
Map<String, String> hdProps =
catalog.getCatalogProperty().getHadoopProperties();
- Assertions.assertEquals(21, hdProps.size());
+ Assertions.assertNull(hdProps.get("fs.s3.impl.disable.cache"));
+ Assertions.assertEquals(20, hdProps.size());
}
@Test
@@ -442,7 +443,8 @@ public class PropertyConverterTest extends
TestWithFeService {
Assertions.assertEquals("s3.us-east-1.amazonaws.com",
properties.get(S3Properties.ENDPOINT));
Map<String, String> hdProps =
catalog.getCatalogProperty().getHadoopProperties();
- Assertions.assertEquals(30, hdProps.size());
+ Assertions.assertEquals(29, hdProps.size());
+ Assertions.assertNull(hdProps.get("fs.s3.impl.disable.cache"));
String query = "create catalog hms_glue properties (\n"
+ " 'type'='hms',\n"
@@ -460,7 +462,8 @@ public class PropertyConverterTest extends
TestWithFeService {
Assertions.assertEquals("s3.us-east-1.amazonaws.com.cn",
propertiesNew.get(S3Properties.ENDPOINT));
Map<String, String> hdPropsNew =
catalogNew.getCatalogProperty().getHadoopProperties();
- Assertions.assertEquals(30, hdPropsNew.size());
+ Assertions.assertNull(hdPropsNew.get("fs.s3.impl.disable.cache"));
+ Assertions.assertEquals(29, hdPropsNew.size());
}
@Test
@@ -474,7 +477,7 @@ public class PropertyConverterTest extends
TestWithFeService {
+ " 'cos.secret_key' = 'skk'\n"
+ ");";
testS3CompatibleCatalogProperties(catalogName0,
CosProperties.COS_PREFIX,
- "cos.ap-beijing.myqcloud.com", query0, 12, 18);
+ "cos.ap-beijing.myqcloud.com", query0, 12, 17);
String catalogName1 = "hms_oss";
String query1 = "create catalog " + catalogName1 + " properties (\n"
@@ -496,7 +499,7 @@ public class PropertyConverterTest extends
TestWithFeService {
+ " 'minio.secret_key' = 'skk'\n"
+ ");";
testS3CompatibleCatalogProperties(catalogName2,
MinioProperties.MINIO_PREFIX,
- "http://127.0.0.1";, query2, 12, 21);
+ "http://127.0.0.1";, query2, 12, 20);
String catalogName3 = "hms_obs";
String query3 = "create catalog hms_obs properties (\n"
@@ -507,7 +510,7 @@ public class PropertyConverterTest extends
TestWithFeService {
+ " 'obs.secret_key' = 'skk'\n"
+ ");";
testS3CompatibleCatalogProperties(catalogName3,
ObsProperties.OBS_PREFIX,
- "obs.cn-north-4.myhuaweicloud.com", query3, 12, 17);
+ "obs.cn-north-4.myhuaweicloud.com", query3, 12, 16);
}
private void testS3CompatibleCatalogProperties(String catalogName, String
prefix,
@@ -521,6 +524,7 @@ public class PropertyConverterTest extends
TestWithFeService {
Map<String, String> hdProps =
catalog.getCatalogProperty().getHadoopProperties();
Assertions.assertEquals(bePropsSize, hdProps.size());
+
Assertions.assertNull(hdProps.get(String.format("fs.%s.impl.disable.cache",
prefix)));
Map<String, String> expectedMetaProperties = new HashMap<>();
expectedMetaProperties.put("endpoint", endpoint);
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org
For additional commands, e-mail: commits-h...@doris.apache.org
