This is an automated email from the ASF dual-hosted git repository.
morningman pushed a commit to branch branch-3.0
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/branch-3.0 by this push:
new 7e6c77d2f72 branch-3.0: [Fix](catalog)Remove the fs.disable.cache
parameter to prevent excessive FS-associated objects and memory leaks #46184
(#46189)
7e6c77d2f72 is described below
commit 7e6c77d2f726f42f282cb797e390249f2097c2d3
Author: github-actions[bot]
<41898282+github-actions[bot]@users.noreply.github.com>
AuthorDate: Tue Dec 31 17:02:40 2024 +0800
branch-3.0: [Fix](catalog)Remove the fs.disable.cache parameter to prevent
excessive FS-associated objects and memory leaks #46184 (#46189)
Cherry-picked from #46184
Co-authored-by: Calvin Kirs <guoqi...@selectdb.com>
---
.../doris/datasource/hive/HiveMetaStoreCache.java | 3 ---
.../doris/datasource/property/PropertyConverter.java | 5 -----
.../datasource/property/PropertyConverterTest.java | 18 +++++++++++-------
3 files changed, 11 insertions(+), 15 deletions(-)
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HiveMetaStoreCache.java
b/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HiveMetaStoreCache.java
index ea42dfa2f52..90af6b3f394 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HiveMetaStoreCache.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HiveMetaStoreCache.java
@@ -402,7 +402,6 @@ public class HiveMetaStoreCache {
URI uri = finalLocation.getPath().toUri();
if (uri.getScheme() != null) {
String scheme = uri.getScheme();
- updateJobConf("fs." + scheme + ".impl.disable.cache",
"true");
if (jobConf.get("fs." + scheme + ".impl") == null) {
if (!scheme.equals("hdfs") &&
!scheme.equals("viewfs")) {
updateJobConf("fs." + scheme + ".impl",
PropertyConverter.getHadoopFSImplByScheme(scheme));
@@ -451,8 +450,6 @@ public class HiveMetaStoreCache {
//
https://blog.actorsfit.com/a?ID=00550-ce56ec63-1bff-4b0c-a6f7-447b93efaa31
jobConf.set("mapreduce.input.fileinputformat.input.dir.recursive",
"true");
// disable FileSystem's cache
- jobConf.set("fs.hdfs.impl.disable.cache", "true");
- jobConf.set("fs.file.impl.disable.cache", "true");
}
private synchronized void updateJobConf(String key, String value) {
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/datasource/property/PropertyConverter.java
b/fe/fe-core/src/main/java/org/apache/doris/datasource/property/PropertyConverter.java
index 7b65411aa6b..cbdf3d4ce6a 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/datasource/property/PropertyConverter.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/datasource/property/PropertyConverter.java
@@ -174,7 +174,6 @@ public class PropertyConverter {
CloudCredential
credential) {
Map<String, String> obsProperties = Maps.newHashMap();
obsProperties.put(OBSConstants.ENDPOINT,
props.get(ObsProperties.ENDPOINT));
- obsProperties.put(ObsProperties.FS.IMPL_DISABLE_CACHE, "true");
obsProperties.put("fs.obs.impl", getHadoopFSImplByScheme("obs"));
if (credential.isWhole()) {
obsProperties.put(OBSConstants.ACCESS_KEY,
credential.getAccessKey());
@@ -275,7 +274,6 @@ public class PropertyConverter {
private static void setS3FsAccess(Map<String, String> s3Properties,
Map<String, String> properties,
CloudCredential credential) {
s3Properties.put(Constants.MAX_ERROR_RETRIES, "2");
- s3Properties.put("fs.s3.impl.disable.cache", "true");
s3Properties.putIfAbsent("fs.s3.impl", S3AFileSystem.class.getName());
String credentialsProviders = getAWSCredentialsProviders(properties);
s3Properties.put(Constants.AWS_CREDENTIALS_PROVIDER,
credentialsProviders);
@@ -286,8 +284,6 @@ public class PropertyConverter {
if (credential.isTemporary()) {
s3Properties.put(Constants.SESSION_TOKEN,
credential.getSessionToken());
s3Properties.put(Constants.AWS_CREDENTIALS_PROVIDER,
TemporaryAWSCredentialsProvider.class.getName());
- s3Properties.put("fs.s3.impl.disable.cache", "true");
- s3Properties.put("fs.s3a.impl.disable.cache", "true");
}
s3Properties.put(Constants.PATH_STYLE_ACCESS,
properties.getOrDefault(USE_PATH_STYLE, "false"));
for (Map.Entry<String, String> entry : properties.entrySet()) {
@@ -362,7 +358,6 @@ public class PropertyConverter {
private static Map<String, String> convertToCOSProperties(Map<String,
String> props, CloudCredential credential) {
Map<String, String> cosProperties = Maps.newHashMap();
cosProperties.put(CosNConfigKeys.COSN_ENDPOINT_SUFFIX_KEY,
props.get(CosProperties.ENDPOINT));
- cosProperties.put("fs.cosn.impl.disable.cache", "true");
cosProperties.put("fs.cosn.impl", getHadoopFSImplByScheme("cosn"));
cosProperties.put("fs.lakefs.impl", getHadoopFSImplByScheme("lakefs"));
if (credential.isWhole()) {
diff --git
a/fe/fe-core/src/test/java/org/apache/doris/datasource/property/PropertyConverterTest.java
b/fe/fe-core/src/test/java/org/apache/doris/datasource/property/PropertyConverterTest.java
index d034440bb6d..8967ca5fae0 100644
---
a/fe/fe-core/src/test/java/org/apache/doris/datasource/property/PropertyConverterTest.java
+++
b/fe/fe-core/src/test/java/org/apache/doris/datasource/property/PropertyConverterTest.java
@@ -273,7 +273,7 @@ public class PropertyConverterTest extends
TestWithFeService {
Assertions.assertEquals(13, properties.size());
Map<String, String> hdProps =
catalog.getCatalogProperty().getHadoopProperties();
- Assertions.assertEquals(22, hdProps.size());
+ Assertions.assertEquals(21, hdProps.size());
}
@Test
@@ -291,7 +291,8 @@ public class PropertyConverterTest extends
TestWithFeService {
Assertions.assertEquals(12, properties.size());
Map<String, String> hdProps =
catalog.getCatalogProperty().getHadoopProperties();
- Assertions.assertEquals(21, hdProps.size());
+ Assertions.assertNull(hdProps.get("fs.s3.impl.disable.cache"));
+ Assertions.assertEquals(20, hdProps.size());
}
@Test
@@ -442,7 +443,8 @@ public class PropertyConverterTest extends
TestWithFeService {
Assertions.assertEquals("s3.us-east-1.amazonaws.com",
properties.get(S3Properties.ENDPOINT));
Map<String, String> hdProps =
catalog.getCatalogProperty().getHadoopProperties();
- Assertions.assertEquals(30, hdProps.size());
+ Assertions.assertEquals(29, hdProps.size());
+ Assertions.assertNull(hdProps.get("fs.s3.impl.disable.cache"));
String query = "create catalog hms_glue properties (\n"
+ " 'type'='hms',\n"
@@ -460,7 +462,8 @@ public class PropertyConverterTest extends
TestWithFeService {
Assertions.assertEquals("s3.us-east-1.amazonaws.com.cn",
propertiesNew.get(S3Properties.ENDPOINT));
Map<String, String> hdPropsNew =
catalogNew.getCatalogProperty().getHadoopProperties();
- Assertions.assertEquals(30, hdPropsNew.size());
+ Assertions.assertNull(hdPropsNew.get("fs.s3.impl.disable.cache"));
+ Assertions.assertEquals(29, hdPropsNew.size());
}
@Test
@@ -474,7 +477,7 @@ public class PropertyConverterTest extends
TestWithFeService {
+ " 'cos.secret_key' = 'skk'\n"
+ ");";
testS3CompatibleCatalogProperties(catalogName0,
CosProperties.COS_PREFIX,
- "cos.ap-beijing.myqcloud.com", query0, 12, 18);
+ "cos.ap-beijing.myqcloud.com", query0, 12, 17);
String catalogName1 = "hms_oss";
String query1 = "create catalog " + catalogName1 + " properties (\n"
@@ -496,7 +499,7 @@ public class PropertyConverterTest extends
TestWithFeService {
+ " 'minio.secret_key' = 'skk'\n"
+ ");";
testS3CompatibleCatalogProperties(catalogName2,
MinioProperties.MINIO_PREFIX,
- "http://127.0.0.1";, query2, 12, 21);
+ "http://127.0.0.1";, query2, 12, 20);
String catalogName3 = "hms_obs";
String query3 = "create catalog hms_obs properties (\n"
@@ -507,7 +510,7 @@ public class PropertyConverterTest extends
TestWithFeService {
+ " 'obs.secret_key' = 'skk'\n"
+ ");";
testS3CompatibleCatalogProperties(catalogName3,
ObsProperties.OBS_PREFIX,
- "obs.cn-north-4.myhuaweicloud.com", query3, 12, 17);
+ "obs.cn-north-4.myhuaweicloud.com", query3, 12, 16);
}
private void testS3CompatibleCatalogProperties(String catalogName, String
prefix,
@@ -521,6 +524,7 @@ public class PropertyConverterTest extends
TestWithFeService {
Map<String, String> hdProps =
catalog.getCatalogProperty().getHadoopProperties();
Assertions.assertEquals(bePropsSize, hdProps.size());
+
Assertions.assertNull(hdProps.get(String.format("fs.%s.impl.disable.cache",
prefix)));
Map<String, String> expectedMetaProperties = new HashMap<>();
expectedMetaProperties.put("endpoint", endpoint);
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org
For additional commands, e-mail: commits-h...@doris.apache.org
