This is an automated email from the ASF dual-hosted git repository.
kirs pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/master by this push:
new 3cf4045cf7d [feat](param-refactor)Unify FS Factory Creation Logic and
Support Broker Properties for Catalog Compatibility (#51888)
3cf4045cf7d is described below
commit 3cf4045cf7d1d9967a99c49b18ecd73e57ae5a37
Author: Calvin Kirs <[email protected]>
AuthorDate: Thu Jun 19 09:48:25 2025 +0800
[feat](param-refactor)Unify FS Factory Creation Logic and Support Broker
Properties for Catalog Compatibility (#51888)
### What problem does this PR solve?
#50238
**Eliminated Redundant bind.broker Usage in Catalog:**
Previously, the Catalog used a custom bind.broker field—derived from the
user-defined ↳broker.name—to determine whether to use a
BrokerFileSystem, and to identify the specific broker.
Since we support many storage backends, keeping this value separately is
redundant.
This PR migrates the logic into brokerProperties, allowing the factory
to determine everything it needs from a single source of truth.
**Refactored FSFactory Interface:**
The original FSFactory contained too many scattered methods, making it
harder to maintain and extend.
We've consolidated and organized its APIs for better clarity, reducing
duplication and improving its usability across modules.
### Release note
---
.../doris/analysis/CreateRepositoryStmt.java | 5 ++
.../org/apache/doris/analysis/StorageDesc.java | 21 ++++---
.../org/apache/doris/backup/BackupHandler.java | 4 +-
.../java/org/apache/doris/backup/Repository.java | 4 +-
.../org/apache/doris/common/util/BrokerUtil.java | 2 +-
.../property/storage/BrokerProperties.java | 43 ++++++++++++++
.../property/storage/StorageProperties.java | 3 +
.../java/org/apache/doris/fs/FileSystemCache.java | 2 +-
.../org/apache/doris/fs/FileSystemFactory.java | 41 +------------
.../org/apache/doris/fs/PersistentFileSystem.java | 10 +++-
.../org/apache/doris/fs/StorageTypeMapper.java | 3 +
.../apache/doris/fs/remote/BrokerFileSystem.java | 13 ++--
.../plans/commands/CreateRepositoryCommand.java | 5 ++
.../org/apache/doris/backup/BackupJobTest.java | 3 +-
.../org/apache/doris/backup/BrokerStorageTest.java | 4 +-
.../org/apache/doris/backup/RepositoryTest.java | 3 +-
.../org/apache/doris/backup/RestoreJobTest.java | 3 +-
.../property/storage/BrokerPropertiesTest.java | 69 ++++++++++++++++++++++
.../external/iceberg/IcebergHadoopCatalogTest.java | 3 +-
.../org/apache/doris/fs/obj/S3FileSystemTest.java | 8 ++-
20 files changed, 177 insertions(+), 72 deletions(-)
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/analysis/CreateRepositoryStmt.java
b/fe/fe-core/src/main/java/org/apache/doris/analysis/CreateRepositoryStmt.java
index e4beff27671..b7781413057 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/analysis/CreateRepositoryStmt.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/analysis/CreateRepositoryStmt.java
@@ -22,6 +22,7 @@ import org.apache.doris.common.ErrorCode;
import org.apache.doris.common.ErrorReport;
import org.apache.doris.common.FeNameFormat;
import org.apache.doris.common.UserException;
+import org.apache.doris.datasource.property.storage.StorageProperties;
import org.apache.doris.mysql.privilege.PrivPredicate;
import org.apache.doris.qe.ConnectContext;
@@ -62,6 +63,10 @@ public class CreateRepositoryStmt extends DdlStmt implements
NotFallbackInParser
return storage.getStorageDesc().getProperties();
}
+ public StorageProperties getStorageProperties() {
+ return storage.getStorageDesc().getStorageProperties();
+ }
+
@Override
public void analyze(Analyzer analyzer) throws UserException {
super.analyze(analyzer);
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/analysis/StorageDesc.java
b/fe/fe-core/src/main/java/org/apache/doris/analysis/StorageDesc.java
index bbfcd8e9d2c..b74a3b6dfa8 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/analysis/StorageDesc.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/StorageDesc.java
@@ -17,7 +17,7 @@
package org.apache.doris.analysis;
-import org.apache.doris.common.UserException;
+import org.apache.doris.datasource.property.storage.BrokerProperties;
import org.apache.doris.datasource.property.storage.StorageProperties;
import com.google.gson.annotations.SerializedName;
@@ -50,17 +50,16 @@ public class StorageDesc extends ResourceDesc {
public StorageDesc(String name, StorageBackend.StorageType storageType,
Map<String, String> properties) {
this.name = name;
this.storageType = storageType;
- if (!storageType.equals(StorageBackend.StorageType.BROKER)) {
- this.storageProperties =
StorageProperties.createPrimary(properties);
- }
this.properties = properties;
+ initStorageProperties();
}
- public StorageDesc(String name, Map<String, String> properties) throws
UserException {
- this.name = name;
- this.properties = properties;
- this.storageProperties = StorageProperties.createPrimary(properties);
- this.storageType =
StorageBackend.StorageType.convertToStorageType(storageProperties.getStorageName());
+ private void initStorageProperties() {
+ if (null != storageType &&
storageType.equals(StorageBackend.StorageType.BROKER)) {
+ this.storageProperties = BrokerProperties.of(name, properties);
+ } else {
+ this.storageProperties =
StorageProperties.createPrimary(properties);
+ }
}
public void setName(String name) {
@@ -93,4 +92,8 @@ public class StorageDesc extends ResourceDesc {
}
return storageProperties.getBackendConfigProperties();
}
+
+ public StorageProperties getStorageProperties() {
+ return storageProperties;
+ }
}
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/backup/BackupHandler.java
b/fe/fe-core/src/main/java/org/apache/doris/backup/BackupHandler.java
index a79f962d747..73cf44860f1 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/backup/BackupHandler.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/backup/BackupHandler.java
@@ -221,7 +221,7 @@ public class BackupHandler extends MasterDaemon implements
Writable {
}
RemoteFileSystem fileSystem;
- fileSystem = FileSystemFactory.get(command.getStorageType(),
command.getBrokerName(), command.getProperties());
+ fileSystem = FileSystemFactory.get(command.getStorageProperties());
long repoId = env.getNextId();
Repository repo = new Repository(repoId, command.getName(),
command.isReadOnly(), command.getLocation(),
fileSystem);
@@ -246,7 +246,7 @@ public class BackupHandler extends MasterDaemon implements
Writable {
}
RemoteFileSystem fileSystem;
- fileSystem = FileSystemFactory.get(stmt.getStorageType(),
stmt.getBrokerName(), stmt.getProperties());
+ fileSystem = FileSystemFactory.get(stmt.getStorageProperties());
long repoId = env.getNextId();
Repository repo = new Repository(repoId, stmt.getName(),
stmt.isReadOnly(), stmt.getLocation(),
fileSystem);
diff --git a/fe/fe-core/src/main/java/org/apache/doris/backup/Repository.java
b/fe/fe-core/src/main/java/org/apache/doris/backup/Repository.java
index 967a5f25359..7e4fa7e9b19 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/backup/Repository.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/backup/Repository.java
@@ -31,6 +31,7 @@ import org.apache.doris.common.io.Writable;
import org.apache.doris.common.util.PrintableMap;
import org.apache.doris.common.util.TimeUtils;
import org.apache.doris.datasource.property.constants.S3Properties;
+import org.apache.doris.datasource.property.storage.BrokerProperties;
import org.apache.doris.datasource.property.storage.StorageProperties;
import
org.apache.doris.datasource.property.storage.exception.StoragePropertiesException;
import org.apache.doris.fs.FileSystemFactory;
@@ -248,7 +249,8 @@ public class Repository implements Writable,
GsonPostProcessable {
} catch (StoragePropertiesException exception) {
LOG.warn("Failed to create file system for repository: {}, error:
{}, roll back to broker"
+ " filesystem", name, exception.getMessage());
- this.fileSystem = FileSystemFactory.get(this.fileSystem.name,
this.fileSystem.properties);
+ BrokerProperties brokerProperties =
BrokerProperties.of(this.fileSystem.name, this.fileSystem.properties);
+ this.fileSystem = FileSystemFactory.get(brokerProperties);
}
}
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/common/util/BrokerUtil.java
b/fe/fe-core/src/main/java/org/apache/doris/common/util/BrokerUtil.java
index 47d94532e0c..e5b7c1a7f45 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/common/util/BrokerUtil.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/common/util/BrokerUtil.java
@@ -85,7 +85,7 @@ public class BrokerUtil {
public static void parseFile(String path, BrokerDesc brokerDesc,
List<TBrokerFileStatus> fileStatuses)
throws UserException {
List<RemoteFile> rfiles = new ArrayList<>();
- try (RemoteFileSystem fileSystem = FileSystemFactory.get(brokerDesc)) {
+ try (RemoteFileSystem fileSystem =
FileSystemFactory.get(brokerDesc.getStorageProperties())) {
Status st = fileSystem.globList(path, rfiles, false);
if (!st.ok()) {
throw new UserException(st.getErrMsg());
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/datasource/property/storage/BrokerProperties.java
b/fe/fe-core/src/main/java/org/apache/doris/datasource/property/storage/BrokerProperties.java
index b53a65ecdc7..bc55bf89892 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/datasource/property/storage/BrokerProperties.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/datasource/property/storage/BrokerProperties.java
@@ -18,14 +18,57 @@
package org.apache.doris.datasource.property.storage;
import org.apache.doris.common.UserException;
+import org.apache.doris.datasource.property.ConnectorProperty;
+import org.apache.doris.datasource.property.PropertyConverter;
+import lombok.Getter;
+import lombok.Setter;
+
+import java.util.HashMap;
import java.util.Map;
public class BrokerProperties extends StorageProperties {
+
+ @Setter
+ @Getter
+ @ConnectorProperty(names = {"broker.name"},
+ required = false,
+ description = "The name of the broker. "
+ + "This is used to identify the broker in the system.")
+ private String brokerName = "";
+
+ @Getter
+ private Map<String, String> brokerParams;
+
public BrokerProperties(Map<String, String> origProps) {
super(Type.BROKER, origProps);
}
+ public static BrokerProperties of(String brokerName, Map<String, String>
origProps) {
+ BrokerProperties properties = new BrokerProperties(origProps);
+ properties.setBrokerName(brokerName);
+ properties.initNormalizeAndCheckProps();
+ return properties;
+ }
+
+ private static final String BIND_BROKER_NAME_KEY = "broker.name";
+
+ public static boolean guessIsMe(Map<String, String> props) {
+ if (props == null || props.isEmpty()) {
+ return false;
+ }
+ return props.keySet().stream()
+ .anyMatch(key -> key.equalsIgnoreCase(BIND_BROKER_NAME_KEY));
+ }
+
+ @Override
+ protected void initNormalizeAndCheckProps() {
+ super.initNormalizeAndCheckProps();
+ this.brokerParams = new HashMap<>(origProps);
+ //why need this convert
+
this.brokerParams.putAll(PropertyConverter.convertToHadoopFSProperties(origProps));
+ }
+
@Override
public Map<String, String> getBackendConfigProperties() {
return origProps;
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/datasource/property/storage/StorageProperties.java
b/fe/fe-core/src/main/java/org/apache/doris/datasource/property/storage/StorageProperties.java
index 0be4eb87f48..d003764bba5 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/datasource/property/storage/StorageProperties.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/datasource/property/storage/StorageProperties.java
@@ -37,6 +37,7 @@ public abstract class StorageProperties extends
ConnectionProperties {
public static final String FS_S3_SUPPORT = "fs.s3.support";
public static final String FS_GCS_SUPPORT = "fs.gcs.support";
public static final String FS_MINIO_SUPPORT = "fs.minio.support";
+ public static final String FS_BROKER_SUPPORT = "fs.broker.support";
public static final String FS_AZURE_SUPPORT = "fs.azure.support";
public static final String FS_OSS_SUPPORT = "fs.oss.support";
public static final String FS_OBS_SUPPORT = "fs.obs.support";
@@ -134,6 +135,8 @@ public abstract class StorageProperties extends
ConnectionProperties {
|| AzureProperties.guessIsMe(props)) ? new
AzureProperties(props) : null,
props -> (isFsSupport(props, FS_MINIO_SUPPORT)
|| MinioProperties.guessIsMe(props)) ? new
MinioProperties(props) : null,
+ props -> (isFsSupport(props, FS_BROKER_SUPPORT)
+ || BrokerProperties.guessIsMe(props)) ? new
BrokerProperties(props) : null,
props -> (isFsSupport(props, FS_LOCAL_SUPPORT)
|| LocalProperties.guessIsMe(props)) ? new
LocalProperties(props) : null
);
diff --git a/fe/fe-core/src/main/java/org/apache/doris/fs/FileSystemCache.java
b/fe/fe-core/src/main/java/org/apache/doris/fs/FileSystemCache.java
index 01730092182..3d316189e36 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/fs/FileSystemCache.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/fs/FileSystemCache.java
@@ -47,7 +47,7 @@ public class FileSystemCache {
}
private RemoteFileSystem loadFileSystem(FileSystemCacheKey key) throws
UserException {
- return FileSystemFactory.get(key.type, key.getFsProperties(),
key.bindBrokerName);
+ return FileSystemFactory.get(key.type, key.getFsProperties());
}
public RemoteFileSystem getRemoteFileSystem(FileSystemCacheKey key) {
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/fs/FileSystemFactory.java
b/fe/fe-core/src/main/java/org/apache/doris/fs/FileSystemFactory.java
index 9d00700f894..11536418fd9 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/fs/FileSystemFactory.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/fs/FileSystemFactory.java
@@ -17,11 +17,8 @@
package org.apache.doris.fs;
-import org.apache.doris.analysis.BrokerDesc;
-import org.apache.doris.analysis.StorageBackend;
import org.apache.doris.common.UserException;
import org.apache.doris.datasource.property.storage.StorageProperties;
-import org.apache.doris.fs.remote.BrokerFileSystem;
import org.apache.doris.fs.remote.RemoteFileSystem;
import java.util.List;
@@ -29,36 +26,13 @@ import java.util.Map;
public class FileSystemFactory {
- public static RemoteFileSystem get(Map<String, String> properties) throws
UserException {
- StorageProperties storageProperties =
StorageProperties.createPrimary(properties);
- return get(storageProperties);
- }
-
- public static RemoteFileSystem get(StorageBackend.StorageType storageType,
String bindBreakName,
- Map<String, String> properties) {
- if (storageType.equals(StorageBackend.StorageType.BROKER)) {
- return new BrokerFileSystem(bindBreakName, properties);
- }
- StorageProperties storageProperties =
StorageProperties.createPrimary(properties);
- return get(storageProperties);
- }
-
public static RemoteFileSystem get(StorageProperties storageProperties) {
return StorageTypeMapper.create(storageProperties);
}
- // This method is a temporary workaround for handling properties.
- // It will be removed when broker properties are officially supported.
- public static RemoteFileSystem get(String name, Map<String, String>
properties) {
- return new BrokerFileSystem(name, properties);
- }
-
- public static RemoteFileSystem get(FileSystemType fileSystemType,
Map<String, String> properties,
- String bindBrokerName)
+ //todo remove when catalog use storage properties
+ public static RemoteFileSystem get(FileSystemType fileSystemType,
Map<String, String> properties)
throws UserException {
- if (fileSystemType == FileSystemType.BROKER) {
- return new BrokerFileSystem(bindBrokerName, properties);
- }
List<StorageProperties> storagePropertiesList =
StorageProperties.createAll(properties);
for (StorageProperties storageProperties : storagePropertiesList) {
@@ -68,15 +42,4 @@ public class FileSystemFactory {
}
throw new RuntimeException("Unsupported file system type: " +
fileSystemType);
}
-
- public static RemoteFileSystem get(BrokerDesc brokerDesc) {
- if (null != brokerDesc.getStorageProperties()) {
- return get(brokerDesc.getStorageProperties());
- }
- if (null != brokerDesc.getStorageType()
- &&
brokerDesc.getStorageType().equals(StorageBackend.StorageType.BROKER)) {
- return new BrokerFileSystem(brokerDesc.getName(),
brokerDesc.getProperties());
- }
- throw new RuntimeException("Unexpected storage type: " +
brokerDesc.getStorageType());
- }
}
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/fs/PersistentFileSystem.java
b/fe/fe-core/src/main/java/org/apache/doris/fs/PersistentFileSystem.java
index ec8f1a8b338..e2f565907ef 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/fs/PersistentFileSystem.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/fs/PersistentFileSystem.java
@@ -19,6 +19,7 @@ package org.apache.doris.fs;
import org.apache.doris.analysis.StorageBackend;
import org.apache.doris.common.io.Text;
+import org.apache.doris.datasource.property.storage.BrokerProperties;
import org.apache.doris.datasource.property.storage.StorageProperties;
import com.google.common.collect.Maps;
@@ -59,7 +60,6 @@ public abstract class PersistentFileSystem implements
FileSystem {
}
/**
- *
* @param in persisted data
* @return file systerm
*/
@@ -77,6 +77,12 @@ public abstract class PersistentFileSystem implements
FileSystem {
if (properties.containsKey(STORAGE_TYPE)) {
type =
StorageBackend.StorageType.valueOf(properties.get(STORAGE_TYPE));
}
- return FileSystemFactory.get(type, name, properties);
+ StorageProperties storageProperties;
+ if (type.equals(StorageBackend.StorageType.BROKER)) {
+ storageProperties = BrokerProperties.of(name, properties);
+ } else {
+ storageProperties = StorageProperties.createPrimary(properties);
+ }
+ return FileSystemFactory.get(storageProperties);
}
}
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/fs/StorageTypeMapper.java
b/fe/fe-core/src/main/java/org/apache/doris/fs/StorageTypeMapper.java
index 2ac1964bdbb..476d14e7198 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/fs/StorageTypeMapper.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/fs/StorageTypeMapper.java
@@ -18,6 +18,7 @@
package org.apache.doris.fs;
import org.apache.doris.datasource.property.storage.AzureProperties;
+import org.apache.doris.datasource.property.storage.BrokerProperties;
import org.apache.doris.datasource.property.storage.COSProperties;
import org.apache.doris.datasource.property.storage.HdfsProperties;
import org.apache.doris.datasource.property.storage.MinioProperties;
@@ -27,6 +28,7 @@ import
org.apache.doris.datasource.property.storage.OSSProperties;
import org.apache.doris.datasource.property.storage.S3Properties;
import org.apache.doris.datasource.property.storage.StorageProperties;
import org.apache.doris.fs.remote.AzureFileSystem;
+import org.apache.doris.fs.remote.BrokerFileSystem;
import org.apache.doris.fs.remote.RemoteFileSystem;
import org.apache.doris.fs.remote.S3FileSystem;
import org.apache.doris.fs.remote.dfs.DFSFileSystem;
@@ -42,6 +44,7 @@ public enum StorageTypeMapper {
AZURE(AzureProperties.class, AzureFileSystem::new),
S3(S3Properties.class, S3FileSystem::new),
HDFS(HdfsProperties.class, DFSFileSystem::new),
+ BROKER(BrokerProperties.class, BrokerFileSystem::new),
OSS_HDFS(OSSHdfsProperties.class, DFSFileSystem::new);
private final Class<? extends StorageProperties> propClass;
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/fs/remote/BrokerFileSystem.java
b/fe/fe-core/src/main/java/org/apache/doris/fs/remote/BrokerFileSystem.java
index 85379a8582a..224381f10d1 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/fs/remote/BrokerFileSystem.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/fs/remote/BrokerFileSystem.java
@@ -26,7 +26,6 @@ import org.apache.doris.common.ClientPool;
import org.apache.doris.common.Pair;
import org.apache.doris.common.UserException;
import org.apache.doris.common.util.BrokerUtil;
-import org.apache.doris.datasource.property.PropertyConverter;
import org.apache.doris.datasource.property.storage.BrokerProperties;
import org.apache.doris.datasource.property.storage.StorageProperties;
import org.apache.doris.fs.operations.BrokerFileOperations;
@@ -72,7 +71,6 @@ import java.nio.file.Path;
import java.nio.file.Paths;
import java.util.Comparator;
import java.util.List;
-import java.util.Map;
public class BrokerFileSystem extends RemoteFileSystem {
private static final Logger LOG =
LogManager.getLogger(BrokerFileSystem.class);
@@ -80,13 +78,10 @@ public class BrokerFileSystem extends RemoteFileSystem {
private final BrokerProperties brokerProperties;
//todo The method parameter should use the interface type
StorageProperties instead of a specific implementation.
- public BrokerFileSystem(String name, Map<String, String> properties) {
- super(name, StorageBackend.StorageType.BROKER);
-
properties.putAll(PropertyConverter.convertToHadoopFSProperties(properties));
- this.properties = properties;
- this.operations = new BrokerFileOperations(name, properties);
- // support broker properties in future
- this.brokerProperties = new BrokerProperties(properties);
+ public BrokerFileSystem(BrokerProperties brokerProperties) {
+ super(brokerProperties.getBrokerName(),
StorageBackend.StorageType.BROKER);
+ this.brokerProperties = brokerProperties;
+ this.operations = new BrokerFileOperations(name,
brokerProperties.getBrokerParams());
}
public Pair<TPaloBrokerService.Client, TNetworkAddress> getBroker() {
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/CreateRepositoryCommand.java
b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/CreateRepositoryCommand.java
index c9c8c82fcdf..d4b396349cd 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/CreateRepositoryCommand.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/CreateRepositoryCommand.java
@@ -24,6 +24,7 @@ import org.apache.doris.common.ErrorCode;
import org.apache.doris.common.ErrorReport;
import org.apache.doris.common.FeNameFormat;
import org.apache.doris.common.UserException;
+import org.apache.doris.datasource.property.storage.StorageProperties;
import org.apache.doris.mysql.privilege.PrivPredicate;
import org.apache.doris.nereids.trees.plans.PlanType;
import org.apache.doris.nereids.trees.plans.visitor.PlanVisitor;
@@ -103,4 +104,8 @@ public class CreateRepositoryCommand extends Command
implements ForwardWithSync,
public Map<String, String> getProperties() {
return storage.getStorageDesc().getProperties();
}
+
+ public StorageProperties getStorageProperties() {
+ return storage.getStorageDesc().getStorageProperties();
+ }
}
diff --git
a/fe/fe-core/src/test/java/org/apache/doris/backup/BackupJobTest.java
b/fe/fe-core/src/test/java/org/apache/doris/backup/BackupJobTest.java
index 935c560178b..f067c8f00be 100644
--- a/fe/fe-core/src/test/java/org/apache/doris/backup/BackupJobTest.java
+++ b/fe/fe-core/src/test/java/org/apache/doris/backup/BackupJobTest.java
@@ -32,6 +32,7 @@ import org.apache.doris.common.UserException;
import org.apache.doris.common.jmockit.Deencapsulation;
import org.apache.doris.common.util.UnitTestUtil;
import org.apache.doris.datasource.InternalCatalog;
+import org.apache.doris.datasource.property.storage.BrokerProperties;
import org.apache.doris.fs.FileSystemFactory;
import org.apache.doris.persist.EditLog;
import org.apache.doris.task.AgentBatchTask;
@@ -127,7 +128,7 @@ public class BackupJobTest {
private EditLog editLog;
private Repository repo = new Repository(repoId, "repo", false, "my_repo",
- FileSystemFactory.get("broker", Maps.newHashMap()));
+ FileSystemFactory.get(BrokerProperties.of("broker",
Maps.newHashMap())));
@BeforeClass
public static void start() {
diff --git
a/fe/fe-core/src/test/java/org/apache/doris/backup/BrokerStorageTest.java
b/fe/fe-core/src/test/java/org/apache/doris/backup/BrokerStorageTest.java
index 704bc17c538..4946e63642d 100644
--- a/fe/fe-core/src/test/java/org/apache/doris/backup/BrokerStorageTest.java
+++ b/fe/fe-core/src/test/java/org/apache/doris/backup/BrokerStorageTest.java
@@ -21,6 +21,7 @@ import org.apache.doris.common.ClientPool;
import org.apache.doris.common.GenericPool;
import org.apache.doris.common.Pair;
import org.apache.doris.common.jmockit.Deencapsulation;
+import org.apache.doris.datasource.property.storage.BrokerProperties;
import org.apache.doris.fs.remote.BrokerFileSystem;
import org.apache.doris.fs.remote.RemoteFile;
import org.apache.doris.thrift.TNetworkAddress;
@@ -87,7 +88,8 @@ public class BrokerStorageTest {
properties.put("bos_accesskey",
System.getenv().getOrDefault("AWS_AK", ""));
properties.put("bos_secret_accesskey",
System.getenv().getOrDefault("AWS_SK", ""));
properties.put("bos_endpoint", "http://bj.bcebos.com";);
- fileSystem = new BrokerFileSystem("bos_broker", properties);
+ BrokerProperties brokerProperties = BrokerProperties.of("bos_broker",
properties);
+ fileSystem = new BrokerFileSystem(brokerProperties);
testFile = bucket + basePath + "/Ode_to_the_West_Wind";
content =
"O wild West Wind, thou breath of Autumn's being\n"
diff --git
a/fe/fe-core/src/test/java/org/apache/doris/backup/RepositoryTest.java
b/fe/fe-core/src/test/java/org/apache/doris/backup/RepositoryTest.java
index 464509d5051..935c9623c68 100644
--- a/fe/fe-core/src/test/java/org/apache/doris/backup/RepositoryTest.java
+++ b/fe/fe-core/src/test/java/org/apache/doris/backup/RepositoryTest.java
@@ -21,6 +21,7 @@ import org.apache.doris.catalog.BrokerMgr;
import org.apache.doris.catalog.FsBroker;
import org.apache.doris.common.AnalysisException;
import org.apache.doris.common.UserException;
+import org.apache.doris.datasource.property.storage.StorageProperties;
import org.apache.doris.fs.FileSystemFactory;
import org.apache.doris.fs.remote.RemoteFile;
import org.apache.doris.fs.remote.RemoteFileSystem;
@@ -323,7 +324,7 @@ public class RepositoryTest {
properties.put("bos_endpoint", "http://gz.bcebos.com";);
properties.put("bos_accesskey", "a");
properties.put("bos_secret_accesskey", "b");
- RemoteFileSystem fs = FileSystemFactory.get(properties);
+ RemoteFileSystem fs =
FileSystemFactory.get(StorageProperties.createPrimary(properties));
repo = new Repository(10000, "repo", false, location, fs);
File file = new File("./Repository");
diff --git
a/fe/fe-core/src/test/java/org/apache/doris/backup/RestoreJobTest.java
b/fe/fe-core/src/test/java/org/apache/doris/backup/RestoreJobTest.java
index 696f669b9c2..854a1b46437 100644
--- a/fe/fe-core/src/test/java/org/apache/doris/backup/RestoreJobTest.java
+++ b/fe/fe-core/src/test/java/org/apache/doris/backup/RestoreJobTest.java
@@ -40,6 +40,7 @@ import org.apache.doris.common.MarkedCountDownLatch;
import org.apache.doris.common.UserException;
import org.apache.doris.common.jmockit.Deencapsulation;
import org.apache.doris.datasource.InternalCatalog;
+import org.apache.doris.datasource.property.storage.BrokerProperties;
import org.apache.doris.fs.FileSystemFactory;
import org.apache.doris.persist.EditLog;
import org.apache.doris.resource.Tag;
@@ -125,7 +126,7 @@ public class RestoreJobTest {
@Injectable
private Repository repo = new Repository(repoId, "repo", false,
"bos://my_repo",
- FileSystemFactory.get("broker", Maps.newHashMap()));
+ FileSystemFactory.get(BrokerProperties.of("broker",
Maps.newHashMap())));
private BackupMeta backupMeta;
diff --git
a/fe/fe-core/src/test/java/org/apache/doris/datasource/property/storage/BrokerPropertiesTest.java
b/fe/fe-core/src/test/java/org/apache/doris/datasource/property/storage/BrokerPropertiesTest.java
new file mode 100644
index 00000000000..d9fb972e98c
--- /dev/null
+++
b/fe/fe-core/src/test/java/org/apache/doris/datasource/property/storage/BrokerPropertiesTest.java
@@ -0,0 +1,69 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.doris.datasource.property.storage;
+
+import com.google.common.collect.ImmutableMap;
+import org.junit.jupiter.api.Assertions;
+import org.junit.jupiter.api.Test;
+
+import java.util.HashMap;
+import java.util.Map;
+
+public class BrokerPropertiesTest {
+
+ @Test
+ void testOfMethod_initializesBrokerNameAndParams() {
+ Map<String, String> inputProps = new HashMap<>();
+ inputProps.put("fs.s3a.access.key", "abc");
+ inputProps.put("broker.name", "broker-01");
+ BrokerProperties props = BrokerProperties.of("broker-01", inputProps);
+ Assertions.assertEquals("broker-01", props.getBrokerName());
+ Assertions.assertEquals("abc",
props.getBackendConfigProperties().get("fs.s3a.access.key"));
+ }
+
+ @Test
+ void testGuessIsMe_returnsTrueWhenBrokerNamePresent() {
+ Map<String, String> props1 = ImmutableMap.of("broker.name", "test");
+ Map<String, String> props2 = ImmutableMap.of("BROKER.NAME", "test");
+ Map<String, String> props3 = ImmutableMap.of("some.other.key",
"value");
+ Assertions.assertTrue(BrokerProperties.guessIsMe(props1));
+ Assertions.assertTrue(BrokerProperties.guessIsMe(props2));
+ Assertions.assertFalse(BrokerProperties.guessIsMe(props3));
+ }
+
+ @Test
+ void testValidateAndNormalizeUri_returnsInput() throws Exception {
+ BrokerProperties props = BrokerProperties.of("broker", new
HashMap<>());
+ String uri = "hdfs://localhost:9000/path";
+ Assertions.assertEquals(uri,
props.validateAndGetUri(ImmutableMap.of("uri", uri)));
+ }
+
+ @Test
+ void testValidateAndGetUri_returnsUriFromProps() throws Exception {
+ Map<String, String> loadProps = ImmutableMap.of("uri",
"s3://bucket/file");
+ BrokerProperties props = BrokerProperties.of("broker", loadProps);
+ String result = props.validateAndGetUri(loadProps);
+ Assertions.assertEquals("s3://bucket/file", result);
+ }
+
+ @Test
+ void testGetStorageNameReturnsBroker() {
+ BrokerProperties props = BrokerProperties.of("broker", new
HashMap<>());
+ Assertions.assertEquals("BROKER", props.getStorageName());
+ }
+}
diff --git
a/fe/fe-core/src/test/java/org/apache/doris/external/iceberg/IcebergHadoopCatalogTest.java
b/fe/fe-core/src/test/java/org/apache/doris/external/iceberg/IcebergHadoopCatalogTest.java
index 1ae9ee7e50c..d1e8d2d459a 100644
---
a/fe/fe-core/src/test/java/org/apache/doris/external/iceberg/IcebergHadoopCatalogTest.java
+++
b/fe/fe-core/src/test/java/org/apache/doris/external/iceberg/IcebergHadoopCatalogTest.java
@@ -19,6 +19,7 @@ package org.apache.doris.external.iceberg;
import org.apache.doris.common.UserException;
import org.apache.doris.datasource.property.PropertyConverter;
+import org.apache.doris.datasource.property.storage.StorageProperties;
import org.apache.doris.fs.FileSystemFactory;
import org.apache.doris.fs.remote.dfs.DFSFileSystem;
@@ -50,7 +51,7 @@ public class IcebergHadoopCatalogTest {
properties.put("cos.region", "ap-beijing");
Map<String, String> hadoopProps =
PropertyConverter.convertToHadoopFSProperties(properties);
String pathStr = "cosn://bucket1/namespace";
- DFSFileSystem fs = (DFSFileSystem) FileSystemFactory.get(hadoopProps);
+ DFSFileSystem fs = (DFSFileSystem)
FileSystemFactory.get(StorageProperties.createPrimary(hadoopProps));
nativeFs = fs.nativeFileSystem(pathStr);
RemoteIterator<FileStatus> it = nativeFs.listStatusIterator(new
Path(pathStr));
diff --git
a/fe/fe-core/src/test/java/org/apache/doris/fs/obj/S3FileSystemTest.java
b/fe/fe-core/src/test/java/org/apache/doris/fs/obj/S3FileSystemTest.java
index 83fd3598360..f3db1561490 100644
--- a/fe/fe-core/src/test/java/org/apache/doris/fs/obj/S3FileSystemTest.java
+++ b/fe/fe-core/src/test/java/org/apache/doris/fs/obj/S3FileSystemTest.java
@@ -104,10 +104,12 @@ public class S3FileSystemTest {
return mockedClient;
}
};
- S3ObjStorage mockedStorage = new
S3ObjStorage((AbstractS3CompatibleProperties)
StorageProperties.createPrimary(properties));
+ AbstractS3CompatibleProperties s3CompatibleProperties =
+ (AbstractS3CompatibleProperties)
StorageProperties.createPrimary(properties);
+ S3ObjStorage mockedStorage = new
S3ObjStorage(s3CompatibleProperties);
Assertions.assertTrue(mockedStorage.getClient() instanceof
MockedS3Client);
// inject storage to file system.
- fileSystem = (S3FileSystem) FileSystemFactory.get(properties);
+ fileSystem = (S3FileSystem)
FileSystemFactory.get(s3CompatibleProperties);
new MockUp<S3FileSystem>(S3FileSystem.class) {
@Mock
public Status globList(String remotePath, List<RemoteFile>
result, boolean fileNameOnly) {
@@ -126,7 +128,7 @@ public class S3FileSystemTest {
};
} else {
// can also real file system to test.
- fileSystem = (S3FileSystem) FileSystemFactory.get(properties);
+ fileSystem = (S3FileSystem)
FileSystemFactory.get(StorageProperties.createPrimary(properties));
}
testFile = bucket + basePath + "/Ode_to_the_West_Wind";
Assertions.assertEquals(Status.OK, fileSystem.directUpload(content,
testFile));
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
