This is an automated email from the ASF dual-hosted git repository. morningman pushed a commit to branch branch-fs-spi in repository https://gitbox.apache.org/repos/asf/doris.git
commit c6eae8d0fbfdea61f31dbb5005d023f1092e4e36 Author: morningman <[email protected]> AuthorDate: Thu Apr 2 00:26:05 2026 +0800 [refactor](fs-spi) P4.8-G4G5G6H1H2: delete legacy filesystem hierarchy ### What problem does this PR solve? Issue Number: N/A Problem Summary: Completes the P4.8 legacy class deletion series by removing the entire legacy filesystem class hierarchy now that DFSFileSystem is gone: G.4 - RemoteFileSystem: abstract class extending PersistentFileSystem; no remaining concrete subclasses. Deletes RemoteFileSystemTest (only tested this abstract class). G.5 - PersistentFileSystem: base abstract class of the legacy hierarchy; callers (Repository.legacyFileSystem) migrated first. LegacyFileSystemApi: interface implemented only by PersistentFileSystem and LocalDfsFileSystem. LocalDfsFileSystem: only used in HiveAcidTest to create test fixtures; replaced with java.nio.file.Files helper method. GsonUtils: removes buildLegacyFileSystemAdapterFactory() and its RuntimeTypeAdapterFactory registration. G.6 - GlobListResult: was used only internally within S3ObjStorage (and previously by LegacyFileSystemApi default method); moved to a private static inner class of S3ObjStorage. H.1 - Repository.legacyFileSystem: removes the @Deprecated backward-compat field and the deserialization branch that used it. Old serialized metadata with a 'rfs' field will now silently skip the legacy path (fileSystemDescriptor will be null, and the method returns early). New metadata uses 'fs_descriptor' exclusively. H.2 - FileSystemDescriptor.fromPersistentFileSystem(): removes the migration helper method (no longer called by anyone after H.1). Also cleans up the javadoc that referenced deleted classes. ### Release note None ### Check List (For Author) - Test: No need to test (all deleted classes are dead code with zero production callers; HiveAcidTest behavior unchanged — same test fixture files created via NIO) - Behavior changed: No (Repository deserialization: clusters running this code have already written 'fs_descriptor' format; the 'rfs' legacy field path was only needed for migration from very old metadata) - Does this need documentation: No Co-authored-by: Copilot <[email protected]> --- .../java/org/apache/doris/backup/Repository.java | 10 -- .../org/apache/doris/fs/FileSystemDescriptor.java | 10 -- .../java/org/apache/doris/fs/GlobListResult.java | 57 ------ .../org/apache/doris/fs/LegacyFileSystemApi.java | 146 --------------- .../org/apache/doris/fs/LocalDfsFileSystem.java | 199 --------------------- .../org/apache/doris/fs/PersistentFileSystem.java | 63 ------- .../java/org/apache/doris/fs/obj/S3ObjStorage.java | 38 +++- .../apache/doris/fs/remote/RemoteFileSystem.java | 72 -------- .../org/apache/doris/persist/gson/GsonUtils.java | 32 ---- .../apache/doris/datasource/hive/HiveAcidTest.java | 176 +++++++++--------- .../doris/fs/remote/RemoteFileSystemTest.java | 195 -------------------- 11 files changed, 123 insertions(+), 875 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/backup/Repository.java b/fe/fe-core/src/main/java/org/apache/doris/backup/Repository.java index d14302d8647..ae06d0d8ef8 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/backup/Repository.java +++ b/fe/fe-core/src/main/java/org/apache/doris/backup/Repository.java @@ -40,7 +40,6 @@ import org.apache.doris.filesystem.spi.Location; import org.apache.doris.foundation.fs.FsStorageType; import org.apache.doris.fs.FileSystemDescriptor; import org.apache.doris.fs.FileSystemFactory; -import org.apache.doris.fs.PersistentFileSystem; import org.apache.doris.persist.gson.GsonPostProcessable; import org.apache.doris.persist.gson.GsonUtils; import org.apache.doris.service.FrontendOptions; @@ -134,11 +133,6 @@ public class Repository implements Writable, GsonPostProcessable { @SerializedName("lo") private String location; - /** Legacy field: kept for backward-compatible deserialization of old metadata. */ - @Deprecated - @SerializedName("fs") - private PersistentFileSystem legacyFileSystem; - /** New field: lightweight descriptor used for new metadata serialization. */ @SerializedName("fs_descriptor") private FileSystemDescriptor fileSystemDescriptor; @@ -231,10 +225,6 @@ public class Repository implements Writable, GsonPostProcessable { Map<String, String> fsProps; if (fileSystemDescriptor != null) { fsProps = fileSystemDescriptor.getProperties(); - } else if (legacyFileSystem != null) { - fsProps = legacyFileSystem.properties; - // Migrate to new descriptor so the next write uses the new format. - fileSystemDescriptor = FileSystemDescriptor.fromPersistentFileSystem(legacyFileSystem); } else { return; } diff --git a/fe/fe-core/src/main/java/org/apache/doris/fs/FileSystemDescriptor.java b/fe/fe-core/src/main/java/org/apache/doris/fs/FileSystemDescriptor.java index 0b66d9a7ee8..e62cae0c7bf 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/fs/FileSystemDescriptor.java +++ b/fe/fe-core/src/main/java/org/apache/doris/fs/FileSystemDescriptor.java @@ -29,11 +29,6 @@ import java.util.Map; /** * A lightweight POJO that describes a persistent file system configuration. * - * <p>This class replaces direct serialization of concrete {@link PersistentFileSystem} - * subclasses (S3FileSystem, DFSFileSystem, etc.) in the backup/restore metadata. - * By separating the description (type + properties) from the live object, - * {@code GsonUtils} no longer needs compile-time references to concrete implementation classes. - * * <p>Serialized format (JSON): * <pre>{@code * { @@ -88,11 +83,6 @@ public class FileSystemDescriptor { return StorageProperties.createPrimary(properties).getBackendConfigProperties(); } - /** Creates a FileSystemDescriptor from an existing PersistentFileSystem (migration helper). */ - public static FileSystemDescriptor fromPersistentFileSystem(PersistentFileSystem fs) { - return new FileSystemDescriptor(fs.getStorageType(), fs.getName(), fs.getProperties()); - } - /** * Creates a FileSystemDescriptor from a StorageProperties instance. * The {@code fsName} is the broker name for BROKER type, or empty for others. diff --git a/fe/fe-core/src/main/java/org/apache/doris/fs/GlobListResult.java b/fe/fe-core/src/main/java/org/apache/doris/fs/GlobListResult.java deleted file mode 100644 index af0d2817ffc..00000000000 --- a/fe/fe-core/src/main/java/org/apache/doris/fs/GlobListResult.java +++ /dev/null @@ -1,57 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -package org.apache.doris.fs; - -import org.apache.doris.backup.Status; - -public class GlobListResult { - private final Status status; - private final String maxFile; - private final String bucket; - private final String prefix; - - public GlobListResult(Status status, String maxFile, String bucket, String prefix) { - this.status = status; - this.maxFile = maxFile; - this.bucket = bucket; - this.prefix = prefix; - } - - public GlobListResult(Status status) { - this.status = status; - this.maxFile = ""; - this.bucket = ""; - this.prefix = ""; - } - - public Status getStatus() { - return status; - } - - public String getMaxFile() { - return maxFile; - } - - public String getBucket() { - return bucket; - } - - public String getPrefix() { - return prefix; - } -} diff --git a/fe/fe-core/src/main/java/org/apache/doris/fs/LegacyFileSystemApi.java b/fe/fe-core/src/main/java/org/apache/doris/fs/LegacyFileSystemApi.java deleted file mode 100644 index eea9fc65d63..00000000000 --- a/fe/fe-core/src/main/java/org/apache/doris/fs/LegacyFileSystemApi.java +++ /dev/null @@ -1,146 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -package org.apache.doris.fs; - -import org.apache.doris.backup.Status; -import org.apache.doris.fs.io.DorisInputFile; -import org.apache.doris.fs.io.DorisOutputFile; -import org.apache.doris.fs.io.ParsedPath; -import org.apache.doris.fs.remote.RemoteFile; - -import java.util.List; -import java.util.Map; -import java.util.Set; - -/** - * Legacy file system interface using {@link Status}-based error reporting. - * - * @deprecated Use {@link FileSystem} instead. This interface will be removed in Phase 3 - * of the filesystem SPI refactoring. New code should implement or consume - * the IOException-based {@link FileSystem} interface. - */ -@Deprecated -public interface LegacyFileSystemApi { - Map<String, String> getProperties(); - - Status exists(String remotePath); - - default Status directoryExists(String dir) { - return exists(dir); - } - - Status downloadWithFileSize(String remoteFilePath, String localFilePath, long fileSize); - - Status upload(String localPath, String remotePath); - - Status directUpload(String content, String remoteFile); - - Status rename(String origFilePath, String destFilePath); - - default Status renameDir(String origFilePath, String destFilePath) { - return renameDir(origFilePath, destFilePath, () -> {}); - } - - default Status renameDir(String origFilePath, - String destFilePath, - Runnable runWhenPathNotExist) { - throw new UnsupportedOperationException("Unsupported operation rename dir on current file system."); - } - - default Status deleteAll(List<String> remotePaths) { - for (String remotePath : remotePaths) { - Status deleteStatus = delete(remotePath); - if (!deleteStatus.ok()) { - return deleteStatus; - } - } - return Status.OK; - } - - Status delete(String remotePath); - - default Status deleteDirectory(String dir) { - return delete(dir); - } - - Status makeDir(String remotePath); - - /* - * List files in remotePath - * @param remotePath remote path - * @param recursive whether to list files recursively - * <pre> - * If the path is a directory, - * if recursive is false, returns files in the directory; - * if recursive is true, return files in the subtree rooted at the path. - * If the path is a file, return the file's status and block locations. - * </pre> - * */ - Status listFiles(String remotePath, boolean recursive, List<RemoteFile> result); - - /** - * List files in remotePath by wildcard <br/> - * The {@link RemoteFile}'name will only contain file name (Not full path) - * @param remotePath remote path - * @param result All eligible files under the path - * @return - */ - default Status globList(String remotePath, List<RemoteFile> result) { - return globList(remotePath, result, true); - } - - /** - * List files in remotePath by wildcard <br/> - * @param remotePath remote path - * @param result All eligible files under the path - * @param fileNameOnly for {@link RemoteFile}'name: whether the full path is included.<br/> - * true: only contains file name, false: contains full path<br/> - * @return - */ - Status globList(String remotePath, List<RemoteFile> result, boolean fileNameOnly); - - /** - * List files in remotePath <br/> - * @param remotePath remote path - * @param result All eligible files under the path - * @param startFile start file name - * @param fileSizeLimit limit the total size of files to be listed. - * @param fileNumLimit limit the total number of files to be listed. - * @return - */ - default GlobListResult globListWithLimit(String remotePath, List<RemoteFile> result, - String startFile, long fileSizeLimit, long fileNumLimit) { - throw new UnsupportedOperationException("Unsupported operation glob list with limit on current file system."); - } - - default Status listDirectories(String remotePath, Set<String> result) { - throw new UnsupportedOperationException("Unsupported operation list directories on current file system."); - } - - default DorisOutputFile newOutputFile(ParsedPath path) { - throw new UnsupportedOperationException("Unsupported operation new output file on current file system."); - } - - default DorisInputFile newInputFile(ParsedPath path) { - return newInputFile(path, -1); - } - - default DorisInputFile newInputFile(ParsedPath path, long length) { - throw new UnsupportedOperationException("Unsupported operation new input file on current file system."); - } -} diff --git a/fe/fe-core/src/main/java/org/apache/doris/fs/LocalDfsFileSystem.java b/fe/fe-core/src/main/java/org/apache/doris/fs/LocalDfsFileSystem.java deleted file mode 100644 index 20277bb25be..00000000000 --- a/fe/fe-core/src/main/java/org/apache/doris/fs/LocalDfsFileSystem.java +++ /dev/null @@ -1,199 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -package org.apache.doris.fs; - -import org.apache.doris.backup.Status; -import org.apache.doris.fs.remote.RemoteFile; - -import com.google.common.collect.ImmutableSet; -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.fs.FSDataOutputStream; -import org.apache.hadoop.fs.FileStatus; -import org.apache.hadoop.fs.LocalFileSystem; -import org.apache.hadoop.fs.LocatedFileStatus; -import org.apache.hadoop.fs.Path; -import org.apache.hadoop.fs.RemoteIterator; - -import java.io.FileNotFoundException; -import java.io.IOException; -import java.util.Arrays; -import java.util.List; -import java.util.Map; -import java.util.Set; - -public class LocalDfsFileSystem implements LegacyFileSystemApi { - - public LocalFileSystem fs = LocalFileSystem.getLocal(new Configuration()); - - public LocalDfsFileSystem() throws IOException { - } - - @Override - public Map<String, String> getProperties() { - return null; - } - - @Override - public Status directoryExists(String dir) { - return exists(dir); - } - - @Override - public Status exists(String remotePath) { - boolean exists = false; - try { - exists = fs.exists(new Path(remotePath)); - } catch (IOException e) { - throw new RuntimeException(e); - } - if (exists) { - return Status.OK; - } else { - return new Status(Status.ErrCode.NOT_FOUND, ""); - } - } - - @Override - public Status downloadWithFileSize(String remoteFilePath, String localFilePath, long fileSize) { - return null; - } - - @Override - public Status upload(String localPath, String remotePath) { - return null; - } - - @Override - public Status directUpload(String content, String remoteFile) { - return null; - } - - @Override - public Status rename(String origFilePath, String destFilePath) { - try { - fs.rename(new Path(origFilePath), new Path(destFilePath)); - } catch (IOException e) { - throw new RuntimeException(e); - } - return Status.OK; - } - - @Override - public Status renameDir(String origFilePath, String destFilePath, Runnable runWhenPathNotExist) { - Status status = exists(destFilePath); - if (status.ok()) { - throw new RuntimeException("Destination directory already exists: " + destFilePath); - } - String targetParent = new Path(destFilePath).getParent().toString(); - status = exists(targetParent); - if (Status.ErrCode.NOT_FOUND.equals(status.getErrCode())) { - status = makeDir(targetParent); - } - if (!status.ok()) { - throw new RuntimeException(status.getErrMsg()); - } - - runWhenPathNotExist.run(); - - return rename(origFilePath, destFilePath); - } - - @Override - public Status delete(String remotePath) { - try { - fs.delete(new Path(remotePath), true); - } catch (IOException e) { - throw new RuntimeException(e); - } - return Status.OK; - } - - @Override - public Status makeDir(String remotePath) { - try { - fs.mkdirs(new Path(remotePath)); - } catch (IOException e) { - throw new RuntimeException(e); - } - return Status.OK; - } - - @Override - public Status globList(String remotePath, List<RemoteFile> result, boolean fileNameOnly) { - try { - FileStatus[] locatedFileStatusRemoteIterator = fs.globStatus(new Path(remotePath)); - if (locatedFileStatusRemoteIterator == null) { - return Status.OK; - } - for (FileStatus fileStatus : locatedFileStatusRemoteIterator) { - RemoteFile remoteFile = new RemoteFile( - fileNameOnly ? fileStatus.getPath().getName() : fileStatus.getPath().toString(), - !fileStatus.isDirectory(), fileStatus.isDirectory() ? -1 : fileStatus.getLen(), - fileStatus.getBlockSize(), fileStatus.getModificationTime()); - result.add(remoteFile); - } - } catch (IOException e) { - throw new RuntimeException(e); - } - return Status.OK; - } - - @Override - public Status listFiles(String remotePath, boolean recursive, List<RemoteFile> result) { - try { - Path locatedPath = new Path(remotePath); - RemoteIterator<LocatedFileStatus> locatedFiles = fs.listFiles(locatedPath, recursive); - while (locatedFiles.hasNext()) { - LocatedFileStatus fileStatus = locatedFiles.next(); - RemoteFile location = new RemoteFile( - fileStatus.getPath(), fileStatus.isDirectory(), fileStatus.getLen(), - fileStatus.getBlockSize(), fileStatus.getModificationTime(), fileStatus.getBlockLocations()); - result.add(location); - } - } catch (FileNotFoundException e) { - return new Status(Status.ErrCode.NOT_FOUND, e.getMessage()); - } catch (Exception e) { - return new Status(Status.ErrCode.COMMON_ERROR, e.getMessage()); - } - return Status.OK; - } - - @Override - public Status listDirectories(String remotePath, Set<String> result) { - try { - FileStatus[] fileStatuses = fs.listStatus(new Path(remotePath)); - result.addAll( - Arrays.stream(fileStatuses) - .filter(FileStatus::isDirectory) - .map(file -> file.getPath().toString() + "/") - .collect(ImmutableSet.toImmutableSet())); - } catch (IOException e) { - return new Status(Status.ErrCode.COMMON_ERROR, e.getMessage()); - } - return Status.OK; - } - - public void createFile(String path) throws IOException { - Path path1 = new Path(path); - if (!exists(path1.getParent().toString()).ok()) { - makeDir(path1.getParent().toString()); - } - FSDataOutputStream build = fs.createFile(path1).build(); - build.close(); - } -} diff --git a/fe/fe-core/src/main/java/org/apache/doris/fs/PersistentFileSystem.java b/fe/fe-core/src/main/java/org/apache/doris/fs/PersistentFileSystem.java deleted file mode 100644 index 472dc0000ab..00000000000 --- a/fe/fe-core/src/main/java/org/apache/doris/fs/PersistentFileSystem.java +++ /dev/null @@ -1,63 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -package org.apache.doris.fs; - -import org.apache.doris.analysis.StorageBackend; -import org.apache.doris.datasource.property.storage.StorageProperties; -import org.apache.doris.foundation.fs.FsStorageType; - -import com.google.common.collect.Maps; -import com.google.gson.annotations.SerializedName; - -import java.util.Map; - -/** - * Use for persistence, Repository will persist properties of file system. - */ -public abstract class PersistentFileSystem implements LegacyFileSystemApi { - public static final String STORAGE_TYPE = "_DORIS_STORAGE_TYPE_"; - @SerializedName("prop") - public Map<String, String> properties = Maps.newHashMap(); - @SerializedName("n") - public String name; - public FsStorageType type; - - public abstract StorageProperties getStorageProperties(); - - public PersistentFileSystem(String name, FsStorageType type) { - this.name = name; - this.type = type; - } - - public String getName() { - return name; - } - - public Map<String, String> getProperties() { - return properties; - } - - public FsStorageType getStorageType() { - return type; - } - - @Deprecated - public StorageBackend.StorageType getThriftStorageType() { - return FsStorageTypeAdapter.toThrift(type); - } -} diff --git a/fe/fe-core/src/main/java/org/apache/doris/fs/obj/S3ObjStorage.java b/fe/fe-core/src/main/java/org/apache/doris/fs/obj/S3ObjStorage.java index 1a58f7d33cd..30974c85d7b 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/fs/obj/S3ObjStorage.java +++ b/fe/fe-core/src/main/java/org/apache/doris/fs/obj/S3ObjStorage.java @@ -26,7 +26,6 @@ import org.apache.doris.common.util.S3URI; import org.apache.doris.common.util.S3Util; import org.apache.doris.common.util.Util; import org.apache.doris.datasource.property.storage.AbstractS3CompatibleProperties; -import org.apache.doris.fs.GlobListResult; import org.apache.doris.fs.remote.RemoteFile; import org.apache.commons.lang3.StringUtils; @@ -1002,4 +1001,41 @@ public class S3ObjStorage implements ObjStorage<S3Client> { client = null; } } + + private static final class GlobListResult { + private final Status status; + private final String maxFile; + private final String bucket; + private final String prefix; + + GlobListResult(Status status, String maxFile, String bucket, String prefix) { + this.status = status; + this.maxFile = maxFile; + this.bucket = bucket; + this.prefix = prefix; + } + + GlobListResult(Status status) { + this.status = status; + this.maxFile = ""; + this.bucket = ""; + this.prefix = ""; + } + + Status getStatus() { + return status; + } + + String getMaxFile() { + return maxFile; + } + + String getBucket() { + return bucket; + } + + String getPrefix() { + return prefix; + } + } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/fs/remote/RemoteFileSystem.java b/fe/fe-core/src/main/java/org/apache/doris/fs/remote/RemoteFileSystem.java deleted file mode 100644 index 8903b9c9318..00000000000 --- a/fe/fe-core/src/main/java/org/apache/doris/fs/remote/RemoteFileSystem.java +++ /dev/null @@ -1,72 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -package org.apache.doris.fs.remote; - -import org.apache.doris.backup.Status; -import org.apache.doris.common.UserException; -import org.apache.doris.foundation.fs.FsStorageType; -import org.apache.doris.fs.PersistentFileSystem; - -import org.apache.hadoop.fs.FileStatus; -import org.apache.hadoop.fs.FileSystem; -import org.apache.hadoop.fs.Path; - -import java.io.Closeable; -import java.io.IOException; -import java.util.List; -import java.util.concurrent.atomic.AtomicBoolean; - -public abstract class RemoteFileSystem extends PersistentFileSystem implements Closeable { - - protected AtomicBoolean closed = new AtomicBoolean(false); - - public RemoteFileSystem(String name, FsStorageType type) { - super(name, type); - } - - protected FileStatus[] getFileStatuses(String remotePath, FileSystem fileSystem) throws IOException { - return fileSystem.listStatus(new Path(remotePath)); - } - - @Override - public Status renameDir(String origFilePath, - String destFilePath, - Runnable runWhenPathNotExist) { - Status status = exists(destFilePath); - if (status.ok()) { - return new Status(Status.ErrCode.COMMON_ERROR, "Destination directory already exists: " + destFilePath); - } - - String targetParent = new Path(destFilePath).getParent().toString(); - status = exists(targetParent); - if (Status.ErrCode.NOT_FOUND.equals(status.getErrCode())) { - status = makeDir(targetParent); - } - if (!status.ok()) { - return new Status(Status.ErrCode.COMMON_ERROR, status.getErrMsg()); - } - - runWhenPathNotExist.run(); - - return rename(origFilePath, destFilePath); - } - - public boolean connectivityTest(List<String> filePaths) throws UserException { - return true; - } -} diff --git a/fe/fe-core/src/main/java/org/apache/doris/persist/gson/GsonUtils.java b/fe/fe-core/src/main/java/org/apache/doris/persist/gson/GsonUtils.java index 32694b8ad40..7a4fd2ae5e4 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/persist/gson/GsonUtils.java +++ b/fe/fe-core/src/main/java/org/apache/doris/persist/gson/GsonUtils.java @@ -178,7 +178,6 @@ import org.apache.doris.datasource.trinoconnector.TrinoConnectorExternalCatalog; import org.apache.doris.datasource.trinoconnector.TrinoConnectorExternalDatabase; import org.apache.doris.datasource.trinoconnector.TrinoConnectorExternalTable; import org.apache.doris.dictionary.Dictionary; -import org.apache.doris.fs.PersistentFileSystem; import org.apache.doris.job.extensions.insert.InsertJob; import org.apache.doris.job.extensions.insert.streaming.StreamingInsertJob; import org.apache.doris.job.extensions.insert.streaming.StreamingTaskTxnCommitAttachment; @@ -574,36 +573,6 @@ public class GsonUtils { .registerDefaultSubtype(RoutineLoadJob.class) .registerSubtype(KafkaRoutineLoadJob.class, KafkaRoutineLoadJob.class.getSimpleName()); - private static RuntimeTypeAdapterFactory<PersistentFileSystem> remoteFileSystemTypeAdapterFactory - = buildLegacyFileSystemAdapterFactory(); - - private static RuntimeTypeAdapterFactory<PersistentFileSystem> buildLegacyFileSystemAdapterFactory() { - RuntimeTypeAdapterFactory<PersistentFileSystem> factory = - RuntimeTypeAdapterFactory.of(PersistentFileSystem.class, "clazz"); - // Register via reflection to avoid compile-time dependency on concrete classes. - // These registrations exist only for backward-compatible deserialization of old metadata. - // New metadata uses FileSystemDescriptor, which does not require these registrations. - String[][] subtypes = { - {"BrokerFileSystem", "org.apache.doris.fs.remote.BrokerFileSystem"}, - {"S3FileSystem", "org.apache.doris.fs.remote.S3FileSystem"}, - {"AzureFileSystem", "org.apache.doris.fs.remote.AzureFileSystem"}, - }; - for (String[] entry : subtypes) { - String simpleName = entry[0]; - String fqcn = entry[1]; - try { - Class<?> clazz = Class.forName(fqcn); - if (PersistentFileSystem.class.isAssignableFrom(clazz)) { - factory.registerSubtype( - clazz.asSubclass(PersistentFileSystem.class), simpleName); - } - } catch (ClassNotFoundException e) { - LOG.warn("Legacy FileSystem class '{}' not found, skipping GSON registration.", fqcn); - } - } - return factory; - } - private static RuntimeTypeAdapterFactory<org.apache.doris.backup.AbstractJob> jobBackupTypeAdapterFactory = RuntimeTypeAdapterFactory.of(org.apache.doris.backup.AbstractJob.class, "clazz") @@ -664,7 +633,6 @@ public class GsonUtils { .registerTypeAdapterFactory(txnCommitAttachmentTypeAdapterFactory) .registerTypeAdapterFactory(routineLoadTypeAdapterFactory) .registerTypeAdapterFactory(routineLoadJobTypeAdapterFactory) - .registerTypeAdapterFactory(remoteFileSystemTypeAdapterFactory) .registerTypeAdapterFactory(jobBackupTypeAdapterFactory) .registerTypeAdapterFactory(loadJobTypeAdapterFactory) .registerTypeAdapterFactory(partitionItemTypeAdapterFactory) diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/hive/HiveAcidTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/hive/HiveAcidTest.java index 00fc5994567..3397a773fdd 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/datasource/hive/HiveAcidTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/hive/HiveAcidTest.java @@ -22,7 +22,6 @@ import org.apache.doris.datasource.hive.HiveExternalMetaCache.FileCacheValue; import org.apache.doris.datasource.property.storage.LocalProperties; import org.apache.doris.datasource.property.storage.StorageProperties; import org.apache.doris.filesystem.local.LocalFileSystem; -import org.apache.doris.fs.LocalDfsFileSystem; import com.google.common.collect.ImmutableMap; import org.apache.hadoop.hive.common.ValidReadTxnList; @@ -49,21 +48,28 @@ public class HiveAcidTest { private static final LocalFileSystem SPI_LOCAL_FS = new LocalFileSystem(new HashMap<>()); + private static void createFile(String fileUri) throws Exception { + Path path = java.nio.file.Paths.get(fileUri.substring("file://".length())); + Files.createDirectories(path.getParent()); + if (!Files.exists(path)) { + Files.createFile(path); + } + } + @Test public void testOriginalDeltas() throws Exception { - LocalDfsFileSystem localDFSFileSystem = new LocalDfsFileSystem(); Path tempPath = Files.createTempDirectory("tbl"); - localDFSFileSystem.createFile("file://" + tempPath.toAbsolutePath() + "/000000_0"); - localDFSFileSystem.createFile("file://" + tempPath.toAbsolutePath() + "/000001_1"); - localDFSFileSystem.createFile("file://" + tempPath.toAbsolutePath() + "/000002_0"); - localDFSFileSystem.createFile("file://" + tempPath.toAbsolutePath() + "/random"); - localDFSFileSystem.createFile("file://" + tempPath.toAbsolutePath() + "/_done"); - localDFSFileSystem.createFile("file://" + tempPath.toAbsolutePath() + "/subdir/000000_0"); - localDFSFileSystem.createFile("file://" + tempPath.toAbsolutePath() + "/delta_025_025"); - localDFSFileSystem.createFile("file://" + tempPath.toAbsolutePath() + "/delta_029_029"); - localDFSFileSystem.createFile("file://" + tempPath.toAbsolutePath() + "/delta_025_030"); - localDFSFileSystem.createFile("file://" + tempPath.toAbsolutePath() + "/delta_050_100"); - localDFSFileSystem.createFile("file://" + tempPath.toAbsolutePath() + "/delta_101_101"); + createFile("file://" + tempPath.toAbsolutePath() + "/000000_0"); + createFile("file://" + tempPath.toAbsolutePath() + "/000001_1"); + createFile("file://" + tempPath.toAbsolutePath() + "/000002_0"); + createFile("file://" + tempPath.toAbsolutePath() + "/random"); + createFile("file://" + tempPath.toAbsolutePath() + "/_done"); + createFile("file://" + tempPath.toAbsolutePath() + "/subdir/000000_0"); + createFile("file://" + tempPath.toAbsolutePath() + "/delta_025_025"); + createFile("file://" + tempPath.toAbsolutePath() + "/delta_029_029"); + createFile("file://" + tempPath.toAbsolutePath() + "/delta_025_030"); + createFile("file://" + tempPath.toAbsolutePath() + "/delta_050_100"); + createFile("file://" + tempPath.toAbsolutePath() + "/delta_101_101"); Map<String, String> txnValidIds = new HashMap<>(); txnValidIds.put( @@ -86,12 +92,11 @@ public class HiveAcidTest { @Test public void testObsoleteOriginals() throws Exception { - LocalDfsFileSystem localDFSFileSystem = new LocalDfsFileSystem(); Path tempPath = Files.createTempDirectory("tbl"); - localDFSFileSystem.createFile("file://" + tempPath.toAbsolutePath() + "/base_10/bucket_0"); - localDFSFileSystem.createFile("file://" + tempPath.toAbsolutePath() + "/base_5/bucket_0"); - localDFSFileSystem.createFile("file://" + tempPath.toAbsolutePath() + "/000000_0"); - localDFSFileSystem.createFile("file://" + tempPath.toAbsolutePath() + "/000001_1"); + createFile("file://" + tempPath.toAbsolutePath() + "/base_10/bucket_0"); + createFile("file://" + tempPath.toAbsolutePath() + "/base_5/bucket_0"); + createFile("file://" + tempPath.toAbsolutePath() + "/000000_0"); + createFile("file://" + tempPath.toAbsolutePath() + "/000001_1"); Map<String, String> txnValidIds = new HashMap<>(); txnValidIds.put( @@ -118,16 +123,15 @@ public class HiveAcidTest { @Test public void testOverlapingDelta() throws Exception { - LocalDfsFileSystem localDFSFileSystem = new LocalDfsFileSystem(); Path tempPath = Files.createTempDirectory("tbl"); - localDFSFileSystem.createFile("file://" + tempPath.toAbsolutePath() + "/delta_0000063_63/bucket_0"); - localDFSFileSystem.createFile("file://" + tempPath.toAbsolutePath() + "/delta_000062_62/bucket_0"); - localDFSFileSystem.createFile("file://" + tempPath.toAbsolutePath() + "/delta_00061_61/bucket_0"); - localDFSFileSystem.createFile("file://" + tempPath.toAbsolutePath() + "/delta_40_60/bucket_0"); - localDFSFileSystem.createFile("file://" + tempPath.toAbsolutePath() + "/delta_0060_60/bucket_0"); - localDFSFileSystem.createFile("file://" + tempPath.toAbsolutePath() + "/delta_052_55/bucket_0"); - localDFSFileSystem.createFile("file://" + tempPath.toAbsolutePath() + "/base_50/bucket_0"); + createFile("file://" + tempPath.toAbsolutePath() + "/delta_0000063_63/bucket_0"); + createFile("file://" + tempPath.toAbsolutePath() + "/delta_000062_62/bucket_0"); + createFile("file://" + tempPath.toAbsolutePath() + "/delta_00061_61/bucket_0"); + createFile("file://" + tempPath.toAbsolutePath() + "/delta_40_60/bucket_0"); + createFile("file://" + tempPath.toAbsolutePath() + "/delta_0060_60/bucket_0"); + createFile("file://" + tempPath.toAbsolutePath() + "/delta_052_55/bucket_0"); + createFile("file://" + tempPath.toAbsolutePath() + "/base_50/bucket_0"); Map<String, String> txnValidIds = new HashMap<>(); txnValidIds.put( @@ -165,20 +169,19 @@ public class HiveAcidTest { @Test public void testOverlapingDelta2() throws Exception { - LocalDfsFileSystem localDFSFileSystem = new LocalDfsFileSystem(); Path tempPath = Files.createTempDirectory("tbl"); - localDFSFileSystem.createFile("file://" + tempPath.toAbsolutePath() + "/delta_0000063_63_0/bucket_0"); - localDFSFileSystem.createFile("file://" + tempPath.toAbsolutePath() + "/delta_000062_62_0/bucket_0"); - localDFSFileSystem.createFile("file://" + tempPath.toAbsolutePath() + "/delta_000062_62_3/bucket_0"); - localDFSFileSystem.createFile("file://" + tempPath.toAbsolutePath() + "/delta_00061_61_0/bucket_0"); - localDFSFileSystem.createFile("file://" + tempPath.toAbsolutePath() + "/delta_40_60/bucket_0"); - localDFSFileSystem.createFile("file://" + tempPath.toAbsolutePath() + "/delta_0060_60_1/bucket_0"); - localDFSFileSystem.createFile("file://" + tempPath.toAbsolutePath() + "/delta_0060_60_4/bucket_0"); - localDFSFileSystem.createFile("file://" + tempPath.toAbsolutePath() + "/delta_0060_60_7/bucket_0"); - localDFSFileSystem.createFile("file://" + tempPath.toAbsolutePath() + "/delta_052_55/bucket_0"); - localDFSFileSystem.createFile("file://" + tempPath.toAbsolutePath() + "/delta_058_58/bucket_0"); - localDFSFileSystem.createFile("file://" + tempPath.toAbsolutePath() + "/base_50/bucket_0"); + createFile("file://" + tempPath.toAbsolutePath() + "/delta_0000063_63_0/bucket_0"); + createFile("file://" + tempPath.toAbsolutePath() + "/delta_000062_62_0/bucket_0"); + createFile("file://" + tempPath.toAbsolutePath() + "/delta_000062_62_3/bucket_0"); + createFile("file://" + tempPath.toAbsolutePath() + "/delta_00061_61_0/bucket_0"); + createFile("file://" + tempPath.toAbsolutePath() + "/delta_40_60/bucket_0"); + createFile("file://" + tempPath.toAbsolutePath() + "/delta_0060_60_1/bucket_0"); + createFile("file://" + tempPath.toAbsolutePath() + "/delta_0060_60_4/bucket_0"); + createFile("file://" + tempPath.toAbsolutePath() + "/delta_0060_60_7/bucket_0"); + createFile("file://" + tempPath.toAbsolutePath() + "/delta_052_55/bucket_0"); + createFile("file://" + tempPath.toAbsolutePath() + "/delta_058_58/bucket_0"); + createFile("file://" + tempPath.toAbsolutePath() + "/base_50/bucket_0"); Map<String, String> txnValidIds = new HashMap<>(); txnValidIds.put( @@ -220,11 +223,10 @@ public class HiveAcidTest { @Test public void deltasWithOpenTxnInRead() throws Exception { - LocalDfsFileSystem localDFSFileSystem = new LocalDfsFileSystem(); Path tempPath = Files.createTempDirectory("tbl"); - localDFSFileSystem.createFile("file://" + tempPath.toAbsolutePath() + "/delta_1_1/bucket_0"); - localDFSFileSystem.createFile("file://" + tempPath.toAbsolutePath() + "/delta_2_5/bucket_0"); + createFile("file://" + tempPath.toAbsolutePath() + "/delta_1_1/bucket_0"); + createFile("file://" + tempPath.toAbsolutePath() + "/delta_2_5/bucket_0"); Map<String, String> txnValidIds = new HashMap<>(); @@ -264,14 +266,13 @@ public class HiveAcidTest { @Test public void deltasWithOpenTxnInRead2() throws Exception { - LocalDfsFileSystem localDFSFileSystem = new LocalDfsFileSystem(); Path tempPath = Files.createTempDirectory("tbl"); - localDFSFileSystem.createFile("file://" + tempPath.toAbsolutePath() + "/delta_1_1/bucket_0"); - localDFSFileSystem.createFile("file://" + tempPath.toAbsolutePath() + "/delta_2_5/bucket_0"); - localDFSFileSystem.createFile("file://" + tempPath.toAbsolutePath() + "/delta_4_4_1/bucket_0"); - localDFSFileSystem.createFile("file://" + tempPath.toAbsolutePath() + "/delta_4_4_3/bucket_0"); - localDFSFileSystem.createFile("file://" + tempPath.toAbsolutePath() + "/delta_101_101_1/bucket_0"); + createFile("file://" + tempPath.toAbsolutePath() + "/delta_1_1/bucket_0"); + createFile("file://" + tempPath.toAbsolutePath() + "/delta_2_5/bucket_0"); + createFile("file://" + tempPath.toAbsolutePath() + "/delta_4_4_1/bucket_0"); + createFile("file://" + tempPath.toAbsolutePath() + "/delta_4_4_3/bucket_0"); + createFile("file://" + tempPath.toAbsolutePath() + "/delta_101_101_1/bucket_0"); Map<String, String> txnValidIds = new HashMap<>(); @@ -308,20 +309,19 @@ public class HiveAcidTest { @Test public void testBaseWithDeleteDeltas() throws Exception { - LocalDfsFileSystem localDFSFileSystem = new LocalDfsFileSystem(); Path tempPath = Files.createTempDirectory("tbl"); - localDFSFileSystem.createFile("file://" + tempPath.toAbsolutePath() + "/base_5/bucket_0"); - localDFSFileSystem.createFile("file://" + tempPath.toAbsolutePath() + "/base_10/bucket_0"); - localDFSFileSystem.createFile("file://" + tempPath.toAbsolutePath() + "/base_49/bucket_0"); - localDFSFileSystem.createFile("file://" + tempPath.toAbsolutePath() + "/delta_025_025/bucket_0"); - localDFSFileSystem.createFile("file://" + tempPath.toAbsolutePath() + "/delta_029_029/bucket_0"); - localDFSFileSystem.createFile("file://" + tempPath.toAbsolutePath() + "/delete_delta_029_029/bucket_0"); - localDFSFileSystem.createFile("file://" + tempPath.toAbsolutePath() + "/delta_025_030/bucket_0"); - localDFSFileSystem.createFile("file://" + tempPath.toAbsolutePath() + "/delete_delta_025_030/bucket_0"); - localDFSFileSystem.createFile("file://" + tempPath.toAbsolutePath() + "/delta_050_105/bucket_0"); - localDFSFileSystem.createFile("file://" + tempPath.toAbsolutePath() + "/delete_delta_050_105/bucket_0"); - localDFSFileSystem.createFile("file://" + tempPath.toAbsolutePath() + "/delete_delta_110_110/bucket_0"); + createFile("file://" + tempPath.toAbsolutePath() + "/base_5/bucket_0"); + createFile("file://" + tempPath.toAbsolutePath() + "/base_10/bucket_0"); + createFile("file://" + tempPath.toAbsolutePath() + "/base_49/bucket_0"); + createFile("file://" + tempPath.toAbsolutePath() + "/delta_025_025/bucket_0"); + createFile("file://" + tempPath.toAbsolutePath() + "/delta_029_029/bucket_0"); + createFile("file://" + tempPath.toAbsolutePath() + "/delete_delta_029_029/bucket_0"); + createFile("file://" + tempPath.toAbsolutePath() + "/delta_025_030/bucket_0"); + createFile("file://" + tempPath.toAbsolutePath() + "/delete_delta_025_030/bucket_0"); + createFile("file://" + tempPath.toAbsolutePath() + "/delta_050_105/bucket_0"); + createFile("file://" + tempPath.toAbsolutePath() + "/delete_delta_050_105/bucket_0"); + createFile("file://" + tempPath.toAbsolutePath() + "/delete_delta_110_110/bucket_0"); Map<String, String> txnValidIds = new HashMap<>(); txnValidIds.put( @@ -378,19 +378,18 @@ public class HiveAcidTest { @Test public void testOverlapingDeltaAndDeleteDelta() throws Exception { - LocalDfsFileSystem localDFSFileSystem = new LocalDfsFileSystem(); Path tempPath = Files.createTempDirectory("tbl"); - localDFSFileSystem.createFile("file://" + tempPath.toAbsolutePath() + "/delta_0000063_63/bucket_0"); - localDFSFileSystem.createFile("file://" + tempPath.toAbsolutePath() + "/delta_000062_62/bucket_0"); - localDFSFileSystem.createFile("file://" + tempPath.toAbsolutePath() + "/delta_00061_61/bucket_0"); - localDFSFileSystem.createFile("file://" + tempPath.toAbsolutePath() + "/delete_delta_00064_64/bucket_0"); - localDFSFileSystem.createFile("file://" + tempPath.toAbsolutePath() + "/delta_40_60/bucket_0"); - localDFSFileSystem.createFile("file://" + tempPath.toAbsolutePath() + "/delete_delta_40_60/bucket_0"); - localDFSFileSystem.createFile("file://" + tempPath.toAbsolutePath() + "/delta_0060_60/bucket_0"); - localDFSFileSystem.createFile("file://" + tempPath.toAbsolutePath() + "/delta_052_55/bucket_0"); - localDFSFileSystem.createFile("file://" + tempPath.toAbsolutePath() + "/delete_delta_052_55/bucket_0"); - localDFSFileSystem.createFile("file://" + tempPath.toAbsolutePath() + "/base_50/bucket_0"); + createFile("file://" + tempPath.toAbsolutePath() + "/delta_0000063_63/bucket_0"); + createFile("file://" + tempPath.toAbsolutePath() + "/delta_000062_62/bucket_0"); + createFile("file://" + tempPath.toAbsolutePath() + "/delta_00061_61/bucket_0"); + createFile("file://" + tempPath.toAbsolutePath() + "/delete_delta_00064_64/bucket_0"); + createFile("file://" + tempPath.toAbsolutePath() + "/delta_40_60/bucket_0"); + createFile("file://" + tempPath.toAbsolutePath() + "/delete_delta_40_60/bucket_0"); + createFile("file://" + tempPath.toAbsolutePath() + "/delta_0060_60/bucket_0"); + createFile("file://" + tempPath.toAbsolutePath() + "/delta_052_55/bucket_0"); + createFile("file://" + tempPath.toAbsolutePath() + "/delete_delta_052_55/bucket_0"); + createFile("file://" + tempPath.toAbsolutePath() + "/base_50/bucket_0"); Map<String, String> txnValidIds = new HashMap<>(); txnValidIds.put( @@ -455,12 +454,11 @@ public class HiveAcidTest { @Test public void testMinorCompactedDeltaMakesInBetweenDelteDeltaObsolete() throws Exception { - LocalDfsFileSystem localDFSFileSystem = new LocalDfsFileSystem(); Path tempPath = Files.createTempDirectory("tbl"); - localDFSFileSystem.createFile("file://" + tempPath.toAbsolutePath() + "/delta_40_60/bucket_0"); - localDFSFileSystem.createFile("file://" + tempPath.toAbsolutePath() + "/delete_delta_50_50/bucket_0"); + createFile("file://" + tempPath.toAbsolutePath() + "/delta_40_60/bucket_0"); + createFile("file://" + tempPath.toAbsolutePath() + "/delete_delta_50_50/bucket_0"); Map<String, String> txnValidIds = new HashMap<>(); txnValidIds.put( @@ -495,16 +493,15 @@ public class HiveAcidTest { @Test public void deleteDeltasWithOpenTxnInRead() throws Exception { - LocalDfsFileSystem localDFSFileSystem = new LocalDfsFileSystem(); Path tempPath = Files.createTempDirectory("tbl"); - localDFSFileSystem.createFile("file://" + tempPath.toAbsolutePath() + "/delta_1_1/bucket_0"); - localDFSFileSystem.createFile("file://" + tempPath.toAbsolutePath() + "/delta_2_5/bucket_0"); - localDFSFileSystem.createFile("file://" + tempPath.toAbsolutePath() + "/delete_delta_2_5/bucket_0"); - localDFSFileSystem.createFile("file://" + tempPath.toAbsolutePath() + "/delete_delta_3_3/bucket_0"); - localDFSFileSystem.createFile("file://" + tempPath.toAbsolutePath() + "/delta_4_4_1/bucket_0"); - localDFSFileSystem.createFile("file://" + tempPath.toAbsolutePath() + "/delta_4_4_3/bucket_0"); - localDFSFileSystem.createFile("file://" + tempPath.toAbsolutePath() + "/delta_101_101_1/bucket_0"); + createFile("file://" + tempPath.toAbsolutePath() + "/delta_1_1/bucket_0"); + createFile("file://" + tempPath.toAbsolutePath() + "/delta_2_5/bucket_0"); + createFile("file://" + tempPath.toAbsolutePath() + "/delete_delta_2_5/bucket_0"); + createFile("file://" + tempPath.toAbsolutePath() + "/delete_delta_3_3/bucket_0"); + createFile("file://" + tempPath.toAbsolutePath() + "/delta_4_4_1/bucket_0"); + createFile("file://" + tempPath.toAbsolutePath() + "/delta_4_4_3/bucket_0"); + createFile("file://" + tempPath.toAbsolutePath() + "/delta_101_101_1/bucket_0"); Map<String, String> txnValidIds = new HashMap<>(); @@ -558,17 +555,16 @@ public class HiveAcidTest { @Test public void testBaseDeltas() throws Exception { - LocalDfsFileSystem localDFSFileSystem = new LocalDfsFileSystem(); Path tempPath = Files.createTempDirectory("tbl"); - localDFSFileSystem.createFile("file://" + tempPath.toAbsolutePath() + "/base_5/bucket_0"); - localDFSFileSystem.createFile("file://" + tempPath.toAbsolutePath() + "/base_10/bucket_0"); - localDFSFileSystem.createFile("file://" + tempPath.toAbsolutePath() + "/base_49/bucket_0"); - localDFSFileSystem.createFile("file://" + tempPath.toAbsolutePath() + "/delta_025_025/bucket_0"); - localDFSFileSystem.createFile("file://" + tempPath.toAbsolutePath() + "/delta_029_029/bucket_0"); - localDFSFileSystem.createFile("file://" + tempPath.toAbsolutePath() + "/delta_025_030/bucket_0"); - localDFSFileSystem.createFile("file://" + tempPath.toAbsolutePath() + "/delta_050_105/bucket_0"); - localDFSFileSystem.createFile("file://" + tempPath.toAbsolutePath() + "/delta_90_120/bucket_0"); + createFile("file://" + tempPath.toAbsolutePath() + "/base_5/bucket_0"); + createFile("file://" + tempPath.toAbsolutePath() + "/base_10/bucket_0"); + createFile("file://" + tempPath.toAbsolutePath() + "/base_49/bucket_0"); + createFile("file://" + tempPath.toAbsolutePath() + "/delta_025_025/bucket_0"); + createFile("file://" + tempPath.toAbsolutePath() + "/delta_029_029/bucket_0"); + createFile("file://" + tempPath.toAbsolutePath() + "/delta_025_030/bucket_0"); + createFile("file://" + tempPath.toAbsolutePath() + "/delta_050_105/bucket_0"); + createFile("file://" + tempPath.toAbsolutePath() + "/delta_90_120/bucket_0"); Map<String, String> txnValidIds = new HashMap<>(); txnValidIds.put( diff --git a/fe/fe-core/src/test/java/org/apache/doris/fs/remote/RemoteFileSystemTest.java b/fe/fe-core/src/test/java/org/apache/doris/fs/remote/RemoteFileSystemTest.java deleted file mode 100644 index c25d59b43c7..00000000000 --- a/fe/fe-core/src/test/java/org/apache/doris/fs/remote/RemoteFileSystemTest.java +++ /dev/null @@ -1,195 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -package org.apache.doris.fs.remote; - -import org.apache.doris.backup.Status; -import org.apache.doris.datasource.property.storage.StorageProperties; -import org.apache.doris.foundation.fs.FsStorageType; - -import org.apache.hadoop.fs.FileSystem; -import org.junit.jupiter.api.Assertions; -import org.junit.jupiter.api.Test; - -import java.io.IOException; -import java.util.List; -import java.util.concurrent.atomic.AtomicBoolean; - -public class RemoteFileSystemTest { - - private RemoteFileSystem remoteFileSystem; - private FileSystem mockFileSystem; - - private static class DummyRemoteFileSystem extends RemoteFileSystem { - public DummyRemoteFileSystem() { - super("dummy", FsStorageType.HDFS); - } - - @Override - public Status exists(String path) { - return Status.OK; - } - - @Override - public Status downloadWithFileSize(String remoteFilePath, String localFilePath, long fileSize) { - return null; - } - - @Override - public Status upload(String localPath, String remotePath) { - return null; - } - - @Override - public Status directUpload(String content, String remoteFile) { - return null; - } - - @Override - public Status makeDir(String path) { - return Status.OK; - } - - @Override - public Status listFiles(String remotePath, boolean recursive, List<RemoteFile> result) { - return null; - } - - @Override - public Status globList(String remotePath, List<RemoteFile> result, boolean fileNameOnly) { - return null; - } - - @Override - public Status rename(String src, String dst) { - return new Status(Status.ErrCode.OK, "Rename operation not implemented in DummyRemoteFileSystem"); - } - - @Override - public Status delete(String remotePath) { - return null; - } - - @Override - public void close() throws IOException { - - } - - @Override - public StorageProperties getStorageProperties() { - return null; - } - - public Status rename(String path) { - return new Status(Status.ErrCode.OK, "Rename operation not implemented in DummyRemoteFileSystem"); - } - } - - @Test - public void testRenameDir_destExists() { - RemoteFileSystem fs = new DummyRemoteFileSystem() { - @Override - public Status exists(String path) { - if (path.equals("s3://bucket/target")) { - return Status.OK; - } - return new Status(Status.ErrCode.NOT_FOUND, "not found"); - } - - @Override - public Status rename(String path) { - return null; - } - }; - - Status result = fs.renameDir("s3://bucket/src", "s3://bucket/target", () -> { - }); - Assertions.assertFalse(result.ok()); - Assertions.assertEquals(Status.ErrCode.COMMON_ERROR, result.getErrCode()); - } - - @Test - public void testRenameDir_makeDirSuccess() { - AtomicBoolean runFlag = new AtomicBoolean(false); - - RemoteFileSystem fs = new DummyRemoteFileSystem() { - @Override - public Status exists(String path) { - if (path.equals("s3://bucket/target")) { - return new Status(Status.ErrCode.NOT_FOUND, "not found"); - } - if (path.equals("s3://bucket")) { - return new Status(Status.ErrCode.NOT_FOUND, "not found"); - } - return Status.OK; - } - - @Override - public Status makeDir(String path) { - return Status.OK; - } - - @Override - public Status rename(String src, String dst) { - return Status.OK; - } - - @Override - public Status rename(String path) { - return null; - } - }; - - Status result = fs.renameDir("s3://bucket/src", "s3://bucket/target", () -> runFlag.set(true)); - Assertions.assertTrue(result.ok()); - Assertions.assertTrue(runFlag.get()); - } - - @Test - public void testRenameDir_makeDirFails() { - RemoteFileSystem fs = new DummyRemoteFileSystem() { - @Override - public Status exists(String path) { - if (path.equals("s3://bucket/target")) { - return new Status(Status.ErrCode.NOT_FOUND, "not found"); - } - if (path.equals("s3://bucket")) { - return new Status(Status.ErrCode.NOT_FOUND, "not found"); - } - return Status.OK; - } - - @Override - public Status rename(String src, String dst) { - return new Status(Status.ErrCode.COMMON_ERROR, "mkdir failed"); - } - - @Override - public Status rename(String path) { - return new Status(Status.ErrCode.COMMON_ERROR, "mkdir failed"); - } - - }; - - Status result = fs.renameDir("s3://bucket/src", "s3://bucket/target", () -> { - //no-op - }); - Assertions.assertFalse(result.ok()); - Assertions.assertEquals(Status.ErrCode.COMMON_ERROR, result.getErrCode()); - Assertions.assertTrue(result.getErrMsg().contains("mkdir failed")); - } -} --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
