This is an automated email from the ASF dual-hosted git repository. morningman pushed a commit to branch branch-fs-spi in repository https://gitbox.apache.org/repos/asf/doris.git
commit 86486c8507654c7ee193a239646e25b445a90719 Author: morningman <[email protected]> AuthorDate: Wed Apr 1 15:11:38 2026 +0800 [refactor](fs-spi) P4.8-B: migrate DirectoryLister from RemoteFile to SPI FileEntry ### What problem does this PR solve? Issue Number: N/A Problem Summary: The DirectoryLister interface and its implementations returned RemoteFile (a Hadoop-dependent legacy class), forcing callers like HiveExternalMetaCache and AcidUtil to depend on Hadoop Path objects. This phase eliminates that dependency by: 1. Adding modificationTime to SPI FileEntry and RemoteObject so no metadata is lost during migration. 2. All SPI providers (HDFS, S3, Azure, Broker, Local) now populate modificationTime from their underlying SDKs. 3. DirectoryLister interface now returns RemoteIterator<FileEntry> (SPI). 4. FileSystemDirectoryLister, SimpleRemoteIterator, and TransactionScopeCachingDirectoryLister all updated accordingly. 5. RemoteFileRemoteIterator and RemoteFiles deleted (no remaining callers). 6. HiveExternalMetaCache.addFile() now accepts FileEntry; converts List<BlockInfo> to BlockLocation[] for HiveFileStatus. 7. AcidUtil removes toRemoteFiles() bridge; works with FileEntry directly. 8. PathVisibleTest updated to use String-based isFileVisible() signature. ### Release note None ### Check List (For Author) - Test: Regression test / Unit Test / Manual test / No need to test (with reason) - Build verification (FE build succeeds) - PathVisibleTest, TransactionScopeCachingDirectoryListerTest, RepositoryTest, CopyLoadPendingTaskTest updated and compile clean - Behavior changed: No (modificationTime was 0L before; now populated from SDK) - Does this need documentation: No Co-authored-by: Copilot <[email protected]> --- .../org/apache/doris/datasource/hive/AcidUtil.java | 46 +++++++++---------- .../datasource/hive/HiveExternalMetaCache.java | 37 +++++++++------- .../java/org/apache/doris/fs/DirectoryLister.java | 4 +- .../apache/doris/fs/FileSystemDirectoryLister.java | 13 +----- .../apache/doris/fs/RemoteFileRemoteIterator.java | 51 ---------------------- .../main/java/org/apache/doris/fs/RemoteFiles.java | 39 ----------------- .../org/apache/doris/fs/SimpleRemoteIterator.java | 10 ++--- .../fs/TransactionScopeCachingDirectoryLister.java | 28 ++++++------ .../org/apache/doris/backup/RepositoryTest.java | 12 ++--- .../doris/cloud/load/CopyLoadPendingTaskTest.java | 8 ++-- .../apache/doris/datasource/PathVisibleTest.java | 33 +++++++------- ...TransactionScopeCachingDirectoryListerTest.java | 49 +++++++++++---------- .../doris/filesystem/azure/AzureFileSystem.java | 2 +- .../doris/filesystem/azure/AzureObjStorage.java | 7 ++- .../filesystem/broker/BrokerSpiFileSystem.java | 1 + .../doris/filesystem/hdfs/HdfsFileIterator.java | 3 +- .../doris/filesystem/local/LocalFileSystem.java | 3 +- .../apache/doris/filesystem/s3/S3FileSystem.java | 3 +- .../apache/doris/filesystem/s3/S3ObjStorage.java | 12 +++-- .../org/apache/doris/filesystem/spi/FileEntry.java | 10 ++++- .../apache/doris/filesystem/spi/RemoteObject.java | 9 +++- 21 files changed, 156 insertions(+), 224 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/AcidUtil.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/AcidUtil.java index 3c5fdda4a9c..e5c2ddcd0b0 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/AcidUtil.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/AcidUtil.java @@ -25,7 +25,6 @@ import org.apache.doris.filesystem.spi.FileEntry; import org.apache.doris.filesystem.spi.FileSystem; import org.apache.doris.filesystem.spi.Location; import org.apache.doris.fs.FileSystemTransferUtil; -import org.apache.doris.fs.remote.RemoteFile; import lombok.EqualsAndHashCode; import lombok.Getter; @@ -252,8 +251,8 @@ public class AcidUtil { String partitionPath = partition.getPath(); //hdfs://xxxxx/user/hive/warehouse/username/data_id=200103 - List<RemoteFile> lsPartitionPath = toRemoteFiles( - FileSystemTransferUtil.globList(fileSystem, partitionPath + "/*", false)); + List<FileEntry> lsPartitionPath = + FileSystemTransferUtil.globList(fileSystem, partitionPath + "/*", false); // List all files and folders, without recursion. String oldestBase = null; @@ -263,9 +262,9 @@ public class AcidUtil { boolean haveOriginalFiles = false; List<ParsedDelta> workingDeltas = new ArrayList<>(); - for (RemoteFile remotePath : lsPartitionPath) { - if (remotePath.isDirectory()) { - String dirName = remotePath.getName(); //dirName: base_xxx,delta_xxx,... + for (FileEntry entry : lsPartitionPath) { + if (entry.isDirectory()) { + String dirName = locationName(entry.location()); //dirName: base_xxx,delta_xxx,... String dirPath = partitionPath + "/" + dirName; if (dirName.startsWith("base_")) { @@ -388,29 +387,27 @@ public class AcidUtil { for (ParsedDelta delta : deltas) { String location = delta.getPath(); - List<RemoteFile> remoteFiles = toRemoteFiles( - FileSystemTransferUtil.globList(fileSystem, location, false)); + List<FileEntry> entries = FileSystemTransferUtil.globList(fileSystem, location, false); if (delta.isDeleteDelta()) { - List<String> deleteDeltaFileNames = remoteFiles.stream() - .map(RemoteFile::getName).filter(fileFilter::accept) + List<String> deleteDeltaFileNames = entries.stream() + .map(e -> locationName(e.location())).filter(fileFilter::accept) .collect(Collectors.toList()); deleteDeltas.add(new DeleteDeltaInfo(location, deleteDeltaFileNames)); continue; } - remoteFiles.stream().filter(f -> fileFilter.accept(f.getName())).forEach(file -> { - LocationPath path = LocationPath.of(file.getPath().toString(), storagePropertiesMap); - fileCacheValue.addFile(file, path); + entries.stream().filter(e -> fileFilter.accept(locationName(e.location()))).forEach(entry -> { + LocationPath path = LocationPath.of(entry.location().uri(), storagePropertiesMap); + fileCacheValue.addFile(entry, path); }); } // base if (bestBasePath != null) { - List<RemoteFile> remoteFiles = toRemoteFiles( - FileSystemTransferUtil.globList(fileSystem, bestBasePath, false)); - remoteFiles.stream().filter(f -> fileFilter.accept(f.getName())) - .forEach(file -> { - LocationPath path = LocationPath.of(file.getPath().toString(), storagePropertiesMap); - fileCacheValue.addFile(file, path); + List<FileEntry> entries = FileSystemTransferUtil.globList(fileSystem, bestBasePath, false); + entries.stream().filter(e -> fileFilter.accept(locationName(e.location()))) + .forEach(entry -> { + LocationPath path = LocationPath.of(entry.location().uri(), storagePropertiesMap); + fileCacheValue.addFile(entry, path); }); } @@ -422,12 +419,9 @@ public class AcidUtil { return fileCacheValue; } - private static List<RemoteFile> toRemoteFiles(List<FileEntry> entries) { - List<RemoteFile> result = new ArrayList<>(entries.size()); - for (FileEntry e : entries) { - org.apache.hadoop.fs.Path hadoopPath = new org.apache.hadoop.fs.Path(e.location().uri()); - result.add(new RemoteFile(hadoopPath, e.isDirectory(), e.length(), -1L, 0L, null)); - } - return result; + private static String locationName(org.apache.doris.filesystem.spi.Location location) { + String uri = location.uri(); + int idx = uri.lastIndexOf('/'); + return idx >= 0 ? uri.substring(idx + 1) : uri; } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HiveExternalMetaCache.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HiveExternalMetaCache.java index c5b63383d46..82c6e5a0cd0 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HiveExternalMetaCache.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HiveExternalMetaCache.java @@ -45,13 +45,14 @@ import org.apache.doris.datasource.metacache.AbstractExternalMetaCache; import org.apache.doris.datasource.metacache.CacheSpec; import org.apache.doris.datasource.metacache.MetaCacheEntry; import org.apache.doris.datasource.metacache.MetaCacheEntryDef; +import org.apache.doris.filesystem.spi.BlockInfo; +import org.apache.doris.filesystem.spi.FileEntry; import org.apache.doris.filesystem.spi.FileSystem; import org.apache.doris.fs.DirectoryLister; import org.apache.doris.fs.FileSystemCache; import org.apache.doris.fs.FileSystemDirectoryLister; import org.apache.doris.fs.FileSystemIOException; import org.apache.doris.fs.RemoteIterator; -import org.apache.doris.fs.remote.RemoteFile; import org.apache.doris.nereids.rules.expression.rules.SortedPartitionRanges; import org.apache.doris.planner.ListPartitionPrunerV2; @@ -65,7 +66,6 @@ import com.google.common.collect.Maps; import com.google.common.collect.Streams; import lombok.Data; import org.apache.hadoop.fs.BlockLocation; -import org.apache.hadoop.fs.Path; import org.apache.hadoop.hive.metastore.api.Partition; import org.apache.hadoop.hive.metastore.api.StorageDescriptor; import org.apache.hadoop.hive.metastore.utils.FileUtils; @@ -402,13 +402,13 @@ public class HiveExternalMetaCache extends AbstractExternalMetaCache { boolean isRecursiveDirectories = Boolean.valueOf( catalog.getProperties().getOrDefault("hive.recursive_directories", "true")); try { - RemoteIterator<RemoteFile> iterator = directoryLister.listFiles(fs, isRecursiveDirectories, + RemoteIterator<FileEntry> iterator = directoryLister.listFiles(fs, isRecursiveDirectories, table, path.getNormalizedLocation()); while (iterator.hasNext()) { - RemoteFile remoteFile = iterator.next(); - String srcPath = remoteFile.getPath().toString(); + FileEntry entry = iterator.next(); + String srcPath = entry.location().uri().toString(); LocationPath locationPath = LocationPath.of(srcPath, path.getStorageProperties()); - result.addFile(remoteFile, locationPath); + result.addFile(entry, locationPath); } } catch (FileSystemIOException e) { if (e.getErrorCode().isPresent() && e.getErrorCode().get().equals(ErrCode.NOT_FOUND)) { @@ -984,14 +984,22 @@ public class HiveExternalMetaCache extends AbstractExternalMetaCache { protected List<String> partitionValues; private AcidInfo acidInfo; - public void addFile(RemoteFile file, LocationPath locationPath) { - if (isFileVisible(file.getPath())) { + public void addFile(FileEntry entry, LocationPath locationPath) { + if (isFileVisible(entry.location().uri().toString())) { HiveFileStatus status = new HiveFileStatus(); - status.setBlockLocations(file.getBlockLocations()); + List<BlockInfo> blocks = entry.blocks(); + if (!blocks.isEmpty()) { + BlockLocation[] blockLocations = new BlockLocation[blocks.size()]; + for (int i = 0; i < blocks.size(); i++) { + BlockInfo b = blocks.get(i); + blockLocations[i] = new BlockLocation(null, b.hosts(), b.offset(), b.length()); + } + status.setBlockLocations(blockLocations); + } status.setPath(locationPath); - status.length = file.getSize(); - status.blockSize = file.getBlockSize(); - status.modificationTime = file.getModificationTime(); + status.length = entry.length(); + status.blockSize = blocks.isEmpty() ? 0 : blocks.get(0).length(); + status.modificationTime = entry.modificationTime(); files.add(status); } } @@ -1001,11 +1009,10 @@ public class HiveExternalMetaCache extends AbstractExternalMetaCache { } @VisibleForTesting - public static boolean isFileVisible(Path path) { - if (path == null) { + public static boolean isFileVisible(String pathStr) { + if (pathStr == null) { return false; } - String pathStr = path.toUri().toString(); if (containsHiddenPath(pathStr)) { return false; } diff --git a/fe/fe-core/src/main/java/org/apache/doris/fs/DirectoryLister.java b/fe/fe-core/src/main/java/org/apache/doris/fs/DirectoryLister.java index 3c8e94ef1b0..d1732c7707e 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/fs/DirectoryLister.java +++ b/fe/fe-core/src/main/java/org/apache/doris/fs/DirectoryLister.java @@ -21,10 +21,10 @@ package org.apache.doris.fs; import org.apache.doris.catalog.TableIf; +import org.apache.doris.filesystem.spi.FileEntry; import org.apache.doris.filesystem.spi.FileSystem; -import org.apache.doris.fs.remote.RemoteFile; public interface DirectoryLister { - RemoteIterator<RemoteFile> listFiles(FileSystem fs, boolean recursive, TableIf table, String location) + RemoteIterator<FileEntry> listFiles(FileSystem fs, boolean recursive, TableIf table, String location) throws FileSystemIOException; } diff --git a/fe/fe-core/src/main/java/org/apache/doris/fs/FileSystemDirectoryLister.java b/fe/fe-core/src/main/java/org/apache/doris/fs/FileSystemDirectoryLister.java index a23b5bbe79a..ea2075b4c71 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/fs/FileSystemDirectoryLister.java +++ b/fe/fe-core/src/main/java/org/apache/doris/fs/FileSystemDirectoryLister.java @@ -20,26 +20,17 @@ package org.apache.doris.fs; import org.apache.doris.catalog.TableIf; import org.apache.doris.filesystem.spi.FileEntry; import org.apache.doris.filesystem.spi.FileSystem; -import org.apache.doris.fs.remote.RemoteFile; - -import org.apache.hadoop.fs.Path; import java.io.IOException; -import java.util.ArrayList; import java.util.List; public class FileSystemDirectoryLister implements DirectoryLister { - public RemoteIterator<RemoteFile> listFiles(FileSystem fs, boolean recursive, + public RemoteIterator<FileEntry> listFiles(FileSystem fs, boolean recursive, TableIf table, String location) throws FileSystemIOException { try { List<FileEntry> entries = FileSystemTransferUtil.globList(fs, location, recursive); - List<RemoteFile> result = new ArrayList<>(entries.size()); - for (FileEntry e : entries) { - Path hadoopPath = new Path(e.location().uri()); - result.add(new RemoteFile(hadoopPath, e.isDirectory(), e.length(), -1L, 0L, null)); - } - return new RemoteFileRemoteIterator(result); + return new SimpleRemoteIterator(entries.iterator()); } catch (IOException ex) { throw new FileSystemIOException(ex.getMessage(), ex); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/fs/RemoteFileRemoteIterator.java b/fe/fe-core/src/main/java/org/apache/doris/fs/RemoteFileRemoteIterator.java deleted file mode 100644 index e67cf7d3157..00000000000 --- a/fe/fe-core/src/main/java/org/apache/doris/fs/RemoteFileRemoteIterator.java +++ /dev/null @@ -1,51 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -package org.apache.doris.fs; - -import org.apache.doris.fs.remote.RemoteFile; - -import java.util.List; -import java.util.NoSuchElementException; -import java.util.Objects; - -/** - * @deprecated use {@link FileIterator} instead - */ -@Deprecated -public class RemoteFileRemoteIterator - implements RemoteIterator<RemoteFile> { - private final List<RemoteFile> remoteFileList; - private int currentIndex = 0; - - public RemoteFileRemoteIterator(List<RemoteFile> remoteFileList) { - this.remoteFileList = Objects.requireNonNull(remoteFileList, "iterator is null"); - } - - @Override - public boolean hasNext() throws FileSystemIOException { - return currentIndex < remoteFileList.size(); - } - - @Override - public RemoteFile next() throws FileSystemIOException { - if (!hasNext()) { - throw new NoSuchElementException("No more elements in RemoteFileRemoteIterator"); - } - return remoteFileList.get(currentIndex++); - } -} diff --git a/fe/fe-core/src/main/java/org/apache/doris/fs/RemoteFiles.java b/fe/fe-core/src/main/java/org/apache/doris/fs/RemoteFiles.java deleted file mode 100644 index a5ba09e5b28..00000000000 --- a/fe/fe-core/src/main/java/org/apache/doris/fs/RemoteFiles.java +++ /dev/null @@ -1,39 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -package org.apache.doris.fs; - -import org.apache.doris.fs.remote.RemoteFile; - -import java.util.List; - -/** - * @deprecated use {@link FileIterator} instead - */ -@Deprecated -public class RemoteFiles { - - private final List<RemoteFile> files; - - public RemoteFiles(List<RemoteFile> files) { - this.files = files; - } - - public List<RemoteFile> files() { - return files; - } -} diff --git a/fe/fe-core/src/main/java/org/apache/doris/fs/SimpleRemoteIterator.java b/fe/fe-core/src/main/java/org/apache/doris/fs/SimpleRemoteIterator.java index 4332a5fed35..370014ce7c6 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/fs/SimpleRemoteIterator.java +++ b/fe/fe-core/src/main/java/org/apache/doris/fs/SimpleRemoteIterator.java @@ -18,7 +18,7 @@ package org.apache.doris.fs; -import org.apache.doris.fs.remote.RemoteFile; +import org.apache.doris.filesystem.spi.FileEntry; import java.util.Iterator; import java.util.Objects; @@ -26,10 +26,10 @@ import java.util.Objects; // https://github.com/trinodb/trino/blob/438/plugin/trino-hive/src/main/java/io/trino/plugin/hive/fs/SimpleRemoteIterator.java // and modified by Doris -class SimpleRemoteIterator implements RemoteIterator<RemoteFile> { - private final Iterator<RemoteFile> iterator; +class SimpleRemoteIterator implements RemoteIterator<FileEntry> { + private final Iterator<FileEntry> iterator; - public SimpleRemoteIterator(Iterator<RemoteFile> iterator) { + public SimpleRemoteIterator(Iterator<FileEntry> iterator) { this.iterator = Objects.requireNonNull(iterator, "iterator is null"); } @@ -39,7 +39,7 @@ class SimpleRemoteIterator implements RemoteIterator<RemoteFile> { } @Override - public RemoteFile next() throws FileSystemIOException { + public FileEntry next() throws FileSystemIOException { return iterator.next(); } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/fs/TransactionScopeCachingDirectoryLister.java b/fe/fe-core/src/main/java/org/apache/doris/fs/TransactionScopeCachingDirectoryLister.java index 122d5face8c..0598eedf54f 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/fs/TransactionScopeCachingDirectoryLister.java +++ b/fe/fe-core/src/main/java/org/apache/doris/fs/TransactionScopeCachingDirectoryLister.java @@ -21,7 +21,7 @@ package org.apache.doris.fs; import org.apache.doris.catalog.TableIf; -import org.apache.doris.fs.remote.RemoteFile; +import org.apache.doris.filesystem.spi.FileEntry; import com.google.common.annotations.VisibleForTesting; import com.google.common.base.Preconditions; @@ -68,13 +68,13 @@ public class TransactionScopeCachingDirectoryLister implements DirectoryLister { } @Override - public RemoteIterator<RemoteFile> listFiles(org.apache.doris.filesystem.spi.FileSystem fs, boolean recursive, + public RemoteIterator<FileEntry> listFiles(org.apache.doris.filesystem.spi.FileSystem fs, boolean recursive, TableIf table, String location) throws FileSystemIOException { return listInternal(fs, recursive, table, new TransactionDirectoryListingCacheKey(transactionId, location)); } - private RemoteIterator<RemoteFile> listInternal(org.apache.doris.filesystem.spi.FileSystem fs, + private RemoteIterator<FileEntry> listInternal(org.apache.doris.filesystem.spi.FileSystem fs, boolean recursive, TableIf table, TransactionDirectoryListingCacheKey cacheKey) throws FileSystemIOException { @@ -96,7 +96,7 @@ public class TransactionScopeCachingDirectoryLister implements DirectoryLister { return cachingRemoteIterator(cachedValueHolder, cacheKey); } - private RemoteIterator<RemoteFile> createListingRemoteIterator( + private RemoteIterator<FileEntry> createListingRemoteIterator( org.apache.doris.filesystem.spi.FileSystem fs, boolean recursive, TableIf table, @@ -106,9 +106,9 @@ public class TransactionScopeCachingDirectoryLister implements DirectoryLister { } - private RemoteIterator<RemoteFile> cachingRemoteIterator(FetchingValueHolder cachedValueHolder, + private RemoteIterator<FileEntry> cachingRemoteIterator(FetchingValueHolder cachedValueHolder, TransactionDirectoryListingCacheKey cacheKey) { - return new RemoteIterator<RemoteFile>() { + return new RemoteIterator<FileEntry>() { private int fileIndex; @Override @@ -130,7 +130,7 @@ public class TransactionScopeCachingDirectoryLister implements DirectoryLister { } @Override - public RemoteFile next() + public FileEntry next() throws FileSystemIOException { // force cache entry weight update in case next file is cached Preconditions.checkState(hasNext()); @@ -152,16 +152,16 @@ public class TransactionScopeCachingDirectoryLister implements DirectoryLister { static class FetchingValueHolder { - private final List<RemoteFile> cachedFiles = ListUtils.synchronizedList(new ArrayList<RemoteFile>()); + private final List<FileEntry> cachedFiles = ListUtils.synchronizedList(new ArrayList<FileEntry>()); @GuardedBy("this") @Nullable - private RemoteIterator<RemoteFile> fileIterator; + private RemoteIterator<FileEntry> fileIterator; @GuardedBy("this") @Nullable private Exception exception; - public FetchingValueHolder(RemoteIterator<RemoteFile> fileIterator) { + public FetchingValueHolder(RemoteIterator<FileEntry> fileIterator) { this.fileIterator = Objects.requireNonNull(fileIterator, "fileIterator is null"); } @@ -173,12 +173,12 @@ public class TransactionScopeCachingDirectoryLister implements DirectoryLister { return cachedFiles.size(); } - public Iterator<RemoteFile> getCachedFiles() { + public Iterator<FileEntry> getCachedFiles() { Preconditions.checkState(isFullyCached()); return cachedFiles.iterator(); } - public Optional<RemoteFile> getCachedFile(int index) + public Optional<FileEntry> getCachedFile(int index) throws FileSystemIOException { int filesSize = cachedFiles.size(); Preconditions.checkArgument(index >= 0 && index <= filesSize, @@ -192,7 +192,7 @@ public class TransactionScopeCachingDirectoryLister implements DirectoryLister { return fetchNextCachedFile(index); } - private synchronized Optional<RemoteFile> fetchNextCachedFile(int index) + private synchronized Optional<FileEntry> fetchNextCachedFile(int index) throws FileSystemIOException { if (exception != null) { throw new FileSystemIOException("Exception while listing directory", exception); @@ -210,7 +210,7 @@ public class TransactionScopeCachingDirectoryLister implements DirectoryLister { return Optional.empty(); } - RemoteFile fileStatus = fileIterator.next(); + FileEntry fileStatus = fileIterator.next(); cachedFiles.add(fileStatus); return Optional.of(fileStatus); } catch (Exception exception) { diff --git a/fe/fe-core/src/test/java/org/apache/doris/backup/RepositoryTest.java b/fe/fe-core/src/test/java/org/apache/doris/backup/RepositoryTest.java index efad2835fbf..2ad2d805567 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/backup/RepositoryTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/backup/RepositoryTest.java @@ -221,10 +221,10 @@ public class RepositoryTest { new FileEntry( Location.of(location + "/__palo_repository_repo/" + Repository.PREFIX_SNAPSHOT_DIR + "a"), - 100, true, null), + 100, true, 0L, null), new FileEntry( Location.of(location + "/__palo_repository_repo/_ss_b"), - 100, false, null)); + 100, false, 0L, null)); return new FileIterator() { private int idx = 0; @@ -320,7 +320,7 @@ public class RepositoryTest { new FileEntry( Location.of(location + "/remote_file" + ".0cc175b9c0f1b6a831c399e269772661"), - 1, false, null)); + 1, false, 0L, null)); mockFs.newInputFile((Location) any); minTimes = 0; @@ -381,11 +381,11 @@ public class RepositoryTest { new FileEntry( Location.of(location + "/__palo_repository_repo/" + Repository.PREFIX_SNAPSHOT_DIR + "s1"), - 100, true, null), + 100, true, 0L, null), new FileEntry( Location.of(location + "/__palo_repository_repo/" + Repository.PREFIX_SNAPSHOT_DIR + "s2"), - 100, true, null)); + 100, true, 0L, null)); return new FileIterator() { private int idx = 0; @@ -414,7 +414,7 @@ public class RepositoryTest { Location.of(location + "/__palo_repository_repo/__ss_s1/" + "__info_2018-04-18-20-11-00" + ".12345678123456781234567812345678"), - 100, false, null)); + 100, false, 0L, null)); } catch (IOException e) { throw new RuntimeException(e); } diff --git a/fe/fe-core/src/test/java/org/apache/doris/cloud/load/CopyLoadPendingTaskTest.java b/fe/fe-core/src/test/java/org/apache/doris/cloud/load/CopyLoadPendingTaskTest.java index b2dbadf0d19..27a9fff39b9 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/cloud/load/CopyLoadPendingTaskTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/cloud/load/CopyLoadPendingTaskTest.java @@ -157,7 +157,7 @@ public class CopyLoadPendingTaskTest extends TestWithFeService { String etag = ""; RemoteObject objectFile = new RemoteObject( STORAGE_PREFIX + (STORAGE_PREFIX.isEmpty() ? "" : "/") + relativePath, relativePath, etag, - (j + 1) * 10); + (j + 1) * 10, 0L); objectStore.put(objectFile.getKey(), objectFile); System.out.println( "object file=" + objectFile.getKey() + ", " + objectFile.getRelativePath() + ", size: " @@ -246,7 +246,7 @@ public class CopyLoadPendingTaskTest extends TestWithFeService { + ".csv"; String etag = ""; RemoteObject objectFile = new RemoteObject(prefix + (prefix.isEmpty() ? "" : "/") + relativePath, - relativePath, etag, (j + 1) * 10); + relativePath, etag, (j + 1) * 10, 0L); objectFiles.add(objectFile); System.out.println( "object file=" + objectFile.getKey() + ", " + objectFile.getRelativePath() + ", size: " @@ -294,7 +294,7 @@ public class CopyLoadPendingTaskTest extends TestWithFeService { for (int j = 0; j < 10; j++) { String relativePath = subPrefix + (subPrefix.isEmpty() ? "" : "/") + "file" + j + ".csv"; RemoteObject objectFile = new RemoteObject(STORAGE_PREFIX + "/" + relativePath, relativePath, "", - (j + 1) * 10); + (j + 1) * 10, 0L); objectStore.put(objectFile.getKey(), objectFile); System.out.println("Add " + objectFile); } @@ -303,7 +303,7 @@ public class CopyLoadPendingTaskTest extends TestWithFeService { List<String> specialNames = Lists.newArrayList("sf,csv", "sd/sf,csv", "sf?csv", "sd/sf?csv", "sf*csv", "sf-csv", "sf[csv", "sf]csv", "sf{csv", "sf}csv"); for (String specialName : specialNames) { - RemoteObject objectFile = new RemoteObject(STORAGE_PREFIX + "/" + specialName, specialName, "", 1); + RemoteObject objectFile = new RemoteObject(STORAGE_PREFIX + "/" + specialName, specialName, "", 1, 0L); objectStore.put(objectFile.getKey(), objectFile); } setupObjFsMock(); diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/PathVisibleTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/PathVisibleTest.java index 1fd42b1dcb0..8282aeabb1c 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/datasource/PathVisibleTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/PathVisibleTest.java @@ -19,7 +19,6 @@ package org.apache.doris.datasource; import org.apache.doris.datasource.hive.HiveExternalMetaCache.FileCacheValue; -import org.apache.hadoop.fs.Path; import org.junit.Assert; import org.junit.Test; @@ -27,24 +26,24 @@ public class PathVisibleTest { @Test public void shouldReturnFalseWhenPathIsNull() { Assert.assertFalse(FileCacheValue.isFileVisible(null)); - Assert.assertFalse(FileCacheValue.isFileVisible(new Path("s3://visible/.hidden/path"))); - Assert.assertFalse(FileCacheValue.isFileVisible(new Path("/visible/.hidden/path"))); - Assert.assertFalse(FileCacheValue.isFileVisible(new Path("hdfs://visible/path/.file"))); - Assert.assertFalse(FileCacheValue.isFileVisible(new Path("/visible/path/_temporary_xx"))); - Assert.assertFalse(FileCacheValue.isFileVisible(new Path("/visible/path/_impala_insert_staging"))); + Assert.assertFalse(FileCacheValue.isFileVisible("s3://visible/.hidden/path")); + Assert.assertFalse(FileCacheValue.isFileVisible("/visible/.hidden/path")); + Assert.assertFalse(FileCacheValue.isFileVisible("hdfs://visible/path/.file")); + Assert.assertFalse(FileCacheValue.isFileVisible("/visible/path/_temporary_xx")); + Assert.assertFalse(FileCacheValue.isFileVisible("/visible/path/_impala_insert_staging")); - Assert.assertFalse(FileCacheValue.isFileVisible(new Path("/visible//.hidden/path"))); - Assert.assertFalse(FileCacheValue.isFileVisible(new Path("s3://visible/.hidden/path"))); - Assert.assertFalse(FileCacheValue.isFileVisible(new Path("///visible/path/.file"))); - Assert.assertFalse(FileCacheValue.isFileVisible(new Path("/visible/path///_temporary_xx"))); - Assert.assertFalse(FileCacheValue.isFileVisible(new Path("hdfs://visible//path/_impala_insert_staging"))); + Assert.assertFalse(FileCacheValue.isFileVisible("/visible//.hidden/path")); + Assert.assertFalse(FileCacheValue.isFileVisible("s3://visible/.hidden/path")); + Assert.assertFalse(FileCacheValue.isFileVisible("///visible/path/.file")); + Assert.assertFalse(FileCacheValue.isFileVisible("/visible/path///_temporary_xx")); + Assert.assertFalse(FileCacheValue.isFileVisible("hdfs://visible//path/_impala_insert_staging")); Assert.assertFalse(FileCacheValue.isFileVisible( - new Path("hdfs://hacluster/user/hive/warehouse/db1.db/tbl1/_spark_metadata/"))); + "hdfs://hacluster/user/hive/warehouse/db1.db/tbl1/_spark_metadata/")); - Assert.assertTrue(FileCacheValue.isFileVisible(new Path("s3://visible/path"))); - Assert.assertTrue(FileCacheValue.isFileVisible(new Path("path"))); - Assert.assertTrue(FileCacheValue.isFileVisible(new Path("hdfs://visible/path./1.txt"))); - Assert.assertTrue(FileCacheValue.isFileVisible(new Path("/1.txt"))); - Assert.assertTrue(FileCacheValue.isFileVisible(new Path("hdfs://vis_ible_/pa.th./1_.txt__"))); + Assert.assertTrue(FileCacheValue.isFileVisible("s3://visible/path")); + Assert.assertTrue(FileCacheValue.isFileVisible("path")); + Assert.assertTrue(FileCacheValue.isFileVisible("hdfs://visible/path./1.txt")); + Assert.assertTrue(FileCacheValue.isFileVisible("/1.txt")); + Assert.assertTrue(FileCacheValue.isFileVisible("hdfs://vis_ible_/pa.th./1_.txt__")); } } diff --git a/fe/fe-core/src/test/java/org/apache/doris/fs/TransactionScopeCachingDirectoryListerTest.java b/fe/fe-core/src/test/java/org/apache/doris/fs/TransactionScopeCachingDirectoryListerTest.java index 457a36dff31..1b32ef501ab 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/fs/TransactionScopeCachingDirectoryListerTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/fs/TransactionScopeCachingDirectoryListerTest.java @@ -21,7 +21,8 @@ package org.apache.doris.fs; import org.apache.doris.catalog.TableIf; -import org.apache.doris.fs.remote.RemoteFile; +import org.apache.doris.filesystem.spi.FileEntry; +import org.apache.doris.filesystem.spi.Location; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; @@ -39,12 +40,16 @@ import java.util.Objects; // some tests may invalidate the whole cache affecting therefore other concurrent tests @Execution(ExecutionMode.SAME_THREAD) public class TransactionScopeCachingDirectoryListerTest { + private static FileEntry fileEntry(String uri) { + return new FileEntry(Location.of(uri), 1, false, 0L, null); + } + @Test public void testConcurrentDirectoryListing(@Mocked TableIf table) throws FileSystemIOException { - RemoteFile firstFile = new RemoteFile("file:/x/x", true, 1, 1); - RemoteFile secondFile = new RemoteFile("file:/x/y", true, 1, 1); - RemoteFile thirdFile = new RemoteFile("file:/y/z", true, 1, 1); + FileEntry firstFile = fileEntry("file:/x/x"); + FileEntry secondFile = fileEntry("file:/x/y"); + FileEntry thirdFile = fileEntry("file:/y/z"); String path1 = "file:/x"; String path2 = "file:/y"; @@ -68,15 +73,15 @@ public class TransactionScopeCachingDirectoryListerTest { // start listing path1 concurrently - RemoteIterator<RemoteFile> path1FilesA = cachingLister.listFiles(null, true, table, path1); - RemoteIterator<RemoteFile> path1FilesB = cachingLister.listFiles(null, true, table, path1); + RemoteIterator<FileEntry> path1FilesA = cachingLister.listFiles(null, true, table, path1); + RemoteIterator<FileEntry> path1FilesB = cachingLister.listFiles(null, true, table, path1); Assert.assertEquals(2, countingLister.getListCount()); // list path1 files using both iterators concurrently - Assert.assertEquals(firstFile, path1FilesA.next()); - Assert.assertEquals(firstFile, path1FilesB.next()); - Assert.assertEquals(secondFile, path1FilesB.next()); - Assert.assertEquals(secondFile, path1FilesA.next()); + Assert.assertSame(firstFile, path1FilesA.next()); + Assert.assertSame(firstFile, path1FilesB.next()); + Assert.assertSame(secondFile, path1FilesB.next()); + Assert.assertSame(secondFile, path1FilesA.next()); Assert.assertFalse(path1FilesA.hasNext()); Assert.assertFalse(path1FilesB.hasNext()); Assert.assertEquals(2, countingLister.getListCount()); @@ -89,7 +94,7 @@ public class TransactionScopeCachingDirectoryListerTest { @Test public void testConcurrentDirectoryListingException(@Mocked TableIf table) throws FileSystemIOException { - RemoteFile file = new RemoteFile("file:/x/x", true, 1, 1); + FileEntry file = fileEntry("file:/x/x"); String path = "file:/x"; @@ -98,8 +103,8 @@ public class TransactionScopeCachingDirectoryListerTest { // start listing path concurrently countingLister.setThrowException(true); - RemoteIterator<RemoteFile> filesA = cachingLister.listFiles(null, true, table, path); - RemoteIterator<RemoteFile> filesB = cachingLister.listFiles(null, true, table, path); + RemoteIterator<FileEntry> filesA = cachingLister.listFiles(null, true, table, path); + RemoteIterator<FileEntry> filesB = cachingLister.listFiles(null, true, table, path); Assert.assertEquals(1, countingLister.getListCount()); // listing should throw an exception @@ -116,9 +121,9 @@ public class TransactionScopeCachingDirectoryListerTest { } - private void assertFiles(RemoteIterator<RemoteFile> iterator, List<RemoteFile> expectedFiles) + private void assertFiles(RemoteIterator<FileEntry> iterator, List<FileEntry> expectedFiles) throws FileSystemIOException { - ImmutableList.Builder<RemoteFile> actualFiles = ImmutableList.builder(); + ImmutableList.Builder<FileEntry> actualFiles = ImmutableList.builder(); while (iterator.hasNext()) { actualFiles.add(iterator.next()); } @@ -127,16 +132,16 @@ public class TransactionScopeCachingDirectoryListerTest { private static class CountingDirectoryLister implements DirectoryLister { - private final Map<String, List<RemoteFile>> fileStatuses; + private final Map<String, List<FileEntry>> fileStatuses; private int listCount; private boolean throwException; - public CountingDirectoryLister(Map<String, List<RemoteFile>> fileStatuses) { + public CountingDirectoryLister(Map<String, List<FileEntry>> fileStatuses) { this.fileStatuses = Objects.requireNonNull(fileStatuses, "fileStatuses is null"); } @Override - public RemoteIterator<RemoteFile> listFiles(org.apache.doris.filesystem.spi.FileSystem fs, boolean recursive, + public RemoteIterator<FileEntry> listFiles(org.apache.doris.filesystem.spi.FileSystem fs, boolean recursive, TableIf table, String location) throws FileSystemIOException { // No specific recursive files-only listing implementation @@ -153,9 +158,9 @@ public class TransactionScopeCachingDirectoryListerTest { } } - static RemoteIterator<RemoteFile> throwingRemoteIterator(List<RemoteFile> files, boolean throwException) { - return new RemoteIterator<RemoteFile>() { - private final Iterator<RemoteFile> iterator = ImmutableList.copyOf(files).iterator(); + static RemoteIterator<FileEntry> throwingRemoteIterator(List<FileEntry> files, boolean throwException) { + return new RemoteIterator<FileEntry>() { + private final Iterator<FileEntry> iterator = ImmutableList.copyOf(files).iterator(); @Override public boolean hasNext() @@ -167,7 +172,7 @@ public class TransactionScopeCachingDirectoryListerTest { } @Override - public RemoteFile next() { + public FileEntry next() { return iterator.next(); } }; diff --git a/fe/fe-filesystem/fe-filesystem-azure/src/main/java/org/apache/doris/filesystem/azure/AzureFileSystem.java b/fe/fe-filesystem/fe-filesystem-azure/src/main/java/org/apache/doris/filesystem/azure/AzureFileSystem.java index edfc664c87f..cde5888861b 100644 --- a/fe/fe-filesystem/fe-filesystem-azure/src/main/java/org/apache/doris/filesystem/azure/AzureFileSystem.java +++ b/fe/fe-filesystem/fe-filesystem-azure/src/main/java/org/apache/doris/filesystem/azure/AzureFileSystem.java @@ -176,7 +176,7 @@ public class AzureFileSystem extends ObjFileSystem { bufferIdx = 0; for (RemoteObject obj : page.getObjectList()) { Location loc = Location.of(rebuildUri(prefix, obj.getKey())); - buffer.add(new FileEntry(loc, obj.getSize(), false, List.of())); + buffer.add(new FileEntry(loc, obj.getSize(), false, obj.modificationTime(), List.of())); } if (page.isTruncated()) { continuationToken = page.getContinuationToken(); diff --git a/fe/fe-filesystem/fe-filesystem-azure/src/main/java/org/apache/doris/filesystem/azure/AzureObjStorage.java b/fe/fe-filesystem/fe-filesystem-azure/src/main/java/org/apache/doris/filesystem/azure/AzureObjStorage.java index 3383219a5b7..84558303858 100644 --- a/fe/fe-filesystem/fe-filesystem-azure/src/main/java/org/apache/doris/filesystem/azure/AzureObjStorage.java +++ b/fe/fe-filesystem/fe-filesystem-azure/src/main/java/org/apache/doris/filesystem/azure/AzureObjStorage.java @@ -185,7 +185,9 @@ public class AzureObjStorage implements ObjStorage<BlobServiceClient> { item.getName().startsWith(uri.key()) ? item.getName().substring(uri.key().length()) : "", item.getProperties().getETag(), item.getProperties().getContentLength() != null - ? item.getProperties().getContentLength() : 0L)); + ? item.getProperties().getContentLength() : 0L, + item.getProperties().getLastModified() != null + ? item.getProperties().getLastModified().toInstant().toEpochMilli() : 0L)); } String nextToken = page.getContinuationToken(); return new RemoteObjects(objects, nextToken != null && !nextToken.isEmpty(), nextToken); @@ -201,7 +203,8 @@ public class AzureObjStorage implements ObjStorage<BlobServiceClient> { BlobClient blobClient = getClient().getBlobContainerClient(uri.container()) .getBlobClient(uri.key()); BlobProperties props = blobClient.getProperties(); - return new RemoteObject(uri.key(), "", props.getETag(), props.getBlobSize()); + return new RemoteObject(uri.key(), "", props.getETag(), props.getBlobSize(), + props.getLastModified() != null ? props.getLastModified().toInstant().toEpochMilli() : 0L); } catch (BlobStorageException e) { if (e.getStatusCode() == HTTP_NOT_FOUND) { throw new FileNotFoundException("404: Object not found: " + remotePath); diff --git a/fe/fe-filesystem/fe-filesystem-broker/src/main/java/org/apache/doris/filesystem/broker/BrokerSpiFileSystem.java b/fe/fe-filesystem/fe-filesystem-broker/src/main/java/org/apache/doris/filesystem/broker/BrokerSpiFileSystem.java index 21c49da926d..dc99c7ec563 100644 --- a/fe/fe-filesystem/fe-filesystem-broker/src/main/java/org/apache/doris/filesystem/broker/BrokerSpiFileSystem.java +++ b/fe/fe-filesystem/fe-filesystem-broker/src/main/java/org/apache/doris/filesystem/broker/BrokerSpiFileSystem.java @@ -165,6 +165,7 @@ public class BrokerSpiFileSystem implements FileSystem { Location.of(s.getPath()), s.getSize(), s.isIsDir(), + s.getModificationTime(), null)); } Iterator<FileEntry> it = entries.iterator(); diff --git a/fe/fe-filesystem/fe-filesystem-hdfs/src/main/java/org/apache/doris/filesystem/hdfs/HdfsFileIterator.java b/fe/fe-filesystem/fe-filesystem-hdfs/src/main/java/org/apache/doris/filesystem/hdfs/HdfsFileIterator.java index 8df248142ca..b6cb6674f93 100644 --- a/fe/fe-filesystem/fe-filesystem-hdfs/src/main/java/org/apache/doris/filesystem/hdfs/HdfsFileIterator.java +++ b/fe/fe-filesystem/fe-filesystem-hdfs/src/main/java/org/apache/doris/filesystem/hdfs/HdfsFileIterator.java @@ -46,7 +46,8 @@ class HdfsFileIterator implements FileIterator { public FileEntry next() throws IOException { FileStatus status = statuses[index++]; Location loc = Location.of(status.getPath().toString()); - return new FileEntry(loc, status.getLen(), status.isDirectory(), null); + return new FileEntry(loc, status.getLen(), status.isDirectory(), + status.getModificationTime(), null); } @Override diff --git a/fe/fe-filesystem/fe-filesystem-local/src/main/java/org/apache/doris/filesystem/local/LocalFileSystem.java b/fe/fe-filesystem/fe-filesystem-local/src/main/java/org/apache/doris/filesystem/local/LocalFileSystem.java index 947c9343b94..be69690ad77 100644 --- a/fe/fe-filesystem/fe-filesystem-local/src/main/java/org/apache/doris/filesystem/local/LocalFileSystem.java +++ b/fe/fe-filesystem/fe-filesystem-local/src/main/java/org/apache/doris/filesystem/local/LocalFileSystem.java @@ -103,7 +103,8 @@ public class LocalFileSystem implements FileSystem { for (Path child : stream) { boolean isDir = Files.isDirectory(child); long length = isDir ? 0L : Files.size(child); - entries.add(new FileEntry(Location.of(child.toUri().toString()), length, isDir, null)); + entries.add(new FileEntry(Location.of(child.toUri().toString()), length, isDir, + Files.getLastModifiedTime(child).toMillis(), null)); } } Iterator<FileEntry> it = entries.iterator(); diff --git a/fe/fe-filesystem/fe-filesystem-s3/src/main/java/org/apache/doris/filesystem/s3/S3FileSystem.java b/fe/fe-filesystem/fe-filesystem-s3/src/main/java/org/apache/doris/filesystem/s3/S3FileSystem.java index ea0b105f9cf..e8cc3d2259f 100644 --- a/fe/fe-filesystem/fe-filesystem-s3/src/main/java/org/apache/doris/filesystem/s3/S3FileSystem.java +++ b/fe/fe-filesystem/fe-filesystem-s3/src/main/java/org/apache/doris/filesystem/s3/S3FileSystem.java @@ -169,7 +169,7 @@ public class S3FileSystem extends ObjFileSystem { bufferIdx = 0; for (RemoteObject obj : page.getObjectList()) { Location loc = Location.of(reconstructUri(prefix, obj.getKey())); - buffer.add(new FileEntry(loc, obj.getSize(), false, List.of())); + buffer.add(new FileEntry(loc, obj.getSize(), false, obj.modificationTime(), List.of())); } if (page.isTruncated()) { continuationToken = page.getContinuationToken(); @@ -425,6 +425,7 @@ public class S3FileSystem extends ObjFileSystem { Location.of("s3://" + bucket + "/" + obj.key()), obj.size(), false, + obj.lastModified() != null ? obj.lastModified().toEpochMilli() : 0L, null)); totalSize += obj.size(); lastMatchedKey = obj.key(); diff --git a/fe/fe-filesystem/fe-filesystem-s3/src/main/java/org/apache/doris/filesystem/s3/S3ObjStorage.java b/fe/fe-filesystem/fe-filesystem-s3/src/main/java/org/apache/doris/filesystem/s3/S3ObjStorage.java index 3b68e084a8e..201be37ae24 100644 --- a/fe/fe-filesystem/fe-filesystem-s3/src/main/java/org/apache/doris/filesystem/s3/S3ObjStorage.java +++ b/fe/fe-filesystem/fe-filesystem-s3/src/main/java/org/apache/doris/filesystem/s3/S3ObjStorage.java @@ -184,7 +184,8 @@ public class S3ObjStorage implements ObjStorage<S3Client> { s3Obj.key(), getRelativePath(uri.key(), s3Obj.key()), s3Obj.eTag(), - s3Obj.size())) + s3Obj.size(), + s3Obj.lastModified() != null ? s3Obj.lastModified().toEpochMilli() : 0L)) .collect(Collectors.toList()); return new RemoteObjects(objects, response.isTruncated(), response.nextContinuationToken()); @@ -204,7 +205,8 @@ public class S3ObjStorage implements ObjStorage<S3Client> { HeadObjectResponse response = getClient().headObject( HeadObjectRequest.builder().bucket(uri.bucket()).key(uri.key()).build()); return new org.apache.doris.filesystem.spi.RemoteObject( - uri.key(), uri.key(), response.eTag(), response.contentLength()); + uri.key(), uri.key(), response.eTag(), response.contentLength(), + response.lastModified() != null ? response.lastModified().toEpochMilli() : 0L); } catch (NoSuchKeyException e) { throw new FileNotFoundException("Object not found: " + remotePath); } catch (S3Exception e) { @@ -426,7 +428,8 @@ public class S3ObjStorage implements ObjStorage<S3Client> { s3Obj.key(), getRelativePathSafe(prefix, s3Obj.key()), s3Obj.eTag(), - s3Obj.size())) + s3Obj.size(), + s3Obj.lastModified() != null ? s3Obj.lastModified().toEpochMilli() : 0L)) .collect(Collectors.toList()); return new RemoteObjects(files, resp.isTruncated(), resp.isTruncated() ? resp.nextContinuationToken() : null); @@ -447,7 +450,8 @@ public class S3ObjStorage implements ObjStorage<S3Client> { .key(fullKey) .build()); RemoteObject obj = new RemoteObject( - fullKey, getRelativePathSafe(prefix, fullKey), resp.eTag(), resp.contentLength()); + fullKey, getRelativePathSafe(prefix, fullKey), resp.eTag(), resp.contentLength(), + resp.lastModified() != null ? resp.lastModified().toEpochMilli() : 0L); return new RemoteObjects(Collections.singletonList(obj), false, null); } catch (NoSuchKeyException e) { LOG.warn("Key not found in headObjectWithMeta, key={}", fullKey); diff --git a/fe/fe-filesystem/fe-filesystem-spi/src/main/java/org/apache/doris/filesystem/spi/FileEntry.java b/fe/fe-filesystem/fe-filesystem-spi/src/main/java/org/apache/doris/filesystem/spi/FileEntry.java index 61a8ee74eea..1aa705c9861 100644 --- a/fe/fe-filesystem/fe-filesystem-spi/src/main/java/org/apache/doris/filesystem/spi/FileEntry.java +++ b/fe/fe-filesystem/fe-filesystem-spi/src/main/java/org/apache/doris/filesystem/spi/FileEntry.java @@ -27,12 +27,15 @@ public final class FileEntry { private final Location location; private final long length; private final boolean isDirectory; + private final long modificationTime; private final List<BlockInfo> blocks; - public FileEntry(Location location, long length, boolean isDirectory, List<BlockInfo> blocks) { + public FileEntry(Location location, long length, boolean isDirectory, + long modificationTime, List<BlockInfo> blocks) { this.location = location; this.length = length; this.isDirectory = isDirectory; + this.modificationTime = modificationTime; this.blocks = blocks == null ? List.of() : List.copyOf(blocks); } @@ -48,6 +51,11 @@ public final class FileEntry { return isDirectory; } + /** Last-modified time in milliseconds since epoch. 0 if not available. */ + public long modificationTime() { + return modificationTime; + } + public List<BlockInfo> blocks() { return blocks; } diff --git a/fe/fe-filesystem/fe-filesystem-spi/src/main/java/org/apache/doris/filesystem/spi/RemoteObject.java b/fe/fe-filesystem/fe-filesystem-spi/src/main/java/org/apache/doris/filesystem/spi/RemoteObject.java index 309af604607..6f05392fde6 100644 --- a/fe/fe-filesystem/fe-filesystem-spi/src/main/java/org/apache/doris/filesystem/spi/RemoteObject.java +++ b/fe/fe-filesystem/fe-filesystem-spi/src/main/java/org/apache/doris/filesystem/spi/RemoteObject.java @@ -26,12 +26,14 @@ public final class RemoteObject { private final String relativePath; private final String etag; private final long size; + private final long modificationTime; - public RemoteObject(String key, String relativePath, String etag, long size) { + public RemoteObject(String key, String relativePath, String etag, long size, long modificationTime) { this.key = key; this.relativePath = relativePath; this.etag = etag; this.size = size; + this.modificationTime = modificationTime; } public String getKey() { @@ -49,4 +51,9 @@ public final class RemoteObject { public long getSize() { return size; } + + /** Last-modified time in milliseconds since epoch. 0 if not available. */ + public long modificationTime() { + return modificationTime; + } } --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
