This is an automated email from the ASF dual-hosted git repository.
sollhui pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/master by this push:
new 54cbb04dbed [perf](storage) Prune Azure glob listing with expanded
prefixes (#69383)
54cbb04dbed is described below
commit 54cbb04dbeddb2b5437b4958cd69508f598632fe
Author: hui lai <[email protected]>
AuthorDate: Mon Oct 26 11:28:56 2026 +0800
[perf](storage) Prune Azure glob listing with expanded prefixes (#69383)
### What problem does this PR solve?
Related PR: #64684
Azure glob listing scans only the longest static prefix. For example,
`date=2025-{0[3-9],1[0-2]}-01/*` scans everything under `date=2025-`,
including unrelated dates.
Extract the object-storage glob helpers into `ObjectStorageGlob` and use
them for Azure and S3-compatible listing. Bounded alternatives become
sorted, non-overlapping remote list prefixes; unsafe or excessive
expansion falls back to a conservative prefix and still applies the
complete matcher. Numeric ranges retain zero padding and can match large
ranges without enumerating every value. Azure limits and resume cursors
continue across expanded prefixes, and object-storage cursors
consistently use unsigned UTF-8 order. S3 directory buckets retain
slash-terminated prefixes and reject unsupported key-based pagination.
### Release note
Reduce Azure glob listing scans for bounded patterns. Correct
object-storage numeric-range and UTF-8 cursor handling. Reject
limited/resumed S3 directory-bucket glob listing because it cannot
support key-based pagination.
---
.../job/offset/s3/S3SourceOffsetProvider.java | 14 +-
.../job/offset/s3/S3SourceOffsetProviderTest.java | 51 ++
.../doris/filesystem/azure/AzureFileSystem.java | 201 ++----
.../apache/doris/filesystem/azure/AzureUri.java | 2 +-
.../filesystem/azure/AzureFileSystemTest.java | 269 ++++++++
.../apache/doris/filesystem/s3/S3FileSystem.java | 14 +
.../doris/filesystem/s3/S3FileSystemTest.java | 92 +++
.../doris/filesystem/spi/ObjectStorageGlob.java | 728 +++++++++++++++++++++
.../filesystem/spi/S3CompatibleFileSystem.java | 368 +----------
.../filesystem/spi/ObjectStorageGlobTest.java | 196 ++++++
.../filesystem/spi/S3CompatibleFileSystemTest.java | 39 +-
11 files changed, 1465 insertions(+), 509 deletions(-)
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/job/offset/s3/S3SourceOffsetProvider.java
b/fe/fe-core/src/main/java/org/apache/doris/job/offset/s3/S3SourceOffsetProvider.java
index 1a4149ead45..286d0ac2dbb 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/job/offset/s3/S3SourceOffsetProvider.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/job/offset/s3/S3SourceOffsetProvider.java
@@ -19,10 +19,12 @@ package org.apache.doris.job.offset.s3;
import org.apache.doris.common.util.DebugPointUtil;
import org.apache.doris.datasource.storage.StorageAdapter;
+import org.apache.doris.datasource.storage.StorageTypeId;
import org.apache.doris.filesystem.FileEntry;
import org.apache.doris.filesystem.FileSystem;
import org.apache.doris.filesystem.GlobListing;
import org.apache.doris.filesystem.Location;
+import org.apache.doris.filesystem.spi.ObjectStorageGlob;
import org.apache.doris.fs.FileSystemFactory;
import org.apache.doris.job.extensions.insert.streaming.StreamingInsertJob;
import org.apache.doris.job.extensions.insert.streaming.StreamingJobProperties;
@@ -45,6 +47,7 @@ import lombok.extern.log4j.Log4j2;
import org.apache.commons.lang3.StringUtils;
import java.util.Arrays;
+import java.util.Comparator;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
@@ -54,6 +57,7 @@ import java.util.stream.Collectors;
public class S3SourceOffsetProvider implements SourceOffsetProvider {
private final boolean oneTimeMode;
private volatile S3Offset noMoreFilesAfterOffset;
+ private volatile Comparator<String> fileKeyOrder =
ObjectStorageGlob::compareUtf8Binary;
volatile S3Offset currentOffset;
volatile String maxEndFile;
@@ -78,6 +82,7 @@ public class S3SourceOffsetProvider implements
SourceOffsetProvider {
String startFile = currentOffset == null ? null :
currentOffset.endFile;
String filePath = null;
StorageAdapter storageAdapter = StorageAdapter.of(copiedProps);
+ updateFileKeyOrder(storageAdapter);
try (FileSystem fileSystem =
FileSystemFactory.getFileSystem(storageAdapter)) {
String uri = storageAdapter.validateAndGetUri(copiedProps);
filePath = storageAdapter.validateAndNormalizeUri(uri);
@@ -188,6 +193,7 @@ public class S3SourceOffsetProvider implements
SourceOffsetProvider {
Map<String, String> copiedProps =
Maps.newTreeMap(String.CASE_INSENSITIVE_ORDER);
copiedProps.putAll(properties);
StorageAdapter storageAdapter = StorageAdapter.of(copiedProps);
+ updateFileKeyOrder(storageAdapter);
S3Offset offsetAtScan = currentOffset;
String startFile = offsetAtScan == null ? null : offsetAtScan.endFile;
try (FileSystem fileSystem =
FileSystemFactory.getFileSystem(storageAdapter)) {
@@ -216,12 +222,18 @@ public class S3SourceOffsetProvider implements
SourceOffsetProvider {
return true;
}
- if (maxEndFile != null && currentOffset.endFile.compareTo(maxEndFile)
< 0) {
+ if (maxEndFile != null
+ && fileKeyOrder.compare(currentOffset.endFile, maxEndFile) <
0) {
return true;
}
return false;
}
+ private void updateFileKeyOrder(StorageAdapter storageAdapter) {
+ fileKeyOrder = storageAdapter.getType() == StorageTypeId.AZURE
+ ? Comparator.naturalOrder() :
ObjectStorageGlob::compareUtf8Binary;
+ }
+
@Override
public boolean hasReachedEnd() {
S3Offset offset = currentOffset;
diff --git
a/fe/fe-core/src/test/java/org/apache/doris/job/offset/s3/S3SourceOffsetProviderTest.java
b/fe/fe-core/src/test/java/org/apache/doris/job/offset/s3/S3SourceOffsetProviderTest.java
index 863c144c1da..04ce5b010ab 100644
---
a/fe/fe-core/src/test/java/org/apache/doris/job/offset/s3/S3SourceOffsetProviderTest.java
+++
b/fe/fe-core/src/test/java/org/apache/doris/job/offset/s3/S3SourceOffsetProviderTest.java
@@ -19,10 +19,12 @@ package org.apache.doris.job.offset.s3;
import org.apache.doris.common.util.S3URI;
import org.apache.doris.datasource.storage.StorageAdapter;
+import org.apache.doris.datasource.storage.StorageTypeId;
import org.apache.doris.filesystem.FileEntry;
import org.apache.doris.filesystem.FileSystem;
import org.apache.doris.filesystem.GlobListing;
import org.apache.doris.filesystem.Location;
+import org.apache.doris.filesystem.spi.ObjectStorageGlob;
import org.apache.doris.filesystem.spi.S3CompatibleFileSystem;
import org.apache.doris.fs.FileSystemFactory;
import org.apache.doris.job.extensions.insert.streaming.StreamingJobProperties;
@@ -229,4 +231,53 @@ public class S3SourceOffsetProviderTest {
Assertions.assertFalse(matcher.matcher(key).matches(), "Unexpected
file: " + key);
}
}
+
+ @Test
+ void hasMoreDataToConsumeUsesUtf8BinaryOrder() {
+ String privateUseKey = "data/" + Character.toString(0xE000) +
"/file.csv";
+ String emojiKey = "data/" + Character.toString(0x1F600) + "/file.csv";
+ Assertions.assertTrue(privateUseKey.compareTo(emojiKey) > 0);
+
Assertions.assertTrue(ObjectStorageGlob.compareUtf8Binary(privateUseKey,
emojiKey) < 0);
+
+ S3Offset currentOffset = new S3Offset();
+ currentOffset.setEndFile(privateUseKey);
+ S3SourceOffsetProvider provider = new S3SourceOffsetProvider();
+ provider.currentOffset = currentOffset;
+ provider.maxEndFile = emojiKey;
+
+ Assertions.assertTrue(provider.hasMoreDataToConsume());
+ }
+
+ @Test
+ void hasMoreDataToConsumeUsesAzureStringOrder() throws Exception {
+ String emojiKey = "data/" + Character.toString(0x1F600) + "/file.csv";
+ String fullwidthKey = "data/" + Character.toString(0xFF21) +
"/file.csv";
+ StorageAdapter adapter = Mockito.mock(StorageAdapter.class);
+ Mockito.when(adapter.getType()).thenReturn(StorageTypeId.AZURE);
+
Mockito.when(adapter.validateAndGetUri(Mockito.anyMap())).thenReturn("s3://bucket/data/*/file.csv");
+
Mockito.when(adapter.validateAndNormalizeUri(Mockito.anyString())).thenReturn("s3://bucket/data/*/file.csv");
+ FileSystem fs = Mockito.mock(FileSystem.class);
+ Mockito.when(fs.globListWithLimit(Mockito.any(Location.class),
Mockito.eq(emojiKey),
+ Mockito.anyLong(),
Mockito.anyLong())).thenReturn(page(fullwidthKey, fullwidthKey));
+
+ try (MockedStatic<StorageAdapter> adapters =
Mockito.mockStatic(StorageAdapter.class);
+ MockedStatic<FileSystemFactory> factory =
Mockito.mockStatic(FileSystemFactory.class)) {
+ adapters.when(() ->
StorageAdapter.of(Mockito.anyMap())).thenReturn(adapter);
+ factory.when(() ->
FileSystemFactory.getFileSystem(adapter)).thenReturn(fs);
+ for (boolean metadataProbe : List.of(true, false)) {
+ S3Offset offset = new S3Offset();
+ offset.setEndFile(emojiKey);
+ S3SourceOffsetProvider provider = new S3SourceOffsetProvider();
+ provider.currentOffset = offset;
+ if (metadataProbe) {
+ provider.fetchRemoteMeta(TVF_PROPS);
+ } else {
+ provider.getNextOffset(ONCE_PROPS, TVF_PROPS);
+ }
+ Assertions.assertTrue(provider.hasMoreDataToConsume());
+ offset.setEndFile(fullwidthKey);
+ Assertions.assertFalse(provider.hasMoreDataToConsume());
+ }
+ }
+ }
}
diff --git
a/fe/fe-filesystem/fe-filesystem-azure/src/main/java/org/apache/doris/filesystem/azure/AzureFileSystem.java
b/fe/fe-filesystem/fe-filesystem-azure/src/main/java/org/apache/doris/filesystem/azure/AzureFileSystem.java
index 02cb3479ef0..6fb588fbd45 100644
---
a/fe/fe-filesystem/fe-filesystem-azure/src/main/java/org/apache/doris/filesystem/azure/AzureFileSystem.java
+++
b/fe/fe-filesystem/fe-filesystem-azure/src/main/java/org/apache/doris/filesystem/azure/AzureFileSystem.java
@@ -26,6 +26,7 @@ import org.apache.doris.filesystem.GlobListing;
import org.apache.doris.filesystem.Location;
import org.apache.doris.filesystem.UploadPartResult;
import org.apache.doris.filesystem.spi.ObjFileSystem;
+import org.apache.doris.filesystem.spi.ObjectStorageGlob;
import org.apache.doris.filesystem.spi.RemoteObject;
import org.apache.doris.filesystem.spi.RemoteObjects;
import org.apache.doris.filesystem.spi.RequestBody;
@@ -39,6 +40,7 @@ import java.io.InputStream;
import java.io.OutputStream;
import java.util.ArrayList;
import java.util.Arrays;
+import java.util.Comparator;
import java.util.List;
import java.util.NoSuchElementException;
import java.util.UUID;
@@ -394,54 +396,70 @@ public class AzureFileSystem extends ObjFileSystem {
AzureUri parsed = AzureUri.parse(uri);
String container = parsed.container();
String keyPattern = parsed.key();
- // base = uri with the key portion stripped, preserving the original
scheme/host syntax.
- String base = uri.substring(0, uri.length() - keyPattern.length());
+ // The parsed key is decoded, so its length cannot locate the key in
the original URI.
+ int keySeparator = uri.indexOf('/', uri.indexOf("://") + 3);
+ if (parsed.scheme().equals("https") || parsed.scheme().equals("http"))
{
+ keySeparator = uri.indexOf('/', keySeparator + 1);
+ }
+ String base = keySeparator < 0 ? uri + "/" : uri.substring(0,
keySeparator + 1);
- Pattern matcher = Pattern.compile(globToRegex(keyPattern));
- String listKeyPrefix = longestNonGlobPrefix(keyPattern);
- String listPrefixUri = base + listKeyPrefix;
+ String expandedKeyPattern =
ObjectStorageGlob.expandNumericRanges(keyPattern);
+ Pattern matcher = Pattern.compile(globToRegex(expandedKeyPattern));
+ String listKeyPrefix = longestNonGlobPrefix(expandedKeyPattern);
+ List<String> listKeyPrefixes =
ObjectStorageGlob.expandedGlobListPrefixes(
+ expandedKeyPattern, Comparator.naturalOrder());
List<FileEntry> files = new ArrayList<>();
long totalSize = 0L;
- String maxFile = "";
- String continuationToken = null;
+ String nextMatchAfterLimit = "";
+ String lastMatchedKey = "";
boolean reachLimit = false;
- outer:
- do {
- RemoteObjects page = objStorage.listObjects(listPrefixUri,
continuationToken);
- for (RemoteObject obj : page.getObjectList()) {
- String key = obj.getKey();
- if (key.endsWith(DIR_MARKER_SUFFIX)) {
- continue;
- }
- if (startAfter != null && !startAfter.isEmpty() &&
key.compareTo(startAfter) <= 0) {
- continue;
- }
- if (!matcher.matcher(key).matches()) {
- continue;
- }
- if ((maxFiles > 0 && files.size() >= maxFiles)
- || (maxBytes > 0 && totalSize >= maxBytes)) {
- maxFile = key;
- reachLimit = true;
- break outer;
+ for (String currentListKeyPrefix : listKeyPrefixes) {
+ String continuationToken = null;
+ String listPrefixUri = base +
AzureUri.encodeKey(currentListKeyPrefix);
+ do {
+ RemoteObjects page = objStorage.listObjects(listPrefixUri,
continuationToken);
+ for (RemoteObject obj : page.getObjectList()) {
+ String key = obj.getKey();
+ if (key.endsWith(DIR_MARKER_SUFFIX)) {
+ continue;
+ }
+ if (startAfter != null && !startAfter.isEmpty()
+ && key.compareTo(startAfter) <= 0) {
+ continue;
+ }
+ if (reachLimit) {
+ if (nextMatchAfterLimit.isEmpty() &&
matcher.matcher(key).matches()) {
+ nextMatchAfterLimit = key;
+ }
+ continue;
+ }
+ if (!matcher.matcher(key).matches()) {
+ continue;
+ }
+ files.add(new FileEntry(
+ Location.of(base + key),
+ obj.getSize(),
+ false,
+ obj.getModificationTime(),
+ null));
+ totalSize += obj.getSize();
+ lastMatchedKey = key;
+
+ if ((maxFiles > 0 && files.size() >= maxFiles)
+ || (maxBytes > 0 && totalSize >= maxBytes)) {
+ reachLimit = true;
+ }
}
- files.add(new FileEntry(
- Location.of(base + key),
- obj.getSize(),
- false,
- obj.getModificationTime(),
- null));
- totalSize += obj.getSize();
- maxFile = key;
- }
- continuationToken = page.isTruncated() ?
page.getContinuationToken() : null;
- } while (continuationToken != null);
-
- if (!reachLimit && files.isEmpty()) {
- maxFile = "";
+ continuationToken = page.isTruncated() ?
page.getContinuationToken() : null;
+ } while (continuationToken != null && (!reachLimit ||
nextMatchAfterLimit.isEmpty()));
+ if (reachLimit && !nextMatchAfterLimit.isEmpty()) {
+ break;
+ }
}
+
+ String maxFile = nextMatchAfterLimit.isEmpty() ? lastMatchedKey :
nextMatchAfterLimit;
return new GlobListing(files, container, listKeyPrefix, maxFile);
}
@@ -477,111 +495,16 @@ public class AzureFileSystem extends ObjFileSystem {
* Azure list-blobs call.
*/
static String longestNonGlobPrefix(String globPattern) {
- int earliest = globPattern.length();
- for (char c : new char[]{'*', '?', '[', '{', '\\'}) {
- int idx = globPattern.indexOf(c);
- if (idx >= 0 && idx < earliest) {
- earliest = idx;
- }
- }
- return globPattern.substring(0, earliest);
+ return ObjectStorageGlob.longestNonGlobPrefix(globPattern);
}
/**
* Translates a Java NIO glob pattern to a regex matched against raw blob
keys.
- * Mirrors the JDK glob → regex conversion but operates on plain strings,
so it
- * tolerates characters that the host OS path syntax would refuse.
Supports:
- * <ul>
- * <li>{@code *} → any run of non-{@code /} characters;</li>
- * <li>{@code **} → any run of characters including {@code /};</li>
- * <li>{@code ?} → any single non-{@code /} character;</li>
- * <li>{@code [...]} character class (with {@code !} for negation);</li>
- * <li>{@code {a,b,c}} alternation;</li>
- * <li>{@code \X} escapes the next character literally.</li>
- * </ul>
+ * Mirrors the JDK glob to regex conversion but operates on plain strings,
so it
+ * tolerates characters that the host OS path syntax would refuse.
*/
static String globToRegex(String glob) {
- StringBuilder sb = new StringBuilder("^");
- boolean inClass = false;
- boolean inGroup = false;
- int i = 0;
- while (i < glob.length()) {
- char c = glob.charAt(i);
- if (c == '\\') {
- if (i + 1 < glob.length()) {
- sb.append(Pattern.quote(String.valueOf(glob.charAt(i +
1))));
- i += 2;
- } else {
- sb.append("\\\\");
- i++;
- }
- continue;
- }
- if (inClass) {
- if (c == ']') {
- inClass = false;
- sb.append(']');
- } else if (c == '\\' || c == '[') {
- sb.append('\\').append(c);
- } else {
- sb.append(c);
- }
- i++;
- continue;
- }
- switch (c) {
- case '*':
- if (i + 1 < glob.length() && glob.charAt(i + 1) == '*') {
- sb.append(".*");
- i += 2;
- } else {
- sb.append("[^/]*");
- i++;
- }
- break;
- case '?':
- sb.append("[^/]");
- i++;
- break;
- case '[':
- inClass = true;
- sb.append('[');
- i++;
- if (i < glob.length() && glob.charAt(i) == '!') {
- sb.append('^');
- i++;
- }
- break;
- case '{':
- inGroup = true;
- sb.append("(?:");
- i++;
- break;
- case '}':
- inGroup = false;
- sb.append(')');
- i++;
- break;
- case ',':
- if (inGroup) {
- sb.append('|');
- } else {
- sb.append(',');
- }
- i++;
- break;
- default:
- if ("\\.^$|+()".indexOf(c) >= 0) {
- sb.append('\\').append(c);
- } else {
- sb.append(c);
- }
- i++;
- break;
- }
- }
- sb.append('$');
- return sb.toString();
+ return ObjectStorageGlob.globToRegex(glob);
}
/** Lazy-paginating FileIterator over Azure list results. */
diff --git
a/fe/fe-filesystem/fe-filesystem-azure/src/main/java/org/apache/doris/filesystem/azure/AzureUri.java
b/fe/fe-filesystem/fe-filesystem-azure/src/main/java/org/apache/doris/filesystem/azure/AzureUri.java
index 2668d1bb458..777108a384c 100644
---
a/fe/fe-filesystem/fe-filesystem-azure/src/main/java/org/apache/doris/filesystem/azure/AzureUri.java
+++
b/fe/fe-filesystem/fe-filesystem-azure/src/main/java/org/apache/doris/filesystem/azure/AzureUri.java
@@ -190,7 +190,7 @@ public final class AzureUri {
return scheme + "://" + container + "@" + accountName + "/" +
encodeKey(key);
}
- private static String encodeKey(String raw) {
+ static String encodeKey(String raw) {
if (raw.isEmpty()) {
return raw;
}
diff --git
a/fe/fe-filesystem/fe-filesystem-azure/src/test/java/org/apache/doris/filesystem/azure/AzureFileSystemTest.java
b/fe/fe-filesystem/fe-filesystem-azure/src/test/java/org/apache/doris/filesystem/azure/AzureFileSystemTest.java
index 596f1c88ff0..995649e8e7b 100644
---
a/fe/fe-filesystem/fe-filesystem-azure/src/test/java/org/apache/doris/filesystem/azure/AzureFileSystemTest.java
+++
b/fe/fe-filesystem/fe-filesystem-azure/src/test/java/org/apache/doris/filesystem/azure/AzureFileSystemTest.java
@@ -21,6 +21,7 @@ import org.apache.doris.filesystem.DorisOutputFile;
import org.apache.doris.filesystem.FileEntry;
import org.apache.doris.filesystem.GlobListing;
import org.apache.doris.filesystem.Location;
+import org.apache.doris.filesystem.spi.ObjectStorageGlob;
import org.apache.doris.filesystem.spi.RemoteObject;
import org.apache.doris.filesystem.spi.RemoteObjects;
import org.apache.doris.filesystem.spi.RequestBody;
@@ -35,6 +36,7 @@ import java.io.FileNotFoundException;
import java.io.IOException;
import java.io.OutputStream;
import java.util.List;
+import java.util.regex.Pattern;
/**
* Unit tests for {@link AzureFileSystem} using a mock {@link AzureObjStorage}.
@@ -236,6 +238,273 @@ class AzureFileSystemTest {
Assertions.assertEquals("data/d.csv", listing.getMaxFile());
}
+ @Test
+ void globListWithLimit_resumesInStringOrderWithinOnePrefix() throws
IOException {
+ String emojiKey = "data/" + Character.toString(0x1F600) + ".csv";
+ String fullwidthKey = "data/" + Character.toString(0xFF21) + ".csv";
+ Assertions.assertTrue(emojiKey.compareTo(fullwidthKey) < 0);
+ Assertions.assertTrue(ObjectStorageGlob.compareUtf8Binary(emojiKey,
fullwidthKey) > 0);
+ Mockito.when(mockStorage.listObjects(ArgumentMatchers.anyString(),
ArgumentMatchers.isNull()))
+ .thenReturn(new RemoteObjects(List.of(new
RemoteObject(emojiKey, "", null, 10L, 0L)),
+ true, "next-page"));
+ Mockito.when(mockStorage.listObjects(ArgumentMatchers.anyString(),
ArgumentMatchers.eq("next-page")))
+ .thenReturn(new RemoteObjects(List.of(new
RemoteObject(fullwidthKey, "", null, 10L, 0L)),
+ false, null));
+ Location glob = Location.of("wasbs://[email protected]/data/*.csv");
+
+ GlobListing first = fs.globListWithLimit(glob, null, 0L, 1L);
+ Assertions.assertEquals(List.of("wasbs://[email protected]/" + emojiKey),
+ first.getFiles().stream().map(file ->
file.location().uri()).toList());
+ Assertions.assertEquals(fullwidthKey, first.getMaxFile());
+ GlobListing second = fs.globListWithLimit(glob, emojiKey, 0L, 1L);
+ Assertions.assertEquals(List.of("wasbs://[email protected]/" + fullwidthKey),
+ second.getFiles().stream().map(file ->
file.location().uri()).toList());
+ Assertions.assertEquals(fullwidthKey, second.getMaxFile());
+ }
+
+ @Test
+ void globListWithLimit_resumesInStringOrderAcrossExpandedPrefixes() throws
IOException {
+ String fullwidth = Character.toString(0xFF21);
+ String emoji = Character.toString(0x1F600);
+ String fullwidthKey = "data/" + fullwidth + "/file.csv";
+ String emojiKey = "data/" + emoji + "/file.csv";
+ mockListingWithUriParsing(List.of(fullwidthKey, emojiKey));
+ Location glob = Location.of("wasbs://[email protected]/data/{" + fullwidth +
"," + emoji + "}/file.csv");
+
+ GlobListing first = fs.globListWithLimit(glob, null, 0L, 1L);
+ Assertions.assertEquals(List.of("wasbs://[email protected]/" + emojiKey),
+ first.getFiles().stream().map(file ->
file.location().uri()).toList());
+ Assertions.assertEquals(fullwidthKey, first.getMaxFile());
+ GlobListing second = fs.globListWithLimit(glob, emojiKey, 0L, 1L);
+ Assertions.assertEquals(List.of("wasbs://[email protected]/" + fullwidthKey),
+ second.getFiles().stream().map(file ->
file.location().uri()).toList());
+ Assertions.assertEquals(fullwidthKey, second.getMaxFile());
+ }
+
+ @Test
+ void globListWithLimit_encodesGeneratedReservedCharacters() throws
IOException {
+ for (String glob : List.of("data/[$-&]/*/file.csv",
"data/[)-,]/*/file.csv", "data/[\"-$]/*/file.csv")) {
+ List<String> keys = List.of("data/%/sub/file.csv",
"data/+/sub/file.csv", "data/#/sub/file.csv",
+ "data/$/sub/file.csv", "data/&/sub/file.csv");
+ mockListingWithUriParsing(keys);
+ List<String> expected = keys.stream()
+ .filter(key ->
Pattern.matches(ObjectStorageGlob.globToRegex(glob), key))
+ .sorted().map(key -> "wasbs://[email protected]/" +
key).toList();
+
+ GlobListing listing =
fs.globListWithLimit(Location.of("wasbs://[email protected]/" + glob), null, 0L, 0L);
+ Assertions.assertEquals(expected,
+ listing.getFiles().stream().map(file ->
file.location().uri()).toList(), glob);
+ }
+ Mockito.verify(mockStorage).listObjects(
+ ArgumentMatchers.eq("wasbs://[email protected]/data/%25/"),
ArgumentMatchers.any());
+ Mockito.verify(mockStorage).listObjects(
+ ArgumentMatchers.eq("wasbs://[email protected]/data/%2B/"),
ArgumentMatchers.any());
+ Mockito.verify(mockStorage).listObjects(
+ ArgumentMatchers.eq("wasbs://[email protected]/data/%23/"),
ArgumentMatchers.any());
+ }
+
+ @Test
+ void globListWithLimit_preservesBaseForEncodedInputKeys() throws
IOException {
+ for (String base : List.of("wasbs://[email protected]/",
"https://a.blob.core.windows.net/c/";,
+ "s3://c/")) {
+ mockListingWithUriParsing(List.of("data/a+b/1/file.csv"));
+ GlobListing listing = fs.globListWithLimit(
+ Location.of(base + "data/a%2Bb/[12]/*"), null, 0L, 0L);
+ Assertions.assertEquals(List.of(base + "data/a+b/1/file.csv"),
+ listing.getFiles().stream().map(file ->
file.location().uri()).toList());
+ Mockito.verify(mockStorage).listObjects(
+ ArgumentMatchers.eq(base + "data/a%2Bb/1/"),
ArgumentMatchers.any());
+ }
+ }
+
+ @Test
+ void globListWithLimit_fallsBackForEscapedRangeEndpoints() throws
IOException {
+ mockListingWithUriParsing(List.of("data/b/sub/file.csv",
"data/m/sub/file.csv"));
+ for (String characterClass : List.of("[\\a-z]", "[a-\\c]")) {
+ String glob = "data/" + characterClass + "/*/file.csv";
+ GlobListing listing =
fs.globListWithLimit(Location.of("wasbs://[email protected]/" + glob), null, 0L, 0L);
+ Assertions.assertEquals("wasbs://[email protected]/data/b/sub/file.csv",
+ listing.getFiles().get(0).location().uri(), glob);
+ Assertions.assertEquals(characterClass.contains("z") ? 2 : 1,
listing.getFiles().size());
+ }
+ }
+
+ @Test
+ void globListWithLimit_preservesPaddedMinimumLong() throws IOException {
+ String padded = "data_-09223372036854775808.csv";
+ mockListingWithUriParsing(List.of(padded,
"data_-9223372036854775808.csv"));
+ GlobListing listing = fs.globListWithLimit(Location.of(
+
"wasbs://[email protected]/data_{-09223372036854775808..-09223372036854775807,x}.csv"),
null, 0L, 0L);
+ Assertions.assertEquals(List.of("wasbs://[email protected]/" + padded),
+ listing.getFiles().stream().map(file ->
file.location().uri()).toList());
+ }
+
+ private void mockListingWithUriParsing(List<String> keys) throws
IOException {
+ Mockito.doAnswer(invocation -> {
+ // Exercise the same second URI parse as
AzureObjStorage.listObjects.
+ String prefix = AzureUri.parse(invocation.getArgument(0)).key();
+ List<RemoteObject> objects = keys.stream().filter(key ->
key.startsWith(prefix)).sorted()
+ .map(key -> new RemoteObject(key, "", null, 10L,
0L)).toList();
+ return new RemoteObjects(objects, false, null);
+ }).when(mockStorage).listObjects(ArgumentMatchers.anyString(),
ArgumentMatchers.any());
+ }
+
+ @Test
+ void globListWithLimit_listsExpandedDatePrefixesInsteadOfBroadDatePrefix()
throws IOException {
+ RemoteObjects empty = new RemoteObjects(List.of(), false, null);
+ Mockito.when(mockStorage.listObjects(ArgumentMatchers.anyString(),
ArgumentMatchers.any()))
+ .thenReturn(empty);
+ Mockito.when(mockStorage.listObjects(
+
ArgumentMatchers.eq("wasbs://[email protected]/asin_trend/sale/month/"
+ + "date%3D2025-03-01/mp_id%3D8/0/0/436/"),
+ ArgumentMatchers.any()))
+ .thenReturn(new RemoteObjects(
+ List.of(new RemoteObject(
+
"asin_trend/sale/month/date=2025-03-01/mp_id=8/0/0/436/a.parquet",
+ "a.parquet", null, 11L, 0L)),
+ false, null));
+ Mockito.when(mockStorage.listObjects(
+
ArgumentMatchers.eq("wasbs://[email protected]/asin_trend/sale/month/"
+ + "date%3D2025-10-01/mp_id%3D8/0/0/436/"),
+ ArgumentMatchers.any()))
+ .thenReturn(new RemoteObjects(
+ List.of(new RemoteObject(
+
"asin_trend/sale/month/date=2025-10-01/mp_id=8/0/0/436/b.parquet",
+ "b.parquet", null, 12L, 0L)),
+ false, null));
+
+ GlobListing listing =
fs.globListWithLimit(Location.of("wasbs://[email protected]/asin_trend/sale/month/"
+ + "date=2025-{0[3-9],1[0-2]}-01/mp_id=8/0/0/436/*"), null, 0L,
0L);
+
+ Assertions.assertEquals(2, listing.getFiles().size());
+ Assertions.assertEquals("asin_trend/sale/month/date=2025-",
listing.getPrefix());
+ Mockito.verify(mockStorage, Mockito.never()).listObjects(
+
ArgumentMatchers.eq("wasbs://[email protected]/asin_trend/sale/month/date%3D2025-"),
+ ArgumentMatchers.any());
+ Mockito.verify(mockStorage).listObjects(
+
ArgumentMatchers.eq("wasbs://[email protected]/asin_trend/sale/month/"
+ + "date%3D2025-03-01/mp_id%3D8/0/0/436/"),
+ ArgumentMatchers.any());
+ Mockito.verify(mockStorage).listObjects(
+
ArgumentMatchers.eq("wasbs://[email protected]/asin_trend/sale/month/"
+ + "date%3D2025-12-01/mp_id%3D8/0/0/436/"),
+ ArgumentMatchers.any());
+ }
+
+ @Test
+ void globListWithLimit_preservesZeroPaddedNumericRangePrefixes() throws
IOException {
+ RemoteObjects empty = new RemoteObjects(List.of(), false, null);
+ Mockito.when(mockStorage.listObjects(ArgumentMatchers.anyString(),
ArgumentMatchers.any()))
+ .thenReturn(empty);
+ Mockito.when(mockStorage.listObjects(
+
ArgumentMatchers.eq("wasbs://[email protected]/date%3D2025-01-01/"),
+ ArgumentMatchers.any()))
+ .thenReturn(new RemoteObjects(
+ List.of(new
RemoteObject("date=2025-01-01/file.parquet",
+ "file.parquet", null, 10L, 0L)),
+ false, null));
+
+ GlobListing listing = fs.globListWithLimit(
+ Location.of("wasbs://[email protected]/date=2025-{01..03}-01/*"),
null, 0L, 0L);
+
+ Assertions.assertEquals(1, listing.getFiles().size());
+
Assertions.assertEquals("wasbs://[email protected]/date=2025-01-01/file.parquet",
+ listing.getFiles().get(0).location().uri());
+ Mockito.verify(mockStorage).listObjects(
+ ArgumentMatchers.eq("wasbs://[email protected]/date%3D2025-01-01/"),
ArgumentMatchers.any());
+ Mockito.verify(mockStorage).listObjects(
+ ArgumentMatchers.eq("wasbs://[email protected]/date%3D2025-03-01/"),
ArgumentMatchers.any());
+ Mockito.verify(mockStorage, Mockito.never()).listObjects(
+ ArgumentMatchers.eq("wasbs://[email protected]/date%3D2025-1-01/"),
ArgumentMatchers.any());
+ }
+
+ @Test
+ void globListWithLimit_preservesOversizedNumericRangeAfterPrefixFallback()
+ throws IOException {
+ Mockito.when(mockStorage.listObjects(
+ ArgumentMatchers.eq("wasbs://[email protected]/date%3D"),
+ ArgumentMatchers.any()))
+ .thenReturn(new RemoteObjects(
+ List.of(
+ new RemoteObject("date=123/file.parquet",
+ "file.parquet", null, 10L, 0L),
+ new RemoteObject("date=10000000/file.parquet",
+ "file.parquet", null, 20L, 0L),
+ new RemoteObject("date=0/file.parquet",
+ "file.parquet", null, 30L, 0L),
+ new RemoteObject("date=10000001/file.parquet",
+ "file.parquet", null, 40L, 0L),
+ new RemoteObject("date=0001/file.parquet",
+ "file.parquet", null, 50L, 0L)),
+ false, null));
+
+ GlobListing listing = fs.globListWithLimit(
+ Location.of("wasbs://[email protected]/date={1..10000000}/*"), null,
0L, 0L);
+
+ Assertions.assertEquals("date=", listing.getPrefix());
+ Assertions.assertEquals(2, listing.getFiles().size());
+ Assertions.assertEquals("wasbs://[email protected]/date=123/file.parquet",
+ listing.getFiles().get(0).location().uri());
+
Assertions.assertEquals("wasbs://[email protected]/date=10000000/file.parquet",
+ listing.getFiles().get(1).location().uri());
+ Mockito.verify(mockStorage).listObjects(
+ ArgumentMatchers.eq("wasbs://[email protected]/date%3D"),
ArgumentMatchers.any());
+ }
+
+ @Test
+ void globListWithLimit_doesNotAppendSuffixToPartialBraceArmPrefix() throws
IOException {
+ Mockito.when(mockStorage.listObjects(
+ ArgumentMatchers.eq("wasbs://[email protected]/data/"),
+ ArgumentMatchers.any()))
+ .thenReturn(new RemoteObjects(
+ List.of(
+ new RemoteObject("data/foobar/part.parquet",
+ "part.parquet", null, 10L, 0L),
+ new RemoteObject("data/barbaz/part.parquet",
+ "part.parquet", null, 20L, 0L)),
+ false, null));
+
+ GlobListing listing = fs.globListWithLimit(
+
Location.of("wasbs://[email protected]/data/{foo*,bar*}/part.parquet"), null, 0L, 0L);
+
+ Assertions.assertEquals(2, listing.getFiles().size());
+ Mockito.verify(mockStorage).listObjects(
+ ArgumentMatchers.eq("wasbs://[email protected]/data/"),
ArgumentMatchers.any());
+ Mockito.verify(mockStorage, Mockito.never()).listObjects(
+
ArgumentMatchers.eq("wasbs://[email protected]/data/foo/part.parquet"),
ArgumentMatchers.any());
+ Mockito.verify(mockStorage, Mockito.never()).listObjects(
+
ArgumentMatchers.eq("wasbs://[email protected]/data/bar/part.parquet"),
ArgumentMatchers.any());
+ }
+
+ @Test
+ void globListWithLimit_findsNextMatchAcrossExpandedPrefixesAfterLimit()
throws IOException {
+ Mockito.when(mockStorage.listObjects(
+
ArgumentMatchers.eq("wasbs://[email protected]/date%3D2025-01/file"),
+ ArgumentMatchers.any()))
+ .thenReturn(new RemoteObjects(
+ List.of(
+ new RemoteObject("date=2025-01/file-a.csv",
+ "file-a.csv", null, 10L, 0L),
+ new RemoteObject("date=2025-01/file-b.csv",
+ "file-b.csv", null, 20L, 0L)),
+ false, null));
+ Mockito.when(mockStorage.listObjects(
+
ArgumentMatchers.eq("wasbs://[email protected]/date%3D2025-02/file"),
+ ArgumentMatchers.any()))
+ .thenReturn(new RemoteObjects(
+ List.of(new RemoteObject("date=2025-02/file-c.csv",
+ "file-c.csv", null, 30L, 0L)),
+ false, null));
+
+ GlobListing listing = fs.globListWithLimit(
+ Location.of("wasbs://[email protected]/date=2025-0[12]/file*.csv"),
null, 0L, 2L);
+
+ Assertions.assertEquals(2, listing.getFiles().size());
+ Assertions.assertEquals("date=2025-02/file-c.csv",
listing.getMaxFile());
+ Mockito.verify(mockStorage).listObjects(
+
ArgumentMatchers.eq("wasbs://[email protected]/date%3D2025-02/file"),
ArgumentMatchers.any());
+ }
+
// ---------------------------------------------------------------------
// F04 — listFiles(Location) glob-aware dispatch
// ---------------------------------------------------------------------
diff --git
a/fe/fe-filesystem/fe-filesystem-s3-base/src/main/java/org/apache/doris/filesystem/s3/S3FileSystem.java
b/fe/fe-filesystem/fe-filesystem-s3-base/src/main/java/org/apache/doris/filesystem/s3/S3FileSystem.java
index 38b6c23a352..8e1cbfb947d 100644
---
a/fe/fe-filesystem/fe-filesystem-s3-base/src/main/java/org/apache/doris/filesystem/s3/S3FileSystem.java
+++
b/fe/fe-filesystem/fe-filesystem-s3-base/src/main/java/org/apache/doris/filesystem/s3/S3FileSystem.java
@@ -17,8 +17,11 @@
package org.apache.doris.filesystem.s3;
+import org.apache.doris.filesystem.GlobListing;
+import org.apache.doris.filesystem.Location;
import org.apache.doris.filesystem.spi.S3CompatibleFileSystem;
+import java.io.IOException;
import java.util.List;
import java.util.Optional;
@@ -53,6 +56,17 @@ public class S3FileSystem extends S3CompatibleFileSystem {
return Optional.ofNullable(properties);
}
+ @Override
+ public GlobListing globListWithLimit(Location path, String startAfter,
long maxBytes,
+ long maxFiles) throws IOException {
+ if (isDirectoryBucketEndpoint()
+ && ((startAfter != null && !startAfter.isEmpty()) || maxBytes
> 0 || maxFiles > 0)) {
+ throw new IOException("S3 directory bucket glob listing does not
support key-based pagination "
+ + "because StartAfter is unsupported and object order is
not lexicographical");
+ }
+ return super.globListWithLimit(path, startAfter, maxBytes, maxFiles);
+ }
+
@Override
protected String globListPrefix(String globPattern) {
if (isDirectoryBucketEndpoint()) {
diff --git
a/fe/fe-filesystem/fe-filesystem-s3-base/src/test/java/org/apache/doris/filesystem/s3/S3FileSystemTest.java
b/fe/fe-filesystem/fe-filesystem-s3-base/src/test/java/org/apache/doris/filesystem/s3/S3FileSystemTest.java
index fe9b28dd8c9..e751bc20a18 100644
---
a/fe/fe-filesystem/fe-filesystem-s3-base/src/test/java/org/apache/doris/filesystem/s3/S3FileSystemTest.java
+++
b/fe/fe-filesystem/fe-filesystem-s3-base/src/test/java/org/apache/doris/filesystem/s3/S3FileSystemTest.java
@@ -702,6 +702,26 @@ class S3FileSystemTest {
ArgumentMatchers.eq("s3://bucket/data/b"),
ArgumentMatchers.any());
}
+ @Test
+ void globListWithLimit_rejectsDirectoryBucketKeyCursorPagination() throws
IOException {
+ S3FileSystemProperties properties = S3FileSystemProperties.of(Map.of(
+ "s3.endpoint",
"https://s3express-usw2-az1.us-west-2.amazonaws.com";,
+ "s3.region", "us-west-2"));
+ S3FileSystem directoryBucketFs = new S3FileSystem(properties,
mockStorage);
+ Location path = Location.of("s3://bucket/data/*.csv");
+
+ IOException resumed = Assertions.assertThrows(IOException.class,
+ () -> directoryBucketFs.globListWithLimit(path, "data/a.csv",
0L, 0L));
+ Assertions.assertTrue(resumed.getMessage().contains("directory
bucket"));
+ Assertions.assertThrows(IOException.class,
+ () -> directoryBucketFs.globListWithLimit(path, null, 0L, 1L));
+ Assertions.assertThrows(IOException.class,
+ () -> directoryBucketFs.globListWithLimit(path, null, 10L,
0L));
+
+ Mockito.verify(mockStorage, Mockito.never()).listObjectsWithOptions(
+ ArgumentMatchers.anyString(),
ArgumentMatchers.<ObjectListOptions>any());
+ }
+
@Test
void globListWithLimit_listsExpandedDatePrefixesInsteadOfBroadDatePrefix()
throws IOException {
Mockito.when(mockStorage.listObjects(ArgumentMatchers.anyString(),
ArgumentMatchers.isNull()))
@@ -899,6 +919,16 @@ class S3FileSystemTest {
Assertions.assertFalse(p.matcher("file_z.csv").matches());
}
+ @Test
+ void globToRegex_oversizedNumericRangeKeepsRangeSemantics() {
+ java.util.regex.Pattern p = java.util.regex.Pattern.compile(
+ S3FileSystem.globToRegex("date={1..10000000}/*"));
+ Assertions.assertTrue(p.matcher("date=123/file.csv").matches());
+ Assertions.assertTrue(p.matcher("date=10000000/file.csv").matches());
+ Assertions.assertFalse(p.matcher("date=0/file.csv").matches());
+ Assertions.assertFalse(p.matcher("date=10000001/file.csv").matches());
+ }
+
@Test
void globToRegex_keysWithSpaceAreLiteral() {
java.util.regex.Pattern p = java.util.regex.Pattern.compile(
@@ -935,6 +965,68 @@ class S3FileSystemTest {
Assertions.assertFalse(p.matcher("filea.csv").matches());
}
+ @Test
+ void globListWithLimit_preservesOversizedNumericRangeAfterPrefixFallback()
+ throws IOException {
+ Mockito.when(mockStorage.listObjects(
+ ArgumentMatchers.eq("s3://bucket/date="),
+ ArgumentMatchers.any()))
+ .thenReturn(new RemoteObjects(
+ List.of(
+ new RemoteObject("date=123/file.parquet",
+ "file.parquet", null, 10L, 0L),
+ new RemoteObject("date=10000000/file.parquet",
+ "file.parquet", null, 20L, 0L),
+ new RemoteObject("date=0/file.parquet",
+ "file.parquet", null, 30L, 0L),
+ new RemoteObject("date=10000001/file.parquet",
+ "file.parquet", null, 40L, 0L)),
+ false, null));
+
+ GlobListing listing = fs.globListWithLimit(
+ Location.of("s3://bucket/date={1..10000000}/*"), null, 0L, 0L);
+
+ Assertions.assertEquals("date=", listing.getPrefix());
+ Assertions.assertEquals(2, listing.getFiles().size());
+ Assertions.assertEquals("s3://bucket/date=123/file.parquet",
+ listing.getFiles().get(0).location().uri());
+ Assertions.assertEquals("s3://bucket/date=10000000/file.parquet",
+ listing.getFiles().get(1).location().uri());
+ Mockito.verify(mockStorage).listObjects(
+ ArgumentMatchers.eq("s3://bucket/date="),
ArgumentMatchers.any());
+ }
+
+ @Test
+ void globListWithLimit_usesSlashTerminatedPrefixForDirectoryBucket()
+ throws IOException {
+ S3FileSystemProperties properties = S3FileSystemProperties.of(Map.of(
+ S3FileSystemProperties.ENDPOINT,
+ "https://s3express-control.us-west-2.amazonaws.com";));
+ S3FileSystem directoryBucketFs = new S3FileSystem(properties,
mockStorage);
+ Mockito.when(mockStorage.listObjects(
+ ArgumentMatchers.eq("s3://bucket/data/"),
+ ArgumentMatchers.any()))
+ .thenReturn(new RemoteObjects(
+ List.of(
+ new RemoteObject("data/file1.csv",
+ "file1.csv", null, 10L, 0L),
+ new RemoteObject("data/file10.csv",
+ "file10.csv", null, 20L, 0L)),
+ false, null));
+
+ GlobListing listing = directoryBucketFs.globListWithLimit(
+ Location.of("s3://bucket/data/file?.csv"), null, 0L, 0L);
+
+ Assertions.assertEquals("data/", listing.getPrefix());
+ Assertions.assertEquals(1, listing.getFiles().size());
+ Assertions.assertEquals("s3://bucket/data/file1.csv",
+ listing.getFiles().get(0).location().uri());
+ Mockito.verify(mockStorage).listObjects(
+ ArgumentMatchers.eq("s3://bucket/data/"),
ArgumentMatchers.any());
+ Mockito.verify(mockStorage, Mockito.never()).listObjects(
+ ArgumentMatchers.eq("s3://bucket/data/file"),
ArgumentMatchers.any());
+ }
+
// ------------------------------------------------------------------
// containsGlob() — only '*' and '?' are glob metacharacters (#17)
// ------------------------------------------------------------------
diff --git
a/fe/fe-filesystem/fe-filesystem-spi/src/main/java/org/apache/doris/filesystem/spi/ObjectStorageGlob.java
b/fe/fe-filesystem/fe-filesystem-spi/src/main/java/org/apache/doris/filesystem/spi/ObjectStorageGlob.java
new file mode 100644
index 00000000000..4c8bfb7314b
--- /dev/null
+++
b/fe/fe-filesystem/fe-filesystem-spi/src/main/java/org/apache/doris/filesystem/spi/ObjectStorageGlob.java
@@ -0,0 +1,728 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.doris.filesystem.spi;
+
+import java.math.BigInteger;
+import java.nio.charset.StandardCharsets;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.Comparator;
+import java.util.LinkedHashSet;
+import java.util.List;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+
+/**
+ * Provider-neutral glob helpers for object-storage keys.
+ */
+public final class ObjectStorageGlob {
+
+ private static final int MAX_EXPANDED_GLOB_LIST_PREFIXES = 256;
+ private static final Comparator<String> UTF8_BINARY_ORDER =
ObjectStorageGlob::compareUtf8Binary;
+ private static final Pattern NUMERIC_RANGE_PATTERN =
Pattern.compile("-?\\d+\\.\\.-?\\d+");
+ private static final Pattern NUMERIC_RANGE_ALTERNATIVE =
Pattern.compile("(-?\\d+)\\.\\.(-?\\d+)");
+
+ private ObjectStorageGlob() {
+ }
+
+ /**
+ * Returns the longest key prefix that contains no glob metacharacters.
+ */
+ public static String longestNonGlobPrefix(String globPattern) {
+ int earliest = globPattern.length();
+ for (char c : new char[]{'*', '?', '[', '{', '\\'}) {
+ int idx = globPattern.indexOf(c);
+ if (idx >= 0 && idx < earliest) {
+ earliest = idx;
+ }
+ }
+ return globPattern.substring(0, earliest);
+ }
+
+ /**
+ * Returns object-store list prefixes that are safe to push down for a
glob pattern.
+ *
+ * <p>Unlike {@link #longestNonGlobPrefix(String)}, this expands bounded
glob constructs
+ * ({@code {...}} alternation and positive {@code [...]} character
classes) before the first
+ * unbounded wildcard. For example,
+ * {@code date=2025-{0[3-9],1[0-2]}-01/mp_id=8/*} becomes concrete date
prefixes instead
+ * of one broad {@code date=2025-} scan. If expansion would be too large
or unsafe, it
+ * falls back to the conservative longest static prefix.
+ */
+ public static List<String> expandedGlobListPrefixes(String globPattern) {
+ return expandedGlobListPrefixes(globPattern, UTF8_BINARY_ORDER);
+ }
+
+ /** Returns safe list prefixes sorted in the provider's key order. */
+ public static List<String> expandedGlobListPrefixes(String globPattern,
Comparator<String> keyOrder) {
+ List<String> prefixes = expandGlobListPrefixes(globPattern, true);
+ if (prefixes == null) {
+ return List.of(longestNonGlobPrefix(globPattern));
+ }
+ List<String> sorted = new ArrayList<>(prefixes);
+ sorted.sort(keyOrder);
+ return sorted;
+ }
+
+ private static List<String> expandGlobListPrefixes(String globPattern,
boolean allowPartialPrefix) {
+ List<String> prefixes = new ArrayList<>();
+ prefixes.add("");
+ int i = 0;
+ while (i < globPattern.length()) {
+ char c = globPattern.charAt(i);
+ if (c == '*' || c == '?') {
+ return allowPartialPrefix ? compactPrefixes(prefixes) : null;
+ }
+ if (c == '\\') {
+ if (i + 1 < globPattern.length()) {
+ appendLiteral(prefixes, globPattern.charAt(i + 1));
+ i += 2;
+ } else {
+ appendLiteral(prefixes, c);
+ i++;
+ }
+ continue;
+ }
+ if (c == '[') {
+ PrefixExpansion charClass = expandCharacterClass(globPattern,
i);
+ if (charClass == null) {
+ return allowPartialPrefix ? compactPrefixes(prefixes) :
null;
+ }
+ prefixes = appendAlternatives(prefixes, charClass.values);
+ if (prefixes == null) {
+ return null;
+ }
+ i = charClass.nextIndex;
+ continue;
+ }
+ if (c == '{') {
+ PrefixExpansion brace = expandBraceGroup(globPattern, i);
+ if (brace == null) {
+ return allowPartialPrefix ? compactPrefixes(prefixes) :
null;
+ }
+ prefixes = appendAlternatives(prefixes, brace.values);
+ if (prefixes == null) {
+ return null;
+ }
+ i = brace.nextIndex;
+ continue;
+ }
+ appendLiteral(prefixes, c);
+ i++;
+ }
+ // Complete brace arms must retain distinct values until the
surrounding suffix is appended.
+ return allowPartialPrefix ? compactPrefixes(prefixes) : prefixes;
+ }
+
+ private static void appendLiteral(List<String> prefixes, char c) {
+ for (int i = 0; i < prefixes.size(); i++) {
+ prefixes.set(i, prefixes.get(i) + c);
+ }
+ }
+
+ private static List<String> appendAlternatives(List<String> prefixes,
List<String> alternatives) {
+ long expandedSize = (long) prefixes.size() * alternatives.size();
+ if (expandedSize > MAX_EXPANDED_GLOB_LIST_PREFIXES) {
+ return null;
+ }
+ List<String> expanded = new ArrayList<>((int) expandedSize);
+ for (String prefix : prefixes) {
+ for (String alternative : alternatives) {
+ expanded.add(prefix + alternative);
+ }
+ }
+ return expanded;
+ }
+
+ private static PrefixExpansion expandCharacterClass(String globPattern,
int openIndex) {
+ int closeIndex = findClosingBracket(globPattern, openIndex);
+ if (closeIndex < 0 || closeIndex == openIndex + 1) {
+ return null;
+ }
+ int i = openIndex + 1;
+ char first = globPattern.charAt(i);
+ if (first == '!' || first == '^') {
+ return null;
+ }
+ // Escaped characters can be range endpoints in the full matcher. Keep
a conservative
+ // prefix rather than interpreting those endpoints differently during
expansion.
+ if (containsSurrogate(globPattern, i, closeIndex)
+ || globPattern.substring(i, closeIndex).indexOf('\\') >= 0) {
+ return null;
+ }
+ List<String> values = new ArrayList<>();
+ while (i < closeIndex) {
+ char current = globPattern.charAt(i);
+ if (i + 2 < closeIndex && globPattern.charAt(i + 1) == '-') {
+ char rangeEnd = globPattern.charAt(i + 2);
+ int step = current <= rangeEnd ? 1 : -1;
+ for (char ch = current; step > 0 ? ch <= rangeEnd : ch >=
rangeEnd; ch += step) {
+ values.add(String.valueOf(ch));
+ if (values.size() > MAX_EXPANDED_GLOB_LIST_PREFIXES) {
+ return null;
+ }
+ }
+ i += 3;
+ continue;
+ }
+ values.add(String.valueOf(current));
+ i++;
+ }
+ return new PrefixExpansion(values, closeIndex + 1);
+ }
+
+ private static int findClosingBracket(String globPattern, int openIndex) {
+ for (int i = openIndex + 1; i < globPattern.length(); i++) {
+ char c = globPattern.charAt(i);
+ if (c == '\\') {
+ i++;
+ continue;
+ }
+ if (c == ']') {
+ return i;
+ }
+ }
+ return -1;
+ }
+
+ private static boolean containsSurrogate(String text, int start, int end) {
+ for (int i = start; i < end; i++) {
+ if (Character.isSurrogate(text.charAt(i))) {
+ return true;
+ }
+ }
+ return false;
+ }
+
+ private static PrefixExpansion expandBraceGroup(String globPattern, int
openIndex) {
+ int closeIndex = findClosingBrace(globPattern, openIndex);
+ if (closeIndex < 0) {
+ return null;
+ }
+ List<String> alternatives = splitBraceAlternatives(
+ globPattern.substring(openIndex + 1, closeIndex));
+ if (alternatives.isEmpty()) {
+ return null;
+ }
+ List<String> values = new ArrayList<>();
+ for (String alternative : alternatives) {
+ if (containsNumericRange(alternative)) {
+ return null;
+ }
+ List<String> expandedAlternative =
expandGlobListPrefixes(alternative, false);
+ if (expandedAlternative == null) {
+ return null;
+ }
+ values.addAll(expandedAlternative);
+ if (values.size() > MAX_EXPANDED_GLOB_LIST_PREFIXES) {
+ return null;
+ }
+ }
+ return new PrefixExpansion(values, closeIndex + 1);
+ }
+
+ private static int findClosingBrace(String globPattern, int openIndex) {
+ int depth = 0;
+ for (int i = openIndex; i < globPattern.length(); i++) {
+ char c = globPattern.charAt(i);
+ if (c == '\\') {
+ i++;
+ continue;
+ }
+ if (c == '[') {
+ int closeBracket = findClosingBracket(globPattern, i);
+ if (closeBracket >= 0) {
+ i = closeBracket;
+ continue;
+ }
+ }
+ if (c == '{') {
+ depth++;
+ } else if (c == '}') {
+ depth--;
+ if (depth == 0) {
+ return i;
+ }
+ }
+ }
+ return -1;
+ }
+
+ private static List<String> splitBraceAlternatives(String content) {
+ List<String> alternatives = new ArrayList<>();
+ int depth = 0;
+ int start = 0;
+ for (int i = 0; i < content.length(); i++) {
+ char c = content.charAt(i);
+ if (c == '\\') {
+ i++;
+ continue;
+ }
+ if (c == '[') {
+ int closeBracket = findClosingBracket(content, i);
+ if (closeBracket >= 0) {
+ i = closeBracket;
+ continue;
+ }
+ }
+ if (c == '{') {
+ depth++;
+ } else if (c == '}') {
+ if (depth == 0) {
+ return Collections.emptyList();
+ }
+ depth--;
+ } else if (c == ',' && depth == 0) {
+ alternatives.add(content.substring(start, i));
+ start = i + 1;
+ }
+ }
+ if (depth != 0) {
+ return Collections.emptyList();
+ }
+ alternatives.add(content.substring(start));
+ return alternatives;
+ }
+
+ private static List<String> compactPrefixes(List<String> prefixes) {
+ List<String> sorted = new ArrayList<>(prefixes);
+ sorted.sort(UTF8_BINARY_ORDER);
+ List<String> compact = new ArrayList<>();
+ for (String prefix : sorted) {
+ if (prefix.isEmpty()) {
+ return List.of("");
+ }
+ boolean covered = false;
+ for (String existing : compact) {
+ if (prefix.startsWith(existing)) {
+ covered = true;
+ break;
+ }
+ }
+ if (!covered) {
+ compact.add(prefix);
+ }
+ }
+ return compact;
+ }
+
+ /**
+ * Compares object keys by unsigned UTF-8 byte order, matching S3 listing
order.
+ */
+ public static int compareUtf8Binary(String left, String right) {
+ byte[] leftBytes = left.getBytes(StandardCharsets.UTF_8);
+ byte[] rightBytes = right.getBytes(StandardCharsets.UTF_8);
+ int commonLength = Math.min(leftBytes.length, rightBytes.length);
+ for (int i = 0; i < commonLength; i++) {
+ int result = Integer.compare(
+ Byte.toUnsignedInt(leftBytes[i]),
Byte.toUnsignedInt(rightBytes[i]));
+ if (result != 0) {
+ return result;
+ }
+ }
+ return Integer.compare(leftBytes.length, rightBytes.length);
+ }
+
+ /**
+ * Expands bounded {N..M} numeric ranges inside brace groups into
comma-separated alternatives.
+ */
+ public static String expandNumericRanges(String pattern) {
+ StringBuilder expanded = new StringBuilder();
+ for (int i = 0; i < pattern.length(); i++) {
+ char c = pattern.charAt(i);
+ if (c == '\\') {
+ expanded.append(c);
+ if (i + 1 < pattern.length()) {
+ expanded.append(pattern.charAt(++i));
+ }
+ } else if (c == '[') {
+ int closeIndex = findClosingBracket(pattern, i);
+ if (closeIndex >= 0) {
+ expanded.append(pattern, i, closeIndex + 1);
+ i = closeIndex;
+ } else {
+ expanded.append(c);
+ }
+ } else if (c == '{') {
+ int closeIndex = findClosingBrace(pattern, i);
+ if (closeIndex >= 0) {
+ expanded.append('{').append(expandNumericBraceGroup(
+ pattern.substring(i + 1, closeIndex))).append('}');
+ i = closeIndex;
+ } else {
+ expanded.append(c);
+ }
+ } else {
+ expanded.append(c);
+ }
+ }
+ return expanded.toString();
+ }
+
+ private static String expandNumericBraceGroup(String content) {
+ List<String> segments = splitBraceAlternatives(content);
+ boolean isMixed = segments.size() > 1;
+ LinkedHashSet<String> values = new LinkedHashSet<>();
+ for (String segment : segments) {
+ Matcher range = numericRangeMatcher(segment, isMixed);
+ if (range == null) {
+ values.add(expandNumericRanges(segment));
+ } else {
+ long from;
+ long to;
+ try {
+ from = Long.parseLong(range.group(1));
+ to = Long.parseLong(range.group(2));
+ } catch (NumberFormatException e) {
+ return content;
+ }
+ long cardinality = numericRangeCardinality(from, to);
+ if (cardinality < 0 || cardinality >
MAX_EXPANDED_GLOB_LIST_PREFIXES - values.size()) {
+ return content;
+ }
+ long step = from <= to ? 1L : -1L;
+ boolean zeroPad = hasLeadingZero(range.group(1)) ||
hasLeadingZero(range.group(2));
+ int width = Math.max(digitWidth(range.group(1)),
digitWidth(range.group(2)));
+ for (long offset = 0; offset < cardinality; offset++) {
+ long value = step > 0 ? from + offset : from - offset;
+ values.add(formatRangeValue(value, width, zeroPad));
+ }
+ }
+ if (values.size() > MAX_EXPANDED_GLOB_LIST_PREFIXES) {
+ return content;
+ }
+ }
+ return String.join(",", values);
+ }
+
+ private static Matcher numericRangeMatcher(String alternative, boolean
isMixed) {
+ Matcher matcher = NUMERIC_RANGE_ALTERNATIVE.matcher(isMixed ?
alternative.trim() : alternative);
+ if (!matcher.matches() || (!isMixed
+ && (matcher.group(1).startsWith("-") ||
matcher.group(2).startsWith("-")))) {
+ return null;
+ }
+ return matcher;
+ }
+
+ private static boolean containsNumericRange(String text) {
+ return NUMERIC_RANGE_PATTERN.matcher(text).find();
+ }
+
+ private static long numericRangeCardinality(long from, long to) {
+ try {
+ long distance = from <= to
+ ? Math.subtractExact(to, from)
+ : Math.subtractExact(from, to);
+ return Math.addExact(distance, 1L);
+ } catch (ArithmeticException e) {
+ return -1L;
+ }
+ }
+
+ private static boolean hasLeadingZero(String value) {
+ String digits = value.startsWith("-") ? value.substring(1) : value;
+ return digits.length() > 1 && digits.charAt(0) == '0';
+ }
+
+ private static int digitWidth(String value) {
+ return value.startsWith("-") ? value.length() - 1 : value.length();
+ }
+
+ private static String formatRangeValue(long value, int width, boolean
zeroPad) {
+ if (!zeroPad) {
+ return Long.toString(value);
+ }
+ String sign = value < 0 ? "-" : "";
+ String text = Long.toString(value);
+ String digits = value < 0 ? text.substring(1) : text;
+ StringBuilder padded = new StringBuilder(sign);
+ for (int i = digits.length(); i < width; i++) {
+ padded.append('0');
+ }
+ return padded.append(digits).toString();
+ }
+
+ /**
+ * Translates a Java NIO glob pattern to a regex matched against raw
object keys.
+ */
+ public static String globToRegex(String glob) {
+ StringBuilder sb = new StringBuilder("^");
+ appendGlobRegex(sb, glob);
+ sb.append('$');
+ return sb.toString();
+ }
+
+ private static void appendGlobRegex(StringBuilder sb, String glob) {
+ boolean inClass = false;
+ boolean inGroup = false;
+ int i = 0;
+ while (i < glob.length()) {
+ char c = glob.charAt(i);
+ if (c == '\\') {
+ if (i + 1 < glob.length()) {
+ sb.append(Pattern.quote(String.valueOf(glob.charAt(i +
1))));
+ i += 2;
+ } else {
+ sb.append("\\\\");
+ i++;
+ }
+ continue;
+ }
+ if (inClass) {
+ if (c == ']') {
+ inClass = false;
+ sb.append(']');
+ } else if (c == '\\' || c == '[') {
+ sb.append('\\').append(c);
+ } else {
+ sb.append(c);
+ }
+ i++;
+ continue;
+ }
+ switch (c) {
+ case '*':
+ if (i + 1 < glob.length() && glob.charAt(i + 1) == '*') {
+ sb.append(".*");
+ i += 2;
+ } else {
+ sb.append("[^/]*");
+ i++;
+ }
+ break;
+ case '?':
+ sb.append("[^/]");
+ i++;
+ break;
+ case '[':
+ inClass = true;
+ sb.append('[');
+ i++;
+ if (i < glob.length() && glob.charAt(i) == '!') {
+ sb.append('^');
+ i++;
+ }
+ break;
+ case '{':
+ int closeIndex = findClosingBrace(glob, i);
+ if (closeIndex >= 0) {
+ appendBraceGroupRegex(sb, glob.substring(i + 1,
closeIndex));
+ i = closeIndex + 1;
+ break;
+ }
+ inGroup = true;
+ sb.append("(?:");
+ i++;
+ break;
+ case '}':
+ inGroup = false;
+ sb.append(')');
+ i++;
+ break;
+ case ',':
+ if (inGroup) {
+ sb.append('|');
+ } else {
+ sb.append(',');
+ }
+ i++;
+ break;
+ default:
+ if ("\\.^$|+()".indexOf(c) >= 0) {
+ sb.append('\\').append(c);
+ } else {
+ sb.append(c);
+ }
+ i++;
+ break;
+ }
+ }
+ }
+
+ private static void appendBraceGroupRegex(StringBuilder sb, String
content) {
+ List<String> alternatives = splitBraceAlternatives(content);
+ if (alternatives.isEmpty()) {
+ sb.append("(?:)");
+ return;
+ }
+ boolean isMixed = alternatives.size() > 1;
+ sb.append("(?:");
+ for (int i = 0; i < alternatives.size(); i++) {
+ if (i > 0) {
+ sb.append('|');
+ }
+ String rangeRegex = numericRangeRegex(alternatives.get(i),
isMixed);
+ if (rangeRegex != null) {
+ sb.append(rangeRegex);
+ } else {
+ appendGlobRegex(sb, alternatives.get(i));
+ }
+ }
+ sb.append(')');
+ }
+
+ private static String numericRangeRegex(String alternative, boolean
isMixed) {
+ Matcher matcher = numericRangeMatcher(alternative, isMixed);
+ if (matcher == null) {
+ return null;
+ }
+ String fromText = matcher.group(1);
+ String toText = matcher.group(2);
+ BigInteger from = new BigInteger(fromText);
+ BigInteger to = new BigInteger(toText);
+ BigInteger min = from.min(to);
+ BigInteger max = from.max(to);
+ boolean zeroPad = hasLeadingZero(fromText) || hasLeadingZero(toText);
+ int width = Math.max(digitWidth(fromText), digitWidth(toText));
+
+ List<String> alternatives = new ArrayList<>();
+ if (min.signum() < 0) {
+ BigInteger negativeUpper = max.min(BigInteger.ONE.negate());
+ alternatives.add("-" + unsignedRangeRegex(
+ negativeUpper.abs(), min.abs(), width, zeroPad));
+ }
+ if (min.signum() <= 0 && max.signum() >= 0) {
+ alternatives.add(zeroPad ? repeatDigit('0', width) : "0");
+ }
+ if (max.signum() > 0) {
+ alternatives.add(unsignedRangeRegex(
+ min.max(BigInteger.ONE), max, width, zeroPad));
+ }
+ return joinRegexAlternatives(alternatives);
+ }
+
+ private static String unsignedRangeRegex(BigInteger from, BigInteger to,
+ int width, boolean zeroPad) {
+ if (zeroPad) {
+ return sameLengthRangeRegex(leftPadWithZeros(from.toString(),
width),
+ leftPadWithZeros(to.toString(), width));
+ }
+ List<String> alternatives = new ArrayList<>();
+ int minLength = from.toString().length();
+ int maxLength = to.toString().length();
+ for (int length = minLength; length <= maxLength; length++) {
+ BigInteger low = from.max(minValueForLength(length));
+ BigInteger high = to.min(maxValueForLength(length));
+ if (low.compareTo(high) <= 0) {
+ alternatives.add(sameLengthRangeRegex(low.toString(),
high.toString()));
+ }
+ }
+ return joinRegexAlternatives(alternatives);
+ }
+
+ private static BigInteger minValueForLength(int length) {
+ if (length == 1) {
+ return BigInteger.ZERO;
+ }
+ return BigInteger.TEN.pow(length - 1);
+ }
+
+ private static BigInteger maxValueForLength(int length) {
+ return BigInteger.TEN.pow(length).subtract(BigInteger.ONE);
+ }
+
+ private static String sameLengthRangeRegex(String low, String high) {
+ if (low.isEmpty()) {
+ return "";
+ }
+ if (low.equals(high)) {
+ return low;
+ }
+ if (isAll(low, '0') && isAll(high, '9')) {
+ return digitRepeat(low.length());
+ }
+
+ char lowFirst = low.charAt(0);
+ char highFirst = high.charAt(0);
+ String lowRest = low.substring(1);
+ String highRest = high.substring(1);
+ if (lowFirst == highFirst) {
+ return lowFirst + sameLengthRangeRegex(lowRest, highRest);
+ }
+ int restLength = low.length() - 1;
+
+ List<String> alternatives = new ArrayList<>();
+ alternatives.add(lowFirst + sameLengthRangeRegex(
+ lowRest, repeatDigit('9', restLength)));
+ if (lowFirst + 1 <= highFirst - 1) {
+ alternatives.add(digitClass((char) (lowFirst + 1), (char)
(highFirst - 1))
+ + digitRepeat(restLength));
+ }
+ alternatives.add(highFirst + sameLengthRangeRegex(
+ repeatDigit('0', restLength), highRest));
+ return joinRegexAlternatives(alternatives);
+ }
+
+ private static boolean isAll(String value, char expected) {
+ for (int i = 0; i < value.length(); i++) {
+ if (value.charAt(i) != expected) {
+ return false;
+ }
+ }
+ return true;
+ }
+
+ private static String leftPadWithZeros(String value, int width) {
+ if (value.length() >= width) {
+ return value;
+ }
+ return repeatDigit('0', width - value.length()) + value;
+ }
+
+ private static String digitClass(char from, char to) {
+ if (from == to) {
+ return String.valueOf(from);
+ }
+ return "[" + from + "-" + to + "]";
+ }
+
+ private static String digitRepeat(int count) {
+ if (count == 0) {
+ return "";
+ }
+ if (count == 1) {
+ return "\\d";
+ }
+ return "\\d{" + count + "}";
+ }
+
+ private static String repeatDigit(char digit, int count) {
+ StringBuilder sb = new StringBuilder(count);
+ for (int i = 0; i < count; i++) {
+ sb.append(digit);
+ }
+ return sb.toString();
+ }
+
+ private static String joinRegexAlternatives(List<String> alternatives) {
+ if (alternatives.size() == 1) {
+ return alternatives.get(0);
+ }
+ return "(?:" + String.join("|", alternatives) + ")";
+ }
+
+ private static class PrefixExpansion {
+ private final List<String> values;
+ private final int nextIndex;
+
+ private PrefixExpansion(List<String> values, int nextIndex) {
+ this.values = values;
+ this.nextIndex = nextIndex;
+ }
+ }
+}
diff --git
a/fe/fe-filesystem/fe-filesystem-spi/src/main/java/org/apache/doris/filesystem/spi/S3CompatibleFileSystem.java
b/fe/fe-filesystem/fe-filesystem-spi/src/main/java/org/apache/doris/filesystem/spi/S3CompatibleFileSystem.java
index 93fc8c0ba8d..5ea66ca5c63 100644
---
a/fe/fe-filesystem/fe-filesystem-spi/src/main/java/org/apache/doris/filesystem/spi/S3CompatibleFileSystem.java
+++
b/fe/fe-filesystem/fe-filesystem-spi/src/main/java/org/apache/doris/filesystem/spi/S3CompatibleFileSystem.java
@@ -28,12 +28,10 @@ import org.apache.doris.filesystem.Location;
import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStream;
-import java.nio.charset.StandardCharsets;
import java.nio.file.FileAlreadyExistsException;
import java.util.ArrayList;
import java.util.Collection;
import java.util.Collections;
-import java.util.Comparator;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
@@ -56,8 +54,6 @@ public abstract class S3CompatibleFileSystem extends
ObjFileSystem {
// Object stores do not have real directories; use a zero-byte marker with
trailing slash.
private static final String DIR_MARKER_SUFFIX = "/";
- private static final int MAX_EXPANDED_GLOB_LIST_PREFIXES = 256;
- private static final Comparator<String> UTF8_BINARY_ORDER =
S3CompatibleFileSystem::compareUtf8Binary;
private final boolean usePathStyle;
private final Set<String> supportedSchemes;
@@ -847,29 +843,11 @@ public abstract class S3CompatibleFileSystem extends
ObjFileSystem {
* ({@code * ? [ { \}). Used as the {@code prefix} parameter for object
storage {@code ListObjectsV2}.
*/
protected static String longestNonGlobPrefix(String globPattern) {
- int earliest = globPattern.length();
- for (char c : new char[]{'*', '?', '[', '{', '\\'}) {
- int idx = globPattern.indexOf(c);
- if (idx >= 0 && idx < earliest) {
- earliest = idx;
- }
- }
- return globPattern.substring(0, earliest);
+ return ObjectStorageGlob.longestNonGlobPrefix(globPattern);
}
- /**
- * Returns object-store list prefixes that are safe to push down for a
glob pattern.
- *
- * <p>Unlike {@link #longestNonGlobPrefix(String)}, this expands bounded
glob constructs
- * ({@code {...}} alternation and positive {@code [...]} character
classes) before the first
- * unbounded wildcard. That lets patterns such as
- * {@code date=2025-{0[3-9],1[0-2]}-01/mp_id=8/*} list the concrete
date/mp prefixes instead
- * of scanning everything under {@code date=2025-}. If expansion would be
too large or a glob
- * construct is not safely enumerable, it falls back to the conservative
longest static prefix.
- */
protected static List<String> expandedGlobListPrefixes(String globPattern)
{
- List<String> prefixes = expandGlobListPrefixes(globPattern, true);
- return prefixes == null ? List.of(longestNonGlobPrefix(globPattern)) :
prefixes;
+ return ObjectStorageGlob.expandedGlobListPrefixes(globPattern);
}
protected String globListPrefix(String globPattern) {
@@ -880,264 +858,9 @@ public abstract class S3CompatibleFileSystem extends
ObjFileSystem {
return expandedGlobListPrefixes(globPattern);
}
- private static List<String> expandGlobListPrefixes(String globPattern,
boolean allowPartialPrefix) {
- List<String> prefixes = new ArrayList<>();
- prefixes.add("");
- int i = 0;
- while (i < globPattern.length()) {
- char c = globPattern.charAt(i);
- if (c == '*' || c == '?') {
- return allowPartialPrefix ? compactPrefixes(prefixes) : null;
- }
- if (c == '\\') {
- if (i + 1 < globPattern.length()) {
- appendLiteral(prefixes, globPattern.charAt(i + 1));
- i += 2;
- } else {
- appendLiteral(prefixes, c);
- i++;
- }
- continue;
- }
- if (c == '[') {
- PrefixExpansion charClass = expandCharacterClass(globPattern,
i);
- if (charClass == null) {
- return allowPartialPrefix ? compactPrefixes(prefixes) :
null;
- }
- prefixes = appendAlternatives(prefixes, charClass.values);
- if (prefixes == null) {
- return null;
- }
- i = charClass.nextIndex;
- continue;
- }
- if (c == '{') {
- PrefixExpansion brace = expandBraceGroup(globPattern, i);
- if (brace == null) {
- return allowPartialPrefix ? compactPrefixes(prefixes) :
null;
- }
- prefixes = appendAlternatives(prefixes, brace.values);
- if (prefixes == null) {
- return null;
- }
- i = brace.nextIndex;
- continue;
- }
- appendLiteral(prefixes, c);
- i++;
- }
- // A complete brace arm is not yet a listing prefix: a later suffix
can distinguish
- // "1.csv" from "101.csv". Compact only after the surrounding
expression is expanded.
- return allowPartialPrefix ? compactPrefixes(prefixes) : prefixes;
- }
-
- private static void appendLiteral(List<String> prefixes, char c) {
- for (int i = 0; i < prefixes.size(); i++) {
- prefixes.set(i, prefixes.get(i) + c);
- }
- }
-
- private static List<String> appendAlternatives(List<String> prefixes,
List<String> alternatives) {
- long expandedSize = (long) prefixes.size() * alternatives.size();
- if (expandedSize > MAX_EXPANDED_GLOB_LIST_PREFIXES) {
- return null;
- }
- List<String> expanded = new ArrayList<>((int) expandedSize);
- for (String prefix : prefixes) {
- for (String alternative : alternatives) {
- expanded.add(prefix + alternative);
- }
- }
- return expanded;
- }
-
- private static PrefixExpansion expandCharacterClass(String globPattern,
int openIndex) {
- int closeIndex = findClosingBracket(globPattern, openIndex);
- if (closeIndex < 0 || closeIndex == openIndex + 1) {
- return null;
- }
- int i = openIndex + 1;
- char first = globPattern.charAt(i);
- if (first == '!' || first == '^') {
- return null;
- }
- if (containsSurrogate(globPattern, i, closeIndex)) {
- return null;
- }
- List<String> values = new ArrayList<>();
- while (i < closeIndex) {
- char current = globPattern.charAt(i);
- if (current == '\\') {
- if (i + 1 >= closeIndex) {
- return null;
- }
- values.add(String.valueOf(globPattern.charAt(i + 1)));
- i += 2;
- continue;
- }
- if (i + 2 < closeIndex && globPattern.charAt(i + 1) == '-') {
- char rangeEnd = globPattern.charAt(i + 2);
- int step = current <= rangeEnd ? 1 : -1;
- for (char ch = current; step > 0 ? ch <= rangeEnd : ch >=
rangeEnd; ch += step) {
- values.add(String.valueOf(ch));
- if (values.size() > MAX_EXPANDED_GLOB_LIST_PREFIXES) {
- return null;
- }
- }
- i += 3;
- continue;
- }
- values.add(String.valueOf(current));
- i++;
- }
- return new PrefixExpansion(values, closeIndex + 1);
- }
-
- private static int findClosingBracket(String globPattern, int openIndex) {
- for (int i = openIndex + 1; i < globPattern.length(); i++) {
- char c = globPattern.charAt(i);
- if (c == '\\') {
- i++;
- continue;
- }
- if (c == ']') {
- return i;
- }
- }
- return -1;
- }
-
- private static boolean containsSurrogate(String text, int start, int end) {
- for (int i = start; i < end; i++) {
- if (Character.isSurrogate(text.charAt(i))) {
- return true;
- }
- }
- return false;
- }
-
- private static PrefixExpansion expandBraceGroup(String globPattern, int
openIndex) {
- int closeIndex = findClosingBrace(globPattern, openIndex);
- if (closeIndex < 0) {
- return null;
- }
- List<String> alternatives = splitBraceAlternatives(
- globPattern.substring(openIndex + 1, closeIndex));
- if (alternatives.isEmpty()) {
- return null;
- }
- List<String> values = new ArrayList<>();
- for (String alternative : alternatives) {
- List<String> expandedAlternative =
expandGlobListPrefixes(alternative, false);
- if (expandedAlternative == null) {
- return null;
- }
- values.addAll(expandedAlternative);
- if (values.size() > MAX_EXPANDED_GLOB_LIST_PREFIXES) {
- return null;
- }
- }
- return new PrefixExpansion(values, closeIndex + 1);
- }
-
- private static int findClosingBrace(String globPattern, int openIndex) {
- int depth = 0;
- for (int i = openIndex; i < globPattern.length(); i++) {
- char c = globPattern.charAt(i);
- if (c == '\\') {
- i++;
- continue;
- }
- if (c == '{') {
- depth++;
- } else if (c == '}') {
- depth--;
- if (depth == 0) {
- return i;
- }
- }
- }
- return -1;
- }
-
- private static List<String> splitBraceAlternatives(String content) {
- List<String> alternatives = new ArrayList<>();
- int depth = 0;
- int start = 0;
- for (int i = 0; i < content.length(); i++) {
- char c = content.charAt(i);
- if (c == '\\') {
- i++;
- continue;
- }
- if (c == '{') {
- depth++;
- } else if (c == '}') {
- if (depth == 0) {
- return Collections.emptyList();
- }
- depth--;
- } else if (c == ',' && depth == 0) {
- alternatives.add(content.substring(start, i));
- start = i + 1;
- }
- }
- if (depth != 0) {
- return Collections.emptyList();
- }
- alternatives.add(content.substring(start));
- return alternatives;
- }
-
- private static List<String> compactPrefixes(List<String> prefixes) {
- List<String> sorted = new ArrayList<>(prefixes);
- sorted.sort(UTF8_BINARY_ORDER);
- List<String> compact = new ArrayList<>();
- for (String prefix : sorted) {
- if (prefix.isEmpty()) {
- return List.of("");
- }
- boolean covered = false;
- for (String existing : compact) {
- if (prefix.startsWith(existing)) {
- covered = true;
- break;
- }
- }
- if (!covered) {
- compact.add(prefix);
- }
- }
- return compact;
- }
-
- private static int compareUtf8Binary(String left, String right) {
- byte[] leftBytes = left.getBytes(StandardCharsets.UTF_8);
- byte[] rightBytes = right.getBytes(StandardCharsets.UTF_8);
- int commonLength = Math.min(leftBytes.length, rightBytes.length);
- for (int i = 0; i < commonLength; i++) {
- int result = Integer.compare(
- Byte.toUnsignedInt(leftBytes[i]),
Byte.toUnsignedInt(rightBytes[i]));
- if (result != 0) {
- return result;
- }
- }
- return Integer.compare(leftBytes.length, rightBytes.length);
- }
-
- private static class PrefixExpansion {
- private final List<String> values;
- private final int nextIndex;
-
- private PrefixExpansion(List<String> values, int nextIndex) {
- this.values = values;
- this.nextIndex = nextIndex;
- }
- }
-
/** Compiles the same numeric-range and glob syntax used by LIST for
matching raw object keys. */
public static Pattern compileGlobPattern(String keyPattern) {
- return
Pattern.compile(globToRegex(GlobPattern.expandNumericRanges(keyPattern)));
+ return
Pattern.compile(globToRegex(ObjectStorageGlob.expandNumericRanges(keyPattern)));
}
@Override
@@ -1150,7 +873,7 @@ public abstract class S3CompatibleFileSystem extends
ObjFileSystem {
String keyPattern = parsed.key();
String base = uriBase(uri, parsed);
- String expandedKeyPattern =
GlobPattern.expandNumericRanges(keyPattern);
+ String expandedKeyPattern =
ObjectStorageGlob.expandNumericRanges(keyPattern);
// Translate the glob to a regex and match against the raw object
storage key. We do NOT
// route through java.nio.file.Paths because (a) on Windows the
filesystem
// separator is '\' which would corrupt object storage keys, and (b)
Paths.get rejects keys
@@ -1244,87 +967,6 @@ public abstract class S3CompatibleFileSystem extends
ObjFileSystem {
* a single {@code *} does not cross directory levels.
*/
protected static String globToRegex(String glob) {
- StringBuilder sb = new StringBuilder("^");
- boolean inClass = false;
- int groupDepth = 0;
- int i = 0;
- while (i < glob.length()) {
- char c = glob.charAt(i);
- if (c == '\\') {
- if (i + 1 < glob.length()) {
- sb.append(Pattern.quote(String.valueOf(glob.charAt(i +
1))));
- i += 2;
- } else {
- sb.append("\\\\");
- i++;
- }
- continue;
- }
- if (inClass) {
- if (c == ']') {
- inClass = false;
- sb.append(']');
- } else if (c == '\\' || c == '[') {
- sb.append('\\').append(c);
- } else {
- sb.append(c);
- }
- i++;
- continue;
- }
- switch (c) {
- case '*':
- if (i + 1 < glob.length() && glob.charAt(i + 1) == '*') {
- sb.append(".*");
- i += 2;
- } else {
- sb.append("[^/]*");
- i++;
- }
- break;
- case '?':
- sb.append("[^/]");
- i++;
- break;
- case '[':
- inClass = true;
- sb.append('[');
- i++;
- if (i < glob.length() && glob.charAt(i) == '!') {
- sb.append('^');
- i++;
- }
- break;
- case '{':
- groupDepth++;
- sb.append("(?:");
- i++;
- break;
- case '}':
- groupDepth--;
- sb.append(')');
- i++;
- break;
- case ',':
- // Closing an inner group must not turn the outer group's
alternatives into literals.
- if (groupDepth > 0) {
- sb.append('|');
- } else {
- sb.append(',');
- }
- i++;
- break;
- default:
- if ("\\.^$|+()".indexOf(c) >= 0) {
- sb.append('\\').append(c);
- } else {
- sb.append(c);
- }
- i++;
- break;
- }
- }
- sb.append('$');
- return sb.toString();
+ return ObjectStorageGlob.globToRegex(glob);
}
}
diff --git
a/fe/fe-filesystem/fe-filesystem-spi/src/test/java/org/apache/doris/filesystem/spi/ObjectStorageGlobTest.java
b/fe/fe-filesystem/fe-filesystem-spi/src/test/java/org/apache/doris/filesystem/spi/ObjectStorageGlobTest.java
new file mode 100644
index 00000000000..6a11a9c1aaf
--- /dev/null
+++
b/fe/fe-filesystem/fe-filesystem-spi/src/test/java/org/apache/doris/filesystem/spi/ObjectStorageGlobTest.java
@@ -0,0 +1,196 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.doris.filesystem.spi;
+
+import org.junit.jupiter.api.Assertions;
+import org.junit.jupiter.api.Test;
+
+import java.util.Comparator;
+import java.util.List;
+import java.util.regex.Pattern;
+
+class ObjectStorageGlobTest {
+
+ @Test
+ void expandNumericRanges_preservesZeroPadding() {
+ Assertions.assertEquals("date=2025-{01,02,03}-01/",
+
ObjectStorageGlob.expandNumericRanges("date=2025-{01..03}-01/"));
+ }
+
+ @Test
+ void expandNumericRanges_preservesPaddedMinimumLong() {
+ for (String range :
List.of("-09223372036854775808..-09223372036854775807",
+ "-09223372036854775807..-09223372036854775808")) {
+ String glob = "data_{" + range + ",x}.csv";
+ String expanded = ObjectStorageGlob.expandNumericRanges(glob);
+ Pattern matcher =
Pattern.compile(ObjectStorageGlob.globToRegex(expanded));
+ Pattern fallback =
Pattern.compile(ObjectStorageGlob.globToRegex(glob));
+ for (String key : List.of("data_-09223372036854775808.csv",
"data_-09223372036854775807.csv",
+ "data_x.csv")) {
+ Assertions.assertTrue(matcher.matcher(key).matches(), key);
+ Assertions.assertTrue(fallback.matcher(key).matches(), key);
+
Assertions.assertTrue(ObjectStorageGlob.expandedGlobListPrefixes(expanded).contains(key),
key);
+ }
+
Assertions.assertFalse(matcher.matcher("data_-9223372036854775808.csv").matches());
+
Assertions.assertFalse(matcher.matcher("data_-09223372036854775806.csv").matches());
+ }
+ }
+
+ @Test
+ void expandedGlobListPrefixes_fallsBackForEscapedCharacterClassRanges() {
+ for (String characterClass : List.of("[\\a-z]", "[a-\\c]",
"[\\a-\\c]")) {
+ String glob = "data/" + characterClass + "/*/file.csv";
+ Assertions.assertEquals(List.of("data/"),
ObjectStorageGlob.expandedGlobListPrefixes(glob));
+
Assertions.assertTrue(Pattern.matches(ObjectStorageGlob.globToRegex(glob),
"data/b/sub/file.csv"));
+ Assertions.assertEquals(List.of("data/"),
+ ObjectStorageGlob.expandedGlobListPrefixes("data/{fixed,"
+ characterClass + "}/*"));
+ }
+ }
+
+ @Test
+ void expandedGlobListPrefixes_usesProviderOrder() {
+ String fullwidth = Character.toString(0xFF21);
+ String emoji = Character.toString(0x1F600);
+ String glob = "data/{" + fullwidth + "," + emoji + "}/file.csv";
+ Assertions.assertEquals(List.of("data/" + fullwidth + "/file.csv",
"data/" + emoji + "/file.csv"),
+ ObjectStorageGlob.expandedGlobListPrefixes(glob));
+ Assertions.assertEquals(List.of("data/" + emoji + "/file.csv", "data/"
+ fullwidth + "/file.csv"),
+ ObjectStorageGlob.expandedGlobListPrefixes(glob,
Comparator.naturalOrder()));
+ Assertions.assertEquals(List.of(""),
+ ObjectStorageGlob.expandedGlobListPrefixes("*",
Comparator.naturalOrder()));
+ }
+
+ @Test
+ void expandNumericRanges_skipsOversizedRanges() {
+ String pattern = "date={1..10000000}/*";
+
+ Assertions.assertEquals(pattern,
ObjectStorageGlob.expandNumericRanges(pattern));
+ }
+
+ @Test
+ void expandNumericRanges_skipsMixedOversizedRanges() {
+ String pattern = "date={x,0..9223372036854775807}/*";
+
+ Assertions.assertEquals(pattern,
ObjectStorageGlob.expandNumericRanges(pattern));
+ }
+
+ @Test
+ void expandNumericRanges_preservesLiteralSpacesInMixedBraceArms() {
+ Assertions.assertEquals("data/{ x,1,2}/*.csv",
+ ObjectStorageGlob.expandNumericRanges("data/{ x,1..2}/*.csv"));
+ }
+
+ @Test
+ void expandedGlobListPrefixes_fallsBackForUnexpandedNumericRanges() {
+ Assertions.assertEquals(List.of("date="),
+
ObjectStorageGlob.expandedGlobListPrefixes("date={1..10000000}/*"));
+ }
+
+ @Test
+ void compareUtf8Binary_matchesObjectStoreOrderForNonBmpKeys() {
+ String privateUseKey = "data/" + Character.toString(0xE000) +
"/file.csv";
+ String emojiKey = "data/" + Character.toString(0x1F600) + "/file.csv";
+
+ Assertions.assertTrue(privateUseKey.compareTo(emojiKey) > 0);
+
Assertions.assertTrue(ObjectStorageGlob.compareUtf8Binary(privateUseKey,
emojiKey) < 0);
+ }
+
+ @Test
+ void globToRegex_preservesOversizedNumericRangeSemantics() {
+ Pattern matcher =
Pattern.compile(ObjectStorageGlob.globToRegex("date={1..10000000}/*"));
+
+ Assertions.assertTrue(matcher.matcher("date=123/file.csv").matches());
+
Assertions.assertTrue(matcher.matcher("date=10000000/file.csv").matches());
+ Assertions.assertFalse(matcher.matcher("date=0/file.csv").matches());
+
Assertions.assertFalse(matcher.matcher("date=10000001/file.csv").matches());
+
Assertions.assertFalse(matcher.matcher("date=0001/file.csv").matches());
+ }
+
+ @Test
+ void globToRegex_preservesLiteralSpacesInMixedBraceArms() {
+ Pattern matcher = Pattern.compile(ObjectStorageGlob.globToRegex(
+ ObjectStorageGlob.expandNumericRanges("data/{
x,1..2}/*.csv")));
+
+ Assertions.assertTrue(matcher.matcher("data/ x/file.csv").matches());
+ Assertions.assertTrue(matcher.matcher("data/1/file.csv").matches());
+ Assertions.assertFalse(matcher.matcher("data/x/file.csv").matches());
+ }
+
+ @Test
+ void globToRegex_preservesMixedOversizedNumericRangeSemantics() {
+ Pattern matcher = Pattern.compile(ObjectStorageGlob.globToRegex(
+ "date={x,0..9223372036854775807}/*"));
+
+ Assertions.assertTrue(matcher.matcher("date=x/file.csv").matches());
+
Assertions.assertTrue(matcher.matcher("date=9223372036854775807/file.csv").matches());
+ Assertions.assertFalse(matcher.matcher("date=-1/file.csv").matches());
+
Assertions.assertFalse(matcher.matcher("date=9223372036854775808/file.csv").matches());
+ }
+
+ @Test
+ void globToRegex_preservesOversizedZeroPaddedNumericRangeSemantics() {
+ Pattern matcher =
Pattern.compile(ObjectStorageGlob.globToRegex("date={0001..1000}/*"));
+
+ Assertions.assertTrue(matcher.matcher("date=0001/file.csv").matches());
+ Assertions.assertTrue(matcher.matcher("date=0999/file.csv").matches());
+ Assertions.assertTrue(matcher.matcher("date=1000/file.csv").matches());
+ Assertions.assertFalse(matcher.matcher("date=999/file.csv").matches());
+
Assertions.assertFalse(matcher.matcher("date=1001/file.csv").matches());
+ }
+
+ @Test
+ void expandNumericRanges_preservesEscapesAndCharacterClasses() {
+ Assertions.assertEquals("data_\\{1..3\\}.csv",
+ ObjectStorageGlob.expandNumericRanges("data_\\{1..3\\}.csv"));
+ Assertions.assertEquals("[{1..3}]/{1,2}",
+ ObjectStorageGlob.expandNumericRanges("[{1..3}]/{1..2}"));
+ Assertions.assertEquals("{1,2,a\\,b,\\{literal\\},[{},]}",
+
ObjectStorageGlob.expandNumericRanges("{1..2,a\\,b,\\{literal\\},[{},]}"));
+ Pattern matcher = Pattern.compile(ObjectStorageGlob.globToRegex(
+
ObjectStorageGlob.expandNumericRanges("data_{1..2,a\\,b,\\{literal\\},[{},]}.csv")));
+ for (String name : List.of("1", "2", "a,b", "{literal}", "{", "}",
",")) {
+ Assertions.assertTrue(matcher.matcher("data_" + name +
".csv").matches(), name);
+ }
+ Assertions.assertFalse(matcher.matcher("data_3.csv").matches());
+ }
+
+ @Test
+ void expandedGlobListPrefixes_preservesBraceArmsUntilSuffixIsAppended() {
+ Assertions.assertEquals(List.of("data_1.csv", "data_101.csv"),
+
ObjectStorageGlob.expandedGlobListPrefixes("data_{1,101}.csv"));
+ Assertions.assertEquals(List.of("a101suffix", "a1suffix", "bsuffix"),
+
ObjectStorageGlob.expandedGlobListPrefixes("{a{1,101},b}suffix"));
+ Assertions.assertEquals(List.of("asuffix", "suffix"),
+ ObjectStorageGlob.expandedGlobListPrefixes("{,a}suffix"));
+ }
+
+ @Test
+ void globToRegex_preservesNestedRangeAlternatives() {
+ String glob = "data_{a{01..02},b{3..10000000},plain}.csv";
+ String expanded = ObjectStorageGlob.expandNumericRanges(glob);
+ Assertions.assertEquals(List.of("data_"),
ObjectStorageGlob.expandedGlobListPrefixes(expanded));
+ Pattern matcher =
Pattern.compile(ObjectStorageGlob.globToRegex(expanded));
+ for (String name : List.of("a01", "a02", "b3", "b10000000", "plain")) {
+ Assertions.assertTrue(matcher.matcher("data_" + name +
".csv").matches(), name);
+ }
+ for (String name : List.of("a1", "a03", "b2", "b10000001",
"a01,b3,plain")) {
+ Assertions.assertFalse(matcher.matcher("data_" + name +
".csv").matches(), name);
+ }
+ }
+
+}
diff --git
a/fe/fe-filesystem/fe-filesystem-spi/src/test/java/org/apache/doris/filesystem/spi/S3CompatibleFileSystemTest.java
b/fe/fe-filesystem/fe-filesystem-spi/src/test/java/org/apache/doris/filesystem/spi/S3CompatibleFileSystemTest.java
index 710a6eb7a31..c6c1c5f7739 100644
---
a/fe/fe-filesystem/fe-filesystem-spi/src/test/java/org/apache/doris/filesystem/spi/S3CompatibleFileSystemTest.java
+++
b/fe/fe-filesystem/fe-filesystem-spi/src/test/java/org/apache/doris/filesystem/spi/S3CompatibleFileSystemTest.java
@@ -42,11 +42,40 @@ import java.util.regex.Pattern;
class S3CompatibleFileSystemTest {
@Test
- void numericRangesRejectExcessiveExpansion() {
- Assertions.assertThrows(IllegalArgumentException.class,
- () ->
S3CompatibleFileSystem.compileGlobPattern("data_{1..10001}.csv"));
- Assertions.assertThrows(IllegalArgumentException.class,
- () ->
S3CompatibleFileSystem.compileGlobPattern("{1..101}/data_{1..101}.csv"));
+ void numericRangesPreservePaddedMinimumLong() {
+ Pattern matcher = S3CompatibleFileSystem.compileGlobPattern(
+ "data_{-09223372036854775808..-09223372036854775807,x}.csv");
+
Assertions.assertTrue(matcher.matcher("data_-09223372036854775808.csv").matches());
+
Assertions.assertTrue(matcher.matcher("data_-09223372036854775807.csv").matches());
+
Assertions.assertFalse(matcher.matcher("data_-9223372036854775808.csv").matches());
+ }
+
+ @Test
+ void escapedRangeEndpointsUseConservativeListPrefix() throws IOException {
+ for (String characterClass : List.of("[\\a-z]", "[a-\\c]")) {
+ PagingObjStorage storage = new PagingObjStorage(List.of(new
RemoteObjects(
+ List.of(remoteObject("data/b/sub/file.csv")), false,
null))) {
+ @Override
+ public RemoteObjects listObjects(String remotePath, String
continuationToken) throws IOException {
+ Assertions.assertEquals("s3://bucket/data/", remotePath);
+ return super.listObjects(remotePath, continuationToken);
+ }
+ };
+ GlobListing listing = new
TestFileSystem(storage).globListWithLimit(
+ Location.of("s3://bucket/data/" + characterClass +
"/*/file.csv"), null, 0L, 0L);
+ Assertions.assertEquals(1, listing.getFiles().size());
+ Assertions.assertEquals("s3://bucket/data/b/sub/file.csv",
listing.getFiles().get(0).location().uri());
+ }
+ }
+
+ @Test
+ void numericRangesMatchWithoutExcessiveExpansion() {
+ Pattern largeRange =
S3CompatibleFileSystem.compileGlobPattern("data_{1..10001}.csv");
+ Assertions.assertTrue(largeRange.matcher("data_10001.csv").matches());
+ Assertions.assertFalse(largeRange.matcher("data_10002.csv").matches());
+ Pattern product =
S3CompatibleFileSystem.compileGlobPattern("{1..101}/data_{1..101}.csv");
+ Assertions.assertTrue(product.matcher("101/data_101.csv").matches());
+ Assertions.assertFalse(product.matcher("102/data_101.csv").matches());
}
@Test
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
