This is an automated email from the ASF dual-hosted git repository.
JingsongLi pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/paimon.git
The following commit(s) were added to refs/heads/master by this push:
new fdfd718729 [core] Narrow format table partition listing by name
pattern (#9025)
fdfd718729 is described below
commit fdfd71872923aee6ea8a67d9c014bc568140b33d
Author: Dapeng Sun(孙大鹏) <[email protected]>
AuthorDate: Tue Aug 4 18:51:53 2026 +0800
[core] Narrow format table partition listing by name pattern (#9025)
---
.../format/CatalogFormatTablePartitionManager.java | 3 +-
.../paimon/table/format/PartitionNamePatterns.java | 249 +++++++++++++++++++++
.../CatalogFormatTablePartitionManagerTest.java | 110 +++++++++
.../table/format/PartitionNamePatternsTest.java | 220 ++++++++++++++++++
4 files changed, 580 insertions(+), 2 deletions(-)
diff --git
a/paimon-core/src/main/java/org/apache/paimon/table/format/CatalogFormatTablePartitionManager.java
b/paimon-core/src/main/java/org/apache/paimon/table/format/CatalogFormatTablePartitionManager.java
index ed266db2a5..4d1009f196 100644
---
a/paimon-core/src/main/java/org/apache/paimon/table/format/CatalogFormatTablePartitionManager.java
+++
b/paimon-core/src/main/java/org/apache/paimon/table/format/CatalogFormatTablePartitionManager.java
@@ -25,7 +25,6 @@ import org.apache.paimon.catalog.Identifier;
import org.apache.paimon.partition.Partition;
import org.apache.paimon.predicate.Predicate;
import org.apache.paimon.utils.FunctionWithException;
-import org.apache.paimon.utils.PartitionPathUtils;
import org.apache.paimon.utils.StringUtils;
import javax.annotation.Nullable;
@@ -73,7 +72,7 @@ class CatalogFormatTablePartitionManager implements
FormatTablePartitionManager
prefix,
partitionKeys,
identifier.getFullName());
- String pattern =
PartitionPathUtils.buildPartitionNamePrefixPattern(partitionKeys, ordered);
+ String pattern = PartitionNamePatterns.build(partitionKeys, ordered,
filter);
return execute(
catalog -> {
if (filter != null) {
diff --git
a/paimon-core/src/main/java/org/apache/paimon/table/format/PartitionNamePatterns.java
b/paimon-core/src/main/java/org/apache/paimon/table/format/PartitionNamePatterns.java
new file mode 100644
index 0000000000..7a9eee3549
--- /dev/null
+++
b/paimon-core/src/main/java/org/apache/paimon/table/format/PartitionNamePatterns.java
@@ -0,0 +1,249 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.paimon.table.format;
+
+import org.apache.paimon.casting.CastExecutor;
+import org.apache.paimon.casting.CastExecutors;
+import org.apache.paimon.data.BinaryString;
+import org.apache.paimon.predicate.Between;
+import org.apache.paimon.predicate.Equal;
+import org.apache.paimon.predicate.FieldRef;
+import org.apache.paimon.predicate.GreaterOrEqual;
+import org.apache.paimon.predicate.GreaterThan;
+import org.apache.paimon.predicate.LeafFunction;
+import org.apache.paimon.predicate.LeafPredicate;
+import org.apache.paimon.predicate.LessOrEqual;
+import org.apache.paimon.predicate.LessThan;
+import org.apache.paimon.predicate.Predicate;
+import org.apache.paimon.predicate.PredicateBuilder;
+import org.apache.paimon.predicate.StartsWith;
+import org.apache.paimon.predicate.SubstringTransform;
+import org.apache.paimon.predicate.Transform;
+import org.apache.paimon.types.DataType;
+import org.apache.paimon.types.DataTypeFamily;
+import org.apache.paimon.types.VarCharType;
+import org.apache.paimon.utils.PartitionPathUtils;
+
+import javax.annotation.Nullable;
+
+import java.util.LinkedHashMap;
+import java.util.List;
+
+/**
+ * Builds the {@code partitionNamePattern} sent to the catalog when a format
table lists its
+ * partitions.
+ *
+ * <p>Without a pattern the client drains the whole registry one page at a
time, which costs one
+ * round trip per page no matter how few partitions the query needs. A pattern
lets the catalog
+ * filter by name prefix and can reduce the number of pages returned.
+ *
+ * <p>{@link PartitionPathUtils#buildPartitionNamePrefixPattern} already
covers the case where
+ * leading partition keys are pinned by equality. This class extends the
pattern by one more key
+ * using predicates that constrain only a <em>prefix of that key's value</em>.
+ *
+ * <h2>Why a too-wide pattern is safe</h2>
+ *
+ * <p>The pattern only narrows the candidate set fetched from the catalog.
Every candidate is still
+ * checked against the full partition predicate before its files are listed
(see {@code
+ * CatalogSplitEnumerator}). So a pattern that matches extra partitions merely
wastes a little
+ * listing; a pattern that misses a matching partition would silently drop
data. Every rule below is
+ * therefore chosen so that <b>every partition satisfying the predicate
matches the pattern</b>.
+ */
+final class PartitionNamePatterns {
+
+ private PartitionNamePatterns() {}
+
+ /**
+ * Returns the pattern for a listing constrained by {@code equalityPrefix}
and {@code filter},
+ * or null when no useful prefix can be derived.
+ */
+ @Nullable
+ static String build(
+ List<String> partitionKeys,
+ LinkedHashMap<String, String> equalityPrefix,
+ @Nullable Predicate filter) {
+ String base =
+
PartitionPathUtils.buildPartitionNamePrefixPattern(partitionKeys,
equalityPrefix);
+ if (filter == null || equalityPrefix.size() >= partitionKeys.size()) {
+ // Either there is nothing left to narrow, or equality already
pinned every key.
+ return base;
+ }
+ if (base == null && !equalityPrefix.isEmpty()) {
+ return null;
+ }
+
+ String nextKey = partitionKeys.get(equalityPrefix.size());
+ String valuePrefix = leadingValuePrefix(filter, nextKey);
+ if (valuePrefix == null || valuePrefix.isEmpty()) {
+ return base;
+ }
+
+ LinkedHashMap<String, String> extended = new
LinkedHashMap<>(equalityPrefix);
+ extended.put(nextKey, valuePrefix);
+ // Escaping is per character, so escaping a value prefix yields a
prefix of the escaped
+ // value. generatePartitionPath appends a trailing separator that a
partial value must not
+ // carry.
+ String path = PartitionPathUtils.generatePartitionPath(extended);
+ String escaped = path.substring(0, path.length() - 1);
+ if (escaped.indexOf('%') >= 0) {
+ // A percent-escaped character would read as a LIKE wildcard.
Widening is safe but
+ // pointless, so fall back rather than send a pattern the server
cannot use as an index
+ // prefix.
+ return base;
+ }
+ return escaped + '%';
+ }
+
+ /**
+ * Derives a prefix of {@code key}'s value that every partition satisfying
{@code filter} must
+ * start with, or null if the predicates do not constrain the value's
prefix.
+ */
+ @Nullable
+ private static String leadingValuePrefix(Predicate filter, String key) {
+ String substringPrefix = null;
+ String startsWithPrefix = null;
+ String lower = null;
+ String upper = null;
+
+ for (Predicate conjunct : PredicateBuilder.splitAnd(filter)) {
+ if (!(conjunct instanceof LeafPredicate)) {
+ continue;
+ }
+ LeafPredicate leaf = (LeafPredicate) conjunct;
+ LeafFunction function = leaf.function();
+ List<Object> literals = leaf.literals();
+ if (literals.isEmpty() || literals.get(0) == null) {
+ continue;
+ }
+
+ if (function instanceof Equal) {
+ String prefix = substringPrefixOf(leaf, key);
+ if (prefix != null) {
+ substringPrefix = prefix;
+ }
+ }
+
+ FieldRef ref = leaf.fieldRefOptional().orElse(null);
+ if (ref == null || !key.equals(ref.name()) ||
!isStringType(ref.type())) {
+ // Prefix reasoning below compares values lexicographically.
That only matches the
+ // predicate's own ordering for string types: on a numeric
key, `k >= 9 AND k <= 99`
+ // holds for 10, whose name does not start with the common
prefix "9".
+ continue;
+ }
+ String literal = literalToString(ref.type(), literals.get(0));
+ if (literal == null) {
+ continue;
+ }
+ if (function instanceof StartsWith) {
+ startsWithPrefix = literal;
+ } else if (function instanceof Between) {
+ // `k >= lo AND k <= hi` reaches us already folded into one
Between leaf, which is
+ // the shape real query plans carry; the split form below is
the fallback.
+ if (literals.size() == 2 && literals.get(1) != null) {
+ lower = literal;
+ upper = literalToString(ref.type(), literals.get(1));
+ }
+ } else if (function instanceof GreaterOrEqual || function
instanceof GreaterThan) {
+ lower = literal;
+ } else if (function instanceof LessOrEqual || function instanceof
LessThan) {
+ upper = literal;
+ }
+ }
+
+ if (substringPrefix != null) {
+ return substringPrefix;
+ }
+ if (startsWithPrefix != null) {
+ return startsWithPrefix;
+ }
+ if (lower != null && upper != null) {
+ // For lexicographic order, lo <= v <= hi implies v starts with
the common prefix of lo
+ // and hi: if v differed from them at some position inside that
prefix, it would fall
+ // outside the range on that very position.
+ return commonPrefix(lower, upper);
+ }
+ return null;
+ }
+
+ /** Returns the literal of {@code substr(key, 1, n) = 'literal'}, or null
if it is not that. */
+ @Nullable
+ private static String substringPrefixOf(LeafPredicate leaf, String key) {
+ Transform transform = leaf.transform();
+ if (!(transform instanceof SubstringTransform)) {
+ return null;
+ }
+ List<Object> inputs = transform.inputs();
+ if (inputs.size() != 3 || !(inputs.get(0) instanceof FieldRef)) {
+ return null;
+ }
+ FieldRef ref = (FieldRef) inputs.get(0);
+ if (!key.equals(ref.name()) || !isStringType(ref.type())) {
+ return null;
+ }
+ // Only a substring anchored at the first character bounds the value's
prefix.
+ if (!isOne(inputs.get(1))) {
+ return null;
+ }
+ Object literal = leaf.literals().get(0);
+ return literal instanceof BinaryString ? literal.toString() : null;
+ }
+
+ private static boolean isOne(Object begin) {
+ if (begin instanceof Number) {
+ return ((Number) begin).longValue() == 1L;
+ }
+ try {
+ return begin != null && Long.parseLong(begin.toString()) == 1L;
+ } catch (NumberFormatException e) {
+ return false;
+ }
+ }
+
+ private static boolean isStringType(DataType type) {
+ return
type.getTypeRoot().getFamilies().contains(DataTypeFamily.CHARACTER_STRING);
+ }
+
+ @Nullable
+ private static String literalToString(DataType type, Object literal) {
+ @SuppressWarnings("unchecked")
+ CastExecutor<Object, BinaryString> executor =
+ (CastExecutor<Object, BinaryString>)
+ CastExecutors.resolve(type, VarCharType.STRING_TYPE);
+ if (executor == null) {
+ return null;
+ }
+ BinaryString value = executor.cast(literal);
+ return value == null ? null : value.toString();
+ }
+
+ private static String commonPrefix(String a, String b) {
+ int max = Math.min(a.length(), b.length());
+ int i = 0;
+ while (i < max) {
+ int codePoint = a.codePointAt(i);
+ if (codePoint != b.codePointAt(i)
+ || (Character.isSurrogate(a.charAt(i))
+ && Character.charCount(codePoint) == 1)) {
+ break;
+ }
+ i += Character.charCount(codePoint);
+ }
+ return a.substring(0, i);
+ }
+}
diff --git
a/paimon-core/src/test/java/org/apache/paimon/table/format/CatalogFormatTablePartitionManagerTest.java
b/paimon-core/src/test/java/org/apache/paimon/table/format/CatalogFormatTablePartitionManagerTest.java
index 7d246bf4f5..dfcc422be3 100644
---
a/paimon-core/src/test/java/org/apache/paimon/table/format/CatalogFormatTablePartitionManagerTest.java
+++
b/paimon-core/src/test/java/org/apache/paimon/table/format/CatalogFormatTablePartitionManagerTest.java
@@ -287,6 +287,72 @@ class CatalogFormatTablePartitionManagerTest {
assertThat(partitions).containsExactly(matching);
}
+ @Test
+ void testRangeOnLeadingKeyIsPushedDownAsValuePattern() throws Exception {
+ Catalog catalog = mock(Catalog.class);
+ when(catalog.listPartitionsByFilterPaged(any(), any(), any(), any(),
any()))
+ .thenReturn(new PagedList<>(Collections.emptyList(), null));
+
+ partitionManager(catalog)
+ .listPartitions(Collections.emptyMap(), yearBetween("2019",
"2021"));
+
+ verify(catalog)
+ .listPartitionsByFilterPaged(
+ eq(IDENTIFIER), any(), eq(REQUEST_SIZE), isNull(),
eq("year=20%"));
+ }
+
+ @Test
+ void testValuePatternExtendsAnEqualityPrefixByOneKey() throws Exception {
+ Catalog catalog = mock(Catalog.class);
+ when(catalog.listPartitionsByFilterPaged(any(), any(), any(), any(),
any()))
+ .thenReturn(new PagedList<>(Collections.emptyList(), null));
+
+ PredicateBuilder builder = partitionPredicates();
+ Predicate monthRange =
+ PredicateBuilder.and(
+ builder.greaterOrEqual(1,
BinaryString.fromString("01")),
+ builder.lessOrEqual(1, BinaryString.fromString("09")));
+
+ partitionManager(catalog)
+ .listPartitions(Collections.singletonMap("year", "2025"),
monthRange);
+
+ verify(catalog)
+ .listPartitionsByFilterPaged(
+ eq(IDENTIFIER),
+ any(),
+ eq(REQUEST_SIZE),
+ isNull(),
+ eq("year=2025/month=0%"));
+ }
+
+ /**
+ * The pattern only narrows what the catalog returns; the scan still tests
every candidate. So a
+ * pattern that matches extra partitions is harmless, while one that
misses a matching partition
+ * silently drops data. This pins the direction that matters against a
catalog that really
+ * applies the LIKE pattern.
+ */
+ @Test
+ void testValuePatternNeverDropsAMatchingPartition() throws Exception {
+ List<Partition> universe = new ArrayList<>();
+ for (int year = 2018; year <= 2021; year++) {
+ for (int month = 1; month <= 12; month++) {
+ universe.add(partition(String.valueOf(year),
String.format("%02d", month)));
+ }
+ }
+ Catalog catalog = catalogHonouringPattern(universe);
+
+ Predicate filter = yearBetween("2019", "2021");
+ List<Partition> returned =
+
partitionManager(catalog).listPartitions(Collections.emptyMap(), filter);
+
+ List<Partition> shouldSurvive =
+ universe.stream()
+ .filter(p -> p.spec().get("year").compareTo("2019") >=
0)
+ .filter(p -> p.spec().get("year").compareTo("2021") <=
0)
+ .collect(Collectors.toList());
+ assertThat(returned).containsAll(shouldSurvive);
+ }
+
@Test
void testNonLeadingPrefixIsRejected() {
Catalog catalog = mock(Catalog.class);
@@ -506,6 +572,50 @@ class CatalogFormatTablePartitionManagerTest {
return
batches.stream().flatMap(List::stream).collect(Collectors.toList());
}
+ private static PredicateBuilder partitionPredicates() {
+ return new PredicateBuilder(
+ RowType.of(
+ new DataType[] {DataTypes.STRING(),
DataTypes.STRING()},
+ new String[] {"year", "month"}));
+ }
+
+ private static Predicate yearBetween(String lower, String upper) {
+ PredicateBuilder builder = partitionPredicates();
+ return PredicateBuilder.and(
+ builder.greaterOrEqual(0, BinaryString.fromString(lower)),
+ builder.lessOrEqual(0, BinaryString.fromString(upper)));
+ }
+
+ /** A catalog that really applies the LIKE pattern, so an over-narrow
pattern shows up. */
+ private static Catalog catalogHonouringPattern(List<Partition> universe)
throws Exception {
+ Catalog catalog = mock(Catalog.class);
+ when(catalog.listPartitionsByFilterPaged(any(), any(), any(), any(),
any()))
+ .thenAnswer(
+ invocation -> {
+ String pattern = invocation.getArgument(4);
+ java.util.regex.Pattern like =
+ pattern == null
+ ? null
+ : java.util.regex.Pattern.compile(
+
java.util.regex.Pattern.quote(pattern)
+ .replace("%",
"\\E.*\\Q")
+ .replace("_",
"\\E.\\Q"));
+ List<Partition> matched = new ArrayList<>();
+ for (Partition partition : universe) {
+ String name =
+ "year="
+ + partition.spec().get("year")
+ + "/month="
+ +
partition.spec().get("month");
+ if (like == null ||
like.matcher(name).matches()) {
+ matched.add(partition);
+ }
+ }
+ return new PagedList<>(matched, null);
+ });
+ return catalog;
+ }
+
private static Partition partition(String year, String month) {
return new Partition(spec(year, month), 0, 0, 0, 0, -1, false);
}
diff --git
a/paimon-core/src/test/java/org/apache/paimon/table/format/PartitionNamePatternsTest.java
b/paimon-core/src/test/java/org/apache/paimon/table/format/PartitionNamePatternsTest.java
new file mode 100644
index 0000000000..d11d91b4a2
--- /dev/null
+++
b/paimon-core/src/test/java/org/apache/paimon/table/format/PartitionNamePatternsTest.java
@@ -0,0 +1,220 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.paimon.table.format;
+
+import org.apache.paimon.data.BinaryString;
+import org.apache.paimon.predicate.Equal;
+import org.apache.paimon.predicate.FieldRef;
+import org.apache.paimon.predicate.LeafPredicate;
+import org.apache.paimon.predicate.Predicate;
+import org.apache.paimon.predicate.PredicateBuilder;
+import org.apache.paimon.predicate.SubstringTransform;
+import org.apache.paimon.rest.RESTApi;
+import org.apache.paimon.rest.requests.ListPartitionsByFilterRequest;
+import org.apache.paimon.types.DataTypes;
+import org.apache.paimon.types.RowType;
+
+import org.junit.jupiter.api.Test;
+
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.LinkedHashMap;
+import java.util.List;
+
+import static org.assertj.core.api.Assertions.assertThat;
+
+/** Tests for {@link PartitionNamePatterns}. */
+class PartitionNamePatternsTest {
+
+ private static final List<String> ONE_KEY =
Collections.singletonList("dt");
+ private static final List<String> TWO_KEYS = Arrays.asList("dt", "hour");
+
+ private static PredicateBuilder stringBuilder() {
+ return new PredicateBuilder(
+ RowType.of(
+ new org.apache.paimon.types.DataType[] {
+ DataTypes.STRING(), DataTypes.STRING()
+ },
+ new String[] {"dt", "hour"}));
+ }
+
+ private static PredicateBuilder intBuilder() {
+ return new PredicateBuilder(
+ RowType.of(
+ new org.apache.paimon.types.DataType[]
{DataTypes.INT()},
+ new String[] {"dt"}));
+ }
+
+ private static BinaryString s(String v) {
+ return BinaryString.fromString(v);
+ }
+
+ private static LinkedHashMap<String, String> noPrefix() {
+ return new LinkedHashMap<>();
+ }
+
+ @Test
+ void noFilterKeepsTheExistingEqualityBehaviour() {
+ LinkedHashMap<String, String> prefix = new LinkedHashMap<>();
+ prefix.put("dt", "20190702");
+ assertThat(PartitionNamePatterns.build(TWO_KEYS, prefix,
null)).isEqualTo("dt=20190702/%");
+ assertThat(PartitionNamePatterns.build(ONE_KEY, prefix,
null)).isEqualTo("dt=20190702");
+ assertThat(PartitionNamePatterns.build(ONE_KEY, noPrefix(),
null)).isNull();
+ }
+
+ @Test
+ void rangeOnLeadingKeyNarrowsToTheCommonPrefix() {
+ PredicateBuilder b = stringBuilder();
+ Predicate filter =
+ PredicateBuilder.and(
+ b.greaterOrEqual(0, s("2019070100")), b.lessOrEqual(0,
s("2019070123")));
+ assertThat(PartitionNamePatterns.build(ONE_KEY, noPrefix(), filter))
+ .isEqualTo("dt=20190701%");
+ }
+
+ @Test
+ void supplementaryRangeDoesNotProduceMalformedPatternAcrossJson() throws
Exception {
+ PredicateBuilder b = stringBuilder();
+ Predicate filter = b.between(0, s("😀-a"), s("🙏-z"));
+
+ String pattern = PartitionNamePatterns.build(ONE_KEY, noPrefix(),
filter);
+ ListPartitionsByFilterRequest request =
+ new ListPartitionsByFilterRequest("filter-json", pattern,
1000, null);
+ ListPartitionsByFilterRequest roundTripped =
+ RESTApi.fromJson(RESTApi.toJson(request),
ListPartitionsByFilterRequest.class);
+
+ assertThat(roundTripped.getPartitionNamePattern()).isNull();
+ }
+
+ @Test
+ void strictBoundsNarrowTheSameWay() {
+ PredicateBuilder b = stringBuilder();
+ Predicate filter =
+ PredicateBuilder.and(
+ b.greaterThan(0, s("2019070100")), b.lessThan(0,
s("2019070123")));
+ assertThat(PartitionNamePatterns.build(ONE_KEY, noPrefix(), filter))
+ .isEqualTo("dt=20190701%");
+ }
+
+ @Test
+ void rangeUsesAnyNonEmptyCommonPrefix() {
+ PredicateBuilder b = stringBuilder();
+ Predicate filter =
+ PredicateBuilder.and(
+ b.greaterOrEqual(0, s("2019010100")), b.lessOrEqual(0,
s("2019123123")));
+ // The year is shared, which is still a useful prefix.
+ assertThat(PartitionNamePatterns.build(ONE_KEY, noPrefix(),
filter)).isEqualTo("dt=2019%");
+
+ Predicate spanning =
+ PredicateBuilder.and(
+ b.greaterOrEqual(0, s("2019010100")), b.lessOrEqual(0,
s("2020123123")));
+ assertThat(PartitionNamePatterns.build(ONE_KEY, noPrefix(),
spanning)).isEqualTo("dt=20%");
+ }
+
+ @Test
+ void oneSidedRangeGivesNoPrefix() {
+ PredicateBuilder b = stringBuilder();
+ assertThat(
+ PartitionNamePatterns.build(
+ ONE_KEY, noPrefix(), b.greaterOrEqual(0,
s("2019070100"))))
+ .isNull();
+ }
+
+ @Test
+ void betweenWithNullBoundaryFallsBack() {
+ PredicateBuilder b = stringBuilder();
+
+ assertThat(PartitionNamePatterns.build(ONE_KEY, noPrefix(),
b.between(0, null, s("z"))))
+ .isNull();
+ assertThat(PartitionNamePatterns.build(ONE_KEY, noPrefix(),
b.between(0, s("a"), null)))
+ .isNull();
+ }
+
+ /**
+ * The counterexample that forces the string-only guard: on an INT key,
{@code dt >= 9 AND dt <=
+ * 99} holds for 10, whose partition name {@code dt=10} does not start
with the common prefix
+ * "9". Deriving a pattern there would silently drop a matching partition.
+ */
+ @Test
+ void numericKeyNeverGetsAPrefix() {
+ PredicateBuilder b = intBuilder();
+ Predicate filter = PredicateBuilder.and(b.greaterOrEqual(0, 9),
b.lessOrEqual(0, 99));
+ assertThat(PartitionNamePatterns.build(ONE_KEY, noPrefix(),
filter)).isNull();
+ }
+
+ @Test
+ void substringAnchoredAtOneNarrowsToItsLiteral() {
+ Predicate filter = substringEqual(1, 6, "201907");
+ assertThat(PartitionNamePatterns.build(ONE_KEY, noPrefix(), filter))
+ .isEqualTo("dt=201907%");
+ }
+
+ @Test
+ void substringNotAnchoredAtOneGivesNoPrefix() {
+ // substr(dt, 3, 4) says nothing about the first two characters.
+ assertThat(PartitionNamePatterns.build(ONE_KEY, noPrefix(),
substringEqual(3, 4, "1907")))
+ .isNull();
+ }
+
+ @Test
+ void valuePrefixExtendsAnEqualityPrefixByExactlyOneKey() {
+ LinkedHashMap<String, String> prefix = new LinkedHashMap<>();
+ prefix.put("dt", "20190702");
+ PredicateBuilder b = stringBuilder();
+ Predicate filter =
+ PredicateBuilder.and(b.greaterOrEqual(1, s("06")),
b.lessOrEqual(1, s("09")));
+ assertThat(PartitionNamePatterns.build(TWO_KEYS, prefix, filter))
+ .isEqualTo("dt=20190702/hour=0%");
+ }
+
+ @Test
+ void unrepresentableEqualityPrefixIsNotExtended() {
+ LinkedHashMap<String, String> prefix = new LinkedHashMap<>();
+ prefix.put("dt", "");
+ PredicateBuilder b = stringBuilder();
+ Predicate filter =
+ PredicateBuilder.and(b.greaterOrEqual(1, s("06")),
b.lessOrEqual(1, s("09")));
+
+ assertThat(PartitionNamePatterns.build(TWO_KEYS, prefix,
filter)).isNull();
+ }
+
+ @Test
+ void predicateOnALaterKeyDoesNotLeakIntoTheLeadingPosition() {
+ // Constraining `hour` says nothing about `dt`, so the pattern must
stay unconstrained.
+ PredicateBuilder b = stringBuilder();
+ Predicate filter =
+ PredicateBuilder.and(b.greaterOrEqual(1, s("06")),
b.lessOrEqual(1, s("09")));
+ assertThat(PartitionNamePatterns.build(TWO_KEYS, noPrefix(),
filter)).isNull();
+ }
+
+ @Test
+ void orPredicatesAreIgnored() {
+ PredicateBuilder b = stringBuilder();
+ Predicate filter =
+ PredicateBuilder.or(b.equal(0, s("2019070212")), b.equal(0,
s("2019070213")));
+ // splitAnd yields the OR itself, which is not a leaf: no prefix, and
no wrong narrowing.
+ assertThat(PartitionNamePatterns.build(ONE_KEY, noPrefix(),
filter)).isNull();
+ }
+
+ private static Predicate substringEqual(int begin, int length, String
literal) {
+ FieldRef ref = new FieldRef(0, "dt", DataTypes.STRING());
+ SubstringTransform transform = new
SubstringTransform(Arrays.asList(ref, begin, length));
+ return LeafPredicate.of(transform, Equal.INSTANCE,
Collections.singletonList(s(literal)));
+ }
+}
