This is an automated email from the ASF dual-hosted git repository.
clintropolis pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/druid.git
The following commit(s) were added to refs/heads/master by this push:
new 7f02bce9602 perf: optimize some multi-value string operators to allow
planning to specialized ListFilteredVirtualColumn without extractionFn (#20469)
7f02bce9602 is described below
commit 7f02bce9602e2ccb75dd260f196721476776bb6a
Author: Clint Wylie <[email protected]>
AuthorDate: Wed Oct 21 04:37:18 2026 -0700
perf: optimize some multi-value string operators to allow planning to
specialized ListFilteredVirtualColumn without extractionFn (#20469)
---
.../druid/benchmark/query/SqlBaseBenchmark.java | 10 +-
.../query/SqlMultiValueStringFilterBenchmark.java | 236 ++++++++++++
.../query/dimension/ListFilteredDimensionSpec.java | 26 +-
.../segment/virtual/ListFilteredVirtualColumn.java | 40 ++-
.../virtual/PrefixFilteredVirtualColumn.java | 10 +-
.../virtual/RegexFilteredVirtualColumn.java | 10 +-
...FilteredDimensionSpecDimensionSelectorTest.java | 59 +++
.../ListFilteredVirtualColumnSelectorTest.java | 109 ++++++
.../virtual/ListFilteredVirtualColumnTest.java | 23 ++
.../virtual/PrefixFilteredVirtualColumnTest.java | 22 ++
.../virtual/RegexFilteredVirtualColumnTest.java | 22 ++
.../sql/calcite/expression/DruidExpression.java | 19 +
.../MultiValueStringOperatorConversions.java | 224 ++++++++----
.../calcite/CalciteMultiValueStringQueryTest.java | 396 ++++++++++++++++++++-
14 files changed, 1100 insertions(+), 106 deletions(-)
diff --git
a/benchmarks/src/test/java/org/apache/druid/benchmark/query/SqlBaseBenchmark.java
b/benchmarks/src/test/java/org/apache/druid/benchmark/query/SqlBaseBenchmark.java
index bdd196618ac..3c0ed62c817 100644
---
a/benchmarks/src/test/java/org/apache/druid/benchmark/query/SqlBaseBenchmark.java
+++
b/benchmarks/src/test/java/org/apache/druid/benchmark/query/SqlBaseBenchmark.java
@@ -237,6 +237,14 @@ public class SqlBaseBenchmark
return context;
}
+ /**
+ * Additional lookups, by name, to make available to queries via LOOKUP
+ */
+ protected Map<String, LookupExtractor> getLookups()
+ {
+ return Collections.emptyMap();
+ }
+
protected IndexSpec getIndexSpec()
{
return IndexSpec.builder()
@@ -331,7 +339,7 @@ public class SqlBaseBenchmark
final Pair<PlannerFactory, SqlEngine> sqlSystem = createSqlSystem(
segments,
realtimeSegments,
- Collections.emptyMap(),
+ getLookups(),
storageType,
closer
);
diff --git
a/benchmarks/src/test/java/org/apache/druid/benchmark/query/SqlMultiValueStringFilterBenchmark.java
b/benchmarks/src/test/java/org/apache/druid/benchmark/query/SqlMultiValueStringFilterBenchmark.java
new file mode 100644
index 00000000000..fba88fb2ddf
--- /dev/null
+++
b/benchmarks/src/test/java/org/apache/druid/benchmark/query/SqlMultiValueStringFilterBenchmark.java
@@ -0,0 +1,236 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.druid.benchmark.query;
+
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.ImmutableMap;
+import com.google.common.collect.Maps;
+import org.apache.druid.java.util.common.Pair;
+import org.apache.druid.java.util.common.StringUtils;
+import org.apache.druid.query.QueryContexts;
+import org.apache.druid.query.lookup.ImmutableLookupMap;
+import org.apache.druid.query.lookup.LookupExtractor;
+import org.apache.druid.sql.calcite.planner.PlannerConfig;
+import org.apache.druid.sql.calcite.planner.PlannerContext;
+import org.openjdk.jmh.annotations.Fork;
+import org.openjdk.jmh.annotations.Measurement;
+import org.openjdk.jmh.annotations.Param;
+import org.openjdk.jmh.annotations.Scope;
+import org.openjdk.jmh.annotations.State;
+import org.openjdk.jmh.annotations.Warmup;
+
+import java.util.Collections;
+import java.util.List;
+import java.util.Map;
+
+/**
+ * Benchmark for the multi-value string filtering functions MV_FILTER_ONLY,
MV_FILTER_NONE, MV_FILTER_REGEX, and
+ * MV_FILTER_PREFIX applied to expressions (LOOKUP, SUBSTRING, REGEXP_EXTRACT)
of multi-value string columns, comparing
+ * the plain expression plan with the specialized filtering virtual column
plan (with an expression virtual column
+ * delegate), and with the extractionFn plan used when {@link
PlannerContext#CTX_SQL_USE_EXTRACTION_FNS} is set.
+ */
+@State(Scope.Benchmark)
+@Fork(value = 1)
+@Warmup(iterations = 3)
+@Measurement(iterations = 5)
+public class SqlMultiValueStringFilterBenchmark extends SqlBaseQueryBenchmark
+{
+ private static final String ENUM_LOOKUP = "enum-lookup";
+ private static final String NUMBER_LOOKUP = "number-lookup";
+ private static final int NUMBER_LOOKUP_KEYS = 1_000_000;
+ private static final int NUMBER_LOOKUP_VALUES = 1_000;
+
+ private static final List<Pair<String, String>> QUERIES = ImmutableList.of(
+ // 0: low cardinality multi-value string column (5 values)
+ Pair.of(
+ SqlBenchmarkDatasets.BASIC,
+ "SELECT MV_FILTER_ONLY(LOOKUP(dimMultivalEnumerated, 'enum-lookup'),
ARRAY['greeting']), COUNT(*) "
+ + "FROM basic GROUP BY 1"
+ ),
+ // 1:
+ Pair.of(
+ SqlBenchmarkDatasets.BASIC,
+ "SELECT MV_FILTER_NONE(LOOKUP(dimMultivalEnumerated, 'enum-lookup'),
ARRAY['greeting']), COUNT(*) "
+ + "FROM basic GROUP BY 1"
+ ),
+ // 2:
+ Pair.of(
+ SqlBenchmarkDatasets.BASIC,
+ "SELECT MV_FILTER_REGEX(LOOKUP(dimMultivalEnumerated,
'enum-lookup'), '^g.*'), COUNT(*) "
+ + "FROM basic GROUP BY 1"
+ ),
+ // 3:
+ Pair.of(
+ SqlBenchmarkDatasets.BASIC,
+ "SELECT MV_FILTER_PREFIX(LOOKUP(dimMultivalEnumerated,
'enum-lookup'), 'g'), COUNT(*) "
+ + "FROM basic GROUP BY 1"
+ ),
+ // 4:
+ Pair.of(
+ SqlBenchmarkDatasets.BASIC,
+ "SELECT COUNT(*) FROM basic "
+ + "WHERE MV_FILTER_ONLY(LOOKUP(dimMultivalEnumerated,
'enum-lookup'), ARRAY['greeting']) = 'greeting'"
+ ),
+ // 5: high cardinality multi-value string columns (1M values)
+ Pair.of(
+ SqlBenchmarkDatasets.GROUPER,
+ "SELECT MV_FILTER_ONLY(LOOKUP(\"multi-string-Uniform-1_000_000\",
'number-lookup'), ARRAY['bucket-1', 'bucket-2']), COUNT(*) "
+ + "FROM grouper GROUP BY 1"
+ ),
+ // 6:
+ Pair.of(
+ SqlBenchmarkDatasets.GROUPER,
+ "SELECT MV_FILTER_ONLY(LOOKUP(\"multi-string-ZipF-1_000_000\",
'number-lookup'), ARRAY['bucket-1', 'bucket-2']), COUNT(*) "
+ + "FROM grouper GROUP BY 1"
+ ),
+ // 7:
+ Pair.of(
+ SqlBenchmarkDatasets.GROUPER,
+ "SELECT MV_FILTER_PREFIX(LOOKUP(\"multi-string-Uniform-1_000_000\",
'number-lookup'), 'bucket-1'), COUNT(*) "
+ + "FROM grouper GROUP BY 1"
+ ),
+ // 8:
+ Pair.of(
+ SqlBenchmarkDatasets.GROUPER,
+ "SELECT COUNT(*) FROM grouper "
+ + "WHERE MV_FILTER_ONLY(LOOKUP(\"multi-string-Uniform-1_000_000\",
'number-lookup'), ARRAY['bucket-1']) = 'bucket-1'"
+ ),
+ // 9: other functions which plan to an extractionFn when
sqlUseExtractionFns is set
+ Pair.of(
+ SqlBenchmarkDatasets.BASIC,
+ "SELECT MV_FILTER_ONLY(SUBSTRING(dimMultivalEnumerated, 1, 1),
ARRAY['B']), COUNT(*) "
+ + "FROM basic GROUP BY 1"
+ ),
+ // 10:
+ Pair.of(
+ SqlBenchmarkDatasets.BASIC,
+ "SELECT MV_FILTER_ONLY(REGEXP_EXTRACT(dimMultivalEnumerated,
'^[A-Z][a-z]'), ARRAY['Ba']), COUNT(*) "
+ + "FROM basic GROUP BY 1"
+ ),
+ // 11:
+ Pair.of(
+ SqlBenchmarkDatasets.GROUPER,
+ "SELECT MV_FILTER_ONLY(SUBSTRING(\"multi-string-Uniform-1_000_000\",
1, 2), ARRAY['12', '34']), COUNT(*) "
+ + "FROM grouper GROUP BY 1"
+ ),
+ // 12:
+ Pair.of(
+ SqlBenchmarkDatasets.GROUPER,
+ "SELECT
MV_FILTER_ONLY(REGEXP_EXTRACT(\"multi-string-Uniform-1_000_000\", '[0-9]{2}$'),
ARRAY['12', '34']), COUNT(*) "
+ + "FROM grouper GROUP BY 1"
+ )
+ );
+
+ @Param({
+ "0",
+ "1",
+ "2",
+ "3",
+ "4",
+ "5",
+ "6",
+ "7",
+ "8",
+ "9",
+ "10",
+ "11",
+ "12"
+ })
+ private String query;
+
+ /**
+ * expression: plain expression virtual columns, forced with {@link
PlannerConfig#CTX_KEY_FORCE_EXPRESSION_VIRTUAL_COLUMNS}
+ * specialized: default planning, specialized filtering virtual columns with
an expression virtual column delegate
+ * extractionFn: specialized filtering virtual columns with an extractionFn
delegate dimension spec
+ */
+ @Param({
+ "expression",
+ "specialized",
+ "extractionFn"
+ })
+ private String plan;
+
+ @Override
+ public String getQuery()
+ {
+ return QUERIES.get(Integer.parseInt(query)).rhs;
+ }
+
+ @Override
+ public List<String> getDatasources()
+ {
+ return Collections.singletonList(QUERIES.get(Integer.parseInt(query)).lhs);
+ }
+
+ @Override
+ protected Map<String, Object> getContext()
+ {
+ final ImmutableMap.Builder<String, Object> builder =
+ ImmutableMap.<String, Object>builder().putAll(super.getContext());
+ switch (plan) {
+ case "expression":
+ builder.put(PlannerConfig.CTX_KEY_FORCE_EXPRESSION_VIRTUAL_COLUMNS,
true);
+ break;
+ case "extractionFn":
+ builder.put(PlannerContext.CTX_SQL_USE_EXTRACTION_FNS, true);
+ break;
+ default:
+ break;
+ }
+ return builder.build();
+ }
+
+ @Override
+ protected Map<String, LookupExtractor> getLookups()
+ {
+ // many-to-one lookups, so filtered values include duplicates
+ final Map<String, String> enumLookup = ImmutableMap.of(
+ "Hello", "greeting",
+ "World", "greeting",
+ "Foo", "placeholder",
+ "Bar", "placeholder",
+ "Baz", "placeholder"
+ );
+ final Map<String, String> numberLookup =
Maps.newHashMapWithExpectedSize(NUMBER_LOOKUP_KEYS + 1);
+ for (int i = 0; i <= NUMBER_LOOKUP_KEYS; i++) {
+ numberLookup.put(String.valueOf(i), "bucket-" + (i %
NUMBER_LOOKUP_VALUES));
+ }
+ return ImmutableMap.of(
+ ENUM_LOOKUP,
+ ImmutableLookupMap.fromMap(enumLookup).asLookupExtractor(false, () ->
StringUtils.toUtf8(ENUM_LOOKUP)),
+ NUMBER_LOOKUP,
+ ImmutableLookupMap.fromMap(numberLookup).asLookupExtractor(false, ()
-> StringUtils.toUtf8(NUMBER_LOOKUP))
+ );
+ }
+
+ @Override
+ protected void checkIncompatibleParameters()
+ {
+ // none of the plans can be vectorized
+ if (QueryContexts.Vectorize.FORCE.equals(vectorizeContext)) {
+ System.exit(0);
+ }
+ // auto schema ingests multi-value strings as arrays, but this benchmark
is about multi-value strings
+ if ("auto".equals(schemaType)) {
+ System.exit(0);
+ }
+ super.checkIncompatibleParameters();
+ }
+}
diff --git
a/processing/src/main/java/org/apache/druid/query/dimension/ListFilteredDimensionSpec.java
b/processing/src/main/java/org/apache/druid/query/dimension/ListFilteredDimensionSpec.java
index fdf73c17ab6..06f09f2e2e7 100644
---
a/processing/src/main/java/org/apache/druid/query/dimension/ListFilteredDimensionSpec.java
+++
b/processing/src/main/java/org/apache/druid/query/dimension/ListFilteredDimensionSpec.java
@@ -76,9 +76,9 @@ public class ListFilteredDimensionSpec extends
BaseFilteredDimensionSpec
}
if (isWhitelist) {
- return filterAllowList(values, selector, delegate.getExtractionFn() !=
null);
+ return filterAllowList(values, selector);
} else {
- return filterDenyList(values, selector, delegate.getExtractionFn() !=
null);
+ return filterDenyList(values, selector);
}
}
@@ -89,19 +89,21 @@ public class ListFilteredDimensionSpec extends
BaseFilteredDimensionSpec
IndexedGetter<String> fn
)
{
- final IdMapping.Builder builder =
IdMapping.Builder.ofCardinality(values.size());
if (idLookup != null) {
+ final IdMapping.Builder builder =
IdMapping.Builder.ofCardinality(values.size());
for (String value : values) {
int i = idLookup.lookupId(value);
if (i >= 0) {
builder.addMapping(i);
}
}
- } else {
- for (int i = 0; i < cardinality; i++) {
- if (values.contains(fn.get(i))) {
- builder.addMapping(i);
- }
+ return builder.build();
+ }
+
+ final IdMapping.Builder builder = IdMapping.Builder.ofUnknownCardinality();
+ for (int i = 0; i < cardinality; i++) {
+ if (values.contains(fn.get(i))) {
+ builder.addForwardMapping(i);
}
}
return builder.build();
@@ -122,9 +124,9 @@ public class ListFilteredDimensionSpec extends
BaseFilteredDimensionSpec
return builder.build();
}
- public static DimensionSelector filterAllowList(Set<String> values,
DimensionSelector selector, boolean forcePredicateFilter)
+ public static DimensionSelector filterAllowList(Set<String> values,
DimensionSelector selector)
{
- if (forcePredicateFilter || selector.getValueCardinality() < 0 ||
!selector.nameLookupPossibleInAdvance()) {
+ if (selector.getValueCardinality() < 0 ||
!selector.nameLookupPossibleInAdvance()) {
return new PredicateFilteredDimensionSelector(selector,
Predicates.in(values));
}
final IdMapping idMapping = buildAllowListIdMapping(
@@ -136,9 +138,9 @@ public class ListFilteredDimensionSpec extends
BaseFilteredDimensionSpec
return new ForwardingFilteredDimensionSelector(selector, idMapping);
}
- public static DimensionSelector filterDenyList(Set<String> values,
DimensionSelector selector, boolean forcePredicateFilter)
+ public static DimensionSelector filterDenyList(Set<String> values,
DimensionSelector selector)
{
- if (forcePredicateFilter || selector.getValueCardinality() < 0 ||
!selector.nameLookupPossibleInAdvance()) {
+ if (selector.getValueCardinality() < 0 ||
!selector.nameLookupPossibleInAdvance()) {
return new PredicateFilteredDimensionSelector(
selector,
input -> !values.contains(input)
diff --git
a/processing/src/main/java/org/apache/druid/segment/virtual/ListFilteredVirtualColumn.java
b/processing/src/main/java/org/apache/druid/segment/virtual/ListFilteredVirtualColumn.java
index 41b58b04a39..b24c1b7c837 100644
---
a/processing/src/main/java/org/apache/druid/segment/virtual/ListFilteredVirtualColumn.java
+++
b/processing/src/main/java/org/apache/druid/segment/virtual/ListFilteredVirtualColumn.java
@@ -142,17 +142,9 @@ public class ListFilteredVirtualColumn implements
VirtualColumn
)
{
if (allowList) {
- return ListFilteredDimensionSpec.filterAllowList(
- values,
- factory.makeDimensionSelector(delegate),
- delegate.getExtractionFn() != null
- );
+ return ListFilteredDimensionSpec.filterAllowList(values,
factory.makeDimensionSelector(delegate));
} else {
- return ListFilteredDimensionSpec.filterDenyList(
- values,
- factory.makeDimensionSelector(delegate),
- delegate.getExtractionFn() != null
- );
+ return ListFilteredDimensionSpec.filterDenyList(values,
factory.makeDimensionSelector(delegate));
}
}
@@ -170,14 +162,20 @@ public class ListFilteredVirtualColumn implements
VirtualColumn
{
return new ColumnCapabilitiesImpl().setType(delegate.getOutputType())
.setDictionaryEncoded(true)
- .setHasBitmapIndexes(true);
+ .setHasBitmapIndexes(true)
+ .setHasMultipleValues(true);
}
@Nullable
@Override
public ColumnCapabilities capabilities(ColumnInspector inspector, String
columnName)
{
- return inspector.getColumnCapabilities(delegate.getDimension());
+ final ColumnCapabilities delegateCapabilities =
inspector.getColumnCapabilities(delegate.getDimension());
+ if (delegateCapabilities == null) {
+ return null;
+ }
+ // filtering can leave a row with no values, so even if the delegate is
single-valued this column is not
+ return
ColumnCapabilitiesImpl.copyOf(delegateCapabilities).setHasMultipleValues(true).setHasNulls(true);
}
@Override
@@ -301,17 +299,29 @@ public class ListFilteredVirtualColumn implements
VirtualColumn
IdMapping idMapping
)
{
- final int start = delegateIndex.getValue(idMapping.getReverseId(0)) ==
null ? 1 : 0;
return getBitmapsInRange(
delegateIndex,
idMapping,
DruidObjectPredicate.alwaysTrue(),
- start,
+ getFirstNonNullIndex(delegateIndex, idMapping),
idMapping.getValueCardinality(),
false
);
}
+ /**
+ * Returns the first filtered dictionary index which is not null. Values are
sorted with null first, so this is 1 if
+ * the filtered dictionary starts with null, otherwise 0, including when the
filtered dictionary is empty because
+ * none of the values matched.
+ */
+ private static int getFirstNonNullIndex(DictionaryEncodedStringValueIndex
delegateIndex, IdMapping idMapping)
+ {
+ if (idMapping.getValueCardinality() == 0) {
+ return 0;
+ }
+ return delegateIndex.getValue(idMapping.getReverseId(0)) == null ? 1 : 0;
+ }
+
private static Iterable<ImmutableBitmap> getBitmapsInRange(
DictionaryEncodedStringValueIndex delegate,
IdMapping idMapping,
@@ -662,7 +672,7 @@ public class ListFilteredVirtualColumn implements
VirtualColumn
)
{
int startIndex, endIndex;
- final int firstValue = delegate.getValue(idMapping.getReverseId(0)) ==
null ? 1 : 0;
+ final int firstValue = getFirstNonNullIndex(delegate, idMapping);
if (startValue == null) {
startIndex = firstValue;
} else {
diff --git
a/processing/src/main/java/org/apache/druid/segment/virtual/PrefixFilteredVirtualColumn.java
b/processing/src/main/java/org/apache/druid/segment/virtual/PrefixFilteredVirtualColumn.java
index 5b201edc528..49a656eb469 100644
---
a/processing/src/main/java/org/apache/druid/segment/virtual/PrefixFilteredVirtualColumn.java
+++
b/processing/src/main/java/org/apache/druid/segment/virtual/PrefixFilteredVirtualColumn.java
@@ -116,14 +116,20 @@ public class PrefixFilteredVirtualColumn implements
VirtualColumn
return new ColumnCapabilitiesImpl()
.setType(delegate.getOutputType())
.setDictionaryEncoded(true)
- .setHasBitmapIndexes(true);
+ .setHasBitmapIndexes(true)
+ .setHasMultipleValues(true);
}
@Nullable
@Override
public ColumnCapabilities capabilities(ColumnInspector inspector, String
columnName)
{
- return inspector.getColumnCapabilities(delegate.getDimension());
+ final ColumnCapabilities delegateCapabilities =
inspector.getColumnCapabilities(delegate.getDimension());
+ if (delegateCapabilities == null) {
+ return null;
+ }
+ // filtering can leave a row with no values, so even if the delegate is
single-valued this column is not
+ return
ColumnCapabilitiesImpl.copyOf(delegateCapabilities).setHasMultipleValues(true).setHasNulls(true);
}
@Override
diff --git
a/processing/src/main/java/org/apache/druid/segment/virtual/RegexFilteredVirtualColumn.java
b/processing/src/main/java/org/apache/druid/segment/virtual/RegexFilteredVirtualColumn.java
index e4c6a8f7327..dcc68c5ae2d 100644
---
a/processing/src/main/java/org/apache/druid/segment/virtual/RegexFilteredVirtualColumn.java
+++
b/processing/src/main/java/org/apache/druid/segment/virtual/RegexFilteredVirtualColumn.java
@@ -116,14 +116,20 @@ public class RegexFilteredVirtualColumn implements
VirtualColumn
return new ColumnCapabilitiesImpl()
.setType(delegate.getOutputType())
.setDictionaryEncoded(true)
- .setHasBitmapIndexes(true);
+ .setHasBitmapIndexes(true)
+ .setHasMultipleValues(true);
}
@Nullable
@Override
public ColumnCapabilities capabilities(ColumnInspector inspector, String
columnName)
{
- return inspector.getColumnCapabilities(delegate.getDimension());
+ final ColumnCapabilities delegateCapabilities =
inspector.getColumnCapabilities(delegate.getDimension());
+ if (delegateCapabilities == null) {
+ return null;
+ }
+ // filtering can leave a row with no values, so even if the delegate is
single-valued this column is not
+ return
ColumnCapabilitiesImpl.copyOf(delegateCapabilities).setHasMultipleValues(true).setHasNulls(true);
}
@Override
diff --git
a/processing/src/test/java/org/apache/druid/query/dimension/ListFilteredDimensionSpecDimensionSelectorTest.java
b/processing/src/test/java/org/apache/druid/query/dimension/ListFilteredDimensionSpecDimensionSelectorTest.java
index 907dae70b48..37753cf43c4 100644
---
a/processing/src/test/java/org/apache/druid/query/dimension/ListFilteredDimensionSpecDimensionSelectorTest.java
+++
b/processing/src/test/java/org/apache/druid/query/dimension/ListFilteredDimensionSpecDimensionSelectorTest.java
@@ -27,8 +27,11 @@ import it.unimi.dsi.fastutil.objects.Object2IntMap;
import it.unimi.dsi.fastutil.objects.Object2IntOpenHashMap;
import org.apache.commons.lang3.mutable.MutableInt;
import org.apache.druid.java.util.common.NonnullPair;
+import org.apache.druid.math.expr.ExprMacroTable;
+import org.apache.druid.math.expr.Parser;
import org.apache.druid.segment.DimensionSelector;
import org.apache.druid.segment.data.IndexedInts;
+import
org.apache.druid.segment.virtual.SingleStringInputDeferredEvaluationExpressionDimensionSelector;
import org.apache.druid.testing.InitializedNullHandlingTest;
import org.junit.jupiter.api.Assertions;
import org.junit.jupiter.api.Test;
@@ -109,6 +112,62 @@ public class
ListFilteredDimensionSpecDimensionSelectorTest extends InitializedN
testDenyList(false, true, false, PredicateFilteredDimensionSelector.class);
}
+ @Test
+ public void testAllowListWithDuplicateDictionaryValues()
+ {
+ RowSupplier rowSupplier = new RowSupplier();
+ DimensionSelector selector = ListFilteredDimensionSpec.filterAllowList(
+ ImmutableSet.of("val2"),
+ makeDuplicateValueSelector(rowSupplier)
+ );
+ Assertions.assertSame(ForwardingFilteredDimensionSelector.class,
selector.getClass());
+ Assertions.assertNull(selector.idLookup());
+ Assertions.assertEquals(3, selector.getValueCardinality());
+
+ rowSupplier.set(data.get(0));
+ Assertions.assertEquals(0, selector.getRow().size());
+
+ rowSupplier.set(data.get(1));
+ Assertions.assertEquals(ImmutableList.of("val2", "val2", "val2"),
selector.getObject());
+ Assertions.assertTrue(selector.makeValueMatcher("val2").matches(false));
+
+ rowSupplier.set(data.get(2));
+ Assertions.assertEquals(0, selector.getRow().size());
+ Assertions.assertFalse(selector.makeValueMatcher("val3").matches(false));
+ }
+
+ @Test
+ public void testDenyListWithDuplicateDictionaryValues()
+ {
+ RowSupplier rowSupplier = new RowSupplier();
+ DimensionSelector selector = ListFilteredDimensionSpec.filterDenyList(
+ ImmutableSet.of("val2"),
+ makeDuplicateValueSelector(rowSupplier)
+ );
+ Assertions.assertSame(ForwardingFilteredDimensionSelector.class,
selector.getClass());
+ Assertions.assertNull(selector.idLookup());
+ Assertions.assertEquals(3, selector.getValueCardinality());
+
+ rowSupplier.set(data.get(0));
+ Assertions.assertEquals(ImmutableList.of("val1", "val1"),
selector.getObject());
+
+ rowSupplier.set(data.get(1));
+ Assertions.assertEquals(0, selector.getRow().size());
+ Assertions.assertFalse(selector.makeValueMatcher("val2").matches(false));
+
+ rowSupplier.set(data.get(2));
+ Assertions.assertEquals("val3", selector.getObject());
+ }
+
+ private DimensionSelector makeDuplicateValueSelector(RowSupplier rowSupplier)
+ {
+ NonnullPair<Object2IntMap<String>, Int2ObjectMap<String>> dictionaries =
createDictionaries(data);
+ return new SingleStringInputDeferredEvaluationExpressionDimensionSelector(
+ new StringDimensionSelectorForTest(rowSupplier, dictionaries.lhs,
dictionaries.rhs, false, true, true),
+ Parser.parse("substring(\"foo\", 0, 4)", ExprMacroTable.nil())
+ );
+ }
+
private void testAllowList(
boolean unknownCardinality,
boolean validIdLookup,
diff --git
a/processing/src/test/java/org/apache/druid/segment/virtual/ListFilteredVirtualColumnSelectorTest.java
b/processing/src/test/java/org/apache/druid/segment/virtual/ListFilteredVirtualColumnSelectorTest.java
index a9d2ca1f76f..bb9de3ab19a 100644
---
a/processing/src/test/java/org/apache/druid/segment/virtual/ListFilteredVirtualColumnSelectorTest.java
+++
b/processing/src/test/java/org/apache/druid/segment/virtual/ListFilteredVirtualColumnSelectorTest.java
@@ -27,13 +27,19 @@ import org.apache.druid.collections.bitmap.ImmutableBitmap;
import org.apache.druid.data.input.MapBasedRow;
import org.apache.druid.java.util.common.io.Closer;
import org.apache.druid.query.dimension.DefaultDimensionSpec;
+import org.apache.druid.query.expression.TestExprMacroTable;
import org.apache.druid.query.filter.ColumnIndexSelector;
import org.apache.druid.segment.ColumnCache;
import org.apache.druid.segment.ColumnValueSelector;
+import org.apache.druid.segment.Cursor;
+import org.apache.druid.segment.CursorBuildSpec;
+import org.apache.druid.segment.CursorHolder;
import org.apache.druid.segment.DimensionSelector;
import org.apache.druid.segment.QueryableIndex;
+import org.apache.druid.segment.QueryableIndexCursorFactory;
import org.apache.druid.segment.RowAdapters;
import org.apache.druid.segment.RowBasedColumnSelectorFactory;
+import org.apache.druid.segment.TestIndex;
import org.apache.druid.segment.VirtualColumns;
import org.apache.druid.segment.column.BaseColumnHolder;
import org.apache.druid.segment.column.ColumnCapabilities;
@@ -43,14 +49,20 @@ import org.apache.druid.segment.column.ColumnIndexSupplier;
import org.apache.druid.segment.column.ColumnType;
import org.apache.druid.segment.column.RowSignature;
import org.apache.druid.segment.column.ValueType;
+import org.apache.druid.segment.data.IndexedInts;
import org.apache.druid.segment.filter.SelectorFilter;
import
org.apache.druid.segment.index.semantic.DictionaryEncodedStringValueIndex;
+import org.apache.druid.segment.index.semantic.DruidPredicateIndexes;
+import org.apache.druid.segment.index.semantic.NullValueIndex;
+import org.apache.druid.segment.index.semantic.StringValueSetIndexes;
import org.apache.druid.testing.InitializedNullHandlingTest;
import org.easymock.EasyMock;
import org.junit.jupiter.api.Assertions;
import org.junit.jupiter.api.Test;
import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
public class ListFilteredVirtualColumnSelectorTest extends
InitializedNullHandlingTest
{
@@ -284,6 +296,103 @@ public class ListFilteredVirtualColumnSelectorTest
extends InitializedNullHandli
}
}
+ @Test
+ public void testListFilteredVirtualColumnExpressionDelegate() throws
IOException
+ {
+ // expression maps many dictionary values to the same value, so the
delegate selector has duplicate names
+ final ExpressionVirtualColumn expressionVirtualColumn = new
ExpressionVirtualColumn(
+ "expr",
+ "if(placementish == 'preferred', 'p', 'x')",
+ ColumnType.STRING,
+ TestExprMacroTable.INSTANCE
+ );
+ final ListFilteredVirtualColumn allowVirtualColumn = new
ListFilteredVirtualColumn(
+ ALLOW_VIRTUAL_NAME,
+ DefaultDimensionSpec.of("expr"),
+ ImmutableSet.of("x"),
+ true
+ );
+ final ListFilteredVirtualColumn denyVirtualColumn = new
ListFilteredVirtualColumn(
+ DENY_VIRTUAL_NAME,
+ DefaultDimensionSpec.of("expr"),
+ ImmutableSet.of("x"),
+ false
+ );
+ final VirtualColumns virtualColumns = VirtualColumns.create(
+ expressionVirtualColumn,
+ allowVirtualColumn,
+ denyVirtualColumn
+ );
+ final QueryableIndex index = TestIndex.getMMappedTestIndex();
+
+ try (final Closer closer = Closer.create()) {
+ final CursorHolder cursorHolder = closer.register(
+ new QueryableIndexCursorFactory(index).makeCursorHolder(
+
CursorBuildSpec.builder().setVirtualColumns(virtualColumns).build()
+ )
+ );
+ final Cursor cursor = cursorHolder.asCursor();
+ final DimensionSelector baseSelector =
+
cursor.getColumnSelectorFactory().makeDimensionSelector(DefaultDimensionSpec.of("placementish"));
+ final DimensionSelector allowSelector =
+
cursor.getColumnSelectorFactory().makeDimensionSelector(DefaultDimensionSpec.of(ALLOW_VIRTUAL_NAME));
+ final DimensionSelector denySelector =
+
cursor.getColumnSelectorFactory().makeDimensionSelector(DefaultDimensionSpec.of(DENY_VIRTUAL_NAME));
+
+ // delegate is dictionary backed, so the filtered selector can remap
dictionary ids instead of evaluating the
+ // expression for every row
+ Assertions.assertEquals(
+
"org.apache.druid.query.dimension.ForwardingFilteredDimensionSelector",
+ allowSelector.getClass().getName()
+ );
+ Assertions.assertEquals(
+
"org.apache.druid.query.dimension.ForwardingFilteredDimensionSelector",
+ denySelector.getClass().getName()
+ );
+ Assertions.assertNull(allowSelector.idLookup());
+
+ int rows = 0;
+ while (!cursor.isDone()) {
+ final List<String> expectedAllow = new ArrayList<>();
+ final List<String> expectedDeny = new ArrayList<>();
+ final IndexedInts baseRow = baseSelector.getRow();
+ for (int i = 0; i < baseRow.size(); i++) {
+ if ("preferred".equals(baseSelector.lookupName(baseRow.get(i)))) {
+ expectedDeny.add("p");
+ } else {
+ expectedAllow.add("x");
+ }
+ }
+ Assertions.assertEquals(expectedAllow, lookupRow(allowSelector));
+ Assertions.assertEquals(expectedDeny, lookupRow(denySelector));
+ Assertions.assertEquals(!expectedAllow.isEmpty(),
allowSelector.makeValueMatcher("x").matches(false));
+ Assertions.assertEquals(!expectedDeny.isEmpty(),
denySelector.makeValueMatcher("p").matches(false));
+ rows++;
+ cursor.advance();
+ }
+ Assertions.assertEquals(index.getNumRows(), rows);
+
+ // the expression virtual column does not provide a dictionary index, so
there are no list filtered indexes
+ final ColumnIndexSelector indexSelector = new ColumnCache(index,
virtualColumns, closer);
+ final ColumnIndexSupplier indexSupplier =
indexSelector.getIndexSupplier(ALLOW_VIRTUAL_NAME);
+ Assertions.assertNotNull(indexSupplier);
+
Assertions.assertNull(indexSupplier.as(DictionaryEncodedStringValueIndex.class));
+ Assertions.assertNull(indexSupplier.as(StringValueSetIndexes.class));
+ Assertions.assertNull(indexSupplier.as(DruidPredicateIndexes.class));
+ Assertions.assertNull(indexSupplier.as(NullValueIndex.class));
+ }
+ }
+
+ private static List<String> lookupRow(DimensionSelector selector)
+ {
+ final IndexedInts row = selector.getRow();
+ final List<String> values = new ArrayList<>(row.size());
+ for (int i = 0; i < row.size(); i++) {
+ values.add(selector.lookupName(row.get(i)));
+ }
+ return values;
+ }
+
private void assertCapabilities(VirtualizedColumnSelectorFactory
selectorFactory, String columnName)
{
ColumnCapabilities capabilities =
selectorFactory.getColumnCapabilities(columnName);
diff --git
a/processing/src/test/java/org/apache/druid/segment/virtual/ListFilteredVirtualColumnTest.java
b/processing/src/test/java/org/apache/druid/segment/virtual/ListFilteredVirtualColumnTest.java
index c50c175394f..c5312243f3f 100644
---
a/processing/src/test/java/org/apache/druid/segment/virtual/ListFilteredVirtualColumnTest.java
+++
b/processing/src/test/java/org/apache/druid/segment/virtual/ListFilteredVirtualColumnTest.java
@@ -24,7 +24,10 @@ import com.fasterxml.jackson.databind.ObjectMapper;
import com.google.common.collect.ImmutableSet;
import nl.jqno.equalsverifier.EqualsVerifier;
import org.apache.druid.query.dimension.DefaultDimensionSpec;
+import org.apache.druid.segment.ColumnInspector;
import org.apache.druid.segment.TestHelper;
+import org.apache.druid.segment.column.ColumnCapabilities;
+import org.apache.druid.segment.column.ColumnCapabilitiesImpl;
import org.apache.druid.segment.column.ColumnType;
import org.junit.jupiter.api.Assertions;
import org.junit.jupiter.api.Test;
@@ -61,6 +64,26 @@ public class ListFilteredVirtualColumnTest
Assertions.assertArrayEquals(virtualColumn.getCacheKey(),
roundTrip.getCacheKey());
}
+ @Test
+ public void testCapabilitiesSingleValuedDelegate()
+ {
+ // filtering can leave rows with no values, so the virtual column is
multi-valued even if the delegate is not
+ ListFilteredVirtualColumn virtualColumn = new ListFilteredVirtualColumn(
+ "hello",
+ new DefaultDimensionSpec("column", "column", ColumnType.STRING),
+ ImmutableSet.of("foo", "bar"),
+ true
+ );
+ final ColumnInspector inspector =
+ column -> "column".equals(column) ?
ColumnCapabilitiesImpl.createSimpleSingleValueStringColumnCapabilities() : null;
+ final ColumnCapabilities capabilities =
virtualColumn.capabilities(inspector, "hello");
+ Assertions.assertEquals(ColumnType.STRING, capabilities.toColumnType());
+ Assertions.assertTrue(capabilities.hasMultipleValues().isTrue());
+ Assertions.assertTrue(capabilities.hasNulls().isTrue());
+
Assertions.assertTrue(virtualColumn.capabilities("hello").hasMultipleValues().isTrue());
+ Assertions.assertNull(virtualColumn.capabilities(column -> null, "hello"));
+ }
+
@Test
public void testEqualsAndHashcode()
{
diff --git
a/processing/src/test/java/org/apache/druid/segment/virtual/PrefixFilteredVirtualColumnTest.java
b/processing/src/test/java/org/apache/druid/segment/virtual/PrefixFilteredVirtualColumnTest.java
index 881035eccac..78b92b58097 100644
---
a/processing/src/test/java/org/apache/druid/segment/virtual/PrefixFilteredVirtualColumnTest.java
+++
b/processing/src/test/java/org/apache/druid/segment/virtual/PrefixFilteredVirtualColumnTest.java
@@ -23,7 +23,10 @@ import com.fasterxml.jackson.core.JsonProcessingException;
import com.fasterxml.jackson.databind.ObjectMapper;
import nl.jqno.equalsverifier.EqualsVerifier;
import org.apache.druid.query.dimension.DefaultDimensionSpec;
+import org.apache.druid.segment.ColumnInspector;
import org.apache.druid.segment.TestHelper;
+import org.apache.druid.segment.column.ColumnCapabilities;
+import org.apache.druid.segment.column.ColumnCapabilitiesImpl;
import org.apache.druid.segment.column.ColumnType;
import org.junit.jupiter.api.Assertions;
import org.junit.jupiter.api.Test;
@@ -45,6 +48,25 @@ public class PrefixFilteredVirtualColumnTest
Assertions.assertArrayEquals(virtualColumn.getCacheKey(),
roundTrip.getCacheKey());
}
+ @Test
+ public void testCapabilitiesSingleValuedDelegate()
+ {
+ // filtering can leave rows with no values, so the virtual column is
multi-valued even if the delegate is not
+ PrefixFilteredVirtualColumn virtualColumn = new
PrefixFilteredVirtualColumn(
+ "hello",
+ new DefaultDimensionSpec("column", "column", ColumnType.STRING),
+ "a"
+ );
+ final ColumnInspector inspector =
+ column -> "column".equals(column) ?
ColumnCapabilitiesImpl.createSimpleSingleValueStringColumnCapabilities() : null;
+ final ColumnCapabilities capabilities =
virtualColumn.capabilities(inspector, "hello");
+ Assertions.assertEquals(ColumnType.STRING, capabilities.toColumnType());
+ Assertions.assertTrue(capabilities.hasMultipleValues().isTrue());
+ Assertions.assertTrue(capabilities.hasNulls().isTrue());
+
Assertions.assertTrue(virtualColumn.capabilities("hello").hasMultipleValues().isTrue());
+ Assertions.assertNull(virtualColumn.capabilities(column -> null, "hello"));
+ }
+
@Test
public void testEqualsAndHashcode()
{
diff --git
a/processing/src/test/java/org/apache/druid/segment/virtual/RegexFilteredVirtualColumnTest.java
b/processing/src/test/java/org/apache/druid/segment/virtual/RegexFilteredVirtualColumnTest.java
index ded4b583584..364e94ebfc6 100644
---
a/processing/src/test/java/org/apache/druid/segment/virtual/RegexFilteredVirtualColumnTest.java
+++
b/processing/src/test/java/org/apache/druid/segment/virtual/RegexFilteredVirtualColumnTest.java
@@ -23,7 +23,10 @@ import com.fasterxml.jackson.core.JsonProcessingException;
import com.fasterxml.jackson.databind.ObjectMapper;
import nl.jqno.equalsverifier.EqualsVerifier;
import org.apache.druid.query.dimension.DefaultDimensionSpec;
+import org.apache.druid.segment.ColumnInspector;
import org.apache.druid.segment.TestHelper;
+import org.apache.druid.segment.column.ColumnCapabilities;
+import org.apache.druid.segment.column.ColumnCapabilitiesImpl;
import org.apache.druid.segment.column.ColumnType;
import org.junit.jupiter.api.Assertions;
import org.junit.jupiter.api.Test;
@@ -45,6 +48,25 @@ public class RegexFilteredVirtualColumnTest
Assertions.assertArrayEquals(virtualColumn.getCacheKey(),
roundTrip.getCacheKey());
}
+ @Test
+ public void testCapabilitiesSingleValuedDelegate()
+ {
+ // filtering can leave rows with no values, so the virtual column is
multi-valued even if the delegate is not
+ RegexFilteredVirtualColumn virtualColumn = new RegexFilteredVirtualColumn(
+ "hello",
+ new DefaultDimensionSpec("column", "column", ColumnType.STRING),
+ "^.*abc$"
+ );
+ final ColumnInspector inspector =
+ column -> "column".equals(column) ?
ColumnCapabilitiesImpl.createSimpleSingleValueStringColumnCapabilities() : null;
+ final ColumnCapabilities capabilities =
virtualColumn.capabilities(inspector, "hello");
+ Assertions.assertEquals(ColumnType.STRING, capabilities.toColumnType());
+ Assertions.assertTrue(capabilities.hasMultipleValues().isTrue());
+ Assertions.assertTrue(capabilities.hasNulls().isTrue());
+
Assertions.assertTrue(virtualColumn.capabilities("hello").hasMultipleValues().isTrue());
+ Assertions.assertNull(virtualColumn.capabilities(column -> null, "hello"));
+ }
+
@Test
public void testEqualsAndHashcode()
{
diff --git
a/sql/src/main/java/org/apache/druid/sql/calcite/expression/DruidExpression.java
b/sql/src/main/java/org/apache/druid/sql/calcite/expression/DruidExpression.java
index e5fa89ecd4c..6c2f2e526a3 100644
---
a/sql/src/main/java/org/apache/druid/sql/calcite/expression/DruidExpression.java
+++
b/sql/src/main/java/org/apache/druid/sql/calcite/expression/DruidExpression.java
@@ -505,6 +505,25 @@ public class DruidExpression
);
}
+ /**
+ * Returns a copy of this expression as {@link NodeType#SPECIALIZED} backed
by a plain
+ * {@link ExpressionVirtualColumn}. When used as an argument of another
{@link NodeType#SPECIALIZED} expression, the
+ * {@link org.apache.druid.sql.calcite.rel.VirtualColumnRegistry} will lift
this expression into its own virtual
+ * column and replace the argument with a direct column reference to it,
which allows the parent virtual column to
+ * reference it.
+ */
+ public DruidExpression asSpecialized()
+ {
+ return new DruidExpression(
+ NodeType.SPECIALIZED,
+ druidType,
+ simpleExtraction,
+ expressionGenerator,
+ arguments,
+ DEFAULT_VIRTUAL_COLUMN_BUILDER
+ );
+ }
+
/**
* Visit all sub {@link DruidExpression} (the {@link #arguments} of this
expression), allowing the
* {@link DruidExpressionShuttle} to potentially rewrite these arguments
with new {@link DruidExpression}, finally
diff --git
a/sql/src/main/java/org/apache/druid/sql/calcite/expression/builtin/MultiValueStringOperatorConversions.java
b/sql/src/main/java/org/apache/druid/sql/calcite/expression/builtin/MultiValueStringOperatorConversions.java
index 5b6b23f8224..4277903a13b 100644
---
a/sql/src/main/java/org/apache/druid/sql/calcite/expression/builtin/MultiValueStringOperatorConversions.java
+++
b/sql/src/main/java/org/apache/druid/sql/calcite/expression/builtin/MultiValueStringOperatorConversions.java
@@ -19,6 +19,7 @@
package org.apache.druid.sql.calcite.expression.builtin;
+import com.google.common.collect.Iterables;
import com.google.common.collect.Sets;
import org.apache.calcite.rex.RexCall;
import org.apache.calcite.rex.RexLiteral;
@@ -33,8 +34,11 @@ import org.apache.calcite.sql.type.SqlTypeName;
import org.apache.druid.math.expr.Evals;
import org.apache.druid.math.expr.Expr;
import org.apache.druid.math.expr.InputBindings;
+import org.apache.druid.query.dimension.DimensionSpec;
+import org.apache.druid.segment.VirtualColumn;
import org.apache.druid.segment.column.ColumnType;
import org.apache.druid.segment.column.RowSignature;
+import org.apache.druid.segment.virtual.ExpressionVirtualColumn;
import org.apache.druid.segment.virtual.ListFilteredVirtualColumn;
import org.apache.druid.segment.virtual.PrefixFilteredVirtualColumn;
import org.apache.druid.segment.virtual.RegexFilteredVirtualColumn;
@@ -48,6 +52,7 @@ import org.apache.druid.sql.calcite.planner.Calcites;
import org.apache.druid.sql.calcite.planner.PlannerContext;
import javax.annotation.Nullable;
+import java.util.ArrayList;
import java.util.HashSet;
import java.util.List;
@@ -375,48 +380,152 @@ public class MultiValueStringOperatorConversions
};
Expr expr =
plannerContext.parseExpression(druidExpressions.get(1).getExpression());
- if (druidExpressions.get(0).isSimpleExtraction() && expr.isLiteral()) {
+ if (expr.isLiteral()) {
Object[] lit = expr.eval(InputBindings.nilBindings()).asArray();
if (lit == null || lit.length == 0) {
- return null;
+ return druidExpressions.get(0).isSimpleExtraction()
+ ? null
+ : DruidExpression.ofExpression(ColumnType.STRING, builder,
druidExpressions);
}
HashSet<String> literals = Sets.newHashSetWithExpectedSize(lit.length);
for (Object o : lit) {
literals.add(Evals.asString(o));
}
- DruidExpression druidExpression = DruidExpression.ofVirtualColumn(
- Calcites.getColumnTypeForRelDataType(rexNode.getType()),
- builder,
+ final DruidExpression druidExpression =
toFilteredVirtualColumnExpression(
+ plannerContext,
+ rowSignature,
+ rexNode,
druidExpressions,
- (name, outputType, parser, self) -> new ListFilteredVirtualColumn(
- name,
- self.getArguments().get(0)
- .getSimpleExtraction()
-
.toDimensionSpec(druidExpressions.get(0).getDirectColumn(), outputType),
- literals,
- isAllowList()
- )
+ builder,
+ (name, delegate) -> new ListFilteredVirtualColumn(name, delegate,
literals, isAllowList())
);
-
- // if the join expression VC registry is present, it means that this
expression is part of a join condition
- // and since that's the case, create virtual column here itself for
optimized usage in join matching
- if (plannerContext.getJoinExpressionVirtualColumnRegistry() != null) {
- String virtualColumnName =
plannerContext.getJoinExpressionVirtualColumnRegistry()
-
.getOrCreateVirtualColumnForExpression(
- druidExpression,
- ColumnType.STRING
- );
- return DruidExpression.ofColumn(ColumnType.STRING,
virtualColumnName);
+ if (druidExpression != null) {
+ return druidExpression;
}
-
- return druidExpression;
}
return DruidExpression.ofExpression(ColumnType.STRING, builder,
druidExpressions);
}
}
+ /**
+ * Creates a {@link VirtualColumn} for a multi-value string filtering
function given the {@link DimensionSpec} of the
+ * multi-value string to filter
+ */
+ @FunctionalInterface
+ private interface FilteredVirtualColumnCreator
+ {
+ VirtualColumn create(String name, DimensionSpec delegate);
+ }
+
+ /**
+ * Builds a {@link DruidExpression.NodeType#SPECIALIZED} expression for a
multi-value string filtering function whose
+ * first argument is the multi-value string to filter, if the first argument
can be used as the delegate
+ * {@link DimensionSpec} of a specialized filtering virtual column. Returns
null if it cannot, in which case callers
+ * should fall back to a plain expression.
+ *
+ * @see #toFilterDelegate(PlannerContext, RowSignature, DruidExpression)
+ */
+ @Nullable
+ private static DruidExpression toFilteredVirtualColumnExpression(
+ PlannerContext plannerContext,
+ RowSignature rowSignature,
+ RexNode rexNode,
+ List<DruidExpression> druidExpressions,
+ DruidExpression.ExpressionGenerator builder,
+ FilteredVirtualColumnCreator virtualColumnCreator
+ )
+ {
+ final DruidExpression delegate = toFilterDelegate(plannerContext,
rowSignature, druidExpressions.get(0));
+ if (delegate == null) {
+ return null;
+ }
+ final List<DruidExpression> arguments = new ArrayList<>(druidExpressions);
+ arguments.set(0, delegate);
+
+ final DruidExpression druidExpression = DruidExpression.ofVirtualColumn(
+ Calcites.getColumnTypeForRelDataType(rexNode.getType()),
+ builder,
+ arguments,
+ (name, outputType, parser, self) -> {
+ final DruidExpression arg = self.getArguments().get(0);
+ if (!arg.isSimpleExtraction()) {
+ // the delegate expression was not lifted into its own virtual
column by specialization, so we cannot
+ // reference it by name, fall back to a plain expression virtual
column
+ return self.toExpressionVirtualColumn(name, outputType, parser);
+ }
+ return virtualColumnCreator.create(
+ name,
+ arg.getSimpleExtraction().toDimensionSpec(arg.getDirectColumn(),
outputType)
+ );
+ }
+ );
+
+ // if the join expression VC registry is present, it means that this
expression is part of a join condition
+ // and since that's the case, create virtual column here itself for
optimized usage in join matching
+ if (plannerContext.getJoinExpressionVirtualColumnRegistry() != null) {
+ String virtualColumnName =
plannerContext.getJoinExpressionVirtualColumnRegistry()
+
.getOrCreateVirtualColumnForExpression(
+ druidExpression,
+ ColumnType.STRING
+ );
+ return DruidExpression.ofColumn(ColumnType.STRING, virtualColumnName);
+ }
+
+ return druidExpression;
+ }
+
+ /**
+ * Returns a {@link DruidExpression} suitable to use as the delegate {@link
DimensionSpec} of a specialized multi-value
+ * string filtering virtual column, or null if there is no such expression.
+ *
+ * Simple extractions are used directly. Expressions of a single {@link
ColumnType#STRING} input are marked
+ * {@link DruidExpression#asSpecialized()} so that they are lifted into
their own {@link ExpressionVirtualColumn}
+ * which the filtering virtual column can use as its delegate. When the
input column is dictionary encoded, the
+ * expression virtual column produces a dimension selector backed by the
input column dictionary which defers
+ * expression evaluation until values are looked up, allowing the filtering
virtual column to evaluate the expression
+ * once per dictionary value instead of for every row.
+ */
+ @Nullable
+ private static DruidExpression toFilterDelegate(
+ PlannerContext plannerContext,
+ RowSignature rowSignature,
+ DruidExpression arg
+ )
+ {
+ if (arg.isSimpleExtraction()) {
+ return arg;
+ }
+ // virtual columns created for join conditions are not always carried into
the native query when joins are nested,
+ // so leave expressions in join conditions inline
+ if (plannerContext.getJoinExpressionVirtualColumnRegistry() != null) {
+ return null;
+ }
+ if (arg.getType() == DruidExpression.NodeType.SPECIALIZED ||
!ColumnType.STRING.equals(arg.getDruidType())) {
+ return null;
+ }
+ final Expr expr = plannerContext.parseExpression(arg.getExpression());
+ final Expr.BindingAnalysis bindingAnalysis = expr.analyzeInputs();
+ if (bindingAnalysis.getRequiredBindings().size() != 1
+ || bindingAnalysis.hasInputArrays()
+ || bindingAnalysis.isOutputArray()) {
+ return null;
+ }
+ final String inputColumn =
Iterables.getOnlyElement(bindingAnalysis.getRequiredBindings());
+ if
(!rowSignature.getColumnType(inputColumn).map(ColumnType.STRING::equals).orElse(false))
{
+ return null;
+ }
+ // a plain expression evaluates an empty multi-value row as if it were
null, while the dictionary backed selector of
+ // an expression virtual column evaluates the expression once for each
value in a row, so an empty row produces no
+ // values at all. These are only equivalent if the expression produces
null for a null input, so don't lift
+ // expressions like NVL or COALESCE which replace null values
+ if (expr.eval(InputBindings.nilBindings()).value() != null) {
+ return null;
+ }
+ return arg.asSpecialized();
+ }
+
public static class RegexFilter implements SqlOperatorConversion
{
private static final SqlFunction SQL_FUNCTION = OperatorConversions
@@ -465,31 +574,21 @@ public class MultiValueStringOperatorConversions
}
final DruidExpression.ExpressionGenerator builder = (args) ->
"filter((x) -> regexp_like(x, " +
DruidExpression.stringLiteral(pattern) + "), " + args.get(0).getExpression() +
")";
+ // RegexFilteredVirtualColumn requires the entire value to match the
pattern, while regexp_like matches if the
+ // pattern is found anywhere in the value, so only use the virtual
column for simple extractions, which were
+ // already planned this way, and leave other expressions as plain
expressions to not change their results
if (druidExpressions.get(0).isSimpleExtraction()) {
- DruidExpression druidExpression = DruidExpression.ofVirtualColumn(
- Calcites.getColumnTypeForRelDataType(rexNode.getType()),
- builder,
+ final DruidExpression druidExpression =
toFilteredVirtualColumnExpression(
+ plannerContext,
+ rowSignature,
+ rexNode,
druidExpressions,
- (name, outputType, parser, self) -> new RegexFilteredVirtualColumn(
- name,
- self.getArguments().get(0)
- .getSimpleExtraction()
-
.toDimensionSpec(druidExpressions.get(0).getDirectColumn(), outputType),
- pattern
- )
+ builder,
+ (name, delegate) -> new RegexFilteredVirtualColumn(name, delegate,
pattern)
);
-
- // If in a join context, create the VC immediately
- if (plannerContext.getJoinExpressionVirtualColumnRegistry() != null) {
- String virtualColumnName =
plannerContext.getJoinExpressionVirtualColumnRegistry()
-
.getOrCreateVirtualColumnForExpression(
- druidExpression,
- ColumnType.STRING
- );
- return DruidExpression.ofColumn(ColumnType.STRING,
virtualColumnName);
+ if (druidExpression != null) {
+ return druidExpression;
}
-
- return druidExpression;
}
return DruidExpression.ofExpression(ColumnType.STRING, builder,
druidExpressions);
@@ -550,30 +649,15 @@ public class MultiValueStringOperatorConversions
final DruidExpression.ExpressionGenerator builder = (args) ->
"filter((x) -> substring(x, 0, " + prefix.length() + ") == " +
DruidExpression.stringLiteral(prefix) + ", " + args.get(0).getExpression() +
")";
- if (druidExpressions.get(0).isSimpleExtraction()) {
- DruidExpression druidExpression = DruidExpression.ofVirtualColumn(
- Calcites.getColumnTypeForRelDataType(rexNode.getType()),
- builder,
- druidExpressions,
- (name, outputType, parser, self) -> new
PrefixFilteredVirtualColumn(
- name,
- self.getArguments().get(0)
- .getSimpleExtraction()
-
.toDimensionSpec(druidExpressions.get(0).getDirectColumn(), outputType),
- prefix
- )
- );
-
- // If in a join context, create the VC immediately
- if (plannerContext.getJoinExpressionVirtualColumnRegistry() != null) {
- String virtualColumnName =
plannerContext.getJoinExpressionVirtualColumnRegistry()
-
.getOrCreateVirtualColumnForExpression(
- druidExpression,
- ColumnType.STRING
- );
- return DruidExpression.ofColumn(ColumnType.STRING,
virtualColumnName);
- }
-
+ final DruidExpression druidExpression =
toFilteredVirtualColumnExpression(
+ plannerContext,
+ rowSignature,
+ rexNode,
+ druidExpressions,
+ builder,
+ (name, delegate) -> new PrefixFilteredVirtualColumn(name, delegate,
prefix)
+ );
+ if (druidExpression != null) {
return druidExpression;
}
diff --git
a/sql/src/test/java/org/apache/druid/sql/calcite/CalciteMultiValueStringQueryTest.java
b/sql/src/test/java/org/apache/druid/sql/calcite/CalciteMultiValueStringQueryTest.java
index 9c10b3e284d..e877e0fa46c 100644
---
a/sql/src/test/java/org/apache/druid/sql/calcite/CalciteMultiValueStringQueryTest.java
+++
b/sql/src/test/java/org/apache/druid/sql/calcite/CalciteMultiValueStringQueryTest.java
@@ -43,6 +43,7 @@ import
org.apache.druid.query.groupby.orderby.OrderByColumnSpec;
import org.apache.druid.query.lookup.RegisteredLookupExtractionFn;
import org.apache.druid.query.ordering.StringComparators;
import org.apache.druid.query.scan.ScanQuery;
+import org.apache.druid.segment.VirtualColumns;
import org.apache.druid.segment.column.ColumnType;
import org.apache.druid.segment.virtual.ExpressionVirtualColumn;
import org.apache.druid.segment.virtual.ListFilteredVirtualColumn;
@@ -2114,11 +2115,13 @@ public class CalciteMultiValueStringQueryTest extends
BaseCalciteQueryTest
.setInterval(querySegmentSpec(Filtration.eternity()))
.setGranularity(Granularities.ALL)
.setVirtualColumns(
- expressionVirtualColumn(
+ new ListFilteredVirtualColumn(
"v0",
- "filter((x) -> array_contains(array(null), x),
lookup(\"dim3\",'lookyloo'))",
- ColumnType.STRING
- )
+ DefaultDimensionSpec.of("v1"),
+ Collections.singleton(null),
+ true
+ ),
+ expressionVirtualColumn("v1",
"lookup(\"dim3\",'lookyloo')", ColumnType.STRING)
)
.setDimensions(dimensions(new
DefaultDimensionSpec("v0", "d0", ColumnType.STRING)))
.setAggregatorSpecs(aggregators(new
CountAggregatorFactory("a0")))
@@ -2172,6 +2175,340 @@ public class CalciteMultiValueStringQueryTest extends
BaseCalciteQueryTest
.run();
}
+ @Test
+ public void testMultiValuedFilterOnlyLookup()
+ {
+ testGroupByFilteredMultiValueExpression(
+ "SELECT MV_FILTER_ONLY(LOOKUP(dim3, 'lookyloo'), ARRAY['xa']),
COUNT(*) FROM druid.foo GROUP BY 1",
+ VirtualColumns.create(
+ new ListFilteredVirtualColumn("v0", DefaultDimensionSpec.of("v1"),
ImmutableSet.of("xa"), true),
+ expressionVirtualColumn("v1", "lookup(\"dim3\",'lookyloo')",
ColumnType.STRING)
+ ),
+ ImmutableList.of(
+ new Object[]{null, 5L},
+ new Object[]{"xa", 1L}
+ )
+ );
+ }
+
+ @Test
+ public void testMultiValuedFilterNoneLookup()
+ {
+ testGroupByFilteredMultiValueExpression(
+ "SELECT MV_FILTER_NONE(LOOKUP(dim3, 'lookyloo'), ARRAY['xa']),
COUNT(*) FROM druid.foo GROUP BY 1",
+ VirtualColumns.create(
+ new ListFilteredVirtualColumn("v0", DefaultDimensionSpec.of("v1"),
ImmutableSet.of("xa"), false),
+ expressionVirtualColumn("v1", "lookup(\"dim3\",'lookyloo')",
ColumnType.STRING)
+ ),
+ ImmutableList.of(
+ new Object[]{null, 7L}
+ )
+ );
+ }
+
+ @Test
+ public void testMultiValuedFilterRegexLookup()
+ {
+ // RegexFilteredVirtualColumn requires a full match while the expression
finds the pattern anywhere, so expressions
+ // are not specialized
+ testGroupByFilteredMultiValueExpression(
+ "SELECT MV_FILTER_REGEX(LOOKUP(dim3, 'lookyloo'), '^x.*'), COUNT(*)
FROM druid.foo GROUP BY 1",
+ VirtualColumns.create(
+ expressionVirtualColumn(
+ "v0",
+ "filter((x) -> regexp_like(x, '^x.*'),
lookup(\"dim3\",'lookyloo'))",
+ ColumnType.STRING
+ )
+ ),
+ ImmutableList.of(
+ new Object[]{null, 5L},
+ new Object[]{"xa", 1L}
+ )
+ );
+ }
+
+ @Test
+ public void testMultiValuedFilterPrefixLookup()
+ {
+ testGroupByFilteredMultiValueExpression(
+ "SELECT MV_FILTER_PREFIX(LOOKUP(dim3, 'lookyloo'), 'x'), COUNT(*) FROM
druid.foo GROUP BY 1",
+ VirtualColumns.create(
+ new PrefixFilteredVirtualColumn("v0",
DefaultDimensionSpec.of("v1"), "x"),
+ expressionVirtualColumn("v1", "lookup(\"dim3\",'lookyloo')",
ColumnType.STRING)
+ ),
+ ImmutableList.of(
+ new Object[]{null, 5L},
+ new Object[]{"xa", 1L}
+ )
+ );
+ }
+
+ @Test
+ public void testMultiValuedFilterOnlySingleInputExpression()
+ {
+ testGroupByFilteredMultiValueExpression(
+ "SELECT MV_FILTER_ONLY(UPPER(dim3), ARRAY['B']), COUNT(*) FROM
druid.foo GROUP BY 1",
+ VirtualColumns.create(
+ new ListFilteredVirtualColumn("v0", DefaultDimensionSpec.of("v1"),
ImmutableSet.of("B"), true),
+ expressionVirtualColumn("v1", "upper(\"dim3\")", ColumnType.STRING)
+ ),
+ ImmutableList.of(
+ new Object[]{null, 4L},
+ new Object[]{"B", 2L}
+ )
+ );
+ }
+
+ @Test
+ public void testMultiValuedFilterOnlySubstring()
+ {
+ testGroupByFilteredMultiValueExpression(
+ "SELECT MV_FILTER_ONLY(SUBSTRING(dim3, 1, 1), ARRAY['b']), COUNT(*)
FROM druid.foo GROUP BY 1",
+ VirtualColumns.create(
+ new ListFilteredVirtualColumn("v0", DefaultDimensionSpec.of("v1"),
ImmutableSet.of("b"), true),
+ expressionVirtualColumn("v1", "substring(\"dim3\", 0, 1)",
ColumnType.STRING)
+ ),
+ ImmutableList.of(
+ new Object[]{null, 4L},
+ new Object[]{"b", 2L}
+ )
+ );
+ }
+
+ @Test
+ public void testMultiValuedFilterOnlyRegexpExtract()
+ {
+ testGroupByFilteredMultiValueExpression(
+ "SELECT MV_FILTER_ONLY(REGEXP_EXTRACT(dim3, '^[a-b]'), ARRAY['b']),
COUNT(*) FROM druid.foo GROUP BY 1",
+ VirtualColumns.create(
+ new ListFilteredVirtualColumn("v0", DefaultDimensionSpec.of("v1"),
ImmutableSet.of("b"), true),
+ expressionVirtualColumn("v1", "regexp_extract(\"dim3\",'^[a-b]')",
ColumnType.STRING)
+ ),
+ ImmutableList.of(
+ new Object[]{null, 4L},
+ new Object[]{"b", 2L}
+ )
+ );
+ }
+
+ @Test
+ public void testMultiValuedFilterOnlyNoMatchesIsNullFilter()
+ {
+ // none of the allowed values exist, so the list filtered index has an
empty dictionary
+ testFilteredMultiValueResults(
+ "SELECT COUNT(*) FROM druid.foo WHERE MV_FILTER_ONLY(dim3,
ARRAY['zzz']) IS NULL",
+ ImmutableList.of(new Object[]{6L})
+ );
+ }
+
+ @Test
+ public void testMultiValuedFilterOnlyNoMatchesRangeFilter()
+ {
+ testFilteredMultiValueResults(
+ "SELECT COUNT(*) FROM druid.foo WHERE MV_FILTER_ONLY(dim3,
ARRAY['zzz']) >= 'a'",
+ ImmutableList.of(new Object[]{0L})
+ );
+ }
+
+ @Test
+ public void testMultiValuedFilterOnlyExpressionOnJoinColumnFilter()
+ {
+ testFilteredMultiValueResults(
+ "SELECT COUNT(*) FROM druid.foo f INNER JOIN lookup.lookyloo l ON
f.dim2 = l.k "
+ + "WHERE MV_FILTER_ONLY(UPPER(l.v), ARRAY['XA']) = 'XA'",
+ ImmutableList.of(new Object[]{2L})
+ );
+ }
+
+ @Test
+ public void testMultiValuedFilterOnlyExpressionInNestedJoinCondition()
+ {
+ testVectorizableFilteredMultiValueResults(
+ "SELECT f.dim1, l.v, l2.v FROM druid.foo f "
+ + "INNER JOIN lookup.lookyloo l ON MV_FILTER_ONLY(LOWER(f.dim3),
ARRAY['a']) = l.k "
+ + "INNER JOIN lookup.lookyloo l2 ON f.dim2 = l2.k",
+ ImmutableList.of(new Object[]{"", "xa", "xa"})
+ );
+ }
+
+ @Test
+ public void testMultiValuedFilterOnlyExpressionOnJoinColumnInJoinCondition()
+ {
+ testVectorizableFilteredMultiValueResults(
+ "SELECT f.dim1, l.v, l2.v FROM druid.foo f "
+ + "INNER JOIN lookup.lookyloo l ON f.dim2 = l.k "
+ + "INNER JOIN lookup.lookyloo l2 ON MV_FILTER_ONLY(SUBSTRING(l.v, 2),
ARRAY['a']) = l2.k",
+ ImmutableList.of(
+ new Object[]{"", "xa", "xa"},
+ new Object[]{"1", "xa", "xa"}
+ )
+ );
+ }
+
+ @Test
+ public void testMultiValuedFilterRegexExpressionNotSpecialized()
+ {
+ // expressions are not specialized for MV_FILTER_REGEX, so the pattern is
still found anywhere in the value and
+ // 'abc' matches
+ testFilteredMultiValueResults(
+ "SELECT MV_FILTER_REGEX(LOWER(dim2), '^a'), COUNT(*) FROM druid.foo
GROUP BY 1",
+ ImmutableList.of(
+ new Object[]{null, 3L},
+ new Object[]{"a", 2L},
+ new Object[]{"abc", 1L}
+ )
+ );
+ }
+
+ @Test
+ public void testMultiValuedFilterOnlyNvlNotSpecialized()
+ {
+ // NVL replaces nulls, so an empty row is not equivalent to a null row and
the expression is not specialized
+ cannotVectorize();
+ testBuilder()
+ .sql("SELECT MV_FILTER_ONLY(NVL(dim3, 'x'), ARRAY['x']), COUNT(*) FROM
druid.foo GROUP BY 1")
+ .expectedQuery(
+ GroupByQuery.builder()
+ .setDataSource(CalciteTests.DATASOURCE1)
+ .setInterval(querySegmentSpec(Filtration.eternity()))
+ .setGranularity(Granularities.ALL)
+ .setVirtualColumns(
+ expressionVirtualColumn(
+ "v0",
+ "filter((x) -> array_contains(array('x'), x),
nvl(\"dim3\",'x'))",
+ ColumnType.STRING
+ )
+ )
+ .setDimensions(dimensions(new
DefaultDimensionSpec("v0", "d0", ColumnType.STRING)))
+ .setAggregatorSpecs(aggregators(new
CountAggregatorFactory("a0")))
+ .setContext(QUERY_CONTEXT_DEFAULT)
+ .build()
+ )
+ .expectedResults(
+ ImmutableList.of(
+ new Object[]{null, 4L},
+ new Object[]{"x", 2L}
+ )
+ )
+ .run();
+ testFilteredMultiValueResults(
+ "SELECT MV_FILTER_ONLY(NVL(dim3, 'x'), ARRAY['x', 'a']), COUNT(*) FROM
(SELECT dim3 FROM druid.foo LIMIT 10) GROUP BY 1",
+ ImmutableList.of(
+ new Object[]{null, 3L},
+ new Object[]{"a", 1L},
+ new Object[]{"x", 2L}
+ )
+ );
+ }
+
+ @Test
+ public void testMultiValuedFilterOnlySpecializedInsideExpression()
+ {
+ // COALESCE applies to each value of the filtered multi-value string, the
same as when filtering a column directly,
+ // so rows where everything was filtered are null instead of 'none'
+ testFilteredMultiValueResults(
+ "SELECT COALESCE(MV_FILTER_ONLY(UPPER(dim3), ARRAY['B']), 'none'),
COUNT(*) FROM druid.foo GROUP BY 1",
+ ImmutableList.of(
+ new Object[]{null, 4L},
+ new Object[]{"B", 2L}
+ )
+ );
+ testFilteredMultiValueResults(
+ "SELECT COALESCE(MV_FILTER_ONLY(dim3, ARRAY['b']), 'none'), COUNT(*)
FROM druid.foo GROUP BY 1",
+ ImmutableList.of(
+ new Object[]{null, 4L},
+ new Object[]{"b", 2L}
+ )
+ );
+ }
+
+ @Test
+ public void testMultiValuedFilterNoneLookupTopN()
+ {
+ // topN does not count empty multi-value rows, the same as when filtering
a column directly
+ testVectorizableFilteredMultiValueResults(
+ "SELECT MV_FILTER_NONE(LOOKUP(dim3, 'lookyloo'), ARRAY['xa']),
COUNT(*) FROM druid.foo "
+ + "GROUP BY 1 ORDER BY 2 DESC LIMIT 2",
+ ImmutableList.of(new Object[]{null, 6L})
+ );
+ }
+
+ @Test
+ public void testMultiValuedFilterExpressionOnSingleValuedColumnFilter()
+ {
+ // filtering a single valued column can leave rows with no values, which
must still be treated as null even
+ // though the expression delegate has no indexes to answer the filter
+ testFilteredMultiValueResults(
+ "SELECT COUNT(*) FROM druid.foo WHERE MV_FILTER_ONLY(UPPER(dim1),
ARRAY['ABC']) IS NULL",
+ ImmutableList.of(new Object[]{5L})
+ );
+ testFilteredMultiValueResults(
+ "SELECT COUNT(*) FROM druid.foo WHERE MV_FILTER_ONLY(UPPER(dim1),
ARRAY['ABC']) = 'ABC'",
+ ImmutableList.of(new Object[]{1L})
+ );
+ testFilteredMultiValueResults(
+ "SELECT COUNT(*) FROM druid.foo WHERE MV_FILTER_PREFIX(UPPER(dim1),
'AB') IS NULL",
+ ImmutableList.of(new Object[]{5L})
+ );
+ }
+
+ @Test
+ public void testMultiValuedFilterOnlyMultiInputExpression()
+ {
+ // expressions with more than one input cannot use a dictionary backed
delegate, so plan to a plain expression
+ testGroupByFilteredMultiValueExpression(
+ "SELECT MV_FILTER_ONLY(CONCAT(dim3, dim2), ARRAY['aa']), COUNT(*) FROM
druid.foo GROUP BY 1",
+ VirtualColumns.create(
+ expressionVirtualColumn(
+ "v0",
+ "filter((x) -> array_contains(array('aa'), x),
concat(\"dim3\",\"dim2\"))",
+ ColumnType.STRING
+ )
+ ),
+ ImmutableList.of(
+ new Object[]{null, 5L},
+ new Object[]{"aa", 1L}
+ )
+ );
+ }
+
+ @Test
+ public void testMultiValuedFilterOnlyLookupFilter()
+ {
+ cannotVectorize();
+ final String sql = "SELECT COUNT(*) FROM druid.foo WHERE
MV_FILTER_ONLY(LOOKUP(dim3, 'lookyloo'), ARRAY['xa']) = 'xa'";
+ final List<Object[]> expectedResults = ImmutableList.of(new Object[]{1L});
+ testBuilder()
+ .sql(sql)
+ .expectedQuery(
+ Druids.newTimeseriesQueryBuilder()
+ .dataSource(CalciteTests.DATASOURCE1)
+ .intervals(querySegmentSpec(Filtration.eternity()))
+ .granularity(Granularities.ALL)
+ .virtualColumns(
+ new ListFilteredVirtualColumn("v0",
DefaultDimensionSpec.of("v1"), ImmutableSet.of("xa"), true),
+ expressionVirtualColumn("v1",
"lookup(\"dim3\",'lookyloo')", ColumnType.STRING)
+ )
+ .filters(equality("v0", "xa", ColumnType.STRING))
+ .aggregators(aggregators(new CountAggregatorFactory("a0")))
+ .context(QUERY_CONTEXT_DEFAULT)
+ .build()
+ )
+ .expectedResults(expectedResults)
+ .run();
+
+ testBuilder()
+ .sql(sql)
+ .queryContext(
+ QueryContexts.override(
+ QUERY_CONTEXT_DEFAULT,
+
QueryContext.ofMap(QueryContextParameters.SQL_USE_EXTRACTION_FNS, true)
+ )
+ )
+ .expectedResults(expectedResults)
+ .run();
+ }
+
@Test
public void testMvContainsFilterWithExtractionFn()
{
@@ -2632,4 +2969,55 @@ public class CalciteMultiValueStringQueryTest extends
BaseCalciteQueryTest
)
.run();
}
+
+ /**
+ * Tests a query grouping on a multi-value string filtering function,
verifying the native query using the default
+ * query context, and that results match those of the same query with
+ * {@link QueryContextParameters#SQL_USE_EXTRACTION_FNS} enabled.
+ */
+ private void testGroupByFilteredMultiValueExpression(
+ String sql,
+ VirtualColumns expectedVirtualColumns,
+ List<Object[]> expectedResults
+ )
+ {
+ cannotVectorize();
+ testBuilder()
+ .sql(sql)
+ .expectedQuery(
+ GroupByQuery.builder()
+ .setDataSource(CalciteTests.DATASOURCE1)
+ .setInterval(querySegmentSpec(Filtration.eternity()))
+ .setGranularity(Granularities.ALL)
+ .setVirtualColumns(expectedVirtualColumns)
+ .setDimensions(dimensions(new
DefaultDimensionSpec("v0", "d0", ColumnType.STRING)))
+ .setAggregatorSpecs(aggregators(new
CountAggregatorFactory("a0")))
+ .setContext(QUERY_CONTEXT_DEFAULT)
+ .build()
+ )
+ .expectedResults(expectedResults)
+ .run();
+
+ testBuilder()
+ .sql(sql)
+ .queryContext(
+ QueryContexts.override(
+ QUERY_CONTEXT_DEFAULT,
+
QueryContext.ofMap(QueryContextParameters.SQL_USE_EXTRACTION_FNS, true)
+ )
+ )
+ .expectedResults(expectedResults)
+ .run();
+ }
+
+ private void testFilteredMultiValueResults(String sql, List<Object[]>
expectedResults)
+ {
+ cannotVectorize();
+ testBuilder().sql(sql).expectedResults(expectedResults).run();
+ }
+
+ private void testVectorizableFilteredMultiValueResults(String sql,
List<Object[]> expectedResults)
+ {
+ testBuilder().sql(sql).expectedResults(expectedResults).run();
+ }
}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
