This is an automated email from the ASF dual-hosted git repository.
jackie pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/pinot.git
The following commit(s) were added to refs/heads/master by this push:
new f951cce Support ID_SET aggregation on MV columns (#7355)
f951cce is described below
commit f951cce796839ca62185fd6a26d1838096731b25
Author: Xiaotian (Jackie) Jiang <17555551+jackie-ji...@users.noreply.github.com>
AuthorDate: Tue Aug 24 09:53:43 2021 -0700
Support ID_SET aggregation on MV columns (#7355)
Support ID_SET aggregation function on MV columns, which collects all the
entries into the IdSet.
---
.../function/IdSetAggregationFunction.java | 428 ++++++++++++++-------
.../org/apache/pinot/queries/IdSetQueriesTest.java | 153 +++++++-
2 files changed, 437 insertions(+), 144 deletions(-)
diff --git
a/pinot-core/src/main/java/org/apache/pinot/core/query/aggregation/function/IdSetAggregationFunction.java
b/pinot-core/src/main/java/org/apache/pinot/core/query/aggregation/function/IdSetAggregationFunction.java
index b2ed594..a72882c 100644
---
a/pinot-core/src/main/java/org/apache/pinot/core/query/aggregation/function/IdSetAggregationFunction.java
+++
b/pinot-core/src/main/java/org/apache/pinot/core/query/aggregation/function/IdSetAggregationFunction.java
@@ -37,8 +37,8 @@ import org.apache.pinot.spi.data.FieldSpec.DataType;
/**
- * The {@code IdSetAggregationFunction} collects the values for a given
single-value expression into an IdSet, which can
- * be used in the second query to optimize the query with huge IN clause
generated from another query.
+ * The {@code IdSetAggregationFunction} collects the values for the given
expression into an IdSet, which can be used in
+ * the second query to optimize the query with huge IN clause generated from
another query.
* <p>The generated IdSet can be backed by RoaringBitmap,
Roaring64NavigableMap or BloomFilter based on type of the ids
* and the function parameters.
* <p>The function takes an optional second argument as the parameters for the
function. There are 3 parameters for the
@@ -131,45 +131,92 @@ public class IdSetAggregationFunction extends
BaseSingleInputAggregationFunction
BlockValSet blockValSet = blockValSetMap.get(_expression);
DataType storedType = blockValSet.getValueType().getStoredType();
IdSet idSet = getIdSet(aggregationResultHolder, storedType);
- switch (storedType) {
- case INT:
- int[] intValues = blockValSet.getIntValuesSV();
- for (int i = 0; i < length; i++) {
- idSet.add(intValues[i]);
- }
- break;
- case LONG:
- long[] longValues = blockValSet.getLongValuesSV();
- for (int i = 0; i < length; i++) {
- idSet.add(longValues[i]);
- }
- break;
- case FLOAT:
- float[] floatValues = blockValSet.getFloatValuesSV();
- for (int i = 0; i < length; i++) {
- idSet.add(floatValues[i]);
- }
- break;
- case DOUBLE:
- double[] doubleValues = blockValSet.getDoubleValuesSV();
- for (int i = 0; i < length; i++) {
- idSet.add(doubleValues[i]);
- }
- break;
- case STRING:
- String[] stringValues = blockValSet.getStringValuesSV();
- for (int i = 0; i < length; i++) {
- idSet.add(stringValues[i]);
- }
- break;
- case BYTES:
- byte[][] bytesValues = blockValSet.getBytesValuesSV();
- for (int i = 0; i < length; i++) {
- idSet.add(bytesValues[i]);
- }
- break;
- default:
- throw new IllegalStateException("Illegal data type for ID_SET
aggregation function: " + storedType);
+ if (blockValSet.isSingleValue()) {
+ switch (storedType) {
+ case INT:
+ int[] intValuesSV = blockValSet.getIntValuesSV();
+ for (int i = 0; i < length; i++) {
+ idSet.add(intValuesSV[i]);
+ }
+ break;
+ case LONG:
+ long[] longValuesSV = blockValSet.getLongValuesSV();
+ for (int i = 0; i < length; i++) {
+ idSet.add(longValuesSV[i]);
+ }
+ break;
+ case FLOAT:
+ float[] floatValuesSV = blockValSet.getFloatValuesSV();
+ for (int i = 0; i < length; i++) {
+ idSet.add(floatValuesSV[i]);
+ }
+ break;
+ case DOUBLE:
+ double[] doubleValuesSV = blockValSet.getDoubleValuesSV();
+ for (int i = 0; i < length; i++) {
+ idSet.add(doubleValuesSV[i]);
+ }
+ break;
+ case STRING:
+ String[] stringValuesSV = blockValSet.getStringValuesSV();
+ for (int i = 0; i < length; i++) {
+ idSet.add(stringValuesSV[i]);
+ }
+ break;
+ case BYTES:
+ byte[][] bytesValuesSV = blockValSet.getBytesValuesSV();
+ for (int i = 0; i < length; i++) {
+ idSet.add(bytesValuesSV[i]);
+ }
+ break;
+ default:
+ throw new IllegalStateException("Illegal SV data type for ID_SET
aggregation function: " + storedType);
+ }
+ } else {
+ switch (storedType) {
+ case INT:
+ int[][] intValuesMV = blockValSet.getIntValuesMV();
+ for (int i = 0; i < length; i++) {
+ for (int intValue : intValuesMV[i]) {
+ idSet.add(intValue);
+ }
+ }
+ break;
+ case LONG:
+ long[][] longValuesMV = blockValSet.getLongValuesMV();
+ for (int i = 0; i < length; i++) {
+ for (long longValue : longValuesMV[i]) {
+ idSet.add(longValue);
+ }
+ }
+ break;
+ case FLOAT:
+ float[][] floatValuesMV = blockValSet.getFloatValuesMV();
+ for (int i = 0; i < length; i++) {
+ for (float floatValue : floatValuesMV[i]) {
+ idSet.add(floatValue);
+ }
+ }
+ break;
+ case DOUBLE:
+ double[][] doubleValuesMV = blockValSet.getDoubleValuesMV();
+ for (int i = 0; i < length; i++) {
+ for (double doubleValue : doubleValuesMV[i]) {
+ idSet.add(doubleValue);
+ }
+ }
+ break;
+ case STRING:
+ String[][] stringValuesMV = blockValSet.getStringValuesMV();
+ for (int i = 0; i < length; i++) {
+ for (String stringValue : stringValuesMV[i]) {
+ idSet.add(stringValue);
+ }
+ }
+ break;
+ default:
+ throw new IllegalStateException("Illegal MV data type for ID_SET
aggregation function: " + storedType);
+ }
}
}
@@ -178,45 +225,97 @@ public class IdSetAggregationFunction extends
BaseSingleInputAggregationFunction
Map<ExpressionContext, BlockValSet> blockValSetMap) {
BlockValSet blockValSet = blockValSetMap.get(_expression);
DataType storedType = blockValSet.getValueType().getStoredType();
- switch (storedType) {
- case INT:
- int[] intValues = blockValSet.getIntValuesSV();
- for (int i = 0; i < length; i++) {
- getIdSet(groupByResultHolder, groupKeyArray[i],
DataType.INT).add(intValues[i]);
- }
- break;
- case LONG:
- long[] longValues = blockValSet.getLongValuesSV();
- for (int i = 0; i < length; i++) {
- getIdSet(groupByResultHolder, groupKeyArray[i],
DataType.LONG).add(longValues[i]);
- }
- break;
- case FLOAT:
- float[] floatValues = blockValSet.getFloatValuesSV();
- for (int i = 0; i < length; i++) {
- getIdSet(groupByResultHolder, groupKeyArray[i],
DataType.FLOAT).add(floatValues[i]);
- }
- break;
- case DOUBLE:
- double[] doubleValues = blockValSet.getDoubleValuesSV();
- for (int i = 0; i < length; i++) {
- getIdSet(groupByResultHolder, groupKeyArray[i],
DataType.DOUBLE).add(doubleValues[i]);
- }
- break;
- case STRING:
- String[] stringValues = blockValSet.getStringValuesSV();
- for (int i = 0; i < length; i++) {
- getIdSet(groupByResultHolder, groupKeyArray[i],
DataType.STRING).add(stringValues[i]);
- }
- break;
- case BYTES:
- byte[][] bytesValues = blockValSet.getBytesValuesSV();
- for (int i = 0; i < length; i++) {
- getIdSet(groupByResultHolder, groupKeyArray[i],
DataType.BYTES).add(bytesValues[i]);
- }
- break;
- default:
- throw new IllegalStateException("Illegal data type for ID_SET
aggregation function: " + storedType);
+ if (blockValSet.isSingleValue()) {
+ switch (storedType) {
+ case INT:
+ int[] intValuesSV = blockValSet.getIntValuesSV();
+ for (int i = 0; i < length; i++) {
+ getIdSet(groupByResultHolder, groupKeyArray[i],
DataType.INT).add(intValuesSV[i]);
+ }
+ break;
+ case LONG:
+ long[] longValuesSV = blockValSet.getLongValuesSV();
+ for (int i = 0; i < length; i++) {
+ getIdSet(groupByResultHolder, groupKeyArray[i],
DataType.LONG).add(longValuesSV[i]);
+ }
+ break;
+ case FLOAT:
+ float[] floatValuesSV = blockValSet.getFloatValuesSV();
+ for (int i = 0; i < length; i++) {
+ getIdSet(groupByResultHolder, groupKeyArray[i],
DataType.FLOAT).add(floatValuesSV[i]);
+ }
+ break;
+ case DOUBLE:
+ double[] doubleValuesSV = blockValSet.getDoubleValuesSV();
+ for (int i = 0; i < length; i++) {
+ getIdSet(groupByResultHolder, groupKeyArray[i],
DataType.DOUBLE).add(doubleValuesSV[i]);
+ }
+ break;
+ case STRING:
+ String[] stringValuesSV = blockValSet.getStringValuesSV();
+ for (int i = 0; i < length; i++) {
+ getIdSet(groupByResultHolder, groupKeyArray[i],
DataType.STRING).add(stringValuesSV[i]);
+ }
+ break;
+ case BYTES:
+ byte[][] bytesValuesSV = blockValSet.getBytesValuesSV();
+ for (int i = 0; i < length; i++) {
+ getIdSet(groupByResultHolder, groupKeyArray[i],
DataType.BYTES).add(bytesValuesSV[i]);
+ }
+ break;
+ default:
+ throw new IllegalStateException("Illegal SV data type for ID_SET
aggregation function: " + storedType);
+ }
+ } else {
+ switch (storedType) {
+ case INT:
+ int[][] intValuesMV = blockValSet.getIntValuesMV();
+ for (int i = 0; i < length; i++) {
+ IdSet idSet = getIdSet(groupByResultHolder, groupKeyArray[i],
DataType.INT);
+ for (int intValue : intValuesMV[i]) {
+ idSet.add(intValue);
+ }
+ }
+ break;
+ case LONG:
+ long[][] longValuesMV = blockValSet.getLongValuesMV();
+ for (int i = 0; i < length; i++) {
+ IdSet idSet = getIdSet(groupByResultHolder, groupKeyArray[i],
DataType.LONG);
+ for (long longValue : longValuesMV[i]) {
+ idSet.add(longValue);
+ }
+ }
+ break;
+ case FLOAT:
+ float[][] floatValuesMV = blockValSet.getFloatValuesMV();
+ for (int i = 0; i < length; i++) {
+ IdSet idSet = getIdSet(groupByResultHolder, groupKeyArray[i],
DataType.FLOAT);
+ for (float floatValue : floatValuesMV[i]) {
+ idSet.add(floatValue);
+ }
+ }
+ break;
+ case DOUBLE:
+ double[][] doubleValuesMV = blockValSet.getDoubleValuesMV();
+ for (int i = 0; i < length; i++) {
+ IdSet idSet = getIdSet(groupByResultHolder, groupKeyArray[i],
DataType.DOUBLE);
+ for (double doubleValue : doubleValuesMV[i]) {
+ idSet.add(doubleValue);
+ }
+ }
+ break;
+ case STRING:
+ String[][] stringValuesMV = blockValSet.getStringValuesMV();
+ for (int i = 0; i < length; i++) {
+ IdSet idSet = getIdSet(groupByResultHolder, groupKeyArray[i],
DataType.STRING);
+ for (String stringValue : stringValuesMV[i]) {
+ idSet.add(stringValue);
+ }
+ }
+ break;
+ default:
+ throw new IllegalStateException("Illegal MV data type for ID_SET
aggregation function: " + storedType);
+ }
}
}
@@ -225,63 +324,130 @@ public class IdSetAggregationFunction extends
BaseSingleInputAggregationFunction
Map<ExpressionContext, BlockValSet> blockValSetMap) {
BlockValSet blockValSet = blockValSetMap.get(_expression);
DataType storedType = blockValSet.getValueType().getStoredType();
- switch (storedType) {
- case INT:
- int[] intValues = blockValSet.getIntValuesSV();
- for (int i = 0; i < length; i++) {
- int intValue = intValues[i];
- for (int groupKey : groupKeysArray[i]) {
- getIdSet(groupByResultHolder, groupKey,
DataType.INT).add(intValue);
+ if (blockValSet.isSingleValue()) {
+ switch (storedType) {
+ case INT:
+ int[] intValuesSV = blockValSet.getIntValuesSV();
+ for (int i = 0; i < length; i++) {
+ int intValue = intValuesSV[i];
+ for (int groupKey : groupKeysArray[i]) {
+ getIdSet(groupByResultHolder, groupKey,
DataType.INT).add(intValue);
+ }
}
- }
- break;
- case LONG:
- long[] longValues = blockValSet.getLongValuesSV();
- for (int i = 0; i < length; i++) {
- long longValue = longValues[i];
- for (int groupKey : groupKeysArray[i]) {
- getIdSet(groupByResultHolder, groupKey,
DataType.LONG).add(longValue);
+ break;
+ case LONG:
+ long[] longValuesSV = blockValSet.getLongValuesSV();
+ for (int i = 0; i < length; i++) {
+ long longValue = longValuesSV[i];
+ for (int groupKey : groupKeysArray[i]) {
+ getIdSet(groupByResultHolder, groupKey,
DataType.LONG).add(longValue);
+ }
}
- }
- break;
- case FLOAT:
- float[] floatValues = blockValSet.getFloatValuesSV();
- for (int i = 0; i < length; i++) {
- float floatValue = floatValues[i];
- for (int groupKey : groupKeysArray[i]) {
- getIdSet(groupByResultHolder, groupKey,
DataType.FLOAT).add(floatValue);
+ break;
+ case FLOAT:
+ float[] floatValuesSV = blockValSet.getFloatValuesSV();
+ for (int i = 0; i < length; i++) {
+ float floatValue = floatValuesSV[i];
+ for (int groupKey : groupKeysArray[i]) {
+ getIdSet(groupByResultHolder, groupKey,
DataType.FLOAT).add(floatValue);
+ }
}
- }
- break;
- case DOUBLE:
- double[] doubleValues = blockValSet.getDoubleValuesSV();
- for (int i = 0; i < length; i++) {
- double doubleValue = doubleValues[i];
- for (int groupKey : groupKeysArray[i]) {
- getIdSet(groupByResultHolder, groupKey,
DataType.DOUBLE).add(doubleValue);
+ break;
+ case DOUBLE:
+ double[] doubleValuesSV = blockValSet.getDoubleValuesSV();
+ for (int i = 0; i < length; i++) {
+ double doubleValue = doubleValuesSV[i];
+ for (int groupKey : groupKeysArray[i]) {
+ getIdSet(groupByResultHolder, groupKey,
DataType.DOUBLE).add(doubleValue);
+ }
}
- }
- break;
- case STRING:
- String[] stringValues = blockValSet.getStringValuesSV();
- for (int i = 0; i < length; i++) {
- String stringValue = stringValues[i];
- for (int groupKey : groupKeysArray[i]) {
- getIdSet(groupByResultHolder, groupKey,
DataType.STRING).add(stringValue);
+ break;
+ case STRING:
+ String[] stringValuesSV = blockValSet.getStringValuesSV();
+ for (int i = 0; i < length; i++) {
+ String stringValue = stringValuesSV[i];
+ for (int groupKey : groupKeysArray[i]) {
+ getIdSet(groupByResultHolder, groupKey,
DataType.STRING).add(stringValue);
+ }
}
- }
- break;
- case BYTES:
- byte[][] bytesValues = blockValSet.getBytesValuesSV();
- for (int i = 0; i < length; i++) {
- byte[] bytesValue = bytesValues[i];
- for (int groupKey : groupKeysArray[i]) {
- getIdSet(groupByResultHolder, groupKey,
DataType.BYTES).add(bytesValue);
+ break;
+ case BYTES:
+ byte[][] bytesValuesSV = blockValSet.getBytesValuesSV();
+ for (int i = 0; i < length; i++) {
+ byte[] bytesValue = bytesValuesSV[i];
+ for (int groupKey : groupKeysArray[i]) {
+ getIdSet(groupByResultHolder, groupKey,
DataType.BYTES).add(bytesValue);
+ }
}
- }
- break;
- default:
- throw new IllegalStateException("Illegal data type for ID_SET
aggregation function: " + storedType);
+ break;
+ default:
+ throw new IllegalStateException("Illegal SV data type for ID_SET
aggregation function: " + storedType);
+ }
+ } else {
+ switch (storedType) {
+ case INT:
+ int[][] intValuesMV = blockValSet.getIntValuesMV();
+ for (int i = 0; i < length; i++) {
+ int[] intValues = intValuesMV[i];
+ for (int groupKey : groupKeysArray[i]) {
+ IdSet idSet = getIdSet(groupByResultHolder, groupKey,
DataType.INT);
+ for (int intValue : intValues) {
+ idSet.add(intValue);
+ }
+ }
+ }
+ break;
+ case LONG:
+ long[][] longValuesMV = blockValSet.getLongValuesMV();
+ for (int i = 0; i < length; i++) {
+ long[] longValues = longValuesMV[i];
+ for (int groupKey : groupKeysArray[i]) {
+ IdSet idSet = getIdSet(groupByResultHolder, groupKey,
DataType.LONG);
+ for (long longValue : longValues) {
+ idSet.add(longValue);
+ }
+ }
+ }
+ break;
+ case FLOAT:
+ float[][] floatValuesMV = blockValSet.getFloatValuesMV();
+ for (int i = 0; i < length; i++) {
+ float[] floatValues = floatValuesMV[i];
+ for (int groupKey : groupKeysArray[i]) {
+ IdSet idSet = getIdSet(groupByResultHolder, groupKey,
DataType.FLOAT);
+ for (float floatValue : floatValues) {
+ idSet.add(floatValue);
+ }
+ }
+ }
+ break;
+ case DOUBLE:
+ double[][] doubleValuesMV = blockValSet.getDoubleValuesMV();
+ for (int i = 0; i < length; i++) {
+ double[] doubleValues = doubleValuesMV[i];
+ for (int groupKey : groupKeysArray[i]) {
+ IdSet idSet = getIdSet(groupByResultHolder, groupKey,
DataType.DOUBLE);
+ for (double doubleValue : doubleValues) {
+ idSet.add(doubleValue);
+ }
+ }
+ }
+ break;
+ case STRING:
+ String[][] stringValuesMV = blockValSet.getStringValuesMV();
+ for (int i = 0; i < length; i++) {
+ String[] stringValues = stringValuesMV[i];
+ for (int groupKey : groupKeysArray[i]) {
+ IdSet idSet = getIdSet(groupByResultHolder, groupKey,
DataType.STRING);
+ for (String stringValue : stringValues) {
+ idSet.add(stringValue);
+ }
+ }
+ }
+ break;
+ default:
+ throw new IllegalStateException("Illegal MV data type for ID_SET
aggregation function: " + storedType);
+ }
}
}
diff --git
a/pinot-core/src/test/java/org/apache/pinot/queries/IdSetQueriesTest.java
b/pinot-core/src/test/java/org/apache/pinot/queries/IdSetQueriesTest.java
index da5c957..1d7ed5c 100644
--- a/pinot-core/src/test/java/org/apache/pinot/queries/IdSetQueriesTest.java
+++ b/pinot-core/src/test/java/org/apache/pinot/queries/IdSetQueriesTest.java
@@ -82,10 +82,19 @@ public class IdSetQueriesTest extends BaseQueriesTest {
private static final String DOUBLE_COLUMN = "doubleColumn";
private static final String STRING_COLUMN = "stringColumn";
private static final String BYTES_COLUMN = "bytesColumn";
+ private static final String INT_MV_COLUMN = "intMVColumn";
+ private static final String LONG_MV_COLUMN = "longMVColumn";
+ private static final String FLOAT_MV_COLUMN = "floatMVColumn";
+ private static final String DOUBLE_MV_COLUMN = "doubleMVColumn";
+ private static final String STRING_MV_COLUMN = "stringMVColumn";
private static final Schema SCHEMA = new
Schema.SchemaBuilder().addSingleValueDimension(INT_COLUMN, DataType.INT)
.addSingleValueDimension(LONG_COLUMN,
DataType.LONG).addSingleValueDimension(FLOAT_COLUMN, DataType.FLOAT)
.addSingleValueDimension(DOUBLE_COLUMN,
DataType.DOUBLE).addSingleValueDimension(STRING_COLUMN, DataType.STRING)
- .addSingleValueDimension(BYTES_COLUMN, DataType.BYTES).build();
+ .addSingleValueDimension(BYTES_COLUMN,
DataType.BYTES).addMultiValueDimension(INT_MV_COLUMN, DataType.INT)
+ .addMultiValueDimension(LONG_MV_COLUMN,
DataType.LONG).addMultiValueDimension(FLOAT_MV_COLUMN, DataType.FLOAT)
+ .addMultiValueDimension(DOUBLE_MV_COLUMN, DataType.DOUBLE)
+ .addMultiValueDimension(STRING_MV_COLUMN, DataType.STRING)
+ .build();
private static final TableConfig TABLE_CONFIG =
new
TableConfigBuilder(TableType.OFFLINE).setTableName(RAW_TABLE_NAME).build();
@@ -130,6 +139,11 @@ public class IdSetQueriesTest extends BaseQueriesTest {
record.putValue(DOUBLE_COLUMN, doubleValue);
record.putValue(STRING_COLUMN, stringValue);
record.putValue(BYTES_COLUMN, bytesValue);
+ record.putValue(INT_MV_COLUMN, new Integer[]{intValue, intValue +
MAX_VALUE});
+ record.putValue(LONG_MV_COLUMN, new Long[]{longValue, longValue +
MAX_VALUE});
+ record.putValue(FLOAT_MV_COLUMN, new Float[]{floatValue, floatValue +
MAX_VALUE});
+ record.putValue(DOUBLE_MV_COLUMN, new Double[]{doubleValue, doubleValue
+ MAX_VALUE});
+ record.putValue(STRING_MV_COLUMN, new String[]{stringValue, stringValue
+ MAX_VALUE});
records.add(record);
}
@@ -152,7 +166,8 @@ public class IdSetQueriesTest extends BaseQueriesTest {
throws IOException {
String query =
"SELECT IDSET(intColumn), IDSET(longColumn), IDSET(floatColumn),
IDSET(doubleColumn), IDSET(stringColumn), "
- + "IDSET(bytesColumn) FROM testTable";
+ + "IDSET(bytesColumn), IDSET(intMVColumn), IDSET(longMVColumn),
IDSET(floatMVColumn), "
+ + "IDSET(doubleMVColumn), IDSET(stringMVColumn) FROM testTable";
// Inner segment
{
@@ -160,10 +175,10 @@ public class IdSetQueriesTest extends BaseQueriesTest {
AggregationOperator aggregationOperator = getOperatorForPqlQuery(query);
IntermediateResultsBlock resultsBlock = aggregationOperator.nextBlock();
QueriesTestUtils.testInnerSegmentExecutionStatistics(aggregationOperator.getExecutionStatistics(),
NUM_RECORDS, 0,
- 6 * NUM_RECORDS, NUM_RECORDS);
+ 11 * NUM_RECORDS, NUM_RECORDS);
List<Object> aggregationResult = resultsBlock.getAggregationResult();
assertNotNull(aggregationResult);
- assertEquals(aggregationResult.size(), 6);
+ assertEquals(aggregationResult.size(), 11);
RoaringBitmapIdSet intIdSet = (RoaringBitmapIdSet)
aggregationResult.get(0);
for (int i = 0; i < NUM_RECORDS; i++) {
assertTrue(intIdSet.contains(_values[i]));
@@ -188,6 +203,31 @@ public class IdSetQueriesTest extends BaseQueriesTest {
for (int i = 0; i < NUM_RECORDS; i++) {
assertTrue(bytesIdSet.contains(Integer.toString(_values[i]).getBytes()));
}
+ RoaringBitmapIdSet intMVIdSet = (RoaringBitmapIdSet)
aggregationResult.get(6);
+ for (int i = 0; i < NUM_RECORDS; i++) {
+ assertTrue(intMVIdSet.contains(_values[i]));
+ assertTrue(intMVIdSet.contains(_values[i] + MAX_VALUE));
+ }
+ Roaring64NavigableMapIdSet longMVIdSet = (Roaring64NavigableMapIdSet)
aggregationResult.get(7);
+ for (int i = 0; i < NUM_RECORDS; i++) {
+ assertTrue(longMVIdSet.contains(_values[i] + (long)
Integer.MAX_VALUE));
+ assertTrue(longMVIdSet.contains(_values[i] + (long) Integer.MAX_VALUE
+ MAX_VALUE));
+ }
+ BloomFilterIdSet floatMVIdSet = (BloomFilterIdSet)
aggregationResult.get(8);
+ for (int i = 0; i < NUM_RECORDS; i++) {
+ assertTrue(floatMVIdSet.contains(_values[i] + 0.5f));
+ assertTrue(floatMVIdSet.contains(_values[i] + 0.5f + MAX_VALUE));
+ }
+ BloomFilterIdSet doubleMVIdSet = (BloomFilterIdSet)
aggregationResult.get(9);
+ for (int i = 0; i < NUM_RECORDS; i++) {
+ assertTrue(doubleMVIdSet.contains(_values[i] + 0.25));
+ assertTrue(doubleMVIdSet.contains(_values[i] + 0.25 + MAX_VALUE));
+ }
+ BloomFilterIdSet stringMVIdSet = (BloomFilterIdSet)
aggregationResult.get(10);
+ for (int i = 0; i < NUM_RECORDS; i++) {
+ assertTrue(stringMVIdSet.contains(Integer.toString(_values[i])));
+ assertTrue(stringMVIdSet.contains(Integer.toString(_values[i]) +
MAX_VALUE));
+ }
}
{
// With filter
@@ -197,8 +237,8 @@ public class IdSetQueriesTest extends BaseQueriesTest {
.testInnerSegmentExecutionStatistics(aggregationOperator.getExecutionStatistics(),
0, 0, 0, NUM_RECORDS);
List<Object> aggregationResult = resultsBlock.getAggregationResult();
assertNotNull(aggregationResult);
- assertEquals(aggregationResult.size(), 6);
- for (int i = 0; i < 6; i++) {
+ assertEquals(aggregationResult.size(), 11);
+ for (int i = 0; i < 11; i++) {
assertTrue(aggregationResult.get(i) instanceof EmptyIdSet);
}
}
@@ -209,10 +249,10 @@ public class IdSetQueriesTest extends BaseQueriesTest {
BrokerResponseNative brokerResponse =
getBrokerResponseForPqlQuery(query);
Assert.assertEquals(brokerResponse.getNumDocsScanned(), 4 * NUM_RECORDS);
Assert.assertEquals(brokerResponse.getNumEntriesScannedInFilter(), 0);
- Assert.assertEquals(brokerResponse.getNumEntriesScannedPostFilter(), 4 *
6 * NUM_RECORDS);
+ Assert.assertEquals(brokerResponse.getNumEntriesScannedPostFilter(), 4 *
11 * NUM_RECORDS);
Assert.assertEquals(brokerResponse.getTotalDocs(), 4 * NUM_RECORDS);
List<AggregationResult> aggregationResults =
brokerResponse.getAggregationResults();
- Assert.assertEquals(aggregationResults.size(), 6);
+ Assert.assertEquals(aggregationResults.size(), 11);
RoaringBitmapIdSet intIdSet =
(RoaringBitmapIdSet) IdSets.fromBase64String((String)
aggregationResults.get(0).getValue());
for (int i = 0; i < NUM_RECORDS; i++) {
@@ -243,6 +283,36 @@ public class IdSetQueriesTest extends BaseQueriesTest {
for (int i = 0; i < NUM_RECORDS; i++) {
assertTrue(bytesIdSet.contains(Integer.toString(_values[i]).getBytes()));
}
+ RoaringBitmapIdSet intMVIdSet =
+ (RoaringBitmapIdSet) IdSets.fromBase64String((String)
aggregationResults.get(6).getValue());
+ for (int i = 0; i < NUM_RECORDS; i++) {
+ assertTrue(intMVIdSet.contains(_values[i]));
+ assertTrue(intMVIdSet.contains(_values[i] + MAX_VALUE));
+ }
+ Roaring64NavigableMapIdSet longMVIdSet =
+ (Roaring64NavigableMapIdSet) IdSets.fromBase64String((String)
aggregationResults.get(7).getValue());
+ for (int i = 0; i < NUM_RECORDS; i++) {
+ assertTrue(longMVIdSet.contains(_values[i] + (long)
Integer.MAX_VALUE));
+ assertTrue(longMVIdSet.contains(_values[i] + (long) Integer.MAX_VALUE
+ MAX_VALUE));
+ }
+ BloomFilterIdSet floatMVIdSet =
+ (BloomFilterIdSet) IdSets.fromBase64String((String)
aggregationResults.get(8).getValue());
+ for (int i = 0; i < NUM_RECORDS; i++) {
+ assertTrue(floatMVIdSet.contains(_values[i] + 0.5f));
+ assertTrue(floatMVIdSet.contains(_values[i] + 0.5f + MAX_VALUE));
+ }
+ BloomFilterIdSet doubleMVIdSet =
+ (BloomFilterIdSet) IdSets.fromBase64String((String)
aggregationResults.get(9).getValue());
+ for (int i = 0; i < NUM_RECORDS; i++) {
+ assertTrue(doubleMVIdSet.contains(_values[i] + 0.25));
+ assertTrue(doubleMVIdSet.contains(_values[i] + 0.25 + MAX_VALUE));
+ }
+ BloomFilterIdSet stringMVIdSet =
+ (BloomFilterIdSet) IdSets.fromBase64String((String)
aggregationResults.get(10).getValue());
+ for (int i = 0; i < NUM_RECORDS; i++) {
+ assertTrue(stringMVIdSet.contains(Integer.toString(_values[i])));
+ assertTrue(stringMVIdSet.contains(Integer.toString(_values[i]) +
MAX_VALUE));
+ }
}
{
// With filter
@@ -252,7 +322,7 @@ public class IdSetQueriesTest extends BaseQueriesTest {
Assert.assertEquals(brokerResponse.getNumEntriesScannedPostFilter(), 0);
Assert.assertEquals(brokerResponse.getTotalDocs(), 4 * NUM_RECORDS);
List<AggregationResult> aggregationResults =
brokerResponse.getAggregationResults();
- Assert.assertEquals(aggregationResults.size(), 6);
+ Assert.assertEquals(aggregationResults.size(), 11);
for (int i = 0; i < 6; i++) {
assertTrue(IdSets.fromBase64String((String)
aggregationResults.get(i).getValue()) instanceof EmptyIdSet);
}
@@ -264,7 +334,8 @@ public class IdSetQueriesTest extends BaseQueriesTest {
throws IOException {
String query =
"SELECT IDSET(intColumn), IDSET(longColumn), IDSET(floatColumn),
IDSET(doubleColumn), IDSET(stringColumn), "
- + "IDSET(bytesColumn) FROM testTable GROUP BY 1";
+ + "IDSET(bytesColumn), IDSET(intMVColumn), IDSET(longMVColumn),
IDSET(floatMVColumn), "
+ + "IDSET(doubleMVColumn), IDSET(stringMVColumn) FROM testTable
GROUP BY 1";
// Inner segment
{
@@ -272,7 +343,7 @@ public class IdSetQueriesTest extends BaseQueriesTest {
IntermediateResultsBlock resultsBlock =
aggregationGroupByOperator.nextBlock();
QueriesTestUtils
.testInnerSegmentExecutionStatistics(aggregationGroupByOperator.getExecutionStatistics(),
NUM_RECORDS, 0,
- 6 * NUM_RECORDS, NUM_RECORDS);
+ 11 * NUM_RECORDS, NUM_RECORDS);
AggregationGroupByResult aggregationGroupByResult =
resultsBlock.getAggregationGroupByResult();
assertNotNull(aggregationGroupByResult);
Iterator<GroupKeyGenerator.GroupKey> groupKeyIterator =
aggregationGroupByResult.getGroupKeyIterator();
@@ -302,6 +373,32 @@ public class IdSetQueriesTest extends BaseQueriesTest {
for (int i = 0; i < NUM_RECORDS; i++) {
assertTrue(bytesIdSet.contains(Integer.toString(_values[i]).getBytes()));
}
+ RoaringBitmapIdSet intMVIdSet = (RoaringBitmapIdSet)
aggregationGroupByResult.getResultForGroupId(6, groupId);
+ for (int i = 0; i < NUM_RECORDS; i++) {
+ assertTrue(intMVIdSet.contains(_values[i]));
+ assertTrue(intMVIdSet.contains(_values[i] + MAX_VALUE));
+ }
+ Roaring64NavigableMapIdSet longMVIdSet =
+ (Roaring64NavigableMapIdSet)
aggregationGroupByResult.getResultForGroupId(7, groupId);
+ for (int i = 0; i < NUM_RECORDS; i++) {
+ assertTrue(longMVIdSet.contains(_values[i] + (long)
Integer.MAX_VALUE));
+ assertTrue(longMVIdSet.contains(_values[i] + (long) Integer.MAX_VALUE
+ MAX_VALUE));
+ }
+ BloomFilterIdSet floatMVIdSet = (BloomFilterIdSet)
aggregationGroupByResult.getResultForGroupId(8, groupId);
+ for (int i = 0; i < NUM_RECORDS; i++) {
+ assertTrue(floatMVIdSet.contains(_values[i] + 0.5f));
+ assertTrue(floatMVIdSet.contains(_values[i] + 0.5f + MAX_VALUE));
+ }
+ BloomFilterIdSet doubleMVIdSet = (BloomFilterIdSet)
aggregationGroupByResult.getResultForGroupId(9, groupId);
+ for (int i = 0; i < NUM_RECORDS; i++) {
+ assertTrue(doubleMVIdSet.contains(_values[i] + 0.25));
+ assertTrue(doubleMVIdSet.contains(_values[i] + 0.25 + MAX_VALUE));
+ }
+ BloomFilterIdSet stringMVIdSet = (BloomFilterIdSet)
aggregationGroupByResult.getResultForGroupId(10, groupId);
+ for (int i = 0; i < NUM_RECORDS; i++) {
+ assertTrue(stringMVIdSet.contains(Integer.toString(_values[i])));
+ assertTrue(stringMVIdSet.contains(Integer.toString(_values[i]) +
MAX_VALUE));
+ }
assertFalse(groupKeyIterator.hasNext());
}
@@ -310,10 +407,10 @@ public class IdSetQueriesTest extends BaseQueriesTest {
BrokerResponseNative brokerResponse =
getBrokerResponseForPqlQuery(query);
Assert.assertEquals(brokerResponse.getNumDocsScanned(), 4 * NUM_RECORDS);
Assert.assertEquals(brokerResponse.getNumEntriesScannedInFilter(), 0);
- Assert.assertEquals(brokerResponse.getNumEntriesScannedPostFilter(), 4 *
6 * NUM_RECORDS);
+ Assert.assertEquals(brokerResponse.getNumEntriesScannedPostFilter(), 4 *
11 * NUM_RECORDS);
Assert.assertEquals(brokerResponse.getTotalDocs(), 4 * NUM_RECORDS);
List<AggregationResult> aggregationResults =
brokerResponse.getAggregationResults();
- Assert.assertEquals(aggregationResults.size(), 6);
+ Assert.assertEquals(aggregationResults.size(), 11);
for (AggregationResult aggregationResult : aggregationResults) {
Assert.assertNull(aggregationResult.getValue());
List<GroupByResult> groupByResults =
aggregationResult.getGroupByResult();
@@ -349,6 +446,36 @@ public class IdSetQueriesTest extends BaseQueriesTest {
for (int i = 0; i < NUM_RECORDS; i++) {
assertTrue(bytesIdSet.contains(Integer.toString(_values[i]).getBytes()));
}
+ RoaringBitmapIdSet intMVIdSet = (RoaringBitmapIdSet) IdSets
+ .fromBase64String((String)
aggregationResults.get(6).getGroupByResult().get(0).getValue());
+ for (int i = 0; i < NUM_RECORDS; i++) {
+ assertTrue(intMVIdSet.contains(_values[i]));
+ assertTrue(intMVIdSet.contains(_values[i] + MAX_VALUE));
+ }
+ Roaring64NavigableMapIdSet longMVIdSet = (Roaring64NavigableMapIdSet)
IdSets
+ .fromBase64String((String)
aggregationResults.get(7).getGroupByResult().get(0).getValue());
+ for (int i = 0; i < NUM_RECORDS; i++) {
+ assertTrue(longMVIdSet.contains(_values[i] + (long)
Integer.MAX_VALUE));
+ assertTrue(longMVIdSet.contains(_values[i] + (long) Integer.MAX_VALUE
+ MAX_VALUE));
+ }
+ BloomFilterIdSet floatMVIdSet = (BloomFilterIdSet) IdSets
+ .fromBase64String((String)
aggregationResults.get(8).getGroupByResult().get(0).getValue());
+ for (int i = 0; i < NUM_RECORDS; i++) {
+ assertTrue(floatMVIdSet.contains(_values[i] + 0.5f));
+ assertTrue(floatMVIdSet.contains(_values[i] + 0.5f + MAX_VALUE));
+ }
+ BloomFilterIdSet doubleMVIdSet = (BloomFilterIdSet) IdSets
+ .fromBase64String((String)
aggregationResults.get(9).getGroupByResult().get(0).getValue());
+ for (int i = 0; i < NUM_RECORDS; i++) {
+ assertTrue(doubleMVIdSet.contains(_values[i] + 0.25));
+ assertTrue(doubleMVIdSet.contains(_values[i] + 0.25 + MAX_VALUE));
+ }
+ BloomFilterIdSet stringMVIdSet = (BloomFilterIdSet) IdSets
+ .fromBase64String((String)
aggregationResults.get(10).getGroupByResult().get(0).getValue());
+ for (int i = 0; i < NUM_RECORDS; i++) {
+ assertTrue(stringMVIdSet.contains(Integer.toString(_values[i])));
+ assertTrue(stringMVIdSet.contains(Integer.toString(_values[i]) +
MAX_VALUE));
+ }
}
}
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscr...@pinot.apache.org
For additional commands, e-mail: commits-h...@pinot.apache.org
