This is an automated email from the ASF dual-hosted git repository.
jackie pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/pinot.git
The following commit(s) were added to refs/heads/master by this push:
new e788a0afc6 Fix when there're two index filter operator h3 inclusion
index throw exception (#8707)
e788a0afc6 is described below
commit e788a0afc6693234d5a7625a050c05ce1f0d7c18
Author: WangCHX <wcxz...@gmail.com>
AuthorDate: Tue May 17 01:56:53 2022 +0800
Fix when there're two index filter operator h3 inclusion index throw
exception (#8707)
---
.../core/operator/filter/FilterOperatorUtils.java | 3 +-
.../filter/H3InclusionIndexFilterOperator.java | 52 ++++++++++------------
.../apache/pinot/queries/H3IndexQueriesTest.java | 28 ++++++++++++
3 files changed, 53 insertions(+), 30 deletions(-)
diff --git
a/pinot-core/src/main/java/org/apache/pinot/core/operator/filter/FilterOperatorUtils.java
b/pinot-core/src/main/java/org/apache/pinot/core/operator/filter/FilterOperatorUtils.java
index cbeb16a4f1..e19c331863 100644
---
a/pinot-core/src/main/java/org/apache/pinot/core/operator/filter/FilterOperatorUtils.java
+++
b/pinot-core/src/main/java/org/apache/pinot/core/operator/filter/FilterOperatorUtils.java
@@ -171,7 +171,8 @@ public class FilterOperatorUtils {
}
if (filterOperator instanceof RangeIndexBasedFilterOperator
|| filterOperator instanceof TextContainsFilterOperator ||
filterOperator instanceof TextMatchFilterOperator
- || filterOperator instanceof JsonMatchFilterOperator ||
filterOperator instanceof H3IndexFilterOperator) {
+ || filterOperator instanceof JsonMatchFilterOperator ||
filterOperator instanceof H3IndexFilterOperator
+ || filterOperator instanceof H3InclusionIndexFilterOperator) {
return 2;
}
if (filterOperator instanceof AndFilterOperator) {
diff --git
a/pinot-core/src/main/java/org/apache/pinot/core/operator/filter/H3InclusionIndexFilterOperator.java
b/pinot-core/src/main/java/org/apache/pinot/core/operator/filter/H3InclusionIndexFilterOperator.java
index c64d511697..bcfc9b66c3 100644
---
a/pinot-core/src/main/java/org/apache/pinot/core/operator/filter/H3InclusionIndexFilterOperator.java
+++
b/pinot-core/src/main/java/org/apache/pinot/core/operator/filter/H3InclusionIndexFilterOperator.java
@@ -84,39 +84,33 @@ public class H3InclusionIndexFilterOperator extends
BaseFilterOperator {
LongSet fullyCoverH3Cells = fullCoverAndPotentialCoverCells.getLeft();
LongSet potentialCoverH3Cells = fullCoverAndPotentialCoverCells.getRight();
- // have list of h3 cell ids for polygon provided
- // return filtered num_docs
- ImmutableRoaringBitmap[] potentialMatchDocIds = new
ImmutableRoaringBitmap[potentialCoverH3Cells.size()];
int i = 0;
- LongIterator potentialCoverH3CellsIterator =
potentialCoverH3Cells.iterator();
- while (potentialCoverH3CellsIterator.hasNext()) {
- potentialMatchDocIds[i++] =
_h3IndexReader.getDocIds(potentialCoverH3CellsIterator.nextLong());
+ ImmutableRoaringBitmap[] fullMatchDocIds = new
ImmutableRoaringBitmap[fullyCoverH3Cells.size()];
+ LongIterator fullyCoverH3CellsIterator = fullyCoverH3Cells.iterator();
+ while (fullyCoverH3CellsIterator.hasNext()) {
+ fullMatchDocIds[i++] =
_h3IndexReader.getDocIds(fullyCoverH3CellsIterator.nextLong());
}
- MutableRoaringBitmap potentialMatchMutableRoaringBitmap =
BufferFastAggregation.or(potentialMatchDocIds);
+ MutableRoaringBitmap fullMatch = BufferFastAggregation.or(fullMatchDocIds);
+
+ // remove full match from potential match to get potential not match cells.
+ i = 0;
+ potentialCoverH3Cells.removeAll(fullyCoverH3Cells);
+ ImmutableRoaringBitmap[] potentialNotMatchDocIds = new
ImmutableRoaringBitmap[potentialCoverH3Cells.size()];
+ LongIterator potentialMatchH3CellsIterator =
potentialCoverH3Cells.iterator();
+ while (potentialMatchH3CellsIterator.hasNext()) {
+ potentialNotMatchDocIds[i++] =
_h3IndexReader.getDocIds(potentialMatchH3CellsIterator.nextLong());
+ }
+ MutableRoaringBitmap potentialMatch =
BufferFastAggregation.or(potentialNotMatchDocIds);
+
if (_isPositiveCheck) {
- ImmutableRoaringBitmap[] fullMatchDocIds = new
ImmutableRoaringBitmap[fullyCoverH3Cells.size()];
- i = 0;
- LongIterator fullyCoverH3CellsIterator = fullyCoverH3Cells.iterator();
- while (fullyCoverH3CellsIterator.hasNext()) {
- fullMatchDocIds[i++] =
_h3IndexReader.getDocIds(fullyCoverH3CellsIterator.nextLong());
- }
- MutableRoaringBitmap fullMatchMutableRoaringBitmap =
BufferFastAggregation.or(fullMatchDocIds);
- return getFilterBlock(fullMatchMutableRoaringBitmap,
potentialMatchMutableRoaringBitmap);
+ return getFilterBlock(fullMatch, potentialMatch);
} else {
- i = 0;
- // remove full match from potential match to get potential not match
cells.
- potentialCoverH3Cells.removeAll(fullyCoverH3Cells);
- ImmutableRoaringBitmap[] potentialNotMatchMutableRoaringBitmap =
- new ImmutableRoaringBitmap[potentialCoverH3Cells.size()];
- LongIterator potentialNotMatchH3CellsIterator =
potentialCoverH3Cells.iterator();
- while (potentialNotMatchH3CellsIterator.hasNext()) {
- potentialNotMatchMutableRoaringBitmap[i++] =
-
_h3IndexReader.getDocIds(potentialNotMatchH3CellsIterator.nextLong());
- }
- MutableRoaringBitmap potentialNotMatch =
BufferFastAggregation.or(potentialNotMatchMutableRoaringBitmap);
- // flip potential match bit map to get exactly not match bitmap.
- potentialMatchMutableRoaringBitmap.flip(0L, _numDocs);
- return getFilterBlock(potentialMatchMutableRoaringBitmap,
potentialNotMatch);
+ // potentialMatch also include potential not match docs.
+ // full not match is first flip potentialMatch, then andNot full match.
+ MutableRoaringBitmap fullNotMatch = potentialMatch.clone();
+ fullNotMatch.flip(0L, _numDocs);
+ fullNotMatch.andNot(fullMatch);
+ return getFilterBlock(fullNotMatch, potentialMatch);
}
}
diff --git
a/pinot-core/src/test/java/org/apache/pinot/queries/H3IndexQueriesTest.java
b/pinot-core/src/test/java/org/apache/pinot/queries/H3IndexQueriesTest.java
index 907f6b9ec6..865b83c330 100644
--- a/pinot-core/src/test/java/org/apache/pinot/queries/H3IndexQueriesTest.java
+++ b/pinot-core/src/test/java/org/apache/pinot/queries/H3IndexQueriesTest.java
@@ -384,6 +384,34 @@ public class H3IndexQueriesTest extends BaseQueriesTest {
Assert.assertEquals((long) aggregationResult.get(0), 0);
}
+ @Test
+ public void queryStContainsWithMultipleFilters()
+ throws Exception {
+ List<GenericRow> records = new ArrayList<>(1);
+ addRecord(records, -122.0007277, 37.5005785);
+ setUp(records);
+ // Test point is closed to border of a polygon but outside.
+ String query = "SELECT COUNT(*) FROM testTable WHERE
ST_Contains(ST_GeomFromText('POLYGON ((\n"
+ + " -122.0008564 37.5004316, \n"
+ + " -121.9991291 37.5005168, \n"
+ + " -121.9990325 37.4995294, \n"
+ + " -122.0001268 37.4993506, \n"
+ + " -122.0008564 37.5004316))'), h3Column_geometry) = 1
AND "
+ + " ST_Contains(ST_GeomFromText('POLYGON (( \n"
+ + " -122.0008564 37.5004316, \n"
+ + " -121.9991291 37.5005168, \n"
+ + " -121.9990325 37.4995294, \n"
+ + " -122.0001268 37.4993506, \n"
+ + " -122.0008564 37.5004316))'), h3Column_geometry) = 0";
+
+ AggregationOperator aggregationOperator = getOperator(query);
+ IntermediateResultsBlock resultsBlock = aggregationOperator.nextBlock();
+
QueriesTestUtils.testInnerSegmentExecutionStatistics(aggregationOperator.getExecutionStatistics(),
0, 2, 0, 1);
+ List<Object> aggregationResult = resultsBlock.getAggregationResult();
+ Assert.assertNotNull(aggregationResult);
+ Assert.assertEquals((long) aggregationResult.get(0), 0);
+ }
+
private void testQuery(String queryTemplate) {
String h3IndexQuery = String.format(queryTemplate, H3_INDEX_COLUMN);
String nonH3IndexQuery = String.format(queryTemplate, NON_H3_INDEX_COLUMN);
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscr...@pinot.apache.org
For additional commands, e-mail: commits-h...@pinot.apache.org
