gsmiller commented on code in PR #13974: URL: https://github.com/apache/lucene/pull/13974#discussion_r1890984514
########## lucene/sandbox/src/java/org/apache/lucene/sandbox/search/SortedSetMultiRangeQuery.java: ########## @@ -0,0 +1,300 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.lucene.sandbox.search; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; +import java.util.Objects; +import org.apache.lucene.document.SortedSetDocValuesField; +import org.apache.lucene.index.DocValues; +import org.apache.lucene.index.DocValuesSkipper; +import org.apache.lucene.index.LeafReaderContext; +import org.apache.lucene.index.SortedSetDocValues; +import org.apache.lucene.index.TermsEnum; +import org.apache.lucene.search.BooleanQuery; +import org.apache.lucene.search.ConstantScoreScorer; +import org.apache.lucene.search.ConstantScoreWeight; +import org.apache.lucene.search.DocIdSetIterator; +import org.apache.lucene.search.DocValuesRangeIterator; +import org.apache.lucene.search.IndexSearcher; +import org.apache.lucene.search.Query; +import org.apache.lucene.search.QueryVisitor; +import org.apache.lucene.search.ScoreMode; +import org.apache.lucene.search.Scorer; +import org.apache.lucene.search.ScorerSupplier; +import org.apache.lucene.search.TwoPhaseIterator; +import org.apache.lucene.search.Weight; +import org.apache.lucene.util.ArrayUtil; +import org.apache.lucene.util.BytesRef; +import org.apache.lucene.util.LongBitSet; + +/** A union multiple ranges over SortedSetDocValuesField */ +public class SortedSetMultiRangeQuery extends Query { + private final String field; + private final int bytesPerDim; + private final ArrayUtil.ByteArrayComparator comparator; + List<MultiRangeQuery.RangeClause> rangeClauses; Review Comment: Any particular reason this shouldn't be `private final`? Maybe you're following the pattern in `MultiRangeQuery`, but I'm not sure what the reasoning is over in that class either (and might advocate to change that as well). Seems maybe even more important since `rangeClauses` is part of object identity? ########## lucene/sandbox/src/java/org/apache/lucene/sandbox/search/SortedSetMultiRangeQuery.java: ########## @@ -0,0 +1,300 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.lucene.sandbox.search; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; +import java.util.Objects; +import org.apache.lucene.document.SortedSetDocValuesField; +import org.apache.lucene.index.DocValues; +import org.apache.lucene.index.DocValuesSkipper; +import org.apache.lucene.index.LeafReaderContext; +import org.apache.lucene.index.SortedSetDocValues; +import org.apache.lucene.index.TermsEnum; +import org.apache.lucene.search.BooleanQuery; +import org.apache.lucene.search.ConstantScoreScorer; +import org.apache.lucene.search.ConstantScoreWeight; +import org.apache.lucene.search.DocIdSetIterator; +import org.apache.lucene.search.DocValuesRangeIterator; +import org.apache.lucene.search.IndexSearcher; +import org.apache.lucene.search.Query; +import org.apache.lucene.search.QueryVisitor; +import org.apache.lucene.search.ScoreMode; +import org.apache.lucene.search.Scorer; +import org.apache.lucene.search.ScorerSupplier; +import org.apache.lucene.search.TwoPhaseIterator; +import org.apache.lucene.search.Weight; +import org.apache.lucene.util.ArrayUtil; +import org.apache.lucene.util.BytesRef; +import org.apache.lucene.util.LongBitSet; + +/** A union multiple ranges over SortedSetDocValuesField */ +public class SortedSetMultiRangeQuery extends Query { + private final String field; + private final int bytesPerDim; + private final ArrayUtil.ByteArrayComparator comparator; + List<MultiRangeQuery.RangeClause> rangeClauses; + + SortedSetMultiRangeQuery( + String name, + List<MultiRangeQuery.RangeClause> clauses, + int bytes, + ArrayUtil.ByteArrayComparator comparator) { + this.field = name; + this.rangeClauses = clauses; + this.bytesPerDim = bytes; + this.comparator = comparator; + } + + /** Builder for creating a SortedSetMultiRangeQuery. */ + public static class Builder { + private final String name; + protected final List<MultiRangeQuery.RangeClause> clauses = new ArrayList<>(); + private final int bytes; + private final ArrayUtil.ByteArrayComparator comparator; + + public Builder(String name, int bytes) { + this.name = Objects.requireNonNull(name); + this.bytes = bytes; // TODO assrt positive + this.comparator = ArrayUtil.getUnsignedComparator(bytes); + } + + public Builder add(BytesRef lowerValue, BytesRef upperValue) { + byte[] low = lowerValue.clone().bytes; + byte[] up = upperValue.clone().bytes; + if (this.comparator.compare(low, 0, up, 0) > 0) { + throw new IllegalArgumentException("lowerValue must be <= upperValue"); + } else { + clauses.add(new MultiRangeQuery.RangeClause(low, up)); + } + return this; + } + + public Query build() { + if (clauses.isEmpty()) { + return new BooleanQuery.Builder().build(); Review Comment: Maybe `MatchNoDocsQuery` instead? ########## lucene/sandbox/src/java/org/apache/lucene/sandbox/search/SortedSetMultiRangeQuery.java: ########## @@ -0,0 +1,300 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.lucene.sandbox.search; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; +import java.util.Objects; +import org.apache.lucene.document.SortedSetDocValuesField; +import org.apache.lucene.index.DocValues; +import org.apache.lucene.index.DocValuesSkipper; +import org.apache.lucene.index.LeafReaderContext; +import org.apache.lucene.index.SortedSetDocValues; +import org.apache.lucene.index.TermsEnum; +import org.apache.lucene.search.BooleanQuery; +import org.apache.lucene.search.ConstantScoreScorer; +import org.apache.lucene.search.ConstantScoreWeight; +import org.apache.lucene.search.DocIdSetIterator; +import org.apache.lucene.search.DocValuesRangeIterator; +import org.apache.lucene.search.IndexSearcher; +import org.apache.lucene.search.Query; +import org.apache.lucene.search.QueryVisitor; +import org.apache.lucene.search.ScoreMode; +import org.apache.lucene.search.Scorer; +import org.apache.lucene.search.ScorerSupplier; +import org.apache.lucene.search.TwoPhaseIterator; +import org.apache.lucene.search.Weight; +import org.apache.lucene.util.ArrayUtil; +import org.apache.lucene.util.BytesRef; +import org.apache.lucene.util.LongBitSet; + +/** A union multiple ranges over SortedSetDocValuesField */ +public class SortedSetMultiRangeQuery extends Query { + private final String field; + private final int bytesPerDim; + private final ArrayUtil.ByteArrayComparator comparator; + List<MultiRangeQuery.RangeClause> rangeClauses; + + SortedSetMultiRangeQuery( + String name, + List<MultiRangeQuery.RangeClause> clauses, + int bytes, + ArrayUtil.ByteArrayComparator comparator) { + this.field = name; + this.rangeClauses = clauses; + this.bytesPerDim = bytes; + this.comparator = comparator; + } + + /** Builder for creating a SortedSetMultiRangeQuery. */ + public static class Builder { + private final String name; + protected final List<MultiRangeQuery.RangeClause> clauses = new ArrayList<>(); Review Comment: Is there a reason this can't be `private`? ########## lucene/sandbox/src/java/org/apache/lucene/sandbox/search/SortedSetMultiRangeQuery.java: ########## @@ -0,0 +1,300 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.lucene.sandbox.search; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; +import java.util.Objects; +import org.apache.lucene.document.SortedSetDocValuesField; +import org.apache.lucene.index.DocValues; +import org.apache.lucene.index.DocValuesSkipper; +import org.apache.lucene.index.LeafReaderContext; +import org.apache.lucene.index.SortedSetDocValues; +import org.apache.lucene.index.TermsEnum; +import org.apache.lucene.search.BooleanQuery; +import org.apache.lucene.search.ConstantScoreScorer; +import org.apache.lucene.search.ConstantScoreWeight; +import org.apache.lucene.search.DocIdSetIterator; +import org.apache.lucene.search.DocValuesRangeIterator; +import org.apache.lucene.search.IndexSearcher; +import org.apache.lucene.search.Query; +import org.apache.lucene.search.QueryVisitor; +import org.apache.lucene.search.ScoreMode; +import org.apache.lucene.search.Scorer; +import org.apache.lucene.search.ScorerSupplier; +import org.apache.lucene.search.TwoPhaseIterator; +import org.apache.lucene.search.Weight; +import org.apache.lucene.util.ArrayUtil; +import org.apache.lucene.util.BytesRef; +import org.apache.lucene.util.LongBitSet; + +/** A union multiple ranges over SortedSetDocValuesField */ +public class SortedSetMultiRangeQuery extends Query { + private final String field; + private final int bytesPerDim; + private final ArrayUtil.ByteArrayComparator comparator; + List<MultiRangeQuery.RangeClause> rangeClauses; + + SortedSetMultiRangeQuery( + String name, + List<MultiRangeQuery.RangeClause> clauses, + int bytes, + ArrayUtil.ByteArrayComparator comparator) { + this.field = name; + this.rangeClauses = clauses; + this.bytesPerDim = bytes; + this.comparator = comparator; + } + + /** Builder for creating a SortedSetMultiRangeQuery. */ + public static class Builder { + private final String name; + protected final List<MultiRangeQuery.RangeClause> clauses = new ArrayList<>(); + private final int bytes; + private final ArrayUtil.ByteArrayComparator comparator; + + public Builder(String name, int bytes) { + this.name = Objects.requireNonNull(name); + this.bytes = bytes; // TODO assrt positive + this.comparator = ArrayUtil.getUnsignedComparator(bytes); + } + + public Builder add(BytesRef lowerValue, BytesRef upperValue) { + byte[] low = lowerValue.clone().bytes; + byte[] up = upperValue.clone().bytes; + if (this.comparator.compare(low, 0, up, 0) > 0) { + throw new IllegalArgumentException("lowerValue must be <= upperValue"); + } else { + clauses.add(new MultiRangeQuery.RangeClause(low, up)); + } + return this; + } + + public Query build() { + if (clauses.isEmpty()) { + return new BooleanQuery.Builder().build(); + } + if (clauses.size() == 1) { + return SortedSetDocValuesField.newSlowRangeQuery( + name, + new BytesRef(clauses.getFirst().lowerValue), + new BytesRef(clauses.getFirst().upperValue), + true, + true); + } + return new SortedSetMultiRangeQuery(name, clauses, this.bytes, comparator); + } + } + + @Override + public Query rewrite(IndexSearcher indexSearcher) throws IOException { + ArrayList<MultiRangeQuery.RangeClause> sortedClauses = new ArrayList<>(this.rangeClauses); Review Comment: I'm not sure why this logic exists in rewrite? Do we rely on having the ranges sorted in a specific order? Is so, should we just sort them in the ctor or in the builder? I don't think we need the rewrite hook for this. (Rewrite is useful if there is some state of the IndexSearcher that can be leveraged to optimize the query in some way). ########## lucene/sandbox/src/java/org/apache/lucene/sandbox/search/SortedSetMultiRangeQuery.java: ########## @@ -0,0 +1,300 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.lucene.sandbox.search; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; +import java.util.Objects; +import org.apache.lucene.document.SortedSetDocValuesField; +import org.apache.lucene.index.DocValues; +import org.apache.lucene.index.DocValuesSkipper; +import org.apache.lucene.index.LeafReaderContext; +import org.apache.lucene.index.SortedSetDocValues; +import org.apache.lucene.index.TermsEnum; +import org.apache.lucene.search.BooleanQuery; +import org.apache.lucene.search.ConstantScoreScorer; +import org.apache.lucene.search.ConstantScoreWeight; +import org.apache.lucene.search.DocIdSetIterator; +import org.apache.lucene.search.DocValuesRangeIterator; +import org.apache.lucene.search.IndexSearcher; +import org.apache.lucene.search.Query; +import org.apache.lucene.search.QueryVisitor; +import org.apache.lucene.search.ScoreMode; +import org.apache.lucene.search.Scorer; +import org.apache.lucene.search.ScorerSupplier; +import org.apache.lucene.search.TwoPhaseIterator; +import org.apache.lucene.search.Weight; +import org.apache.lucene.util.ArrayUtil; +import org.apache.lucene.util.BytesRef; +import org.apache.lucene.util.LongBitSet; + +/** A union multiple ranges over SortedSetDocValuesField */ +public class SortedSetMultiRangeQuery extends Query { + private final String field; + private final int bytesPerDim; + private final ArrayUtil.ByteArrayComparator comparator; + List<MultiRangeQuery.RangeClause> rangeClauses; + + SortedSetMultiRangeQuery( + String name, + List<MultiRangeQuery.RangeClause> clauses, + int bytes, + ArrayUtil.ByteArrayComparator comparator) { + this.field = name; + this.rangeClauses = clauses; + this.bytesPerDim = bytes; + this.comparator = comparator; + } + + /** Builder for creating a SortedSetMultiRangeQuery. */ + public static class Builder { + private final String name; + protected final List<MultiRangeQuery.RangeClause> clauses = new ArrayList<>(); + private final int bytes; + private final ArrayUtil.ByteArrayComparator comparator; + + public Builder(String name, int bytes) { + this.name = Objects.requireNonNull(name); + this.bytes = bytes; // TODO assrt positive + this.comparator = ArrayUtil.getUnsignedComparator(bytes); + } + + public Builder add(BytesRef lowerValue, BytesRef upperValue) { + byte[] low = lowerValue.clone().bytes; + byte[] up = upperValue.clone().bytes; + if (this.comparator.compare(low, 0, up, 0) > 0) { + throw new IllegalArgumentException("lowerValue must be <= upperValue"); + } else { + clauses.add(new MultiRangeQuery.RangeClause(low, up)); + } + return this; + } + + public Query build() { + if (clauses.isEmpty()) { + return new BooleanQuery.Builder().build(); + } + if (clauses.size() == 1) { + return SortedSetDocValuesField.newSlowRangeQuery( + name, + new BytesRef(clauses.getFirst().lowerValue), + new BytesRef(clauses.getFirst().upperValue), + true, + true); + } + return new SortedSetMultiRangeQuery(name, clauses, this.bytes, comparator); + } + } + + @Override + public Query rewrite(IndexSearcher indexSearcher) throws IOException { + ArrayList<MultiRangeQuery.RangeClause> sortedClauses = new ArrayList<>(this.rangeClauses); + sortedClauses.sort( + (o1, o2) -> { + // if (result == 0) { + // return comparator.compare(o1.upperValue, 0, o2.upperValue, 0); + // } else { + return comparator.compare(o1.lowerValue, 0, o2.lowerValue, 0); + // } + }); + if (!this.rangeClauses.equals(sortedClauses)) { + return new SortedSetMultiRangeQuery( + this.field, sortedClauses, this.bytesPerDim, this.comparator); + } else { + return this; + } + } + + @Override + public String toString(String fld) { + return "SortedSetMultiRangeQuery{" + + "field='" + + fld + + '\'' + + ", rangeClauses=" + + rangeClauses + + // TODO better toString + '}'; + } + + // what TODO with reverse ranges ??? + @Override + public Weight createWeight(IndexSearcher searcher, ScoreMode scoreMode, float boost) + throws IOException { + return new ConstantScoreWeight(this, boost) { + @Override + public ScorerSupplier scorerSupplier(LeafReaderContext context) throws IOException { + if (context.reader().getFieldInfos().fieldInfo(field) == null) { + return null; + } + DocValuesSkipper skipper = context.reader().getDocValuesSkipper(field); + SortedSetDocValues values = DocValues.getSortedSet(context.reader(), field); + // implement ScorerSupplier, since we do some expensive stuff to make a scorer + return new ScorerSupplier() { + @Override + public Scorer get(long leadCost) throws IOException { + if (rangeClauses.isEmpty()) { + return empty(); + } + TermsEnum termsEnum = values.termsEnum(); + LongBitSet matchingOrdsShifted = null; + long minOrd = 0, maxOrd = values.getValueCount() - 1; + long matchesAbove = + values.getValueCount(); // it's last range goes to maxOrd, by default - no match + long maxSeenOrd = values.getValueCount(); + TermsEnum.SeekStatus seekStatus = TermsEnum.SeekStatus.NOT_FOUND; + for (int r = 0; r < rangeClauses.size(); r++) { + MultiRangeQuery.RangeClause range = rangeClauses.get(r); + long startingOrd; + seekStatus = termsEnum.seekCeil(new BytesRef(range.lowerValue)); + if (matchingOrdsShifted == null) { // first iter + if (seekStatus == TermsEnum.SeekStatus.END) { + return empty(); // no bitset yet, give up + } + minOrd = termsEnum.ord(); + if (skipper != null) { + minOrd = Math.max(minOrd, skipper.minValue()); + maxOrd = Math.min(maxOrd, skipper.maxValue()); + } + if (maxOrd < minOrd) { + return empty(); + } + startingOrd = minOrd; + } else { + if (seekStatus == TermsEnum.SeekStatus.END) { + break; // ranges - we are done, terms are exhausted + } else { + startingOrd = termsEnum.ord(); + } + } + byte[] upper = range.upperValue; // TODO ignore reverse ranges + // looking for overlap + for (int overlap = r + 1; overlap < rangeClauses.size(); overlap++, r++) { + MultiRangeQuery.RangeClause mayOverlap = rangeClauses.get(overlap); + assert comparator.compare(range.lowerValue, 0, mayOverlap.lowerValue, 0) <= 0 + : "since they are sorted"; + // TODO it might be contiguous ranges, it's worth to check but I have no idea how + if (comparator.compare(mayOverlap.lowerValue, 0, upper, 0) <= 0) { + // overlap, expand if needed + if (comparator.compare(upper, 0, mayOverlap.upperValue, 0) < 0) { + upper = mayOverlap.upperValue; + } + // continue; // skip overlapping rng + } else { + break; // no r++ + } + } + seekStatus = termsEnum.seekCeil(new BytesRef(upper)); + + if (seekStatus == TermsEnum.SeekStatus.END) { + maxSeenOrd = maxOrd; // perhaps it's worth to set for skipper + matchesAbove = startingOrd; + break; // no need to create bitset + } + maxSeenOrd = + seekStatus == TermsEnum.SeekStatus.FOUND + ? termsEnum.ord() + : termsEnum.ord() - 1; // floor + + if (matchingOrdsShifted == null) { + matchingOrdsShifted = new LongBitSet(maxOrd + 1 - minOrd); + } + matchingOrdsShifted.set( + startingOrd - minOrd, maxSeenOrd - minOrd + 1); // up is exclusive + } + /// ranges are over, there might be no set!! + TwoPhaseIterator iterator; + long finalMatchesAbove = matchesAbove; + LongBitSet finalMatchingOrdsShifted = matchingOrdsShifted; + long finalMinOrd = minOrd; + iterator = + new TwoPhaseIterator(values) { + // TODO unwrap singleton? + @Override + public boolean matches() throws IOException { + for (int i = 0; i < values.docValueCount(); i++) { + long ord = values.nextOrd(); + if (ord >= finalMinOrd + && ((finalMatchesAbove < values.getValueCount() + && ord >= finalMatchesAbove) + || finalMatchingOrdsShifted.get(ord - finalMinOrd))) { Review Comment: This approach of densely memorizing all ordinals that fall within the ranges has me a bit worried. This could potentially be a quite large bitset. I wonder if we could instead just keep track of all the ranges actually represented in the query and then just check against them for each doc ord? Like what we do in `SortedSetDocValuesRangeQuery` but keeping tracking of multiple min/max tuples to check against instead of only one? Is there a reason you didn't go with that approach and feel the dense bitset is necessary here? ########## lucene/sandbox/src/java/org/apache/lucene/sandbox/search/SortedSetMultiRangeQuery.java: ########## @@ -0,0 +1,300 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.lucene.sandbox.search; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; +import java.util.Objects; +import org.apache.lucene.document.SortedSetDocValuesField; +import org.apache.lucene.index.DocValues; +import org.apache.lucene.index.DocValuesSkipper; +import org.apache.lucene.index.LeafReaderContext; +import org.apache.lucene.index.SortedSetDocValues; +import org.apache.lucene.index.TermsEnum; +import org.apache.lucene.search.BooleanQuery; +import org.apache.lucene.search.ConstantScoreScorer; +import org.apache.lucene.search.ConstantScoreWeight; +import org.apache.lucene.search.DocIdSetIterator; +import org.apache.lucene.search.DocValuesRangeIterator; +import org.apache.lucene.search.IndexSearcher; +import org.apache.lucene.search.Query; +import org.apache.lucene.search.QueryVisitor; +import org.apache.lucene.search.ScoreMode; +import org.apache.lucene.search.Scorer; +import org.apache.lucene.search.ScorerSupplier; +import org.apache.lucene.search.TwoPhaseIterator; +import org.apache.lucene.search.Weight; +import org.apache.lucene.util.ArrayUtil; +import org.apache.lucene.util.BytesRef; +import org.apache.lucene.util.LongBitSet; + +/** A union multiple ranges over SortedSetDocValuesField */ +public class SortedSetMultiRangeQuery extends Query { + private final String field; + private final int bytesPerDim; + private final ArrayUtil.ByteArrayComparator comparator; + List<MultiRangeQuery.RangeClause> rangeClauses; + + SortedSetMultiRangeQuery( + String name, + List<MultiRangeQuery.RangeClause> clauses, + int bytes, + ArrayUtil.ByteArrayComparator comparator) { + this.field = name; + this.rangeClauses = clauses; + this.bytesPerDim = bytes; + this.comparator = comparator; + } + + /** Builder for creating a SortedSetMultiRangeQuery. */ + public static class Builder { + private final String name; + protected final List<MultiRangeQuery.RangeClause> clauses = new ArrayList<>(); + private final int bytes; + private final ArrayUtil.ByteArrayComparator comparator; + + public Builder(String name, int bytes) { + this.name = Objects.requireNonNull(name); + this.bytes = bytes; // TODO assrt positive + this.comparator = ArrayUtil.getUnsignedComparator(bytes); + } + + public Builder add(BytesRef lowerValue, BytesRef upperValue) { + byte[] low = lowerValue.clone().bytes; + byte[] up = upperValue.clone().bytes; + if (this.comparator.compare(low, 0, up, 0) > 0) { + throw new IllegalArgumentException("lowerValue must be <= upperValue"); + } else { + clauses.add(new MultiRangeQuery.RangeClause(low, up)); + } + return this; + } + + public Query build() { + if (clauses.isEmpty()) { + return new BooleanQuery.Builder().build(); + } + if (clauses.size() == 1) { + return SortedSetDocValuesField.newSlowRangeQuery( + name, + new BytesRef(clauses.getFirst().lowerValue), + new BytesRef(clauses.getFirst().upperValue), + true, + true); + } + return new SortedSetMultiRangeQuery(name, clauses, this.bytes, comparator); + } + } + + @Override + public Query rewrite(IndexSearcher indexSearcher) throws IOException { + ArrayList<MultiRangeQuery.RangeClause> sortedClauses = new ArrayList<>(this.rangeClauses); + sortedClauses.sort( + (o1, o2) -> { + // if (result == 0) { + // return comparator.compare(o1.upperValue, 0, o2.upperValue, 0); + // } else { + return comparator.compare(o1.lowerValue, 0, o2.lowerValue, 0); + // } + }); + if (!this.rangeClauses.equals(sortedClauses)) { + return new SortedSetMultiRangeQuery( + this.field, sortedClauses, this.bytesPerDim, this.comparator); + } else { + return this; + } + } + + @Override + public String toString(String fld) { + return "SortedSetMultiRangeQuery{" + + "field='" + + fld + + '\'' + + ", rangeClauses=" + + rangeClauses + + // TODO better toString + '}'; + } + + // what TODO with reverse ranges ??? + @Override + public Weight createWeight(IndexSearcher searcher, ScoreMode scoreMode, float boost) + throws IOException { + return new ConstantScoreWeight(this, boost) { + @Override + public ScorerSupplier scorerSupplier(LeafReaderContext context) throws IOException { + if (context.reader().getFieldInfos().fieldInfo(field) == null) { + return null; + } + DocValuesSkipper skipper = context.reader().getDocValuesSkipper(field); + SortedSetDocValues values = DocValues.getSortedSet(context.reader(), field); + // implement ScorerSupplier, since we do some expensive stuff to make a scorer + return new ScorerSupplier() { + @Override + public Scorer get(long leadCost) throws IOException { + if (rangeClauses.isEmpty()) { + return empty(); + } + TermsEnum termsEnum = values.termsEnum(); + LongBitSet matchingOrdsShifted = null; + long minOrd = 0, maxOrd = values.getValueCount() - 1; + long matchesAbove = + values.getValueCount(); // it's last range goes to maxOrd, by default - no match + long maxSeenOrd = values.getValueCount(); + TermsEnum.SeekStatus seekStatus = TermsEnum.SeekStatus.NOT_FOUND; + for (int r = 0; r < rangeClauses.size(); r++) { + MultiRangeQuery.RangeClause range = rangeClauses.get(r); + long startingOrd; + seekStatus = termsEnum.seekCeil(new BytesRef(range.lowerValue)); + if (matchingOrdsShifted == null) { // first iter + if (seekStatus == TermsEnum.SeekStatus.END) { + return empty(); // no bitset yet, give up + } + minOrd = termsEnum.ord(); + if (skipper != null) { + minOrd = Math.max(minOrd, skipper.minValue()); + maxOrd = Math.min(maxOrd, skipper.maxValue()); + } + if (maxOrd < minOrd) { Review Comment: I'm not sure this can ever happen? ########## lucene/sandbox/src/java/org/apache/lucene/sandbox/search/SortedSetMultiRangeQuery.java: ########## @@ -0,0 +1,300 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.lucene.sandbox.search; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; +import java.util.Objects; +import org.apache.lucene.document.SortedSetDocValuesField; +import org.apache.lucene.index.DocValues; +import org.apache.lucene.index.DocValuesSkipper; +import org.apache.lucene.index.LeafReaderContext; +import org.apache.lucene.index.SortedSetDocValues; +import org.apache.lucene.index.TermsEnum; +import org.apache.lucene.search.BooleanQuery; +import org.apache.lucene.search.ConstantScoreScorer; +import org.apache.lucene.search.ConstantScoreWeight; +import org.apache.lucene.search.DocIdSetIterator; +import org.apache.lucene.search.DocValuesRangeIterator; +import org.apache.lucene.search.IndexSearcher; +import org.apache.lucene.search.Query; +import org.apache.lucene.search.QueryVisitor; +import org.apache.lucene.search.ScoreMode; +import org.apache.lucene.search.Scorer; +import org.apache.lucene.search.ScorerSupplier; +import org.apache.lucene.search.TwoPhaseIterator; +import org.apache.lucene.search.Weight; +import org.apache.lucene.util.ArrayUtil; +import org.apache.lucene.util.BytesRef; +import org.apache.lucene.util.LongBitSet; + +/** A union multiple ranges over SortedSetDocValuesField */ +public class SortedSetMultiRangeQuery extends Query { + private final String field; + private final int bytesPerDim; + private final ArrayUtil.ByteArrayComparator comparator; + List<MultiRangeQuery.RangeClause> rangeClauses; + + SortedSetMultiRangeQuery( + String name, + List<MultiRangeQuery.RangeClause> clauses, + int bytes, + ArrayUtil.ByteArrayComparator comparator) { + this.field = name; + this.rangeClauses = clauses; + this.bytesPerDim = bytes; + this.comparator = comparator; + } + + /** Builder for creating a SortedSetMultiRangeQuery. */ + public static class Builder { + private final String name; + protected final List<MultiRangeQuery.RangeClause> clauses = new ArrayList<>(); + private final int bytes; + private final ArrayUtil.ByteArrayComparator comparator; + + public Builder(String name, int bytes) { + this.name = Objects.requireNonNull(name); + this.bytes = bytes; // TODO assrt positive + this.comparator = ArrayUtil.getUnsignedComparator(bytes); + } + + public Builder add(BytesRef lowerValue, BytesRef upperValue) { + byte[] low = lowerValue.clone().bytes; Review Comment: You need to be aware of `offset` here. It might be better to array-copy the underlying bytes into a new array instead of cloning? ########## lucene/sandbox/src/java/org/apache/lucene/sandbox/search/SortedSetMultiRangeQuery.java: ########## @@ -0,0 +1,300 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.lucene.sandbox.search; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; +import java.util.Objects; +import org.apache.lucene.document.SortedSetDocValuesField; +import org.apache.lucene.index.DocValues; +import org.apache.lucene.index.DocValuesSkipper; +import org.apache.lucene.index.LeafReaderContext; +import org.apache.lucene.index.SortedSetDocValues; +import org.apache.lucene.index.TermsEnum; +import org.apache.lucene.search.BooleanQuery; +import org.apache.lucene.search.ConstantScoreScorer; +import org.apache.lucene.search.ConstantScoreWeight; +import org.apache.lucene.search.DocIdSetIterator; +import org.apache.lucene.search.DocValuesRangeIterator; +import org.apache.lucene.search.IndexSearcher; +import org.apache.lucene.search.Query; +import org.apache.lucene.search.QueryVisitor; +import org.apache.lucene.search.ScoreMode; +import org.apache.lucene.search.Scorer; +import org.apache.lucene.search.ScorerSupplier; +import org.apache.lucene.search.TwoPhaseIterator; +import org.apache.lucene.search.Weight; +import org.apache.lucene.util.ArrayUtil; +import org.apache.lucene.util.BytesRef; +import org.apache.lucene.util.LongBitSet; + +/** A union multiple ranges over SortedSetDocValuesField */ +public class SortedSetMultiRangeQuery extends Query { + private final String field; + private final int bytesPerDim; + private final ArrayUtil.ByteArrayComparator comparator; + List<MultiRangeQuery.RangeClause> rangeClauses; + + SortedSetMultiRangeQuery( + String name, + List<MultiRangeQuery.RangeClause> clauses, + int bytes, + ArrayUtil.ByteArrayComparator comparator) { + this.field = name; + this.rangeClauses = clauses; + this.bytesPerDim = bytes; + this.comparator = comparator; + } + + /** Builder for creating a SortedSetMultiRangeQuery. */ + public static class Builder { + private final String name; + protected final List<MultiRangeQuery.RangeClause> clauses = new ArrayList<>(); + private final int bytes; Review Comment: nit: maybe `bytesPerDim` (and renaming elsewhere as well to be consistent)? ########## lucene/sandbox/src/java/org/apache/lucene/sandbox/search/SortedSetMultiRangeQuery.java: ########## @@ -0,0 +1,300 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.lucene.sandbox.search; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; +import java.util.Objects; +import org.apache.lucene.document.SortedSetDocValuesField; +import org.apache.lucene.index.DocValues; +import org.apache.lucene.index.DocValuesSkipper; +import org.apache.lucene.index.LeafReaderContext; +import org.apache.lucene.index.SortedSetDocValues; +import org.apache.lucene.index.TermsEnum; +import org.apache.lucene.search.BooleanQuery; +import org.apache.lucene.search.ConstantScoreScorer; +import org.apache.lucene.search.ConstantScoreWeight; +import org.apache.lucene.search.DocIdSetIterator; +import org.apache.lucene.search.DocValuesRangeIterator; +import org.apache.lucene.search.IndexSearcher; +import org.apache.lucene.search.Query; +import org.apache.lucene.search.QueryVisitor; +import org.apache.lucene.search.ScoreMode; +import org.apache.lucene.search.Scorer; +import org.apache.lucene.search.ScorerSupplier; +import org.apache.lucene.search.TwoPhaseIterator; +import org.apache.lucene.search.Weight; +import org.apache.lucene.util.ArrayUtil; +import org.apache.lucene.util.BytesRef; +import org.apache.lucene.util.LongBitSet; + +/** A union multiple ranges over SortedSetDocValuesField */ +public class SortedSetMultiRangeQuery extends Query { + private final String field; + private final int bytesPerDim; + private final ArrayUtil.ByteArrayComparator comparator; + List<MultiRangeQuery.RangeClause> rangeClauses; + + SortedSetMultiRangeQuery( + String name, + List<MultiRangeQuery.RangeClause> clauses, + int bytes, + ArrayUtil.ByteArrayComparator comparator) { + this.field = name; + this.rangeClauses = clauses; + this.bytesPerDim = bytes; + this.comparator = comparator; + } + + /** Builder for creating a SortedSetMultiRangeQuery. */ + public static class Builder { + private final String name; + protected final List<MultiRangeQuery.RangeClause> clauses = new ArrayList<>(); + private final int bytes; + private final ArrayUtil.ByteArrayComparator comparator; + + public Builder(String name, int bytes) { + this.name = Objects.requireNonNull(name); + this.bytes = bytes; // TODO assrt positive + this.comparator = ArrayUtil.getUnsignedComparator(bytes); + } + + public Builder add(BytesRef lowerValue, BytesRef upperValue) { + byte[] low = lowerValue.clone().bytes; + byte[] up = upperValue.clone().bytes; + if (this.comparator.compare(low, 0, up, 0) > 0) { + throw new IllegalArgumentException("lowerValue must be <= upperValue"); + } else { + clauses.add(new MultiRangeQuery.RangeClause(low, up)); + } + return this; + } + + public Query build() { + if (clauses.isEmpty()) { + return new BooleanQuery.Builder().build(); + } + if (clauses.size() == 1) { + return SortedSetDocValuesField.newSlowRangeQuery( + name, + new BytesRef(clauses.getFirst().lowerValue), + new BytesRef(clauses.getFirst().upperValue), + true, + true); + } + return new SortedSetMultiRangeQuery(name, clauses, this.bytes, comparator); + } + } + + @Override + public Query rewrite(IndexSearcher indexSearcher) throws IOException { + ArrayList<MultiRangeQuery.RangeClause> sortedClauses = new ArrayList<>(this.rangeClauses); + sortedClauses.sort( + (o1, o2) -> { + // if (result == 0) { + // return comparator.compare(o1.upperValue, 0, o2.upperValue, 0); + // } else { + return comparator.compare(o1.lowerValue, 0, o2.lowerValue, 0); + // } + }); + if (!this.rangeClauses.equals(sortedClauses)) { + return new SortedSetMultiRangeQuery( + this.field, sortedClauses, this.bytesPerDim, this.comparator); + } else { + return this; + } + } + + @Override + public String toString(String fld) { + return "SortedSetMultiRangeQuery{" + + "field='" + + fld + + '\'' + + ", rangeClauses=" + + rangeClauses + + // TODO better toString + '}'; + } + + // what TODO with reverse ranges ??? + @Override + public Weight createWeight(IndexSearcher searcher, ScoreMode scoreMode, float boost) + throws IOException { + return new ConstantScoreWeight(this, boost) { + @Override + public ScorerSupplier scorerSupplier(LeafReaderContext context) throws IOException { + if (context.reader().getFieldInfos().fieldInfo(field) == null) { + return null; + } + DocValuesSkipper skipper = context.reader().getDocValuesSkipper(field); + SortedSetDocValues values = DocValues.getSortedSet(context.reader(), field); + // implement ScorerSupplier, since we do some expensive stuff to make a scorer + return new ScorerSupplier() { + @Override + public Scorer get(long leadCost) throws IOException { + if (rangeClauses.isEmpty()) { Review Comment: Can we make sure `rangeClauses` is never empty at this point? Ideally we would want to have returned a null scorer supplier if this was the case. But I think you have this already covered elsewhere in your builder logic? So maybe we can just `assert` here instead? ########## lucene/sandbox/src/java/org/apache/lucene/sandbox/search/SortedSetMultiRangeQuery.java: ########## @@ -0,0 +1,300 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.lucene.sandbox.search; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; +import java.util.Objects; +import org.apache.lucene.document.SortedSetDocValuesField; +import org.apache.lucene.index.DocValues; +import org.apache.lucene.index.DocValuesSkipper; +import org.apache.lucene.index.LeafReaderContext; +import org.apache.lucene.index.SortedSetDocValues; +import org.apache.lucene.index.TermsEnum; +import org.apache.lucene.search.BooleanQuery; +import org.apache.lucene.search.ConstantScoreScorer; +import org.apache.lucene.search.ConstantScoreWeight; +import org.apache.lucene.search.DocIdSetIterator; +import org.apache.lucene.search.DocValuesRangeIterator; +import org.apache.lucene.search.IndexSearcher; +import org.apache.lucene.search.Query; +import org.apache.lucene.search.QueryVisitor; +import org.apache.lucene.search.ScoreMode; +import org.apache.lucene.search.Scorer; +import org.apache.lucene.search.ScorerSupplier; +import org.apache.lucene.search.TwoPhaseIterator; +import org.apache.lucene.search.Weight; +import org.apache.lucene.util.ArrayUtil; +import org.apache.lucene.util.BytesRef; +import org.apache.lucene.util.LongBitSet; + +/** A union multiple ranges over SortedSetDocValuesField */ +public class SortedSetMultiRangeQuery extends Query { Review Comment: This is a bit of an odd companion to `MultiRangeQuery`. `MultiRangeQuery` seems meant for numeric-based points in the points index, while this is specific to term ranges. I'm not sure I have a great concrete suggestion here, but I think the way it's currently positioned (reusing some internal classes of MRQ and being in the same sandbox package) could be confusing. ########## lucene/sandbox/src/java/org/apache/lucene/sandbox/search/SortedSetMultiRangeQuery.java: ########## @@ -0,0 +1,300 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.lucene.sandbox.search; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; +import java.util.Objects; +import org.apache.lucene.document.SortedSetDocValuesField; +import org.apache.lucene.index.DocValues; +import org.apache.lucene.index.DocValuesSkipper; +import org.apache.lucene.index.LeafReaderContext; +import org.apache.lucene.index.SortedSetDocValues; +import org.apache.lucene.index.TermsEnum; +import org.apache.lucene.search.BooleanQuery; +import org.apache.lucene.search.ConstantScoreScorer; +import org.apache.lucene.search.ConstantScoreWeight; +import org.apache.lucene.search.DocIdSetIterator; +import org.apache.lucene.search.DocValuesRangeIterator; +import org.apache.lucene.search.IndexSearcher; +import org.apache.lucene.search.Query; +import org.apache.lucene.search.QueryVisitor; +import org.apache.lucene.search.ScoreMode; +import org.apache.lucene.search.Scorer; +import org.apache.lucene.search.ScorerSupplier; +import org.apache.lucene.search.TwoPhaseIterator; +import org.apache.lucene.search.Weight; +import org.apache.lucene.util.ArrayUtil; +import org.apache.lucene.util.BytesRef; +import org.apache.lucene.util.LongBitSet; + +/** A union multiple ranges over SortedSetDocValuesField */ +public class SortedSetMultiRangeQuery extends Query { + private final String field; + private final int bytesPerDim; + private final ArrayUtil.ByteArrayComparator comparator; + List<MultiRangeQuery.RangeClause> rangeClauses; + + SortedSetMultiRangeQuery( + String name, + List<MultiRangeQuery.RangeClause> clauses, + int bytes, + ArrayUtil.ByteArrayComparator comparator) { + this.field = name; + this.rangeClauses = clauses; + this.bytesPerDim = bytes; + this.comparator = comparator; + } + + /** Builder for creating a SortedSetMultiRangeQuery. */ + public static class Builder { + private final String name; + protected final List<MultiRangeQuery.RangeClause> clauses = new ArrayList<>(); + private final int bytes; + private final ArrayUtil.ByteArrayComparator comparator; + + public Builder(String name, int bytes) { + this.name = Objects.requireNonNull(name); + this.bytes = bytes; // TODO assrt positive + this.comparator = ArrayUtil.getUnsignedComparator(bytes); + } + + public Builder add(BytesRef lowerValue, BytesRef upperValue) { + byte[] low = lowerValue.clone().bytes; + byte[] up = upperValue.clone().bytes; + if (this.comparator.compare(low, 0, up, 0) > 0) { + throw new IllegalArgumentException("lowerValue must be <= upperValue"); + } else { + clauses.add(new MultiRangeQuery.RangeClause(low, up)); + } + return this; + } + + public Query build() { + if (clauses.isEmpty()) { + return new BooleanQuery.Builder().build(); + } + if (clauses.size() == 1) { + return SortedSetDocValuesField.newSlowRangeQuery( + name, + new BytesRef(clauses.getFirst().lowerValue), + new BytesRef(clauses.getFirst().upperValue), + true, + true); + } + return new SortedSetMultiRangeQuery(name, clauses, this.bytes, comparator); + } + } + + @Override + public Query rewrite(IndexSearcher indexSearcher) throws IOException { + ArrayList<MultiRangeQuery.RangeClause> sortedClauses = new ArrayList<>(this.rangeClauses); + sortedClauses.sort( + (o1, o2) -> { + // if (result == 0) { + // return comparator.compare(o1.upperValue, 0, o2.upperValue, 0); + // } else { + return comparator.compare(o1.lowerValue, 0, o2.lowerValue, 0); + // } + }); + if (!this.rangeClauses.equals(sortedClauses)) { + return new SortedSetMultiRangeQuery( + this.field, sortedClauses, this.bytesPerDim, this.comparator); + } else { + return this; + } + } + + @Override + public String toString(String fld) { + return "SortedSetMultiRangeQuery{" + + "field='" + + fld + + '\'' + + ", rangeClauses=" + + rangeClauses + + // TODO better toString + '}'; + } + + // what TODO with reverse ranges ??? + @Override + public Weight createWeight(IndexSearcher searcher, ScoreMode scoreMode, float boost) + throws IOException { + return new ConstantScoreWeight(this, boost) { + @Override + public ScorerSupplier scorerSupplier(LeafReaderContext context) throws IOException { + if (context.reader().getFieldInfos().fieldInfo(field) == null) { + return null; + } + DocValuesSkipper skipper = context.reader().getDocValuesSkipper(field); Review Comment: Let's move these lines inside `#get`. No need to do this work up-front in case a scorer is never pulled. ########## lucene/sandbox/src/java/org/apache/lucene/sandbox/search/SortedSetMultiRangeQuery.java: ########## @@ -0,0 +1,300 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.lucene.sandbox.search; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; +import java.util.Objects; +import org.apache.lucene.document.SortedSetDocValuesField; +import org.apache.lucene.index.DocValues; +import org.apache.lucene.index.DocValuesSkipper; +import org.apache.lucene.index.LeafReaderContext; +import org.apache.lucene.index.SortedSetDocValues; +import org.apache.lucene.index.TermsEnum; +import org.apache.lucene.search.BooleanQuery; +import org.apache.lucene.search.ConstantScoreScorer; +import org.apache.lucene.search.ConstantScoreWeight; +import org.apache.lucene.search.DocIdSetIterator; +import org.apache.lucene.search.DocValuesRangeIterator; +import org.apache.lucene.search.IndexSearcher; +import org.apache.lucene.search.Query; +import org.apache.lucene.search.QueryVisitor; +import org.apache.lucene.search.ScoreMode; +import org.apache.lucene.search.Scorer; +import org.apache.lucene.search.ScorerSupplier; +import org.apache.lucene.search.TwoPhaseIterator; +import org.apache.lucene.search.Weight; +import org.apache.lucene.util.ArrayUtil; +import org.apache.lucene.util.BytesRef; +import org.apache.lucene.util.LongBitSet; + +/** A union multiple ranges over SortedSetDocValuesField */ +public class SortedSetMultiRangeQuery extends Query { + private final String field; + private final int bytesPerDim; + private final ArrayUtil.ByteArrayComparator comparator; + List<MultiRangeQuery.RangeClause> rangeClauses; + + SortedSetMultiRangeQuery( + String name, + List<MultiRangeQuery.RangeClause> clauses, + int bytes, + ArrayUtil.ByteArrayComparator comparator) { + this.field = name; + this.rangeClauses = clauses; + this.bytesPerDim = bytes; + this.comparator = comparator; + } + + /** Builder for creating a SortedSetMultiRangeQuery. */ + public static class Builder { + private final String name; Review Comment: nit: maybe `fieldName`? ########## lucene/sandbox/src/java/org/apache/lucene/sandbox/search/SortedSetMultiRangeQuery.java: ########## @@ -0,0 +1,300 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.lucene.sandbox.search; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; +import java.util.Objects; +import org.apache.lucene.document.SortedSetDocValuesField; +import org.apache.lucene.index.DocValues; +import org.apache.lucene.index.DocValuesSkipper; +import org.apache.lucene.index.LeafReaderContext; +import org.apache.lucene.index.SortedSetDocValues; +import org.apache.lucene.index.TermsEnum; +import org.apache.lucene.search.BooleanQuery; +import org.apache.lucene.search.ConstantScoreScorer; +import org.apache.lucene.search.ConstantScoreWeight; +import org.apache.lucene.search.DocIdSetIterator; +import org.apache.lucene.search.DocValuesRangeIterator; +import org.apache.lucene.search.IndexSearcher; +import org.apache.lucene.search.Query; +import org.apache.lucene.search.QueryVisitor; +import org.apache.lucene.search.ScoreMode; +import org.apache.lucene.search.Scorer; +import org.apache.lucene.search.ScorerSupplier; +import org.apache.lucene.search.TwoPhaseIterator; +import org.apache.lucene.search.Weight; +import org.apache.lucene.util.ArrayUtil; +import org.apache.lucene.util.BytesRef; +import org.apache.lucene.util.LongBitSet; + +/** A union multiple ranges over SortedSetDocValuesField */ +public class SortedSetMultiRangeQuery extends Query { + private final String field; + private final int bytesPerDim; + private final ArrayUtil.ByteArrayComparator comparator; + List<MultiRangeQuery.RangeClause> rangeClauses; + + SortedSetMultiRangeQuery( + String name, + List<MultiRangeQuery.RangeClause> clauses, + int bytes, + ArrayUtil.ByteArrayComparator comparator) { + this.field = name; + this.rangeClauses = clauses; + this.bytesPerDim = bytes; + this.comparator = comparator; + } + + /** Builder for creating a SortedSetMultiRangeQuery. */ + public static class Builder { + private final String name; + protected final List<MultiRangeQuery.RangeClause> clauses = new ArrayList<>(); + private final int bytes; + private final ArrayUtil.ByteArrayComparator comparator; + + public Builder(String name, int bytes) { + this.name = Objects.requireNonNull(name); + this.bytes = bytes; // TODO assrt positive + this.comparator = ArrayUtil.getUnsignedComparator(bytes); + } + + public Builder add(BytesRef lowerValue, BytesRef upperValue) { + byte[] low = lowerValue.clone().bytes; + byte[] up = upperValue.clone().bytes; + if (this.comparator.compare(low, 0, up, 0) > 0) { + throw new IllegalArgumentException("lowerValue must be <= upperValue"); + } else { + clauses.add(new MultiRangeQuery.RangeClause(low, up)); + } + return this; + } + + public Query build() { + if (clauses.isEmpty()) { + return new BooleanQuery.Builder().build(); + } + if (clauses.size() == 1) { + return SortedSetDocValuesField.newSlowRangeQuery( + name, + new BytesRef(clauses.getFirst().lowerValue), + new BytesRef(clauses.getFirst().upperValue), + true, + true); + } + return new SortedSetMultiRangeQuery(name, clauses, this.bytes, comparator); + } + } + + @Override + public Query rewrite(IndexSearcher indexSearcher) throws IOException { + ArrayList<MultiRangeQuery.RangeClause> sortedClauses = new ArrayList<>(this.rangeClauses); + sortedClauses.sort( + (o1, o2) -> { + // if (result == 0) { + // return comparator.compare(o1.upperValue, 0, o2.upperValue, 0); + // } else { + return comparator.compare(o1.lowerValue, 0, o2.lowerValue, 0); + // } + }); + if (!this.rangeClauses.equals(sortedClauses)) { + return new SortedSetMultiRangeQuery( + this.field, sortedClauses, this.bytesPerDim, this.comparator); + } else { + return this; + } + } + + @Override + public String toString(String fld) { + return "SortedSetMultiRangeQuery{" + + "field='" + + fld + + '\'' + + ", rangeClauses=" + + rangeClauses + + // TODO better toString + '}'; + } + + // what TODO with reverse ranges ??? + @Override + public Weight createWeight(IndexSearcher searcher, ScoreMode scoreMode, float boost) + throws IOException { + return new ConstantScoreWeight(this, boost) { + @Override + public ScorerSupplier scorerSupplier(LeafReaderContext context) throws IOException { + if (context.reader().getFieldInfos().fieldInfo(field) == null) { + return null; + } + DocValuesSkipper skipper = context.reader().getDocValuesSkipper(field); + SortedSetDocValues values = DocValues.getSortedSet(context.reader(), field); + // implement ScorerSupplier, since we do some expensive stuff to make a scorer + return new ScorerSupplier() { + @Override + public Scorer get(long leadCost) throws IOException { + if (rangeClauses.isEmpty()) { + return empty(); + } + TermsEnum termsEnum = values.termsEnum(); + LongBitSet matchingOrdsShifted = null; + long minOrd = 0, maxOrd = values.getValueCount() - 1; + long matchesAbove = + values.getValueCount(); // it's last range goes to maxOrd, by default - no match + long maxSeenOrd = values.getValueCount(); + TermsEnum.SeekStatus seekStatus = TermsEnum.SeekStatus.NOT_FOUND; + for (int r = 0; r < rangeClauses.size(); r++) { + MultiRangeQuery.RangeClause range = rangeClauses.get(r); + long startingOrd; + seekStatus = termsEnum.seekCeil(new BytesRef(range.lowerValue)); + if (matchingOrdsShifted == null) { // first iter + if (seekStatus == TermsEnum.SeekStatus.END) { + return empty(); // no bitset yet, give up + } + minOrd = termsEnum.ord(); + if (skipper != null) { Review Comment: I'm not really sure I understand how the skipper helps us here. Any ordinal you get from the terms enum is going to be present somewhere in the segment, so by definition, I don't see how the global min or max from the skipper could provide tighter bounds. Maybe I'm missing something? ########## lucene/sandbox/src/java/org/apache/lucene/sandbox/search/SortedSetMultiRangeQuery.java: ########## @@ -0,0 +1,300 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.lucene.sandbox.search; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; +import java.util.Objects; +import org.apache.lucene.document.SortedSetDocValuesField; +import org.apache.lucene.index.DocValues; +import org.apache.lucene.index.DocValuesSkipper; +import org.apache.lucene.index.LeafReaderContext; +import org.apache.lucene.index.SortedSetDocValues; +import org.apache.lucene.index.TermsEnum; +import org.apache.lucene.search.BooleanQuery; +import org.apache.lucene.search.ConstantScoreScorer; +import org.apache.lucene.search.ConstantScoreWeight; +import org.apache.lucene.search.DocIdSetIterator; +import org.apache.lucene.search.DocValuesRangeIterator; +import org.apache.lucene.search.IndexSearcher; +import org.apache.lucene.search.Query; +import org.apache.lucene.search.QueryVisitor; +import org.apache.lucene.search.ScoreMode; +import org.apache.lucene.search.Scorer; +import org.apache.lucene.search.ScorerSupplier; +import org.apache.lucene.search.TwoPhaseIterator; +import org.apache.lucene.search.Weight; +import org.apache.lucene.util.ArrayUtil; +import org.apache.lucene.util.BytesRef; +import org.apache.lucene.util.LongBitSet; + +/** A union multiple ranges over SortedSetDocValuesField */ +public class SortedSetMultiRangeQuery extends Query { + private final String field; + private final int bytesPerDim; + private final ArrayUtil.ByteArrayComparator comparator; + List<MultiRangeQuery.RangeClause> rangeClauses; + + SortedSetMultiRangeQuery( + String name, + List<MultiRangeQuery.RangeClause> clauses, + int bytes, + ArrayUtil.ByteArrayComparator comparator) { + this.field = name; + this.rangeClauses = clauses; + this.bytesPerDim = bytes; + this.comparator = comparator; + } + + /** Builder for creating a SortedSetMultiRangeQuery. */ + public static class Builder { + private final String name; + protected final List<MultiRangeQuery.RangeClause> clauses = new ArrayList<>(); + private final int bytes; + private final ArrayUtil.ByteArrayComparator comparator; + + public Builder(String name, int bytes) { + this.name = Objects.requireNonNull(name); + this.bytes = bytes; // TODO assrt positive + this.comparator = ArrayUtil.getUnsignedComparator(bytes); + } + + public Builder add(BytesRef lowerValue, BytesRef upperValue) { + byte[] low = lowerValue.clone().bytes; + byte[] up = upperValue.clone().bytes; + if (this.comparator.compare(low, 0, up, 0) > 0) { + throw new IllegalArgumentException("lowerValue must be <= upperValue"); + } else { + clauses.add(new MultiRangeQuery.RangeClause(low, up)); + } + return this; + } + + public Query build() { + if (clauses.isEmpty()) { + return new BooleanQuery.Builder().build(); + } + if (clauses.size() == 1) { + return SortedSetDocValuesField.newSlowRangeQuery( + name, + new BytesRef(clauses.getFirst().lowerValue), + new BytesRef(clauses.getFirst().upperValue), + true, + true); + } + return new SortedSetMultiRangeQuery(name, clauses, this.bytes, comparator); + } + } + + @Override + public Query rewrite(IndexSearcher indexSearcher) throws IOException { + ArrayList<MultiRangeQuery.RangeClause> sortedClauses = new ArrayList<>(this.rangeClauses); + sortedClauses.sort( + (o1, o2) -> { + // if (result == 0) { + // return comparator.compare(o1.upperValue, 0, o2.upperValue, 0); + // } else { + return comparator.compare(o1.lowerValue, 0, o2.lowerValue, 0); + // } + }); + if (!this.rangeClauses.equals(sortedClauses)) { + return new SortedSetMultiRangeQuery( + this.field, sortedClauses, this.bytesPerDim, this.comparator); + } else { + return this; + } + } + + @Override + public String toString(String fld) { + return "SortedSetMultiRangeQuery{" + + "field='" + + fld + + '\'' + + ", rangeClauses=" + + rangeClauses + + // TODO better toString + '}'; + } + + // what TODO with reverse ranges ??? + @Override + public Weight createWeight(IndexSearcher searcher, ScoreMode scoreMode, float boost) + throws IOException { + return new ConstantScoreWeight(this, boost) { + @Override + public ScorerSupplier scorerSupplier(LeafReaderContext context) throws IOException { + if (context.reader().getFieldInfos().fieldInfo(field) == null) { + return null; + } + DocValuesSkipper skipper = context.reader().getDocValuesSkipper(field); + SortedSetDocValues values = DocValues.getSortedSet(context.reader(), field); + // implement ScorerSupplier, since we do some expensive stuff to make a scorer + return new ScorerSupplier() { + @Override + public Scorer get(long leadCost) throws IOException { + if (rangeClauses.isEmpty()) { + return empty(); + } + TermsEnum termsEnum = values.termsEnum(); + LongBitSet matchingOrdsShifted = null; + long minOrd = 0, maxOrd = values.getValueCount() - 1; + long matchesAbove = + values.getValueCount(); // it's last range goes to maxOrd, by default - no match + long maxSeenOrd = values.getValueCount(); + TermsEnum.SeekStatus seekStatus = TermsEnum.SeekStatus.NOT_FOUND; + for (int r = 0; r < rangeClauses.size(); r++) { + MultiRangeQuery.RangeClause range = rangeClauses.get(r); + long startingOrd; + seekStatus = termsEnum.seekCeil(new BytesRef(range.lowerValue)); Review Comment: There's a fair amount of `BytesRef` creation going on here. Should we just store bytes ref instances in the ranges instead of storing byte arrays then converting here? -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: issues-unsubscr...@lucene.apache.org For queries about this service, please contact Infrastructure at: us...@infra.apache.org --------------------------------------------------------------------- To unsubscribe, e-mail: issues-unsubscr...@lucene.apache.org For additional commands, e-mail: issues-h...@lucene.apache.org
