Jackie-Jiang commented on a change in pull request #7141: URL: https://github.com/apache/incubator-pinot/pull/7141#discussion_r667126687
########## File path: pinot-core/src/main/java/org/apache/pinot/core/operator/query/DictionaryBasedDistinctOperator.java ########## @@ -0,0 +1,142 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.pinot.core.operator.query; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import org.apache.pinot.common.request.context.ExpressionContext; +import org.apache.pinot.common.request.context.OrderByExpressionContext; +import org.apache.pinot.common.utils.DataSchema; +import org.apache.pinot.core.data.table.Record; +import org.apache.pinot.core.operator.ExecutionStatistics; +import org.apache.pinot.core.operator.blocks.IntermediateResultsBlock; +import org.apache.pinot.core.operator.transform.TransformOperator; +import org.apache.pinot.core.query.aggregation.function.AggregationFunction; +import org.apache.pinot.core.query.aggregation.function.DistinctAggregationFunction; +import org.apache.pinot.core.query.distinct.DistinctTable; +import org.apache.pinot.segment.spi.AggregationFunctionType; +import org.apache.pinot.segment.spi.IndexSegment; +import org.apache.pinot.segment.spi.index.reader.Dictionary; +import org.apache.pinot.spi.data.FieldSpec; + + +/** + * Operator which executes DISTINCT operation based on dictionary + */ +public class DictionaryBasedDistinctOperator extends DistinctOperator { + private static final String OPERATOR_NAME = "DictionaryBasedDistinctOperator"; + + private final DistinctAggregationFunction _distinctAggregationFunction; + private final Dictionary _dictionary; + private final int _numTotalDocs; + private final TransformOperator _transformOperator; + + private boolean _hasOrderBy; + private boolean _isAscending; + + public DictionaryBasedDistinctOperator(IndexSegment indexSegment, DistinctAggregationFunction distinctAggregationFunction, + Dictionary dictionary, int numTotalDocs, + TransformOperator transformOperator) { + super(indexSegment, distinctAggregationFunction, transformOperator); + + _distinctAggregationFunction = distinctAggregationFunction; + _dictionary = dictionary; + _numTotalDocs = numTotalDocs; + _transformOperator = transformOperator; + + List<OrderByExpressionContext> orderByExpressionContexts = _distinctAggregationFunction.getOrderByExpressions(); + + if (orderByExpressionContexts != null) { + OrderByExpressionContext orderByExpressionContext = orderByExpressionContexts.get(0); + + _isAscending = orderByExpressionContext.isAsc(); + _hasOrderBy = true; + } + } + + @Override + protected IntermediateResultsBlock getNextBlock() { + DistinctTable distinctTable = buildResult(); + + return new IntermediateResultsBlock(new AggregationFunction[]{_distinctAggregationFunction}, + Collections.singletonList(distinctTable), false); + } + + /** + * Build the final result for this operation + */ + private DistinctTable buildResult() { + + assert _distinctAggregationFunction.getType() == AggregationFunctionType.DISTINCT; + + List<ExpressionContext> expressions = _distinctAggregationFunction.getInputExpressions(); + ExpressionContext expression = expressions.get(0); + FieldSpec.DataType dataType = _transformOperator.getResultMetadata(expression).getDataType(); + + DataSchema dataSchema = new DataSchema(new String[]{expression.toString()}, + new DataSchema.ColumnDataType[]{DataSchema.ColumnDataType.fromDataTypeSV(dataType)}); + List<Record> records; + + int limit = _distinctAggregationFunction.getLimit(); + int actualLimit = Math.min(limit, _dictionary.length()); + + // If ORDER BY is not present, we read the first limit values from the dictionary and return. + // If ORDER BY is present and the dictionary is sorted, then we read the first/last limit values + // from the dictionary. If not sorted, then we read the entire dictionary and return it. + if (!_hasOrderBy) { + records = new ArrayList<>(actualLimit); + + for (int i = 0; i < actualLimit; i++) { + records.add(new Record(new Object[]{_dictionary.getInternal(i)})); + } + } else { + if (_dictionary.isSorted()) { + records = new ArrayList<>(actualLimit); + if (_isAscending) { + for (int i = 0; i < actualLimit; i++) { + records.add(new Record(new Object[]{_dictionary.getInternal(i)})); + } + } else { + for (int i = _dictionary.length() - 1; i >= (_dictionary.length() - actualLimit); i--) { + records.add(new Record(new Object[]{_dictionary.getInternal(i)})); + } + } + } else { + records = new ArrayList<>(_dictionary.length()); Review comment: We should insert records into the `DistinctTable`. `DistinctTable` will only keep the top records. ```suggestion DistinctTable distinctTable = new DistinctTable(dataSchema, _distinctAggregationFunction.getOrderByExpressions(), limit); for (int i = 0; i < dictionarySize; i++) { distinctTable.addWithOrderBy(new Record(new Object[]{_dictionary.getInternal(i)})); } ``` ########## File path: pinot-core/src/test/java/org/apache/pinot/queries/DistinctQueriesTest.java ########## @@ -245,7 +245,34 @@ public void testSingleColumnDistinctOnlyInnerSegment() // String columns //@formatter:off List<String> queries = Arrays Review comment: Since there is only one query, let's use a single value instead of a list. We can also remove the comment to turn off/on the formatter ########## File path: pinot-core/src/test/java/org/apache/pinot/queries/DistinctQueriesTest.java ########## @@ -245,7 +245,34 @@ public void testSingleColumnDistinctOnlyInnerSegment() // String columns //@formatter:off List<String> queries = Arrays - .asList("SELECT DISTINCT(stringColumn) FROM testTable", "SELECT DISTINCT(rawStringColumn) FROM testTable"); + .asList("SELECT DISTINCT(stringColumn) FROM testTable"); + //@formatter:on + Set<Integer> expectedValues = + new HashSet<>(Arrays.asList(0, 16, 17, 1, 10, 11, 12, 13, 14, 15)); Review comment: (nit) for clarity (alphabetical order) ```suggestion new HashSet<>(Arrays.asList(0, 1, 10, 11, 12, 13, 14, 15, 16, 17)); ``` ########## File path: pinot-core/src/main/java/org/apache/pinot/core/operator/query/DictionaryBasedDistinctOperator.java ########## @@ -0,0 +1,163 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.pinot.core.operator.query; + +import it.unimi.dsi.fastutil.ints.IntHeapPriorityQueue; +import it.unimi.dsi.fastutil.ints.IntOpenHashSet; +import it.unimi.dsi.fastutil.ints.IntPriorityQueue; +import java.util.ArrayList; +import java.util.Collections; +import java.util.Iterator; +import java.util.List; +import java.util.Map; +import org.apache.pinot.common.request.context.ExpressionContext; +import org.apache.pinot.common.request.context.OrderByExpressionContext; +import org.apache.pinot.common.utils.DataSchema; +import org.apache.pinot.core.data.table.Record; +import org.apache.pinot.core.operator.ExecutionStatistics; +import org.apache.pinot.core.operator.blocks.IntermediateResultsBlock; +import org.apache.pinot.core.operator.transform.TransformOperator; +import org.apache.pinot.core.query.aggregation.function.AggregationFunction; +import org.apache.pinot.core.query.aggregation.function.DistinctAggregationFunction; +import org.apache.pinot.core.query.distinct.DistinctTable; +import org.apache.pinot.segment.spi.AggregationFunctionType; +import org.apache.pinot.segment.spi.IndexSegment; +import org.apache.pinot.segment.spi.index.reader.Dictionary; +import org.apache.pinot.spi.data.FieldSpec; + +/** + * Operator which executes DISTINCT operation based on dictionary + */ +public class DictionaryBasedDistinctOperator extends DistinctOperator { + private static final String OPERATOR_NAME = "DictionaryBasedDistinctOperator"; Review comment: I don't think it takes effect. The indentation should be 2 spaces instead of 4 ########## File path: pinot-core/src/test/java/org/apache/pinot/queries/DistinctQueriesTest.java ########## @@ -245,7 +245,34 @@ public void testSingleColumnDistinctOnlyInnerSegment() // String columns //@formatter:off List<String> queries = Arrays - .asList("SELECT DISTINCT(stringColumn) FROM testTable", "SELECT DISTINCT(rawStringColumn) FROM testTable"); + .asList("SELECT DISTINCT(stringColumn) FROM testTable"); + //@formatter:on + Set<Integer> expectedValues = + new HashSet<>(Arrays.asList(0, 16, 17, 1, 10, 11, 12, 13, 14, 15)); + for (String query : queries) { + DistinctTable pqlDistinctTable = getDistinctTableInnerSegment(query, true); + DistinctTable pqlDistinctTable2 = DistinctTable.fromByteBuffer(ByteBuffer.wrap(pqlDistinctTable.toBytes())); + DistinctTable sqlDistinctTable = getDistinctTableInnerSegment(query, false); + DistinctTable sqlDistinctTable2 = DistinctTable.fromByteBuffer(ByteBuffer.wrap(sqlDistinctTable.toBytes())); + for (DistinctTable distinctTable : Arrays + .asList(pqlDistinctTable, pqlDistinctTable2, sqlDistinctTable, sqlDistinctTable2)) { + assertEquals(distinctTable.size(), 10); + Set<Integer> actualValues = new HashSet<>(); + for (Record record : distinctTable.getRecords()) { + Object[] values = record.getValues(); + assertEquals(values.length, 1); + assertTrue(values[0] instanceof String); + actualValues.add(Integer.parseInt((String) values[0])); + } + assertEquals(actualValues, expectedValues); + } + } + } + { + // String columns + //@formatter:off + List<String> queries = Arrays Review comment: Same here, use single value ########## File path: pinot-core/src/main/java/org/apache/pinot/core/operator/query/DictionaryBasedDistinctOperator.java ########## @@ -0,0 +1,142 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.pinot.core.operator.query; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import org.apache.pinot.common.request.context.ExpressionContext; +import org.apache.pinot.common.request.context.OrderByExpressionContext; +import org.apache.pinot.common.utils.DataSchema; +import org.apache.pinot.core.data.table.Record; +import org.apache.pinot.core.operator.ExecutionStatistics; +import org.apache.pinot.core.operator.blocks.IntermediateResultsBlock; +import org.apache.pinot.core.operator.transform.TransformOperator; +import org.apache.pinot.core.query.aggregation.function.AggregationFunction; +import org.apache.pinot.core.query.aggregation.function.DistinctAggregationFunction; +import org.apache.pinot.core.query.distinct.DistinctTable; +import org.apache.pinot.segment.spi.AggregationFunctionType; +import org.apache.pinot.segment.spi.IndexSegment; +import org.apache.pinot.segment.spi.index.reader.Dictionary; +import org.apache.pinot.spi.data.FieldSpec; + + +/** + * Operator which executes DISTINCT operation based on dictionary + */ +public class DictionaryBasedDistinctOperator extends DistinctOperator { + private static final String OPERATOR_NAME = "DictionaryBasedDistinctOperator"; + + private final DistinctAggregationFunction _distinctAggregationFunction; + private final Dictionary _dictionary; + private final int _numTotalDocs; + private final TransformOperator _transformOperator; + + private boolean _hasOrderBy; + private boolean _isAscending; + + public DictionaryBasedDistinctOperator(IndexSegment indexSegment, DistinctAggregationFunction distinctAggregationFunction, + Dictionary dictionary, int numTotalDocs, + TransformOperator transformOperator) { + super(indexSegment, distinctAggregationFunction, transformOperator); + + _distinctAggregationFunction = distinctAggregationFunction; + _dictionary = dictionary; + _numTotalDocs = numTotalDocs; + _transformOperator = transformOperator; + + List<OrderByExpressionContext> orderByExpressionContexts = _distinctAggregationFunction.getOrderByExpressions(); + + if (orderByExpressionContexts != null) { + OrderByExpressionContext orderByExpressionContext = orderByExpressionContexts.get(0); + + _isAscending = orderByExpressionContext.isAsc(); + _hasOrderBy = true; + } + } + + @Override + protected IntermediateResultsBlock getNextBlock() { + DistinctTable distinctTable = buildResult(); + + return new IntermediateResultsBlock(new AggregationFunction[]{_distinctAggregationFunction}, + Collections.singletonList(distinctTable), false); + } + + /** + * Build the final result for this operation + */ + private DistinctTable buildResult() { + + assert _distinctAggregationFunction.getType() == AggregationFunctionType.DISTINCT; + + List<ExpressionContext> expressions = _distinctAggregationFunction.getInputExpressions(); + ExpressionContext expression = expressions.get(0); + FieldSpec.DataType dataType = _transformOperator.getResultMetadata(expression).getDataType(); + + DataSchema dataSchema = new DataSchema(new String[]{expression.toString()}, + new DataSchema.ColumnDataType[]{DataSchema.ColumnDataType.fromDataTypeSV(dataType)}); + List<Record> records; + + int limit = _distinctAggregationFunction.getLimit(); + int actualLimit = Math.min(limit, _dictionary.length()); Review comment: Let's cache the value of `_dictionary.length()` into a local variable ########## File path: pinot-core/src/test/java/org/apache/pinot/queries/DistinctQueriesTest.java ########## @@ -245,12 +245,14 @@ public void testSingleColumnDistinctOnlyInnerSegment() // String columns //@formatter:off List<String> queries = Arrays - .asList("SELECT DISTINCT(stringColumn) FROM testTable", "SELECT DISTINCT(rawStringColumn) FROM testTable"); + .asList("SELECT DISTINCT(stringColumn) FROM testTable"); Review comment: I see. This is caused by sorting in alphabetical order. Can you please add some comments explaining the reason? ########## File path: pinot-core/src/main/java/org/apache/pinot/core/plan/DictionaryBasedDistinctPlanNode.java ########## @@ -0,0 +1,64 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.pinot.core.plan; + +import org.apache.pinot.core.operator.query.DictionaryBasedDistinctOperator; +import org.apache.pinot.core.query.aggregation.function.AggregationFunction; +import org.apache.pinot.core.query.aggregation.function.DistinctAggregationFunction; +import org.apache.pinot.core.query.request.context.QueryContext; +import org.apache.pinot.segment.spi.IndexSegment; +import org.apache.pinot.segment.spi.index.reader.Dictionary; + +/** + * Execute a DISTINCT operation using dictionary based plan + */ +public class DictionaryBasedDistinctPlanNode implements PlanNode { + private final IndexSegment _indexSegment; + private final DistinctAggregationFunction _distinctAggregationFunction; + private final Dictionary _dictionary; + private final TransformPlanNode _transformPlanNode; + + /** + * Constructor for the class. + * + * @param indexSegment Segment to process + * @param queryContext Query context + */ + public DictionaryBasedDistinctPlanNode(IndexSegment indexSegment, QueryContext queryContext, Dictionary dictionary) { + _indexSegment = indexSegment; + AggregationFunction[] aggregationFunctions = queryContext.getAggregationFunctions(); + + assert aggregationFunctions != null && aggregationFunctions.length == 1 + && aggregationFunctions[0] instanceof DistinctAggregationFunction; + + _distinctAggregationFunction = (DistinctAggregationFunction) aggregationFunctions[0]; + + _dictionary = dictionary; + + _transformPlanNode = Review comment: Data type is available in by `Dictionary.getValueType()`. We want to short-circuit all the lower level operators -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: commits-unsubscr...@pinot.apache.org For queries about this service, please contact Infrastructure at: us...@infra.apache.org --------------------------------------------------------------------- To unsubscribe, e-mail: commits-unsubscr...@pinot.apache.org For additional commands, e-mail: commits-h...@pinot.apache.org
