Jackie-Jiang commented on a change in pull request #7141: URL: https://github.com/apache/incubator-pinot/pull/7141#discussion_r667211807
########## File path: pinot-core/src/main/java/org/apache/pinot/core/plan/DictionaryBasedDistinctPlanNode.java ########## @@ -0,0 +1,64 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.pinot.core.plan; + +import org.apache.pinot.core.operator.query.DictionaryBasedDistinctOperator; +import org.apache.pinot.core.query.aggregation.function.AggregationFunction; +import org.apache.pinot.core.query.aggregation.function.DistinctAggregationFunction; +import org.apache.pinot.core.query.request.context.QueryContext; +import org.apache.pinot.segment.spi.IndexSegment; +import org.apache.pinot.segment.spi.index.reader.Dictionary; + +/** + * Execute a DISTINCT operation using dictionary based plan + */ +public class DictionaryBasedDistinctPlanNode implements PlanNode { + private final IndexSegment _indexSegment; + private final DistinctAggregationFunction _distinctAggregationFunction; + private final Dictionary _dictionary; + private final TransformPlanNode _transformPlanNode; + + /** + * Constructor for the class. + * + * @param indexSegment Segment to process + * @param queryContext Query context + */ + public DictionaryBasedDistinctPlanNode(IndexSegment indexSegment, QueryContext queryContext, Dictionary dictionary) { + _indexSegment = indexSegment; + AggregationFunction[] aggregationFunctions = queryContext.getAggregationFunctions(); + + assert aggregationFunctions != null && aggregationFunctions.length == 1 + && aggregationFunctions[0] instanceof DistinctAggregationFunction; + + _distinctAggregationFunction = (DistinctAggregationFunction) aggregationFunctions[0]; + + _dictionary = dictionary; + + _transformPlanNode = Review comment: No need to create the transform plan ########## File path: pinot-core/src/main/java/org/apache/pinot/core/operator/query/DictionaryBasedDistinctOperator.java ########## @@ -0,0 +1,146 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.pinot.core.operator.query; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import org.apache.pinot.common.request.context.ExpressionContext; +import org.apache.pinot.common.request.context.OrderByExpressionContext; +import org.apache.pinot.common.utils.DataSchema; +import org.apache.pinot.core.data.table.Record; +import org.apache.pinot.core.operator.ExecutionStatistics; +import org.apache.pinot.core.operator.blocks.IntermediateResultsBlock; +import org.apache.pinot.core.operator.transform.TransformOperator; +import org.apache.pinot.core.query.aggregation.function.AggregationFunction; +import org.apache.pinot.core.query.aggregation.function.DistinctAggregationFunction; +import org.apache.pinot.core.query.distinct.DistinctTable; +import org.apache.pinot.segment.spi.AggregationFunctionType; +import org.apache.pinot.segment.spi.IndexSegment; +import org.apache.pinot.segment.spi.index.reader.Dictionary; +import org.apache.pinot.spi.data.FieldSpec; + + +/** + * Operator which executes DISTINCT operation based on dictionary + */ +public class DictionaryBasedDistinctOperator extends DistinctOperator { Review comment: We should extends `BaseOperator<IntermediateResultsBlock>` instead of `DistinctOperator` ########## File path: pinot-core/src/main/java/org/apache/pinot/core/operator/query/DictionaryBasedDistinctOperator.java ########## @@ -0,0 +1,146 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.pinot.core.operator.query; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import org.apache.pinot.common.request.context.ExpressionContext; +import org.apache.pinot.common.request.context.OrderByExpressionContext; +import org.apache.pinot.common.utils.DataSchema; +import org.apache.pinot.core.data.table.Record; +import org.apache.pinot.core.operator.ExecutionStatistics; +import org.apache.pinot.core.operator.blocks.IntermediateResultsBlock; +import org.apache.pinot.core.operator.transform.TransformOperator; +import org.apache.pinot.core.query.aggregation.function.AggregationFunction; +import org.apache.pinot.core.query.aggregation.function.DistinctAggregationFunction; +import org.apache.pinot.core.query.distinct.DistinctTable; +import org.apache.pinot.segment.spi.AggregationFunctionType; +import org.apache.pinot.segment.spi.IndexSegment; +import org.apache.pinot.segment.spi.index.reader.Dictionary; +import org.apache.pinot.spi.data.FieldSpec; + + +/** + * Operator which executes DISTINCT operation based on dictionary + */ +public class DictionaryBasedDistinctOperator extends DistinctOperator { + private static final String OPERATOR_NAME = "DictionaryBasedDistinctOperator"; + + private final DistinctAggregationFunction _distinctAggregationFunction; + private final Dictionary _dictionary; + private final int _numTotalDocs; + + private boolean _hasOrderBy; + private boolean _isAscending; + + private int _dictLength; + + public DictionaryBasedDistinctOperator(IndexSegment indexSegment, DistinctAggregationFunction distinctAggregationFunction, + Dictionary dictionary, int numTotalDocs, + TransformOperator transformOperator) { + super(indexSegment, distinctAggregationFunction, transformOperator); + + _distinctAggregationFunction = distinctAggregationFunction; + _dictionary = dictionary; + _numTotalDocs = numTotalDocs; + + List<OrderByExpressionContext> orderByExpressionContexts = _distinctAggregationFunction.getOrderByExpressions(); + + if (orderByExpressionContexts != null) { + OrderByExpressionContext orderByExpressionContext = orderByExpressionContexts.get(0); + + _isAscending = orderByExpressionContext.isAsc(); + _hasOrderBy = true; + } + + _dictLength = _dictionary.length(); + } + + @Override + protected IntermediateResultsBlock getNextBlock() { + DistinctTable distinctTable = buildResult(); + + return new IntermediateResultsBlock(new AggregationFunction[]{_distinctAggregationFunction}, + Collections.singletonList(distinctTable), false); + } + + /** + * Build the final result for this operation + */ + private DistinctTable buildResult() { + + assert _distinctAggregationFunction.getType() == AggregationFunctionType.DISTINCT; + + List<ExpressionContext> expressions = _distinctAggregationFunction.getInputExpressions(); + ExpressionContext expression = expressions.get(0); + FieldSpec.DataType dataType = _dictionary.getValueType(); + + DataSchema dataSchema = new DataSchema(new String[]{expression.toString()}, + new DataSchema.ColumnDataType[]{DataSchema.ColumnDataType.fromDataTypeSV(dataType)}); + List<Record> records; + + int limit = _distinctAggregationFunction.getLimit(); + int actualLimit = Math.min(limit, _dictLength); + + // If ORDER BY is not present, we read the first limit values from the dictionary and return. + // If ORDER BY is present and the dictionary is sorted, then we read the first/last limit values + // from the dictionary. If not sorted, then we read the entire dictionary and return it. + if (!_hasOrderBy) { + records = new ArrayList<>(actualLimit); + + for (int i = 0; i < actualLimit; i++) { + records.add(new Record(new Object[]{_dictionary.getInternal(i)})); + } + } else { + DistinctTable distinctTable = new DistinctTable(dataSchema, _distinctAggregationFunction.getOrderByExpressions(), limit); + + if (_dictionary.isSorted()) { + if (_isAscending) { + for (int i = 0; i < actualLimit; i++) { + distinctTable.addWithOrderBy(new Record(new Object[]{_dictionary.getInternal(i)})); + } + } else { + for (int i = _dictLength - 1; i >= (_dictLength - actualLimit); i--) { + distinctTable.addWithOrderBy(new Record(new Object[]{_dictionary.getInternal(i)})); + } + } + } else { + for (int i = 0; i < _dictLength; i++) { + distinctTable.addWithOrderBy(new Record(new Object[]{_dictionary.getInternal(i)})); + } + } + + return distinctTable; + } + + return new DistinctTable(dataSchema, records); + } + + @Override + public String getOperatorName() { + return OPERATOR_NAME; + } + + @Override + public ExecutionStatistics getExecutionStatistics() { + // NOTE: Set numDocsScanned to numTotalDocs for backward compatibility. + return new ExecutionStatistics(_numTotalDocs, 0, 0, _numTotalDocs); Review comment: Let's count the actual values read from the dictionary, and put it as `numDocsScanned` and `numEntriesScannedPostFilter` ########## File path: pinot-core/src/main/java/org/apache/pinot/core/operator/query/DictionaryBasedDistinctOperator.java ########## @@ -0,0 +1,146 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.pinot.core.operator.query; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import org.apache.pinot.common.request.context.ExpressionContext; +import org.apache.pinot.common.request.context.OrderByExpressionContext; +import org.apache.pinot.common.utils.DataSchema; +import org.apache.pinot.core.data.table.Record; +import org.apache.pinot.core.operator.ExecutionStatistics; +import org.apache.pinot.core.operator.blocks.IntermediateResultsBlock; +import org.apache.pinot.core.operator.transform.TransformOperator; +import org.apache.pinot.core.query.aggregation.function.AggregationFunction; +import org.apache.pinot.core.query.aggregation.function.DistinctAggregationFunction; +import org.apache.pinot.core.query.distinct.DistinctTable; +import org.apache.pinot.segment.spi.AggregationFunctionType; +import org.apache.pinot.segment.spi.IndexSegment; +import org.apache.pinot.segment.spi.index.reader.Dictionary; +import org.apache.pinot.spi.data.FieldSpec; + + +/** + * Operator which executes DISTINCT operation based on dictionary + */ +public class DictionaryBasedDistinctOperator extends DistinctOperator { + private static final String OPERATOR_NAME = "DictionaryBasedDistinctOperator"; + + private final DistinctAggregationFunction _distinctAggregationFunction; + private final Dictionary _dictionary; + private final int _numTotalDocs; + + private boolean _hasOrderBy; + private boolean _isAscending; + + private int _dictLength; + + public DictionaryBasedDistinctOperator(IndexSegment indexSegment, DistinctAggregationFunction distinctAggregationFunction, + Dictionary dictionary, int numTotalDocs, + TransformOperator transformOperator) { + super(indexSegment, distinctAggregationFunction, transformOperator); + + _distinctAggregationFunction = distinctAggregationFunction; + _dictionary = dictionary; + _numTotalDocs = numTotalDocs; + + List<OrderByExpressionContext> orderByExpressionContexts = _distinctAggregationFunction.getOrderByExpressions(); + + if (orderByExpressionContexts != null) { + OrderByExpressionContext orderByExpressionContext = orderByExpressionContexts.get(0); + + _isAscending = orderByExpressionContext.isAsc(); + _hasOrderBy = true; + } + + _dictLength = _dictionary.length(); + } + + @Override + protected IntermediateResultsBlock getNextBlock() { + DistinctTable distinctTable = buildResult(); + + return new IntermediateResultsBlock(new AggregationFunction[]{_distinctAggregationFunction}, + Collections.singletonList(distinctTable), false); + } + + /** + * Build the final result for this operation + */ + private DistinctTable buildResult() { + + assert _distinctAggregationFunction.getType() == AggregationFunctionType.DISTINCT; + + List<ExpressionContext> expressions = _distinctAggregationFunction.getInputExpressions(); + ExpressionContext expression = expressions.get(0); + FieldSpec.DataType dataType = _dictionary.getValueType(); + + DataSchema dataSchema = new DataSchema(new String[]{expression.toString()}, + new DataSchema.ColumnDataType[]{DataSchema.ColumnDataType.fromDataTypeSV(dataType)}); + List<Record> records; + + int limit = _distinctAggregationFunction.getLimit(); + int actualLimit = Math.min(limit, _dictLength); + + // If ORDER BY is not present, we read the first limit values from the dictionary and return. + // If ORDER BY is present and the dictionary is sorted, then we read the first/last limit values + // from the dictionary. If not sorted, then we read the entire dictionary and return it. + if (!_hasOrderBy) { + records = new ArrayList<>(actualLimit); + + for (int i = 0; i < actualLimit; i++) { + records.add(new Record(new Object[]{_dictionary.getInternal(i)})); + } + } else { + DistinctTable distinctTable = new DistinctTable(dataSchema, _distinctAggregationFunction.getOrderByExpressions(), limit); + + if (_dictionary.isSorted()) { + if (_isAscending) { + for (int i = 0; i < actualLimit; i++) { + distinctTable.addWithOrderBy(new Record(new Object[]{_dictionary.getInternal(i)})); + } + } else { + for (int i = _dictLength - 1; i >= (_dictLength - actualLimit); i--) { + distinctTable.addWithOrderBy(new Record(new Object[]{_dictionary.getInternal(i)})); + } + } + } else { + for (int i = 0; i < _dictLength; i++) { Review comment: You only need to create main distinct table when dictionary is not sorted. When dictionary is sorted, creating a wrapper distinct table is good enough -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: commits-unsubscr...@pinot.apache.org For queries about this service, please contact Infrastructure at: us...@infra.apache.org --------------------------------------------------------------------- To unsubscribe, e-mail: commits-unsubscr...@pinot.apache.org For additional commands, e-mail: commits-h...@pinot.apache.org
