sajjad-moradi commented on a change in pull request #6869:
URL: https://github.com/apache/incubator-pinot/pull/6869#discussion_r628640366
##########
File path:
pinot-controller/src/main/java/org/apache/pinot/controller/recommender/rules/impl/PinotTablePartitionRule.java
##########
@@ -119,12 +113,15 @@ public void run()
//We define have a desirable segment size OPTIMAL_SIZE_PER_SEGMENT
//Divide the size of data coming in on a given day by
OPTIMAL_SIZE_PER_SEGMENT we get the number of partitions.
if
(!_input.getOverWrittenConfigs().getPartitionConfig().isNumPartitionsOfflineOverwritten())
{
- _output.getPartitionConfig().setNumPartitionsOffline((int)
(optimalOfflinePartitions));
+ int optimalOfflinePartitions = (int)
_output.getSegmentSizeRecommendations().getNumSegments();
+
_output.getPartitionConfig().setNumPartitionsOffline(optimalOfflinePartitions);
}
}
if (_input.getTableType().equalsIgnoreCase(HYBRID)) {
if
(!_input.getOverWrittenConfigs().getPartitionConfig().isNumPartitionsOfflineOverwritten())
{
-
_output.getPartitionConfig().setNumPartitionsOffline(Math.min(optimalOfflinePartitions,
numKafkaPartitions));
+ int optimalOfflinePartitions =
+ Math.min((int)
_output.getSegmentSizeRecommendations().getNumSegments(), numKafkaPartitions);
+
_output.getPartitionConfig().setNumPartitionsOffline(optimalOfflinePartitions);
Review comment:
Done.
##########
File path:
pinot-controller/src/main/java/org/apache/pinot/controller/recommender/rules/impl/SegmentSizeRule.java
##########
@@ -0,0 +1,145 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.pinot.controller.recommender.rules.impl;
+
+import com.google.common.annotations.VisibleForTesting;
+import java.io.File;
+import java.io.IOException;
+import org.apache.commons.io.FileUtils;
+import
org.apache.pinot.controller.recommender.exceptions.InvalidInputException;
+import org.apache.pinot.controller.recommender.io.ConfigManager;
+import org.apache.pinot.controller.recommender.io.InputManager;
+import
org.apache.pinot.controller.recommender.realtime.provisioning.MemoryEstimator;
+import org.apache.pinot.controller.recommender.rules.AbstractRule;
+import
org.apache.pinot.controller.recommender.rules.io.configs.SegmentSizeRecommendations;
+import
org.apache.pinot.controller.recommender.rules.io.params.SegmentSizeRuleParams;
+import org.apache.pinot.spi.config.table.TableConfig;
+import org.apache.pinot.spi.config.table.TableType;
+import org.apache.pinot.spi.data.Schema;
+import org.apache.pinot.spi.utils.builder.TableConfigBuilder;
+
+import static
org.apache.pinot.controller.recommender.rules.io.params.RecommenderConstants.SegmentSizeRule.NOT_PROVIDED;
+
+
+/**
+ * This rule generates a segment based on the provided schema characteristics
and then recommends the followings
+ * using the size and number of records in the generated segments:
+ * - number of segments
+ * - number of records in each segment
+ * - size of each segment
+ * The purpose of generating a segment is to estimate the size and number of
rows of one sample segment. In case user
+ * already have a production segment in hand, they can provide
actualSegmentSize and numRowsInActualSegment parameters
+ * in the input and then this rule uses those parameters instead of generating
a segment to derived those values.
+ */
+public class SegmentSizeRule extends AbstractRule {
+
+ static final int MEGA_BYTE = 1024 * 1024;
+
+ public SegmentSizeRule(InputManager input, ConfigManager output) {
+ super(input, output);
+ }
+
+ @Override
+ public void run()
+ throws InvalidInputException {
+
+ if (_input.getTableType().equalsIgnoreCase("REALTIME")) {
+ // no need to estimate segment size & optimal number of segments for
realtime only tables;
+ // RT Provisioning Rule will have a comprehensive analysis on that
Review comment:
The cardinality specified in schemaWithMetadata is relative to the value
of the field numRecords. So for realtime only tables, there's no need to run
SegmentSizeRule.
##########
File path:
pinot-controller/src/main/java/org/apache/pinot/controller/recommender/rules/impl/SegmentSizeRule.java
##########
@@ -0,0 +1,145 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.pinot.controller.recommender.rules.impl;
+
+import com.google.common.annotations.VisibleForTesting;
+import java.io.File;
+import java.io.IOException;
+import org.apache.commons.io.FileUtils;
+import
org.apache.pinot.controller.recommender.exceptions.InvalidInputException;
+import org.apache.pinot.controller.recommender.io.ConfigManager;
+import org.apache.pinot.controller.recommender.io.InputManager;
+import
org.apache.pinot.controller.recommender.realtime.provisioning.MemoryEstimator;
+import org.apache.pinot.controller.recommender.rules.AbstractRule;
+import
org.apache.pinot.controller.recommender.rules.io.configs.SegmentSizeRecommendations;
+import
org.apache.pinot.controller.recommender.rules.io.params.SegmentSizeRuleParams;
+import org.apache.pinot.spi.config.table.TableConfig;
+import org.apache.pinot.spi.config.table.TableType;
+import org.apache.pinot.spi.data.Schema;
+import org.apache.pinot.spi.utils.builder.TableConfigBuilder;
+
+import static
org.apache.pinot.controller.recommender.rules.io.params.RecommenderConstants.SegmentSizeRule.NOT_PROVIDED;
+
+
+/**
+ * This rule generates a segment based on the provided schema characteristics
and then recommends the followings
+ * using the size and number of records in the generated segments:
+ * - number of segments
+ * - number of records in each segment
+ * - size of each segment
+ * The purpose of generating a segment is to estimate the size and number of
rows of one sample segment. In case user
+ * already have a production segment in hand, they can provide
actualSegmentSize and numRowsInActualSegment parameters
+ * in the input and then this rule uses those parameters instead of generating
a segment to derived those values.
+ */
+public class SegmentSizeRule extends AbstractRule {
+
+ static final int MEGA_BYTE = 1024 * 1024;
+
+ public SegmentSizeRule(InputManager input, ConfigManager output) {
+ super(input, output);
+ }
+
+ @Override
+ public void run()
+ throws InvalidInputException {
+
+ if (_input.getTableType().equalsIgnoreCase("REALTIME")) {
+ // no need to estimate segment size & optimal number of segments for
realtime only tables;
+ // RT Provisioning Rule will have a comprehensive analysis on that
+ return;
+ }
+
+ long segmentSize;
+ int numRows;
+ SegmentSizeRuleParams segmentSizeRuleParams =
_input.getSegmentSizeRuleParams();
+ if (segmentSizeRuleParams.getActualSegmentSizeMB() == NOT_PROVIDED
+ && segmentSizeRuleParams.getNumRowsInActualSegment() == NOT_PROVIDED) {
+
+ // generate a segment
+ TableConfig tableConfig = createTableConfig(_input.getSchema());
+ int numRowsInGeneratedSegment =
segmentSizeRuleParams.getNumRowsInGeneratedSegment();
+ File generatedSegmentDir =
+ new MemoryEstimator.SegmentGenerator(_input._schemaWithMetaData,
_input._schema, tableConfig,
+ numRowsInGeneratedSegment, true).generate();
+ segmentSize = FileUtils.sizeOfDirectory(generatedSegmentDir);
+ numRows = numRowsInGeneratedSegment;
+
+ // cleanup
+ try {
+ FileUtils.deleteDirectory(generatedSegmentDir);
+ } catch (IOException e) {
+ throw new RuntimeException("Cannot delete the generated segment
directory", e);
+ }
+ } else {
+ segmentSize = segmentSizeRuleParams.getActualSegmentSizeMB() * MEGA_BYTE;
+ numRows = segmentSizeRuleParams.getNumRowsInActualSegment();
+ }
+
+ // estimate optimal segment count & size parameters
+ SegmentSizeRecommendations params =
+ estimate(segmentSize, segmentSizeRuleParams.getDesiredSegmentSizeMB()
* MEGA_BYTE, numRows,
+ _input.getNumRecordsPerPush());
+
+ // wire the recommendations and also update the cardinalities in input
manager. The updated cardinalities are used in subsequent rules.
+ _output.setSegmentSizeRecommendations(params);
+ _input.capCardinalities((int) params.getNumRowsPerSegment());
+ }
+
+ /**
+ * Estimate segment size parameters by extrapolation based on the number of
records and size of the generated segment.
+ * The linear extrapolation used here is not optimal because of columnar way
of storing data and usage of different
+ * indices. Another way would be to iteratively generate new segments with
expected number of rows until the ideal
+ * segment is found, but that's costly because of the time it takes to
generate segments. Although the extrapolation
+ * approach seems to be less accurate, it is chosen due to its performance.
+ *
+ * @param GSS generated segment size
+ * @param DSS desired segment size
+ * @param NRGS num records of generated segment
+ * @param NRPP num records per push
+ * @return recommendations on optimal segment count, size, and number of
records
+ */
+ @VisibleForTesting
+ SegmentSizeRecommendations estimate(long GSS, int DSS, int NRGS, long NRPP) {
+
+ // calc num rows in desired segment
+ double sizeRatio = (double) DSS / GSS;
+ long numRowsInDesiredSegment = Math.round(NRGS * sizeRatio);
+
+ // calc optimal num segment
+ long optimalNumSegments = Math.round(NRPP / (double)
numRowsInDesiredSegment);
+ optimalNumSegments = Math.max(optimalNumSegments, 1);
+
+ // revise optimal num rows in segment
+ long optimalNumRowsInSegment = Math.round(NRPP / (double)
optimalNumSegments);
+
+ // calc optimal segment size
+ double rowRatio = (double) optimalNumRowsInSegment / NRGS;
+ long optimalSegmentSize = Math.round(GSS * rowRatio);
+
+ return new SegmentSizeRecommendations(optimalNumRowsInSegment,
optimalNumSegments, optimalSegmentSize);
+ }
+
+ private TableConfig createTableConfig(Schema schema) {
Review comment:
Agree 👍
Updated the doc.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
us...@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscr...@pinot.apache.org
For additional commands, e-mail: commits-h...@pinot.apache.org
