rohityadav1993 commented on code in PR #16727:
URL: https://github.com/apache/pinot/pull/16727#discussion_r2318120825
##########
pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/creator/impl/SegmentIndexCreationDriverImpl.java:
##########
@@ -169,31 +170,73 @@ public void init(SegmentGeneratorConfig config,
RecordReader recordReader)
new TransformPipeline(config.getTableConfig(), config.getSchema()));
}
+ /**
+ * Initialize the driver for columnar segment building using a
ColumnReaderFactory.
+ * This method sets up the driver to use column-wise input data access
instead of row-wise.
+ *
+ * @param config Segment generator configuration
+ * @param columnReaderFactory Factory for creating column readers
+ * @throws Exception if initialization fails
+ */
+ public void init(SegmentGeneratorConfig config, ColumnReaderFactory
columnReaderFactory)
+ throws Exception {
+ // Initialize the column reader factory with target schema
+ columnReaderFactory.init(config.getSchema());
+
+ // Create all column readers for the target schema
+ Map<String, ColumnReader> columnReaders =
columnReaderFactory.getAllColumnReaders();
+
+ // Create columnar data source
+ ColumnarSegmentCreationDataSource columnarDataSource =
+ new ColumnarSegmentCreationDataSource(columnReaderFactory,
columnReaders);
+
+ // Use the existing init method with columnar data source and no transform
pipeline
+ init(config, columnarDataSource, null);
+
+ LOGGER.info("Initialized SegmentIndexCreationDriverImpl for columnar
building with {} columns",
+ columnReaders.size());
+ }
+
public void init(SegmentGeneratorConfig config, SegmentCreationDataSource
dataSource,
TransformPipeline transformPipeline)
throws Exception {
_config = config;
- _recordReader = dataSource.getRecordReader();
_dataSchema = config.getSchema();
_continueOnError = config.isContinueOnError();
+ String readerClassName = null;
+
+ // Handle columnar data sources differently
+ if (dataSource instanceof ColumnarSegmentCreationDataSource) {
+ // For columnar data sources, we don't have a record reader
+ _recordReader = null;
+ _transformPipeline = null; // No transform pipeline for columnar mode
+ _dataSource = dataSource;
+ // Initialize common components with columnar-specific reader class name
Review Comment:
self review: remove stale comment
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
