aishikbh commented on code in PR #12220:
URL: https://github.com/apache/pinot/pull/12220#discussion_r1444226777
##########
pinot-core/src/main/java/org/apache/pinot/core/segment/processing/mapper/SegmentMapper.java:
##########
@@ -141,28 +149,42 @@ private Map<String, GenericRowFileManager> doMap()
RecordReaderFactory.getRecordReader(recordReaderFileConfig._fileFormat,
recordReaderFileConfig._dataFile,
recordReaderFileConfig._fieldsToRead,
recordReaderFileConfig._recordReaderConfig);
mapAndTransformRow(recordReader, reuse, observer, count, totalCount);
+ _recordReaderFileConfigs.get(i)._recordReader = recordReader;
+ if (!_constraintsChecker.canWrite()) {
+ LOGGER.info("Stopping record readers at index: {} as size limit
reached", i);
+ break;
+ }
} finally {
- if (recordReader != null) {
+ if (recordReader != null && !recordReader.hasNext()) {
recordReader.close();
}
}
} else {
+ if (!recordReader.hasNext()) {
+ LOGGER.info("Skipping record reader as it is already processed at
index: {}", i);
+ count++;
+ continue;
+ }
mapAndTransformRow(recordReader, reuse, observer, count, totalCount);
+ _recordReaderFileConfigs.get(i)._recordReader = recordReader;
+ if (!_constraintsChecker.canWrite()) {
+ LOGGER.info("Stopping record readers at index: {} as size limit
reached", i);
+ break;
+ }
}
count++;
}
for (GenericRowFileManager fileManager :
_partitionToFileManagerMap.values()) {
fileManager.closeFileWriter();
}
-
return _partitionToFileManagerMap;
}
private void mapAndTransformRow(RecordReader recordReader, GenericRow reuse,
Consumer<Object> observer, int count, int totalCount) throws Exception {
observer.accept(String.format("Doing map phase on data from RecordReader
(%d out of %d)", count, totalCount));
- while (recordReader.hasNext()) {
+ while (recordReader.hasNext() && (!_isSizeBasedConstraintsCheckerEnabled
|| _constraintsChecker.canWrite())) {
Review Comment:
agreed we do not need separate `isSizeBasedConstraintsCheckerEnabled`. It
was from a previous iteration. The current version does not need it. removed
this.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscr...@pinot.apache.org
For queries about this service, please contact Infrastructure at:
us...@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscr...@pinot.apache.org
For additional commands, e-mail: commits-h...@pinot.apache.org
