xiangfu0 commented on code in PR #8465:
URL: https://github.com/apache/pinot/pull/8465#discussion_r843356509
##########
pinot-plugins/pinot-minion-tasks/pinot-minion-builtin-tasks/src/main/java/org/apache/pinot/plugin/minion/tasks/segmentgenerationandpush/SegmentGenerationAndPushTaskGenerator.java:
##########
@@ -177,6 +180,78 @@ public String getTaskType() {
return pinotTaskConfigs;
}
+ @Override
+ public List<PinotTaskConfig> generateAdhocTasks(TableConfig tableConfig,
Map<String, String> taskConfigs)
+ throws Exception {
+ String taskUUID = UUID.randomUUID().toString();
+ // Only generate tasks for OFFLINE tables
+ String offlineTableName = tableConfig.getTableName();
+ if (tableConfig.getTableType() != TableType.OFFLINE) {
+ LOGGER.warn("Skip generating SegmentGenerationAndPushTask for
non-OFFLINE table: {}", offlineTableName);
+ return ImmutableList.of();
+ }
+
+ TableTaskConfig tableTaskConfig = tableConfig.getTaskConfig();
+ if (tableTaskConfig != null) {
+ Map<String, String> taskConfigsFromTable =
+
tableTaskConfig.getConfigsForTaskType(MinionConstants.SegmentGenerationAndPushTask.TASK_TYPE);
+ if (taskConfigsFromTable != null) {
+ taskConfigsFromTable.entrySet().forEach(entry ->
taskConfigs.putIfAbsent(entry.getKey(), entry.getValue()));
+ }
+ }
+ Map<String, String> batchConfigMap = new HashMap<>(taskConfigs);
+ int tableNumTasks = 0;
+ try {
+ URI inputDirURI =
+
SegmentGenerationUtils.getDirectoryURI(batchConfigMap.get(BatchConfigProperties.INPUT_DIR_URI));
+ List<URI> inputFileURIs = getInputFilesFromDirectory(batchConfigMap,
inputDirURI, Collections.emptySet());
+ if (inputFileURIs.isEmpty()) {
+ LOGGER.warn("Skip generating SegmentGenerationAndPushTask, no input
files found : {}", inputDirURI);
+ return ImmutableList.of();
+ }
+ if (!batchConfigMap.containsKey(BatchConfigProperties.INPUT_FORMAT)) {
+ batchConfigMap.put(BatchConfigProperties.INPUT_FORMAT,
+ extractFormatFromFileSuffix(inputFileURIs.get(0).getPath()));
+ }
+ updateRecordReaderConfigs(batchConfigMap);
+
+ List<PinotTaskConfig> pinotTaskConfigs = new ArrayList<>();
+ LOGGER.info("Final input files for task config generation: {}",
inputFileURIs);
+ for (URI inputFileURI : inputFileURIs) {
+ Map<String, String> singleFileGenerationTaskConfig =
+ getSingleFileGenerationTaskConfig(offlineTableName, tableNumTasks,
batchConfigMap, inputFileURI,
+ generateFixedSegmentName(offlineTableName, taskUUID,
tableNumTasks));
+ pinotTaskConfigs.add(new
PinotTaskConfig(MinionConstants.SegmentGenerationAndPushTask.TASK_TYPE,
+ singleFileGenerationTaskConfig));
+ tableNumTasks++;
+ }
+ if (!batchConfigMap.containsKey(BatchConfigProperties.INPUT_FORMAT)) {
+ batchConfigMap.put(BatchConfigProperties.INPUT_FORMAT,
+ extractFormatFromFileSuffix(inputFileURIs.get(0).getPath()));
+ updateRecordReaderConfigs(batchConfigMap);
+ }
+ return pinotTaskConfigs;
+ } catch (Exception e) {
+ LOGGER.error("Unable to generate the SegmentGenerationAndPush task. [
table configs: {}, task configs: {} ]",
+ tableConfig, taskConfigs, e);
+ throw e;
+ }
+ }
+
+ private String generateFixedSegmentName(String offlineTableName, String
taskUUID, int tableNumTasks) {
+ return String.format("%s_%s_%d", offlineTableName, taskUUID,
tableNumTasks);
+ }
+
+ private String extractFormatFromFileSuffix(String path) {
+ String fileExtension = path.substring(path.lastIndexOf(".") + 1);
+ switch (fileExtension) {
+ case "":
+ throw new UnsupportedOperationException("No file extension found");
Review Comment:
will change the logic to if check.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscr...@pinot.apache.org
For queries about this service, please contact Infrastructure at:
us...@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscr...@pinot.apache.org
For additional commands, e-mail: commits-h...@pinot.apache.org
