rohityadav1993 commented on code in PR #18486:
URL: https://github.com/apache/pinot/pull/18486#discussion_r3239739881
##########
pinot-controller/src/main/java/org/apache/pinot/controller/api/upload/SegmentValidationUtils.java:
##########
@@ -57,6 +63,56 @@ public static void validateTimeInterval(SegmentMetadata
segmentMetadata, TableCo
}
}
+ /**
+ * When {@code controller.segment.upload.rejectOutOfRetention.enabled} is
true on the controller, rejects the upload
+ * if segment end time (or index creation time when {@code
controller.retentionManager.enableCreationTimeFallback} is
+ * enabled and end time is invalid) is past the table's {@code
retentionTimeValue}/{@code retentionTimeUnit} window,
+ * using the same boundary as the controller retention manager.
+ *
+ * <p>Controller wiring: {@link
org.apache.pinot.controller.api.resources.PinotSegmentUploadDownloadRestletResource}
+ * invokes this for single-segment upload only. METADATA batch upload
({@code POST /segments/batchUpload}) and
+ * reingested-segment upload do not call it.
+ *
+ * <p>For <b>OFFLINE</b> tables, when batch segment ingestion type is not
<b>APPEND</b>, this method returns without
+ * evaluating retention (aligned with time retention for completed
segments). <b>REALTIME</b> tables always proceed
+ * to the retention comparison when the flag and parseable retention are set.
+ */
+ public static void rejectUploadIfOutOfRetention(SegmentMetadata
segmentMetadata, TableConfig tableConfig,
+ long currentTimeMs, boolean controllerRejectOutOfRetentionEnabled,
+ boolean retentionCreationTimeFallbackEnabled) {
+ if (!controllerRejectOutOfRetentionEnabled) {
+ return;
+ }
+ SegmentsValidationAndRetentionConfig validationConfig =
tableConfig.getValidationConfig();
+ if (validationConfig == null) {
+ return;
+ }
+ if (tableConfig.getTableType() == TableType.OFFLINE
+ &&
!"APPEND".equalsIgnoreCase(IngestionConfigUtils.getBatchSegmentIngestionType(tableConfig)))
{
+ return;
+ }
+ OptionalLong retentionMsOpt =
RetentionUtils.parseTableDataRetentionMillis(validationConfig);
+ if (retentionMsOpt.isEmpty()) {
+ if (StringUtils.isNotEmpty(validationConfig.getRetentionTimeUnit())
+ && StringUtils.isNotEmpty(validationConfig.getRetentionTimeValue()))
{
+ LOGGER.warn("Invalid retention {} {} for table {}, skipping upload
retention check",
+ validationConfig.getRetentionTimeUnit(),
validationConfig.getRetentionTimeValue(),
+ tableConfig.getTableName());
+ }
+ return;
+ }
+ long retentionMs = retentionMsOpt.getAsLong();
+ if (RetentionUtils.isPurgeable(tableConfig.getTableName(),
segmentMetadata, retentionMs, currentTimeMs,
Review Comment:
Should we return a no op status code instead of an error. This will require
existing clients to do exception handling now.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
