This is an automated email from the ASF dual-hosted git repository.
pratik pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/pinot.git
The following commit(s) were added to refs/heads/master by this push:
new f5a8ea5804 Fix deepstore upload retry for infinite retention tables
(#14406)
f5a8ea5804 is described below
commit f5a8ea58047868bc33db78199210af1cc31c3cd5
Author: Pratik Tibrewal <tibrewalprati...@gmail.com>
AuthorDate: Mon Nov 11 11:26:57 2024 +0530
Fix deepstore upload retry for infinite retention tables (#14406)
---
.../realtime/PinotLLCRealtimeSegmentManager.java | 20 ++++++++++----------
1 file changed, 10 insertions(+), 10 deletions(-)
diff --git
a/pinot-controller/src/main/java/org/apache/pinot/controller/helix/core/realtime/PinotLLCRealtimeSegmentManager.java
b/pinot-controller/src/main/java/org/apache/pinot/controller/helix/core/realtime/PinotLLCRealtimeSegmentManager.java
index ebd3763cfe..71defdb300 100644
---
a/pinot-controller/src/main/java/org/apache/pinot/controller/helix/core/realtime/PinotLLCRealtimeSegmentManager.java
+++
b/pinot-controller/src/main/java/org/apache/pinot/controller/helix/core/realtime/PinotLLCRealtimeSegmentManager.java
@@ -114,6 +114,7 @@ import
org.apache.pinot.spi.utils.CommonConstants.Segment.Realtime.Status;
import org.apache.pinot.spi.utils.IngestionConfigUtils;
import org.apache.pinot.spi.utils.JsonUtils;
import org.apache.pinot.spi.utils.StringUtil;
+import org.apache.pinot.spi.utils.TimeUtils;
import org.apache.pinot.spi.utils.builder.TableNameBuilder;
import org.apache.pinot.spi.utils.retry.RetryPolicies;
import org.apache.zookeeper.data.Stat;
@@ -1489,20 +1490,19 @@ public class PinotLLCRealtimeSegmentManager {
String realtimeTableName = tableConfig.getTableName();
String rawTableName =
TableNameBuilder.extractRawTableName(realtimeTableName);
+ RetentionStrategy retentionStrategy = new
TimeRetentionStrategy(TimeUnit.MILLISECONDS,
+ TimeUtils.VALID_MAX_TIME_MILLIS);
- // If RetentionStrategy cannot be generated from validationConfig, skip
uploading the segments
SegmentsValidationAndRetentionConfig validationConfig =
tableConfig.getValidationConfig();
- if (validationConfig.getRetentionTimeUnit() == null ||
validationConfig.getRetentionTimeUnit().isEmpty()
- || validationConfig.getRetentionTimeValue() == null ||
validationConfig.getRetentionTimeValue().isEmpty()) {
- return;
+ if (validationConfig.getRetentionTimeUnit() != null &&
!validationConfig.getRetentionTimeUnit().isEmpty()
+ && validationConfig.getRetentionTimeValue() != null &&
!validationConfig.getRetentionTimeValue().isEmpty()) {
+ // Use this retention value to avoid the data racing between segment
upload and retention management.
+ long retentionMs =
TimeUnit.valueOf(validationConfig.getRetentionTimeUnit().toUpperCase())
+ .toMillis(Long.parseLong(validationConfig.getRetentionTimeValue()));
+ retentionStrategy = new TimeRetentionStrategy(TimeUnit.MILLISECONDS,
+ retentionMs -
MIN_TIME_BEFORE_SEGMENT_EXPIRATION_FOR_FIXING_DEEP_STORE_COPY_MILLIS);
}
- // Use this retention value to avoid the data racing between segment
upload and retention management.
- long retentionMs =
TimeUnit.valueOf(validationConfig.getRetentionTimeUnit().toUpperCase())
- .toMillis(Long.parseLong(validationConfig.getRetentionTimeValue()));
- RetentionStrategy retentionStrategy = new
TimeRetentionStrategy(TimeUnit.MILLISECONDS,
- retentionMs -
MIN_TIME_BEFORE_SEGMENT_EXPIRATION_FOR_FIXING_DEEP_STORE_COPY_MILLIS);
-
PinotFS pinotFS =
PinotFSFactory.create(URIUtils.getUri(_controllerConf.getDataDir()).getScheme());
// Iterate through LLC segments and upload missing deep store copy by
following steps:
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscr...@pinot.apache.org
For additional commands, e-mail: commits-h...@pinot.apache.org
