Jackie-Jiang commented on code in PR #15528:
URL: https://github.com/apache/pinot/pull/15528#discussion_r2045679139
##########
pinot-segment-local/src/main/java/org/apache/pinot/segment/local/dedup/BaseTableDedupMetadataManager.java:
##########
@@ -40,64 +39,91 @@ public abstract class BaseTableDedupMetadataManager
implements TableDedupMetadat
protected final Map<Integer, PartitionDedupMetadataManager>
_partitionMetadataManagerMap = new ConcurrentHashMap<>();
protected String _tableNameWithType;
- protected DedupContext _dedupContext;
- private boolean _enablePreload;
+ protected DedupContext _context;
@Override
- public void init(TableConfig tableConfig, Schema schema, TableDataManager
tableDataManager,
- ServerMetrics serverMetrics) {
+ public void init(PinotConfiguration instanceDedupConfig, TableConfig
tableConfig, Schema schema,
+ TableDataManager tableDataManager) {
_tableNameWithType = tableConfig.getTableName();
+ Preconditions.checkArgument(tableConfig.isDedupEnabled(), "Dedup must be
enabled for table: %s",
+ _tableNameWithType);
+ DedupConfig dedupConfig = tableConfig.getDedupConfig();
+ assert dedupConfig != null;
+
List<String> primaryKeyColumns = schema.getPrimaryKeyColumns();
Preconditions.checkArgument(!CollectionUtils.isEmpty(primaryKeyColumns),
"Primary key columns must be configured for dedup enabled table: %s",
_tableNameWithType);
- DedupConfig dedupConfig = tableConfig.getDedupConfig();
- Preconditions.checkArgument(dedupConfig != null, "Dedup must be enabled
for table: %s", _tableNameWithType);
double metadataTTL = dedupConfig.getMetadataTTL();
String dedupTimeColumn = dedupConfig.getDedupTimeColumn();
if (dedupTimeColumn == null) {
dedupTimeColumn = tableConfig.getValidationConfig().getTimeColumnName();
}
if (metadataTTL > 0) {
Preconditions.checkArgument(dedupTimeColumn != null,
- "When metadataTTL is configured, metadata time column or time column
must be configured for "
- + "dedup enabled table: %s", _tableNameWithType);
+ "When metadataTTL is configured, metadata time column or time column
must be configured for dedup enabled "
+ + "table: %s", _tableNameWithType);
+ }
+
+ boolean enablePreload =
+ dedupConfig.getPreload().isEnabled(() ->
instanceDedupConfig.getProperty(Dedup.DEFAULT_ENABLE_PRELOAD, false));
+ if (enablePreload) {
+ if (tableDataManager.getSegmentPreloadExecutor() == null) {
+ LOGGER.warn("Preload cannot be enabled without segment preload
executor for table: {}", _tableNameWithType);
+ enablePreload = false;
+ }
}
- _enablePreload = dedupConfig.isEnablePreload() &&
tableDataManager.getSegmentPreloadExecutor() != null;
- HashFunction hashFunction = dedupConfig.getHashFunction();
- File tableIndexDir = tableDataManager.getTableDataDir();
- DedupContext.Builder dedupContextBuider = new DedupContext.Builder();
-
dedupContextBuider.setTableConfig(tableConfig).setSchema(schema).setPrimaryKeyColumns(primaryKeyColumns)
-
.setHashFunction(hashFunction).setEnablePreload(_enablePreload).setMetadataTTL(metadataTTL)
-
.setDedupTimeColumn(dedupTimeColumn).setTableIndexDir(tableIndexDir).setTableDataManager(tableDataManager);
- _dedupContext = dedupContextBuider.build();
- LOGGER.info(
- "Initialized {} for table: {} with primary key columns: {}, hash
function: {}, enable preload: {}, metadata "
- + "TTL: {}, dedup time column: {}, table index dir: {}",
getClass().getSimpleName(), _tableNameWithType,
- primaryKeyColumns, hashFunction, _enablePreload, metadataTTL,
dedupTimeColumn, tableIndexDir);
+
+ // NOTE: This field doesn't follow enablement override, and always take
instance config if set to true.
Review Comment:
If instance config is set to `true`. Let me revise the comment lol
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscr...@pinot.apache.org
For queries about this service, please contact Infrastructure at:
us...@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscr...@pinot.apache.org
For additional commands, e-mail: commits-h...@pinot.apache.org
