chia7712 commented on code in PR #19465:
URL: https://github.com/apache/kafka/pull/19465#discussion_r2043855296
##########
raft/src/main/java/org/apache/kafka/raft/MetadataLogConfig.java:
##########
@@ -16,25 +16,119 @@
*/
package org.apache.kafka.raft;
-/**
- * Configuration for the metadata log
- * @param logSegmentBytes The maximum size of a single metadata log file
- * @param logSegmentMinBytes The minimum size of a single metadata log file
- * @param logSegmentMillis The maximum time before a new metadata log file is
rolled out
- * @param retentionMaxBytes The size of the metadata log and snapshots before
deleting old snapshots and log files
- * @param retentionMillis The time to keep a metadata log file or snapshot
before deleting it
- * @param maxBatchSizeInBytes The largest record batch size allowed in the
metadata log
- * @param maxFetchSizeInBytes The maximum number of bytes to read when
fetching from the metadata log
- * @param deleteDelayMillis The amount of time to wait before deleting a file
from the filesystem
- * @param nodeId The node id
- */
-public record MetadataLogConfig(int logSegmentBytes,
- int logSegmentMinBytes,
- long logSegmentMillis,
- long retentionMaxBytes,
- long retentionMillis,
- int maxBatchSizeInBytes,
- int maxFetchSizeInBytes,
- long deleteDelayMillis,
- int nodeId) {
+import org.apache.kafka.common.config.ConfigDef;
+import org.apache.kafka.common.record.Records;
+
+import java.util.concurrent.TimeUnit;
+
+import static org.apache.kafka.common.config.ConfigDef.Importance.HIGH;
+import static org.apache.kafka.common.config.ConfigDef.Importance.LOW;
+import static org.apache.kafka.common.config.ConfigDef.Range.atLeast;
+import static org.apache.kafka.common.config.ConfigDef.Type.INT;
+import static org.apache.kafka.common.config.ConfigDef.Type.LONG;
+import static org.apache.kafka.common.config.ConfigDef.Type.STRING;
+
+public class MetadataLogConfig {
+
+ public static final String METADATA_LOG_DIR_CONFIG = "metadata.log.dir";
+ public static final String METADATA_LOG_DIR_DOC = "This configuration
determines where we put the metadata log. " +
+ "If it is not set, the metadata log is placed in the first log
directory from log.dirs.";
+
+ public static final String METADATA_SNAPSHOT_MAX_INTERVAL_MS_CONFIG =
"metadata.log.max.snapshot.interval.ms";
+ public static final long METADATA_SNAPSHOT_MAX_INTERVAL_MS_DEFAULT =
TimeUnit.HOURS.toMillis(1);
+ public static final String METADATA_SNAPSHOT_MAX_NEW_RECORD_BYTES_CONFIG =
"metadata.log.max.record.bytes.between.snapshots";
+ public static final int METADATA_SNAPSHOT_MAX_NEW_RECORD_BYTES = 20 * 1024
* 1024;
+ public static final String METADATA_SNAPSHOT_MAX_NEW_RECORD_BYTES_DOC =
"This is the maximum number of bytes in the log between the latest " +
+ "snapshot and the high-watermark needed before generating a new
snapshot. The default value is " +
+ METADATA_SNAPSHOT_MAX_NEW_RECORD_BYTES + ". To generate snapshots
based on the time elapsed, see the <code>" +
+ METADATA_SNAPSHOT_MAX_INTERVAL_MS_CONFIG + "</code> configuration.
The Kafka node will generate a snapshot when " +
+ "either the maximum time interval is reached or the maximum bytes
limit is reached.";
+ public static final String METADATA_SNAPSHOT_MAX_INTERVAL_MS_DOC = "This
is the maximum number of milliseconds to wait to generate a snapshot " +
+ "if there are committed records in the log that are not included
in the latest snapshot. A value of zero disables " +
+ "time based snapshot generation. The default value is " +
METADATA_SNAPSHOT_MAX_INTERVAL_MS_DEFAULT + ". To generate " +
+ "snapshots based on the number of metadata bytes, see the <code>"
+ METADATA_SNAPSHOT_MAX_NEW_RECORD_BYTES_CONFIG + "</code> " +
+ "configuration. The Kafka node will generate a snapshot when
either the maximum time interval is reached or the " +
+ "maximum bytes limit is reached.";
+
+ public static final String METADATA_LOG_SEGMENT_MIN_BYTES_CONFIG =
"metadata.log.segment.min.bytes";
+ public static final String METADATA_LOG_SEGMENT_MIN_BYTES_DOC = "Override
the minimum size for a single metadata log file. This should be used for
testing only.";
+ public static final int METADATA_LOG_SEGMENT_MIN_BYTES_DEFAULT = 8 * 1024
* 1024;
+
+ public static final String METADATA_LOG_SEGMENT_BYTES_CONFIG =
"metadata.log.segment.bytes";
+ public static final String METADATA_LOG_SEGMENT_BYTES_DOC = "The maximum
size of a single metadata log file.";
+ public static final int METADATA_LOG_SEGMENT_BYTES_DEFAULT = 1024 * 1024 *
1024;
+
+ public static final String METADATA_LOG_SEGMENT_MILLIS_CONFIG =
"metadata.log.segment.ms";
+ public static final String METADATA_LOG_SEGMENT_MILLIS_DOC = "The maximum
time before a new metadata log file is rolled out (in milliseconds).";
+ public static final long METADATA_LOG_SEGMENT_MILLIS_DEFAULT = 24 * 7 * 60
* 60 * 1000L;
+
+ public static final String METADATA_MAX_RETENTION_BYTES_CONFIG =
"metadata.max.retention.bytes";
+ public static final int METADATA_MAX_RETENTION_BYTES_DEFAULT = 100 * 1024
* 1024;
+ public static final String METADATA_MAX_RETENTION_BYTES_DOC = "The maximum
combined size of the metadata log and snapshots before deleting old " +
+ "snapshots and log files. Since at least one snapshot must exist
before any logs can be deleted, this is a soft limit.";
+
+ public static final String METADATA_MAX_RETENTION_MILLIS_CONFIG =
"metadata.max.retention.ms";
+ public static final String METADATA_MAX_RETENTION_MILLIS_DOC = "The number
of milliseconds to keep a metadata log file or snapshot before " +
+ "deleting it. Since at least one snapshot must exist before any
logs can be deleted, this is a soft limit.";
+ public static final long METADATA_MAX_RETENTION_MILLIS_DEFAULT = 24 * 7 *
60 * 60 * 1000L;
+
+ public static final String METADATA_MAX_IDLE_INTERVAL_MS_CONFIG =
"metadata.max.idle.interval.ms";
+ public static final int METADATA_MAX_IDLE_INTERVAL_MS_DEFAULT = 500;
+ public static final String METADATA_MAX_IDLE_INTERVAL_MS_DOC = "This
configuration controls how often the active " +
+ "controller should write no-op records to the metadata partition.
If the value is 0, no-op records " +
+ "are not appended to the metadata partition. The default value is
" + METADATA_MAX_IDLE_INTERVAL_MS_DEFAULT;
+
+ public static final ConfigDef CONFIG_DEF = new ConfigDef()
+ .define(METADATA_SNAPSHOT_MAX_NEW_RECORD_BYTES_CONFIG, LONG,
METADATA_SNAPSHOT_MAX_NEW_RECORD_BYTES, atLeast(1), HIGH,
METADATA_SNAPSHOT_MAX_NEW_RECORD_BYTES_DOC)
+ .define(METADATA_SNAPSHOT_MAX_INTERVAL_MS_CONFIG, LONG,
METADATA_SNAPSHOT_MAX_INTERVAL_MS_DEFAULT, atLeast(0), HIGH,
METADATA_SNAPSHOT_MAX_INTERVAL_MS_DOC)
+ .define(METADATA_LOG_DIR_CONFIG, STRING, null, null, HIGH,
METADATA_LOG_DIR_DOC)
+ .define(METADATA_LOG_SEGMENT_BYTES_CONFIG, INT,
METADATA_LOG_SEGMENT_BYTES_DEFAULT, atLeast(Records.LOG_OVERHEAD), HIGH,
METADATA_LOG_SEGMENT_BYTES_DOC)
+ .defineInternal(METADATA_LOG_SEGMENT_MIN_BYTES_CONFIG, INT,
METADATA_LOG_SEGMENT_MIN_BYTES_DEFAULT, atLeast(Records.LOG_OVERHEAD), HIGH,
METADATA_LOG_SEGMENT_MIN_BYTES_DOC)
+ .define(METADATA_LOG_SEGMENT_MILLIS_CONFIG, LONG,
METADATA_LOG_SEGMENT_MILLIS_DEFAULT, null, HIGH,
METADATA_LOG_SEGMENT_MILLIS_DOC)
+ .define(METADATA_MAX_RETENTION_BYTES_CONFIG, LONG,
METADATA_MAX_RETENTION_BYTES_DEFAULT, null, HIGH,
METADATA_MAX_RETENTION_BYTES_DOC)
+ .define(METADATA_MAX_RETENTION_MILLIS_CONFIG, LONG,
METADATA_MAX_RETENTION_MILLIS_DEFAULT, null, HIGH,
METADATA_MAX_RETENTION_MILLIS_DOC)
+ .define(METADATA_MAX_IDLE_INTERVAL_MS_CONFIG, INT,
METADATA_MAX_IDLE_INTERVAL_MS_DEFAULT, atLeast(0), LOW,
METADATA_MAX_IDLE_INTERVAL_MS_DOC);
+
+ public final int logSegmentBytes;
+ public final int logSegmentMinBytes;
+ public final long logSegmentMillis;
+ public final long retentionMaxBytes;
+ public final long retentionMillis;
+ public final int maxBatchSizeInBytes;
+ public final int maxFetchSizeInBytes;
+ public final long deleteDelayMillis;
+ public final int nodeId;
+
+ /**
+ * Configuration for the metadata log
+ * @param logSegmentBytes The maximum size of a single metadata log file
+ * @param logSegmentMinBytes The minimum size of a single metadata log file
+ * @param logSegmentMillis The maximum time before a new metadata log file
is rolled out
+ * @param retentionMaxBytes The size of the metadata log and snapshots
before deleting old snapshots and log files
+ * @param retentionMillis The time to keep a metadata log file or snapshot
before deleting it
+ * @param maxBatchSizeInBytes The largest record batch size allowed in the
metadata log
+ * @param maxFetchSizeInBytes The maximum number of bytes to read when
fetching from the metadata log
+ * @param deleteDelayMillis The amount of time to wait before deleting a
file from the filesystem
+ * @param nodeId The node id
+ */
+ public MetadataLogConfig(int logSegmentBytes,
+ int logSegmentMinBytes,
+ long logSegmentMillis,
+ long retentionMaxBytes,
+ long retentionMillis,
+ int maxBatchSizeInBytes,
+ int maxFetchSizeInBytes,
+ long deleteDelayMillis,
Review Comment:
I guess they will be configurable in the future, so maybe
`MetadataLogConfig` can initialize them based on the constants for now rather
than removing them from `MetadataLogConfig`
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
