This is an automated email from the ASF dual-hosted git repository.
JingsongLi pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/paimon.git
The following commit(s) were added to refs/heads/master by this push:
new 67d24eb20e [core] Improve committer initialization with
TagAutoCreation (#8699)
67d24eb20e is described below
commit 67d24eb20e537996048f52e01c9f27b53e5b4e18
Author: yuzelin <[email protected]>
AuthorDate: Thu Jul 16 21:59:53 2026 +0800
[core] Improve committer initialization with TagAutoCreation (#8699)
Currently, TagAutoCreation reads and deserializes all existing
auto-created tag files during initialization to find the latest tag.
This can significantly increase committer initialization time for tables
retaining many tags, especially on object storage.
This change lists tag names first, identifies the latest auto-created
tag by its timestamp, and reads only that tag file to restore
nextSnapshot and nextTag. As a result, initialization is reduced from
one directory listing plus N tag file reads to one directory listing
plus a single tag file read.
---
.../main/java/org/apache/paimon/tag/TagAutoCreation.java | 14 +++++++++-----
.../src/main/java/org/apache/paimon/utils/TagManager.java | 12 ++++++++++++
2 files changed, 21 insertions(+), 5 deletions(-)
diff --git
a/paimon-core/src/main/java/org/apache/paimon/tag/TagAutoCreation.java
b/paimon-core/src/main/java/org/apache/paimon/tag/TagAutoCreation.java
index 5f72763c70..47a186cbd4 100644
--- a/paimon-core/src/main/java/org/apache/paimon/tag/TagAutoCreation.java
+++ b/paimon-core/src/main/java/org/apache/paimon/tag/TagAutoCreation.java
@@ -37,6 +37,7 @@ import java.time.Duration;
import java.time.Instant;
import java.time.LocalDateTime;
import java.time.ZoneId;
+import java.util.Comparator;
import java.util.List;
import java.util.Optional;
import java.util.SortedMap;
@@ -94,16 +95,19 @@ public class TagAutoCreation {
this.periodHandler.validateDelay(delay);
- SortedMap<Snapshot, List<String>> tags =
tagManager.tags(periodHandler::isAutoTag);
+ // Auto-created tag times advance with snapshot IDs. Read only the
latest tag to restore
+ // progress instead of reading every tag file.
+ List<String> tagNames = tagManager.tagNames(periodHandler::isAutoTag);
+ tagNames.sort(Comparator.comparing(periodHandler::tagToTime,
Comparator.reverseOrder()));
- if (tags.isEmpty()) {
+ if (tagNames.isEmpty()) {
this.nextSnapshot =
firstNonNull(snapshotManager.earliestSnapshotId(),
FIRST_SNAPSHOT_ID);
} else {
- Snapshot lastTag = tags.lastKey();
- this.nextSnapshot = lastTag.id() + 1;
+ String tagName = tagNames.get(0);
+ Tag tag = tagManager.getOrThrow(tagName);
+ this.nextSnapshot = tag.trimToSnapshot().id() + 1;
- String tagName = checkAndGetOneAutoTag(tags.get(lastTag));
LocalDateTime time = periodHandler.tagToTime(tagName);
this.nextTag = periodHandler.nextTagTime(time);
}
diff --git a/paimon-core/src/main/java/org/apache/paimon/utils/TagManager.java
b/paimon-core/src/main/java/org/apache/paimon/utils/TagManager.java
index 4a0a18e51e..8bbcf55791 100644
--- a/paimon-core/src/main/java/org/apache/paimon/utils/TagManager.java
+++ b/paimon-core/src/main/java/org/apache/paimon/utils/TagManager.java
@@ -370,6 +370,18 @@ public class TagManager {
}
}
+ /** Get tag names without reading tag files. */
+ public List<String> tagNames(Predicate<String> filter) {
+ try {
+ return tagPaths(path -> true).stream()
+ .map(path -> path.getName().substring(TAG_PREFIX.length()))
+ .filter(filter)
+ .collect(Collectors.toList());
+ } catch (IOException e) {
+ throw new RuntimeException(e);
+ }
+ }
+
/** Get all tagged snapshots sorted by snapshot id. */
public List<Snapshot> taggedSnapshots() {
return new ArrayList<>(tags().keySet());
