This is an automated email from the ASF dual-hosted git repository.
devesh pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/ozone.git
The following commit(s) were added to refs/heads/master by this push:
new c187de0080 HDDS-12228. Fix Duplicate Key Violation Condition in
FileSizeCountTask. (#7824)
c187de0080 is described below
commit c187de0080c4b61827000d0fa35fb767f914962b
Author: Arafat2198 <[email protected]>
AuthorDate: Fri Feb 7 13:11:22 2025 +0530
HDDS-12228. Fix Duplicate Key Violation Condition in FileSizeCountTask.
(#7824)
---
.../hadoop/ozone/recon/tasks/FileSizeCountTask.java | 20 +++++++++++++++-----
1 file changed, 15 insertions(+), 5 deletions(-)
diff --git
a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/tasks/FileSizeCountTask.java
b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/tasks/FileSizeCountTask.java
index 6906d4bbb8..1e83043da8 100644
---
a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/tasks/FileSizeCountTask.java
+++
b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/tasks/FileSizeCountTask.java
@@ -95,7 +95,7 @@ public Pair<String, Boolean> reprocess(OMMetadataManager
omMetadataManager) {
if (!statusFSO && !statusOBS) {
return new ImmutablePair<>(getTaskName(), false);
}
- writeCountsToDB(true, fileSizeCountMap);
+ writeCountsToDB(fileSizeCountMap);
LOG.debug("Completed a 'reprocess' run of FileSizeCountTask.");
return new ImmutablePair<>(getTaskName(), true);
}
@@ -112,7 +112,7 @@ private boolean reprocessBucketLayout(BucketLayout
bucketLayout,
handlePutKeyEvent(kv.getValue(), fileSizeCountMap);
// The time complexity of .size() method is constant time, O(1)
if (fileSizeCountMap.size() >= 100000) {
- writeCountsToDB(true, fileSizeCountMap);
+ writeCountsToDB(fileSizeCountMap);
fileSizeCountMap.clear();
}
}
@@ -198,7 +198,7 @@ public Pair<String, Boolean> process(OMUpdateEventBatch
events) {
value.getClass().getName(), updatedKey);
}
}
- writeCountsToDB(false, fileSizeCountMap);
+ writeCountsToDB(fileSizeCountMap);
LOG.debug("{} successfully processed in {} milliseconds",
getTaskName(), (System.currentTimeMillis() - startTime));
return new ImmutablePair<>(getTaskName(), true);
@@ -209,10 +209,11 @@ public Pair<String, Boolean> process(OMUpdateEventBatch
events) {
* using the dao.
*
*/
- private void writeCountsToDB(boolean isDbTruncated,
- Map<FileSizeCountKey, Long> fileSizeCountMap) {
+ private void writeCountsToDB(Map<FileSizeCountKey, Long> fileSizeCountMap) {
+
List<FileCountBySize> insertToDb = new ArrayList<>();
List<FileCountBySize> updateInDb = new ArrayList<>();
+ boolean isDbTruncated = isFileCountBySizeTableEmpty(); // Check if table
is empty
fileSizeCountMap.keySet().forEach((FileSizeCountKey key) -> {
FileCountBySize newRecord = new FileCountBySize();
@@ -295,6 +296,15 @@ private void handleDeleteKeyEvent(String key, OmKeyInfo
omKeyInfo,
}
}
+ /**
+ * Checks if the FILE_COUNT_BY_SIZE table is empty.
+ *
+ * @return true if the table is empty, false otherwise.
+ */
+ private boolean isFileCountBySizeTableEmpty() {
+ return dslContext.fetchCount(FILE_COUNT_BY_SIZE) == 0;
+ }
+
private static class FileSizeCountKey {
private String volume;
private String bucket;
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
