This is an automated email from the ASF dual-hosted git repository.
kturner pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/accumulo.git
commit 220cdc54da5ba779675d635bcbcc02a6514cd5e6
Merge: 7baa5c76ab c88d368b2d
Author: Keith Turner <ktur...@apache.org>
AuthorDate: Mon May 13 17:35:55 2024 -0400
Merge branch '2.1'
.../accumulo/tserver/tablet/CompactableImpl.java | 31 +++++++++--
.../org/apache/accumulo/tserver/tablet/Tablet.java | 8 +++
.../tablet/CompactableImplFileManagerTest.java | 61 ++++++++++++++++++----
3 files changed, 86 insertions(+), 14 deletions(-)
diff --cc
server/tserver/src/main/java/org/apache/accumulo/tserver/tablet/CompactableImpl.java
index d519ffd172,37b23dc6c4..c17f808ab2
---
a/server/tserver/src/main/java/org/apache/accumulo/tserver/tablet/CompactableImpl.java
+++
b/server/tserver/src/main/java/org/apache/accumulo/tserver/tablet/CompactableImpl.java
@@@ -232,15 -247,32 +232,33 @@@ public class CompactableImpl implement
protected abstract long getNanoTime();
- boolean initiateSelection(CompactionKind kind) {
+ /**
+ * @return the last id of the last successful user compaction
+ */
+ protected abstract long getLastCompactId();
+
+ boolean initiateSelection(CompactionKind kind, Long compactionId) {
- Preconditions
- .checkArgument(kind == DeprecatedCompactionKind.SELECTOR || kind ==
CompactionKind.USER);
+ Preconditions.checkArgument(
- kind == CompactionKind.SELECTOR && compactionId == null
++ kind == DeprecatedCompactionKind.SELECTOR && compactionId == null
+ || kind == CompactionKind.USER && compactionId != null,
+ "Unexpected kind and/or compaction id: %s %s", kind, compactionId);
- if (selectStatus == FileSelectionStatus.NOT_ACTIVE || (kind ==
CompactionKind.USER
- && selectKind == CompactionKind.SELECTOR &&
noneRunning(CompactionKind.SELECTOR)
- && selectStatus != FileSelectionStatus.SELECTING)) {
+ if (selectStatus == FileSelectionStatus.NOT_ACTIVE
+ || (kind == CompactionKind.USER && selectKind ==
DeprecatedCompactionKind.SELECTOR
+ && noneRunning(DeprecatedCompactionKind.SELECTOR)
+ && selectStatus != FileSelectionStatus.SELECTING)) {
+
+ // Check compaction id when a lock is held and no other user
compactions have files
+ // selected, at this point the results of any previous user
compactions should be seen. If
+ // user compaction is currently running, then will not get this far
because of the checks a
+ // few lines up.
+ if (kind == CompactionKind.USER && getLastCompactId() >=
compactionId) {
+ // This user compaction has already completed, so no need to
initiate selection of files
+ // for user compaction.
+ return false;
+ }
+
selectStatus = FileSelectionStatus.NEW;
selectKind = kind;
selectedFiles.clear();
diff --cc
server/tserver/src/test/java/org/apache/accumulo/tserver/tablet/CompactableImplFileManagerTest.java
index 35d5119dfa,743ac8feca..2f04a875f2
---
a/server/tserver/src/test/java/org/apache/accumulo/tserver/tablet/CompactableImplFileManagerTest.java
+++
b/server/tserver/src/test/java/org/apache/accumulo/tserver/tablet/CompactableImplFileManagerTest.java
@@@ -28,8 -30,10 +29,9 @@@ import static org.junit.jupiter.api.Ass
import java.time.Duration;
import java.util.HashSet;
-import java.util.List;
import java.util.Optional;
import java.util.Set;
+ import java.util.concurrent.atomic.AtomicLong;
import java.util.stream.Collectors;
import java.util.stream.Stream;
@@@ -111,10 -117,10 +113,10 @@@ public class CompactableImplFileManager
assertFalse(fileMgr.reserveFiles(staleJob));
assertEquals(newFiles("F00000.rf", "F00001.rf", "F00002.rf", "F00003.rf"),
- fileMgr.getCandidates(tabletFiles, SYSTEM, false));
- assertNoCandidates(fileMgr, tabletFiles, CHOP, USER, SELECTOR);
+ fileMgr.getCandidates(tabletFiles, SYSTEM));
+ assertNoCandidates(fileMgr, tabletFiles, USER, SELECTOR);
- assertTrue(fileMgr.initiateSelection(USER));
+ assertTrue(fileMgr.initiateSelection(USER, 1L));
assertFalse(fileMgr.reserveFiles(staleJob));
@@@ -372,6 -380,96 +374,39 @@@
}
- @Test
- public void testChop() {
- TestFileManager fileMgr = new TestFileManager();
-
- // simulate a compaction because files that were created by compaction
are remembered as not
- // needing a chop
- var job1 = newJob(SYSTEM, "F00000.rf", "F00001.rf");
- assertTrue(fileMgr.reserveFiles(job1));
- fileMgr.completed(job1, newFile("C00005.rf"));
-
- var tabletFiles = newFiles("C00005.rf", "F00002.rf", "F00003.rf",
"F00004.rf");
-
- ChopSelector chopSel = fileMgr.initiateChop(tabletFiles);
- assertEquals(ChopSelectionStatus.SELECTING, fileMgr.getChopStatus());
-
- assertEquals(Set.of(), fileMgr.getCandidates(tabletFiles, CHOP, false));
-
- // this should not include C00005.rf because it was created by a
compaction observed by the file
- // manager
- assertEquals(newFiles("F00002.rf", "F00003.rf", "F00004.rf"),
chopSel.getFilesToExamine());
-
- chopSel.selectChopFiles(newFiles("F00002.rf", "F00004.rf"));
- assertEquals(ChopSelectionStatus.SELECTED, fileMgr.getChopStatus());
-
- assertEquals(newFiles("F00002.rf", "F00004.rf"),
- fileMgr.getCandidates(tabletFiles, CHOP, false));
-
- // simulate compacting one of the files that needs to be chopped, but
this should not finish the
- // chop because more files need to be chopped
- var job2 = newJob(CHOP, "F00002.rf");
- assertTrue(fileMgr.reserveFiles(job2));
- fileMgr.completed(job2, newFile("C00006.rf"));
- tabletFiles = newFiles("C00004.rf", "C00006.rf", "F00003.rf",
"F00004.rf");
- assertFalse(fileMgr.finishChop(tabletFiles));
- assertEquals(ChopSelectionStatus.SELECTED, fileMgr.getChopStatus());
- assertThrows(IllegalStateException.class, fileMgr::finishMarkingChop);
-
- assertEquals(newFiles("F00004.rf"), fileMgr.getCandidates(tabletFiles,
CHOP, false));
-
- // simulate compacting the last file to chop. should cause the chop finish
- var job3 = newJob(CHOP, "F00004.rf");
- assertTrue(fileMgr.reserveFiles(job3));
- fileMgr.completed(job3, newFile("C00007.rf"));
- tabletFiles = newFiles("C00004.rf", "C00006.rf", "F00003.rf",
"C00007.rf");
- assertTrue(fileMgr.finishChop(tabletFiles));
-
- assertEquals(Set.of(), fileMgr.getCandidates(tabletFiles, CHOP, false));
- assertEquals(ChopSelectionStatus.MARKING, fileMgr.getChopStatus());
- assertEquals(Set.of(), fileMgr.getCompactingFiles());
-
- fileMgr.finishMarkingChop();
- assertEquals(ChopSelectionStatus.NOT_ACTIVE, fileMgr.getChopStatus());
-
- }
-
+ @Test
+ public void testComletedUserCompaction() {
+ TestFileManager fileMgr = new TestFileManager();
+
+ fileMgr.lastCompactId.set(2);
+ // should fail to initiate because the last compact id is equal
+ assertFalse(fileMgr.initiateSelection(USER, 2L));
+ assertEquals(FileSelectionStatus.NOT_ACTIVE, fileMgr.selectStatus);
+
+ fileMgr.lastCompactId.set(3);
+ // should fail to initiate because the last compact id is greater than
+ assertFalse(fileMgr.initiateSelection(USER, 2L));
+ assertEquals(FileSelectionStatus.NOT_ACTIVE, fileMgr.selectStatus);
+
+ fileMgr.lastCompactId.set(1);
+ assertTrue(fileMgr.initiateSelection(USER, 2L));
+ assertEquals(FileSelectionStatus.NEW, fileMgr.selectStatus);
+
+ assertTrue(fileMgr.beginSelection());
+ fileMgr.finishSelection(
+ newFiles("F00000.rf", "F00001.rf", "F00002.rf", "F00003.rf",
"F00004.rf"), false);
+ assertEquals(FileSelectionStatus.SELECTED, fileMgr.getSelectionStatus());
+ }
+
+ @Test
+ public void testIllegalInitiateArgs() {
+ TestFileManager fileMgr = new TestFileManager();
+ assertThrows(IllegalArgumentException.class, () ->
fileMgr.initiateSelection(USER, null));
+ assertThrows(IllegalArgumentException.class, () ->
fileMgr.initiateSelection(SELECTOR, 2L));
- for (var kind : List.of(SYSTEM, CHOP)) {
- assertThrows(IllegalArgumentException.class, () ->
fileMgr.initiateSelection(kind, 2L));
- assertThrows(IllegalArgumentException.class, () ->
fileMgr.initiateSelection(kind, null));
- }
++ assertThrows(IllegalArgumentException.class, () ->
fileMgr.initiateSelection(SYSTEM, 2L));
++ assertThrows(IllegalArgumentException.class, () ->
fileMgr.initiateSelection(SYSTEM, null));
+ }
+
private void assertNoCandidates(TestFileManager fileMgr,
Set<StoredTabletFile> tabletFiles,
CompactionKind... kinds) {
for (CompactionKind kind : kinds) {
