This is an automated email from the ASF dual-hosted git repository.
kturner pushed a commit to branch elasticity
in repository https://gitbox.apache.org/repos/asf/accumulo.git
The following commit(s) were added to refs/heads/elasticity by this push:
new 08cdb0e8b2 adds support for splitting tablets with walogs (#3914)
08cdb0e8b2 is described below
commit 08cdb0e8b228118161b553e5cc529139d8611138
Author: Keith Turner <ktur...@apache.org>
AuthorDate: Wed Feb 21 15:21:07 2024 -0500
adds support for splitting tablets with walogs (#3914)
Updates the FATE operations that split tablets to handle walogs. These
changes should wait for a tablet with walogs to recover before
proceeding. The changes to actually make recovery happen were already
done in #3904 with changes to TabletGoalState.compute().
fixes #3844
---
.../accumulo/core/metadata/schema/Ample.java | 2 ++
.../metadata/ConditionalTabletMutatorImpl.java | 8 +++++++
.../manager/tableOps/merge/DeleteTablets.java | 2 +-
.../manager/tableOps/merge/MergeTablets.java | 4 ++--
.../manager/tableOps/split/DeleteOperationIds.java | 2 +-
.../accumulo/manager/tableOps/split/PreSplit.java | 25 ++++++++++++++--------
.../manager/tableOps/split/UpdateTablets.java | 7 ++++--
7 files changed, 35 insertions(+), 15 deletions(-)
diff --git
a/core/src/main/java/org/apache/accumulo/core/metadata/schema/Ample.java
b/core/src/main/java/org/apache/accumulo/core/metadata/schema/Ample.java
index ce8f32ceb9..47bd5e4d1d 100644
--- a/core/src/main/java/org/apache/accumulo/core/metadata/schema/Ample.java
+++ b/core/src/main/java/org/apache/accumulo/core/metadata/schema/Ample.java
@@ -529,6 +529,8 @@ public interface Ample {
ConditionalTabletMutator requireSame(TabletMetadata tabletMetadata,
ColumnType type,
ColumnType... otherTypes);
+ ConditionalTabletMutator requireAbsentLogs();
+
/**
* <p>
* Ample provides the following features on top of the conditional writer
to help automate
diff --git
a/server/base/src/main/java/org/apache/accumulo/server/metadata/ConditionalTabletMutatorImpl.java
b/server/base/src/main/java/org/apache/accumulo/server/metadata/ConditionalTabletMutatorImpl.java
index bd35c00713..34ecdeeebd 100644
---
a/server/base/src/main/java/org/apache/accumulo/server/metadata/ConditionalTabletMutatorImpl.java
+++
b/server/base/src/main/java/org/apache/accumulo/server/metadata/ConditionalTabletMutatorImpl.java
@@ -247,6 +247,14 @@ public class ConditionalTabletMutatorImpl extends
TabletMutatorBase<Ample.Condit
return this;
}
+ @Override
+ public Ample.ConditionalTabletMutator requireAbsentLogs() {
+ Preconditions.checkState(updatesEnabled, "Cannot make updates after
calling mutate.");
+ // create a tablet metadata with an empty set of logs and require the same
as that
+ requireSameSingle(TabletMetadata.builder(extent).build(ColumnType.LOGS),
ColumnType.LOGS);
+ return this;
+ }
+
@Override
public void submit(Ample.RejectionHandler rejectionCheck) {
Preconditions.checkState(updatesEnabled, "Cannot make updates after
calling mutate.");
diff --git
a/server/manager/src/main/java/org/apache/accumulo/manager/tableOps/merge/DeleteTablets.java
b/server/manager/src/main/java/org/apache/accumulo/manager/tableOps/merge/DeleteTablets.java
index f48885d546..4ef90497e5 100644
---
a/server/manager/src/main/java/org/apache/accumulo/manager/tableOps/merge/DeleteTablets.java
+++
b/server/manager/src/main/java/org/apache/accumulo/manager/tableOps/merge/DeleteTablets.java
@@ -89,7 +89,7 @@ public class DeleteTablets extends ManagerRepo {
MergeTablets.validateTablet(tabletMeta, fateId, opid, data.tableId);
var tabletMutator = tabletsMutator.mutateTablet(tabletMeta.getExtent())
- .requireOperation(opid).requireAbsentLocation();
+
.requireOperation(opid).requireAbsentLocation().requireAbsentLogs();
// do not delete the last tablet
if (Objects.equals(tabletMeta.getExtent().endRow(), lastEndRow)) {
diff --git
a/server/manager/src/main/java/org/apache/accumulo/manager/tableOps/merge/MergeTablets.java
b/server/manager/src/main/java/org/apache/accumulo/manager/tableOps/merge/MergeTablets.java
index 2b5b9c969c..d48528a3f7 100644
---
a/server/manager/src/main/java/org/apache/accumulo/manager/tableOps/merge/MergeTablets.java
+++
b/server/manager/src/main/java/org/apache/accumulo/manager/tableOps/merge/MergeTablets.java
@@ -153,8 +153,8 @@ public class MergeTablets extends ManagerRepo {
// update the last tablet
try (var tabletsMutator =
manager.getContext().getAmple().conditionallyMutateTablets()) {
var lastExtent = lastTabletMeta.getExtent();
- var tabletMutator =
-
tabletsMutator.mutateTablet(lastExtent).requireOperation(opid).requireAbsentLocation();
+ var tabletMutator =
tabletsMutator.mutateTablet(lastExtent).requireOperation(opid)
+ .requireAbsentLocation().requireAbsentLogs();
// fence the files in the last tablet if needed
lastTabletMeta.getFilesMap().forEach((file, dfv) -> {
diff --git
a/server/manager/src/main/java/org/apache/accumulo/manager/tableOps/split/DeleteOperationIds.java
b/server/manager/src/main/java/org/apache/accumulo/manager/tableOps/split/DeleteOperationIds.java
index dbf0d04f42..9748f2d21f 100644
---
a/server/manager/src/main/java/org/apache/accumulo/manager/tableOps/split/DeleteOperationIds.java
+++
b/server/manager/src/main/java/org/apache/accumulo/manager/tableOps/split/DeleteOperationIds.java
@@ -54,7 +54,7 @@ public class DeleteOperationIds extends ManagerRepo {
splitInfo.getTablets().forEach(extent -> {
tabletsMutator.mutateTablet(extent).requireOperation(opid).requireAbsentLocation()
- .deleteOperation().submit(rejectionHandler);
+ .requireAbsentLogs().deleteOperation().submit(rejectionHandler);
});
var results = tabletsMutator.process();
diff --git
a/server/manager/src/main/java/org/apache/accumulo/manager/tableOps/split/PreSplit.java
b/server/manager/src/main/java/org/apache/accumulo/manager/tableOps/split/PreSplit.java
index a973015ddc..f94afbdb1e 100644
---
a/server/manager/src/main/java/org/apache/accumulo/manager/tableOps/split/PreSplit.java
+++
b/server/manager/src/main/java/org/apache/accumulo/manager/tableOps/split/PreSplit.java
@@ -19,6 +19,7 @@
package org.apache.accumulo.manager.tableOps.split;
import static
org.apache.accumulo.core.metadata.schema.TabletMetadata.ColumnType.LOCATION;
+import static
org.apache.accumulo.core.metadata.schema.TabletMetadata.ColumnType.LOGS;
import static
org.apache.accumulo.core.metadata.schema.TabletMetadata.ColumnType.OPID;
import static
org.apache.accumulo.core.metadata.schema.TabletMetadata.ColumnType.PREV_ROW;
@@ -77,7 +78,7 @@ public class PreSplit extends ManagerRepo {
// through as quickly as possible.
var tabletMetadata =
manager.getContext().getAmple().readTablet(splitInfo.getOriginal(),
- PREV_ROW, LOCATION, OPID);
+ PREV_ROW, LOCATION, OPID, LOGS);
log.trace("Attempting tablet split {} {} {}", fateId,
splitInfo.getOriginal(),
tabletMetadata == null ? null : tabletMetadata.getLocation());
@@ -87,28 +88,30 @@ public class PreSplit extends ManagerRepo {
// tablet no longer exists or is reserved by another operation
return 0;
} else if (opid.equals(tabletMetadata.getOperationId())) {
- if (tabletMetadata.getLocation() == null) {
- // the operation id is set and there is no location, so can proceed to
split
+ if (tabletMetadata.getLocation() == null &&
tabletMetadata.getLogs().isEmpty()) {
+ // the operation id is set and there is no location or wals, so can
proceed to split
return 0;
} else {
- // the operation id was set, but a location is also set wait for it be
unset
+ // the operation id was set, but a location or wals are also set, so
wait for them to be
+ // unset
return 1000;
}
} else {
try (var tabletsMutator =
manager.getContext().getAmple().conditionallyMutateTablets()) {
tabletsMutator.mutateTablet(splitInfo.getOriginal()).requireAbsentOperation()
- .requireSame(tabletMetadata, LOCATION).putOperation(opid)
+ .requireSame(tabletMetadata, LOCATION, LOGS).putOperation(opid)
.submit(tmeta -> opid.equals(tmeta.getOperationId()));
Map<KeyExtent,Ample.ConditionalResult> results =
tabletsMutator.process();
if (results.get(splitInfo.getOriginal()).getStatus() ==
Status.ACCEPTED) {
log.trace("Successfully set operation id for split {}", fateId);
- if (tabletMetadata.getLocation() == null) {
- // the operation id was set and there is no location, so can move
on
+ if (tabletMetadata.getLocation() == null &&
tabletMetadata.getLogs().isEmpty()) {
+ // the operation id was set and there is no location or wals, so
can move on
return 0;
} else {
- // now that the operation id set, generate an event to unload the
tablet
+ // now that the operation id set, generate an event to unload the
tablet or recover the
+ // logs
manager.getEventCoordinator().event(splitInfo.getOriginal(),
"Set operation id %s on tablet for split", fateId);
// the operation id was set, but a location is also set wait for
it be unset
@@ -129,7 +132,7 @@ public class PreSplit extends ManagerRepo {
manager.getSplitter().removeSplitStarting(splitInfo.getOriginal());
TabletMetadata tabletMetadata = manager.getContext().getAmple()
- .readTablet(splitInfo.getOriginal(), PREV_ROW, LOCATION, OPID);
+ .readTablet(splitInfo.getOriginal(), PREV_ROW, LOCATION, OPID, LOGS);
var opid = TabletOperationId.from(TabletOperationType.SPLITTING, fateId);
@@ -149,6 +152,10 @@ public class PreSplit extends ManagerRepo {
"Tablet unexpectedly had location set %s %s %s", fateId,
tabletMetadata.getLocation(),
tabletMetadata.getExtent());
+ Preconditions.checkState(tabletMetadata.getLogs().isEmpty(),
+ "Tablet unexpectedly had walogs %s %s %s", fateId,
tabletMetadata.getLogs(),
+ tabletMetadata.getExtent());
+
// Create the dir name here for the next step. If the next step fails it
will always have the
// same dir name each time it runs again making it idempotent.
diff --git
a/server/manager/src/main/java/org/apache/accumulo/manager/tableOps/split/UpdateTablets.java
b/server/manager/src/main/java/org/apache/accumulo/manager/tableOps/split/UpdateTablets.java
index afc1a77b7a..2fe3c56399 100644
---
a/server/manager/src/main/java/org/apache/accumulo/manager/tableOps/split/UpdateTablets.java
+++
b/server/manager/src/main/java/org/apache/accumulo/manager/tableOps/split/UpdateTablets.java
@@ -90,6 +90,10 @@ public class UpdateTablets extends ManagerRepo {
"Tablet %s unexpectedly has a location %s", splitInfo.getOriginal(),
tabletMetadata.getLocation());
+ Preconditions.checkState(tabletMetadata.getLogs().isEmpty(),
+ "Tablet unexpectedly had walogs %s %s %s", fateId,
tabletMetadata.getLogs(),
+ tabletMetadata.getExtent());
+
var newTablets = splitInfo.getTablets();
var newTabletsFiles = getNewTabletFiles(newTablets, tabletMetadata,
@@ -99,7 +103,6 @@ public class UpdateTablets extends ManagerRepo {
// Only update the original tablet after successfully creating the new
tablets, this is
// important for failure cases where this operation partially runs a then
runs again.
-
updateExistingTablet(fateId, manager, tabletMetadata, opid, newTablets,
newTabletsFiles);
return new DeleteOperationIds(splitInfo);
@@ -216,7 +219,7 @@ public class UpdateTablets extends ManagerRepo {
var newExtent = newTablets.last();
var mutator =
tabletsMutator.mutateTablet(splitInfo.getOriginal()).requireOperation(opid)
- .requireAbsentLocation();
+ .requireAbsentLocation().requireAbsentLogs();
mutator.putPrevEndRow(newExtent.prevEndRow());
