This is an automated email from the ASF dual-hosted git repository.
kturner pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/accumulo.git
commit f210f860d7ab08017bc1dbccf7c078f8cfa33b26
Merge: 62cd737399 bb9a967f56
Author: Keith Turner <ktur...@apache.org>
AuthorDate: Wed Jul 24 19:22:13 2024 +0000
Merge branch '2.1'
.../java/org/apache/accumulo/manager/Manager.java | 83 ++++++++-------
.../accumulo/tserver/tablet/DatafileManager.java | 6 +-
.../org/apache/accumulo/tserver/tablet/Tablet.java | 111 ++++++++++++---------
3 files changed, 112 insertions(+), 88 deletions(-)
diff --cc server/manager/src/main/java/org/apache/accumulo/manager/Manager.java
index 039282cb66,514c0271d4..923bfc33b4
--- a/server/manager/src/main/java/org/apache/accumulo/manager/Manager.java
+++ b/server/manager/src/main/java/org/apache/accumulo/manager/Manager.java
@@@ -56,10 -55,7 +57,8 @@@ import java.util.concurrent.atomic.Atom
import java.util.stream.Collectors;
import org.apache.accumulo.core.Constants;
+import org.apache.accumulo.core.cli.ConfigOpts;
- import org.apache.accumulo.core.client.AccumuloClient;
- import org.apache.accumulo.core.client.Scanner;
- import org.apache.accumulo.core.client.TableNotFoundException;
+ import org.apache.accumulo.core.clientImpl.ClientContext;
import org.apache.accumulo.core.clientImpl.thrift.TableOperation;
import org.apache.accumulo.core.clientImpl.thrift.TableOperationExceptionType;
import
org.apache.accumulo.core.clientImpl.thrift.ThriftTableOperationException;
@@@ -93,17 -84,19 +90,15 @@@ import org.apache.accumulo.core.manager
import org.apache.accumulo.core.manager.thrift.ManagerGoalState;
import org.apache.accumulo.core.manager.thrift.ManagerMonitorInfo;
import org.apache.accumulo.core.manager.thrift.ManagerState;
-import org.apache.accumulo.core.master.thrift.BulkImportState;
-import org.apache.accumulo.core.master.thrift.TableInfo;
-import org.apache.accumulo.core.master.thrift.TabletServerStatus;
-import org.apache.accumulo.core.metadata.MetadataTable;
-import org.apache.accumulo.core.metadata.RootTable;
+import org.apache.accumulo.core.manager.thrift.TableInfo;
+import org.apache.accumulo.core.manager.thrift.TabletServerStatus;
+import org.apache.accumulo.core.metadata.AccumuloTable;
import org.apache.accumulo.core.metadata.TServerInstance;
import org.apache.accumulo.core.metadata.TabletLocationState;
-import org.apache.accumulo.core.metadata.TabletState;
import org.apache.accumulo.core.metadata.schema.Ample.DataLevel;
- import org.apache.accumulo.core.metadata.schema.MetadataSchema;
- import
org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSection.TabletColumnFamily;
+ import org.apache.accumulo.core.metadata.schema.TabletMetadata;
import org.apache.accumulo.core.metrics.MetricsInfo;
import org.apache.accumulo.core.metrics.MetricsProducer;
- import org.apache.accumulo.core.security.Authorizations;
-import org.apache.accumulo.core.replication.thrift.ReplicationCoordinator;
import org.apache.accumulo.core.spi.balancer.BalancerEnvironment;
import org.apache.accumulo.core.spi.balancer.SimpleLoadBalancer;
import org.apache.accumulo.core.spi.balancer.TabletBalancer;
diff --cc
server/tserver/src/main/java/org/apache/accumulo/tserver/tablet/DatafileManager.java
index 1c534de4e4,4697439421..10a8e309f9
---
a/server/tserver/src/main/java/org/apache/accumulo/tserver/tablet/DatafileManager.java
+++
b/server/tserver/src/main/java/org/apache/accumulo/tserver/tablet/DatafileManager.java
@@@ -365,9 -363,47 +363,9 @@@ class DatafileManager
newFile =
tablet.updateTabletDataFile(commitSession.getMaxCommittedTime(), newDatafile,
dfv,
unusedWalLogs, flushId);
- // Mark that we have data we want to replicate
- // This WAL could still be in use by other Tablets *from the same
table*, so we can only
- // mark
- // that there is data to replicate,
- // but it is *not* closed. We know it is not closed by the fact that
this MinC triggered. A
- // MinC cannot happen unless the
- // tablet is online and thus these WALs are referenced by that
tablet. Therefore, the WAL
- // replication status cannot be 'closed'.
- @SuppressWarnings("deprecation")
- boolean replicate =
org.apache.accumulo.core.replication.ReplicationConfigurationUtil
- .isEnabled(tablet.getExtent(), tablet.getTableConfiguration());
- if (replicate) {
- // unusedWalLogs is of the form host/fileURI, need to strip off the
host portion
- Set<String> logFileOnly = new HashSet<>();
- for (String unusedWalLog : unusedWalLogs) {
- int index = unusedWalLog.indexOf('/');
- if (index == -1) {
- log.warn(
- "Could not find host component to strip from DFSLogger
representation of WAL");
- } else {
- unusedWalLog = unusedWalLog.substring(index + 1);
- }
- logFileOnly.add(unusedWalLog);
- }
-
- if (log.isDebugEnabled()) {
- log.debug("Recording that data has been ingested into {} using
{}", tablet.getExtent(),
- logFileOnly);
- }
- for (String logFile : logFileOnly) {
- @SuppressWarnings("deprecation")
- Status status =
-
org.apache.accumulo.server.replication.StatusUtil.openWithUnknownLength();
- ReplicationTableUtil.updateFiles(tablet.getContext(),
tablet.getExtent(), logFile,
- status);
- }
- }
-
tablet.finishClearingUnusedLogs();
} finally {
- tablet.getLogLock().unlock();
+ logLock.unlock();
}
do {
diff --cc
server/tserver/src/main/java/org/apache/accumulo/tserver/tablet/Tablet.java
index 4ea148046b,544b6fc60a..1bd0067bd8
---
a/server/tserver/src/main/java/org/apache/accumulo/tserver/tablet/Tablet.java
+++
b/server/tserver/src/main/java/org/apache/accumulo/tserver/tablet/Tablet.java
@@@ -1606,68 -1634,75 +1616,75 @@@ public class Tablet extends TabletBase
}
// obtain this info outside of synch block since it will involve opening
- // the map files... it is ok if the set of map files changes, because
- // this info is used for optimization... it is ok if map files are missing
+ // the data files... it is ok if the set of data files changes, because
+ // this info is used for optimization... it is ok if data files are
missing
// from the set... can still query and insert into the tablet while this
- // map file operation is happening
- Map<TabletFile,FileUtil.FileInfo> firstAndLastRows =
FileUtil.tryToGetFirstAndLastRows(context,
- tableConfiguration, getDatafileManager().getFiles());
+ // data file operation is happening
+ Map<StoredTabletFile,FileUtil.FileInfo> firstAndLastRows = FileUtil
+ .tryToGetFirstAndLastRows(context, tableConfiguration,
getDatafileManager().getFiles());
- synchronized (this) {
- // java needs tuples ...
- TreeMap<KeyExtent,TabletData> newTablets = new TreeMap<>();
+ // This must be acquired before the tablet lock AND completeClose expects
it to be acquired when
+ // called.
+ final var lock = lockLogLock();
+ try {
+ synchronized (this) {
+ // java needs tuples ...
+ TreeMap<KeyExtent,TabletData> newTablets = new TreeMap<>();
- long t1 = System.currentTimeMillis();
+ long t1 = System.currentTimeMillis();
- closeState = CloseState.CLOSING;
- completeClose(true, false);
+ closeState = CloseState.CLOSING;
+ completeClose(true, false);
- Text midRow = splitPoint.row;
- double splitRatio = splitPoint.splitRatio;
+ Text midRow = splitPoint.row;
+ double splitRatio = splitPoint.splitRatio;
- KeyExtent low = new KeyExtent(extent.tableId(), midRow,
extent.prevEndRow());
- KeyExtent high = new KeyExtent(extent.tableId(), extent.endRow(),
midRow);
+ KeyExtent low = new KeyExtent(extent.tableId(), midRow,
extent.prevEndRow());
+ KeyExtent high = new KeyExtent(extent.tableId(), extent.endRow(),
midRow);
- String lowDirectoryName = createTabletDirectoryName(context, midRow);
+ String lowDirectoryName = createTabletDirectoryName(context, midRow);
- // write new tablet information to MetadataTable
- SortedMap<StoredTabletFile,DataFileValue> lowDatafileSizes = new
TreeMap<>();
- SortedMap<StoredTabletFile,DataFileValue> highDatafileSizes = new
TreeMap<>();
- List<StoredTabletFile> highDatafilesToRemove = new ArrayList<>();
+ // write new tablet information to MetadataTable
+ SortedMap<StoredTabletFile,DataFileValue> lowDatafileSizes = new
TreeMap<>();
+ SortedMap<StoredTabletFile,DataFileValue> highDatafileSizes = new
TreeMap<>();
+ List<StoredTabletFile> highDatafilesToRemove = new ArrayList<>();
- MetadataTableUtil.splitDatafiles(midRow, splitRatio, firstAndLastRows,
- getDatafileManager().getDatafileSizes(), lowDatafileSizes,
highDatafileSizes,
- highDatafilesToRemove);
+ MetadataTableUtil.splitDatafiles(midRow, splitRatio, firstAndLastRows,
+ getDatafileManager().getDatafileSizes(), lowDatafileSizes,
highDatafileSizes,
+ highDatafilesToRemove);
- log.debug("Files for low split {} {}", low, lowDatafileSizes.keySet());
- log.debug("Files for high split {} {}", high,
highDatafileSizes.keySet());
+ log.debug("Files for low split {} {}", low,
lowDatafileSizes.keySet());
+ log.debug("Files for high split {} {}", high,
highDatafileSizes.keySet());
- MetadataTime time = tabletTime.getMetadataTime();
+ MetadataTime time = tabletTime.getMetadataTime();
- HashSet<ExternalCompactionId> ecids = new HashSet<>();
- compactable.getExternalCompactionIds(ecids::add);
+ HashSet<ExternalCompactionId> ecids = new HashSet<>();
+ compactable.getExternalCompactionIds(ecids::add);
- MetadataTableUtil.splitTablet(high, extent.prevEndRow(), splitRatio,
- getTabletServer().getContext(), getTabletServer().getLock(), ecids);
- ManagerMetadataUtil.addNewTablet(getTabletServer().getContext(), low,
lowDirectoryName,
- getTabletServer().getTabletSession(), lowDatafileSizes,
bulkImported, time,
- lastFlushID.get(), lastCompactID.get(),
getTabletServer().getLock());
- MetadataTableUtil.finishSplit(high, highDatafileSizes,
highDatafilesToRemove,
- getTabletServer().getContext(), getTabletServer().getLock());
+ MetadataTableUtil.splitTablet(high, extent.prevEndRow(), splitRatio,
+ getTabletServer().getContext(), getTabletServer().getLock(),
ecids);
+ ManagerMetadataUtil.addNewTablet(getTabletServer().getContext(), low,
lowDirectoryName,
+ getTabletServer().getTabletSession(), lowDatafileSizes,
bulkImported, time,
+ lastFlushID.get(), lastCompactID.get(),
getTabletServer().getLock());
+ MetadataTableUtil.finishSplit(high, highDatafileSizes,
highDatafilesToRemove,
+ getTabletServer().getContext(), getTabletServer().getLock());
- TabletLogger.split(extent, low, high,
getTabletServer().getTabletSession());
+ TabletLogger.split(extent, low, high,
getTabletServer().getTabletSession());
- newTablets.put(high, new TabletData(dirName, highDatafileSizes, time,
lastFlushID.get(),
- lastCompactID.get(), lastLocation, bulkImported));
- newTablets.put(low, new TabletData(lowDirectoryName, lowDatafileSizes,
time,
- lastFlushID.get(), lastCompactID.get(), lastLocation,
bulkImported));
+ newTablets.put(high, new TabletData(dirName, highDatafileSizes, time,
lastFlushID.get(),
+ lastCompactID.get(), lastLocation, bulkImported));
+ newTablets.put(low, new TabletData(lowDirectoryName,
lowDatafileSizes, time,
+ lastFlushID.get(), lastCompactID.get(), lastLocation,
bulkImported));
- long t2 = System.currentTimeMillis();
+ long t2 = System.currentTimeMillis();
- log.debug(String.format("offline split time : %6.2f secs", (t2 - t1) /
1000.0));
+ log.debug(String.format("offline split time : %6.2f secs", (t2 - t1)
/ 1000.0));
- closeState = CloseState.COMPLETE;
- return newTablets;
+ closeState = CloseState.COMPLETE;
+ return newTablets;
+ }
+ } finally {
+ lock.unlock();
}
}
