Repository: accumulo Updated Branches: refs/heads/master e0ef0ca42 -> dd99cbff2
ACCUMULO-3339 found some more metadata clean-up to move into TabletData Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/0721a6b6 Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/0721a6b6 Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/0721a6b6 Branch: refs/heads/master Commit: 0721a6b6183dd0c268ed7092b0b588f49596de0d Parents: 3e5acd5 Author: Eric C. Newton <eric.new...@gmail.com> Authored: Mon Jul 6 15:24:29 2015 -0400 Committer: Eric C. Newton <eric.new...@gmail.com> Committed: Mon Jul 6 15:24:29 2015 -0400 ---------------------------------------------------------------------- .../apache/accumulo/tserver/TabletServer.java | 5 +- .../apache/accumulo/tserver/tablet/Tablet.java | 85 ++++++-------------- .../accumulo/tserver/tablet/TabletData.java | 45 ++++++++--- 3 files changed, 58 insertions(+), 77 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/accumulo/blob/0721a6b6/server/tserver/src/main/java/org/apache/accumulo/tserver/TabletServer.java ---------------------------------------------------------------------- diff --git a/server/tserver/src/main/java/org/apache/accumulo/tserver/TabletServer.java b/server/tserver/src/main/java/org/apache/accumulo/tserver/TabletServer.java index a8be243..5d3e0f0 100644 --- a/server/tserver/src/main/java/org/apache/accumulo/tserver/TabletServer.java +++ b/server/tserver/src/main/java/org/apache/accumulo/tserver/TabletServer.java @@ -159,7 +159,6 @@ import org.apache.accumulo.server.fs.FileRef; import org.apache.accumulo.server.fs.VolumeManager; import org.apache.accumulo.server.fs.VolumeManager.FileType; import org.apache.accumulo.server.fs.VolumeManagerImpl; -import org.apache.accumulo.server.fs.VolumeUtil; import org.apache.accumulo.server.log.SortedLogState; import org.apache.accumulo.server.log.WalStateManager; import org.apache.accumulo.server.log.WalStateManager.WalMarkerException; @@ -2131,12 +2130,10 @@ public class TabletServer extends AccumuloServerContext implements Runnable { TabletResourceManager trm = resourceManager.createTabletResourceManager(extent, getTableConfiguration(extent)); TabletData data; if (extent.isRootTablet()) { - data = new TabletData(fs, ZooReaderWriter.getInstance()); + data = new TabletData(fs, ZooReaderWriter.getInstance(), getTableConfiguration(extent)); } else { data = new TabletData(extent, fs, tabletsKeyValues.entrySet().iterator()); } - // this opens the tablet file and fills in the endKey in the extent - data.setDirectory(VolumeUtil.switchRootTabletVolume(extent, data.getDirectory())); tablet = new Tablet(TabletServer.this, extent, trm, data); // If a minor compaction starts after a tablet opens, this indicates a log recovery occurred. This recovered data must be minor compacted. http://git-wip-us.apache.org/repos/asf/accumulo/blob/0721a6b6/server/tserver/src/main/java/org/apache/accumulo/tserver/tablet/Tablet.java ---------------------------------------------------------------------- diff --git a/server/tserver/src/main/java/org/apache/accumulo/tserver/tablet/Tablet.java b/server/tserver/src/main/java/org/apache/accumulo/tserver/tablet/Tablet.java index 307044f..8fcf035 100644 --- a/server/tserver/src/main/java/org/apache/accumulo/tserver/tablet/Tablet.java +++ b/server/tserver/src/main/java/org/apache/accumulo/tserver/tablet/Tablet.java @@ -173,7 +173,7 @@ public class Tablet implements TabletCommitter { private final Object timeLock = new Object(); private long persistedTime; - private TServerInstance lastLocation; + private TServerInstance lastLocation = null; private volatile boolean tableDirChecked = false; private final AtomicLong dataSourceDeletions = new AtomicLong(0); @@ -309,6 +309,17 @@ public class Tablet implements TabletCommitter { public Tablet(final TabletServer tabletServer, final KeyExtent extent, final TabletResourceManager trm, TabletData data) throws IOException { + this.tabletServer = tabletServer; + this.extent = extent; + this.tabletResources = trm; + this.lastLocation = data.getLastLocation(); + this.lastFlushID = data.getFlushID(); + this.lastCompactID = data.getCompactID(); + this.splitCreationTime = data.getSplitTime(); + this.tabletTime = TabletTime.getInstance(data.getTime()); + this.persistedTime = tabletTime.getTime(); + this.logId = tabletServer.createLogId(extent); + TableConfiguration tblConf = tabletServer.getTableConfiguration(extent); if (null == tblConf) { Tables.clearCache(tabletServer.getInstance()); @@ -318,71 +329,28 @@ public class Tablet implements TabletCommitter { this.tableConfiguration = tblConf; - TabletFiles tabletPaths = VolumeUtil - .updateTabletVolumes(tabletServer, tabletServer.getLock(), tabletServer.getFileSystem(), extent, - new TabletFiles(data.getDirectory(), data.getLogEntris(), data.getDataFiles()), - ReplicationConfigurationUtil.isEnabled(extent, this.tableConfiguration)); + // translate any volume changes + VolumeManager fs = tabletServer.getFileSystem(); + boolean replicationEnabled = ReplicationConfigurationUtil.isEnabled(extent, this.tableConfiguration); + TabletFiles tabletPaths = new TabletFiles(data.getDirectory(), data.getLogEntris(), data.getDataFiles()); + tabletPaths = VolumeUtil.updateTabletVolumes(tabletServer, tabletServer.getLock(), fs, extent, tabletPaths, replicationEnabled); + // deal with relative path for the directory Path locationPath; - if (tabletPaths.dir.contains(":")) { - locationPath = new Path(tabletPaths.dir.toString()); + locationPath = new Path(tabletPaths.dir); } else { - locationPath = tabletServer.getFileSystem().getFullPath(FileType.TABLE, extent.getTableId().toString() + tabletPaths.dir.toString()); + locationPath = tabletServer.getFileSystem().getFullPath(FileType.TABLE, extent.getTableId().toString() + tabletPaths.dir); } - - final List<LogEntry> logEntries = tabletPaths.logEntries; - final SortedMap<FileRef,DataFileValue> datafiles = tabletPaths.datafiles; - this.location = locationPath; - this.lastLocation = data.getLastLocation(); this.tabletDirectory = tabletPaths.dir; - - this.extent = extent; - this.tabletResources = trm; - - this.lastFlushID = data.getFlushID(); - this.lastCompactID = data.getCompactID(); - this.splitCreationTime = data.getSplitTime(); - String time = data.getTime(); - - if (extent.isRootTablet()) { - long rtime = Long.MIN_VALUE; - for (FileRef ref : datafiles.keySet()) { - Path path = ref.path(); - FileSystem ns = tabletServer.getFileSystem().getVolumeByPath(path).getFileSystem(); - FileSKVIterator reader = FileOperations.getInstance().openReader(path.toString(), true, ns, ns.getConf(), tabletServer.getTableConfiguration(extent)); - long maxTime = -1; - try { - - while (reader.hasTop()) { - maxTime = Math.max(maxTime, reader.getTopKey().getTimestamp()); - reader.next(); - } - - } finally { - reader.close(); - } - - if (maxTime > rtime) { - time = TabletTime.LOGICAL_TIME_ID + "" + maxTime; - rtime = maxTime; - } - } - } - if (time == null && datafiles.isEmpty() && extent.equals(RootTable.OLD_EXTENT)) { - // recovery... old root tablet has no data, so time doesn't matter: - time = TabletTime.LOGICAL_TIME_ID + "" + Long.MIN_VALUE; + for (Entry<Long,List<FileRef>> entry : data.getBulkImported().entrySet()) { + this.bulkImported.put(entry.getKey(), new CopyOnWriteArrayList<FileRef>(entry.getValue())); } - - this.tabletServer = tabletServer; - this.logId = tabletServer.createLogId(extent); - setupDefaultSecurityLabels(extent); - tabletMemory = new TabletMemory(this); - tabletTime = TabletTime.getInstance(time); - persistedTime = tabletTime.getTime(); + final List<LogEntry> logEntries = tabletPaths.logEntries; + final SortedMap<FileRef,DataFileValue> datafiles = tabletPaths.datafiles; tableConfiguration.addObserver(configObserver = new ConfigurationObserver() { @@ -425,13 +393,10 @@ public class Tablet implements TabletCommitter { }); tableConfiguration.getNamespaceConfiguration().addObserver(configObserver); + tabletMemory = new TabletMemory(this); // Force a load of any per-table properties configObserver.propertiesChanged(); - for (Entry<Long,List<FileRef>> entry : data.getBulkImported().entrySet()) { - this.bulkImported.put(entry.getKey(), new CopyOnWriteArrayList<FileRef>(entry.getValue())); - } - if (!logEntries.isEmpty()) { log.info("Starting Write-Ahead Log recovery for " + this.extent); final AtomicLong entriesUsedOnTablet = new AtomicLong(0); http://git-wip-us.apache.org/repos/asf/accumulo/blob/0721a6b6/server/tserver/src/main/java/org/apache/accumulo/tserver/tablet/TabletData.java ---------------------------------------------------------------------- diff --git a/server/tserver/src/main/java/org/apache/accumulo/tserver/tablet/TabletData.java b/server/tserver/src/main/java/org/apache/accumulo/tserver/tablet/TabletData.java index cb9015e..bfbf33f 100644 --- a/server/tserver/src/main/java/org/apache/accumulo/tserver/tablet/TabletData.java +++ b/server/tserver/src/main/java/org/apache/accumulo/tserver/tablet/TabletData.java @@ -33,9 +33,12 @@ import java.util.Map.Entry; import java.util.SortedMap; import java.util.TreeMap; +import org.apache.accumulo.core.conf.AccumuloConfiguration; import org.apache.accumulo.core.data.Key; import org.apache.accumulo.core.data.Value; import org.apache.accumulo.core.data.impl.KeyExtent; +import org.apache.accumulo.core.file.FileOperations; +import org.apache.accumulo.core.file.FileSKVIterator; import org.apache.accumulo.core.metadata.RootTable; import org.apache.accumulo.core.metadata.schema.DataFileValue; import org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSection.BulkFileColumnFamily; @@ -48,9 +51,12 @@ import org.apache.accumulo.core.tabletserver.log.LogEntry; import org.apache.accumulo.fate.zookeeper.ZooReader; import org.apache.accumulo.server.fs.FileRef; import org.apache.accumulo.server.fs.VolumeManager; +import org.apache.accumulo.server.fs.VolumeUtil; import org.apache.accumulo.server.master.state.TServerInstance; +import org.apache.accumulo.server.tablets.TabletTime; import org.apache.accumulo.server.util.MetadataTableUtil; import org.apache.hadoop.fs.FileStatus; +import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; import org.apache.hadoop.io.Text; import org.slf4j.Logger; @@ -119,38 +125,55 @@ public class TabletData { } } } + if (time == null && dataFiles.isEmpty() && extent.equals(RootTable.OLD_EXTENT)) { + // recovery... old root tablet has no data, so time doesn't matter: + time = TabletTime.LOGICAL_TIME_ID + "" + Long.MIN_VALUE; + } } // Read basic root table metadata from zookeeper - public TabletData(VolumeManager fs, ZooReader rdr) throws IOException { + public TabletData(VolumeManager fs, ZooReader rdr, AccumuloConfiguration conf) throws IOException { Path location = new Path(MetadataTableUtil.getRootTabletDir()); // cleanReplacement() has special handling for deleting files FileStatus[] files = fs.listStatus(location); Collection<String> goodPaths = RootFiles.cleanupReplacement(fs, files, true); + long rtime = Long.MIN_VALUE; for (String good : goodPaths) { Path path = new Path(good); String filename = path.getName(); FileRef ref = new FileRef(location.toString() + "/" + filename, path); DataFileValue dfv = new DataFileValue(0, 0); dataFiles.put(ref, dfv); + + FileSystem ns = fs.getVolumeByPath(path).getFileSystem(); + FileSKVIterator reader = FileOperations.getInstance().openReader(path.toString(), true, ns, ns.getConf(), conf); + long maxTime = -1; + try { + while (reader.hasTop()) { + maxTime = Math.max(maxTime, reader.getTopKey().getTimestamp()); + reader.next(); + } + } finally { + reader.close(); + } + if (maxTime > rtime) { + time = TabletTime.LOGICAL_TIME_ID + "" + maxTime; + rtime = maxTime; + } } + try { logEntris = MetadataTableUtil.getLogEntries(null, RootTable.EXTENT); } catch (Exception ex) { throw new RuntimeException("Unable to read tablet log entries", ex); } - directory = MetadataTableUtil.getRootTabletDir(); + directory = VolumeUtil.switchRootTabletVolume(RootTable.EXTENT, MetadataTableUtil.getRootTabletDir()); } // Data pulled from an existing tablet to make a split - public TabletData(String tabletDirectory, - SortedMap<FileRef,DataFileValue> highDatafileSizes, - String time, - long lastFlushID, - long lastCompactID, - TServerInstance lastLocation, - Map<Long,List<FileRef>> bulkIngestedFiles) { + public TabletData(String tabletDirectory, SortedMap<FileRef,DataFileValue> highDatafileSizes, String time, long lastFlushID, long lastCompactID, + TServerInstance lastLocation, Map<Long,List<FileRef>> bulkIngestedFiles) { this.directory = tabletDirectory; this.dataFiles = highDatafileSizes; this.time = time; @@ -201,10 +224,6 @@ public class TabletData { return directory; } - public void setDirectory(String directory) { - this.directory = directory; - } - public long getSplitTime() { return splitTime; }
