ACCUMULO-1764 Iterate over each volume and compute an overall summary of usage.
Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/75aac15d Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/75aac15d Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/75aac15d Branch: refs/heads/master Commit: 75aac15d568d5a2d0a50b9bbe813551615d14d1f Parents: ec4d4d8 Author: Josh Elser <els...@apache.org> Authored: Thu Feb 13 19:25:11 2014 -0500 Committer: Josh Elser <els...@apache.org> Committed: Thu Feb 13 19:26:56 2014 -0500 ---------------------------------------------------------------------- .../monitor/servlets/DefaultServlet.java | 70 ++++++++++++++------ 1 file changed, 51 insertions(+), 19 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/accumulo/blob/75aac15d/server/monitor/src/main/java/org/apache/accumulo/monitor/servlets/DefaultServlet.java ---------------------------------------------------------------------- diff --git a/server/monitor/src/main/java/org/apache/accumulo/monitor/servlets/DefaultServlet.java b/server/monitor/src/main/java/org/apache/accumulo/monitor/servlets/DefaultServlet.java index 4a7838a..75aa400 100644 --- a/server/monitor/src/main/java/org/apache/accumulo/monitor/servlets/DefaultServlet.java +++ b/server/monitor/src/main/java/org/apache/accumulo/monitor/servlets/DefaultServlet.java @@ -33,10 +33,7 @@ import javax.servlet.http.HttpServletResponse; import org.apache.accumulo.core.Constants; import org.apache.accumulo.core.conf.DefaultConfiguration; -import org.apache.accumulo.core.conf.Property; -import org.apache.accumulo.core.file.FileUtil; import org.apache.accumulo.core.master.thrift.MasterMonitorInfo; -import org.apache.accumulo.core.util.CachedConfiguration; import org.apache.accumulo.core.util.Duration; import org.apache.accumulo.core.util.NumUtil; import org.apache.accumulo.core.util.Pair; @@ -44,9 +41,10 @@ import org.apache.accumulo.monitor.Monitor; import org.apache.accumulo.monitor.ZooKeeperStatus; import org.apache.accumulo.monitor.ZooKeeperStatus.ZooKeeperState; import org.apache.accumulo.monitor.util.celltypes.NumberType; +import org.apache.accumulo.server.ServerConstants; import org.apache.accumulo.server.conf.ServerConfiguration; -import org.apache.accumulo.server.trace.TraceFileSystem; -import org.apache.hadoop.conf.Configuration; +import org.apache.accumulo.server.fs.VolumeManager; +import org.apache.accumulo.server.fs.VolumeManagerImpl; import org.apache.hadoop.fs.ContentSummary; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; @@ -259,31 +257,65 @@ public class DefaultServlet extends BasicServlet { private void doAccumuloTable(StringBuilder sb) throws IOException { // Accumulo - Configuration conf = CachedConfiguration.getInstance(); - FileSystem fs = TraceFileSystem.wrap(FileUtil.getFileSystem(conf, ServerConfiguration.getSiteConfiguration())); + VolumeManager vm = VolumeManagerImpl.get(ServerConfiguration.getSiteConfiguration()); MasterMonitorInfo info = Monitor.getMmi(); sb.append("<table>\n"); sb.append("<tr><th colspan='2'><a href='/master'>Accumulo Master</a></th></tr>\n"); if (info == null) { sb.append("<tr><td colspan='2'><span class='error'>Master is Down</span></td></tr>\n"); } else { - String consumed = "Unknown"; - String diskUsed = "Unknown"; + long totalAcuBytesUsed = 0l; + long totalHdfsBytesUsed = 0l; + try { - Path path = new Path(Monitor.getSystemConfiguration().get(Property.INSTANCE_DFS_DIR)); - log.debug("Reading the content summary for " + path); - try { - ContentSummary acu = fs.getContentSummary(path); - diskUsed = bytes(acu.getSpaceConsumed()); - ContentSummary rootSummary = fs.getContentSummary(new Path("/")); - consumed = String.format("%.2f%%", acu.getSpaceConsumed() * 100. / rootSummary.getSpaceConsumed()); - } catch (Exception ex) { - log.trace("Unable to get disk usage information from hdfs", ex); + for (String baseDir : ServerConstants.getConfiguredBaseDirs()) { + final Path basePath = new Path(baseDir); + final FileSystem fs = vm.getFileSystemByPath(basePath); + + try { + // Calculate the amount of space used by Accumulo on the FileSystem + ContentSummary accumuloSummary = fs.getContentSummary(basePath); + long bytesUsedByAcuOnFs = accumuloSummary.getSpaceConsumed(); + totalAcuBytesUsed += bytesUsedByAcuOnFs; + + // Catch the overflow -- this is big data + if (totalAcuBytesUsed < bytesUsedByAcuOnFs) { + log.debug("Overflowed long in bytes used by Accumulo for " + baseDir); + totalAcuBytesUsed = 0l; + break; + } + + // Calculate the total amount of space used on the FileSystem + ContentSummary volumeSummary = fs.getContentSummary(new Path("/")); + long bytesUsedOnVolume = volumeSummary.getSpaceConsumed(); + totalHdfsBytesUsed += bytesUsedOnVolume; + + // Catch the overflow -- this is big data + if (totalHdfsBytesUsed < bytesUsedOnVolume) { + log.debug("Overflowed long in bytes used in HDFS for " + baseDir); + totalHdfsBytesUsed = 0; + break; + } + } catch (Exception ex) { + log.trace("Unable to get disk usage information for " + baseDir, ex); + } + } + + String diskUsed = "Unknown"; + String consumed = null; + if (totalAcuBytesUsed > 0) { + // Convert Accumulo usage to a readable String + diskUsed = bytes(totalAcuBytesUsed); + + if (totalHdfsBytesUsed > 0) { + // Compute amount of space used by Accumulo as a percentage of total space usage. + consumed = String.format("%.2f%%", totalAcuBytesUsed * 100. / totalHdfsBytesUsed); + } } boolean highlight = false; tableRow(sb, (highlight = !highlight), "Disk Used", diskUsed); - if (fs.getUsed() != 0) + if (null != consumed) tableRow(sb, (highlight = !highlight), "% of Used DFS", consumed); tableRow(sb, (highlight = !highlight), "<a href='/tables'>Tables</a>", NumberType.commas(Monitor.getTotalTables())); tableRow(sb, (highlight = !highlight), "<a href='/tservers'>Tablet Servers</a>", NumberType.commas(info.tServerInfo.size(), 1, Long.MAX_VALUE));
