Repository: kylin Updated Branches: refs/heads/master 03bea8d94 -> 511de715f
KYLIN-2757 support getting counters for non-hdfs storage Project: http://git-wip-us.apache.org/repos/asf/kylin/repo Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/511de715 Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/511de715 Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/511de715 Branch: refs/heads/master Commit: 511de715f9894cfe5f66af4ce9ead7c2331b68fb Parents: 03bea8d Author: shaofengshi <shaofeng...@apache.org> Authored: Thu Jul 27 21:53:12 2017 +0800 Committer: shaofengshi <shaofeng...@apache.org> Committed: Thu Jul 27 21:53:53 2017 +0800 ---------------------------------------------------------------------- .../kylin/engine/mr/common/HadoopCmdOutput.java | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/kylin/blob/511de715/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/HadoopCmdOutput.java ---------------------------------------------------------------------- diff --git a/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/HadoopCmdOutput.java b/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/HadoopCmdOutput.java index 5da1947..11d6d2c 100644 --- a/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/HadoopCmdOutput.java +++ b/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/HadoopCmdOutput.java @@ -24,6 +24,7 @@ import java.util.Map; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.mapreduce.Counters; +import org.apache.hadoop.mapreduce.FileSystemCounter; import org.apache.hadoop.mapreduce.Job; import org.apache.hadoop.mapreduce.TaskCounter; import org.apache.kylin.engine.mr.steps.FactDistinctColumnsMapper.RawDataCounter; @@ -94,17 +95,17 @@ public class HadoopCmdOutput { } this.output.append(counters.toString()).append("\n"); logger.debug(counters.toString()); - - String bytsWrittenCounterName = "HDFS_BYTES_WRITTEN"; - String fsScheme = FileSystem.get(job.getConfiguration()).getScheme(); - if (("wasb").equalsIgnoreCase(fsScheme)) { - // for Azure blob store - bytsWrittenCounterName = "WASB_BYTES_WRITTEN"; - } mapInputRecords = String.valueOf(counters.findCounter(TaskCounter.MAP_INPUT_RECORDS).getValue()); - hdfsBytesWritten = String.valueOf(counters.findCounter("FileSystemCounters", bytsWrittenCounterName).getValue()); rawInputBytesRead = String.valueOf(counters.findCounter(RawDataCounter.BYTES).getValue()); + + String fsScheme = FileSystem.get(job.getConfiguration()).getScheme(); + long bytesWritten = counters.findCounter(fsScheme, FileSystemCounter.BYTES_WRITTEN).getValue(); + if (bytesWritten == 0) { + bytesWritten = counters.findCounter("FileSystemCounters", "HDFS_BYTES_WRITTEN").getValue(); + } + hdfsBytesWritten = String.valueOf(bytesWritten); + } catch (Exception e) { logger.error(e.getLocalizedMessage(), e); output.append(e.getLocalizedMessage());
