Repository: kylin Updated Branches: refs/heads/yang23 e43602597 -> a950586e0
KYLIN-2757 support getting counters for non-hdfs storage Project: http://git-wip-us.apache.org/repos/asf/kylin/repo Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/a950586e Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/a950586e Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/a950586e Branch: refs/heads/yang23 Commit: a950586e05f406b90b68d4c519412a03b0896ebc Parents: e436025 Author: shaofengshi <shaofeng...@apache.org> Authored: Thu Jul 27 21:53:12 2017 +0800 Committer: shaofengshi <shaofeng...@apache.org> Committed: Mon Jul 31 17:21:10 2017 +0800 ---------------------------------------------------------------------- .../kylin/engine/mr/common/HadoopCmdOutput.java | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/kylin/blob/a950586e/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/HadoopCmdOutput.java ---------------------------------------------------------------------- diff --git a/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/HadoopCmdOutput.java b/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/HadoopCmdOutput.java index 5da1947..11d6d2c 100644 --- a/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/HadoopCmdOutput.java +++ b/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/HadoopCmdOutput.java @@ -24,6 +24,7 @@ import java.util.Map; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.mapreduce.Counters; +import org.apache.hadoop.mapreduce.FileSystemCounter; import org.apache.hadoop.mapreduce.Job; import org.apache.hadoop.mapreduce.TaskCounter; import org.apache.kylin.engine.mr.steps.FactDistinctColumnsMapper.RawDataCounter; @@ -94,17 +95,17 @@ public class HadoopCmdOutput { } this.output.append(counters.toString()).append("\n"); logger.debug(counters.toString()); - - String bytsWrittenCounterName = "HDFS_BYTES_WRITTEN"; - String fsScheme = FileSystem.get(job.getConfiguration()).getScheme(); - if (("wasb").equalsIgnoreCase(fsScheme)) { - // for Azure blob store - bytsWrittenCounterName = "WASB_BYTES_WRITTEN"; - } mapInputRecords = String.valueOf(counters.findCounter(TaskCounter.MAP_INPUT_RECORDS).getValue()); - hdfsBytesWritten = String.valueOf(counters.findCounter("FileSystemCounters", bytsWrittenCounterName).getValue()); rawInputBytesRead = String.valueOf(counters.findCounter(RawDataCounter.BYTES).getValue()); + + String fsScheme = FileSystem.get(job.getConfiguration()).getScheme(); + long bytesWritten = counters.findCounter(fsScheme, FileSystemCounter.BYTES_WRITTEN).getValue(); + if (bytesWritten == 0) { + bytesWritten = counters.findCounter("FileSystemCounters", "HDFS_BYTES_WRITTEN").getValue(); + } + hdfsBytesWritten = String.valueOf(bytesWritten); + } catch (Exception e) { logger.error(e.getLocalizedMessage(), e); output.append(e.getLocalizedMessage());
