Repository: kylin Updated Branches: refs/heads/yang22 6617955b4 -> 09e754508
support azure hdinsight counters Project: http://git-wip-us.apache.org/repos/asf/kylin/repo Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/09e75450 Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/09e75450 Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/09e75450 Branch: refs/heads/yang22 Commit: 09e75450879281b5921ffa24cd5fc6edc554ee73 Parents: 6617955 Author: shaofengshi <shaofeng...@apache.org> Authored: Wed Mar 8 11:23:29 2017 +0800 Committer: shaofengshi <shaofeng...@apache.org> Committed: Sat Apr 1 08:56:09 2017 +0800 ---------------------------------------------------------------------- .../kylin/engine/mr/common/HadoopCmdOutput.java | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/kylin/blob/09e75450/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/HadoopCmdOutput.java ---------------------------------------------------------------------- diff --git a/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/HadoopCmdOutput.java b/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/HadoopCmdOutput.java index 9d016cc..023717f 100644 --- a/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/HadoopCmdOutput.java +++ b/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/HadoopCmdOutput.java @@ -22,6 +22,7 @@ import java.util.Collections; import java.util.HashMap; import java.util.Map; +import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.mapreduce.Counters; import org.apache.hadoop.mapreduce.Job; import org.apache.hadoop.mapreduce.TaskCounter; @@ -93,9 +94,18 @@ public class HadoopCmdOutput { this.output.append(counters.toString()).append("\n"); logger.debug(counters.toString()); + String bytsWrittenCounterName = "HDFS_BYTES_WRITTEN"; + String bytsReadCounterName = "HDFS_BYTES_READ"; + String fsScheme = FileSystem.get(job.getConfiguration()).getScheme(); + if (("wasb").equalsIgnoreCase(fsScheme)) { + // for Azure blob store + bytsWrittenCounterName = "WASB_BYTES_WRITTEN"; + bytsReadCounterName = "WASB_BYTES_READ"; + } + mapInputRecords = String.valueOf(counters.findCounter(TaskCounter.MAP_INPUT_RECORDS).getValue()); - hdfsBytesWritten = String.valueOf(counters.findCounter("FileSystemCounters", "HDFS_BYTES_WRITTEN").getValue()); - hdfsBytesRead = String.valueOf(counters.findCounter("FileSystemCounters", "HDFS_BYTES_READ").getValue()); + hdfsBytesWritten = String.valueOf(counters.findCounter("FileSystemCounters", bytsWrittenCounterName).getValue()); + hdfsBytesRead = String.valueOf(counters.findCounter("FileSystemCounters", bytsReadCounterName).getValue()); } catch (Exception e) { logger.error(e.getLocalizedMessage(), e); output.append(e.getLocalizedMessage());
