Repository: kylin Updated Branches: refs/heads/v1.6.0-rc1 0ba460d44 -> 75c981cb3
KYLIN-1839 bug fix Project: http://git-wip-us.apache.org/repos/asf/kylin/repo Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/75c981cb Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/75c981cb Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/75c981cb Branch: refs/heads/v1.6.0-rc1 Commit: 75c981cb342b188418052b9768ea129bea26469d Parents: 0ba460d Author: Yang Li <liy...@apache.org> Authored: Sun Nov 6 15:50:49 2016 +0800 Committer: Yang Li <liy...@apache.org> Committed: Sun Nov 6 15:51:55 2016 +0800 ---------------------------------------------------------------------- .../kylin/engine/mr/common/AbstractHadoopJob.java | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/kylin/blob/75c981cb/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/AbstractHadoopJob.java ---------------------------------------------------------------------- diff --git a/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/AbstractHadoopJob.java b/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/AbstractHadoopJob.java index 21bb10e..1fd1631 100644 --- a/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/AbstractHadoopJob.java +++ b/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/AbstractHadoopJob.java @@ -295,9 +295,9 @@ public abstract class AbstractHadoopJob extends Configured implements Tool { continue; } FileSystem fs; - if (hdfs.exists(p)) { + if (exists(hdfs, p)) { fs = hdfs; - } else if (localfs.exists(p)) { + } else if (exists(localfs, p)) { fs = localfs; } else { logger.warn("The directory of kylin dependency '" + fileName + "' does not exist, skip"); @@ -386,6 +386,15 @@ public abstract class AbstractHadoopJob extends Configured implements Tool { return classpath; } + private static boolean exists(FileSystem fs, Path p) throws IOException { + try { + return fs.exists(p); + } catch (IllegalArgumentException ex) { + // can happen when FS mismatch + return false; + } + } + public static int addInputDirs(String input, Job job) throws IOException { int folderNum = addInputDirs(StringSplitter.split(input, ","), job); logger.info("Number of added folders:" + folderNum); @@ -401,7 +410,7 @@ public abstract class AbstractHadoopJob extends Configured implements Tool { FileSystem fs = FileSystem.get(job.getConfiguration()); Path path = new Path(inp); - if (!fs.exists(path)) { + if (!exists(fs, path)) { logger.warn("Path not exist:" + path.toString()); continue; }
