Repository: incubator-ignite Updated Branches: refs/heads/ignite-418 [created] 8c4d6ac30
[IGNITE-418]: fixed file/dir problem in DUAL modes. Project: http://git-wip-us.apache.org/repos/asf/incubator-ignite/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-ignite/commit/8c4d6ac3 Tree: http://git-wip-us.apache.org/repos/asf/incubator-ignite/tree/8c4d6ac3 Diff: http://git-wip-us.apache.org/repos/asf/incubator-ignite/diff/8c4d6ac3 Branch: refs/heads/ignite-418 Commit: 8c4d6ac305180aa05a13d4231eb990e93ddfd2a7 Parents: 9b0ba86 Author: iveselovskiy <iveselovs...@gridgain.com> Authored: Fri Mar 6 21:54:48 2015 +0300 Committer: iveselovskiy <iveselovs...@gridgain.com> Committed: Fri Mar 6 21:54:48 2015 +0300 ---------------------------------------------------------------------- .../ignite/internal/processors/igfs/IgfsFileImpl.java | 7 +++++++ .../ignite/internal/processors/igfs/IgfsFileInfo.java | 3 ++- .../ignite/internal/processors/igfs/IgfsImpl.java | 4 ++-- .../hadoop/fs/IgniteHadoopIgfsSecondaryFileSystem.java | 7 +++++-- .../ignite/hadoop/fs/v1/IgniteHadoopFileSystem.java | 13 ++++++++++--- .../ignite/hadoop/fs/v2/IgniteHadoopFileSystem.java | 6 +++--- pom.xml | 2 +- 7 files changed, 30 insertions(+), 12 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/8c4d6ac3/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsFileImpl.java ---------------------------------------------------------------------- diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsFileImpl.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsFileImpl.java index dc1248c..d32b658 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsFileImpl.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsFileImpl.java @@ -77,6 +77,9 @@ public final class IgfsFileImpl implements IgfsFile, Externalizable { if (info.isFile()) { blockSize = info.blockSize(); + + assert blockSize > 0; // By contract file must have blockSize > 0, while directory's blockSize == 0. + len = info.length(); grpBlockSize = info.affinityKey() == null ? globalGrpBlockSize : @@ -107,6 +110,10 @@ public final class IgfsFileImpl implements IgfsFile, Externalizable { blockSize = entry.blockSize(); + // By contract file must have blockSize > 0, while directory's blockSize == 0: + assert entry.isFile() == (blockSize > 0); + assert entry.isDirectory() == (blockSize == 0); + grpBlockSize = entry.affinityKey() == null ? globalGrpSize : entry.length() == 0 ? globalGrpSize : entry.length(); http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/8c4d6ac3/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsFileInfo.java ---------------------------------------------------------------------- diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsFileInfo.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsFileInfo.java index e88503b..de7d077 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsFileInfo.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsFileInfo.java @@ -189,7 +189,8 @@ public final class IgfsFileInfo implements Externalizable { * @param evictExclude Evict exclude flag. */ IgfsFileInfo(int blockSize, long len, boolean evictExclude, @Nullable Map<String, String> props) { - this(false, null, blockSize, len, null, null, props, null, true, System.currentTimeMillis(), evictExclude); + this(blockSize == 0, // NB The contract is: (blockSize == null) <=> isDirectory() + null, blockSize, len, null, null, props, null, true, System.currentTimeMillis(), evictExclude); } /** http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/8c4d6ac3/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsImpl.java ---------------------------------------------------------------------- diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsImpl.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsImpl.java index 614815f..d24a5c0 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsImpl.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsImpl.java @@ -1018,8 +1018,8 @@ public final class IgfsImpl implements IgfsEx { Collection<IgfsFile> children = secondaryFs.listFiles(path); for (IgfsFile child : children) { - IgfsFileInfo fsInfo = new IgfsFileInfo(cfg.getBlockSize(), child.length(), - evictExclude(path, false), child.properties()); + IgfsFileInfo fsInfo = new IgfsFileInfo( + child.blockSize(), child.length(), evictExclude(path, false), child.properties()); files.add(new IgfsFileImpl(child.path(), fsInfo, data.groupBlockSize())); } http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/8c4d6ac3/modules/hadoop/src/main/java/org/apache/ignite/hadoop/fs/IgniteHadoopIgfsSecondaryFileSystem.java ---------------------------------------------------------------------- diff --git a/modules/hadoop/src/main/java/org/apache/ignite/hadoop/fs/IgniteHadoopIgfsSecondaryFileSystem.java b/modules/hadoop/src/main/java/org/apache/ignite/hadoop/fs/IgniteHadoopIgfsSecondaryFileSystem.java index 3320a96..36f1dda 100644 --- a/modules/hadoop/src/main/java/org/apache/ignite/hadoop/fs/IgniteHadoopIgfsSecondaryFileSystem.java +++ b/modules/hadoop/src/main/java/org/apache/ignite/hadoop/fs/IgniteHadoopIgfsSecondaryFileSystem.java @@ -40,7 +40,6 @@ import static org.apache.ignite.internal.processors.igfs.IgfsEx.*; * Adapter to use any Hadoop file system {@link FileSystem} as {@link IgfsSecondaryFileSystem}. */ public class IgniteHadoopIgfsSecondaryFileSystem implements IgfsSecondaryFileSystem, AutoCloseable { - /** Hadoop file system. */ private final FileSystem fileSys; @@ -331,7 +330,7 @@ public class IgniteHadoopIgfsSecondaryFileSystem implements IgfsSecondaryFileSys final Map<String, String> props = properties(status); - return new IgfsFile() { + IgfsFile igfsFile = new IgfsFile() { @Override public IgfsPath path() { return path; } @@ -385,6 +384,10 @@ public class IgniteHadoopIgfsSecondaryFileSystem implements IgfsSecondaryFileSys } }; + // By convention directory has blockSize == 0, while file has blockSize > 0: + assert igfsFile.isDirectory() == (igfsFile.blockSize() == 0); + + return igfsFile; } catch (FileNotFoundException ignore) { return null; http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/8c4d6ac3/modules/hadoop/src/main/java/org/apache/ignite/hadoop/fs/v1/IgniteHadoopFileSystem.java ---------------------------------------------------------------------- diff --git a/modules/hadoop/src/main/java/org/apache/ignite/hadoop/fs/v1/IgniteHadoopFileSystem.java b/modules/hadoop/src/main/java/org/apache/ignite/hadoop/fs/v1/IgniteHadoopFileSystem.java index 93c0df4..92f5b85 100644 --- a/modules/hadoop/src/main/java/org/apache/ignite/hadoop/fs/v1/IgniteHadoopFileSystem.java +++ b/modules/hadoop/src/main/java/org/apache/ignite/hadoop/fs/v1/IgniteHadoopFileSystem.java @@ -694,12 +694,19 @@ public class IgniteHadoopFileSystem extends FileSystem { return secondaryFs.rename(toSecondary(src), toSecondary(dst)); } else { - // Will throw exception if failed. - rmtClient.rename(srcPath, dstPath); - if (clientLog.isLogEnabled()) clientLog.logRename(srcPath, mode, dstPath); + try { + rmtClient.rename(srcPath, dstPath); + } + catch (IOException ioe) { + // Log the exception before rethrowing since it may be ignored: + LOG.warn("Failed to rename [srcPath="+srcPath+", dstPath="+dstPath+", mode="+mode+"]: ", ioe); + + throw ioe; + } + return true; } } http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/8c4d6ac3/modules/hadoop/src/main/java/org/apache/ignite/hadoop/fs/v2/IgniteHadoopFileSystem.java ---------------------------------------------------------------------- diff --git a/modules/hadoop/src/main/java/org/apache/ignite/hadoop/fs/v2/IgniteHadoopFileSystem.java b/modules/hadoop/src/main/java/org/apache/ignite/hadoop/fs/v2/IgniteHadoopFileSystem.java index 70ad99f..fea8739 100644 --- a/modules/hadoop/src/main/java/org/apache/ignite/hadoop/fs/v2/IgniteHadoopFileSystem.java +++ b/modules/hadoop/src/main/java/org/apache/ignite/hadoop/fs/v2/IgniteHadoopFileSystem.java @@ -581,6 +581,9 @@ public class IgniteHadoopFileSystem extends AbstractFileSystem implements Closea IgfsPath dstPath = convert(dst); Set<IgfsMode> childrenModes = modeRslvr.resolveChildrenModes(srcPath); + if (clientLog.isLogEnabled()) + clientLog.logRename(srcPath, modeRslvr.resolveMode(srcPath), dstPath); + if (childrenModes.contains(PROXY)) { if (clientLog.isLogEnabled()) clientLog.logRename(srcPath, PROXY, dstPath); @@ -589,9 +592,6 @@ public class IgniteHadoopFileSystem extends AbstractFileSystem implements Closea } rmtClient.rename(srcPath, dstPath); - - if (clientLog.isLogEnabled()) - clientLog.logRename(srcPath, modeRslvr.resolveMode(srcPath), dstPath); } finally { leaveBusy(); http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/8c4d6ac3/pom.xml ---------------------------------------------------------------------- diff --git a/pom.xml b/pom.xml index 9c6ef75..94b0c36 100644 --- a/pom.xml +++ b/pom.xml @@ -35,7 +35,7 @@ <properties> <ignite.version>1.0.0-RC2-SNAPSHOT</ignite.version> <ignite.edition>fabric</ignite.edition> - <hadoop.version>2.4.1</hadoop.version> + <hadoop.version>2.6.0</hadoop.version> <spring.version>4.1.0.RELEASE</spring.version> <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding> <maven.build.timestamp.format>MMMM d yyyy</maven.build.timestamp.format>
