http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/8e2a08b9/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheProcessor.java ---------------------------------------------------------------------- diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheProcessor.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheProcessor.java index 3b2ca47..72c2b49 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheProcessor.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheProcessor.java @@ -550,10 +550,10 @@ public class GridCacheProcessor extends GridProcessorAdapter { maxPreloadOrder = validatePreloadOrder(ctx.config().getCacheConfiguration()); // Internal caches which should not be returned to user. - IgfsConfiguration[] igfsCfgs = ctx.grid().configuration().getIgfsConfiguration(); + FileSystemConfiguration[] igfsCfgs = ctx.grid().configuration().getFileSystemConfiguration(); if (igfsCfgs != null) { - for (IgfsConfiguration igfsCfg : igfsCfgs) { + for (FileSystemConfiguration igfsCfg : igfsCfgs) { sysCaches.add(igfsCfg.getMetaCacheName()); sysCaches.add(igfsCfg.getDataCacheName()); }
http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/8e2a08b9/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheUtils.java ---------------------------------------------------------------------- diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheUtils.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheUtils.java index 1e67907..b07c14e 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheUtils.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheUtils.java @@ -1594,10 +1594,10 @@ public class GridCacheUtils { * @return {@code True} in this is IGFS data or meta cache. */ public static boolean isIgfsCache(IgniteConfiguration cfg, @Nullable String cacheName) { - IgfsConfiguration[] igfsCfgs = cfg.getIgfsConfiguration(); + FileSystemConfiguration[] igfsCfgs = cfg.getFileSystemConfiguration(); if (igfsCfgs != null) { - for (IgfsConfiguration igfsCfg : igfsCfgs) { + for (FileSystemConfiguration igfsCfg : igfsCfgs) { // IGFS config probably has not been validated yet => possible NPE, so we check for null. if (igfsCfg != null && (F.eq(cacheName, igfsCfg.getDataCacheName()) || F.eq(cacheName, igfsCfg.getMetaCacheName()))) http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/8e2a08b9/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsAsyncImpl.java ---------------------------------------------------------------------- diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsAsyncImpl.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsAsyncImpl.java index 7aeeb46..48a32f4 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsAsyncImpl.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsAsyncImpl.java @@ -193,7 +193,7 @@ public class IgfsAsyncImpl extends AsyncSupportAdapter<IgniteFileSystem> impleme } /** {@inheritDoc} */ - @Override public IgfsConfiguration configuration() { + @Override public FileSystemConfiguration configuration() { return igfs.configuration(); } http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/8e2a08b9/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsContext.java ---------------------------------------------------------------------- diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsContext.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsContext.java index 245a1dd..475a7fc 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsContext.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsContext.java @@ -37,7 +37,7 @@ public class IgfsContext { private final GridKernalContext ctx; /** Configuration. */ - private final IgfsConfiguration cfg; + private final FileSystemConfiguration cfg; /** Managers. */ private List<IgfsManager> mgrs = new LinkedList<>(); @@ -68,7 +68,7 @@ public class IgfsContext { */ public IgfsContext( GridKernalContext ctx, - IgfsConfiguration cfg, + FileSystemConfiguration cfg, IgfsMetaManager metaMgr, IgfsDataManager dataMgr, IgfsServerManager srvMgr, @@ -102,7 +102,7 @@ public class IgfsContext { /** * @return IGFS configuration. */ - public IgfsConfiguration configuration() { + public FileSystemConfiguration configuration() { return cfg; } http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/8e2a08b9/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsDataManager.java ---------------------------------------------------------------------- diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsDataManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsDataManager.java index 333c5ca..72bd60a 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsDataManager.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsDataManager.java @@ -307,7 +307,7 @@ public class IgfsDataManager extends IgfsManager { IgniteDataLoader<IgfsBlockKey, byte[]> ldr = igfsCtx.kernalContext().<IgfsBlockKey, byte[]>dataLoad().dataLoader(dataCachePrj.name()); - IgfsConfiguration cfg = igfsCtx.configuration(); + FileSystemConfiguration cfg = igfsCtx.configuration(); if (cfg.getPerNodeBatchSize() > 0) ldr.perNodeBufferSize(cfg.getPerNodeBatchSize()); http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/8e2a08b9/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsFileInfo.java ---------------------------------------------------------------------- diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsFileInfo.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsFileInfo.java index 43def03..e88503b 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsFileInfo.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsFileInfo.java @@ -99,13 +99,13 @@ public final class IgfsFileInfo implements Externalizable { } /** - * Constructs directory or file info with {@link org.apache.ignite.configuration.IgfsConfiguration#DFLT_BLOCK_SIZE default} block size. + * Constructs directory or file info with {@link org.apache.ignite.configuration.FileSystemConfiguration#DFLT_BLOCK_SIZE default} block size. * * @param isDir Constructs directory info if {@code true} or file info if {@code false}. * @param props Meta properties to set. */ public IgfsFileInfo(boolean isDir, @Nullable Map<String, String> props) { - this(isDir, null, isDir ? 0 : IgfsConfiguration.DFLT_BLOCK_SIZE, 0, null, null, props, null, false, + this(isDir, null, isDir ? 0 : FileSystemConfiguration.DFLT_BLOCK_SIZE, 0, null, null, props, null, false, System.currentTimeMillis(), false); } http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/8e2a08b9/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsImpl.java ---------------------------------------------------------------------- diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsImpl.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsImpl.java index f3fcc83..614815f 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsImpl.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsImpl.java @@ -72,7 +72,7 @@ public final class IgfsImpl implements IgfsEx { private IgfsDataManager data; /** FS configuration. */ - private IgfsConfiguration cfg; + private FileSystemConfiguration cfg; /** IGFS context. */ private IgfsContext igfsCtx; @@ -378,7 +378,7 @@ public final class IgfsImpl implements IgfsEx { } /** {@inheritDoc} */ - @Override public IgfsConfiguration configuration() { + @Override public FileSystemConfiguration configuration() { return cfg; } @@ -1789,7 +1789,7 @@ public final class IgfsImpl implements IgfsEx { /** * Executes IGFS task with overridden maximum range length (see - * {@link org.apache.ignite.configuration.IgfsConfiguration#getMaximumTaskRangeLength()} for more information). + * {@link org.apache.ignite.configuration.FileSystemConfiguration#getMaximumTaskRangeLength()} for more information). * * @param task Task to execute. * @param rslvr Optional resolver to control split boundaries. @@ -1823,7 +1823,7 @@ public final class IgfsImpl implements IgfsEx { /** * Executes IGFS task asynchronously with overridden maximum range length (see - * {@link org.apache.ignite.configuration.IgfsConfiguration#getMaximumTaskRangeLength()} for more information). + * {@link org.apache.ignite.configuration.FileSystemConfiguration#getMaximumTaskRangeLength()} for more information). * * @param taskCls Task class to execute. * @param rslvr Optional resolver to control split boundaries. http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/8e2a08b9/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsMetaManager.java ---------------------------------------------------------------------- diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsMetaManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsMetaManager.java index f2fc9f9..adc0254 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsMetaManager.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsMetaManager.java @@ -52,7 +52,7 @@ import static org.apache.ignite.transactions.TransactionIsolation.*; @SuppressWarnings("all") public class IgfsMetaManager extends IgfsManager { /** IGFS configuration. */ - private IgfsConfiguration cfg; + private FileSystemConfiguration cfg; /** Metadata cache. */ private GridCache<Object, Object> metaCache; http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/8e2a08b9/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsProcessor.java ---------------------------------------------------------------------- diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsProcessor.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsProcessor.java index 20bfcca..e1b5114 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsProcessor.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsProcessor.java @@ -73,17 +73,17 @@ public class IgfsProcessor extends IgfsProcessorAdapter { if (ctx.config().isDaemon()) return; - IgfsConfiguration[] cfgs = ctx.config().getIgfsConfiguration(); + FileSystemConfiguration[] cfgs = ctx.config().getFileSystemConfiguration(); assert cfgs != null && cfgs.length > 0; validateLocalIgfsConfigurations(cfgs); // Start IGFS instances. - for (IgfsConfiguration cfg : cfgs) { + for (FileSystemConfiguration cfg : cfgs) { IgfsContext igfsCtx = new IgfsContext( ctx, - new IgfsConfiguration(cfg), + new FileSystemConfiguration(cfg), new IgfsMetaManager(), new IgfsDataManager(), new IgfsServerManager(), @@ -202,7 +202,7 @@ public class IgfsProcessor extends IgfsProcessorAdapter { // is daemon; // doesn't have configured IGFS; // doesn't have configured caches. - if (gridCfg.isDaemon() || F.isEmpty(gridCfg.getIgfsConfiguration()) || + if (gridCfg.isDaemon() || F.isEmpty(gridCfg.getFileSystemConfiguration()) || F.isEmpty(gridCfg.getCacheConfiguration())) return; @@ -216,9 +216,9 @@ public class IgfsProcessor extends IgfsProcessorAdapter { Collection<IgfsAttributes> attrVals = new ArrayList<>(); - assert gridCfg.getIgfsConfiguration() != null; + assert gridCfg.getFileSystemConfiguration() != null; - for (IgfsConfiguration igfsCfg : gridCfg.getIgfsConfiguration()) { + for (FileSystemConfiguration igfsCfg : gridCfg.getFileSystemConfiguration()) { CacheConfiguration cacheCfg = cacheCfgs.get(igfsCfg.getDataCacheName()); if (cacheCfg == null) @@ -258,10 +258,10 @@ public class IgfsProcessor extends IgfsProcessorAdapter { * @param cfgs IGFS configurations * @throws IgniteCheckedException If any of IGFS configurations is invalid. */ - private void validateLocalIgfsConfigurations(IgfsConfiguration[] cfgs) throws IgniteCheckedException { + private void validateLocalIgfsConfigurations(FileSystemConfiguration[] cfgs) throws IgniteCheckedException { Collection<String> cfgNames = new HashSet<>(); - for (IgfsConfiguration cfg : cfgs) { + for (FileSystemConfiguration cfg : cfgs) { String name = cfg.getName(); if (cfgNames.contains(name)) http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/8e2a08b9/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsServerManager.java ---------------------------------------------------------------------- diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsServerManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsServerManager.java index e1f4a0f..643eeff 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsServerManager.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsServerManager.java @@ -29,7 +29,7 @@ import org.apache.ignite.thread.*; import java.util.*; import java.util.concurrent.*; -import static org.apache.ignite.configuration.IgfsConfiguration.*; +import static org.apache.ignite.configuration.FileSystemConfiguration.*; /** * IGFS server manager. @@ -49,7 +49,7 @@ public class IgfsServerManager extends IgfsManager { /** {@inheritDoc} */ @Override protected void start0() throws IgniteCheckedException { - IgfsConfiguration igfsCfg = igfsCtx.configuration(); + FileSystemConfiguration igfsCfg = igfsCtx.configuration(); Map<String,String> cfg = igfsCfg.getIpcEndpointConfiguration(); if (F.isEmpty(cfg)) { http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/8e2a08b9/modules/core/src/main/java/org/apache/ignite/internal/visor/node/VisorGridConfiguration.java ---------------------------------------------------------------------- diff --git a/modules/core/src/main/java/org/apache/ignite/internal/visor/node/VisorGridConfiguration.java b/modules/core/src/main/java/org/apache/ignite/internal/visor/node/VisorGridConfiguration.java index 96c69d9..cc6ae63 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/visor/node/VisorGridConfiguration.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/visor/node/VisorGridConfiguration.java @@ -114,7 +114,7 @@ public class VisorGridConfiguration implements Serializable { rest = VisorRestConfiguration.from(c); userAttrs = c.getUserAttributes(); caches = VisorCacheConfiguration.list(ignite, c.getCacheConfiguration()); - igfss = VisorIgfsConfiguration.list(c.getIgfsConfiguration()); + igfss = VisorIgfsConfiguration.list(c.getFileSystemConfiguration()); streamers = VisorStreamerConfiguration.list(c.getStreamerConfiguration()); env = new HashMap<>(getenv()); sysProps = getProperties(); http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/8e2a08b9/modules/core/src/main/java/org/apache/ignite/internal/visor/node/VisorIgfsConfiguration.java ---------------------------------------------------------------------- diff --git a/modules/core/src/main/java/org/apache/ignite/internal/visor/node/VisorIgfsConfiguration.java b/modules/core/src/main/java/org/apache/ignite/internal/visor/node/VisorIgfsConfiguration.java index fbb751a..f29dac5 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/visor/node/VisorIgfsConfiguration.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/visor/node/VisorIgfsConfiguration.java @@ -126,7 +126,7 @@ public class VisorIgfsConfiguration implements Serializable { * @param igfs IGFS configuration. * @return Data transfer object for IGFS configuration properties. */ - public static VisorIgfsConfiguration from(IgfsConfiguration igfs) { + public static VisorIgfsConfiguration from(FileSystemConfiguration igfs) { VisorIgfsConfiguration cfg = new VisorIgfsConfiguration(); cfg.name = igfs.getName(); @@ -177,13 +177,13 @@ public class VisorIgfsConfiguration implements Serializable { * @param igfss Igfs configurations. * @return igfs configurations properties. */ - public static Iterable<VisorIgfsConfiguration> list(IgfsConfiguration[] igfss) { + public static Iterable<VisorIgfsConfiguration> list(FileSystemConfiguration[] igfss) { if (igfss == null) return Collections.emptyList(); final Collection<VisorIgfsConfiguration> cfgs = new ArrayList<>(igfss.length); - for (IgfsConfiguration igfs : igfss) + for (FileSystemConfiguration igfs : igfss) cfgs.add(from(igfs)); return cfgs; http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/8e2a08b9/modules/core/src/main/java/org/apache/ignite/internal/visor/util/VisorTaskUtils.java ---------------------------------------------------------------------- diff --git a/modules/core/src/main/java/org/apache/ignite/internal/visor/util/VisorTaskUtils.java b/modules/core/src/main/java/org/apache/ignite/internal/visor/util/VisorTaskUtils.java index 31acae7..51e3c6e 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/visor/util/VisorTaskUtils.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/visor/util/VisorTaskUtils.java @@ -46,7 +46,7 @@ import java.util.concurrent.atomic.*; import java.util.zip.*; import static java.lang.System.*; -import static org.apache.ignite.configuration.IgfsConfiguration.*; +import static org.apache.ignite.configuration.FileSystemConfiguration.*; import static org.apache.ignite.events.EventType.*; /** http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/8e2a08b9/modules/core/src/test/config/igfs-loopback.xml ---------------------------------------------------------------------- diff --git a/modules/core/src/test/config/igfs-loopback.xml b/modules/core/src/test/config/igfs-loopback.xml index 81ce501..ab4b172 100644 --- a/modules/core/src/test/config/igfs-loopback.xml +++ b/modules/core/src/test/config/igfs-loopback.xml @@ -80,7 +80,7 @@ <property name="igfsConfiguration"> <list> - <bean class="org.apache.ignite.configuration.IgfsConfiguration"> + <bean class="org.apache.ignite.configuration.FileSystemConfiguration"> <property name="name" value="igfs"/> <property name="metaCacheName" value="igfs-meta"/> <property name="dataCacheName" value="igfs-data"/> http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/8e2a08b9/modules/core/src/test/config/igfs-shmem.xml ---------------------------------------------------------------------- diff --git a/modules/core/src/test/config/igfs-shmem.xml b/modules/core/src/test/config/igfs-shmem.xml index d30051b..bb0b0c1 100644 --- a/modules/core/src/test/config/igfs-shmem.xml +++ b/modules/core/src/test/config/igfs-shmem.xml @@ -80,7 +80,7 @@ <property name="igfsConfiguration"> <list> - <bean class="org.apache.ignite.configuration.IgfsConfiguration"> + <bean class="org.apache.ignite.configuration.FileSystemConfiguration"> <property name="name" value="igfs"/> <property name="metaCacheName" value="igfs-meta"/> <property name="dataCacheName" value="igfs-data"/> http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/8e2a08b9/modules/core/src/test/java/org/apache/ignite/igfs/IgfsEventsAbstractSelfTest.java ---------------------------------------------------------------------- diff --git a/modules/core/src/test/java/org/apache/ignite/igfs/IgfsEventsAbstractSelfTest.java b/modules/core/src/test/java/org/apache/ignite/igfs/IgfsEventsAbstractSelfTest.java index c1b36a6..2d9d269 100644 --- a/modules/core/src/test/java/org/apache/ignite/igfs/IgfsEventsAbstractSelfTest.java +++ b/modules/core/src/test/java/org/apache/ignite/igfs/IgfsEventsAbstractSelfTest.java @@ -84,8 +84,8 @@ public abstract class IgfsEventsAbstractSelfTest extends GridCommonAbstractTest /** * @return IGFS configuration for this test. */ - protected IgfsConfiguration getIgfsConfiguration() throws IgniteCheckedException { - IgfsConfiguration igfsCfg = new IgfsConfiguration(); + protected FileSystemConfiguration getIgfsConfiguration() throws IgniteCheckedException { + FileSystemConfiguration igfsCfg = new FileSystemConfiguration(); igfsCfg.setDataCacheName("dataCache"); igfsCfg.setMetaCacheName("metaCache"); @@ -109,7 +109,7 @@ public abstract class IgfsEventsAbstractSelfTest extends GridCommonAbstractTest * @return Grid configuration. * @throws Exception If failed. */ - protected IgniteConfiguration getConfiguration(String gridName, IgfsConfiguration igfsCfg) throws Exception { + protected IgniteConfiguration getConfiguration(String gridName, FileSystemConfiguration igfsCfg) throws Exception { IgniteConfiguration cfg = IgnitionEx.loadConfiguration("config/hadoop/default-config.xml").get1(); assert cfg != null; @@ -118,7 +118,7 @@ public abstract class IgfsEventsAbstractSelfTest extends GridCommonAbstractTest cfg.setIncludeEventTypes(concat(EVTS_IGFS, EVT_TASK_FAILED, EVT_TASK_FINISHED, EVT_JOB_MAPPED)); - cfg.setIgfsConfiguration(igfsCfg); + cfg.setFileSystemConfiguration(igfsCfg); cfg.setCacheConfiguration(getCacheConfiguration(gridName)); http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/8e2a08b9/modules/core/src/test/java/org/apache/ignite/igfs/IgfsFragmentizerAbstractSelfTest.java ---------------------------------------------------------------------- diff --git a/modules/core/src/test/java/org/apache/ignite/igfs/IgfsFragmentizerAbstractSelfTest.java b/modules/core/src/test/java/org/apache/ignite/igfs/IgfsFragmentizerAbstractSelfTest.java index 6ae9c86..3fa0804 100644 --- a/modules/core/src/test/java/org/apache/ignite/igfs/IgfsFragmentizerAbstractSelfTest.java +++ b/modules/core/src/test/java/org/apache/ignite/igfs/IgfsFragmentizerAbstractSelfTest.java @@ -63,7 +63,7 @@ public class IgfsFragmentizerAbstractSelfTest extends IgfsCommonAbstractTest { cfg.setCacheConfiguration(metaConfiguration(), dataConfiguration()); - IgfsConfiguration igfsCfg = new IgfsConfiguration(); + FileSystemConfiguration igfsCfg = new FileSystemConfiguration(); igfsCfg.setName("igfs"); igfsCfg.setMetaCacheName(META_CACHE_NAME); @@ -76,7 +76,7 @@ public class IgfsFragmentizerAbstractSelfTest extends IgfsCommonAbstractTest { igfsCfg.setFragmentizerThrottlingBlockLength(16 * IGFS_BLOCK_SIZE); igfsCfg.setFragmentizerThrottlingDelay(10); - cfg.setIgfsConfiguration(igfsCfg); + cfg.setFileSystemConfiguration(igfsCfg); return cfg; } http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/8e2a08b9/modules/core/src/test/java/org/apache/ignite/internal/processors/igfs/IgfsAbstractSelfTest.java ---------------------------------------------------------------------- diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/igfs/IgfsAbstractSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/igfs/IgfsAbstractSelfTest.java index 31fc4a7..9a99611 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/igfs/IgfsAbstractSelfTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/igfs/IgfsAbstractSelfTest.java @@ -183,7 +183,7 @@ public abstract class IgfsAbstractSelfTest extends IgfsCommonAbstractTest { */ protected Ignite startGridWithIgfs(String gridName, String igfsName, IgfsMode mode, @Nullable IgfsSecondaryFileSystem secondaryFs, @Nullable Map<String, String> restCfg) throws Exception { - IgfsConfiguration igfsCfg = new IgfsConfiguration(); + FileSystemConfiguration igfsCfg = new FileSystemConfiguration(); igfsCfg.setDataCacheName("dataCache"); igfsCfg.setMetaCacheName("metaCache"); @@ -226,7 +226,7 @@ public abstract class IgfsAbstractSelfTest extends IgfsCommonAbstractTest { cfg.setDiscoverySpi(discoSpi); cfg.setCacheConfiguration(dataCacheCfg, metaCacheCfg); - cfg.setIgfsConfiguration(igfsCfg); + cfg.setFileSystemConfiguration(igfsCfg); cfg.setLocalHost("127.0.0.1"); cfg.setConnectorConfiguration(null); http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/8e2a08b9/modules/core/src/test/java/org/apache/ignite/internal/processors/igfs/IgfsCachePerBlockLruEvictionPolicySelfTest.java ---------------------------------------------------------------------- diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/igfs/IgfsCachePerBlockLruEvictionPolicySelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/igfs/IgfsCachePerBlockLruEvictionPolicySelfTest.java index e4b2913..fd590f5 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/igfs/IgfsCachePerBlockLruEvictionPolicySelfTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/igfs/IgfsCachePerBlockLruEvictionPolicySelfTest.java @@ -79,7 +79,7 @@ public class IgfsCachePerBlockLruEvictionPolicySelfTest extends IgfsCommonAbstra * @throws Exception If failed. */ private void startPrimary() throws Exception { - IgfsConfiguration igfsCfg = new IgfsConfiguration(); + FileSystemConfiguration igfsCfg = new FileSystemConfiguration(); igfsCfg.setDataCacheName("dataCache"); igfsCfg.setMetaCacheName("metaCache"); @@ -130,7 +130,7 @@ public class IgfsCachePerBlockLruEvictionPolicySelfTest extends IgfsCommonAbstra cfg.setDiscoverySpi(discoSpi); cfg.setCacheConfiguration(dataCacheCfg, metaCacheCfg); - cfg.setIgfsConfiguration(igfsCfg); + cfg.setFileSystemConfiguration(igfsCfg); cfg.setLocalHost("127.0.0.1"); cfg.setConnectorConfiguration(null); @@ -149,7 +149,7 @@ public class IgfsCachePerBlockLruEvictionPolicySelfTest extends IgfsCommonAbstra * @throws Exception If failed. */ private void startSecondary() throws Exception { - IgfsConfiguration igfsCfg = new IgfsConfiguration(); + FileSystemConfiguration igfsCfg = new FileSystemConfiguration(); igfsCfg.setDataCacheName("dataCache"); igfsCfg.setMetaCacheName("metaCache"); @@ -188,7 +188,7 @@ public class IgfsCachePerBlockLruEvictionPolicySelfTest extends IgfsCommonAbstra cfg.setDiscoverySpi(discoSpi); cfg.setCacheConfiguration(dataCacheCfg, metaCacheCfg); - cfg.setIgfsConfiguration(igfsCfg); + cfg.setFileSystemConfiguration(igfsCfg); cfg.setLocalHost("127.0.0.1"); cfg.setConnectorConfiguration(null); http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/8e2a08b9/modules/core/src/test/java/org/apache/ignite/internal/processors/igfs/IgfsCacheSelfTest.java ---------------------------------------------------------------------- diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/igfs/IgfsCacheSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/igfs/IgfsCacheSelfTest.java index bde395b..255e319 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/igfs/IgfsCacheSelfTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/igfs/IgfsCacheSelfTest.java @@ -57,13 +57,13 @@ public class IgfsCacheSelfTest extends IgfsCommonAbstractTest { cfg.setDiscoverySpi(discoSpi); - IgfsConfiguration igfsCfg = new IgfsConfiguration(); + FileSystemConfiguration igfsCfg = new FileSystemConfiguration(); igfsCfg.setMetaCacheName(META_CACHE_NAME); igfsCfg.setDataCacheName(DATA_CACHE_NAME); igfsCfg.setName("igfs"); - cfg.setIgfsConfiguration(igfsCfg); + cfg.setFileSystemConfiguration(igfsCfg); return cfg; } http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/8e2a08b9/modules/core/src/test/java/org/apache/ignite/internal/processors/igfs/IgfsDataManagerSelfTest.java ---------------------------------------------------------------------- diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/igfs/IgfsDataManagerSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/igfs/IgfsDataManagerSelfTest.java index 2af01bf..297cc7b 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/igfs/IgfsDataManagerSelfTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/igfs/IgfsDataManagerSelfTest.java @@ -95,7 +95,7 @@ public class IgfsDataManagerSelfTest extends IgfsCommonAbstractTest { cfg.setDiscoverySpi(discoSpi); - IgfsConfiguration igfsCfg = new IgfsConfiguration(); + FileSystemConfiguration igfsCfg = new FileSystemConfiguration(); igfsCfg.setMetaCacheName(META_CACHE_NAME); igfsCfg.setDataCacheName(DATA_CACHE_NAME); @@ -103,7 +103,7 @@ public class IgfsDataManagerSelfTest extends IgfsCommonAbstractTest { igfsCfg.setName("igfs"); igfsCfg.setBlockSize(BLOCK_SIZE); - cfg.setIgfsConfiguration(igfsCfg); + cfg.setFileSystemConfiguration(igfsCfg); return cfg; } http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/8e2a08b9/modules/core/src/test/java/org/apache/ignite/internal/processors/igfs/IgfsMetaManagerSelfTest.java ---------------------------------------------------------------------- diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/igfs/IgfsMetaManagerSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/igfs/IgfsMetaManagerSelfTest.java index 1a52b7c..11d6cc5 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/igfs/IgfsMetaManagerSelfTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/igfs/IgfsMetaManagerSelfTest.java @@ -75,13 +75,13 @@ public class IgfsMetaManagerSelfTest extends IgfsCommonAbstractTest { cfg.setDiscoverySpi(discoSpi); - IgfsConfiguration igfsCfg = new IgfsConfiguration(); + FileSystemConfiguration igfsCfg = new FileSystemConfiguration(); igfsCfg.setMetaCacheName(META_CACHE_NAME); igfsCfg.setDataCacheName(DATA_CACHE_NAME); igfsCfg.setName("igfs"); - cfg.setIgfsConfiguration(igfsCfg); + cfg.setFileSystemConfiguration(igfsCfg); return cfg; } http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/8e2a08b9/modules/core/src/test/java/org/apache/ignite/internal/processors/igfs/IgfsMetricsSelfTest.java ---------------------------------------------------------------------- diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/igfs/IgfsMetricsSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/igfs/IgfsMetricsSelfTest.java index 0739148..0af1dea 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/igfs/IgfsMetricsSelfTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/igfs/IgfsMetricsSelfTest.java @@ -101,7 +101,7 @@ public class IgfsMetricsSelfTest extends IgfsCommonAbstractTest { * @throws Exception If failed. */ private IgniteConfiguration primaryConfiguration(int idx) throws Exception { - IgfsConfiguration igfsCfg = new IgfsConfiguration(); + FileSystemConfiguration igfsCfg = new FileSystemConfiguration(); igfsCfg.setDataCacheName("dataCache"); igfsCfg.setMetaCacheName("metaCache"); @@ -146,7 +146,7 @@ public class IgfsMetricsSelfTest extends IgfsCommonAbstractTest { cfg.setDiscoverySpi(discoSpi); cfg.setCacheConfiguration(dataCacheCfg, metaCacheCfg); - cfg.setIgfsConfiguration(igfsCfg); + cfg.setFileSystemConfiguration(igfsCfg); cfg.setLocalHost("127.0.0.1"); @@ -159,7 +159,7 @@ public class IgfsMetricsSelfTest extends IgfsCommonAbstractTest { * @throws Exception If failed. */ private void startSecondary() throws Exception { - IgfsConfiguration igfsCfg = new IgfsConfiguration(); + FileSystemConfiguration igfsCfg = new FileSystemConfiguration(); igfsCfg.setDataCacheName("dataCache"); igfsCfg.setMetaCacheName("metaCache"); @@ -198,7 +198,7 @@ public class IgfsMetricsSelfTest extends IgfsCommonAbstractTest { cfg.setDiscoverySpi(discoSpi); cfg.setCacheConfiguration(dataCacheCfg, metaCacheCfg); - cfg.setIgfsConfiguration(igfsCfg); + cfg.setFileSystemConfiguration(igfsCfg); cfg.setLocalHost("127.0.0.1"); http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/8e2a08b9/modules/core/src/test/java/org/apache/ignite/internal/processors/igfs/IgfsModesSelfTest.java ---------------------------------------------------------------------- diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/igfs/IgfsModesSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/igfs/IgfsModesSelfTest.java index 1a37e93..ea136ac 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/igfs/IgfsModesSelfTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/igfs/IgfsModesSelfTest.java @@ -82,7 +82,7 @@ public class IgfsModesSelfTest extends IgfsCommonAbstractTest { private void startUp() throws Exception { startUpSecondary(); - IgfsConfiguration igfsCfg = new IgfsConfiguration(); + FileSystemConfiguration igfsCfg = new FileSystemConfiguration(); igfsCfg.setDataCacheName("partitioned"); igfsCfg.setMetaCacheName("replicated"); @@ -128,7 +128,7 @@ public class IgfsModesSelfTest extends IgfsCommonAbstractTest { cfg.setDiscoverySpi(discoSpi); cfg.setCacheConfiguration(metaCacheCfg, cacheCfg); - cfg.setIgfsConfiguration(igfsCfg); + cfg.setFileSystemConfiguration(igfsCfg); cfg.setLocalHost("127.0.0.1"); cfg.setConnectorConfiguration(null); @@ -144,7 +144,7 @@ public class IgfsModesSelfTest extends IgfsCommonAbstractTest { * @throws Exception If failed. */ private void startUpSecondary() throws Exception { - IgfsConfiguration igfsCfg = new IgfsConfiguration(); + FileSystemConfiguration igfsCfg = new FileSystemConfiguration(); igfsCfg.setDataCacheName("partitioned"); igfsCfg.setMetaCacheName("replicated"); @@ -185,7 +185,7 @@ public class IgfsModesSelfTest extends IgfsCommonAbstractTest { cfg.setDiscoverySpi(discoSpi); cfg.setCacheConfiguration(metaCacheCfg, cacheCfg); - cfg.setIgfsConfiguration(igfsCfg); + cfg.setFileSystemConfiguration(igfsCfg); cfg.setLocalHost("127.0.0.1"); cfg.setConnectorConfiguration(null); http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/8e2a08b9/modules/core/src/test/java/org/apache/ignite/internal/processors/igfs/IgfsProcessorSelfTest.java ---------------------------------------------------------------------- diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/igfs/IgfsProcessorSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/igfs/IgfsProcessorSelfTest.java index 135e543..0e03aa8 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/igfs/IgfsProcessorSelfTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/igfs/IgfsProcessorSelfTest.java @@ -73,7 +73,7 @@ public class IgfsProcessorSelfTest extends IgfsCommonAbstractTest { igfs = grid.fileSystem(igfsName()); - IgfsConfiguration[] cfgs = grid.configuration().getIgfsConfiguration(); + FileSystemConfiguration[] cfgs = grid.configuration().getFileSystemConfiguration(); assert cfgs.length == 1; @@ -111,13 +111,13 @@ public class IgfsProcessorSelfTest extends IgfsCommonAbstractTest { cfg.setDiscoverySpi(discoSpi); - IgfsConfiguration igfsCfg = new IgfsConfiguration(); + FileSystemConfiguration igfsCfg = new FileSystemConfiguration(); igfsCfg.setMetaCacheName(META_CACHE_NAME); igfsCfg.setDataCacheName(DATA_CACHE_NAME); igfsCfg.setName("igfs"); - cfg.setIgfsConfiguration(igfsCfg); + cfg.setFileSystemConfiguration(igfsCfg); return cfg; } http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/8e2a08b9/modules/core/src/test/java/org/apache/ignite/internal/processors/igfs/IgfsProcessorValidationSelfTest.java ---------------------------------------------------------------------- diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/igfs/IgfsProcessorValidationSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/igfs/IgfsProcessorValidationSelfTest.java index 6f444c0..063c970 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/igfs/IgfsProcessorValidationSelfTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/igfs/IgfsProcessorValidationSelfTest.java @@ -37,7 +37,7 @@ import static org.apache.ignite.igfs.IgfsMode.*; * Tests for node validation logic in {@link IgfsProcessor}. * <p> * Tests starting with "testLocal" are checking - * {@link IgfsProcessor#validateLocalIgfsConfigurations(org.apache.ignite.configuration.IgfsConfiguration[])}. + * {@link IgfsProcessor#validateLocalIgfsConfigurations(org.apache.ignite.configuration.FileSystemConfiguration[])}. * <p> * Tests starting with "testRemote" are checking {@link IgfsProcessor#checkIgfsOnRemoteNode(org.apache.ignite.cluster.ClusterNode)}. */ @@ -61,10 +61,10 @@ public class IgfsProcessorValidationSelfTest extends IgfsCommonAbstractTest { private static final String metaCache2Name = "metaCache2"; /** First IGFS config in grid #1. */ - private IgfsConfiguration g1IgfsCfg1 = new IgfsConfiguration(); + private FileSystemConfiguration g1IgfsCfg1 = new FileSystemConfiguration(); /** Second IGFS config in grid#1. */ - private IgfsConfiguration g1IgfsCfg2 = new IgfsConfiguration(); + private FileSystemConfiguration g1IgfsCfg2 = new FileSystemConfiguration(); /** {@inheritDoc} */ @Override protected void beforeTest() throws Exception { @@ -89,7 +89,7 @@ public class IgfsProcessorValidationSelfTest extends IgfsCommonAbstractTest { g1IgfsCfg2.setDataCacheName(dataCache2Name); g1IgfsCfg2.setMetaCacheName(metaCache2Name); - cfg.setIgfsConfiguration(g1IgfsCfg1, g1IgfsCfg2); + cfg.setFileSystemConfiguration(g1IgfsCfg1, g1IgfsCfg2); cfg.setLocalHost("127.0.0.1"); @@ -278,11 +278,11 @@ public class IgfsProcessorValidationSelfTest extends IgfsCommonAbstractTest { g1Cfg.setCacheConfiguration(concat(dataCaches(1024), metaCaches(), CacheConfiguration.class)); g2Cfg.setCacheConfiguration(concat(dataCaches(1024), metaCaches(), CacheConfiguration.class)); - IgfsConfiguration g2IgfsCfg1 = new IgfsConfiguration(g1IgfsCfg1); + FileSystemConfiguration g2IgfsCfg1 = new FileSystemConfiguration(g1IgfsCfg1); g2IgfsCfg1.setBlockSize(g2IgfsCfg1.getBlockSize() + 100); - g2Cfg.setIgfsConfiguration(g2IgfsCfg1, g1IgfsCfg2); + g2Cfg.setFileSystemConfiguration(g2IgfsCfg1, g1IgfsCfg2); G.start(g1Cfg); @@ -310,8 +310,8 @@ public class IgfsProcessorValidationSelfTest extends IgfsCommonAbstractTest { public void testRemoteIfMetaCacheNameDiffers() throws Exception { IgniteConfiguration g2Cfg = getConfiguration("g2"); - IgfsConfiguration g2IgfsCfg1 = new IgfsConfiguration(g1IgfsCfg1); - IgfsConfiguration g2IgfsCfg2 = new IgfsConfiguration(g1IgfsCfg2); + FileSystemConfiguration g2IgfsCfg1 = new FileSystemConfiguration(g1IgfsCfg1); + FileSystemConfiguration g2IgfsCfg2 = new FileSystemConfiguration(g1IgfsCfg2); g2IgfsCfg1.setMetaCacheName("g2MetaCache1"); g2IgfsCfg2.setMetaCacheName("g2MetaCache2"); @@ -320,7 +320,7 @@ public class IgfsProcessorValidationSelfTest extends IgfsCommonAbstractTest { g2Cfg.setCacheConfiguration(concat(dataCaches(1024), metaCaches("g2MetaCache1", "g2MetaCache2"), CacheConfiguration.class)); - g2Cfg.setIgfsConfiguration(g2IgfsCfg1, g2IgfsCfg2); + g2Cfg.setFileSystemConfiguration(g2IgfsCfg1, g2IgfsCfg2); G.start(g1Cfg); @@ -333,8 +333,8 @@ public class IgfsProcessorValidationSelfTest extends IgfsCommonAbstractTest { public void testRemoteIfMetaCacheNameEquals() throws Exception { IgniteConfiguration g2Cfg = getConfiguration("g2"); - IgfsConfiguration g2IgfsCfg1 = new IgfsConfiguration(g1IgfsCfg1); - IgfsConfiguration g2IgfsCfg2 = new IgfsConfiguration(g1IgfsCfg2); + FileSystemConfiguration g2IgfsCfg1 = new FileSystemConfiguration(g1IgfsCfg1); + FileSystemConfiguration g2IgfsCfg2 = new FileSystemConfiguration(g1IgfsCfg2); g2IgfsCfg1.setName("g2IgfsCfg1"); g2IgfsCfg2.setName("g2IgfsCfg2"); @@ -346,7 +346,7 @@ public class IgfsProcessorValidationSelfTest extends IgfsCommonAbstractTest { g2Cfg.setCacheConfiguration(concat(dataCaches(1024, "g2DataCache1", "g2DataCache2"), metaCaches(), CacheConfiguration.class)); - g2Cfg.setIgfsConfiguration(g2IgfsCfg1, g2IgfsCfg2); + g2Cfg.setFileSystemConfiguration(g2IgfsCfg1, g2IgfsCfg2); G.start(g1Cfg); @@ -359,8 +359,8 @@ public class IgfsProcessorValidationSelfTest extends IgfsCommonAbstractTest { public void testRemoteIfDataCacheNameDiffers() throws Exception { IgniteConfiguration g2Cfg = getConfiguration("g2"); - IgfsConfiguration g2IgfsCfg1 = new IgfsConfiguration(g1IgfsCfg1); - IgfsConfiguration g2IgfsCfg2 = new IgfsConfiguration(g1IgfsCfg2); + FileSystemConfiguration g2IgfsCfg1 = new FileSystemConfiguration(g1IgfsCfg1); + FileSystemConfiguration g2IgfsCfg2 = new FileSystemConfiguration(g1IgfsCfg2); g2IgfsCfg1.setDataCacheName("g2DataCache1"); g2IgfsCfg2.setDataCacheName("g2DataCache2"); @@ -369,7 +369,7 @@ public class IgfsProcessorValidationSelfTest extends IgfsCommonAbstractTest { g2Cfg.setCacheConfiguration(concat(dataCaches(1024, "g2DataCache1", "g2DataCache2"), metaCaches(), CacheConfiguration.class)); - g2Cfg.setIgfsConfiguration(g2IgfsCfg1, g2IgfsCfg2); + g2Cfg.setFileSystemConfiguration(g2IgfsCfg1, g2IgfsCfg2); G.start(g1Cfg); @@ -382,8 +382,8 @@ public class IgfsProcessorValidationSelfTest extends IgfsCommonAbstractTest { public void testRemoteIfDataCacheNameEquals() throws Exception { IgniteConfiguration g2Cfg = getConfiguration("g2"); - IgfsConfiguration g2IgfsCfg1 = new IgfsConfiguration(g1IgfsCfg1); - IgfsConfiguration g2IgfsCfg2 = new IgfsConfiguration(g1IgfsCfg2); + FileSystemConfiguration g2IgfsCfg1 = new FileSystemConfiguration(g1IgfsCfg1); + FileSystemConfiguration g2IgfsCfg2 = new FileSystemConfiguration(g1IgfsCfg2); g2IgfsCfg1.setName("g2IgfsCfg1"); g2IgfsCfg2.setName("g2IgfsCfg2"); @@ -395,7 +395,7 @@ public class IgfsProcessorValidationSelfTest extends IgfsCommonAbstractTest { g2Cfg.setCacheConfiguration(concat(dataCaches(1024), metaCaches("g2MetaCache1", "g2MetaCache2"), CacheConfiguration.class)); - g2Cfg.setIgfsConfiguration(g2IgfsCfg1, g2IgfsCfg2); + g2Cfg.setFileSystemConfiguration(g2IgfsCfg1, g2IgfsCfg2); G.start(g1Cfg); @@ -408,8 +408,8 @@ public class IgfsProcessorValidationSelfTest extends IgfsCommonAbstractTest { public void testRemoteIfDefaultModeDiffers() throws Exception { IgniteConfiguration g2Cfg = getConfiguration("g2"); - IgfsConfiguration g2IgfsCfg1 = new IgfsConfiguration(g1IgfsCfg1); - IgfsConfiguration g2IgfsCfg2 = new IgfsConfiguration(g1IgfsCfg2); + FileSystemConfiguration g2IgfsCfg1 = new FileSystemConfiguration(g1IgfsCfg1); + FileSystemConfiguration g2IgfsCfg2 = new FileSystemConfiguration(g1IgfsCfg2); g1IgfsCfg1.setDefaultMode(DUAL_ASYNC); g1IgfsCfg2.setDefaultMode(DUAL_ASYNC); @@ -420,7 +420,7 @@ public class IgfsProcessorValidationSelfTest extends IgfsCommonAbstractTest { g1Cfg.setCacheConfiguration(concat(dataCaches(1024), metaCaches(), CacheConfiguration.class)); g2Cfg.setCacheConfiguration(concat(dataCaches(1024), metaCaches(), CacheConfiguration.class)); - g2Cfg.setIgfsConfiguration(g2IgfsCfg1, g2IgfsCfg2); + g2Cfg.setFileSystemConfiguration(g2IgfsCfg1, g2IgfsCfg2); G.start(g1Cfg); @@ -433,8 +433,8 @@ public class IgfsProcessorValidationSelfTest extends IgfsCommonAbstractTest { public void testRemoteIfPathModeDiffers() throws Exception { IgniteConfiguration g2Cfg = getConfiguration("g2"); - IgfsConfiguration g2IgfsCfg1 = new IgfsConfiguration(g1IgfsCfg1); - IgfsConfiguration g2IgfsCfg2 = new IgfsConfiguration(g1IgfsCfg2); + FileSystemConfiguration g2IgfsCfg1 = new FileSystemConfiguration(g1IgfsCfg1); + FileSystemConfiguration g2IgfsCfg2 = new FileSystemConfiguration(g1IgfsCfg2); g2IgfsCfg1.setPathModes(Collections.singletonMap("/somePath", DUAL_SYNC)); g2IgfsCfg2.setPathModes(Collections.singletonMap("/somePath", DUAL_SYNC)); @@ -442,7 +442,7 @@ public class IgfsProcessorValidationSelfTest extends IgfsCommonAbstractTest { g1Cfg.setCacheConfiguration(concat(dataCaches(1024), metaCaches(), CacheConfiguration.class)); g2Cfg.setCacheConfiguration(concat(dataCaches(1024), metaCaches(), CacheConfiguration.class)); - g2Cfg.setIgfsConfiguration(g2IgfsCfg1, g2IgfsCfg2); + g2Cfg.setFileSystemConfiguration(g2IgfsCfg1, g2IgfsCfg2); G.start(g1Cfg); http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/8e2a08b9/modules/core/src/test/java/org/apache/ignite/internal/processors/igfs/IgfsServerManagerIpcEndpointRegistrationAbstractSelfTest.java ---------------------------------------------------------------------- diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/igfs/IgfsServerManagerIpcEndpointRegistrationAbstractSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/igfs/IgfsServerManagerIpcEndpointRegistrationAbstractSelfTest.java index ee8e7bd..694d5c3 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/igfs/IgfsServerManagerIpcEndpointRegistrationAbstractSelfTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/igfs/IgfsServerManagerIpcEndpointRegistrationAbstractSelfTest.java @@ -36,7 +36,7 @@ import java.util.*; import java.util.concurrent.atomic.*; import static org.apache.ignite.cache.CacheAtomicityMode.*; -import static org.apache.ignite.configuration.IgfsConfiguration.*; +import static org.apache.ignite.configuration.FileSystemConfiguration.*; /** * Base test class for {@link IgfsServer} checking IPC endpoint registrations. @@ -58,7 +58,7 @@ public abstract class IgfsServerManagerIpcEndpointRegistrationAbstractSelfTest e public void testLoopbackEndpointsRegistration() throws Exception { IgniteConfiguration cfg = gridConfiguration(); - cfg.setIgfsConfiguration( + cfg.setFileSystemConfiguration( igfsConfiguration("tcp", DFLT_IPC_PORT, null) ); @@ -77,7 +77,7 @@ public abstract class IgfsServerManagerIpcEndpointRegistrationAbstractSelfTest e public void testLoopbackEndpointsCustomHostRegistration() throws Exception { IgniteConfiguration cfg = gridConfiguration(); - cfg.setIgfsConfiguration( + cfg.setFileSystemConfiguration( igfsConfiguration("tcp", DFLT_IPC_PORT, "127.0.0.1"), igfsConfiguration("tcp", DFLT_IPC_PORT + 1, U.getLocalHost().getHostName())); @@ -154,7 +154,7 @@ public abstract class IgfsServerManagerIpcEndpointRegistrationAbstractSelfTest e * @param endPntHost End point host. * @return test-purposed IgfsConfiguration. */ - protected IgfsConfiguration igfsConfiguration(@Nullable String endPntType, @Nullable Integer endPntPort, + protected FileSystemConfiguration igfsConfiguration(@Nullable String endPntType, @Nullable Integer endPntPort, @Nullable String endPntHost) throws IgniteCheckedException { HashMap<String, String> endPntCfg = null; @@ -170,7 +170,7 @@ public abstract class IgfsServerManagerIpcEndpointRegistrationAbstractSelfTest e endPntCfg.put("host", endPntHost); } - IgfsConfiguration igfsConfiguration = new IgfsConfiguration(); + FileSystemConfiguration igfsConfiguration = new FileSystemConfiguration(); igfsConfiguration.setDataCacheName("partitioned"); igfsConfiguration.setMetaCacheName("replicated"); http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/8e2a08b9/modules/core/src/test/java/org/apache/ignite/internal/processors/igfs/IgfsServerManagerIpcEndpointRegistrationOnLinuxAndMacSelfTest.java ---------------------------------------------------------------------- diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/igfs/IgfsServerManagerIpcEndpointRegistrationOnLinuxAndMacSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/igfs/IgfsServerManagerIpcEndpointRegistrationOnLinuxAndMacSelfTest.java index d7319f4..57f10d9 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/igfs/IgfsServerManagerIpcEndpointRegistrationOnLinuxAndMacSelfTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/igfs/IgfsServerManagerIpcEndpointRegistrationOnLinuxAndMacSelfTest.java @@ -20,7 +20,7 @@ package org.apache.ignite.internal.processors.igfs; import org.apache.ignite.configuration.*; import org.apache.ignite.internal.util.typedef.*; -import static org.apache.ignite.configuration.IgfsConfiguration.*; +import static org.apache.ignite.configuration.FileSystemConfiguration.*; /** * Tests for {@link IgfsServer} that checks all IPC endpoint registration types @@ -34,7 +34,7 @@ public class IgfsServerManagerIpcEndpointRegistrationOnLinuxAndMacSelfTest public void testLoopbackAndShmemEndpointsRegistration() throws Exception { IgniteConfiguration cfg = gridConfiguration(); - cfg.setIgfsConfiguration( + cfg.setFileSystemConfiguration( igfsConfiguration(null, null, null), // Check null IPC endpoint config won't bring any hassles. igfsConfiguration("tcp", DFLT_IPC_PORT + 1, null), igfsConfiguration("shmem", DFLT_IPC_PORT + 2, null)); http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/8e2a08b9/modules/core/src/test/java/org/apache/ignite/internal/processors/igfs/IgfsServerManagerIpcEndpointRegistrationOnWindowsSelfTest.java ---------------------------------------------------------------------- diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/igfs/IgfsServerManagerIpcEndpointRegistrationOnWindowsSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/igfs/IgfsServerManagerIpcEndpointRegistrationOnWindowsSelfTest.java index 259f31a..4f18aff 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/igfs/IgfsServerManagerIpcEndpointRegistrationOnWindowsSelfTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/igfs/IgfsServerManagerIpcEndpointRegistrationOnWindowsSelfTest.java @@ -40,7 +40,7 @@ public class IgfsServerManagerIpcEndpointRegistrationOnWindowsSelfTest @Override public Object call() throws Exception { IgniteConfiguration cfg = gridConfiguration(); - cfg.setIgfsConfiguration(igfsConfiguration("shmem", IpcSharedMemoryServerEndpoint.DFLT_IPC_PORT, + cfg.setFileSystemConfiguration(igfsConfiguration("shmem", IpcSharedMemoryServerEndpoint.DFLT_IPC_PORT, null)); return G.start(cfg); http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/8e2a08b9/modules/core/src/test/java/org/apache/ignite/internal/processors/igfs/IgfsSizeSelfTest.java ---------------------------------------------------------------------- diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/igfs/IgfsSizeSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/igfs/IgfsSizeSelfTest.java index b212f02..c62e759 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/igfs/IgfsSizeSelfTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/igfs/IgfsSizeSelfTest.java @@ -111,7 +111,7 @@ public class IgfsSizeSelfTest extends IgfsCommonAbstractTest { @Override protected IgniteConfiguration getConfiguration(String gridName) throws Exception { IgniteConfiguration cfg = super.getConfiguration(gridName); - IgfsConfiguration igfsCfg = new IgfsConfiguration(); + FileSystemConfiguration igfsCfg = new FileSystemConfiguration(); igfsCfg.setDataCacheName(DATA_CACHE_NAME); igfsCfg.setMetaCacheName(META_CACHE_NAME); @@ -154,7 +154,7 @@ public class IgfsSizeSelfTest extends IgfsCommonAbstractTest { cfg.setDiscoverySpi(discoSpi); cfg.setCacheConfiguration(metaCfg, dataCfg); - cfg.setIgfsConfiguration(igfsCfg); + cfg.setFileSystemConfiguration(igfsCfg); return cfg; } http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/8e2a08b9/modules/core/src/test/java/org/apache/ignite/internal/processors/igfs/IgfsStreamsSelfTest.java ---------------------------------------------------------------------- diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/igfs/IgfsStreamsSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/igfs/IgfsStreamsSelfTest.java index 38d4af5..25816c7 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/igfs/IgfsStreamsSelfTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/igfs/IgfsStreamsSelfTest.java @@ -110,7 +110,7 @@ public class IgfsStreamsSelfTest extends IgfsCommonAbstractTest { cfg.setDiscoverySpi(discoSpi); - IgfsConfiguration igfsCfg = new IgfsConfiguration(); + FileSystemConfiguration igfsCfg = new FileSystemConfiguration(); igfsCfg.setMetaCacheName(META_CACHE_NAME); igfsCfg.setDataCacheName(DATA_CACHE_NAME); @@ -118,7 +118,7 @@ public class IgfsStreamsSelfTest extends IgfsCommonAbstractTest { igfsCfg.setBlockSize(CFG_BLOCK_SIZE); igfsCfg.setFragmentizerEnabled(true); - cfg.setIgfsConfiguration(igfsCfg); + cfg.setFileSystemConfiguration(igfsCfg); return cfg; } http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/8e2a08b9/modules/core/src/test/java/org/apache/ignite/internal/processors/igfs/IgfsTaskSelfTest.java ---------------------------------------------------------------------- diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/igfs/IgfsTaskSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/igfs/IgfsTaskSelfTest.java index 2f096d1..d1778df 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/igfs/IgfsTaskSelfTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/igfs/IgfsTaskSelfTest.java @@ -97,7 +97,7 @@ public class IgfsTaskSelfTest extends IgfsCommonAbstractTest { * @return Grid configuration */ private IgniteConfiguration config(int idx) { - IgfsConfiguration igfsCfg = new IgfsConfiguration(); + FileSystemConfiguration igfsCfg = new FileSystemConfiguration(); igfsCfg.setDataCacheName("dataCache"); igfsCfg.setMetaCacheName("metaCache"); @@ -133,7 +133,7 @@ public class IgfsTaskSelfTest extends IgfsCommonAbstractTest { cfg.setDiscoverySpi(discoSpi); cfg.setCacheConfiguration(dataCacheCfg, metaCacheCfg); - cfg.setIgfsConfiguration(igfsCfg); + cfg.setFileSystemConfiguration(igfsCfg); cfg.setGridName("node-" + idx); http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/8e2a08b9/modules/core/src/test/java/org/apache/ignite/internal/processors/igfs/split/IgfsAbstractRecordResolverSelfTest.java ---------------------------------------------------------------------- diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/igfs/split/IgfsAbstractRecordResolverSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/igfs/split/IgfsAbstractRecordResolverSelfTest.java index e7c819b..487d391 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/igfs/split/IgfsAbstractRecordResolverSelfTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/igfs/split/IgfsAbstractRecordResolverSelfTest.java @@ -50,7 +50,7 @@ public class IgfsAbstractRecordResolverSelfTest extends GridCommonAbstractTest { /** {@inheritDoc} */ @Override protected void beforeTestsStarted() throws Exception { - IgfsConfiguration igfsCfg = new IgfsConfiguration(); + FileSystemConfiguration igfsCfg = new FileSystemConfiguration(); igfsCfg.setDataCacheName("dataCache"); igfsCfg.setMetaCacheName("metaCache"); @@ -87,7 +87,7 @@ public class IgfsAbstractRecordResolverSelfTest extends GridCommonAbstractTest { cfg.setDiscoverySpi(discoSpi); cfg.setCacheConfiguration(dataCacheCfg, metaCacheCfg); - cfg.setIgfsConfiguration(igfsCfg); + cfg.setFileSystemConfiguration(igfsCfg); Ignite g = G.start(cfg); http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/8e2a08b9/modules/core/src/test/java/org/apache/ignite/internal/util/ipc/shmem/IpcSharedMemoryNodeStartup.java ---------------------------------------------------------------------- diff --git a/modules/core/src/test/java/org/apache/ignite/internal/util/ipc/shmem/IpcSharedMemoryNodeStartup.java b/modules/core/src/test/java/org/apache/ignite/internal/util/ipc/shmem/IpcSharedMemoryNodeStartup.java index 400bbb1..8051a3e 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/util/ipc/shmem/IpcSharedMemoryNodeStartup.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/util/ipc/shmem/IpcSharedMemoryNodeStartup.java @@ -41,7 +41,7 @@ public class IpcSharedMemoryNodeStartup { public static void main(String[] args) throws Exception{ IgniteConfiguration cfg = new IgniteConfiguration(); - IgfsConfiguration igfsCfg = new IgfsConfiguration(); + FileSystemConfiguration igfsCfg = new FileSystemConfiguration(); TcpDiscoverySpi discoSpi = new TcpDiscoverySpi(); @@ -60,7 +60,7 @@ public class IpcSharedMemoryNodeStartup { igfsCfg.setMetaCacheName("partitioned"); igfsCfg.setName("igfs"); - cfg.setIgfsConfiguration(igfsCfg); + cfg.setFileSystemConfiguration(igfsCfg); CacheConfiguration cacheCfg = new CacheConfiguration(); http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/8e2a08b9/modules/hadoop/src/main/java/org/apache/ignite/hadoop/fs/IgniteHadoopIgfsSecondaryFileSystem.java ---------------------------------------------------------------------- diff --git a/modules/hadoop/src/main/java/org/apache/ignite/hadoop/fs/IgniteHadoopIgfsSecondaryFileSystem.java b/modules/hadoop/src/main/java/org/apache/ignite/hadoop/fs/IgniteHadoopIgfsSecondaryFileSystem.java index dfdbe91..d79a2d0 100644 --- a/modules/hadoop/src/main/java/org/apache/ignite/hadoop/fs/IgniteHadoopIgfsSecondaryFileSystem.java +++ b/modules/hadoop/src/main/java/org/apache/ignite/hadoop/fs/IgniteHadoopIgfsSecondaryFileSystem.java @@ -103,16 +103,14 @@ public class IgniteHadoopIgfsSecondaryFileSystem implements IgfsSecondaryFileSys * @param detailMsg Detailed error message. * @return Appropriate exception. */ + @SuppressWarnings({"ThrowableResultOfMethodCallIgnored", "unchecked"}) private IgfsException handleSecondaryFsError(IOException e, String detailMsg) { boolean wrongVer = X.hasCause(e, RemoteException.class) || (e.getMessage() != null && e.getMessage().contains("Failed on local")); - IgfsException igfsErr = !wrongVer ? cast(detailMsg, e) : + return !wrongVer ? cast(detailMsg, e) : new IgfsInvalidHdfsVersionException("HDFS version you are connecting to differs from local " + - "version.", e); - - return igfsErr; - } + "version.", e); } /** * Cast IO exception to IGFS exception. http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/8e2a08b9/modules/hadoop/src/main/java/org/apache/ignite/hadoop/fs/v1/IgniteHadoopFileSystem.java ---------------------------------------------------------------------- diff --git a/modules/hadoop/src/main/java/org/apache/ignite/hadoop/fs/v1/IgniteHadoopFileSystem.java b/modules/hadoop/src/main/java/org/apache/ignite/hadoop/fs/v1/IgniteHadoopFileSystem.java index 2ee9b50..f541bce 100644 --- a/modules/hadoop/src/main/java/org/apache/ignite/hadoop/fs/v1/IgniteHadoopFileSystem.java +++ b/modules/hadoop/src/main/java/org/apache/ignite/hadoop/fs/v1/IgniteHadoopFileSystem.java @@ -39,7 +39,7 @@ import java.net.*; import java.util.*; import java.util.concurrent.atomic.*; -import static org.apache.ignite.configuration.IgfsConfiguration.*; +import static org.apache.ignite.configuration.FileSystemConfiguration.*; import static org.apache.ignite.igfs.IgfsMode.*; import static org.apache.ignite.internal.processors.hadoop.fs.HadoopParameters.*; import static org.apache.ignite.internal.processors.hadoop.igfs.HadoopIgfsUtils.*; @@ -192,6 +192,7 @@ public class IgniteHadoopFileSystem extends FileSystem { } /** {@inheritDoc} */ + @SuppressWarnings("ConstantConditions") @Override public void initialize(URI name, Configuration cfg) throws IOException { enterBusy(); @@ -689,6 +690,7 @@ public class IgniteHadoopFileSystem extends FileSystem { } /** {@inheritDoc} */ + @SuppressWarnings("unchecked") @Override public boolean rename(Path src, Path dst) throws IOException { A.notNull(src, "src"); A.notNull(dst, "dst"); @@ -742,6 +744,7 @@ public class IgniteHadoopFileSystem extends FileSystem { } /** {@inheritDoc} */ + @SuppressWarnings("unchecked") @Override public boolean delete(Path f, boolean recursive) throws IOException { A.notNull(f, "f"); @@ -901,6 +904,7 @@ public class IgniteHadoopFileSystem extends FileSystem { } /** {@inheritDoc} */ + @SuppressWarnings("unchecked") @Override public boolean mkdirs(Path f, FsPermission perm) throws IOException { A.notNull(f, "f"); http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/8e2a08b9/modules/hadoop/src/main/java/org/apache/ignite/hadoop/fs/v2/IgniteHadoopFileSystem.java ---------------------------------------------------------------------- diff --git a/modules/hadoop/src/main/java/org/apache/ignite/hadoop/fs/v2/IgniteHadoopFileSystem.java b/modules/hadoop/src/main/java/org/apache/ignite/hadoop/fs/v2/IgniteHadoopFileSystem.java index bcf3029..16d804f 100644 --- a/modules/hadoop/src/main/java/org/apache/ignite/hadoop/fs/v2/IgniteHadoopFileSystem.java +++ b/modules/hadoop/src/main/java/org/apache/ignite/hadoop/fs/v2/IgniteHadoopFileSystem.java @@ -39,7 +39,7 @@ import java.net.*; import java.util.*; import java.util.concurrent.atomic.*; -import static org.apache.ignite.configuration.IgfsConfiguration.*; +import static org.apache.ignite.configuration.FileSystemConfiguration.*; import static org.apache.ignite.igfs.IgfsMode.*; import static org.apache.ignite.internal.processors.hadoop.fs.HadoopParameters.*; import static org.apache.ignite.internal.processors.hadoop.igfs.HadoopIgfsUtils.*; http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/8e2a08b9/modules/hadoop/src/main/java/org/apache/ignite/hadoop/mapreduce/IgniteHadoopMapReducePlanner.java ---------------------------------------------------------------------- diff --git a/modules/hadoop/src/main/java/org/apache/ignite/hadoop/mapreduce/IgniteHadoopMapReducePlanner.java b/modules/hadoop/src/main/java/org/apache/ignite/hadoop/mapreduce/IgniteHadoopMapReducePlanner.java new file mode 100644 index 0000000..9a6b4d8 --- /dev/null +++ b/modules/hadoop/src/main/java/org/apache/ignite/hadoop/mapreduce/IgniteHadoopMapReducePlanner.java @@ -0,0 +1,435 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ignite.hadoop.mapreduce; + +import org.apache.ignite.*; +import org.apache.ignite.cluster.*; +import org.apache.ignite.igfs.*; +import org.apache.ignite.internal.*; +import org.apache.ignite.internal.processors.hadoop.igfs.*; +import org.apache.ignite.internal.processors.hadoop.planner.*; +import org.apache.ignite.internal.processors.igfs.*; +import org.apache.ignite.internal.processors.hadoop.*; +import org.apache.ignite.internal.util.typedef.*; +import org.apache.ignite.internal.util.typedef.internal.*; +import org.apache.ignite.resources.*; +import org.jetbrains.annotations.*; + +import java.util.*; + +import static org.apache.ignite.IgniteFileSystem.*; + +/** + * Default map-reduce planner implementation. + */ +public class IgniteHadoopMapReducePlanner implements HadoopMapReducePlanner { + /** Injected grid. */ + @IgniteInstanceResource + private Ignite ignite; + + /** Logger. */ + @SuppressWarnings("UnusedDeclaration") + @LoggerResource + private IgniteLogger log; + + /** {@inheritDoc} */ + @Override public HadoopMapReducePlan preparePlan(HadoopJob job, Collection<ClusterNode> top, + @Nullable HadoopMapReducePlan oldPlan) throws IgniteCheckedException { + // Convert collection of topology nodes to collection of topology node IDs. + Collection<UUID> topIds = new HashSet<>(top.size(), 1.0f); + + for (ClusterNode topNode : top) + topIds.add(topNode.id()); + + Map<UUID, Collection<HadoopInputSplit>> mappers = mappers(top, topIds, job.input()); + + int rdcCnt = job.info().reducers(); + + if (rdcCnt < 0) + throw new IgniteCheckedException("Number of reducers must be non-negative, actual: " + rdcCnt); + + Map<UUID, int[]> reducers = reducers(top, mappers, rdcCnt); + + return new HadoopDefaultMapReducePlan(mappers, reducers); + } + + /** + * Create plan for mappers. + * + * @param top Topology nodes. + * @param topIds Topology node IDs. + * @param splits Splits. + * @return Mappers map. + * @throws IgniteCheckedException If failed. + */ + private Map<UUID, Collection<HadoopInputSplit>> mappers(Collection<ClusterNode> top, Collection<UUID> topIds, + Iterable<HadoopInputSplit> splits) throws IgniteCheckedException { + Map<UUID, Collection<HadoopInputSplit>> mappers = new HashMap<>(); + + Map<String, Collection<UUID>> nodes = hosts(top); + + Map<UUID, Integer> nodeLoads = new HashMap<>(top.size(), 1.0f); // Track node load. + + for (UUID nodeId : topIds) + nodeLoads.put(nodeId, 0); + + for (HadoopInputSplit split : splits) { + UUID nodeId = nodeForSplit(split, topIds, nodes, nodeLoads); + + if (log.isDebugEnabled()) + log.debug("Mapped split to node [split=" + split + ", nodeId=" + nodeId + ']'); + + Collection<HadoopInputSplit> nodeSplits = mappers.get(nodeId); + + if (nodeSplits == null) { + nodeSplits = new ArrayList<>(); + + mappers.put(nodeId, nodeSplits); + } + + nodeSplits.add(split); + + // Updated node load. + nodeLoads.put(nodeId, nodeLoads.get(nodeId) + 1); + } + + return mappers; + } + + /** + * Groups nodes by host names. + * + * @param top Topology to group. + * @return Map. + */ + private static Map<String, Collection<UUID>> hosts(Collection<ClusterNode> top) { + Map<String, Collection<UUID>> grouped = U.newHashMap(top.size()); + + for (ClusterNode node : top) { + for (String host : node.hostNames()) { + Collection<UUID> nodeIds = grouped.get(host); + + if (nodeIds == null) { + // Expecting 1-2 nodes per host. + nodeIds = new ArrayList<>(2); + + grouped.put(host, nodeIds); + } + + nodeIds.add(node.id()); + } + } + + return grouped; + } + + /** + * Determine the best node for this split. + * + * @param split Split. + * @param topIds Topology node IDs. + * @param nodes Nodes. + * @param nodeLoads Node load tracker. + * @return Node ID. + */ + @SuppressWarnings("unchecked") + private UUID nodeForSplit(HadoopInputSplit split, Collection<UUID> topIds, Map<String, Collection<UUID>> nodes, + Map<UUID, Integer> nodeLoads) throws IgniteCheckedException { + if (split instanceof HadoopFileBlock) { + HadoopFileBlock split0 = (HadoopFileBlock)split; + + if (IGFS_SCHEME.equalsIgnoreCase(split0.file().getScheme())) { + HadoopIgfsEndpoint endpoint = new HadoopIgfsEndpoint(split0.file().getAuthority()); + + IgfsEx igfs = null; + + if (F.eq(ignite.name(), endpoint.grid())) + igfs = (IgfsEx)((IgniteEx)ignite).igfsx(endpoint.igfs()); + + if (igfs != null && !igfs.isProxy(split0.file())) { + Collection<IgfsBlockLocation> blocks; + + try { + blocks = igfs.affinity(new IgfsPath(split0.file()), split0.start(), split0.length()); + } + catch (IgniteException e) { + throw new IgniteCheckedException(e); + } + + assert blocks != null; + + if (blocks.size() == 1) + // Fast-path, split consists of one IGFS block (as in most cases). + return bestNode(blocks.iterator().next().nodeIds(), topIds, nodeLoads, false); + else { + // Slow-path, file consists of multiple IGFS blocks. First, find the most co-located nodes. + Map<UUID, Long> nodeMap = new HashMap<>(); + + List<UUID> bestNodeIds = null; + long bestLen = -1L; + + for (IgfsBlockLocation block : blocks) { + for (UUID blockNodeId : block.nodeIds()) { + if (topIds.contains(blockNodeId)) { + Long oldLen = nodeMap.get(blockNodeId); + long newLen = oldLen == null ? block.length() : oldLen + block.length(); + + nodeMap.put(blockNodeId, newLen); + + if (bestNodeIds == null || bestLen < newLen) { + bestNodeIds = new ArrayList<>(1); + + bestNodeIds.add(blockNodeId); + + bestLen = newLen; + } + else if (bestLen == newLen) { + assert !F.isEmpty(bestNodeIds); + + bestNodeIds.add(blockNodeId); + } + } + } + } + + if (bestNodeIds != null) { + return bestNodeIds.size() == 1 ? bestNodeIds.get(0) : + bestNode(bestNodeIds, topIds, nodeLoads, true); + } + } + } + } + } + + // Cannot use local IGFS for some reason, try selecting the node by host. + Collection<UUID> blockNodes = null; + + for (String host : split.hosts()) { + Collection<UUID> hostNodes = nodes.get(host); + + if (!F.isEmpty(hostNodes)) { + if (blockNodes == null) + blockNodes = new ArrayList<>(hostNodes); + else + blockNodes.addAll(hostNodes); + } + } + + return bestNode(blockNodes, topIds, nodeLoads, false); + } + + /** + * Finds the best (the least loaded) node among the candidates. + * + * @param candidates Candidates. + * @param topIds Topology node IDs. + * @param nodeLoads Known node loads. + * @param skipTopCheck Whether to skip topology check. + * @return The best node. + */ + private UUID bestNode(@Nullable Collection<UUID> candidates, Collection<UUID> topIds, Map<UUID, Integer> nodeLoads, + boolean skipTopCheck) { + UUID bestNode = null; + int bestLoad = Integer.MAX_VALUE; + + if (candidates != null) { + for (UUID candidate : candidates) { + if (skipTopCheck || topIds.contains(candidate)) { + int load = nodeLoads.get(candidate); + + if (bestNode == null || bestLoad > load) { + bestNode = candidate; + bestLoad = load; + + if (bestLoad == 0) + break; // Minimum load possible, no need for further iterations. + } + } + } + } + + if (bestNode == null) { + // Blocks are located on nodes which are not Hadoop-enabled, assign to the least loaded one. + bestLoad = Integer.MAX_VALUE; + + for (UUID nodeId : topIds) { + int load = nodeLoads.get(nodeId); + + if (bestNode == null || bestLoad > load) { + bestNode = nodeId; + bestLoad = load; + + if (bestLoad == 0) + break; // Minimum load possible, no need for further iterations. + } + } + } + + assert bestNode != null; + + return bestNode; + } + + /** + * Create plan for reducers. + * + * @param top Topology. + * @param mappers Mappers map. + * @param reducerCnt Reducers count. + * @return Reducers map. + */ + private Map<UUID, int[]> reducers(Collection<ClusterNode> top, + Map<UUID, Collection<HadoopInputSplit>> mappers, int reducerCnt) { + // Determine initial node weights. + int totalWeight = 0; + + List<WeightedNode> nodes = new ArrayList<>(top.size()); + + for (ClusterNode node : top) { + Collection<HadoopInputSplit> split = mappers.get(node.id()); + + int weight = reducerNodeWeight(node, split != null ? split.size() : 0); + + nodes.add(new WeightedNode(node.id(), weight, weight)); + + totalWeight += weight; + } + + // Adjust weights. + int totalAdjustedWeight = 0; + + for (WeightedNode node : nodes) { + node.floatWeight = ((float)node.weight * reducerCnt) / totalWeight; + + node.weight = Math.round(node.floatWeight); + + totalAdjustedWeight += node.weight; + } + + // Apply redundant/lost reducers. + Collections.sort(nodes); + + if (totalAdjustedWeight > reducerCnt) { + // Too much reducers set. + ListIterator<WeightedNode> iter = nodes.listIterator(nodes.size() - 1); + + while (totalAdjustedWeight != reducerCnt) { + if (!iter.hasPrevious()) + iter = nodes.listIterator(nodes.size() - 1); + + WeightedNode node = iter.previous(); + + if (node.weight > 0) { + node.weight -= 1; + + totalAdjustedWeight--; + } + } + } + else if (totalAdjustedWeight < reducerCnt) { + // Not enough reducers set. + ListIterator<WeightedNode> iter = nodes.listIterator(0); + + while (totalAdjustedWeight != reducerCnt) { + if (!iter.hasNext()) + iter = nodes.listIterator(0); + + WeightedNode node = iter.next(); + + if (node.floatWeight > 0.0f) { + node.weight += 1; + + totalAdjustedWeight++; + } + } + } + + int idx = 0; + + Map<UUID, int[]> reducers = new HashMap<>(nodes.size(), 1.0f); + + for (WeightedNode node : nodes) { + if (node.weight > 0) { + int[] arr = new int[node.weight]; + + for (int i = 0; i < arr.length; i++) + arr[i] = idx++; + + reducers.put(node.nodeId, arr); + } + } + + return reducers; + } + + /** + * Calculate node weight based on node metrics and data co-location. + * + * @param node Node. + * @param splitCnt Splits mapped to this node. + * @return Node weight. + */ + @SuppressWarnings("UnusedParameters") + protected int reducerNodeWeight(ClusterNode node, int splitCnt) { + return splitCnt; + } + + /** + * Weighted node. + */ + private static class WeightedNode implements Comparable<WeightedNode> { + /** Node ID. */ + private final UUID nodeId; + + /** Weight. */ + private int weight; + + /** Floating point weight. */ + private float floatWeight; + + /** + * Constructor. + * + * @param nodeId Node ID. + * @param weight Weight. + * @param floatWeight Floating point weight. + */ + private WeightedNode(UUID nodeId, int weight, float floatWeight) { + this.nodeId = nodeId; + this.weight = weight; + this.floatWeight = floatWeight; + } + + /** {@inheritDoc} */ + @Override public boolean equals(Object obj) { + return obj != null && obj instanceof WeightedNode && F.eq(nodeId, ((WeightedNode)obj).nodeId); + } + + /** {@inheritDoc} */ + @Override public int hashCode() { + return nodeId.hashCode(); + } + + /** {@inheritDoc} */ + @Override public int compareTo(@NotNull WeightedNode other) { + float res = other.floatWeight - floatWeight; + + return res > 0.0f ? 1 : res < 0.0f ? -1 : nodeId.compareTo(other.nodeId); + } + } +} http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/8e2a08b9/modules/hadoop/src/main/java/org/apache/ignite/hadoop/mapreduce/package.html ---------------------------------------------------------------------- diff --git a/modules/hadoop/src/main/java/org/apache/ignite/hadoop/mapreduce/package.html b/modules/hadoop/src/main/java/org/apache/ignite/hadoop/mapreduce/package.html index 1802afd..e289841 100644 --- a/modules/hadoop/src/main/java/org/apache/ignite/hadoop/mapreduce/package.html +++ b/modules/hadoop/src/main/java/org/apache/ignite/hadoop/mapreduce/package.html @@ -19,6 +19,6 @@ <html> <body> <!-- Package description. --> - Ignite Hadoop Accelerator map-reduce client protocol provider. + Ignite Hadoop Accelerator map-reduce classes. </body> </html> http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/8e2a08b9/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopProcessor.java ---------------------------------------------------------------------- diff --git a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopProcessor.java b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopProcessor.java index f17ce66..e0c5916 100644 --- a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopProcessor.java +++ b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopProcessor.java @@ -19,10 +19,10 @@ package org.apache.ignite.internal.processors.hadoop; import org.apache.ignite.*; import org.apache.ignite.configuration.*; +import org.apache.ignite.hadoop.mapreduce.*; import org.apache.ignite.internal.*; import org.apache.ignite.internal.processors.hadoop.counter.HadoopCounters; import org.apache.ignite.internal.processors.hadoop.jobtracker.*; -import org.apache.ignite.internal.processors.hadoop.planner.*; import org.apache.ignite.internal.processors.hadoop.shuffle.*; import org.apache.ignite.internal.processors.hadoop.taskexecutor.*; import org.apache.ignite.internal.processors.hadoop.taskexecutor.external.*; @@ -210,7 +210,7 @@ public class HadoopProcessor extends HadoopProcessorAdapter { */ private void initializeDefaults(HadoopConfiguration cfg) { if (cfg.getMapReducePlanner() == null) - cfg.setMapReducePlanner(new HadoopDefaultMapReducePlanner()); + cfg.setMapReducePlanner(new IgniteHadoopMapReducePlanner()); } /** http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/8e2a08b9/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/fs/HadoopDistributedFileSystem.java ---------------------------------------------------------------------- diff --git a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/fs/HadoopDistributedFileSystem.java b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/fs/HadoopDistributedFileSystem.java index 88c5899..509f443 100644 --- a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/fs/HadoopDistributedFileSystem.java +++ b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/fs/HadoopDistributedFileSystem.java @@ -25,7 +25,7 @@ import org.apache.hadoop.mapreduce.*; import java.io.*; import java.net.*; -import static org.apache.ignite.configuration.IgfsConfiguration.*; +import static org.apache.ignite.configuration.FileSystemConfiguration.*; /** * Wrapper of HDFS for support of separated working directory. http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/8e2a08b9/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/fs/HadoopParameters.java ---------------------------------------------------------------------- diff --git a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/fs/HadoopParameters.java b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/fs/HadoopParameters.java index 36fabb7..7edcec0 100644 --- a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/fs/HadoopParameters.java +++ b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/fs/HadoopParameters.java @@ -24,7 +24,7 @@ package org.apache.ignite.internal.processors.hadoop.fs; * <ul> * <li> * {@code fs.igfs.[name].open.sequential_reads_before_prefetch} - this parameter overrides - * the one specified in {@link org.apache.ignite.configuration.IgfsConfiguration#getSequentialReadsBeforePrefetch()} + * the one specified in {@link org.apache.ignite.configuration.FileSystemConfiguration#getSequentialReadsBeforePrefetch()} * IGFS data node configuration property. * </li> * <li> http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/8e2a08b9/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/igfs/HadoopIgfsEndpoint.java ---------------------------------------------------------------------- diff --git a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/igfs/HadoopIgfsEndpoint.java b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/igfs/HadoopIgfsEndpoint.java index 16c9351..7502f57 100644 --- a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/igfs/HadoopIgfsEndpoint.java +++ b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/igfs/HadoopIgfsEndpoint.java @@ -26,7 +26,7 @@ import org.jetbrains.annotations.*; import java.io.*; import java.net.*; -import static org.apache.ignite.configuration.IgfsConfiguration.*; +import static org.apache.ignite.configuration.FileSystemConfiguration.*; /** * IGFS endpoint abstraction.
