http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/7d46deb2/modules/hadoop/src/test/java/org/apache/ignite/igfs/IgniteHadoopFileSystemHandshakeSelfTest.java ---------------------------------------------------------------------- diff --git a/modules/hadoop/src/test/java/org/apache/ignite/igfs/IgniteHadoopFileSystemHandshakeSelfTest.java b/modules/hadoop/src/test/java/org/apache/ignite/igfs/IgniteHadoopFileSystemHandshakeSelfTest.java new file mode 100644 index 0000000..0eef7e1 --- /dev/null +++ b/modules/hadoop/src/test/java/org/apache/ignite/igfs/IgniteHadoopFileSystemHandshakeSelfTest.java @@ -0,0 +1,311 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ignite.igfs; + +import org.apache.hadoop.conf.*; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.*; +import org.apache.ignite.*; +import org.apache.ignite.configuration.*; +import org.apache.ignite.hadoop.fs.v2.IgniteHadoopFileSystem; +import org.apache.ignite.internal.processors.igfs.*; +import org.apache.ignite.internal.util.typedef.*; +import org.apache.ignite.spi.communication.tcp.*; +import org.apache.ignite.spi.discovery.tcp.*; +import org.apache.ignite.spi.discovery.tcp.ipfinder.*; +import org.apache.ignite.spi.discovery.tcp.ipfinder.vm.*; +import org.apache.ignite.testframework.*; + +import java.io.*; +import java.net.*; +import java.util.*; +import java.util.concurrent.*; + +import static org.apache.ignite.cache.CacheAtomicityMode.*; +import static org.apache.ignite.cache.CacheDistributionMode.*; +import static org.apache.ignite.cache.CacheMode.*; +import static org.apache.ignite.cache.CacheWriteSynchronizationMode.*; +import static org.apache.ignite.igfs.IgfsMode.*; +import static org.apache.ignite.internal.igfs.hadoop.IgfsHadoopUtils.*; +import static org.apache.ignite.internal.util.ipc.shmem.IpcSharedMemoryServerEndpoint.*; + +/** + * Tests for IGFS file system handshake. + */ +public class IgniteHadoopFileSystemHandshakeSelfTest extends IgfsCommonAbstractTest { + /** IP finder. */ + private static final TcpDiscoveryIpFinder IP_FINDER = new TcpDiscoveryVmIpFinder(true); + + /** Grid name. */ + private static final String GRID_NAME = "grid"; + + /** IGFS name. */ + private static final String IGFS_NAME = "igfs"; + + /** IGFS path. */ + private static final IgfsPath PATH = new IgfsPath("/path"); + + /** {@inheritDoc} */ + @Override protected void afterTest() throws Exception { + stopAllGrids(true); + } + + /** + * Tests for Grid and IGFS having normal names. + * + * @throws Exception If failed. + */ + public void testHandshake() throws Exception { + startUp(false, false); + + checkValid(IGFS_NAME + ":" + GRID_NAME + "@"); + checkValid(IGFS_NAME + ":" + GRID_NAME + "@127.0.0.1"); + checkValid(IGFS_NAME + ":" + GRID_NAME + "@127.0.0.1:" + DFLT_IPC_PORT); + + checkInvalid(IGFS_NAME + "@"); + checkInvalid(IGFS_NAME + "@127.0.0.1"); + checkInvalid(IGFS_NAME + "@127.0.0.1:" + DFLT_IPC_PORT); + + checkInvalid(":" + GRID_NAME + "@"); + checkInvalid(":" + GRID_NAME + "@127.0.0.1"); + checkInvalid(":" + GRID_NAME + "@127.0.0.1:" + DFLT_IPC_PORT); + + checkInvalid(""); + checkInvalid("127.0.0.1"); + checkInvalid("127.0.0.1:" + DFLT_IPC_PORT); + } + + /** + * Tests for Grid having {@code null} name and IGFS having normal name. + * + * @throws Exception If failed. + */ + public void testHandshakeDefaultGrid() throws Exception { + startUp(true, false); + + checkInvalid(IGFS_NAME + ":" + GRID_NAME + "@"); + checkInvalid(IGFS_NAME + ":" + GRID_NAME + "@127.0.0.1"); + checkInvalid(IGFS_NAME + ":" + GRID_NAME + "@127.0.0.1:" + DFLT_IPC_PORT); + + checkValid(IGFS_NAME + "@"); + checkValid(IGFS_NAME + "@127.0.0.1"); + checkValid(IGFS_NAME + "@127.0.0.1:" + DFLT_IPC_PORT); + + checkInvalid(":" + GRID_NAME + "@"); + checkInvalid(":" + GRID_NAME + "@127.0.0.1"); + checkInvalid(":" + GRID_NAME + "@127.0.0.1:" + DFLT_IPC_PORT); + + checkInvalid(""); + checkInvalid("127.0.0.1"); + checkInvalid("127.0.0.1:" + DFLT_IPC_PORT); + } + + /** + * Tests for Grid having normal name and IGFS having {@code null} name. + * + * @throws Exception If failed. + */ + public void testHandshakeDefaultIgfs() throws Exception { + startUp(false, true); + + checkInvalid(IGFS_NAME + ":" + GRID_NAME + "@"); + checkInvalid(IGFS_NAME + ":" + GRID_NAME + "@127.0.0.1"); + checkInvalid(IGFS_NAME + ":" + GRID_NAME + "@127.0.0.1:" + DFLT_IPC_PORT); + + checkInvalid(IGFS_NAME + "@"); + checkInvalid(IGFS_NAME + "@127.0.0.1"); + checkInvalid(IGFS_NAME + "@127.0.0.1:" + DFLT_IPC_PORT); + + checkValid(":" + GRID_NAME + "@"); + checkValid(":" + GRID_NAME + "@127.0.0.1"); + checkValid(":" + GRID_NAME + "@127.0.0.1:" + DFLT_IPC_PORT); + + checkInvalid(""); + checkInvalid("127.0.0.1"); + checkInvalid("127.0.0.1:" + DFLT_IPC_PORT); + } + + /** + * Tests for Grid having {@code null} name and IGFS having {@code null} name. + * + * @throws Exception If failed. + */ + public void testHandshakeDefaultGridDefaultIgfs() throws Exception { + startUp(true, true); + + checkInvalid(IGFS_NAME + ":" + GRID_NAME + "@"); + checkInvalid(IGFS_NAME + ":" + GRID_NAME + "@127.0.0.1"); + checkInvalid(IGFS_NAME + ":" + GRID_NAME + "@127.0.0.1:" + DFLT_IPC_PORT); + + checkInvalid(IGFS_NAME + "@"); + checkInvalid(IGFS_NAME + "@127.0.0.1"); + checkInvalid(IGFS_NAME + "@127.0.0.1:" + DFLT_IPC_PORT); + + checkInvalid(":" + GRID_NAME + "@"); + checkInvalid(":" + GRID_NAME + "@127.0.0.1"); + checkInvalid(":" + GRID_NAME + "@127.0.0.1:" + DFLT_IPC_PORT); + + checkValid(""); + checkValid("127.0.0.1"); + checkValid("127.0.0.1:" + DFLT_IPC_PORT); + } + + /** + * Perform startup. + * + * @param dfltGridName Default Grid name. + * @param dfltIgfsName Default IGFS name. + * @throws Exception If failed. + */ + private void startUp(boolean dfltGridName, boolean dfltIgfsName) throws Exception { + Ignite ignite = G.start(gridConfiguration(dfltGridName, dfltIgfsName)); + + IgniteFs igfs = ignite.fileSystem(dfltIgfsName ? null : IGFS_NAME); + + igfs.mkdirs(PATH); + } + + /** + * Create Grid configuration. + * + * @param dfltGridName Default Grid name. + * @param dfltIgfsName Default IGFS name. + * @return Grid configuration. + * @throws Exception If failed. + */ + private IgniteConfiguration gridConfiguration(boolean dfltGridName, boolean dfltIgfsName) throws Exception { + IgniteConfiguration cfg = super.getConfiguration(dfltGridName ? null : GRID_NAME); + + cfg.setLocalHost("127.0.0.1"); + cfg.setConnectorConfiguration(null); + + TcpDiscoverySpi discoSpi = new TcpDiscoverySpi(); + + discoSpi.setIpFinder(IP_FINDER); + + cfg.setDiscoverySpi(discoSpi); + + TcpCommunicationSpi commSpi = new TcpCommunicationSpi(); + + commSpi.setSharedMemoryPort(-1); + + cfg.setCommunicationSpi(commSpi); + + CacheConfiguration metaCacheCfg = defaultCacheConfiguration(); + + metaCacheCfg.setName("replicated"); + metaCacheCfg.setCacheMode(REPLICATED); + metaCacheCfg.setWriteSynchronizationMode(FULL_SYNC); + metaCacheCfg.setQueryIndexEnabled(false); + metaCacheCfg.setAtomicityMode(TRANSACTIONAL); + + CacheConfiguration dataCacheCfg = defaultCacheConfiguration(); + + dataCacheCfg.setName("partitioned"); + dataCacheCfg.setCacheMode(PARTITIONED); + dataCacheCfg.setDistributionMode(PARTITIONED_ONLY); + dataCacheCfg.setWriteSynchronizationMode(FULL_SYNC); + dataCacheCfg.setAffinityMapper(new IgfsGroupDataBlocksKeyMapper(128)); + dataCacheCfg.setBackups(0); + dataCacheCfg.setQueryIndexEnabled(false); + dataCacheCfg.setAtomicityMode(TRANSACTIONAL); + + cfg.setCacheConfiguration(metaCacheCfg, dataCacheCfg); + + IgfsConfiguration igfsCfg = new IgfsConfiguration(); + + igfsCfg.setDataCacheName("partitioned"); + igfsCfg.setMetaCacheName("replicated"); + igfsCfg.setName(dfltIgfsName ? null : IGFS_NAME); + igfsCfg.setPrefetchBlocks(1); + igfsCfg.setDefaultMode(PRIMARY); + igfsCfg.setIpcEndpointConfiguration(new HashMap<String, String>() {{ + put("type", "tcp"); + put("port", String.valueOf(DFLT_IPC_PORT)); + }}); + + igfsCfg.setManagementPort(-1); + igfsCfg.setBlockSize(512 * 1024); + + cfg.setIgfsConfiguration(igfsCfg); + + return cfg; + } + + /** + * Check valid file system endpoint. + * + * @param authority Authority. + * @throws Exception If failed. + */ + private void checkValid(String authority) throws Exception { + FileSystem fs = fileSystem(authority); + + assert fs.exists(new Path(PATH.toString())); + } + + /** + * Check invalid file system endpoint. + * + * @param authority Authority. + * @throws Exception If failed. + */ + @SuppressWarnings("ThrowableResultOfMethodCallIgnored") + private void checkInvalid(final String authority) throws Exception { + GridTestUtils.assertThrows(log, new Callable<Object>() { + @Override public Object call() throws Exception { + fileSystem(authority); + + return null; + } + }, IOException.class, null); + } + + /** + * + * + * @param authority Authority. + * @return File system. + * @throws Exception If failed. + */ + private static FileSystem fileSystem(String authority) throws Exception { + return FileSystem.get(new URI("igfs://" + authority + "/"), configuration(authority)); + } + + /** + * Create configuration for test. + * + * @param authority Authority. + * @return Configuration. + */ + private static Configuration configuration(String authority) { + Configuration cfg = new Configuration(); + + cfg.set("fs.defaultFS", "igfs://" + authority + "/"); + cfg.set("fs.igfs.impl", org.apache.ignite.hadoop.fs.v1.IgniteHadoopFileSystem.class.getName()); + cfg.set("fs.AbstractFileSystem.igfs.impl", + IgniteHadoopFileSystem.class.getName()); + + cfg.setBoolean("fs.igfs.impl.disable.cache", true); + + cfg.setBoolean(String.format(PARAM_IGFS_ENDPOINT_NO_EMBED, authority), true); + cfg.setBoolean(String.format(PARAM_IGFS_ENDPOINT_NO_LOCAL_SHMEM, authority), true); + + return cfg; + } +}
http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/7d46deb2/modules/hadoop/src/test/java/org/apache/ignite/igfs/IgniteHadoopFileSystemIpcCacheSelfTest.java ---------------------------------------------------------------------- diff --git a/modules/hadoop/src/test/java/org/apache/ignite/igfs/IgniteHadoopFileSystemIpcCacheSelfTest.java b/modules/hadoop/src/test/java/org/apache/ignite/igfs/IgniteHadoopFileSystemIpcCacheSelfTest.java new file mode 100644 index 0000000..bdbff8b --- /dev/null +++ b/modules/hadoop/src/test/java/org/apache/ignite/igfs/IgniteHadoopFileSystemIpcCacheSelfTest.java @@ -0,0 +1,207 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ignite.igfs; + +import org.apache.hadoop.conf.*; +import org.apache.hadoop.fs.*; +import org.apache.ignite.cache.*; +import org.apache.ignite.configuration.*; +import org.apache.ignite.internal.igfs.hadoop.*; +import org.apache.ignite.internal.processors.igfs.*; +import org.apache.ignite.internal.util.ipc.shmem.*; +import org.apache.ignite.internal.util.typedef.*; +import org.apache.ignite.internal.util.typedef.internal.*; +import org.apache.ignite.spi.communication.tcp.*; +import org.apache.ignite.spi.discovery.tcp.*; +import org.apache.ignite.spi.discovery.tcp.ipfinder.*; +import org.apache.ignite.spi.discovery.tcp.ipfinder.vm.*; + +import java.lang.reflect.*; +import java.net.*; +import java.util.*; +import java.util.concurrent.atomic.*; + +import static org.apache.ignite.cache.CacheAtomicityMode.*; +import static org.apache.ignite.cache.CacheMode.*; +import static org.apache.ignite.events.EventType.*; + +/** + * IPC cache test. + */ +public class IgniteHadoopFileSystemIpcCacheSelfTest extends IgfsCommonAbstractTest { + /** IP finder. */ + private static final TcpDiscoveryIpFinder IP_FINDER = new TcpDiscoveryVmIpFinder(true); + + /** Path to test hadoop configuration. */ + private static final String HADOOP_FS_CFG = "modules/core/src/test/config/hadoop/core-site.xml"; + + /** Group size. */ + public static final int GRP_SIZE = 128; + + /** Started grid counter. */ + private static int cnt; + + /** {@inheritDoc} */ + @Override protected IgniteConfiguration getConfiguration(String gridName) throws Exception { + IgniteConfiguration cfg = super.getConfiguration(gridName); + + TcpDiscoverySpi discoSpi = new TcpDiscoverySpi(); + discoSpi.setIpFinder(IP_FINDER); + + cfg.setDiscoverySpi(discoSpi); + + IgfsConfiguration igfsCfg = new IgfsConfiguration(); + + igfsCfg.setDataCacheName("partitioned"); + igfsCfg.setMetaCacheName("replicated"); + igfsCfg.setName("igfs"); + igfsCfg.setManagementPort(IgfsConfiguration.DFLT_MGMT_PORT + cnt); + + igfsCfg.setIpcEndpointConfiguration(new HashMap<String, String>() {{ + put("type", "shmem"); + put("port", String.valueOf(IpcSharedMemoryServerEndpoint.DFLT_IPC_PORT + cnt)); + }}); + + igfsCfg.setBlockSize(512 * 1024); // Together with group blocks mapper will yield 64M per node groups. + + cfg.setIgfsConfiguration(igfsCfg); + + cfg.setCacheConfiguration(cacheConfiguration()); + + cfg.setIncludeEventTypes(EVT_TASK_FAILED, EVT_TASK_FINISHED, EVT_JOB_MAPPED); + + TcpCommunicationSpi commSpi = new TcpCommunicationSpi(); + + commSpi.setSharedMemoryPort(-1); + + cfg.setCommunicationSpi(commSpi); + + cnt++; + + return cfg; + } + + /** + * Gets cache configuration. + * + * @return Cache configuration. + */ + private CacheConfiguration[] cacheConfiguration() { + CacheConfiguration cacheCfg = defaultCacheConfiguration(); + + cacheCfg.setName("partitioned"); + cacheCfg.setCacheMode(PARTITIONED); + cacheCfg.setDistributionMode(CacheDistributionMode.PARTITIONED_ONLY); + cacheCfg.setWriteSynchronizationMode(CacheWriteSynchronizationMode.FULL_SYNC); + cacheCfg.setAffinityMapper(new IgfsGroupDataBlocksKeyMapper(GRP_SIZE)); + cacheCfg.setBackups(0); + cacheCfg.setQueryIndexEnabled(false); + cacheCfg.setAtomicityMode(TRANSACTIONAL); + + CacheConfiguration metaCacheCfg = defaultCacheConfiguration(); + + metaCacheCfg.setName("replicated"); + metaCacheCfg.setCacheMode(REPLICATED); + metaCacheCfg.setWriteSynchronizationMode(CacheWriteSynchronizationMode.FULL_SYNC); + metaCacheCfg.setQueryIndexEnabled(false); + metaCacheCfg.setAtomicityMode(TRANSACTIONAL); + + return new CacheConfiguration[] {metaCacheCfg, cacheCfg}; + } + + /** {@inheritDoc} */ + @Override protected void beforeTestsStarted() throws Exception { + startGrids(4); + } + + /** {@inheritDoc} */ + @Override protected void afterTestsStopped() throws Exception { + G.stopAll(true); + } + + /** + * Test how IPC cache map works. + * + * @throws Exception If failed. + */ + @SuppressWarnings("unchecked") + public void testIpcCache() throws Exception { + Field cacheField = IgfsHadoopIpcIo.class.getDeclaredField("ipcCache"); + + cacheField.setAccessible(true); + + Field activeCntField = IgfsHadoopIpcIo.class.getDeclaredField("activeCnt"); + + activeCntField.setAccessible(true); + + Map<String, IgfsHadoopIpcIo> cache = (Map<String, IgfsHadoopIpcIo>)cacheField.get(null); + + String name = "igfs:" + getTestGridName(0) + "@"; + + Configuration cfg = new Configuration(); + + cfg.addResource(U.resolveIgniteUrl(HADOOP_FS_CFG)); + cfg.setBoolean("fs.igfs.impl.disable.cache", true); + cfg.setBoolean(String.format(IgfsHadoopUtils.PARAM_IGFS_ENDPOINT_NO_EMBED, name), true); + + // Ensure that existing IO is reused. + FileSystem fs1 = FileSystem.get(new URI("igfs://" + name + "/"), cfg); + + assertEquals(1, cache.size()); + + IgfsHadoopIpcIo io = null; + + System.out.println("CACHE: " + cache); + + for (String key : cache.keySet()) { + if (key.contains("10500")) { + io = cache.get(key); + + break; + } + } + + assert io != null; + + assertEquals(1, ((AtomicInteger)activeCntField.get(io)).get()); + + // Ensure that when IO is used by multiple file systems and one of them is closed, IO is not stopped. + FileSystem fs2 = FileSystem.get(new URI("igfs://" + name + "/abc"), cfg); + + assertEquals(1, cache.size()); + assertEquals(2, ((AtomicInteger)activeCntField.get(io)).get()); + + fs2.close(); + + assertEquals(1, cache.size()); + assertEquals(1, ((AtomicInteger)activeCntField.get(io)).get()); + + Field stopField = IgfsHadoopIpcIo.class.getDeclaredField("stopping"); + + stopField.setAccessible(true); + + assert !(Boolean)stopField.get(io); + + // Ensure that IO is stopped when nobody else is need it. + fs1.close(); + + assert cache.isEmpty(); + + assert (Boolean)stopField.get(io); + } +} http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/7d46deb2/modules/hadoop/src/test/java/org/apache/ignite/igfs/IgniteHadoopFileSystemLoggerSelfTest.java ---------------------------------------------------------------------- diff --git a/modules/hadoop/src/test/java/org/apache/ignite/igfs/IgniteHadoopFileSystemLoggerSelfTest.java b/modules/hadoop/src/test/java/org/apache/ignite/igfs/IgniteHadoopFileSystemLoggerSelfTest.java new file mode 100644 index 0000000..8d43e08 --- /dev/null +++ b/modules/hadoop/src/test/java/org/apache/ignite/igfs/IgniteHadoopFileSystemLoggerSelfTest.java @@ -0,0 +1,287 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ignite.igfs; + +import org.apache.ignite.internal.igfs.common.*; +import org.apache.ignite.internal.processors.igfs.*; +import org.apache.ignite.internal.util.typedef.internal.*; + +import java.io.*; +import java.util.*; + +import static org.apache.ignite.igfs.IgfsMode.*; +import static org.apache.ignite.internal.igfs.common.IgfsLogger.*; + +/** + * Grid IGFS client logger test. + */ +public class IgniteHadoopFileSystemLoggerSelfTest extends IgfsCommonAbstractTest { + /** Path string. */ + private static final String PATH_STR = "/dir1/dir2/file;test"; + + /** Path string with escaped semicolons. */ + private static final String PATH_STR_ESCAPED = PATH_STR.replace(';', '~'); + + /** Path. */ + private static final IgfsPath PATH = new IgfsPath(PATH_STR); + + /** IGFS name. */ + private static final String IGFS_NAME = "igfs"; + + /** Log file path. */ + private static final String LOG_DIR = U.getIgniteHome(); + + /** Endpoint address. */ + private static final String ENDPOINT = "localhost:10500"; + + /** Log file name. */ + private static final String LOG_FILE = LOG_DIR + File.separator + "igfs-log-" + IGFS_NAME + "-" + U.jvmPid() + + ".csv"; + + /** {@inheritDoc} */ + @Override protected void beforeTestsStarted() throws Exception { + removeLogs(); + } + + /** {@inheritDoc} */ + @Override protected void afterTest() throws Exception { + removeLogs(); + } + + /** + * Remove existing logs. + * + * @throws Exception If failed. + */ + private void removeLogs() throws Exception { + File dir = new File(LOG_DIR); + + File[] logs = dir.listFiles(new FilenameFilter() { + @Override public boolean accept(File dir, String name) { + return name.startsWith("igfs-log-"); + } + }); + + for (File log : logs) + log.delete(); + } + + /** + * Ensure correct static loggers creation/removal as well as file creation. + * + * @throws Exception If failed. + */ + public void testCreateDelete() throws Exception { + IgfsLogger log = IgfsLogger.logger(ENDPOINT, IGFS_NAME, LOG_DIR, 10); + + IgfsLogger sameLog0 = IgfsLogger.logger(ENDPOINT, IGFS_NAME, LOG_DIR, 10); + + // Loggers for the same endpoint must be the same object. + assert log == sameLog0; + + IgfsLogger otherLog = IgfsLogger.logger("other" + ENDPOINT, IGFS_NAME, LOG_DIR, 10); + + // Logger for another endpoint must be different. + assert log != otherLog; + + otherLog.close(); + + log.logDelete(PATH, PRIMARY, false); + + log.close(); + + File logFile = new File(LOG_FILE); + + // When there are multiple loggers, closing one must not force flushing. + assert !logFile.exists(); + + IgfsLogger sameLog1 = IgfsLogger.logger(ENDPOINT, IGFS_NAME, LOG_DIR, 10); + + assert sameLog0 == sameLog1; + + sameLog0.close(); + + assert !logFile.exists(); + + sameLog1.close(); + + // When we cloe the last logger, it must flush data to disk. + assert logFile.exists(); + + logFile.delete(); + + IgfsLogger sameLog2 = IgfsLogger.logger(ENDPOINT, IGFS_NAME, LOG_DIR, 10); + + // This time we expect new logger instance to be created. + assert sameLog0 != sameLog2; + + sameLog2.close(); + + // As we do not add any records to the logger, we do not expect flushing. + assert !logFile.exists(); + } + + /** + * Test read operations logging. + * + * @throws Exception If failed. + */ + public void testLogRead() throws Exception { + IgfsLogger log = IgfsLogger.logger(ENDPOINT, IGFS_NAME, LOG_DIR, 10); + + log.logOpen(1, PATH, PRIMARY, 2, 3L); + log.logRandomRead(1, 4L, 5); + log.logSeek(1, 6L); + log.logSkip(1, 7L); + log.logMark(1, 8L); + log.logReset(1); + log.logCloseIn(1, 9L, 10L, 11); + + log.close(); + + checkLog( + new SB().a(U.jvmPid() + d() + TYPE_OPEN_IN + d() + PATH_STR_ESCAPED + d() + PRIMARY + d() + 1 + d() + 2 + + d() + 3 + d(14)).toString(), + new SB().a(U.jvmPid() + d() + TYPE_RANDOM_READ + d(3) + 1 + d(7) + 4 + d() + 5 + d(8)).toString(), + new SB().a(U.jvmPid() + d() + TYPE_SEEK + d(3) + 1 + d(7) + 6 + d(9)).toString(), + new SB().a(U.jvmPid() + d() + TYPE_SKIP + d(3) + 1 + d(9) + 7 + d(7)).toString(), + new SB().a(U.jvmPid() + d() + TYPE_MARK + d(3) + 1 + d(10) + 8 + d(6)).toString(), + new SB().a(U.jvmPid() + d() + TYPE_RESET + d(3) + 1 + d(16)).toString(), + new SB().a(U.jvmPid() + d() + TYPE_CLOSE_IN + d(3) + 1 + d(11) + 9 + d() + 10 + d() + 11 + d(3)).toString() + ); + } + + /** + * Test write operations logging. + * + * @throws Exception If failed. + */ + public void testLogWrite() throws Exception { + IgfsLogger log = IgfsLogger.logger(ENDPOINT, IGFS_NAME, LOG_DIR, 10); + + log.logCreate(1, PATH, PRIMARY, true, 2, new Integer(3).shortValue(), 4L); + log.logAppend(2, PATH, PRIMARY, 8); + log.logCloseOut(2, 9L, 10L, 11); + + log.close(); + + checkLog( + new SB().a(U.jvmPid() + d() + TYPE_OPEN_OUT + d() + PATH_STR_ESCAPED + d() + PRIMARY + d() + 1 + d() + + 2 + d(2) + 0 + d() + 1 + d() + 3 + d() + 4 + d(10)).toString(), + new SB().a(U.jvmPid() + d() + TYPE_OPEN_OUT + d() + PATH_STR_ESCAPED + d() + PRIMARY + d() + 2 + d() + + 8 + d(2) + 1 + d(13)).toString(), + new SB().a(U.jvmPid() + d() + TYPE_CLOSE_OUT + d(3) + 2 + d(11) + 9 + d() + 10 + d() + 11 + d(3)) + .toString() + ); + } + + /** + * Test miscellaneous operations logging. + * + * @throws Exception If failed. + */ + @SuppressWarnings("TooBroadScope") + public void testLogMisc() throws Exception { + IgfsLogger log = IgfsLogger.logger(ENDPOINT, IGFS_NAME, LOG_DIR, 10); + + String newFile = "/dir3/file.test"; + String file1 = "/dir3/file1.test"; + String file2 = "/dir3/file1.test"; + + log.logMakeDirectory(PATH, PRIMARY); + log.logRename(PATH, PRIMARY, new IgfsPath(newFile)); + log.logListDirectory(PATH, PRIMARY, new String[] { file1, file2 }); + log.logDelete(PATH, PRIMARY, false); + + log.close(); + + checkLog( + new SB().a(U.jvmPid() + d() + TYPE_DIR_MAKE + d() + PATH_STR_ESCAPED + d() + PRIMARY + d(17)).toString(), + new SB().a(U.jvmPid() + d() + TYPE_RENAME + d() + PATH_STR_ESCAPED + d() + PRIMARY + d(15) + newFile + + d(2)).toString(), + new SB().a(U.jvmPid() + d() + TYPE_DIR_LIST + d() + PATH_STR_ESCAPED + d() + PRIMARY + d(17) + file1 + + DELIM_FIELD_VAL + file2).toString(), + new SB().a(U.jvmPid() + d() + TYPE_DELETE + d(1) + PATH_STR_ESCAPED + d() + PRIMARY + d(16) + 0 + + d()).toString() + ); + } + + /** + * Create IGFS file with the given path. + * + * @param path File path. + * @return IGFS file instance. + */ + private IgfsFile file(String path) { + return new IgfsFileImpl(new IgfsPath(path), new IgfsFileInfo(), 64 * 1024 * 1024); + } + + /** + * Ensure that log file has only the following lines. + * + * @param lines Expected lines. + */ + private void checkLog(String... lines) throws Exception { + BufferedReader br = new BufferedReader(new InputStreamReader(new FileInputStream(LOG_FILE))); + + List<String> logLines = new ArrayList<>(lines.length); + + String nextLogLine; + + while ((nextLogLine = br.readLine()) != null) + logLines.add(nextLogLine); + + U.closeQuiet(br); + + assertEquals(lines.length + 1, logLines.size()); + + assertEquals(logLines.get(0), HDR); + + for (int i = 0; i < lines.length; i++) { + String logLine = logLines.get(i + 1); + + logLine = logLine.substring(logLine.indexOf(DELIM_FIELD, logLine.indexOf(DELIM_FIELD) + 1) + 1); + + assertEquals(lines[i], logLine); + } + } + + /** + * Return single field delimiter. + * + * @return Single field delimiter. + */ + private String d() { + return d(1); + } + + /** + * Return a bunch of field delimiters. + * + * @param cnt Amount of field delimiters. + * @return Field delimiters. + */ + private String d(int cnt) { + SB buf = new SB(); + + for (int i = 0; i < cnt; i++) + buf.a(DELIM_FIELD); + + return buf.toString(); + } +} http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/7d46deb2/modules/hadoop/src/test/java/org/apache/ignite/igfs/IgniteHadoopFileSystemLoggerStateSelfTest.java ---------------------------------------------------------------------- diff --git a/modules/hadoop/src/test/java/org/apache/ignite/igfs/IgniteHadoopFileSystemLoggerStateSelfTest.java b/modules/hadoop/src/test/java/org/apache/ignite/igfs/IgniteHadoopFileSystemLoggerStateSelfTest.java new file mode 100644 index 0000000..fdd2711 --- /dev/null +++ b/modules/hadoop/src/test/java/org/apache/ignite/igfs/IgniteHadoopFileSystemLoggerStateSelfTest.java @@ -0,0 +1,325 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ignite.igfs; + +import org.apache.hadoop.conf.*; +import org.apache.hadoop.fs.FileSystem; +import org.apache.ignite.*; +import org.apache.ignite.cache.*; +import org.apache.ignite.configuration.*; +import org.apache.ignite.hadoop.fs.v1.*; +import org.apache.ignite.internal.igfs.common.*; +import org.apache.ignite.internal.processors.igfs.*; +import org.apache.ignite.internal.util.typedef.*; +import org.apache.ignite.internal.util.typedef.internal.*; +import org.apache.ignite.spi.discovery.tcp.*; +import org.apache.ignite.spi.discovery.tcp.ipfinder.vm.*; + +import java.lang.reflect.*; +import java.net.*; +import java.nio.file.*; +import java.util.*; + +import static org.apache.ignite.cache.CacheAtomicityMode.*; +import static org.apache.ignite.cache.CacheMode.*; +import static org.apache.ignite.igfs.IgfsMode.*; +import static org.apache.ignite.internal.processors.hadoop.fs.HadoopParameters.*; + +/** + * Ensures that sampling is really turned on/off. + */ +public class IgniteHadoopFileSystemLoggerStateSelfTest extends IgfsCommonAbstractTest { + /** IGFS. */ + private IgfsEx igfs; + + /** File system. */ + private FileSystem fs; + + /** Whether logging is enabled in FS configuration. */ + private boolean logging; + + /** whether sampling is enabled. */ + private Boolean sampling; + + /** {@inheritDoc} */ + @Override protected void afterTest() throws Exception { + U.closeQuiet(fs); + + igfs = null; + fs = null; + + G.stopAll(true); + + logging = false; + sampling = null; + } + + /** + * Startup the grid and instantiate the file system. + * + * @throws Exception If failed. + */ + private void startUp() throws Exception { + IgfsConfiguration igfsCfg = new IgfsConfiguration(); + + igfsCfg.setDataCacheName("partitioned"); + igfsCfg.setMetaCacheName("replicated"); + igfsCfg.setName("igfs"); + igfsCfg.setBlockSize(512 * 1024); + igfsCfg.setDefaultMode(PRIMARY); + igfsCfg.setIpcEndpointConfiguration(new HashMap<String, String>() {{ + put("type", "tcp"); + put("port", "10500"); + }}); + + CacheConfiguration cacheCfg = defaultCacheConfiguration(); + + cacheCfg.setName("partitioned"); + cacheCfg.setCacheMode(PARTITIONED); + cacheCfg.setDistributionMode(CacheDistributionMode.PARTITIONED_ONLY); + cacheCfg.setWriteSynchronizationMode(CacheWriteSynchronizationMode.FULL_SYNC); + cacheCfg.setAffinityMapper(new IgfsGroupDataBlocksKeyMapper(128)); + cacheCfg.setBackups(0); + cacheCfg.setQueryIndexEnabled(false); + cacheCfg.setAtomicityMode(TRANSACTIONAL); + + CacheConfiguration metaCacheCfg = defaultCacheConfiguration(); + + metaCacheCfg.setName("replicated"); + metaCacheCfg.setCacheMode(REPLICATED); + metaCacheCfg.setWriteSynchronizationMode(CacheWriteSynchronizationMode.FULL_SYNC); + metaCacheCfg.setQueryIndexEnabled(false); + metaCacheCfg.setAtomicityMode(TRANSACTIONAL); + + IgniteConfiguration cfg = new IgniteConfiguration(); + + cfg.setGridName("igfs-grid"); + + TcpDiscoverySpi discoSpi = new TcpDiscoverySpi(); + + discoSpi.setIpFinder(new TcpDiscoveryVmIpFinder(true)); + + cfg.setDiscoverySpi(discoSpi); + cfg.setCacheConfiguration(metaCacheCfg, cacheCfg); + cfg.setIgfsConfiguration(igfsCfg); + + cfg.setLocalHost("127.0.0.1"); + cfg.setConnectorConfiguration(null); + + Ignite g = G.start(cfg); + + igfs = (IgfsEx)g.fileSystem("igfs"); + + igfs.globalSampling(sampling); + + fs = fileSystem(); + } + + /** + * When logging is disabled and sampling is not set no-op logger must be used. + * + * @throws Exception If failed. + */ + public void testLoggingDisabledSamplingNotSet() throws Exception { + startUp(); + + assert !logEnabled(); + } + + /** + * When logging is enabled and sampling is not set file logger must be used. + * + * @throws Exception If failed. + */ + public void testLoggingEnabledSamplingNotSet() throws Exception { + logging = true; + + startUp(); + + assert logEnabled(); + } + + /** + * When logging is disabled and sampling is disabled no-op logger must be used. + * + * @throws Exception If failed. + */ + public void testLoggingDisabledSamplingDisabled() throws Exception { + sampling = false; + + startUp(); + + assert !logEnabled(); + } + + /** + * When logging is enabled and sampling is disabled no-op logger must be used. + * + * @throws Exception If failed. + */ + public void testLoggingEnabledSamplingDisabled() throws Exception { + logging = true; + sampling = false; + + startUp(); + + assert !logEnabled(); + } + + /** + * When logging is disabled and sampling is enabled file logger must be used. + * + * @throws Exception If failed. + */ + public void testLoggingDisabledSamplingEnabled() throws Exception { + sampling = true; + + startUp(); + + assert logEnabled(); + } + + /** + * When logging is enabled and sampling is enabled file logger must be used. + * + * @throws Exception If failed. + */ + public void testLoggingEnabledSamplingEnabled() throws Exception { + logging = true; + sampling = true; + + startUp(); + + assert logEnabled(); + } + + /** + * Ensure sampling change through API causes changes in logging on subsequent client connections. + * + * @throws Exception If failed. + */ + public void testSamplingChange() throws Exception { + // Start with sampling not set. + startUp(); + + assert !logEnabled(); + + fs.close(); + + // "Not set" => true transition. + igfs.globalSampling(true); + + fs = fileSystem(); + + assert logEnabled(); + + fs.close(); + + // True => "not set" transition. + igfs.globalSampling(null); + + fs = fileSystem(); + + assert !logEnabled(); + + // "Not-set" => false transition. + igfs.globalSampling(false); + + fs = fileSystem(); + + assert !logEnabled(); + + fs.close(); + + // False => "not=set" transition. + igfs.globalSampling(null); + + fs = fileSystem(); + + assert !logEnabled(); + + fs.close(); + + // True => false transition. + igfs.globalSampling(true); + igfs.globalSampling(false); + + fs = fileSystem(); + + assert !logEnabled(); + + fs.close(); + + // False => true transition. + igfs.globalSampling(true); + + fs = fileSystem(); + + assert logEnabled(); + } + + /** + * Ensure that log directory is set to IGFS when client FS connects. + * + * @throws Exception If failed. + */ + @SuppressWarnings("ConstantConditions") + public void testLogDirectory() throws Exception { + startUp(); + + assertEquals(Paths.get(U.getIgniteHome()).normalize().toString(), + igfs.clientLogDirectory()); + } + + /** + * Instantiate new file system. + * + * @return New file system. + * @throws Exception If failed. + */ + private IgniteHadoopFileSystem fileSystem() throws Exception { + Configuration fsCfg = new Configuration(); + + fsCfg.addResource(U.resolveIgniteUrl("modules/core/src/test/config/hadoop/core-site-loopback.xml")); + + fsCfg.setBoolean("fs.igfs.impl.disable.cache", true); + + if (logging) + fsCfg.setBoolean(String.format(PARAM_IGFS_LOG_ENABLED, "igfs:igfs-grid@"), logging); + + fsCfg.setStrings(String.format(PARAM_IGFS_LOG_DIR, "igfs:igfs-grid@"), U.getIgniteHome()); + + return (IgniteHadoopFileSystem)FileSystem.get(new URI("igfs://igfs:igfs-grid@/"), fsCfg); + } + + /** + * Ensure that real logger is used by the file system. + * + * @return {@code True} in case path is secondary. + * @throws Exception If failed. + */ + private boolean logEnabled() throws Exception { + assert fs != null; + + Field field = fs.getClass().getDeclaredField("clientLog"); + + field.setAccessible(true); + + return ((IgfsLogger)field.get(fs)).isLogEnabled(); + } +} http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/7d46deb2/modules/hadoop/src/test/java/org/apache/ignite/igfs/IgniteHadoopFileSystemLoopbackAbstractSelfTest.java ---------------------------------------------------------------------- diff --git a/modules/hadoop/src/test/java/org/apache/ignite/igfs/IgniteHadoopFileSystemLoopbackAbstractSelfTest.java b/modules/hadoop/src/test/java/org/apache/ignite/igfs/IgniteHadoopFileSystemLoopbackAbstractSelfTest.java new file mode 100644 index 0000000..e33cde7 --- /dev/null +++ b/modules/hadoop/src/test/java/org/apache/ignite/igfs/IgniteHadoopFileSystemLoopbackAbstractSelfTest.java @@ -0,0 +1,46 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ignite.igfs; + +import java.util.*; + +import static org.apache.ignite.internal.util.ipc.shmem.IpcSharedMemoryServerEndpoint.*; + +/** + * IGFS Hadoop file system IPC loopback self test. + */ +public abstract class IgniteHadoopFileSystemLoopbackAbstractSelfTest extends + IgniteHadoopFileSystemAbstractSelfTest { + /** + * Constructor. + * + * @param mode IGFS mode. + * @param skipEmbed Skip embedded mode flag. + */ + protected IgniteHadoopFileSystemLoopbackAbstractSelfTest(IgfsMode mode, boolean skipEmbed) { + super(mode, skipEmbed, true); + } + + /** {@inheritDoc} */ + @Override protected Map<String, String> primaryIpcEndpointConfiguration(final String gridName) { + return new HashMap<String, String>() {{ + put("type", "tcp"); + put("port", String.valueOf(DFLT_IPC_PORT + getTestGridIndex(gridName))); + }}; + } +} http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/7d46deb2/modules/hadoop/src/test/java/org/apache/ignite/igfs/IgniteHadoopFileSystemLoopbackEmbeddedDualAsyncSelfTest.java ---------------------------------------------------------------------- diff --git a/modules/hadoop/src/test/java/org/apache/ignite/igfs/IgniteHadoopFileSystemLoopbackEmbeddedDualAsyncSelfTest.java b/modules/hadoop/src/test/java/org/apache/ignite/igfs/IgniteHadoopFileSystemLoopbackEmbeddedDualAsyncSelfTest.java new file mode 100644 index 0000000..d46001d --- /dev/null +++ b/modules/hadoop/src/test/java/org/apache/ignite/igfs/IgniteHadoopFileSystemLoopbackEmbeddedDualAsyncSelfTest.java @@ -0,0 +1,33 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ignite.igfs; + +import static org.apache.ignite.igfs.IgfsMode.*; + +/** + * IGFS Hadoop file system IPC loopback self test in DUAL_ASYNC mode. + */ +public class IgniteHadoopFileSystemLoopbackEmbeddedDualAsyncSelfTest extends + IgniteHadoopFileSystemLoopbackAbstractSelfTest { + /** + * Constructor. + */ + public IgniteHadoopFileSystemLoopbackEmbeddedDualAsyncSelfTest() { + super(DUAL_ASYNC, false); + } +} http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/7d46deb2/modules/hadoop/src/test/java/org/apache/ignite/igfs/IgniteHadoopFileSystemLoopbackEmbeddedDualSyncSelfTest.java ---------------------------------------------------------------------- diff --git a/modules/hadoop/src/test/java/org/apache/ignite/igfs/IgniteHadoopFileSystemLoopbackEmbeddedDualSyncSelfTest.java b/modules/hadoop/src/test/java/org/apache/ignite/igfs/IgniteHadoopFileSystemLoopbackEmbeddedDualSyncSelfTest.java new file mode 100644 index 0000000..fa64734 --- /dev/null +++ b/modules/hadoop/src/test/java/org/apache/ignite/igfs/IgniteHadoopFileSystemLoopbackEmbeddedDualSyncSelfTest.java @@ -0,0 +1,33 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ignite.igfs; + +import static org.apache.ignite.igfs.IgfsMode.*; + +/** + * IGFS Hadoop file system IPC loopback self test in DUAL_SYNC mode. + */ +public class IgniteHadoopFileSystemLoopbackEmbeddedDualSyncSelfTest + extends IgniteHadoopFileSystemLoopbackAbstractSelfTest { + /** + * Constructor. + */ + public IgniteHadoopFileSystemLoopbackEmbeddedDualSyncSelfTest() { + super(DUAL_SYNC, false); + } +} http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/7d46deb2/modules/hadoop/src/test/java/org/apache/ignite/igfs/IgniteHadoopFileSystemLoopbackEmbeddedPrimarySelfTest.java ---------------------------------------------------------------------- diff --git a/modules/hadoop/src/test/java/org/apache/ignite/igfs/IgniteHadoopFileSystemLoopbackEmbeddedPrimarySelfTest.java b/modules/hadoop/src/test/java/org/apache/ignite/igfs/IgniteHadoopFileSystemLoopbackEmbeddedPrimarySelfTest.java new file mode 100644 index 0000000..9035acd --- /dev/null +++ b/modules/hadoop/src/test/java/org/apache/ignite/igfs/IgniteHadoopFileSystemLoopbackEmbeddedPrimarySelfTest.java @@ -0,0 +1,33 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ignite.igfs; + +import static org.apache.ignite.igfs.IgfsMode.*; + +/** + * IGFS Hadoop file system IPC loopback self test in PRIMARY mode. + */ +public class IgniteHadoopFileSystemLoopbackEmbeddedPrimarySelfTest + extends IgniteHadoopFileSystemLoopbackAbstractSelfTest { + /** + * Constructor. + */ + public IgniteHadoopFileSystemLoopbackEmbeddedPrimarySelfTest() { + super(PRIMARY, false); + } +} http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/7d46deb2/modules/hadoop/src/test/java/org/apache/ignite/igfs/IgniteHadoopFileSystemLoopbackEmbeddedSecondarySelfTest.java ---------------------------------------------------------------------- diff --git a/modules/hadoop/src/test/java/org/apache/ignite/igfs/IgniteHadoopFileSystemLoopbackEmbeddedSecondarySelfTest.java b/modules/hadoop/src/test/java/org/apache/ignite/igfs/IgniteHadoopFileSystemLoopbackEmbeddedSecondarySelfTest.java new file mode 100644 index 0000000..8198c52 --- /dev/null +++ b/modules/hadoop/src/test/java/org/apache/ignite/igfs/IgniteHadoopFileSystemLoopbackEmbeddedSecondarySelfTest.java @@ -0,0 +1,34 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ignite.igfs; + +import static org.apache.ignite.igfs.IgfsMode.*; + +/** + * IGFS Hadoop file system IPC loopback self test in SECONDARY mode. + */ +public class IgniteHadoopFileSystemLoopbackEmbeddedSecondarySelfTest extends + IgniteHadoopFileSystemLoopbackAbstractSelfTest { + + /** + * Constructor. + */ + public IgniteHadoopFileSystemLoopbackEmbeddedSecondarySelfTest() { + super(PROXY, false); + } +} http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/7d46deb2/modules/hadoop/src/test/java/org/apache/ignite/igfs/IgniteHadoopFileSystemLoopbackExternalDualAsyncSelfTest.java ---------------------------------------------------------------------- diff --git a/modules/hadoop/src/test/java/org/apache/ignite/igfs/IgniteHadoopFileSystemLoopbackExternalDualAsyncSelfTest.java b/modules/hadoop/src/test/java/org/apache/ignite/igfs/IgniteHadoopFileSystemLoopbackExternalDualAsyncSelfTest.java new file mode 100644 index 0000000..246c516 --- /dev/null +++ b/modules/hadoop/src/test/java/org/apache/ignite/igfs/IgniteHadoopFileSystemLoopbackExternalDualAsyncSelfTest.java @@ -0,0 +1,33 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ignite.igfs; + +import static org.apache.ignite.igfs.IgfsMode.*; + +/** + * IGFS Hadoop file system IPC loopback self test in DUAL_ASYNC mode. + */ +public class IgniteHadoopFileSystemLoopbackExternalDualAsyncSelfTest extends + IgniteHadoopFileSystemLoopbackAbstractSelfTest { + /** + * Constructor. + */ + public IgniteHadoopFileSystemLoopbackExternalDualAsyncSelfTest() { + super(DUAL_ASYNC, true); + } +} http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/7d46deb2/modules/hadoop/src/test/java/org/apache/ignite/igfs/IgniteHadoopFileSystemLoopbackExternalDualSyncSelfTest.java ---------------------------------------------------------------------- diff --git a/modules/hadoop/src/test/java/org/apache/ignite/igfs/IgniteHadoopFileSystemLoopbackExternalDualSyncSelfTest.java b/modules/hadoop/src/test/java/org/apache/ignite/igfs/IgniteHadoopFileSystemLoopbackExternalDualSyncSelfTest.java new file mode 100644 index 0000000..b5140af --- /dev/null +++ b/modules/hadoop/src/test/java/org/apache/ignite/igfs/IgniteHadoopFileSystemLoopbackExternalDualSyncSelfTest.java @@ -0,0 +1,33 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ignite.igfs; + +import static org.apache.ignite.igfs.IgfsMode.*; + +/** + * IGFS Hadoop file system IPC loopback self test in DUAL_SYNC mode. + */ +public class IgniteHadoopFileSystemLoopbackExternalDualSyncSelfTest + extends IgniteHadoopFileSystemLoopbackAbstractSelfTest { + /** + * Constructor. + */ + public IgniteHadoopFileSystemLoopbackExternalDualSyncSelfTest() { + super(DUAL_SYNC, true); + } +} http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/7d46deb2/modules/hadoop/src/test/java/org/apache/ignite/igfs/IgniteHadoopFileSystemLoopbackExternalPrimarySelfTest.java ---------------------------------------------------------------------- diff --git a/modules/hadoop/src/test/java/org/apache/ignite/igfs/IgniteHadoopFileSystemLoopbackExternalPrimarySelfTest.java b/modules/hadoop/src/test/java/org/apache/ignite/igfs/IgniteHadoopFileSystemLoopbackExternalPrimarySelfTest.java new file mode 100644 index 0000000..572ac4b --- /dev/null +++ b/modules/hadoop/src/test/java/org/apache/ignite/igfs/IgniteHadoopFileSystemLoopbackExternalPrimarySelfTest.java @@ -0,0 +1,33 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ignite.igfs; + +import static org.apache.ignite.igfs.IgfsMode.*; + +/** + * IGFS Hadoop file system IPC loopback self test in PRIMARY mode. + */ +public class IgniteHadoopFileSystemLoopbackExternalPrimarySelfTest + extends IgniteHadoopFileSystemLoopbackAbstractSelfTest { + /** + * Constructor. + */ + public IgniteHadoopFileSystemLoopbackExternalPrimarySelfTest() { + super(PRIMARY, true); + } +} http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/7d46deb2/modules/hadoop/src/test/java/org/apache/ignite/igfs/IgniteHadoopFileSystemLoopbackExternalSecondarySelfTest.java ---------------------------------------------------------------------- diff --git a/modules/hadoop/src/test/java/org/apache/ignite/igfs/IgniteHadoopFileSystemLoopbackExternalSecondarySelfTest.java b/modules/hadoop/src/test/java/org/apache/ignite/igfs/IgniteHadoopFileSystemLoopbackExternalSecondarySelfTest.java new file mode 100644 index 0000000..006c271 --- /dev/null +++ b/modules/hadoop/src/test/java/org/apache/ignite/igfs/IgniteHadoopFileSystemLoopbackExternalSecondarySelfTest.java @@ -0,0 +1,34 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ignite.igfs; + +import static org.apache.ignite.igfs.IgfsMode.*; + +/** + * IGFS Hadoop file system IPC loopback self test in SECONDARY mode. + */ +public class IgniteHadoopFileSystemLoopbackExternalSecondarySelfTest extends + IgniteHadoopFileSystemLoopbackAbstractSelfTest { + + /** + * Constructor. + */ + public IgniteHadoopFileSystemLoopbackExternalSecondarySelfTest() { + super(PROXY, true); + } +} http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/7d46deb2/modules/hadoop/src/test/java/org/apache/ignite/igfs/IgniteHadoopFileSystemSecondaryModeSelfTest.java ---------------------------------------------------------------------- diff --git a/modules/hadoop/src/test/java/org/apache/ignite/igfs/IgniteHadoopFileSystemSecondaryModeSelfTest.java b/modules/hadoop/src/test/java/org/apache/ignite/igfs/IgniteHadoopFileSystemSecondaryModeSelfTest.java new file mode 100644 index 0000000..e672f4d --- /dev/null +++ b/modules/hadoop/src/test/java/org/apache/ignite/igfs/IgniteHadoopFileSystemSecondaryModeSelfTest.java @@ -0,0 +1,319 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ignite.igfs; + +import org.apache.hadoop.conf.*; +import org.apache.hadoop.fs.*; +import org.apache.ignite.cache.*; +import org.apache.ignite.configuration.*; +import org.apache.ignite.hadoop.fs.*; +import org.apache.ignite.hadoop.fs.v1.*; +import org.apache.ignite.internal.processors.igfs.*; +import org.apache.ignite.internal.util.typedef.*; +import org.apache.ignite.internal.util.typedef.internal.*; +import org.apache.ignite.lang.*; +import org.apache.ignite.spi.discovery.tcp.*; +import org.apache.ignite.spi.discovery.tcp.ipfinder.vm.*; + +import java.net.*; +import java.util.*; + +import static org.apache.ignite.cache.CacheAtomicityMode.*; +import static org.apache.ignite.cache.CacheMode.*; +import static org.apache.ignite.igfs.IgfsMode.*; + +/** + * Ensures correct modes resolution for SECONDARY paths. + */ +public class IgniteHadoopFileSystemSecondaryModeSelfTest extends IgfsCommonAbstractTest { + /** Path to check. */ + private static final Path PATH = new Path("/dir"); + + /** Pattern matching the path. */ + private static final String PATTERN_MATCHES = "/dir"; + + /** Pattern doesn't matching the path. */ + private static final String PATTERN_NOT_MATCHES = "/files"; + + /** Default IGFS mode. */ + private IgfsMode mode; + + /** Path modes. */ + private Map<String, IgfsMode> pathModes; + + /** File system. */ + private IgniteHadoopFileSystem fs; + + /** {@inheritDoc} */ + @Override protected void beforeTest() throws Exception { + mode = null; + pathModes = null; + } + + /** {@inheritDoc} */ + @Override protected void afterTest() throws Exception { + U.closeQuiet(fs); + + fs = null; + + G.stopAll(true); + } + + /** + * Perform initial startup. + * + * @throws Exception If failed. + */ + @SuppressWarnings("NullableProblems") + private void startUp() throws Exception { + startUpSecondary(); + + IgfsConfiguration igfsCfg = new IgfsConfiguration(); + + igfsCfg.setDataCacheName("partitioned"); + igfsCfg.setMetaCacheName("replicated"); + igfsCfg.setName("igfs"); + igfsCfg.setBlockSize(512 * 1024); + igfsCfg.setDefaultMode(mode); + igfsCfg.setPathModes(pathModes); + igfsCfg.setIpcEndpointConfiguration(new HashMap<String, String>() {{ + put("type", "tcp"); + put("port", "10500"); + }}); + + igfsCfg.setManagementPort(-1); + igfsCfg.setSecondaryFileSystem(new IgniteHadoopSecondaryFileSystem( + "igfs://igfs-secondary:igfs-grid-secondary@127.0.0.1:11500/", + "modules/core/src/test/config/hadoop/core-site-loopback-secondary.xml")); + + CacheConfiguration cacheCfg = defaultCacheConfiguration(); + + cacheCfg.setName("partitioned"); + cacheCfg.setCacheMode(PARTITIONED); + cacheCfg.setDistributionMode(CacheDistributionMode.PARTITIONED_ONLY); + cacheCfg.setWriteSynchronizationMode(CacheWriteSynchronizationMode.FULL_SYNC); + cacheCfg.setAffinityMapper(new IgfsGroupDataBlocksKeyMapper(128)); + cacheCfg.setBackups(0); + cacheCfg.setQueryIndexEnabled(false); + cacheCfg.setAtomicityMode(TRANSACTIONAL); + + CacheConfiguration metaCacheCfg = defaultCacheConfiguration(); + + metaCacheCfg.setName("replicated"); + metaCacheCfg.setCacheMode(REPLICATED); + metaCacheCfg.setWriteSynchronizationMode(CacheWriteSynchronizationMode.FULL_SYNC); + metaCacheCfg.setQueryIndexEnabled(false); + metaCacheCfg.setAtomicityMode(TRANSACTIONAL); + + IgniteConfiguration cfg = new IgniteConfiguration(); + + cfg.setGridName("igfs-grid"); + + TcpDiscoverySpi discoSpi = new TcpDiscoverySpi(); + + discoSpi.setIpFinder(new TcpDiscoveryVmIpFinder(true)); + + cfg.setDiscoverySpi(discoSpi); + cfg.setCacheConfiguration(metaCacheCfg, cacheCfg); + cfg.setIgfsConfiguration(igfsCfg); + + cfg.setLocalHost("127.0.0.1"); + + G.start(cfg); + + Configuration fsCfg = new Configuration(); + + fsCfg.addResource(U.resolveIgniteUrl("modules/core/src/test/config/hadoop/core-site-loopback.xml")); + + fsCfg.setBoolean("fs.igfs.impl.disable.cache", true); + + fs = (IgniteHadoopFileSystem)FileSystem.get(new URI("igfs://igfs:igfs-grid@/"), fsCfg); + } + + /** + * Startup secondary file system. + * + * @throws Exception If failed. + */ + private void startUpSecondary() throws Exception { + IgfsConfiguration igfsCfg = new IgfsConfiguration(); + + igfsCfg.setDataCacheName("partitioned"); + igfsCfg.setMetaCacheName("replicated"); + igfsCfg.setName("igfs-secondary"); + igfsCfg.setBlockSize(512 * 1024); + igfsCfg.setDefaultMode(PRIMARY); + igfsCfg.setIpcEndpointConfiguration(new HashMap<String, String>() {{ + put("type", "tcp"); + put("port", "11500"); + }}); + + CacheConfiguration cacheCfg = defaultCacheConfiguration(); + + cacheCfg.setName("partitioned"); + cacheCfg.setCacheMode(PARTITIONED); + cacheCfg.setDistributionMode(CacheDistributionMode.PARTITIONED_ONLY); + cacheCfg.setWriteSynchronizationMode(CacheWriteSynchronizationMode.FULL_SYNC); + cacheCfg.setAffinityMapper(new IgfsGroupDataBlocksKeyMapper(128)); + cacheCfg.setBackups(0); + cacheCfg.setQueryIndexEnabled(false); + cacheCfg.setAtomicityMode(TRANSACTIONAL); + + CacheConfiguration metaCacheCfg = defaultCacheConfiguration(); + + metaCacheCfg.setName("replicated"); + metaCacheCfg.setCacheMode(REPLICATED); + metaCacheCfg.setWriteSynchronizationMode(CacheWriteSynchronizationMode.FULL_SYNC); + metaCacheCfg.setQueryIndexEnabled(false); + metaCacheCfg.setAtomicityMode(TRANSACTIONAL); + + IgniteConfiguration cfg = new IgniteConfiguration(); + + cfg.setGridName("igfs-grid-secondary"); + + TcpDiscoverySpi discoSpi = new TcpDiscoverySpi(); + + discoSpi.setIpFinder(new TcpDiscoveryVmIpFinder(true)); + + cfg.setDiscoverySpi(discoSpi); + cfg.setCacheConfiguration(metaCacheCfg, cacheCfg); + cfg.setIgfsConfiguration(igfsCfg); + + cfg.setLocalHost("127.0.0.1"); + + G.start(cfg); + } + + /** + * Check path resolution when secondary mode is not default and there are no other exclusion paths. + * + * @throws Exception If failed. + */ + public void testSecondaryNotDefaultNoExclusions() throws Exception { + mode = PRIMARY; + + startUp(); + + assert !secondary(PATH); + assert !secondary(PATH); + } + + /** + * Check path resolution when secondary mode is not default and there is no matching exclusion paths. + * + * @throws Exception If failed. + */ + public void testSecondaryNotDefaultNonMatchingExclusion() throws Exception { + mode = PRIMARY; + + pathModes(F.t(PATTERN_NOT_MATCHES, PROXY)); + + startUp(); + + assert !secondary(PATH); + assert !secondary(PATH); + } + + /** + * Check path resolution when secondary mode is not default and there is matching exclusion path. + * + * @throws Exception If failed. + */ + public void testSecondaryNotDefaultMatchingExclusion() throws Exception { + mode = PRIMARY; + + pathModes(F.t(PATTERN_NOT_MATCHES, PROXY), F.t(PATTERN_MATCHES, PROXY)); + + startUp(); + + assert secondary(PATH); + assert secondary(PATH); + } + + /** + * Check path resolution when secondary mode is default and there is no exclusion paths. + * + * @throws Exception If failed. + */ + public void testSecondaryDefaultNoExclusions() throws Exception { + mode = PROXY; + + startUp(); + + assert secondary(PATH); + assert secondary(PATH); + } + + /** + * Check path resolution when secondary mode is default and there is no matching exclusion paths. + * + * @throws Exception If failed. + */ + public void testSecondaryDefaultNonMatchingExclusion() throws Exception { + mode = PROXY; + + pathModes(F.t(PATTERN_NOT_MATCHES, PRIMARY)); + + startUp(); + + assert secondary(PATH); + assert secondary(PATH); + } + + /** + * Check path resolution when secondary mode is default and there is no matching exclusion paths. + * + * @throws Exception If failed. + */ + public void testSecondaryDefaultMatchingExclusion() throws Exception { + mode = PROXY; + + pathModes(F.t(PATTERN_NOT_MATCHES, PRIMARY), F.t(PATTERN_MATCHES, PRIMARY)); + + startUp(); + + assert !secondary(PATH); + assert !secondary(PATH); + } + + /** + * Set IGFS modes for particular paths. + * + * @param modes Modes. + */ + @SafeVarargs + final void pathModes(IgniteBiTuple<String, IgfsMode>... modes) { + assert modes != null; + + pathModes = new LinkedHashMap<>(modes.length, 1.0f); + + for (IgniteBiTuple<String, IgfsMode> mode : modes) + pathModes.put(mode.getKey(), mode.getValue()); + } + + /** + * Check whether the given path is threaten as SECONDARY in the file system. + * + * @param path Path to check. + * @return {@code True} in case path is secondary. + * @throws Exception If failed. + */ + private boolean secondary(Path path) throws Exception { + return fs.mode(path) == PROXY; + } +} http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/7d46deb2/modules/hadoop/src/test/java/org/apache/ignite/igfs/IgniteHadoopFileSystemShmemAbstractSelfTest.java ---------------------------------------------------------------------- diff --git a/modules/hadoop/src/test/java/org/apache/ignite/igfs/IgniteHadoopFileSystemShmemAbstractSelfTest.java b/modules/hadoop/src/test/java/org/apache/ignite/igfs/IgniteHadoopFileSystemShmemAbstractSelfTest.java new file mode 100644 index 0000000..991045b --- /dev/null +++ b/modules/hadoop/src/test/java/org/apache/ignite/igfs/IgniteHadoopFileSystemShmemAbstractSelfTest.java @@ -0,0 +1,88 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ignite.igfs; + +import org.apache.ignite.*; +import org.apache.ignite.internal.util.ipc.*; +import org.apache.ignite.internal.util.typedef.*; +import org.apache.ignite.testframework.*; + +import java.util.*; +import java.util.concurrent.*; + +import static org.apache.ignite.internal.util.ipc.shmem.IpcSharedMemoryServerEndpoint.*; + +/** + * IGFS Hadoop file system IPC self test. + */ +public abstract class IgniteHadoopFileSystemShmemAbstractSelfTest extends IgniteHadoopFileSystemAbstractSelfTest { + /** + * Constructor. + * + * @param mode IGFS mode. + * @param skipEmbed Skip embedded mode flag. + */ + protected IgniteHadoopFileSystemShmemAbstractSelfTest(IgfsMode mode, boolean skipEmbed) { + super(mode, skipEmbed, false); + } + + /** {@inheritDoc} */ + @Override protected Map<String, String> primaryIpcEndpointConfiguration(final String gridName) { + return new HashMap<String, String>() {{ + put("type", "shmem"); + put("port", String.valueOf(DFLT_IPC_PORT + getTestGridIndex(gridName))); + }}; + } + + /** + * Checks correct behaviour in case when we run out of system + * resources. + * + * @throws Exception If error occurred. + */ + @SuppressWarnings("ThrowableResultOfMethodCallIgnored") + public void testOutOfResources() throws Exception { + final Collection<IpcEndpoint> eps = new LinkedList<>(); + + try { + IgniteCheckedException e = (IgniteCheckedException)GridTestUtils.assertThrows(log, new Callable<Object>() { + @SuppressWarnings("InfiniteLoopStatement") + @Override public Object call() throws Exception { + while (true) { + IpcEndpoint ep = IpcEndpointFactory.connectEndpoint("shmem:10500", log); + + eps.add(ep); + } + } + }, IgniteCheckedException.class, null); + + assertNotNull(e); + + String msg = e.getMessage(); + + assertTrue("Invalid exception: " + X.getFullStackTrace(e), + msg.contains("(error code: 28)") || + msg.contains("(error code: 24)") || + msg.contains("(error code: 12)")); + } + finally { + for (IpcEndpoint ep : eps) + ep.close(); + } + } +} http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/7d46deb2/modules/hadoop/src/test/java/org/apache/ignite/igfs/IgniteHadoopFileSystemShmemEmbeddedDualAsyncSelfTest.java ---------------------------------------------------------------------- diff --git a/modules/hadoop/src/test/java/org/apache/ignite/igfs/IgniteHadoopFileSystemShmemEmbeddedDualAsyncSelfTest.java b/modules/hadoop/src/test/java/org/apache/ignite/igfs/IgniteHadoopFileSystemShmemEmbeddedDualAsyncSelfTest.java new file mode 100644 index 0000000..ed34398 --- /dev/null +++ b/modules/hadoop/src/test/java/org/apache/ignite/igfs/IgniteHadoopFileSystemShmemEmbeddedDualAsyncSelfTest.java @@ -0,0 +1,33 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ignite.igfs; + +import static org.apache.ignite.igfs.IgfsMode.*; + +/** + * IGFS Hadoop file system IPC shmem self test in DUAL_ASYNC mode. + */ +public class IgniteHadoopFileSystemShmemEmbeddedDualAsyncSelfTest + extends IgniteHadoopFileSystemShmemAbstractSelfTest { + /** + * Constructor. + */ + public IgniteHadoopFileSystemShmemEmbeddedDualAsyncSelfTest() { + super(DUAL_ASYNC, false); + } +} http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/7d46deb2/modules/hadoop/src/test/java/org/apache/ignite/igfs/IgniteHadoopFileSystemShmemEmbeddedDualSyncSelfTest.java ---------------------------------------------------------------------- diff --git a/modules/hadoop/src/test/java/org/apache/ignite/igfs/IgniteHadoopFileSystemShmemEmbeddedDualSyncSelfTest.java b/modules/hadoop/src/test/java/org/apache/ignite/igfs/IgniteHadoopFileSystemShmemEmbeddedDualSyncSelfTest.java new file mode 100644 index 0000000..4b7bfb1 --- /dev/null +++ b/modules/hadoop/src/test/java/org/apache/ignite/igfs/IgniteHadoopFileSystemShmemEmbeddedDualSyncSelfTest.java @@ -0,0 +1,33 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ignite.igfs; + +import static org.apache.ignite.igfs.IgfsMode.*; + +/** + * IGFS Hadoop file system IPC shmem self test in DUAL_SYNC mode. + */ +public class IgniteHadoopFileSystemShmemEmbeddedDualSyncSelfTest + extends IgniteHadoopFileSystemShmemAbstractSelfTest { + /** + * Constructor. + */ + public IgniteHadoopFileSystemShmemEmbeddedDualSyncSelfTest() { + super(DUAL_SYNC, false); + } +} http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/7d46deb2/modules/hadoop/src/test/java/org/apache/ignite/igfs/IgniteHadoopFileSystemShmemEmbeddedPrimarySelfTest.java ---------------------------------------------------------------------- diff --git a/modules/hadoop/src/test/java/org/apache/ignite/igfs/IgniteHadoopFileSystemShmemEmbeddedPrimarySelfTest.java b/modules/hadoop/src/test/java/org/apache/ignite/igfs/IgniteHadoopFileSystemShmemEmbeddedPrimarySelfTest.java new file mode 100644 index 0000000..c1393d3 --- /dev/null +++ b/modules/hadoop/src/test/java/org/apache/ignite/igfs/IgniteHadoopFileSystemShmemEmbeddedPrimarySelfTest.java @@ -0,0 +1,33 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ignite.igfs; + +import static org.apache.ignite.igfs.IgfsMode.*; + +/** + * IGFS Hadoop file system IPC shmem self test in PRIMARY mode. + */ +public class IgniteHadoopFileSystemShmemEmbeddedPrimarySelfTest + extends IgniteHadoopFileSystemShmemAbstractSelfTest { + /** + * Constructor. + */ + public IgniteHadoopFileSystemShmemEmbeddedPrimarySelfTest() { + super(PRIMARY, false); + } +} http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/7d46deb2/modules/hadoop/src/test/java/org/apache/ignite/igfs/IgniteHadoopFileSystemShmemEmbeddedSecondarySelfTest.java ---------------------------------------------------------------------- diff --git a/modules/hadoop/src/test/java/org/apache/ignite/igfs/IgniteHadoopFileSystemShmemEmbeddedSecondarySelfTest.java b/modules/hadoop/src/test/java/org/apache/ignite/igfs/IgniteHadoopFileSystemShmemEmbeddedSecondarySelfTest.java new file mode 100644 index 0000000..2c97e7b --- /dev/null +++ b/modules/hadoop/src/test/java/org/apache/ignite/igfs/IgniteHadoopFileSystemShmemEmbeddedSecondarySelfTest.java @@ -0,0 +1,33 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ignite.igfs; + +import static org.apache.ignite.igfs.IgfsMode.*; + +/** + * IGFS Hadoop file system IPC shmem self test in SECONDARY mode. + */ +public class IgniteHadoopFileSystemShmemEmbeddedSecondarySelfTest + extends IgniteHadoopFileSystemShmemAbstractSelfTest { + /** + * Constructor. + */ + public IgniteHadoopFileSystemShmemEmbeddedSecondarySelfTest() { + super(PROXY, false); + } +} http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/7d46deb2/modules/hadoop/src/test/java/org/apache/ignite/igfs/IgniteHadoopFileSystemShmemExternalDualAsyncSelfTest.java ---------------------------------------------------------------------- diff --git a/modules/hadoop/src/test/java/org/apache/ignite/igfs/IgniteHadoopFileSystemShmemExternalDualAsyncSelfTest.java b/modules/hadoop/src/test/java/org/apache/ignite/igfs/IgniteHadoopFileSystemShmemExternalDualAsyncSelfTest.java new file mode 100644 index 0000000..6d9973f --- /dev/null +++ b/modules/hadoop/src/test/java/org/apache/ignite/igfs/IgniteHadoopFileSystemShmemExternalDualAsyncSelfTest.java @@ -0,0 +1,33 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ignite.igfs; + +import static org.apache.ignite.igfs.IgfsMode.*; + +/** + * IGFS Hadoop file system IPC shmem self test in DUAL_ASYNC mode. + */ +public class IgniteHadoopFileSystemShmemExternalDualAsyncSelfTest + extends IgniteHadoopFileSystemShmemAbstractSelfTest { + /** + * Constructor. + */ + public IgniteHadoopFileSystemShmemExternalDualAsyncSelfTest() { + super(DUAL_ASYNC, true); + } +} http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/7d46deb2/modules/hadoop/src/test/java/org/apache/ignite/igfs/IgniteHadoopFileSystemShmemExternalDualSyncSelfTest.java ---------------------------------------------------------------------- diff --git a/modules/hadoop/src/test/java/org/apache/ignite/igfs/IgniteHadoopFileSystemShmemExternalDualSyncSelfTest.java b/modules/hadoop/src/test/java/org/apache/ignite/igfs/IgniteHadoopFileSystemShmemExternalDualSyncSelfTest.java new file mode 100644 index 0000000..6ec084d --- /dev/null +++ b/modules/hadoop/src/test/java/org/apache/ignite/igfs/IgniteHadoopFileSystemShmemExternalDualSyncSelfTest.java @@ -0,0 +1,33 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ignite.igfs; + +import static org.apache.ignite.igfs.IgfsMode.*; + +/** + * IGFS Hadoop file system IPC shmem self test in DUAL_SYNC mode. + */ +public class IgniteHadoopFileSystemShmemExternalDualSyncSelfTest + extends IgniteHadoopFileSystemShmemAbstractSelfTest { + /** + * Constructor. + */ + public IgniteHadoopFileSystemShmemExternalDualSyncSelfTest() { + super(DUAL_SYNC, true); + } +} http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/7d46deb2/modules/hadoop/src/test/java/org/apache/ignite/igfs/IgniteHadoopFileSystemShmemExternalPrimarySelfTest.java ---------------------------------------------------------------------- diff --git a/modules/hadoop/src/test/java/org/apache/ignite/igfs/IgniteHadoopFileSystemShmemExternalPrimarySelfTest.java b/modules/hadoop/src/test/java/org/apache/ignite/igfs/IgniteHadoopFileSystemShmemExternalPrimarySelfTest.java new file mode 100644 index 0000000..03f3fee --- /dev/null +++ b/modules/hadoop/src/test/java/org/apache/ignite/igfs/IgniteHadoopFileSystemShmemExternalPrimarySelfTest.java @@ -0,0 +1,33 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ignite.igfs; + +import static org.apache.ignite.igfs.IgfsMode.*; + +/** + * IGFS Hadoop file system IPC shmem self test in PRIMARY mode. + */ +public class IgniteHadoopFileSystemShmemExternalPrimarySelfTest + extends IgniteHadoopFileSystemShmemAbstractSelfTest { + /** + * Constructor. + */ + public IgniteHadoopFileSystemShmemExternalPrimarySelfTest() { + super(PRIMARY, true); + } +} http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/7d46deb2/modules/hadoop/src/test/java/org/apache/ignite/igfs/IgniteHadoopFileSystemShmemExternalSecondarySelfTest.java ---------------------------------------------------------------------- diff --git a/modules/hadoop/src/test/java/org/apache/ignite/igfs/IgniteHadoopFileSystemShmemExternalSecondarySelfTest.java b/modules/hadoop/src/test/java/org/apache/ignite/igfs/IgniteHadoopFileSystemShmemExternalSecondarySelfTest.java new file mode 100644 index 0000000..bc27674 --- /dev/null +++ b/modules/hadoop/src/test/java/org/apache/ignite/igfs/IgniteHadoopFileSystemShmemExternalSecondarySelfTest.java @@ -0,0 +1,33 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ignite.igfs; + +import static org.apache.ignite.igfs.IgfsMode.*; + +/** + * IGFS Hadoop file system IPC shmem self test in SECONDARY mode. + */ +public class IgniteHadoopFileSystemShmemExternalSecondarySelfTest + extends IgniteHadoopFileSystemShmemAbstractSelfTest { + /** + * Constructor. + */ + public IgniteHadoopFileSystemShmemExternalSecondarySelfTest() { + super(PROXY, true); + } +}
