http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/cfcf46df/modules/core/src/test/java/org/apache/ignite/internal/processors/fs/GridGgfsCommonAbstractTest.java ---------------------------------------------------------------------- diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/fs/GridGgfsCommonAbstractTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/fs/GridGgfsCommonAbstractTest.java new file mode 100644 index 0000000..9d84e53 --- /dev/null +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/fs/GridGgfsCommonAbstractTest.java @@ -0,0 +1,67 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ignite.internal.processors.fs; + +import org.apache.ignite.configuration.*; +import org.gridgain.testframework.junits.*; +import org.gridgain.testframework.junits.common.*; + +/** + * Common subclass for all GGFS tests. Aimed to disabled peer class loading which is restricted for Hadoop edition. + */ +public class GridGgfsCommonAbstractTest extends GridCommonAbstractTest { + /** {@inheritDoc} */ + @Override protected IgniteConfiguration getConfiguration() throws Exception { + IgniteConfiguration cfg = super.getConfiguration(); + + cfg.setPeerClassLoadingEnabled(false); + cfg.setLocalHost("127.0.0.1"); + + return cfg; + } + + /** {@inheritDoc} */ + @Override protected IgniteConfiguration getConfiguration(String gridName) throws Exception { + IgniteConfiguration cfg = super.getConfiguration(gridName); + + cfg.setPeerClassLoadingEnabled(false); + cfg.setLocalHost("127.0.0.1"); + + return cfg; + } + + /** {@inheritDoc} */ + @Override protected IgniteConfiguration getConfiguration(GridTestResources rsrcs) throws Exception { + IgniteConfiguration cfg = super.getConfiguration(rsrcs); + + cfg.setPeerClassLoadingEnabled(false); + cfg.setLocalHost("127.0.0.1"); + + return cfg; + } + + /** {@inheritDoc} */ + @Override protected IgniteConfiguration getConfiguration(String gridName, GridTestResources rsrcs) throws Exception { + IgniteConfiguration cfg = super.getConfiguration(gridName, rsrcs); + + cfg.setPeerClassLoadingEnabled(false); + cfg.setLocalHost("127.0.0.1"); + + return cfg; + } +}
http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/cfcf46df/modules/core/src/test/java/org/apache/ignite/internal/processors/fs/GridGgfsDataManagerSelfTest.java ---------------------------------------------------------------------- diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/fs/GridGgfsDataManagerSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/fs/GridGgfsDataManagerSelfTest.java new file mode 100644 index 0000000..b71b523 --- /dev/null +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/fs/GridGgfsDataManagerSelfTest.java @@ -0,0 +1,599 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ignite.internal.processors.fs; + +import org.apache.ignite.cache.*; +import org.apache.ignite.cluster.*; +import org.apache.ignite.configuration.*; +import org.apache.ignite.fs.*; +import org.apache.ignite.lang.*; +import org.gridgain.grid.kernal.processors.cache.*; +import org.apache.ignite.spi.discovery.tcp.*; +import org.apache.ignite.spi.discovery.tcp.ipfinder.*; +import org.apache.ignite.spi.discovery.tcp.ipfinder.vm.*; +import org.gridgain.grid.kernal.processors.cache.transactions.*; +import org.apache.ignite.internal.util.typedef.*; +import org.apache.ignite.internal.util.typedef.internal.*; +import org.gridgain.testframework.*; +import org.jetbrains.annotations.*; + +import java.nio.*; +import java.security.*; +import java.util.*; +import java.util.concurrent.*; + +import static org.apache.ignite.cache.GridCacheAtomicityMode.*; +import static org.apache.ignite.cache.GridCacheMode.*; +import static org.gridgain.testframework.GridTestUtils.*; + +/** + * {@link GridGgfsDataManager} test case. + */ +public class GridGgfsDataManagerSelfTest extends GridGgfsCommonAbstractTest { + /** Test IP finder. */ + private static final TcpDiscoveryIpFinder IP_FINDER = new TcpDiscoveryVmIpFinder(true); + + /** Meta-information cache name. */ + private static final String META_CACHE_NAME = "meta"; + + /** Data cache name. */ + private static final String DATA_CACHE_NAME = "partitioned"; + + /** Groups count for data blocks. */ + private static final int DATA_BLOCK_GROUP_CNT = 2; + + /** GGFS block size. */ + private static final int BLOCK_SIZE = 32 * 1024; + + /** Test nodes count. */ + private static final int NODES_CNT = 4; + + /** Busy wait sleep interval in milliseconds. */ + private static final int BUSY_WAIT_SLEEP_INTERVAL = 200; + + /** GGFS block size. */ + private static final int GGFS_BLOCK_SIZE = 64 * 1024; + + /** Random numbers generator. */ + private final SecureRandom rnd = new SecureRandom(); + + /** Data manager to test. */ + private GridGgfsDataManager mgr; + + /** {@inheritDoc} */ + @Override protected void beforeTest() throws Exception { + GridGgfsEx ggfs = (GridGgfsEx)grid(0).fileSystem("ggfs"); + + mgr = ggfs.context().data(); + } + + /** {@inheritDoc} */ + @Override protected IgniteConfiguration getConfiguration(String gridName) throws Exception { + IgniteConfiguration cfg = super.getConfiguration(gridName); + + cfg.setCacheConfiguration(cacheConfiguration(META_CACHE_NAME), cacheConfiguration(DATA_CACHE_NAME)); + + TcpDiscoverySpi discoSpi = new TcpDiscoverySpi(); + + discoSpi.setIpFinder(IP_FINDER); + + cfg.setDiscoverySpi(discoSpi); + + IgniteFsConfiguration ggfsCfg = new IgniteFsConfiguration(); + + ggfsCfg.setMetaCacheName(META_CACHE_NAME); + ggfsCfg.setDataCacheName(DATA_CACHE_NAME); + ggfsCfg.setBlockSize(GGFS_BLOCK_SIZE); + ggfsCfg.setName("ggfs"); + ggfsCfg.setBlockSize(BLOCK_SIZE); + + cfg.setGgfsConfiguration(ggfsCfg); + + return cfg; + } + + /** {@inheritDoc} */ + protected CacheConfiguration cacheConfiguration(String cacheName) { + CacheConfiguration cacheCfg = defaultCacheConfiguration(); + + cacheCfg.setName(cacheName); + + if (META_CACHE_NAME.equals(cacheName)) + cacheCfg.setCacheMode(REPLICATED); + else { + cacheCfg.setCacheMode(PARTITIONED); + cacheCfg.setDistributionMode(GridCacheDistributionMode.PARTITIONED_ONLY); + + cacheCfg.setBackups(0); + cacheCfg.setAffinityMapper(new IgniteFsGroupDataBlocksKeyMapper(DATA_BLOCK_GROUP_CNT)); + } + + cacheCfg.setWriteSynchronizationMode(GridCacheWriteSynchronizationMode.FULL_SYNC); + cacheCfg.setAtomicityMode(TRANSACTIONAL); + cacheCfg.setQueryIndexEnabled(false); + + return cacheCfg; + } + + /** {@inheritDoc} */ + @Override protected void afterTest() throws Exception { + for (int i = 0; i < NODES_CNT; i++) { + grid(i).cachex(META_CACHE_NAME).clearAll(); + + grid(i).cachex(DATA_CACHE_NAME).clearAll(); + } + } + + /** {@inheritDoc} */ + @Override protected void beforeTestsStarted() throws Exception { + startGrids(NODES_CNT); + } + + /** {@inheritDoc} */ + @Override protected void afterTestsStopped() throws Exception { + stopAllGrids(); + } + + /** + * Test file system structure in meta-cache. + * + * @throws Exception If failed. + */ + @SuppressWarnings("ConstantConditions") + public void testDataStoring() throws Exception { + for (int i = 0; i < 10; i++) { + IgniteFsPath path = new IgniteFsPath(); + GridGgfsFileInfo info = new GridGgfsFileInfo(200, null, false, null); + + assertNull(mgr.dataBlock(info, path, 0, null).get()); + + byte[] data = new byte[rnd.nextInt(20000) + 5]; + + rnd.nextBytes(data); + + IgniteFuture<Boolean> fut = mgr.writeStart(info); + + expectsStoreFail(info, data, "Not enough space reserved to store data"); + + info = new GridGgfsFileInfo(info, info.length() + data.length - 3); + + expectsStoreFail(info, data, "Not enough space reserved to store data"); + + info = new GridGgfsFileInfo(info, info.length() + 3); + + GridGgfsFileAffinityRange range = new GridGgfsFileAffinityRange(); + + byte[] remainder = mgr.storeDataBlocks(info, info.length(), null, 0, ByteBuffer.wrap(data), true, + range, null); + + assert remainder == null; + + mgr.writeClose(info); + + fut.get(3000); + + for (int j = 0; j < NODES_CNT; j++) { + GridCacheContext<Object, Object> ctx = GridTestUtils.getFieldValue(grid(j).cachex(DATA_CACHE_NAME), + "ctx"); + Collection<IgniteTxEx<Object, Object>> txs = ctx.tm().txs(); + + assert txs.isEmpty() : "Incomplete transactions: " + txs; + } + + // Validate data stored in cache. + for (int pos = 0, block = 0; pos < info.length(); block++) { + byte[] stored = mgr.dataBlock(info, path, block, null).get(); + + assertNotNull("Expects data exist [data.length=" + data.length + ", block=" + block + ']', stored); + + for (int j = 0; j < stored.length; j++) + assertEquals(stored[j], data[pos + j]); + + pos += stored.length; + } + + mgr.delete(info); + + long nIters = getTestTimeout() / BUSY_WAIT_SLEEP_INTERVAL; + + assert nIters < Integer.MAX_VALUE; + + boolean rmvBlocks = false; + + // Wait for all blocks to be removed. + for (int j = 0; j < nIters && !rmvBlocks; j = sleepAndIncrement(BUSY_WAIT_SLEEP_INTERVAL, j)) { + boolean b = true; + + for (long block = 0; block < info.blocksCount(); block++) + b &= mgr.dataBlock(info, path, block, null).get() == null; + + rmvBlocks = b; + } + + assertTrue("All blocks should be removed from cache.", rmvBlocks); + } + } + + /** + * Test file system structure in meta-cache. + * + * @throws Exception If failed. + */ + public void testDataStoringRemainder() throws Exception { + final int blockSize = GGFS_BLOCK_SIZE; + + for (int i = 0; i < 10; i++) { + IgniteFsPath path = new IgniteFsPath(); + GridGgfsFileInfo info = new GridGgfsFileInfo(blockSize, null, false, null); + + assertNull(mgr.dataBlock(info, path, 0, null).get()); + + byte[] data = new byte[blockSize]; + + rnd.nextBytes(data); + + byte[] remainder = new byte[blockSize / 2]; + + rnd.nextBytes(remainder); + + info = new GridGgfsFileInfo(info, info.length() + data.length + remainder.length); + + IgniteFuture<Boolean> fut = mgr.writeStart(info); + + GridGgfsFileAffinityRange range = new GridGgfsFileAffinityRange(); + + byte[] left = mgr.storeDataBlocks(info, info.length(), remainder, remainder.length, ByteBuffer.wrap(data), + false, range, null); + + assert left.length == blockSize / 2; + + byte[] remainder2 = new byte[blockSize / 2]; + + info = new GridGgfsFileInfo(info, info.length() + remainder2.length); + + byte[] left2 = mgr.storeDataBlocks(info, info.length(), left, left.length, ByteBuffer.wrap(remainder2), + false, range, null); + + assert left2 == null; + + mgr.writeClose(info); + + fut.get(3000); + + for (int j = 0; j < NODES_CNT; j++) { + GridCacheContext<Object, Object> ctx = GridTestUtils.getFieldValue(grid(j).cachex(DATA_CACHE_NAME), + "ctx"); + Collection<IgniteTxEx<Object, Object>> txs = ctx.tm().txs(); + + assert txs.isEmpty() : "Incomplete transactions: " + txs; + } + + byte[] concat = U.join(remainder, data, remainder2); + + // Validate data stored in cache. + for (int pos = 0, block = 0; pos < info.length(); block++) { + byte[] stored = mgr.dataBlock(info, path, block, null).get(); + + assertNotNull("Expects data exist [data.length=" + concat.length + ", block=" + block + ']', stored); + + for (int j = 0; j < stored.length; j++) + assertEquals(stored[j], concat[pos + j]); + + pos += stored.length; + } + + mgr.delete(info); + + long nIters = getTestTimeout() / BUSY_WAIT_SLEEP_INTERVAL; + + assert nIters < Integer.MAX_VALUE; + + boolean rmvBlocks = false; + + // Wait for all blocks to be removed. + for (int j = 0; j < nIters && !rmvBlocks; j = sleepAndIncrement(BUSY_WAIT_SLEEP_INTERVAL, j)) { + boolean b = true; + + for (long block = 0; block < info.blocksCount(); block++) + b &= mgr.dataBlock(info, path, block, null).get() == null; + + rmvBlocks = b; + } + + assertTrue("All blocks should be removed from cache.", rmvBlocks); + } + } + + /** @throws Exception If failed. */ + public void testDataStoringFlush() throws Exception { + final int blockSize = GGFS_BLOCK_SIZE; + final int writesCnt = 64; + + for (int i = 0; i < 10; i++) { + IgniteFsPath path = new IgniteFsPath(); + GridGgfsFileInfo info = new GridGgfsFileInfo(blockSize, null, false, null); + + GridGgfsFileAffinityRange range = new GridGgfsFileAffinityRange(); + + assertNull(mgr.dataBlock(info, path, 0, null).get()); + + int chunkSize = blockSize / 4; + + byte[] data = new byte[chunkSize]; + + info = new GridGgfsFileInfo(info, info.length() + data.length * writesCnt); + + IgniteFuture<Boolean> fut = mgr.writeStart(info); + + for (int j = 0; j < 64; j++) { + Arrays.fill(data, (byte)(j / 4)); + + byte[] left = mgr.storeDataBlocks(info, (j + 1) * chunkSize, null, 0, ByteBuffer.wrap(data), + true, range, null); + + assert left == null : "No remainder should be returned if flush is true: " + Arrays.toString(left); + } + + mgr.writeClose(info); + + assertTrue(range.regionEqual(new GridGgfsFileAffinityRange(0, writesCnt * chunkSize - 1, null))); + + fut.get(3000); + + for (int j = 0; j < NODES_CNT; j++) { + GridCacheContext<Object, Object> ctx = GridTestUtils.getFieldValue(grid(j).cachex(DATA_CACHE_NAME), + "ctx"); + Collection<IgniteTxEx<Object, Object>> txs = ctx.tm().txs(); + + assert txs.isEmpty() : "Incomplete transactions: " + txs; + } + + // Validate data stored in cache. + for (int pos = 0, block = 0; pos < info.length(); block++) { + byte[] stored = mgr.dataBlock(info, path, block, null).get(); + + assertNotNull("Expects data exist [block=" + block + ']', stored); + + for (byte b : stored) + assertEquals(b, (byte)block); + + pos += stored.length; + } + + IgniteFuture<Object> delFut = mgr.delete(info); + + delFut.get(); + + for (long block = 0; block < info.blocksCount(); block++) + assertNull(mgr.dataBlock(info, path, block, null).get()); + } + } + + /** + * Test affinity. + * + * @throws Exception If failed. + */ + public void testAffinity() throws Exception { + final int blockSize = 10; + final int grpSize = blockSize * DATA_BLOCK_GROUP_CNT; + + //GridGgfsFileInfo info = new GridGgfsFileInfo(blockSize, 0); + GridGgfsFileInfo info = new GridGgfsFileInfo(blockSize, 1024 * 1024, null, null, false, null); + + for (int pos = 0; pos < 5 * grpSize; pos++) { + assertEquals("Expects no affinity for zero length.", Collections.<IgniteFsBlockLocation>emptyList(), + mgr.affinity(info, pos, 0)); + + // Expects grouped data blocks are interpreted as a single block location. + // And no guaranties for blocks out of the group. + for (int len = 1, maxLen = grpSize - pos % grpSize; len < maxLen; len++) { + Collection<IgniteFsBlockLocation> aff = mgr.affinity(info, pos, len); + + assertEquals("Unexpected affinity: " + aff, 1, aff.size()); + + IgniteFsBlockLocation loc = F.first(aff); + + assertEquals("Unexpected block location: " + loc, pos, loc.start()); + assertEquals("Unexpected block location: " + loc, len, loc.length()); + } + + // Validate ranges. + for (int len = grpSize * 4 + 1, maxLen = 5 * grpSize - pos % grpSize; len < maxLen; len++) { + Collection<IgniteFsBlockLocation> aff = mgr.affinity(info, pos, len); + + assertTrue("Unexpected affinity [aff=" + aff + ", pos=" + pos + ", len=" + len + ']', aff.size() <= 5); + + IgniteFsBlockLocation first = F.first(aff); + + assertEquals("Unexpected the first block location [aff=" + aff + ", pos=" + pos + ", len=" + len + ']', + pos, first.start()); + + assertTrue("Unexpected the first block location [aff=" + aff + ", pos=" + pos + ", len=" + len + ']', + first.length() >= grpSize - pos % grpSize); + + IgniteFsBlockLocation last = F.last(aff); + + assertTrue("Unexpected the last block location [aff=" + aff + ", pos=" + pos + ", len=" + len + ']', + last.start() <= (pos / grpSize + 4) * grpSize); + + assertTrue("Unexpected the last block location [aff=" + aff + ", pos=" + pos + ", len=" + len + ']', + last.length() >= (pos + len - 1) % grpSize + 1); + } + } + } + + /** @throws Exception If failed. */ + public void testAffinity2() throws Exception { + int blockSize = BLOCK_SIZE; + + GridGgfsFileInfo info = new GridGgfsFileInfo(blockSize, 1024 * 1024, null, null, false, null); + + Collection<IgniteFsBlockLocation> affinity = mgr.affinity(info, 0, info.length()); + + for (IgniteFsBlockLocation loc : affinity) { + info("Going to check GGFS block location: " + loc); + + int block = (int)(loc.start() / blockSize); + + int endPos; + + do { + GridGgfsBlockKey key = new GridGgfsBlockKey(info.id(), null, false, block); + + ClusterNode affNode = grid(0).cachex(DATA_CACHE_NAME).affinity().mapKeyToNode(key); + + assertTrue("Failed to find node in affinity [dataMgr=" + loc.nodeIds() + + ", nodeId=" + affNode.id() + ", block=" + block + ']', loc.nodeIds().contains(affNode.id())); + + endPos = (block + 1) * blockSize; + + block++; + } + while (endPos < loc.start() + loc.length()); + } + } + + /** @throws Exception If failed. */ + public void testAffinityFileMap() throws Exception { + int blockSize = BLOCK_SIZE; + + GridGgfsFileInfo info = new GridGgfsFileInfo(blockSize, 1024 * 1024, null, null, false, null); + + IgniteUuid affKey = IgniteUuid.randomUuid(); + + GridGgfsFileMap map = new GridGgfsFileMap(); + + map.addRange(new GridGgfsFileAffinityRange(3 * BLOCK_SIZE, 5 * BLOCK_SIZE - 1, affKey)); + map.addRange(new GridGgfsFileAffinityRange(13 * BLOCK_SIZE, 17 * BLOCK_SIZE - 1, affKey)); + + info.fileMap(map); + + Collection<IgniteFsBlockLocation> affinity = mgr.affinity(info, 0, info.length()); + + checkAffinity(blockSize, info, affinity); + + // Check from middle of range. + affinity = mgr.affinity(info, 3 * BLOCK_SIZE + BLOCK_SIZE / 2, info.length()); + + checkAffinity(blockSize, info, affinity); + + // Check from middle of last range. + affinity = mgr.affinity(info, 14 * BLOCK_SIZE, info.length()); + + checkAffinity(blockSize, info, affinity); + + // Check inside one range. + affinity = mgr.affinity(info, 14 * BLOCK_SIZE, 2 * BLOCK_SIZE); + + checkAffinity(blockSize, info, affinity); + + // Check outside last range. + affinity = mgr.affinity(info, 18 * BLOCK_SIZE, info.length()); + + checkAffinity(blockSize, info, affinity); + } + + /** + * Checks affinity validity. + * + * @param blockSize Block size. + * @param info File info. + * @param affinity Affinity block locations to check. + */ + private void checkAffinity(int blockSize, GridGgfsFileInfo info, Iterable<IgniteFsBlockLocation> affinity) { + GridCache<Object, Object> dataCache = grid(0).cachex(DATA_CACHE_NAME); + + for (IgniteFsBlockLocation loc : affinity) { + info("Going to check GGFS block location: " + loc); + + int block = (int)(loc.start() / blockSize); + + int endPos; + + do { + GridGgfsBlockKey key = new GridGgfsBlockKey(info.id(), + info.fileMap().affinityKey(block * blockSize, false), false, block); + + ClusterNode affNode = dataCache.affinity().mapKeyToNode(key); + + assertTrue("Failed to find node in affinity [dataMgr=" + loc.nodeIds() + + ", nodeId=" + affNode.id() + ", block=" + block + ']', loc.nodeIds().contains(affNode.id())); + + endPos = (block + 1) * blockSize; + + block++; + } + while (endPos < loc.start() + loc.length()); + } + } + + /** + * Test expected failures for 'store' operation. + * + * @param reserved Reserved file info. + * @param data Data to store. + * @param msg Expected failure message. + */ + private void expectsStoreFail(final GridGgfsFileInfo reserved, final byte[] data, @Nullable String msg) { + GridTestUtils.assertThrows(log, new Callable() { + @Override public Object call() throws Exception { + GridGgfsFileAffinityRange range = new GridGgfsFileAffinityRange(); + + mgr.storeDataBlocks(reserved, reserved.length(), null, 0, ByteBuffer.wrap(data), false, range, null); + + return null; + } + }, IgniteFsException.class, msg); + } + + /** + * Test expected failures for 'delete' operation. + * + * @param fileInfo File to delete data for. + * @param msg Expected failure message. + */ + private void expectsDeleteFail(final GridGgfsFileInfo fileInfo, @Nullable String msg) { + GridTestUtils.assertThrows(log, new Callable() { + @Override public Object call() throws Exception { + mgr.delete(fileInfo); + + return null; + } + }, IgniteFsException.class, msg); + } + + /** + * Test expected failures for 'affinity' operation. + * + * @param info File info to resolve affinity nodes for. + * @param start Start position in the file. + * @param len File part length to get affinity for. + * @param msg Expected failure message. + */ + private void expectsAffinityFail(final GridGgfsFileInfo info, final long start, final long len, + @Nullable String msg) { + GridTestUtils.assertThrows(log, new Callable() { + @Override public Object call() throws Exception { + mgr.affinity(info, start, len); + + return null; + } + }, IgniteFsException.class, msg); + } +} http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/cfcf46df/modules/core/src/test/java/org/apache/ignite/internal/processors/fs/GridGgfsDualAbstractSelfTest.java ---------------------------------------------------------------------- diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/fs/GridGgfsDualAbstractSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/fs/GridGgfsDualAbstractSelfTest.java new file mode 100644 index 0000000..96570ce --- /dev/null +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/fs/GridGgfsDualAbstractSelfTest.java @@ -0,0 +1,1601 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ignite.internal.processors.fs; + +import org.apache.ignite.*; +import org.apache.ignite.cache.*; +import org.apache.ignite.fs.*; +import org.apache.ignite.internal.util.lang.*; +import org.apache.ignite.internal.util.typedef.internal.*; +import org.gridgain.testframework.*; + +import java.io.*; +import java.util.*; +import java.util.concurrent.*; + +import static org.apache.ignite.IgniteFs.*; +import static org.apache.ignite.fs.IgniteFsMode.*; + +/** + * Tests for GGFS working in mode when remote file system exists: DUAL_SYNC, DUAL_ASYNC. + */ +public abstract class GridGgfsDualAbstractSelfTest extends GridGgfsAbstractSelfTest { + /** + * Constructor. + * + * @param mode GGFS mode. + */ + protected GridGgfsDualAbstractSelfTest(IgniteFsMode mode) { + super(mode); + + assert mode == DUAL_SYNC || mode == DUAL_ASYNC; + } + + /** + * @throws Exception If failed. + */ + public void testDefaultDirectories() throws Exception { + IgniteFsPath gg = new IgniteFsPath("/gridgain"); + IgniteFsPath[] paths = paths( + gg, new IgniteFsPath(gg, "sync"), new IgniteFsPath(gg, "async"), new IgniteFsPath(gg, "primary")); + + create(ggfs, paths, null); + + for (IgniteFsPath p : paths) + assert ggfs.exists(p); + + assert ggfs.listFiles(gg).size() == 3; + assert ggfs.listPaths(gg).size() == 3; + } + + /** + * Test existence check when the path exists only remotely. + * + * @throws Exception If failed. + */ + public void testExistsPathMissing() throws Exception { + create(ggfsSecondary, paths(DIR), null); + + assert ggfs.exists(DIR); + } + + /** + * Test list files routine when the path doesn't exist locally. + * + * @throws Exception If failed. + */ + public void testListFilesPathMissing() throws Exception { + create(ggfsSecondary, paths(DIR, SUBDIR, SUBSUBDIR), paths(FILE)); + + Collection<IgniteFsFile> paths = ggfs.listFiles(SUBDIR); + + assert paths != null; + assert paths.size() == 2; + + Iterator<IgniteFsFile> iter = paths.iterator(); + + IgniteFsFile path1 = iter.next(); + IgniteFsFile path2 = iter.next(); + + assert (SUBSUBDIR.equals(path1.path()) && FILE.equals(path2.path())) || + (FILE.equals(path1.path()) && SUBSUBDIR.equals(path2.path())); + } + + /** + * Test info routine when the path doesn't exist locally. + * + * @throws Exception If failed. + */ + public void testInfoPathMissing() throws Exception { + create(ggfsSecondary, paths(DIR), null); + create(ggfs, null, null); + + IgniteFsFile info = ggfs.info(DIR); + + assert info != null; + + assertEquals(DIR, info.path()); + } + + /** + * Test rename in case source exists partially and the path being renamed is a file. + * + * @throws Exception If failed. + */ + public void testRenameFileSourceMissingPartially() throws Exception { + create(ggfsSecondary, paths(DIR, SUBDIR), paths(FILE)); + create(ggfs, paths(DIR), null); + + ggfs.rename(FILE, FILE2); + + checkExist(ggfs, SUBDIR); + checkExist(ggfs, ggfsSecondary, FILE2); + checkNotExist(ggfs, ggfsSecondary, FILE); + } + + /** + * Test rename in case source doesn't exist and the path being renamed is a file. + * + * @throws Exception If failed. + */ + public void testRenameFileSourceMissing() throws Exception { + create(ggfsSecondary, paths(DIR, SUBDIR), paths(FILE)); + create(ggfs, null, null); + + ggfs.rename(FILE, FILE2); + + checkExist(ggfs, DIR, SUBDIR); + checkExist(ggfs, ggfsSecondary, FILE2); + checkNotExist(ggfs, ggfsSecondary, FILE); + } + + /** + * Test file rename when parent folder is the root and source is missing. + * + * @throws Exception If failed. + */ + public void testRenameFileParentRootSourceMissing() throws Exception { + IgniteFsPath file1 = new IgniteFsPath("/file1"); + IgniteFsPath file2 = new IgniteFsPath("/file2"); + + create(ggfsSecondary, null, paths(file1)); + create(ggfs, null, null); + + ggfs.rename(file1, file2); + + checkExist(ggfs, ggfsSecondary, file2); + checkNotExist(ggfs, ggfsSecondary, file1); + } + + /** + * Test rename in case source exists partially and the path being renamed is a directory. + * + * @throws Exception If failed. + */ + public void testRenameDirectorySourceMissingPartially() throws Exception { + create(ggfsSecondary, paths(DIR, SUBDIR), null); + create(ggfs, paths(DIR), null); + + ggfs.rename(SUBDIR, SUBDIR2); + + checkExist(ggfs, ggfsSecondary, SUBDIR2); + checkNotExist(ggfs, ggfsSecondary, SUBDIR); + } + + /** + * Test rename in case source doesn't exist and the path being renamed is a directory. + * + * @throws Exception If failed. + */ + public void testRenameDirectorySourceMissing() throws Exception { + create(ggfsSecondary, paths(DIR, SUBDIR), null); + create(ggfs, null, null); + + ggfs.rename(SUBDIR, SUBDIR2); + + checkExist(ggfs, DIR); + checkExist(ggfs, ggfsSecondary, SUBDIR2); + checkNotExist(ggfs, ggfsSecondary, SUBDIR); + } + + /** + * Test directory rename when parent folder is the root and source is missing. + * + * @throws Exception If failed. + */ + public void testRenameDirectoryParentRootSourceMissing() throws Exception { + IgniteFsPath dir1 = new IgniteFsPath("/dir1"); + IgniteFsPath dir2 = new IgniteFsPath("/dir2"); + + create(ggfsSecondary, paths(dir1), null); + create(ggfs, null, null); + + ggfs.rename(dir1, dir2); + + checkExist(ggfs, ggfsSecondary, dir2); + checkNotExist(ggfs, ggfsSecondary, dir1); + } + + /** + * Test move in case source exists partially and the path being renamed is a file. + * + * @throws Exception If failed. + */ + public void testMoveFileSourceMissingPartially() throws Exception { + create(ggfsSecondary, paths(DIR, SUBDIR, DIR_NEW, SUBDIR_NEW), paths(FILE)); + create(ggfs, paths(DIR, DIR_NEW, SUBDIR_NEW), paths(FILE)); + + ggfs.rename(FILE, SUBDIR_NEW); + + checkExist(ggfs, SUBDIR); + checkExist(ggfs, ggfsSecondary, new IgniteFsPath(SUBDIR_NEW, FILE.name())); + checkNotExist(ggfs, ggfsSecondary, FILE); + } + + /** + * Test move in case source doesn't exist and the path being renamed is a file. + * + * @throws Exception If failed. + */ + public void testMoveFileSourceMissing() throws Exception { + create(ggfsSecondary, paths(DIR, SUBDIR, DIR_NEW, SUBDIR_NEW), paths(FILE)); + create(ggfs, paths(DIR_NEW, SUBDIR_NEW), paths(FILE)); + + ggfs.rename(FILE, SUBDIR_NEW); + + checkExist(ggfs, DIR, SUBDIR); + checkExist(ggfs, ggfsSecondary, new IgniteFsPath(SUBDIR_NEW, FILE.name())); + checkNotExist(ggfs, ggfsSecondary, FILE); + } + + /** + * Test move in case destination exists partially and the path being renamed is a file. + * + * @throws Exception If failed. + */ + public void testMoveFileDestinationMissingPartially() throws Exception { + create(ggfsSecondary, paths(DIR, SUBDIR, DIR_NEW, SUBDIR_NEW), paths(FILE)); + create(ggfs, paths(DIR, SUBDIR, DIR_NEW), paths(FILE)); + + ggfs.rename(FILE, SUBDIR_NEW); + + checkExist(ggfs, SUBDIR_NEW); + checkExist(ggfs, ggfsSecondary, new IgniteFsPath(SUBDIR_NEW, FILE.name())); + checkNotExist(ggfs, ggfsSecondary, FILE); + } + + /** + * Test move in case destination doesn't exist and the path being renamed is a file. + * + * @throws Exception If failed. + */ + public void testMoveFileDestinationMissing() throws Exception { + create(ggfsSecondary, paths(DIR, SUBDIR, DIR_NEW, SUBDIR_NEW), paths(FILE)); + create(ggfs, paths(DIR, SUBDIR), paths(FILE)); + + ggfs.rename(FILE, SUBDIR_NEW); + + checkExist(ggfs, DIR_NEW, SUBDIR_NEW); + checkExist(ggfs, ggfsSecondary, new IgniteFsPath(SUBDIR_NEW, FILE.name())); + checkNotExist(ggfs, ggfsSecondary, FILE); + } + + /** + * Test move in case source and destination exist partially and the path being renamed is a file. + * + * @throws Exception If failed. + */ + public void testMoveFileSourceAndDestinationMissingPartially() throws Exception { + create(ggfsSecondary, paths(DIR, SUBDIR, DIR_NEW, SUBDIR_NEW), paths(FILE)); + create(ggfs, paths(DIR, DIR_NEW), null); + + ggfs.rename(FILE, SUBDIR_NEW); + + checkExist(ggfs, SUBDIR, SUBDIR_NEW); + checkExist(ggfs, ggfsSecondary, new IgniteFsPath(SUBDIR_NEW, FILE.name())); + checkNotExist(ggfs, ggfsSecondary, FILE); + } + + /** + * Test move in case source and destination don't exist and the path being renamed is a file. + * + * @throws Exception If failed. + */ + public void testMoveFileSourceAndDestinationMissing() throws Exception { + create(ggfsSecondary, paths(DIR, SUBDIR, DIR_NEW, SUBDIR_NEW), paths(FILE)); + create(ggfs, null, null); + + ggfs.rename(FILE, SUBDIR_NEW); + + checkExist(ggfs, DIR, SUBDIR, DIR_NEW, SUBDIR_NEW); + checkExist(ggfs, ggfsSecondary, new IgniteFsPath(SUBDIR_NEW, FILE.name())); + checkNotExist(ggfs, ggfsSecondary, FILE); + } + + /** + * Test file move when destination is the root and source is missing partially. + * + * @throws Exception If failed. + */ + public void testMoveFileDestinationRootSourceMissingPartially() throws Exception { + create(ggfsSecondary, paths(DIR, SUBDIR), paths(FILE)); + create(ggfs, paths(DIR), null); + + ggfs.rename(FILE, new IgniteFsPath()); + + checkExist(ggfs, SUBDIR); + checkExist(ggfs, ggfsSecondary, new IgniteFsPath("/" + FILE.name())); + checkNotExist(ggfs, ggfsSecondary, FILE); + } + + /** + * Test file move when destination is the root and source is missing. + * + * @throws Exception If failed. + */ + public void testMoveFileDestinationRootSourceMissing() throws Exception { + create(ggfsSecondary, paths(DIR, SUBDIR), paths(FILE)); + create(ggfs, null, null); + + ggfs.rename(FILE, new IgniteFsPath()); + + checkExist(ggfs, DIR, SUBDIR); + checkExist(ggfs, ggfsSecondary, new IgniteFsPath("/" + FILE.name())); + checkNotExist(ggfs, ggfsSecondary, FILE); + } + + /** + * Test file move when source parent is the root and the source is missing. + * + * @throws Exception If failed. + */ + public void testMoveFileSourceParentRootSourceMissing() throws Exception { + IgniteFsPath file = new IgniteFsPath("/" + FILE.name()); + + create(ggfsSecondary, paths(DIR_NEW, SUBDIR_NEW), paths(file)); + create(ggfs, paths(DIR_NEW, SUBDIR_NEW), null); + + ggfs.rename(file, SUBDIR_NEW); + + checkExist(ggfs, ggfsSecondary, new IgniteFsPath(SUBDIR_NEW, FILE.name())); + checkNotExist(ggfs, ggfsSecondary, file); + } + + /** + * Test file move when source parent is the root and destination is missing partially. + * + * @throws Exception If failed. + */ + public void testMoveFileSourceParentRootDestinationMissingPartially() throws Exception { + IgniteFsPath file = new IgniteFsPath("/" + FILE.name()); + + create(ggfsSecondary, paths(DIR_NEW, SUBDIR_NEW), paths(file)); + create(ggfs, paths(DIR_NEW), null); + + ggfs.rename(file, SUBDIR_NEW); + + checkExist(ggfs, SUBDIR_NEW); + checkExist(ggfs, ggfsSecondary, new IgniteFsPath(SUBDIR_NEW, FILE.name())); + checkNotExist(ggfs, ggfsSecondary, file); + } + + /** + * Test file move when source parent is the root and destination is missing. + * + * @throws Exception If failed. + */ + public void testMoveFileSourceParentRootDestinationMissing() throws Exception { + IgniteFsPath file = new IgniteFsPath("/" + FILE.name()); + + create(ggfsSecondary, paths(DIR_NEW, SUBDIR_NEW), paths(file)); + create(ggfs, null, null); + + ggfs.rename(file, SUBDIR_NEW); + + checkExist(ggfs, DIR_NEW, SUBDIR_NEW); + checkExist(ggfs, ggfsSecondary, new IgniteFsPath(SUBDIR_NEW, FILE.name())); + checkNotExist(ggfs, ggfsSecondary, file); + } + + /** + * Test move and rename in case source exists partially and the path being renamed is a file. + * + * @throws Exception If failed. + */ + public void testMoveRenameFileSourceMissingPartially() throws Exception { + create(ggfsSecondary, paths(DIR, SUBDIR, DIR_NEW, SUBDIR_NEW), paths(FILE)); + create(ggfs, paths(DIR, DIR_NEW, SUBDIR_NEW), paths(FILE)); + + ggfs.rename(FILE, FILE_NEW); + + checkExist(ggfs, SUBDIR); + checkExist(ggfs, ggfsSecondary, FILE_NEW); + checkNotExist(ggfs, ggfsSecondary, FILE); + } + + /** + * Test move and rename in case source doesn't exist and the path being renamed is a file. + * + * @throws Exception If failed. + */ + public void testMoveRenameFileSourceMissing() throws Exception { + create(ggfsSecondary, paths(DIR, SUBDIR, DIR_NEW, SUBDIR_NEW), paths(FILE)); + create(ggfs, paths(DIR_NEW, SUBDIR_NEW), paths(FILE)); + + ggfs.rename(FILE, FILE_NEW); + + checkExist(ggfs, DIR, SUBDIR); + checkExist(ggfs, ggfsSecondary, FILE_NEW); + checkNotExist(ggfs, ggfsSecondary, FILE); + } + + /** + * Test move and rename in case destination exists partially and the path being renamed is a file. + * + * @throws Exception If failed. + */ + public void testMoveRenameFileDestinationMissingPartially() throws Exception { + create(ggfsSecondary, paths(DIR, SUBDIR, DIR_NEW, SUBDIR_NEW), paths(FILE)); + create(ggfs, paths(DIR, SUBDIR, DIR_NEW), paths(FILE)); + + ggfs.rename(FILE, FILE_NEW); + + checkExist(ggfs, SUBDIR_NEW); + checkExist(ggfs, ggfsSecondary, FILE_NEW); + checkNotExist(ggfs, ggfsSecondary, FILE); + } + + /** + * Test move and rename in case destination doesn't exist and the path being renamed is a file. + * + * @throws Exception If failed. + */ + public void testMoveRenameFileDestinationMissing() throws Exception { + create(ggfsSecondary, paths(DIR, SUBDIR, DIR_NEW, SUBDIR_NEW), paths(FILE)); + create(ggfs, paths(DIR, SUBDIR), paths(FILE)); + + ggfs.rename(FILE, FILE_NEW); + + checkExist(ggfs, DIR_NEW, SUBDIR_NEW); + checkExist(ggfs, ggfsSecondary, FILE_NEW); + checkNotExist(ggfs, ggfsSecondary, FILE); + } + + /** + * Test move and rename in case source and destination exist partially and the path being renamed is a file. + * + * @throws Exception If failed. + */ + public void testMoveRenameFileSourceAndDestinationMissingPartially() throws Exception { + create(ggfsSecondary, paths(DIR, SUBDIR, DIR_NEW, SUBDIR_NEW), paths(FILE)); + create(ggfs, paths(DIR, DIR_NEW), null); + + ggfs.rename(FILE, FILE_NEW); + + checkExist(ggfs, SUBDIR, SUBDIR_NEW); + checkExist(ggfs, ggfsSecondary, FILE_NEW); + checkNotExist(ggfs, ggfsSecondary, FILE); + } + + /** + * Test move and rename in case source and destination don't exist and the path being renamed is a file. + * + * @throws Exception If failed. + */ + public void testMoveRenameFileSourceAndDestinationMissing() throws Exception { + create(ggfsSecondary, paths(DIR, SUBDIR, DIR_NEW, SUBDIR_NEW), paths(FILE)); + create(ggfs, null, null); + + ggfs.rename(FILE, FILE_NEW); + + checkExist(ggfs, DIR, SUBDIR, DIR_NEW, SUBDIR_NEW); + checkExist(ggfs, ggfsSecondary, FILE_NEW); + checkNotExist(ggfs, ggfsSecondary, FILE); + } + + /** + * Test file move and rename when destination is the root and source is missing partially. + * + * @throws Exception If failed. + */ + public void testMoveRenameFileDestinationRootSourceMissingPartially() throws Exception { + IgniteFsPath file = new IgniteFsPath("/" + FILE.name()); + + create(ggfsSecondary, paths(DIR, SUBDIR), paths(FILE)); + create(ggfs, paths(DIR), null); + + ggfs.rename(FILE, file); + + checkExist(ggfs, SUBDIR); + checkExist(ggfs, ggfsSecondary, file); + checkNotExist(ggfs, ggfsSecondary, FILE); + } + + /** + * Test file move and rename when destination is the root and source is missing. + * + * @throws Exception If failed. + */ + public void testMoveRenameFileDestinationRootSourceMissing() throws Exception { + IgniteFsPath file = new IgniteFsPath("/" + FILE.name()); + + create(ggfsSecondary, paths(DIR, SUBDIR), paths(FILE)); + create(ggfs, null, null); + + ggfs.rename(FILE, file); + + checkExist(ggfs, DIR, SUBDIR); + checkExist(ggfs, ggfsSecondary, file); + checkNotExist(ggfs, ggfsSecondary, FILE); + } + + /** + * Test file move and rename when source parent is the root and the source is missing. + * + * @throws Exception If failed. + */ + public void testMoveRenameFileSourceParentRootSourceMissing() throws Exception { + IgniteFsPath file = new IgniteFsPath("/" + FILE_NEW.name()); + + create(ggfsSecondary, paths(DIR_NEW, SUBDIR_NEW), paths(file)); + create(ggfs, paths(DIR_NEW, SUBDIR_NEW), null); + + ggfs.rename(file, FILE_NEW); + + checkExist(ggfs, ggfsSecondary, FILE_NEW); + checkNotExist(ggfs, ggfsSecondary, file); + } + + /** + * Test file move and rename when source parent is the root and destination is missing partially. + * + * @throws Exception If failed. + */ + public void testMoveRenameFileSourceParentRootDestinationMissingPartially() throws Exception { + IgniteFsPath file = new IgniteFsPath("/" + FILE_NEW.name()); + + create(ggfsSecondary, paths(DIR_NEW, SUBDIR_NEW), paths(file)); + create(ggfs, paths(DIR_NEW), null); + + ggfs.rename(file, FILE_NEW); + + checkExist(ggfs, SUBDIR_NEW); + checkExist(ggfs, ggfsSecondary, FILE_NEW); + checkNotExist(ggfs, ggfsSecondary, file); + } + + /** + * Test file move and rename when source parent is the root and destination is missing. + * + * @throws Exception If failed. + */ + public void testMoveRenameFileSourceParentRootDestinationMissing() throws Exception { + IgniteFsPath file = new IgniteFsPath("/" + FILE_NEW.name()); + + create(ggfsSecondary, paths(DIR_NEW, SUBDIR_NEW), paths(file)); + create(ggfs, null, null); + + ggfs.rename(file, FILE_NEW); + + checkExist(ggfs, DIR_NEW, SUBDIR_NEW); + checkExist(ggfs, ggfsSecondary, FILE_NEW); + checkNotExist(ggfs, ggfsSecondary, file); + } + + /** + * Test move in case source exists partially and the path being renamed is a directory. + * + * @throws Exception If failed. + */ + public void testMoveDirectorySourceMissingPartially() throws Exception { + create(ggfsSecondary, paths(DIR, SUBDIR, SUBSUBDIR, DIR_NEW, SUBDIR_NEW), null); + create(ggfs, paths(DIR, DIR_NEW, SUBDIR_NEW), null); + + ggfs.rename(SUBSUBDIR, SUBDIR_NEW); + + checkExist(ggfs, SUBDIR); + checkExist(ggfs, ggfsSecondary, new IgniteFsPath(SUBDIR_NEW, SUBSUBDIR.name())); + checkNotExist(ggfs, ggfsSecondary, SUBSUBDIR); + } + + /** + * Test move in case source doesn't exist and the path being renamed is a directory. + * + * @throws Exception If failed. + */ + public void testMoveDirectorySourceMissing() throws Exception { + create(ggfsSecondary, paths(DIR, SUBDIR, SUBSUBDIR, DIR_NEW, SUBDIR_NEW), null); + create(ggfs, paths(DIR_NEW, SUBDIR_NEW), null); + + ggfs.rename(SUBSUBDIR, SUBDIR_NEW); + + checkExist(ggfs, DIR); + checkExist(ggfs, SUBDIR); + checkExist(ggfs, ggfsSecondary, new IgniteFsPath(SUBDIR_NEW, SUBSUBDIR.name())); + checkNotExist(ggfs, ggfsSecondary, SUBSUBDIR); + } + + /** + * Test move in case destination exists partially and the path being renamed is a directory. + * + * @throws Exception If failed. + */ + public void testMoveDirectoryDestinationMissingPartially() throws Exception { + create(ggfsSecondary, paths(DIR, SUBDIR, SUBSUBDIR, DIR_NEW, SUBDIR_NEW), null); + create(ggfs, paths(DIR, SUBDIR, SUBSUBDIR, DIR_NEW), null); + + ggfs.rename(SUBSUBDIR, SUBDIR_NEW); + + checkExist(ggfs, SUBDIR_NEW); + checkExist(ggfs, ggfsSecondary, new IgniteFsPath(SUBDIR_NEW, SUBSUBDIR.name())); + checkNotExist(ggfs, ggfsSecondary, SUBSUBDIR); + } + + /** + * Test move in case destination doesn't exist and the path being renamed is a directory. + * + * @throws Exception If failed. + */ + public void testMoveDirectoryDestinationMissing() throws Exception { + create(ggfsSecondary, paths(DIR, SUBDIR, SUBSUBDIR, DIR_NEW, SUBDIR_NEW), null); + create(ggfs, paths(DIR, SUBDIR, SUBSUBDIR), null); + + ggfs.rename(SUBSUBDIR, SUBDIR_NEW); + + checkExist(ggfs, DIR_NEW, SUBDIR_NEW); + checkExist(ggfs, ggfsSecondary, new IgniteFsPath(SUBDIR_NEW, SUBSUBDIR.name())); + checkNotExist(ggfs, ggfsSecondary, SUBSUBDIR); + } + + /** + * Test move in case source and destination exist partially and the path being renamed is a directory. + * + * @throws Exception If failed. + */ + public void testMoveDirectorySourceAndDestinationMissingPartially() throws Exception { + create(ggfsSecondary, paths(DIR, SUBDIR, SUBSUBDIR, DIR_NEW, SUBDIR_NEW), null); + create(ggfs, paths(DIR, DIR_NEW), null); + + ggfs.rename(SUBSUBDIR, SUBDIR_NEW); + + checkExist(ggfs, SUBDIR, SUBDIR_NEW); + checkExist(ggfs, ggfsSecondary, new IgniteFsPath(SUBDIR_NEW, SUBSUBDIR.name())); + checkNotExist(ggfs, ggfsSecondary, SUBSUBDIR); + } + + /** + * Test move in case source and destination don't exist and the path being renamed is a directory. + * + * @throws Exception If failed. + */ + public void testMoveDirectorySourceAndDestinationMissing() throws Exception { + create(ggfsSecondary, paths(DIR, SUBDIR, SUBSUBDIR, DIR_NEW, SUBDIR_NEW), null); + create(ggfs, null, null); + + ggfs.rename(SUBSUBDIR, SUBDIR_NEW); + + checkExist(ggfs, DIR, SUBDIR, DIR_NEW, SUBDIR_NEW); + checkExist(ggfs, ggfsSecondary, new IgniteFsPath(SUBDIR_NEW, SUBSUBDIR.name())); + checkNotExist(ggfs, ggfsSecondary, SUBSUBDIR); + } + + /** + * Test directory move when destination is the root and source is missing partially. + * + * @throws Exception If failed. + */ + public void testMoveDirectoryDestinationRootSourceMissingPartially() throws Exception { + create(ggfsSecondary, paths(DIR, SUBDIR, SUBSUBDIR), null); + create(ggfs, paths(DIR), null); + + ggfs.rename(SUBSUBDIR, new IgniteFsPath()); + + checkExist(ggfs, SUBDIR); + checkExist(ggfs, ggfsSecondary, new IgniteFsPath("/" + SUBSUBDIR.name())); + checkNotExist(ggfs, ggfsSecondary, SUBSUBDIR); + } + + /** + * Test directory move when destination is the root and source is missing. + * + * @throws Exception If failed. + */ + public void testMoveDirectoryDestinationRootSourceMissing() throws Exception { + create(ggfsSecondary, paths(DIR, SUBDIR, SUBSUBDIR), null); + create(ggfs, null, null); + + ggfs.rename(SUBSUBDIR, new IgniteFsPath()); + + checkExist(ggfs, DIR, SUBDIR); + checkExist(ggfs, ggfsSecondary, new IgniteFsPath("/" + SUBSUBDIR.name())); + checkNotExist(ggfs, ggfsSecondary, SUBSUBDIR); + } + + /** + * Test directory move when source parent is the root and the source folder is missing locally. + * + * @throws Exception If failed. + */ + public void testMoveDirectorySourceParentRootSourceMissing() throws Exception { + IgniteFsPath dir = new IgniteFsPath("/" + SUBSUBDIR.name()); + + create(ggfsSecondary, paths(DIR_NEW, SUBDIR_NEW, dir), null); + create(ggfs, paths(DIR_NEW, SUBDIR_NEW), null); + + ggfs.rename(dir, SUBDIR_NEW); + + checkExist(ggfs, ggfsSecondary, new IgniteFsPath(SUBDIR_NEW, SUBSUBDIR.name())); + checkNotExist(ggfs, ggfsSecondary, dir); + } + + /** + * Test directory move when source parent is the root and destination is missing partially. + * + * @throws Exception If failed. + */ + public void testMoveDirectorySourceParentRootDestinationMissingPartially() throws Exception { + IgniteFsPath dir = new IgniteFsPath("/" + SUBSUBDIR.name()); + + create(ggfsSecondary, paths(DIR_NEW, SUBDIR_NEW, dir), null); + create(ggfs, paths(DIR_NEW), null); + + ggfs.rename(dir, SUBDIR_NEW); + + checkExist(ggfs, SUBDIR_NEW); + checkExist(ggfs, ggfsSecondary, new IgniteFsPath(SUBDIR_NEW, SUBSUBDIR.name())); + checkNotExist(ggfs, ggfsSecondary, dir); + } + + /** + * Test directory move when source parent is the root and destination is missing. + * + * @throws Exception If failed. + */ + public void testMoveDirectorySourceParentRootDestinationMissing() throws Exception { + IgniteFsPath dir = new IgniteFsPath("/" + SUBSUBDIR.name()); + + create(ggfsSecondary, paths(DIR_NEW, SUBDIR_NEW, dir), null); + create(ggfs, null, null); + + ggfs.rename(dir, SUBDIR_NEW); + + checkExist(ggfs, DIR_NEW, SUBDIR_NEW); + checkExist(ggfs, ggfsSecondary, new IgniteFsPath(SUBDIR_NEW, SUBSUBDIR.name())); + checkNotExist(ggfs, ggfsSecondary, dir); + } + + /** + * Test move and rename in case source exists partially and the path being renamed is a directory. + * + * @throws Exception If failed. + */ + public void testMoveRenameDirectorySourceMissingPartially() throws Exception { + create(ggfsSecondary, paths(DIR, SUBDIR, SUBSUBDIR, DIR_NEW, SUBDIR_NEW), null); + create(ggfs, paths(DIR, DIR_NEW, SUBDIR_NEW), null); + + ggfs.rename(SUBSUBDIR, SUBSUBDIR_NEW); + + checkExist(ggfs, SUBDIR); + checkExist(ggfs, ggfsSecondary, SUBSUBDIR_NEW); + checkNotExist(ggfs, ggfsSecondary, SUBSUBDIR); + } + + /** + * Test move and rename in case source doesn't exist and the path being renamed is a directory. + * + * @throws Exception If failed. + */ + public void testMoveRenameDirectorySourceMissing() throws Exception { + create(ggfsSecondary, paths(DIR, SUBDIR, SUBSUBDIR, DIR_NEW, SUBDIR_NEW), null); + create(ggfs, paths(DIR_NEW, SUBDIR_NEW), null); + + ggfs.rename(SUBSUBDIR, SUBSUBDIR_NEW); + + checkExist(ggfs, DIR); + checkExist(ggfs, SUBDIR); + checkExist(ggfs, ggfsSecondary, SUBSUBDIR_NEW); + checkNotExist(ggfs, ggfsSecondary, SUBSUBDIR); + } + + /** + * Test move and rename in case destination exists partially and the path being renamed is a directory. + * + * @throws Exception If failed. + */ + public void testMoveRenameDirectoryDestinationMissingPartially() throws Exception { + create(ggfsSecondary, paths(DIR, SUBDIR, SUBSUBDIR, DIR_NEW, SUBDIR_NEW), null); + create(ggfs, paths(DIR, SUBDIR, SUBSUBDIR, DIR_NEW), null); + + ggfs.rename(SUBSUBDIR, SUBSUBDIR_NEW); + + checkExist(ggfs, SUBDIR_NEW); + checkExist(ggfs, ggfsSecondary, SUBSUBDIR_NEW); + checkNotExist(ggfs, ggfsSecondary, SUBSUBDIR); + } + + /** + * Test move and rename in case destination doesn't exist and the path being renamed is a directory. + * + * @throws Exception If failed. + */ + public void testMoveRenameDirectoryDestinationMissing() throws Exception { + create(ggfsSecondary, paths(DIR, SUBDIR, SUBSUBDIR, DIR_NEW, SUBDIR_NEW), null); + create(ggfs, paths(DIR, SUBDIR, SUBSUBDIR), null); + + ggfs.rename(SUBSUBDIR, SUBSUBDIR_NEW); + + checkExist(ggfs, DIR_NEW, SUBDIR_NEW); + checkExist(ggfs, ggfsSecondary, SUBSUBDIR_NEW); + checkNotExist(ggfs, ggfsSecondary, SUBSUBDIR); + } + + /** + * Test move and rename in case source and destination exist partially and the path being renamed is a directory. + * + * @throws Exception If failed. + */ + public void testMoveRenameDirectorySourceAndDestinationMissingPartially() throws Exception { + create(ggfsSecondary, paths(DIR, SUBDIR, SUBSUBDIR, DIR_NEW, SUBDIR_NEW), null); + create(ggfs, paths(DIR, DIR_NEW), null); + + ggfs.rename(SUBSUBDIR, SUBSUBDIR_NEW); + + checkExist(ggfs, SUBDIR, SUBDIR_NEW); + checkExist(ggfs, ggfsSecondary, SUBSUBDIR_NEW); + checkNotExist(ggfs, ggfsSecondary, SUBSUBDIR); + } + + /** + * Test move and rename in case source and destination don't exist and the path being renamed is a directory. + * + * @throws Exception If failed. + */ + public void testMoveRenameDirectorySourceAndDestinationMissing() throws Exception { + create(ggfsSecondary, paths(DIR, SUBDIR, SUBSUBDIR, DIR_NEW, SUBDIR_NEW), null); + create(ggfs, null, null); + + ggfs.rename(SUBSUBDIR, SUBSUBDIR_NEW); + + checkExist(ggfs, DIR, SUBDIR, DIR_NEW, SUBDIR_NEW); + checkExist(ggfs, ggfsSecondary, SUBSUBDIR_NEW); + checkNotExist(ggfs, ggfsSecondary, SUBSUBDIR); + } + + /** + * Test directory move and rename when destination is the root and source is missing partially. + * + * @throws Exception If failed. + */ + public void testMoveRenameDirectoryDestinationRootSourceMissingPartially() throws Exception { + IgniteFsPath dir = new IgniteFsPath("/" + SUBSUBDIR.name()); + + create(ggfsSecondary, paths(DIR, SUBDIR, SUBSUBDIR), null); + create(ggfs, paths(DIR), null); + + ggfs.rename(SUBSUBDIR, dir); + + checkExist(ggfs, SUBDIR); + checkExist(ggfs, ggfsSecondary, dir); + checkNotExist(ggfs, ggfsSecondary, SUBSUBDIR); + } + + /** + * Test directory move and rename when destination is the root and source is missing. + * + * @throws Exception If failed. + */ + public void testMoveRenameDirectoryDestinationRootSourceMissing() throws Exception { + IgniteFsPath dir = new IgniteFsPath("/" + SUBSUBDIR.name()); + + create(ggfsSecondary, paths(DIR, SUBDIR, SUBSUBDIR), null); + create(ggfs, null, null); + + ggfs.rename(SUBSUBDIR, dir); + + checkExist(ggfs, DIR, SUBDIR); + checkExist(ggfs, ggfsSecondary, dir); + checkNotExist(ggfs, ggfsSecondary, SUBSUBDIR); + } + + /** + * Test directory move and rename when source parent is the root and the source is missing locally. + * + * @throws Exception If failed. + */ + public void testMoveRenameDirectorySourceParentRootSourceMissing() throws Exception { + IgniteFsPath dir = new IgniteFsPath("/" + SUBSUBDIR_NEW.name()); + + create(ggfsSecondary, paths(DIR_NEW, SUBDIR_NEW, dir), null); + create(ggfs, paths(DIR_NEW, SUBDIR_NEW), null); + + ggfs.rename(dir, SUBSUBDIR_NEW); + + checkExist(ggfs, ggfsSecondary, SUBSUBDIR_NEW); + checkNotExist(ggfs, ggfsSecondary, dir); + } + + /** + * Test directory move and rename when source parent is the root and destination is missing partially. + * + * @throws Exception If failed. + */ + public void testMoveRenameDirectorySourceParentRootDestinationMissingPartially() throws Exception { + IgniteFsPath dir = new IgniteFsPath("/" + SUBSUBDIR_NEW.name()); + + create(ggfsSecondary, paths(DIR_NEW, SUBDIR_NEW, dir), null); + create(ggfs, paths(DIR_NEW), null); + + ggfs.rename(dir, SUBSUBDIR_NEW); + + checkExist(ggfs, SUBDIR_NEW); + checkExist(ggfs, ggfsSecondary, SUBSUBDIR_NEW); + checkNotExist(ggfs, ggfsSecondary, dir); + } + + /** + * Test directory move and rename when source parent is the root and destination is missing. + * + * @throws Exception If failed. + */ + public void testMoveRenameDirectorySourceParentRootDestinationMissing() throws Exception { + IgniteFsPath dir = new IgniteFsPath("/" + SUBSUBDIR_NEW.name()); + + create(ggfsSecondary, paths(DIR_NEW, SUBDIR_NEW, dir), null); + create(ggfs, null, null); + + ggfs.rename(dir, SUBSUBDIR_NEW); + + checkExist(ggfs, DIR_NEW, SUBDIR_NEW); + checkExist(ggfs, ggfsSecondary, SUBSUBDIR_NEW); + checkNotExist(ggfs, ggfsSecondary, dir); + } + + /** + * Test mkdirs in case parent exists remotely, but some part of the parent path doesn't exist locally. + * + * @throws Exception If failed. + */ + public void testMkdirsParentPathMissingPartially() throws Exception { + Map<String, String> props = properties(null, null, "0555"); // mkdirs command doesn't propagate user info. + + create(ggfsSecondary, paths(DIR, SUBDIR), null); + create(ggfs, paths(DIR), null); + + ggfs.mkdirs(SUBSUBDIR, props); + + // Ensure that directory was created and properties are propagated. + checkExist(ggfs, SUBDIR); + checkExist(ggfs, ggfsSecondary, SUBSUBDIR); + + assertEquals(props, ggfsSecondary.info(SUBSUBDIR).properties()); + + // We check only permission because GGFS client adds username and group name explicitly. + assertEquals(props.get(PROP_PERMISSION), ggfs.info(SUBSUBDIR).properties().get(PROP_PERMISSION)); + } + + /** + * Test mkdirs in case parent exists remotely, but no parents exist locally. + * + * @throws Exception If failed. + */ + public void testMkdrisParentPathMissing() throws Exception { + Map<String, String> props = properties(null, null, "0555"); // mkdirs command doesn't propagate user info. + + create(ggfsSecondary, paths(DIR, SUBDIR), null); + create(ggfs, null, null); + + ggfs.mkdirs(SUBSUBDIR, props); + + // Ensure that directory was created and properties are propagated. + checkExist(ggfs, DIR); + checkExist(ggfs, SUBDIR); + checkExist(ggfs, ggfsSecondary, SUBSUBDIR); + + assertEquals(props, ggfsSecondary.info(SUBSUBDIR).properties()); + + // We check only permission because GGFS client adds username and group name explicitly. + assertEquals(props.get(PROP_PERMISSION), ggfs.info(SUBSUBDIR).properties().get(PROP_PERMISSION)); + } + + /** + * Test delete in case parent exists remotely, but some part of the parent path doesn't exist locally. + * + * @throws Exception If failed. + */ + public void testDeletePathMissingPartially() throws Exception { + create(ggfsSecondary, paths(DIR, SUBDIR, SUBSUBDIR), paths(FILE)); + create(ggfs, paths(DIR), null); + + ggfs.delete(SUBDIR, true); + + checkExist(ggfs, DIR); + checkNotExist(ggfs, ggfsSecondary, SUBDIR, SUBSUBDIR, FILE); + } + + /** + * Test delete in case parent exists remotely, but no parents exist locally. + * + * @throws Exception If failed. + */ + public void testDeletePathMissing() throws Exception { + create(ggfsSecondary, paths(DIR, SUBDIR, SUBSUBDIR), paths(FILE)); + create(ggfs, null, null); + + ggfs.delete(SUBDIR, true); + + checkExist(ggfs, DIR); + checkNotExist(ggfs, ggfsSecondary, SUBDIR, SUBSUBDIR, FILE); + } + + /** + * Test delete when the path parent is the root and the path is missing locally. + * + * @throws Exception If failed. + */ + public void testDeleteParentRootPathMissing() throws Exception { + create(ggfsSecondary, paths(DIR, SUBDIR, SUBSUBDIR), paths(FILE)); + create(ggfs, null, null); + + ggfs.delete(DIR, true); + + checkNotExist(ggfs, ggfsSecondary, DIR, SUBDIR, SUBSUBDIR, FILE); + } + + /** + * Test update in case file exists remotely, but some part of the path doesn't exist locally. + * + * @throws Exception If failed. + */ + public void testUpdatePathMissingPartially() throws Exception { + Map<String, String> propsSubDir = properties("subDirOwner", "subDirGroup", "0555"); + Map<String, String> propsFile = properties("fileOwner", "fileGroup", "0666"); + + create(ggfsSecondary, paths(DIR, SUBDIR), paths(FILE)); + create(ggfs, paths(DIR), null); + + // Set different properties to the sub-directory. + ggfsSecondary.update(SUBDIR, propsSubDir); + + ggfs.update(FILE, propsFile); + + // Ensure missing entries were re-created locally. + checkExist(ggfs, SUBDIR, FILE); + + // Ensure properties propagation. + assertEquals(propsSubDir, ggfsSecondary.info(SUBDIR).properties()); + assertEquals(propsSubDir, ggfs.info(SUBDIR).properties()); + + assertEquals(propsFile, ggfsSecondary.info(FILE).properties()); + assertEquals(propsFile, ggfs.info(FILE).properties()); + } + + /** + * Test update in case file exists remotely, but neither the file nor all it's parents exist locally. + * + * @throws Exception If failed. + */ + public void testUpdatePathMissing() throws Exception { + Map<String, String> propsSubDir = properties("subDirOwner", "subDirGroup", "0555"); + Map<String, String> propsFile = properties("fileOwner", "fileGroup", "0666"); + + create(ggfsSecondary, paths(DIR, SUBDIR), paths(FILE)); + create(ggfs, null, null); + + // Set different properties to the sub-directory. + ggfsSecondary.update(SUBDIR, propsSubDir); + + ggfs.update(FILE, propsFile); + + // Ensure missing entries were re-created locally. + checkExist(ggfs, DIR, SUBDIR, FILE); + + // Ensure properties propagation. + assertEquals(propsSubDir, ggfsSecondary.info(SUBDIR).properties()); + assertEquals(propsSubDir, ggfs.info(SUBDIR).properties()); + + assertEquals(propsFile, ggfsSecondary.info(FILE).properties()); + assertEquals(propsFile, ggfs.info(FILE).properties()); + } + + /** + * Test update when parent is the root and the path being updated is missing locally. + * + * @throws Exception If failed. + */ + public void testUpdateParentRootPathMissing() throws Exception { + Map<String, String> props = properties("owner", "group", "0555"); + + create(ggfsSecondary, paths(DIR), null); + create(ggfs, null, null); + + ggfs.update(DIR, props); + + checkExist(ggfs, DIR); + + assertEquals(props, ggfsSecondary.info(DIR).properties()); + assertEquals(props, ggfs.info(DIR).properties()); + } + + /** + * Test file open in case it doesn't exist locally. + * + * @throws Exception If failed. + */ + public void testOpenMissing() throws Exception { + create(ggfsSecondary, paths(DIR, SUBDIR), null); + create(ggfs, null, null); + + createFile(ggfsSecondary, FILE, true, chunk); + + checkFileContent(ggfs, FILE, chunk); + } + + /** + * Ensure that no prefetch occurs in case not enough block are read sequentially. + * + * @throws Exception If failed. + */ + public void testOpenNoPrefetch() throws Exception { + create(ggfsSecondary, paths(DIR, SUBDIR), paths(FILE)); + + // Write enough data to the secondary file system. + final int blockSize = GGFS_BLOCK_SIZE; + + IgniteFsOutputStream out = ggfsSecondary.append(FILE, false); + + int totalWritten = 0; + + while (totalWritten < blockSize * 2 + chunk.length) { + out.write(chunk); + + totalWritten += chunk.length; + } + + out.close(); + + awaitFileClose(ggfsSecondary, FILE); + + // Read the first block. + int totalRead = 0; + + IgniteFsInputStream in = ggfs.open(FILE, blockSize); + + final byte[] readBuf = new byte[1024]; + + while (totalRead + readBuf.length <= blockSize) { + in.read(readBuf); + + totalRead += readBuf.length; + } + + // Now perform seek. + in.seek(blockSize * 2); + + // Read the third block. + totalRead = 0; + + while (totalRead < totalWritten - blockSize * 2) { + in.read(readBuf); + + totalRead += readBuf.length; + } + + // Let's wait for a while because prefetch occurs asynchronously. + U.sleep(300); + + // Remove the file from the secondary file system. + ggfsSecondary.delete(FILE, false); + + // Let's wait for file will be deleted. + U.sleep(300); + + final IgniteFsInputStream in0 = in; + + // Try reading the second block. Should fail. + GridTestUtils.assertThrows(log, new Callable<Object>() { + @Override + public Object call() throws Exception { + in0.seek(blockSize); + + try { + in0.read(readBuf); + } finally { + U.closeQuiet(in0); + } + + return null; + } + }, IOException.class, "Failed to read data due to secondary file system exception: " + + "Failed to retrieve file's data block (corrupted file?) [path=/dir/subdir/file, blockIdx=1"); + } + + /** + * Ensure that prefetch occurs in case several blocks are read sequentially. + * + * @throws Exception If failed. + */ + public void testOpenPrefetch() throws Exception { + create(ggfsSecondary, paths(DIR, SUBDIR), paths(FILE)); + + // Write enough data to the secondary file system. + final int blockSize = ggfs.info(FILE).blockSize(); + + IgniteFsOutputStream out = ggfsSecondary.append(FILE, false); + + int totalWritten = 0; + + while (totalWritten < blockSize * 2 + chunk.length) { + out.write(chunk); + + totalWritten += chunk.length; + } + + out.close(); + + awaitFileClose(ggfsSecondary, FILE); + + // Read the first two blocks. + int totalRead = 0; + + IgniteFsInputStream in = ggfs.open(FILE, blockSize); + + final byte[] readBuf = new byte[1024]; + + while (totalRead + readBuf.length <= blockSize * 2) { + in.read(readBuf); + + totalRead += readBuf.length; + } + + // Wait for a while for prefetch to finish. + GridGgfsMetaManager meta = ggfs.context().meta(); + + GridGgfsFileInfo info = meta.info(meta.fileId(FILE)); + + GridGgfsBlockKey key = new GridGgfsBlockKey(info.id(), info.affinityKey(), info.evictExclude(), 2); + + GridCache<GridGgfsBlockKey, byte[]> dataCache = ggfs.context().kernalContext().cache().cache( + ggfs.configuration().getDataCacheName()); + + for (int i = 0; i < 10; i++) { + if (dataCache.containsKey(key)) + break; + else + U.sleep(100); + } + + // Remove the file from the secondary file system. + ggfsSecondary.delete(FILE, false); + + // Let's wait for file will be deleted. + U.sleep(300); + + // Read the third block. + totalRead = 0; + + in.seek(blockSize * 2); + + while (totalRead + readBuf.length <= blockSize) { + in.read(readBuf); + + totalRead += readBuf.length; + } + + in.close(); + } + + /** + * Test create when parent directory is partially missing locally. + * + * @throws Exception If failed. + */ + public void testCreateParentMissingPartially() throws Exception { + Map<String, String> props = properties("owner", "group", "0555"); + + create(ggfsSecondary, paths(DIR, SUBDIR), null); + create(ggfs, paths(DIR), null); + + ggfsSecondary.update(SUBDIR, props); + + createFile(ggfs, FILE, true, chunk); + + // Ensure that directory structure was created. + checkExist(ggfs, ggfsSecondary, SUBDIR); + checkFile(ggfs, ggfsSecondary, FILE, chunk); + + // Ensure properties propagation of the created subdirectory. + assertEquals(props, ggfs.info(SUBDIR).properties()); + } + + /** + * Test create when parent directory is missing locally. + * + * @throws Exception If failed. + */ + public void testCreateParentMissing() throws Exception { + Map<String, String> propsDir = properties("ownerDir", "groupDir", "0555"); + Map<String, String> propsSubDir = properties("ownerSubDir", "groupSubDir", "0666"); + + create(ggfsSecondary, paths(DIR, SUBDIR), null); + create(ggfs, null, null); + + ggfsSecondary.update(DIR, propsDir); + ggfsSecondary.update(SUBDIR, propsSubDir); + + createFile(ggfs, FILE, true, chunk); + + checkExist(ggfs, ggfsSecondary, SUBDIR); + checkFile(ggfs, ggfsSecondary, FILE, chunk); + + // Ensure properties propagation of the created directories. + assertEquals(propsDir, ggfs.info(DIR).properties()); + assertEquals(propsSubDir, ggfs.info(SUBDIR).properties()); + } + + /** + * Test append when parent directory is partially missing locally. + * + * @throws Exception If failed. + */ + public void testAppendParentMissingPartially() throws Exception { + Map<String, String> props = properties("owner", "group", "0555"); + + create(ggfsSecondary, paths(DIR, SUBDIR), null); + create(ggfs, paths(DIR), null); + + ggfsSecondary.update(SUBDIR, props); + + createFile(ggfsSecondary, FILE, true, BLOCK_SIZE, chunk); + + appendFile(ggfs, FILE, chunk); + + // Ensure that directory structure was created. + checkExist(ggfs, ggfsSecondary, SUBDIR); + checkFile(ggfs, ggfsSecondary, FILE, chunk, chunk); + + // Ensure properties propagation of the created subdirectory. + assertEquals(props, ggfs.info(SUBDIR).properties()); + } + + /** + * Test append when parent directory is missing locally. + * + * @throws Exception If failed. + */ + public void testAppendParentMissing() throws Exception { + Map<String, String> propsDir = properties("ownerDir", "groupDir", "0555"); + Map<String, String> propsSubDir = properties("ownerSubDir", "groupSubDir", "0666"); + + create(ggfsSecondary, paths(DIR, SUBDIR), null); + create(ggfs, null, null); + + ggfsSecondary.update(DIR, propsDir); + ggfsSecondary.update(SUBDIR, propsSubDir); + + createFile(ggfsSecondary, FILE, true, BLOCK_SIZE, chunk); + + appendFile(ggfs, FILE, chunk); + + checkExist(ggfs, ggfsSecondary, SUBDIR); + checkFile(ggfs, ggfsSecondary, FILE, chunk, chunk); + + // Ensure properties propagation of the created directories. + assertEquals(propsDir, ggfs.info(DIR).properties()); + assertEquals(propsSubDir, ggfs.info(SUBDIR).properties()); + } + + /** + * Ensure that in case we rename the folder A and delete it at the same time, only one of these requests succeed. + * Initially file system entries are created only in the secondary file system. + * + * @throws Exception If failed. + */ + public void testConcurrentRenameDeleteSourceRemote() throws Exception { + for (int i = 0; i < REPEAT_CNT; i++) { + final CyclicBarrier barrier = new CyclicBarrier(2); + + create(ggfsSecondary, paths(DIR, SUBDIR, DIR_NEW), paths()); + + GridPlainFuture<Boolean> res1 = execute(new Callable<Boolean>() { + @Override public Boolean call() throws Exception { + U.awaitQuiet(barrier); + + try { + ggfs.rename(SUBDIR, SUBDIR_NEW); + + return true; + } + catch (IgniteCheckedException ignored) { + return false; + } + } + }); + + GridPlainFuture<Boolean> res2 = execute(new Callable<Boolean>() { + @Override public Boolean call() throws Exception { + U.awaitQuiet(barrier); + + return ggfs.delete(SUBDIR, true); + } + }); + + res1.get(); + res2.get(); + + if (res1.get()) { + assert !res2.get(); // Rename succeeded, so delete must fail. + + checkExist(ggfs, ggfsSecondary, DIR, DIR_NEW, SUBDIR_NEW); + checkNotExist(ggfs, ggfsSecondary, SUBDIR); + } + else { + assert res2.get(); // Rename failed because delete succeeded. + + checkExist(ggfs, DIR); // DIR_NEW should not be synchronized with he primary GGFS. + checkExist(ggfsSecondary, DIR, DIR_NEW); + checkNotExist(ggfs, ggfsSecondary, SUBDIR, SUBDIR_NEW); + } + + clear(ggfs, ggfsSecondary); + } + } + + /** + * Ensure that in case we rename the folder A to B and delete B at the same time, FS consistency is not compromised. + * Initially file system entries are created only in the secondary file system. + * + * @throws Exception If failed. + */ + public void testConcurrentRenameDeleteDestinationRemote() throws Exception { + for (int i = 0; i < REPEAT_CNT; i++) { + final CyclicBarrier barrier = new CyclicBarrier(2); + + create(ggfsSecondary, paths(DIR, SUBDIR, DIR_NEW), paths()); + + GridPlainFuture<Boolean> res1 = execute(new Callable<Boolean>() { + @Override public Boolean call() throws Exception { + U.awaitQuiet(barrier); + + try { + ggfs.rename(SUBDIR, SUBDIR_NEW); + + return true; + } + catch (IgniteCheckedException ignored) { + return false; + } + } + }); + + GridPlainFuture<Boolean> res2 = execute(new Callable<Boolean>() { + @Override public Boolean call() throws Exception { + U.awaitQuiet(barrier); + + return ggfs.delete(SUBDIR_NEW, true); + } + }); + + assert res1.get(); + + if (res2.get()) { + // Delete after rename. + checkExist(ggfs, ggfsSecondary, DIR, DIR_NEW); + checkNotExist(ggfs, ggfsSecondary, SUBDIR, SUBDIR_NEW); + } + else { + // Delete before rename. + checkExist(ggfs, ggfsSecondary, DIR, DIR_NEW, SUBDIR_NEW); + checkNotExist(ggfs, ggfsSecondary, SUBDIR); + } + + clear(ggfs, ggfsSecondary); + } + } + + /** + * Ensure file system consistency in case two concurrent rename requests are executed: A -> B and B -> A. + * Initially file system entries are created only in the secondary file system. + * + * @throws Exception If failed. + */ + public void testConcurrentRenamesRemote() throws Exception { + for (int i = 0; i < REPEAT_CNT; i++) { + final CyclicBarrier barrier = new CyclicBarrier(2); + + create(ggfsSecondary, paths(DIR, SUBDIR, DIR_NEW), paths()); + + GridPlainFuture<Boolean> res1 = execute(new Callable<Boolean>() { + @Override public Boolean call() throws Exception { + U.awaitQuiet(barrier); + + try { + ggfs.rename(SUBDIR, SUBDIR_NEW); + + return true; + } + catch (IgniteCheckedException ignored) { + return false; + } + } + }); + + GridPlainFuture<Boolean> res2 = execute(new Callable<Boolean>() { + @Override public Boolean call() throws Exception { + U.awaitQuiet(barrier); + + try { + ggfs.rename(SUBDIR_NEW, SUBDIR); + + return true; + } + catch (IgniteCheckedException ignored) { + return false; + } + } + }); + + res1.get(); + res2.get(); + + assert res1.get(); // First rename must be successful anyway. + + if (res2.get()) { + checkExist(ggfs, ggfsSecondary, DIR, SUBDIR, DIR_NEW); + checkNotExist(ggfs, ggfsSecondary, SUBDIR_NEW); + } + else { + checkExist(ggfs, ggfsSecondary, DIR, DIR_NEW, SUBDIR_NEW); + checkNotExist(ggfs, ggfsSecondary, SUBDIR); + } + + clear(ggfs, ggfsSecondary); + } + } + + /** + * Ensure that in case we delete the folder A and delete its parent at the same time, resulting file system + * structure is consistent. Initially file system entries are created only in the secondary file system. + * + * @throws Exception If failed. + */ + public void testConcurrentDeletesRemote() throws Exception { + for (int i = 0; i < REPEAT_CNT; i++) { + final CyclicBarrier barrier = new CyclicBarrier(2); + + create(ggfsSecondary, paths(DIR, SUBDIR, SUBSUBDIR), paths()); + + GridPlainFuture<Boolean> res1 = execute(new Callable<Boolean>() { + @Override public Boolean call() throws Exception { + U.awaitQuiet(barrier); + + try { + ggfs.delete(SUBDIR, true); + + return true; + } + catch (IgniteCheckedException ignored) { + return false; + } + } + }); + + GridPlainFuture<Boolean> res2 = execute(new Callable<Boolean>() { + @Override public Boolean call() throws Exception { + U.awaitQuiet(barrier); + + try { + ggfs.delete(SUBSUBDIR, true); + + return true; + } + catch (IgniteCheckedException ignored) { + return false; + } + } + }); + + assert res1.get(); // Delete on the parent must succeed anyway. + res2.get(); + + checkExist(ggfs, ggfsSecondary, DIR); + checkNotExist(ggfs, ggfsSecondary, SUBDIR, SUBSUBDIR); + + clear(ggfs, ggfsSecondary); + } + } +} http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/cfcf46df/modules/core/src/test/java/org/apache/ignite/internal/processors/fs/GridGgfsDualAsyncSelfTest.java ---------------------------------------------------------------------- diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/fs/GridGgfsDualAsyncSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/fs/GridGgfsDualAsyncSelfTest.java new file mode 100644 index 0000000..08ddc3b --- /dev/null +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/fs/GridGgfsDualAsyncSelfTest.java @@ -0,0 +1,32 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ignite.internal.processors.fs; + +import static org.apache.ignite.fs.IgniteFsMode.*; + +/** + * Tests for DUAL_ASYNC mode. + */ +public class GridGgfsDualAsyncSelfTest extends GridGgfsDualAbstractSelfTest { + /** + * Constructor. + */ + public GridGgfsDualAsyncSelfTest() { + super(DUAL_ASYNC); + } +} http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/cfcf46df/modules/core/src/test/java/org/apache/ignite/internal/processors/fs/GridGgfsDualSyncSelfTest.java ---------------------------------------------------------------------- diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/fs/GridGgfsDualSyncSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/fs/GridGgfsDualSyncSelfTest.java new file mode 100644 index 0000000..8200fed --- /dev/null +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/fs/GridGgfsDualSyncSelfTest.java @@ -0,0 +1,32 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ignite.internal.processors.fs; + +import static org.apache.ignite.fs.IgniteFsMode.*; + +/** + * Tests for DUAL_SYNC mode. + */ +public class GridGgfsDualSyncSelfTest extends GridGgfsDualAbstractSelfTest { + /** + * Constructor. + */ + public GridGgfsDualSyncSelfTest() { + super(DUAL_SYNC); + } +}
