http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/cfcf46df/modules/core/src/test/java/org/apache/ignite/internal/processors/fs/GridGgfsFileInfoSelfTest.java ---------------------------------------------------------------------- diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/fs/GridGgfsFileInfoSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/fs/GridGgfsFileInfoSelfTest.java new file mode 100644 index 0000000..09f020c --- /dev/null +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/fs/GridGgfsFileInfoSelfTest.java @@ -0,0 +1,88 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ignite.internal.processors.fs; + +import org.apache.ignite.*; +import org.apache.ignite.marshaller.*; +import org.apache.ignite.marshaller.optimized.*; +import org.apache.ignite.internal.util.typedef.*; +import org.jetbrains.annotations.*; + +import java.io.*; +import java.util.*; +import java.util.concurrent.*; + +/** + * {@link org.apache.ignite.internal.processors.fs.GridGgfsFileInfo} test case. + */ +public class GridGgfsFileInfoSelfTest extends GridGgfsCommonAbstractTest { + /** Marshaller to test {@link Externalizable} interface. */ + private final IgniteMarshaller marshaller = new IgniteOptimizedMarshaller(); + + /** + * Test node info serialization. + * + * @throws Exception If failed. + */ + public void testSerialization() throws Exception { + final int max = Integer.MAX_VALUE; + + multithreaded(new Callable<Object>() { + private final Random rnd = new Random(); + + @SuppressWarnings("deprecation") // Suppress due to default constructor should never be used directly. + @Nullable @Override public Object call() throws IgniteCheckedException { + for (int i = 0; i < 10000; i++) { + testSerialization(new GridGgfsFileInfo()); + testSerialization(new GridGgfsFileInfo()); + testSerialization(new GridGgfsFileInfo(true, null)); + testSerialization(new GridGgfsFileInfo(false, null)); + + GridGgfsFileInfo rndInfo = new GridGgfsFileInfo(rnd.nextInt(max), null, false, null); + + testSerialization(rndInfo); + testSerialization(new GridGgfsFileInfo(rndInfo, rnd.nextInt(max))); + testSerialization(new GridGgfsFileInfo(rndInfo, F.asMap("desc", String.valueOf(rnd.nextLong())))); + } + + return null; + } + }, 20); + } + + /** + * Test node info serialization. + * + * @param info Node info to test serialization for. + * @throws IgniteCheckedException If failed. + */ + public void testSerialization(GridGgfsFileInfo info) throws IgniteCheckedException { + assertEquals(info, mu(info)); + } + + /** + * Marshal/unmarshal object. + * + * @param obj Object to marshal/unmarshal. + * @return Marshalled and then unmarshalled object. + * @throws IgniteCheckedException In case of any marshalling exception. + */ + private <T> T mu(T obj) throws IgniteCheckedException { + return marshaller.unmarshal(marshaller.marshal(obj), null); + } +}
http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/cfcf46df/modules/core/src/test/java/org/apache/ignite/internal/processors/fs/GridGgfsFileMapSelfTest.java ---------------------------------------------------------------------- diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/fs/GridGgfsFileMapSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/fs/GridGgfsFileMapSelfTest.java new file mode 100644 index 0000000..87bc6e0 --- /dev/null +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/fs/GridGgfsFileMapSelfTest.java @@ -0,0 +1,335 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ignite.internal.processors.fs; + +import org.apache.ignite.lang.*; +import org.apache.ignite.internal.util.typedef.*; +import org.gridgain.testframework.*; + +import java.util.*; +import java.util.concurrent.*; + +/** + * File map self test. + */ +public class GridGgfsFileMapSelfTest extends GridGgfsCommonAbstractTest { + /** + * @throws Exception If failed. + */ + public void testRanges() throws Exception { + GridGgfsFileMap map = new GridGgfsFileMap(); + + IgniteUuid[] affKeys = new IgniteUuid[20]; + + for (int i = 0; i < affKeys.length; i++) + affKeys[i] = IgniteUuid.randomUuid(); + + int numOfRanges = 0; + + do { + for (int i = 0; i < 2 * numOfRanges + 1; i++) { + long off1 = i * 10; + long off2 = i * 10 + 5; + long off3 = i * 10 + 8; + + IgniteUuid affKey = i % 2 == 0 ? null : affKeys[i / 2]; + + assertEquals("For i: " + i, affKey, map.affinityKey(off1, false)); + assertEquals("For i: " + i, affKey, map.affinityKey(off2, false)); + assertEquals("For i: " + i, affKey, map.affinityKey(off3, false)); + } + + map.addRange(new GridGgfsFileAffinityRange(10 + 20 * numOfRanges, 19 + 20 * numOfRanges, + affKeys[numOfRanges])); + + numOfRanges++; + } while (numOfRanges < 20); + } + + /** + * @throws Exception If failed. + */ + public void testAddUpdateAdd() throws Exception { + GridGgfsFileMap map = new GridGgfsFileMap(); + + IgniteUuid affKey = IgniteUuid.randomUuid(); + + map.addRange(new GridGgfsFileAffinityRange(0, 9, affKey)); + + map.updateRangeStatus(new GridGgfsFileAffinityRange(0, 9, affKey), RANGE_STATUS_MOVING); + + map.addRange(new GridGgfsFileAffinityRange(10, 19, affKey)); + + List<GridGgfsFileAffinityRange> ranges = map.ranges(); + + assertEquals(2, ranges.size()); + + assertEquals(RANGE_STATUS_MOVING, ranges.get(0).status()); + assertTrue(ranges.get(0).regionEqual(new GridGgfsFileAffinityRange(0, 9, affKey))); + + assertEquals(RANGE_STATUS_INITIAL, ranges.get(1).status()); + assertTrue(ranges.get(1).regionEqual(new GridGgfsFileAffinityRange(10, 19, affKey))); + } + + /** + * @throws Exception If failed. + */ + public void testRangeUpdate1() throws Exception { + GridGgfsFileMap map = new GridGgfsFileMap(); + + IgniteUuid affKey = IgniteUuid.randomUuid(); + + for (int i = 0; i < 4; i++) + map.addRange(new GridGgfsFileAffinityRange(i * 20 + 10, i * 20 + 19, affKey)); + + // Middle, first, last. + map.updateRangeStatus(new GridGgfsFileAffinityRange(30, 39, affKey), RANGE_STATUS_MOVING); + map.updateRangeStatus(new GridGgfsFileAffinityRange(10, 19, affKey), RANGE_STATUS_MOVING); + map.updateRangeStatus(new GridGgfsFileAffinityRange(70, 79, affKey), RANGE_STATUS_MOVING); + + List<GridGgfsFileAffinityRange> ranges = map.ranges(); + + assertEquals(RANGE_STATUS_MOVING, ranges.get(0).status()); + assertEquals(RANGE_STATUS_MOVING, ranges.get(1).status()); + assertEquals(RANGE_STATUS_INITIAL, ranges.get(2).status()); + assertEquals(RANGE_STATUS_MOVING, ranges.get(3).status()); + + // Middle, first, last. + map.updateRangeStatus(new GridGgfsFileAffinityRange(30, 39, affKey), RANGE_STATUS_MOVED); + map.updateRangeStatus(new GridGgfsFileAffinityRange(10, 19, affKey), RANGE_STATUS_MOVED); + map.updateRangeStatus(new GridGgfsFileAffinityRange(70, 79, affKey), RANGE_STATUS_MOVED); + + ranges = map.ranges(); + + assertEquals(RANGE_STATUS_MOVED, ranges.get(0).status()); + assertEquals(RANGE_STATUS_MOVED, ranges.get(1).status()); + assertEquals(RANGE_STATUS_INITIAL, ranges.get(2).status()); + assertEquals(RANGE_STATUS_MOVED, ranges.get(3).status()); + + // Middle, first, last. + map.deleteRange(new GridGgfsFileAffinityRange(30, 39, affKey)); + map.deleteRange(new GridGgfsFileAffinityRange(10, 19, affKey)); + map.deleteRange(new GridGgfsFileAffinityRange(70, 79, affKey)); + + ranges = map.ranges(); + + assertEquals(1, ranges.size()); + assertEquals(RANGE_STATUS_INITIAL, ranges.get(0).status()); + assertTrue(ranges.get(0).regionEqual(new GridGgfsFileAffinityRange(50, 59, affKey))); + } + + /** + * @throws Exception If failed. + */ + public void testRangeUpdate2() throws Exception { + GridGgfsFileMap map = new GridGgfsFileMap(); + + IgniteUuid affKey = IgniteUuid.randomUuid(); + + for (int i = 0; i < 4; i++) + map.addRange(new GridGgfsFileAffinityRange(i * 20 + 10, i * 20 + 19, affKey)); + + // Middle, first, last. + map.updateRangeStatus(new GridGgfsFileAffinityRange(30, 35, affKey), RANGE_STATUS_MOVING); + map.updateRangeStatus(new GridGgfsFileAffinityRange(10, 15, affKey), RANGE_STATUS_MOVING); + map.updateRangeStatus(new GridGgfsFileAffinityRange(70, 75, affKey), RANGE_STATUS_MOVING); + + List<GridGgfsFileAffinityRange> ranges = map.ranges(); + + assertEquals(7, ranges.size()); + + int idx = 0; + assertTrue(ranges.get(idx).regionEqual(new GridGgfsFileAffinityRange(10, 15, affKey))); + assertEquals(RANGE_STATUS_MOVING, ranges.get(idx).status()); + idx++; + + assertTrue(ranges.get(idx).regionEqual(new GridGgfsFileAffinityRange(16, 19, affKey))); + assertEquals(RANGE_STATUS_INITIAL, ranges.get(idx).status()); + idx++; + + assertTrue(ranges.get(idx).regionEqual(new GridGgfsFileAffinityRange(30, 35, affKey))); + assertEquals(RANGE_STATUS_MOVING, ranges.get(idx).status()); + idx++; + + assertTrue(ranges.get(idx).regionEqual(new GridGgfsFileAffinityRange(36, 39, affKey))); + assertEquals(RANGE_STATUS_INITIAL, ranges.get(idx).status()); + idx++; + + assertTrue(ranges.get(idx).regionEqual(new GridGgfsFileAffinityRange(50, 59, affKey))); + assertEquals(RANGE_STATUS_INITIAL, ranges.get(idx).status()); + idx++; + + assertTrue(ranges.get(idx).regionEqual(new GridGgfsFileAffinityRange(70, 75, affKey))); + assertEquals(RANGE_STATUS_MOVING, ranges.get(idx).status()); + idx++; + + assertTrue(ranges.get(idx).regionEqual(new GridGgfsFileAffinityRange(76, 79, affKey))); + assertEquals(RANGE_STATUS_INITIAL, ranges.get(idx).status()); + + // Middle, first, last. + map.updateRangeStatus(new GridGgfsFileAffinityRange(30, 35, affKey), RANGE_STATUS_MOVED); + map.updateRangeStatus(new GridGgfsFileAffinityRange(10, 15, affKey), RANGE_STATUS_MOVED); + map.updateRangeStatus(new GridGgfsFileAffinityRange(70, 75, affKey), RANGE_STATUS_MOVED); + + idx = 0; + assertTrue(ranges.get(idx).regionEqual(new GridGgfsFileAffinityRange(10, 15, affKey))); + assertEquals(RANGE_STATUS_MOVED, ranges.get(idx).status()); + idx++; + + assertTrue(ranges.get(idx).regionEqual(new GridGgfsFileAffinityRange(16, 19, affKey))); + assertEquals(RANGE_STATUS_INITIAL, ranges.get(idx).status()); + idx++; + + assertTrue(ranges.get(idx).regionEqual(new GridGgfsFileAffinityRange(30, 35, affKey))); + assertEquals(RANGE_STATUS_MOVED, ranges.get(idx).status()); + idx++; + + assertTrue(ranges.get(idx).regionEqual(new GridGgfsFileAffinityRange(36, 39, affKey))); + assertEquals(RANGE_STATUS_INITIAL, ranges.get(idx).status()); + idx++; + + assertTrue(ranges.get(idx).regionEqual(new GridGgfsFileAffinityRange(50, 59, affKey))); + assertEquals(RANGE_STATUS_INITIAL, ranges.get(idx).status()); + idx++; + + assertTrue(ranges.get(idx).regionEqual(new GridGgfsFileAffinityRange(70, 75, affKey))); + assertEquals(RANGE_STATUS_MOVED, ranges.get(idx).status()); + idx++; + + assertTrue(ranges.get(idx).regionEqual(new GridGgfsFileAffinityRange(76, 79, affKey))); + assertEquals(RANGE_STATUS_INITIAL, ranges.get(idx).status()); + + // Middle, first, last. + map.deleteRange(new GridGgfsFileAffinityRange(30, 35, affKey)); + map.deleteRange(new GridGgfsFileAffinityRange(10, 15, affKey)); + map.deleteRange(new GridGgfsFileAffinityRange(70, 75, affKey)); + + ranges = map.ranges(); + + assertEquals(4, ranges.size()); + + assertEquals(RANGE_STATUS_INITIAL, ranges.get(0).status()); + assertTrue(ranges.get(0).regionEqual(new GridGgfsFileAffinityRange(16, 19, affKey))); + + assertEquals(RANGE_STATUS_INITIAL, ranges.get(1).status()); + assertTrue(ranges.get(1).regionEqual(new GridGgfsFileAffinityRange(36, 39, affKey))); + + assertEquals(RANGE_STATUS_INITIAL, ranges.get(2).status()); + assertTrue(ranges.get(2).regionEqual(new GridGgfsFileAffinityRange(50, 59, affKey))); + + assertEquals(RANGE_STATUS_INITIAL, ranges.get(3).status()); + assertTrue(ranges.get(3).regionEqual(new GridGgfsFileAffinityRange(76, 79, affKey))); + } + + /** + * @throws Exception If failed. + */ + public void testInvalidRangeUpdates() throws Exception { + final GridGgfsFileMap map = new GridGgfsFileMap(); + + final IgniteUuid affKey1 = IgniteUuid.randomUuid(); + final IgniteUuid affKey2 = IgniteUuid.randomUuid(); + + map.addRange(new GridGgfsFileAffinityRange(10, 19, affKey1)); + map.addRange(new GridGgfsFileAffinityRange(30, 39, affKey1)); + + GridTestUtils.assertThrows(log, new Callable<Object>() { + @Override public Object call() throws Exception { + map.updateRangeStatus(new GridGgfsFileAffinityRange(0, 5, affKey1), RANGE_STATUS_MOVING); + + return null; + } + }, GridGgfsInvalidRangeException.class, null); + + GridTestUtils.assertThrows(log, new Callable<Object>() { + @Override public Object call() throws Exception { + map.updateRangeStatus(new GridGgfsFileAffinityRange(15, 19, affKey1), RANGE_STATUS_MOVING); + + return null; + } + }, GridGgfsInvalidRangeException.class, null); + + GridTestUtils.assertThrows(log, new Callable<Object>() { + @Override public Object call() throws Exception { + map.updateRangeStatus(new GridGgfsFileAffinityRange(10, 19, affKey2), RANGE_STATUS_MOVING); + + return null; + } + }, AssertionError.class, null); + + GridTestUtils.assertThrows(log, new Callable<Object>() { + @Override public Object call() throws Exception { + map.updateRangeStatus(new GridGgfsFileAffinityRange(10, 22, affKey1), RANGE_STATUS_MOVING); + + return null; + } + }, AssertionError.class, null); + + assertEquals(2, map.ranges().size()); + } + + /** + * @throws Exception If failed. + */ + public void testRangeSplit() throws Exception { + IgniteUuid affKey = IgniteUuid.randomUuid(); + + GridGgfsFileAffinityRange range = new GridGgfsFileAffinityRange(0, 9999, affKey); + + Collection<GridGgfsFileAffinityRange> split = range.split(10000); + + assertEquals(1, split.size()); + assertTrue(range.regionEqual(F.first(split))); + + split = range.split(5000); + + assertEquals(2, split.size()); + + Iterator<GridGgfsFileAffinityRange> it = split.iterator(); + + GridGgfsFileAffinityRange part = it.next(); + + assertTrue(part.regionEqual(new GridGgfsFileAffinityRange(0, 4999, affKey))); + + part = it.next(); + + assertTrue(part.regionEqual(new GridGgfsFileAffinityRange(5000, 9999, affKey))); + + split = range.split(3000); + + assertEquals(4, split.size()); + + it = split.iterator(); + + part = it.next(); + + assertTrue(part.regionEqual(new GridGgfsFileAffinityRange(0, 2999, affKey))); + + part = it.next(); + + assertTrue(part.regionEqual(new GridGgfsFileAffinityRange(3000, 5999, affKey))); + + part = it.next(); + + assertTrue(part.regionEqual(new GridGgfsFileAffinityRange(6000, 8999, affKey))); + + part = it.next(); + + assertTrue(part.regionEqual(new GridGgfsFileAffinityRange(9000, 9999, affKey))); + } +} http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/cfcf46df/modules/core/src/test/java/org/apache/ignite/internal/processors/fs/GridGgfsGroupDataBlockKeyMapperHashSelfTest.java ---------------------------------------------------------------------- diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/fs/GridGgfsGroupDataBlockKeyMapperHashSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/fs/GridGgfsGroupDataBlockKeyMapperHashSelfTest.java new file mode 100644 index 0000000..4edc602 --- /dev/null +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/fs/GridGgfsGroupDataBlockKeyMapperHashSelfTest.java @@ -0,0 +1,136 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ignite.internal.processors.fs; + +import org.apache.ignite.fs.*; +import org.apache.ignite.lang.*; +import org.apache.ignite.internal.util.typedef.internal.*; + +import java.util.concurrent.*; + +/** + * Tests for {@link org.apache.ignite.fs.IgniteFsGroupDataBlocksKeyMapper} hash. + */ +public class GridGgfsGroupDataBlockKeyMapperHashSelfTest extends GridGgfsCommonAbstractTest { + /** + * @throws Exception If failed. + */ + public void testDistribution() throws Exception { + for (int i = 0; i < 100; i++) { + int grpSize = ThreadLocalRandom.current().nextInt(2, 100000); + int partCnt = ThreadLocalRandom.current().nextInt(1, grpSize); + + checkDistribution(grpSize, partCnt); + } + } + + /** + * @throws Exception If failed. + */ + public void testIntOverflowDistribution() throws Exception { + for (int i = 0; i < 100; i++) + checkIntOverflowDistribution(ThreadLocalRandom.current().nextInt(1, Integer.MAX_VALUE)); + } + + /** + * @param mapper GGFS blocks mapper. + * @param fileId GGFS file ID. + * @param blockId File block ID. + * @param partCnt Total partitions count. + * @return Partition index. + */ + private int partition(IgniteFsGroupDataBlocksKeyMapper mapper, IgniteUuid fileId, long blockId, int partCnt) { + return U.safeAbs((Integer) mapper.affinityKey(new GridGgfsBlockKey(fileId, null, false, blockId)) % partCnt); + } + + /** + * Check hash code generation for the given group size and partitions count. + * + * @throws Exception If failed. + */ + public void checkDistribution(int grpSize, int partCnt) throws Exception { + IgniteUuid fileId = IgniteUuid.randomUuid(); + + IgniteFsGroupDataBlocksKeyMapper mapper = new IgniteFsGroupDataBlocksKeyMapper(grpSize); + + int lastPart = 0; + + boolean first = true; + + for (int i = 0; i < 10; i++) { + // Ensure that all blocks within the group has the same hash codes. + boolean firstInGroup = true; + + for (int j = 0; j < grpSize; j++) { + int part = partition(mapper, fileId, i * grpSize + j, partCnt); + + if (firstInGroup) { + if (first) + first = false; + else + assert checkPartition(lastPart, part, partCnt) : + "[fileId = " + fileId + ", i=" + i + ", j=" + j + ", grpSize= " + grpSize + + ", partCnt=" + partCnt + ", lastPart=" + lastPart + ", part=" + part + ']'; + + firstInGroup = false; + } + else + assert part == lastPart; + + lastPart = part; + } + } + } + + /** + * Check distribution for integer overflow. + * + * @throws Exception If failed. + */ + @SuppressWarnings("NumericOverflow") + public void checkIntOverflowDistribution(int partCnt) throws Exception { + IgniteUuid fileId = IgniteUuid.randomUuid(); + + IgniteFsGroupDataBlocksKeyMapper mapper = new IgniteFsGroupDataBlocksKeyMapper(1); + + int part1 = partition(mapper, fileId, Integer.MAX_VALUE - 1, partCnt); + int part2 = partition(mapper, fileId, Integer.MAX_VALUE, partCnt); + int part3 = partition(mapper, fileId, (long)Integer.MAX_VALUE + 1, partCnt); + + assert checkPartition(part1, part2, partCnt) : + "[fileId = " + fileId + "part1=" + part1 + ", part2=" + part2 + ", partCnt=" + partCnt + ']'; + + assert checkPartition(part2, part3, partCnt) : + "[fileId = " + fileId + "part1=" + part2 + ", part3=" + part3 + ", partCnt=" + partCnt + ']'; + } + + /** + * Check correct partition shift. + * + * @param prevPart Previous partition. + * @param part Current partition. + * @param totalParts Total partitions. + * @return {@code true} if previous and current partitions have correct values. + */ + private boolean checkPartition(int prevPart, int part, int totalParts) { + return U.safeAbs(prevPart - part) == 1 || + (part == 0 && prevPart == totalParts - 1) || + (prevPart == 0 && part == totalParts - 1); + } +} + http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/cfcf46df/modules/core/src/test/java/org/apache/ignite/internal/processors/fs/GridGgfsMetaManagerSelfTest.java ---------------------------------------------------------------------- diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/fs/GridGgfsMetaManagerSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/fs/GridGgfsMetaManagerSelfTest.java new file mode 100644 index 0000000..33788e9 --- /dev/null +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/fs/GridGgfsMetaManagerSelfTest.java @@ -0,0 +1,466 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ignite.internal.processors.fs; + +import org.apache.ignite.cache.*; +import org.apache.ignite.configuration.*; +import org.apache.ignite.fs.*; +import org.apache.ignite.lang.*; +import org.apache.ignite.spi.discovery.tcp.*; +import org.apache.ignite.spi.discovery.tcp.ipfinder.*; +import org.apache.ignite.spi.discovery.tcp.ipfinder.vm.*; +import org.apache.ignite.internal.util.typedef.*; +import org.gridgain.testframework.*; +import org.jetbrains.annotations.*; + +import java.util.*; +import java.util.concurrent.*; + +import static org.apache.ignite.cache.GridCacheAtomicityMode.*; +import static org.apache.ignite.cache.GridCacheDistributionMode.*; +import static org.apache.ignite.cache.GridCacheMode.*; +import static org.apache.ignite.cache.GridCacheWriteSynchronizationMode.*; + +/** + * {@link GridGgfsMetaManager} test case. + */ +public class GridGgfsMetaManagerSelfTest extends GridGgfsCommonAbstractTest { + /** Test IP finder. */ + private static final TcpDiscoveryIpFinder IP_FINDER = new TcpDiscoveryVmIpFinder(true); + + /** Meta-information cache name. */ + private static final String META_CACHE_NAME = "replicated"; + + /** Data cache name. */ + public static final String DATA_CACHE_NAME = "data"; + + /** Test nodes count. */ + private static final int NODES_CNT = 4; + + /** Meta manager to test. */ + private GridGgfsMetaManager mgr; + + /** {@inheritDoc} */ + @Override protected void beforeTest() throws Exception { + GridGgfsEx ggfs = (GridGgfsEx)grid(0).fileSystem("ggfs"); + + mgr = ggfs.context().meta(); + } + + /** {@inheritDoc} */ + @Override protected IgniteConfiguration getConfiguration(String gridName) throws Exception { + IgniteConfiguration cfg = super.getConfiguration(gridName); + + cfg.setCacheConfiguration(cacheConfiguration(META_CACHE_NAME), cacheConfiguration(DATA_CACHE_NAME)); + + TcpDiscoverySpi discoSpi = new TcpDiscoverySpi(); + + discoSpi.setIpFinder(IP_FINDER); + + cfg.setDiscoverySpi(discoSpi); + + IgniteFsConfiguration ggfsCfg = new IgniteFsConfiguration(); + + ggfsCfg.setMetaCacheName(META_CACHE_NAME); + ggfsCfg.setDataCacheName(DATA_CACHE_NAME); + ggfsCfg.setName("ggfs"); + + cfg.setGgfsConfiguration(ggfsCfg); + + return cfg; + } + + /** {@inheritDoc} */ + protected CacheConfiguration cacheConfiguration(String cacheName) { + CacheConfiguration cacheCfg = defaultCacheConfiguration(); + + cacheCfg.setName(cacheName); + + if (META_CACHE_NAME.equals(cacheName)) + cacheCfg.setCacheMode(REPLICATED); + else { + cacheCfg.setCacheMode(PARTITIONED); + cacheCfg.setDistributionMode(PARTITIONED_ONLY); + + cacheCfg.setBackups(0); + cacheCfg.setAffinityMapper(new IgniteFsGroupDataBlocksKeyMapper(128)); + } + + cacheCfg.setQueryIndexEnabled(false); + cacheCfg.setWriteSynchronizationMode(FULL_SYNC); + cacheCfg.setAtomicityMode(TRANSACTIONAL); + + return cacheCfg; + } + + /** {@inheritDoc} */ + @Override protected void afterTest() throws Exception { + mgr.ggfsCtx.ggfs().format(); + } + + /** {@inheritDoc} */ + @Override protected void beforeTestsStarted() throws Exception { + startGrids(NODES_CNT); + } + + /** {@inheritDoc} */ + @Override protected void afterTestsStopped() throws Exception { + stopAllGrids(); + } + + /** + * Test properties management in meta-cache. + * + * @throws Exception If failed. + */ + @SuppressWarnings("NullableProblems") + public void testUpdateProperties() throws Exception { + assertEmpty(mgr.directoryListing(ROOT_ID)); + + GridGgfsFileInfo dir = new GridGgfsFileInfo(true, null); + GridGgfsFileInfo file = new GridGgfsFileInfo(new GridGgfsFileInfo(400, null, false, null), 1); + + assertNull(mgr.putIfAbsent(ROOT_ID, "dir", dir)); + assertNull(mgr.putIfAbsent(ROOT_ID, "file", file)); + + assertEquals(F.asMap("dir", new GridGgfsListingEntry(dir), "file", new GridGgfsListingEntry(file)), + mgr.directoryListing(ROOT_ID)); + + //GridGgfsFileInfo tmp = mgr.info(dir.id()); + + assertEquals(dir, mgr.info(dir.id())); + assertEquals(file, mgr.info(file.id())); + + for (IgniteBiTuple<IgniteUuid, String> tup: Arrays.asList(F.t(dir.id(), "dir"), F.t(file.id(), "file"))) { + IgniteUuid fileId = tup.get1(); + String fileName = tup.get2(); + + for (Map<String, String> props : Arrays.asList(null, Collections.<String, String>emptyMap())) + expectsUpdatePropertiesFail(fileId, props, AssertionError.class, "Expects not-empty file's properties"); + + String key1 = UUID.randomUUID().toString(); + String key2 = UUID.randomUUID().toString(); + + GridGgfsFileInfo info = mgr.info(fileId); + + assertNull("Expects empty properties are not stored: " + info, getFieldValue(info, "props")); + assertEquals("Expects empty properties are not stored: " + info, Collections.<String, String>emptyMap(), + info.properties()); + + info = mgr.updateProperties(ROOT_ID, fileId, fileName, F.asMap(key1, "1")); + + assertEquals("Unexpected stored properties: " + info, F.asMap(key1, "1"), info.properties()); + + info = mgr.updateProperties(ROOT_ID, fileId, fileName, F.asMap(key2, "2")); + + assertEquals("Unexpected stored properties: " + info, F.asMap(key1, "1", key2, "2"), info.properties()); + + info = mgr.updateProperties(ROOT_ID, fileId, fileName, F.<String, String>asMap(key1, null)); + + assertEquals("Unexpected stored properties: " + info, F.asMap(key2, "2"), info.properties()); + + info = mgr.updateProperties(ROOT_ID, fileId, fileName, F.<String, String>asMap(key2, null)); + + assertNull("Expects empty properties are not stored: " + info, getFieldValue(info, "props")); + assertEquals("Expects empty properties are not stored: " + info, Collections.<String, String>emptyMap(), + info.properties()); + + assertNull(mgr.updateProperties(ROOT_ID, fileId, "not_exists", F.<String, String>asMap(key2, null))); + } + + mgr.removeIfEmpty(ROOT_ID, "dir", dir.id(), new IgniteFsPath("/dir"), true); + mgr.removeIfEmpty(ROOT_ID, "file", file.id(), new IgniteFsPath("/file"), true); + + assertNull(mgr.updateProperties(ROOT_ID, dir.id(), "dir", F.asMap("p", "7"))); + assertNull(mgr.updateProperties(ROOT_ID, file.id(), "file", F.asMap("q", "8"))); + } + + /** + * Test file system structure in meta-cache. + * + * @throws Exception If failed. + */ + public void testStructure() throws Exception { + GridGgfsFileInfo rootInfo = new GridGgfsFileInfo(); + // Test empty structure. + assertEmpty(mgr.directoryListing(ROOT_ID)); + assertEquals(rootInfo, mgr.info(ROOT_ID)); + assertEquals(F.asMap(ROOT_ID, rootInfo), mgr.infos(Arrays.asList(ROOT_ID))); + + GridGgfsFileInfo a = new GridGgfsFileInfo(true, null); + GridGgfsFileInfo b = new GridGgfsFileInfo(true, null); + GridGgfsFileInfo f1 = new GridGgfsFileInfo(400, null, false, null); + GridGgfsFileInfo f2 = new GridGgfsFileInfo(new GridGgfsFileInfo(400, null, false, null), 0); + GridGgfsFileInfo f3 = new GridGgfsFileInfo(new GridGgfsFileInfo(400, null, false, null), 200000L); + + // Validate 'add file' operation. + assertNull(mgr.putIfAbsent(ROOT_ID, "a", a)); + assertNull(mgr.putIfAbsent(ROOT_ID, "f1", f1)); + assertNull(mgr.putIfAbsent(a.id(), "b", b)); + assertNull(mgr.putIfAbsent(a.id(), "f2", f2)); + assertNull(mgr.putIfAbsent(b.id(), "f3", f3)); + + assertEquals(b.id(), mgr.putIfAbsent(a.id(), "b", f3)); + expectsPutIfAbsentFail(a.id(), "c", f3, "Failed to add file details into cache"); + + assertEquals(F.asMap("a", new GridGgfsListingEntry(a), "f1", new GridGgfsListingEntry(f1)), + mgr.directoryListing(ROOT_ID)); + + assertEquals(F.asMap("b", new GridGgfsListingEntry(b), "f2", new GridGgfsListingEntry(f2)), + mgr.directoryListing(a.id())); + + assertEquals(F.asMap("f3", new GridGgfsListingEntry(f3)), mgr.directoryListing(b.id())); + + // Validate empty files listings. + for (GridGgfsFileInfo info : Arrays.asList(f1, f2, f3)) { + assertEmpty(mgr.directoryListing(info.id())); + } + + // Validate 'file info' operations. + for (GridGgfsFileInfo info : Arrays.asList(rootInfo, a, b, f1, f2, f3)) { + assertEquals(info, mgr.info(info.id())); + assertEquals(F.asMap(info.id(), info), mgr.infos(Arrays.asList(info.id()))); + } + + // Validate 'file ID' operations. + assertEquals(ROOT_ID, mgr.fileId(new IgniteFsPath("/"))); + assertEquals(a.id(), mgr.fileId(new IgniteFsPath("/a"))); + assertEquals(b.id(), mgr.fileId(new IgniteFsPath("/a/b"))); + assertEquals(f1.id(), mgr.fileId(new IgniteFsPath("/f1"))); + assertEquals(f2.id(), mgr.fileId(new IgniteFsPath("/a/f2"))); + assertEquals(f3.id(), mgr.fileId(new IgniteFsPath("/a/b/f3"))); + assertNull(mgr.fileId(new IgniteFsPath("/f4"))); + assertNull(mgr.fileId(new IgniteFsPath("/a/f5"))); + assertNull(mgr.fileId(new IgniteFsPath("/a/b/f6"))); + + assertEquals(a.id(), mgr.fileId(ROOT_ID, "a")); + assertEquals(b.id(), mgr.fileId(a.id(), "b")); + assertEquals(f1.id(), mgr.fileId(ROOT_ID, "f1")); + assertEquals(f2.id(), mgr.fileId(a.id(), "f2")); + assertEquals(f3.id(), mgr.fileId(b.id(), "f3")); + assertNull(mgr.fileId(ROOT_ID, "f4")); + assertNull(mgr.fileId(a.id(), "f5")); + assertNull(mgr.fileId(b.id(), "f6")); + + assertEquals(Arrays.asList(ROOT_ID), mgr.fileIds(new IgniteFsPath("/"))); + assertEquals(Arrays.asList(ROOT_ID, a.id()), mgr.fileIds(new IgniteFsPath("/a"))); + assertEquals(Arrays.asList(ROOT_ID, a.id(), b.id()), mgr.fileIds(new IgniteFsPath("/a/b"))); + assertEquals(Arrays.asList(ROOT_ID, f1.id()), mgr.fileIds(new IgniteFsPath("/f1"))); + assertEquals(Arrays.asList(ROOT_ID, a.id(), f2.id()), mgr.fileIds(new IgniteFsPath("/a/f2"))); + assertEquals(Arrays.asList(ROOT_ID, a.id(), b.id(), f3.id()), mgr.fileIds(new IgniteFsPath("/a/b/f3"))); + assertEquals(Arrays.asList(ROOT_ID, null), mgr.fileIds(new IgniteFsPath("/f4"))); + assertEquals(Arrays.asList(ROOT_ID, a.id(), null), mgr.fileIds(new IgniteFsPath("/a/f5"))); + assertEquals(Arrays.asList(ROOT_ID, a.id(), b.id(), null), mgr.fileIds(new IgniteFsPath("/a/b/f6"))); + assertEquals(Arrays.asList(ROOT_ID, null, null, null, null), mgr.fileIds(new IgniteFsPath("/f7/a/b/f6"))); + + // Validate 'rename' operation. + final IgniteUuid rndId = IgniteUuid.randomUuid(); + + // One of participated files does not exist in cache. + expectsRenameFail(ROOT_ID, "b", rndId, "b2", rndId, "Failed to lock source directory (not found?)"); + expectsRenameFail(b.id(), "b", rndId, "b2", rndId, "Failed to lock source directory (not found?)"); + expectsRenameFail(ROOT_ID, "b", ROOT_ID, "b2", rndId, "Failed to lock destination directory (not found?)"); + expectsRenameFail(b.id(), "b", ROOT_ID, "b2", rndId, "Failed to lock destination directory (not found?)"); + expectsRenameFail(rndId, "b", ROOT_ID, "b2", ROOT_ID, "Failed to lock target file (not found?)"); + expectsRenameFail(rndId, "b", b.id(), "b2", b.id(), "Failed to lock target file (not found?)"); + + // Target file ID differ from the file ID resolved from the source directory for source file name. + expectsRenameFail(b.id(), "a", ROOT_ID, "q", ROOT_ID, "Failed to remove file name from the source directory"); + expectsRenameFail(f1.id(), "a", ROOT_ID, "q", ROOT_ID, "Failed to remove file name from the source directory"); + expectsRenameFail(f2.id(), "a", ROOT_ID, "q", ROOT_ID, "Failed to remove file name from the source directory"); + expectsRenameFail(f3.id(), "a", ROOT_ID, "q", ROOT_ID, "Failed to remove file name from the source directory"); + + // Invalid source file name (not found). + expectsRenameFail(a.id(), "u1", ROOT_ID, "q", ROOT_ID, "Failed to remove file name from the source"); + expectsRenameFail(a.id(), "u2", ROOT_ID, "q", ROOT_ID, "Failed to remove file name from the source"); + expectsRenameFail(a.id(), "u3", ROOT_ID, "q", ROOT_ID, "Failed to remove file name from the source"); + + // Invalid destination file - already exists. + expectsRenameFail(a.id(), "a", ROOT_ID, "f1", ROOT_ID, "Failed to add file name into the destination"); + expectsRenameFail(f2.id(), "f2", a.id(), "f1", ROOT_ID, "Failed to add file name into the destination"); + expectsRenameFail(f3.id(), "f3", b.id(), "f1", ROOT_ID, "Failed to add file name into the destination"); + expectsRenameFail(b.id(), "b", a.id(), "f2", a.id(), "Failed to add file name into the destination"); + + System.out.println("/: " + mgr.directoryListing(ROOT_ID)); + System.out.println("a: " + mgr.directoryListing(a.id())); + System.out.println("b: " + mgr.directoryListing(b.id())); + System.out.println("f3: " + mgr.directoryListing(f3.id())); + + mgr.move(a.id(), "a", ROOT_ID, "a2", ROOT_ID); + mgr.move(b.id(), "b", a.id(), "b2", a.id()); + + assertNotNull(mgr.info(b.id())); + + mgr.move(f3.id(), "f3", b.id(), "f3-2", a.id()); + + assertNotNull(mgr.info(b.id())); + + mgr.move(f3.id(), "f3-2", a.id(), "f3", b.id()); + mgr.move(b.id(), "b2", a.id(), "b", a.id()); + mgr.move(a.id(), "a2", ROOT_ID, "a", ROOT_ID); + + // Validate 'remove' operation. + for (int i = 0; i < 100; i++) { + // One of participants doesn't exist. + assertNull(mgr.removeIfEmpty(ROOT_ID, "a", IgniteUuid.randomUuid(), new IgniteFsPath("/a"), true)); + assertNull(mgr.removeIfEmpty(IgniteUuid.randomUuid(), "a", IgniteUuid.randomUuid(), + new IgniteFsPath("/" + IgniteUuid.randomUuid() + "/a"), true)); + } + + expectsRemoveFail(ROOT_ID, "a", a.id(), new IgniteFsPath("/a"), + "Failed to remove file (directory is not empty)"); + expectsRemoveFail(a.id(), "b", b.id(), new IgniteFsPath("/a/b"), + "Failed to remove file (directory is not empty)"); + assertNull(mgr.removeIfEmpty(ROOT_ID, "a", f1.id(), new IgniteFsPath("/a"), true)); + assertNull(mgr.removeIfEmpty(a.id(), "b", f1.id(), new IgniteFsPath("/a/b"), true)); + + assertEquals(f3, mgr.removeIfEmpty(b.id(), "f3", f3.id(), new IgniteFsPath("/a/b/f3"), true)); + + assertEquals(F.asMap("a", new GridGgfsListingEntry(a), "f1", new GridGgfsListingEntry(f1)), + mgr.directoryListing(ROOT_ID)); + + assertEquals(F.asMap("b", new GridGgfsListingEntry(b), "f2", new GridGgfsListingEntry(f2)), + mgr.directoryListing(a.id())); + + assertEmpty(mgr.directoryListing(b.id())); + + assertEquals(b, mgr.removeIfEmpty(a.id(), "b", b.id(), new IgniteFsPath("/a/b"), true)); + + assertEquals(F.asMap("a", new GridGgfsListingEntry(a), "f1", new GridGgfsListingEntry(f1)), + mgr.directoryListing(ROOT_ID)); + + assertEquals(F.asMap("f2", new GridGgfsListingEntry(f2)), mgr.directoryListing(a.id())); + + assertEmpty(mgr.directoryListing(b.id())); + + // Validate last actual data received from 'remove' operation. + GridGgfsFileInfo newF2 = mgr.updateInfo(f2.id(), new C1<GridGgfsFileInfo, GridGgfsFileInfo>() { + @Override public GridGgfsFileInfo apply(GridGgfsFileInfo e) { + return new GridGgfsFileInfo(e, e.length() + 20); + } + }); + + assertNotNull(newF2); + assertEquals(f2.id(), newF2.id()); + assertNotSame(f2, newF2); + + assertEquals(newF2, mgr.removeIfEmpty(a.id(), "f2", f2.id(), new IgniteFsPath("/a/f2"), true)); + + assertEquals(F.asMap("a", new GridGgfsListingEntry(a), "f1", new GridGgfsListingEntry(f1)), + mgr.directoryListing(ROOT_ID)); + + assertEmpty(mgr.directoryListing(a.id())); + assertEmpty(mgr.directoryListing(b.id())); + + assertEquals(f1, mgr.removeIfEmpty(ROOT_ID, "f1", f1.id(), new IgniteFsPath("/f1"), true)); + + assertEquals(F.asMap("a", new GridGgfsListingEntry(a)), mgr.directoryListing(ROOT_ID)); + + assertEmpty(mgr.directoryListing(a.id())); + assertEmpty(mgr.directoryListing(b.id())); + + assertEquals(a, mgr.removeIfEmpty(ROOT_ID, "a", a.id(), new IgniteFsPath("/a"), true)); + + assertEmpty(mgr.directoryListing(ROOT_ID)); + assertEmpty(mgr.directoryListing(a.id())); + assertEmpty(mgr.directoryListing(b.id())); + } + + /** + * Validate passed map is empty. + * + * @param map Map to validate it is empty. + */ + private void assertEmpty(Map map) { + assertEquals(Collections.emptyMap(), map); + } + + /** + * Test expected failures for 'update properties' operation. + * + * @param fileId File ID. + * @param props File properties to set. + * @param msg Failure message if expected exception was not thrown. + */ + private void expectsUpdatePropertiesFail(@Nullable final IgniteUuid fileId, @Nullable final Map<String, String> props, + Class<? extends Throwable> cls, @Nullable String msg) { + GridTestUtils.assertThrows(log, new Callable() { + @Override public Object call() throws Exception { + return mgr.updateProperties(null, fileId, "file", props); + } + }, cls, msg); + } + + /** + * Test expected failures for 'add file' operation. + * + * @param parentId Parent file ID. + * @param fileName New file name in the parent's listing. + * @param fileInfo New file initial details. + * @param msg Failure message if expected exception was not thrown. + */ + private void expectsPutIfAbsentFail(final IgniteUuid parentId, final String fileName, final GridGgfsFileInfo fileInfo, + @Nullable String msg) { + GridTestUtils.assertThrows(log, new Callable() { + @Override public Object call() throws Exception { + return mgr.putIfAbsent(parentId, fileName, fileInfo); + } + }, IgniteFsException.class, msg); + } + + /** + * Test expected failures for 'move file' operation. + * + * @param fileId File ID to rename. + * @param srcFileName Original file name in the parent's listing. + * @param srcParentId Source parent directory ID. + * @param destFileName New file name in the parent's listing after renaming. + * @param destParentId Destination parent directory ID. + * @param msg Failure message if expected exception was not thrown. + */ + private void expectsRenameFail(final IgniteUuid fileId, final String srcFileName, final IgniteUuid srcParentId, + final String destFileName, final IgniteUuid destParentId, @Nullable String msg) { + GridTestUtils.assertThrowsInherited(log, new Callable() { + @Override public Object call() throws Exception { + mgr.move(fileId, srcFileName, srcParentId, destFileName, destParentId); + + return null; + } + }, IgniteFsException.class, msg); + } + + /** + * Test expected failures for 'remove file' operation. + * + * @param parentId Parent file ID to remove file from. + * @param fileName File name in the parent's listing. + * @param fileId File ID to remove. + * @param path Removed file path. + * @param msg Failure message if expected exception was not thrown. + */ + private void expectsRemoveFail(final IgniteUuid parentId, final String fileName, final IgniteUuid fileId, + final IgniteFsPath path, @Nullable String msg) { + assertThrows(log, new Callable() { + @Nullable @Override public Object call() throws Exception { + mgr.removeIfEmpty(parentId, fileName, fileId, path, true); + + return null; + } + }, GridGgfsDirectoryNotEmptyException.class, msg); + } +} http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/cfcf46df/modules/core/src/test/java/org/apache/ignite/internal/processors/fs/GridGgfsMetricsSelfTest.java ---------------------------------------------------------------------- diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/fs/GridGgfsMetricsSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/fs/GridGgfsMetricsSelfTest.java new file mode 100644 index 0000000..d61f6a7 --- /dev/null +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/fs/GridGgfsMetricsSelfTest.java @@ -0,0 +1,536 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ignite.internal.processors.fs; + +import org.apache.ignite.*; +import org.apache.ignite.cache.*; +import org.apache.ignite.configuration.*; +import org.apache.ignite.fs.*; +import org.apache.ignite.spi.discovery.tcp.*; +import org.apache.ignite.spi.discovery.tcp.ipfinder.*; +import org.apache.ignite.spi.discovery.tcp.ipfinder.vm.*; +import org.apache.ignite.internal.util.typedef.*; +import org.apache.ignite.internal.util.typedef.internal.U; + +import java.util.*; + +import static org.apache.ignite.cache.GridCacheAtomicityMode.*; +import static org.apache.ignite.cache.GridCacheMode.*; +import static org.apache.ignite.fs.IgniteFsMode.*; + +/** + * Test for GGFS metrics. + */ +public class GridGgfsMetricsSelfTest extends GridGgfsCommonAbstractTest { + /** Primary GGFS name. */ + private static final String GGFS_PRIMARY = "ggfs-primary"; + + /** Primary GGFS name. */ + private static final String GGFS_SECONDARY = "ggfs-secondary"; + + /** Secondary file system REST endpoint configuration map. */ + private static final Map<String, String> SECONDARY_REST_CFG = new HashMap<String, String>(){{ + put("type", "tcp"); + put("port", "11500"); + }}; + + /** Test nodes count. */ + private static final int NODES_CNT = 3; + + /** IP finder for the grid with the primary file system. */ + private static final TcpDiscoveryIpFinder IP_FINDER = new TcpDiscoveryVmIpFinder(true); + + /** Primary GGFS instances. */ + private static IgniteFs[] ggfsPrimary; + + /** Secondary GGFS instance. */ + private static IgniteFs ggfsSecondary; + + /** Primary file system block size. */ + public static final int PRIMARY_BLOCK_SIZE = 512; + + /** Secondary file system block size. */ + public static final int SECONDARY_BLOCK_SIZE = 512; + + /** {@inheritDoc} */ + @Override protected void beforeTestsStarted() throws Exception { + startSecondary(); + startPrimary(); + } + + /** {@inheritDoc} */ + @Override protected void afterTestsStopped() throws Exception { + stopAllGrids(false); + } + + /** + * Start a grid with the primary file system. + * + * @throws Exception If failed. + */ + private void startPrimary() throws Exception { + ggfsPrimary = new IgniteFs[NODES_CNT]; + + for (int i = 0; i < NODES_CNT; i++) { + Ignite g = G.start(primaryConfiguration(i)); + + ggfsPrimary[i] = g.fileSystem(GGFS_PRIMARY); + } + } + + /** + * Get configuration for a grid with the primary file system. + * + * @param idx Node index. + * @return Configuration. + * @throws Exception If failed. + */ + private IgniteConfiguration primaryConfiguration(int idx) throws Exception { + IgniteFsConfiguration ggfsCfg = new IgniteFsConfiguration(); + + ggfsCfg.setDataCacheName("dataCache"); + ggfsCfg.setMetaCacheName("metaCache"); + ggfsCfg.setName(GGFS_PRIMARY); + ggfsCfg.setBlockSize(PRIMARY_BLOCK_SIZE); + ggfsCfg.setDefaultMode(PRIMARY); + ggfsCfg.setSecondaryFileSystem(ggfsSecondary); + + Map<String, IgniteFsMode> pathModes = new HashMap<>(); + + pathModes.put("/fileRemote", DUAL_SYNC); + + ggfsCfg.setPathModes(pathModes); + + CacheConfiguration dataCacheCfg = defaultCacheConfiguration(); + + dataCacheCfg.setName("dataCache"); + dataCacheCfg.setCacheMode(PARTITIONED); + dataCacheCfg.setDistributionMode(GridCacheDistributionMode.PARTITIONED_ONLY); + dataCacheCfg.setWriteSynchronizationMode(GridCacheWriteSynchronizationMode.FULL_SYNC); + dataCacheCfg.setAffinityMapper(new IgniteFsGroupDataBlocksKeyMapper(128)); + dataCacheCfg.setBackups(0); + dataCacheCfg.setQueryIndexEnabled(false); + dataCacheCfg.setAtomicityMode(TRANSACTIONAL); + + CacheConfiguration metaCacheCfg = defaultCacheConfiguration(); + + metaCacheCfg.setName("metaCache"); + metaCacheCfg.setCacheMode(REPLICATED); + metaCacheCfg.setDistributionMode(GridCacheDistributionMode.PARTITIONED_ONLY); + metaCacheCfg.setWriteSynchronizationMode(GridCacheWriteSynchronizationMode.FULL_SYNC); + metaCacheCfg.setQueryIndexEnabled(false); + metaCacheCfg.setAtomicityMode(TRANSACTIONAL); + + IgniteConfiguration cfg = new IgniteConfiguration(); + + cfg.setGridName("grid-" + idx); + + TcpDiscoverySpi discoSpi = new TcpDiscoverySpi(); + + discoSpi.setIpFinder(IP_FINDER); + + cfg.setDiscoverySpi(discoSpi); + cfg.setCacheConfiguration(dataCacheCfg, metaCacheCfg); + cfg.setGgfsConfiguration(ggfsCfg); + + cfg.setLocalHost("127.0.0.1"); + + return cfg; + } + + /** + * Start a grid with the secondary file system. + * + * @throws Exception If failed. + */ + private void startSecondary() throws Exception { + IgniteFsConfiguration ggfsCfg = new IgniteFsConfiguration(); + + ggfsCfg.setDataCacheName("dataCache"); + ggfsCfg.setMetaCacheName("metaCache"); + ggfsCfg.setName(GGFS_SECONDARY); + ggfsCfg.setBlockSize(SECONDARY_BLOCK_SIZE); + ggfsCfg.setDefaultMode(PRIMARY); + ggfsCfg.setIpcEndpointConfiguration(SECONDARY_REST_CFG); + + CacheConfiguration dataCacheCfg = defaultCacheConfiguration(); + + dataCacheCfg.setName("dataCache"); + dataCacheCfg.setCacheMode(PARTITIONED); + dataCacheCfg.setDistributionMode(GridCacheDistributionMode.PARTITIONED_ONLY); + dataCacheCfg.setWriteSynchronizationMode(GridCacheWriteSynchronizationMode.FULL_SYNC); + dataCacheCfg.setAffinityMapper(new IgniteFsGroupDataBlocksKeyMapper(128)); + dataCacheCfg.setBackups(0); + dataCacheCfg.setQueryIndexEnabled(false); + dataCacheCfg.setAtomicityMode(TRANSACTIONAL); + + CacheConfiguration metaCacheCfg = defaultCacheConfiguration(); + + metaCacheCfg.setName("metaCache"); + metaCacheCfg.setCacheMode(REPLICATED); + metaCacheCfg.setDistributionMode(GridCacheDistributionMode.PARTITIONED_ONLY); + metaCacheCfg.setWriteSynchronizationMode(GridCacheWriteSynchronizationMode.FULL_SYNC); + metaCacheCfg.setQueryIndexEnabled(false); + metaCacheCfg.setAtomicityMode(TRANSACTIONAL); + + IgniteConfiguration cfg = new IgniteConfiguration(); + + cfg.setGridName("grid-secondary"); + + TcpDiscoverySpi discoSpi = new TcpDiscoverySpi(); + + discoSpi.setIpFinder(new TcpDiscoveryVmIpFinder(true)); + + cfg.setDiscoverySpi(discoSpi); + cfg.setCacheConfiguration(dataCacheCfg, metaCacheCfg); + cfg.setGgfsConfiguration(ggfsCfg); + + cfg.setLocalHost("127.0.0.1"); + + Ignite g = G.start(cfg); + + ggfsSecondary = g.fileSystem(GGFS_SECONDARY); + } + + /** @throws Exception If failed. */ + public void testMetrics() throws Exception { + IgniteFs fs = ggfsPrimary[0]; + + assertNotNull(fs); + + IgniteFsMetrics m = fs.metrics(); + + assertNotNull(m); + assertEquals(0, m.directoriesCount()); + assertEquals(0, m.filesCount()); + assertEquals(0, m.filesOpenedForRead()); + assertEquals(0, m.filesOpenedForWrite()); + + fs.mkdirs(new IgniteFsPath("/dir1")); + + m = fs.metrics(); + + assertNotNull(m); + assertEquals(1, m.directoriesCount()); + assertEquals(0, m.filesCount()); + assertEquals(0, m.filesOpenedForRead()); + assertEquals(0, m.filesOpenedForWrite()); + + fs.mkdirs(new IgniteFsPath("/dir1/dir2/dir3")); + fs.mkdirs(new IgniteFsPath("/dir4")); + + m = fs.metrics(); + + assertNotNull(m); + assertEquals(4, m.directoriesCount()); + assertEquals(0, m.filesCount()); + assertEquals(0, m.filesOpenedForRead()); + assertEquals(0, m.filesOpenedForWrite()); + + IgniteFsOutputStream out1 = fs.create(new IgniteFsPath("/dir1/file1"), false); + IgniteFsOutputStream out2 = fs.create(new IgniteFsPath("/dir1/file2"), false); + IgniteFsOutputStream out3 = fs.create(new IgniteFsPath("/dir1/dir2/file"), false); + + m = fs.metrics(); + + assertNotNull(m); + assertEquals(4, m.directoriesCount()); + assertEquals(3, m.filesCount()); + assertEquals(0, m.filesOpenedForRead()); + assertEquals(3, m.filesOpenedForWrite()); + + out1.write(new byte[10]); + out2.write(new byte[20]); + out3.write(new byte[30]); + + out1.close(); + + m = fs.metrics(); + + assertNotNull(m); + assertEquals(4, m.directoriesCount()); + assertEquals(3, m.filesCount()); + assertEquals(0, m.filesOpenedForRead()); + assertEquals(2, m.filesOpenedForWrite()); + + out2.close(); + out3.close(); + + m = fs.metrics(); + + assertNotNull(m); + assertEquals(4, m.directoriesCount()); + assertEquals(3, m.filesCount()); + assertEquals(0, m.filesOpenedForRead()); + assertEquals(0, m.filesOpenedForWrite()); + + IgniteFsOutputStream out = fs.append(new IgniteFsPath("/dir1/file1"), false); + + out.write(new byte[20]); + + m = fs.metrics(); + + assertNotNull(m); + assertEquals(4, m.directoriesCount()); + assertEquals(3, m.filesCount()); + assertEquals(0, m.filesOpenedForRead()); + assertEquals(1, m.filesOpenedForWrite()); + + out.write(new byte[20]); + + out.close(); + + m = fs.metrics(); + + assertNotNull(m); + assertEquals(4, m.directoriesCount()); + assertEquals(3, m.filesCount()); + assertEquals(0, m.filesOpenedForRead()); + assertEquals(0, m.filesOpenedForWrite()); + + IgniteFsInputStream in1 = fs.open(new IgniteFsPath("/dir1/file1")); + IgniteFsInputStream in2 = fs.open(new IgniteFsPath("/dir1/file2")); + + m = fs.metrics(); + + assertNotNull(m); + assertEquals(4, m.directoriesCount()); + assertEquals(3, m.filesCount()); + assertEquals(2, m.filesOpenedForRead()); + assertEquals(0, m.filesOpenedForWrite()); + + in1.close(); + in2.close(); + + m = fs.metrics(); + + assertNotNull(m); + assertEquals(4, m.directoriesCount()); + assertEquals(3, m.filesCount()); + assertEquals(0, m.filesOpenedForRead()); + assertEquals(0, m.filesOpenedForWrite()); + + fs.delete(new IgniteFsPath("/dir1/file1"), false); + fs.delete(new IgniteFsPath("/dir1/dir2"), true); + + m = fs.metrics(); + + assertNotNull(m); + assertEquals(2, m.directoriesCount()); + assertEquals(1, m.filesCount()); + assertEquals(0, m.filesOpenedForRead()); + assertEquals(0, m.filesOpenedForWrite()); + + fs.delete(new IgniteFsPath("/"), true); + + m = fs.metrics(); + + assertNotNull(m); + assertEquals(0, m.directoriesCount()); + assertEquals(0, m.filesCount()); + assertEquals(0, m.filesOpenedForRead()); + assertEquals(0, m.filesOpenedForWrite()); + } + + /** @throws Exception If failed. */ + public void testMultipleClose() throws Exception { + IgniteFs fs = ggfsPrimary[0]; + + IgniteFsOutputStream out = fs.create(new IgniteFsPath("/file"), false); + + out.close(); + out.close(); + + IgniteFsInputStream in = fs.open(new IgniteFsPath("/file")); + + in.close(); + in.close(); + + IgniteFsMetrics m = fs.metrics(); + + assertEquals(0, m.filesOpenedForWrite()); + assertEquals(0, m.filesOpenedForRead()); + } + + /** + * Test block metrics. + * + * @throws Exception If failed. + */ + public void testBlockMetrics() throws Exception { + GridGgfsEx ggfs = (GridGgfsEx)ggfsPrimary[0]; + + IgniteFsPath fileRemote = new IgniteFsPath("/fileRemote"); + IgniteFsPath file1 = new IgniteFsPath("/file1"); + IgniteFsPath file2 = new IgniteFsPath("/file2"); + + // Create remote file and write some data to it. + IgniteFsOutputStream out = ggfsSecondary.create(fileRemote, 256, true, null, 1, 256, null); + + int rmtBlockSize = ggfsSecondary.info(fileRemote).blockSize(); + + out.write(new byte[rmtBlockSize]); + out.close(); + + // Start metrics measuring. + IgniteFsMetrics initMetrics = ggfs.metrics(); + + // Create empty file. + ggfs.create(file1, 256, true, null, 1, 256, null).close(); + + int blockSize = ggfs.info(file1).blockSize(); + + checkBlockMetrics(initMetrics, ggfs.metrics(), 0, 0, 0, 0, 0, 0); + + // Write two blocks to the file. + IgniteFsOutputStream os = ggfs.append(file1, false); + os.write(new byte[blockSize * 2]); + os.close(); + + checkBlockMetrics(initMetrics, ggfs.metrics(), 0, 0, 0, 2, 0, blockSize * 2); + + // Write one more file (one block). + os = ggfs.create(file2, 256, true, null, 1, 256, null); + os.write(new byte[blockSize]); + os.close(); + + checkBlockMetrics(initMetrics, ggfs.metrics(), 0, 0, 0, 3, 0, blockSize * 3); + + // Read data from the first file. + GridGgfsInputStreamAdapter is = ggfs.open(file1); + is.readFully(0, new byte[blockSize * 2]); + is.close(); + + checkBlockMetrics(initMetrics, ggfs.metrics(), 2, 0, blockSize * 2, 3, 0, blockSize * 3); + + // Read data from the second file with hits. + is = ggfs.open(file2); + is.readChunks(0, blockSize); + is.close(); + + checkBlockMetrics(initMetrics, ggfs.metrics(), 3, 0, blockSize * 3, 3, 0, blockSize * 3); + + // Clear the first file. + ggfs.create(file1, true).close(); + + checkBlockMetrics(initMetrics, ggfs.metrics(), 3, 0, blockSize * 3, 3, 0, blockSize * 3); + + // Delete the second file. + ggfs.delete(file2, false); + + checkBlockMetrics(initMetrics, ggfs.metrics(), 3, 0, blockSize * 3, 3, 0, blockSize * 3); + + // Read remote file. + is = ggfs.open(fileRemote); + is.readChunks(0, rmtBlockSize); + is.close(); + + checkBlockMetrics(initMetrics, ggfs.metrics(), 4, 1, blockSize * 3 + rmtBlockSize, 3, 0, blockSize * 3); + + // Lets wait for blocks will be placed to cache + U.sleep(300); + + // Read remote file again. + is = ggfs.open(fileRemote); + is.readChunks(0, rmtBlockSize); + is.close(); + + checkBlockMetrics(initMetrics, ggfs.metrics(), 5, 1, blockSize * 3 + rmtBlockSize * 2, 3, 0, blockSize * 3); + + IgniteFsMetrics metrics = ggfs.metrics(); + + assert metrics.secondarySpaceSize() == rmtBlockSize; + + // Write some data to the file working in DUAL mode. + os = ggfs.append(fileRemote, false); + os.write(new byte[rmtBlockSize]); + os.close(); + + // Additional block read here due to file ending synchronization. + checkBlockMetrics(initMetrics, ggfs.metrics(), 5, 1, blockSize * 3 + rmtBlockSize * 2, 4, 1, + blockSize * 3 + rmtBlockSize); + + metrics = ggfs.metrics(); + + assert metrics.secondarySpaceSize() == rmtBlockSize * 2; + + ggfs.delete(fileRemote, false); + + U.sleep(300); + + assert ggfs.metrics().secondarySpaceSize() == 0; + + // Write partial block to the first file. + os = ggfs.append(file1, false); + os.write(new byte[blockSize / 2]); + os.close(); + + checkBlockMetrics(initMetrics, ggfs.metrics(), 5, 1, blockSize * 3 + rmtBlockSize * 2, 5, 1, + blockSize * 7 / 2 + rmtBlockSize); + + // Now read partial block. + // Read remote file again. + is = ggfs.open(file1); + is.seek(blockSize * 2); + is.readChunks(0, blockSize / 2); + is.close(); + + checkBlockMetrics(initMetrics, ggfs.metrics(), 6, 1, blockSize * 7 / 2 + rmtBlockSize * 2, 5, 1, + blockSize * 7 / 2 + rmtBlockSize); + + ggfs.resetMetrics(); + + metrics = ggfs.metrics(); + + assert metrics.blocksReadTotal() == 0; + assert metrics.blocksReadRemote() == 0; + assert metrics.blocksWrittenTotal() == 0; + assert metrics.blocksWrittenRemote() == 0; + assert metrics.bytesRead() == 0; + assert metrics.bytesReadTime() == 0; + assert metrics.bytesWritten() == 0; + assert metrics.bytesWriteTime() == 0; + } + + /** + * Ensure overall block-related metrics correctness. + * + * @param initMetrics Initial metrics. + * @param metrics Metrics to check. + * @param blocksRead Blocks read remote. + * @param blocksReadRemote Blocks read remote. + * @param bytesRead Bytes read. + * @param blocksWrite Blocks write. + * @param blocksWriteRemote Blocks write remote. + * @param bytesWrite Bytes write. + * @throws Exception If failed. + */ + private void checkBlockMetrics(IgniteFsMetrics initMetrics, IgniteFsMetrics metrics, long blocksRead, + long blocksReadRemote, long bytesRead, long blocksWrite, long blocksWriteRemote, long bytesWrite) + throws Exception { + assert metrics != null; + + assertEquals(blocksRead, metrics.blocksReadTotal() - initMetrics.blocksReadTotal()); + assertEquals(blocksReadRemote, metrics.blocksReadRemote() - initMetrics.blocksReadRemote()); + assertEquals(bytesRead, metrics.bytesRead() - initMetrics.bytesRead()); + + assertEquals(blocksWrite, metrics.blocksWrittenTotal() - initMetrics.blocksWrittenTotal()); + assertEquals(blocksWriteRemote, metrics.blocksWrittenRemote() - initMetrics.blocksWrittenRemote()); + assertEquals(bytesWrite, metrics.bytesWritten() - initMetrics.bytesWritten()); + } +} http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/cfcf46df/modules/core/src/test/java/org/apache/ignite/internal/processors/fs/GridGgfsModeResolverSelfTest.java ---------------------------------------------------------------------- diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/fs/GridGgfsModeResolverSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/fs/GridGgfsModeResolverSelfTest.java new file mode 100644 index 0000000..dbc3c3f --- /dev/null +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/fs/GridGgfsModeResolverSelfTest.java @@ -0,0 +1,77 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ignite.internal.processors.fs; + +import junit.framework.*; +import org.apache.ignite.fs.*; +import org.apache.ignite.internal.util.typedef.*; + +import java.util.*; + +import static org.apache.ignite.fs.IgniteFsMode.*; + +/** + * + */ +public class GridGgfsModeResolverSelfTest extends TestCase { + /** */ + private GridGgfsModeResolver resolver; + + /** {@inheritDoc} */ + @Override protected void setUp() throws Exception { + resolver = new GridGgfsModeResolver(DUAL_SYNC, Arrays.asList( + new T2<>(new IgniteFsPath("/a/b/"), PRIMARY), + new T2<>(new IgniteFsPath("/a/b/c/d"), PROXY))); + } + + /** + * @throws Exception If failed. + */ + public void testResolve() throws Exception { + assertEquals(DUAL_SYNC, resolver.resolveMode(new IgniteFsPath("/"))); + assertEquals(DUAL_SYNC, resolver.resolveMode(new IgniteFsPath("/a"))); + assertEquals(DUAL_SYNC, resolver.resolveMode(new IgniteFsPath("/a/1"))); + assertEquals(PRIMARY, resolver.resolveMode(new IgniteFsPath("/a/b"))); + assertEquals(PRIMARY, resolver.resolveMode(new IgniteFsPath("/a/b/c"))); + assertEquals(PRIMARY, resolver.resolveMode(new IgniteFsPath("/a/b/c/2"))); + assertEquals(PROXY, resolver.resolveMode(new IgniteFsPath("/a/b/c/d"))); + assertEquals(PROXY, resolver.resolveMode(new IgniteFsPath("/a/b/c/d/e"))); + } + + /** + * @throws Exception If failed. + */ + public void testResolveChildren() throws Exception { + assertEquals(new HashSet<IgniteFsMode>(){{add(DUAL_SYNC); add(PRIMARY); add(PROXY);}}, + resolver.resolveChildrenModes(new IgniteFsPath("/"))); + assertEquals(new HashSet<IgniteFsMode>(){{add(DUAL_SYNC); add(PRIMARY); add(PROXY);}}, + resolver.resolveChildrenModes(new IgniteFsPath("/a"))); + assertEquals(new HashSet<IgniteFsMode>(){{add(DUAL_SYNC);}}, + resolver.resolveChildrenModes(new IgniteFsPath("/a/1"))); + assertEquals(new HashSet<IgniteFsMode>(){{add(PRIMARY); add(PROXY);}}, + resolver.resolveChildrenModes(new IgniteFsPath("/a/b"))); + assertEquals(new HashSet<IgniteFsMode>(){{add(PRIMARY); add(PROXY);}}, + resolver.resolveChildrenModes(new IgniteFsPath("/a/b/c"))); + assertEquals(new HashSet<IgniteFsMode>(){{add(PRIMARY);}}, + resolver.resolveChildrenModes(new IgniteFsPath("/a/b/c/2"))); + assertEquals(new HashSet<IgniteFsMode>(){{add(PROXY);}}, + resolver.resolveChildrenModes(new IgniteFsPath("/a/b/c/d"))); + assertEquals(new HashSet<IgniteFsMode>(){{add(PROXY);}}, + resolver.resolveChildrenModes(new IgniteFsPath("/a/b/c/d/e"))); + } +} http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/cfcf46df/modules/core/src/test/java/org/apache/ignite/internal/processors/fs/GridGgfsModesSelfTest.java ---------------------------------------------------------------------- diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/fs/GridGgfsModesSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/fs/GridGgfsModesSelfTest.java new file mode 100644 index 0000000..86a33b5 --- /dev/null +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/fs/GridGgfsModesSelfTest.java @@ -0,0 +1,604 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ignite.internal.processors.fs; + +import org.apache.ignite.*; +import org.apache.ignite.cache.*; +import org.apache.ignite.configuration.*; +import org.apache.ignite.fs.*; +import org.apache.ignite.internal.*; +import org.apache.ignite.lang.*; +import org.apache.ignite.spi.discovery.tcp.*; +import org.apache.ignite.spi.discovery.tcp.ipfinder.vm.*; +import org.apache.ignite.internal.util.typedef.*; +import org.apache.ignite.internal.util.typedef.internal.*; + +import java.util.*; + +import static org.apache.ignite.fs.IgniteFsMode.*; +import static org.apache.ignite.cache.GridCacheAtomicityMode.*; +import static org.apache.ignite.cache.GridCacheMode.*; + +/** + * GGFS modes self test. + */ +public class GridGgfsModesSelfTest extends GridGgfsCommonAbstractTest { + /** Grid instance hosting primary GGFS. */ + private GridEx grid; + + /** Primary GGFS. */ + private GridGgfsImpl ggfs; + + /** Secondary GGFS. */ + private GridGgfsImpl ggfsSecondary; + + /** Default GGFS mode. */ + private IgniteFsMode mode; + + /** Modes map. */ + private Map<String, IgniteFsMode> pathModes; + + /** Whether to set "null" mode. */ + private boolean setNullMode; + + /** Whether to set secondary file system URI. */ + private boolean setSecondaryFs; + + /** {@inheritDoc} */ + @Override protected void beforeTest() throws Exception { + mode = null; + pathModes = null; + + setNullMode = false; + setSecondaryFs = false; + } + + /** {@inheritDoc} */ + @Override protected void afterTest() throws Exception { + G.stopAll(true); + } + + /** + * Perform initial startup. + * + * @throws Exception If failed. + */ + @SuppressWarnings("NullableProblems") + private void startUp() throws Exception { + startUpSecondary(); + + IgniteFsConfiguration ggfsCfg = new IgniteFsConfiguration(); + + ggfsCfg.setDataCacheName("partitioned"); + ggfsCfg.setMetaCacheName("replicated"); + ggfsCfg.setName("ggfs"); + ggfsCfg.setBlockSize(512 * 1024); + + if (setNullMode) + ggfsCfg.setDefaultMode(null); + else if (mode != null) + ggfsCfg.setDefaultMode(mode); + + ggfsCfg.setPathModes(pathModes); + + if (setSecondaryFs) + ggfsCfg.setSecondaryFileSystem(ggfsSecondary); + + CacheConfiguration cacheCfg = defaultCacheConfiguration(); + + cacheCfg.setName("partitioned"); + cacheCfg.setCacheMode(PARTITIONED); + cacheCfg.setDistributionMode(GridCacheDistributionMode.PARTITIONED_ONLY); + cacheCfg.setWriteSynchronizationMode(GridCacheWriteSynchronizationMode.FULL_SYNC); + cacheCfg.setAffinityMapper(new IgniteFsGroupDataBlocksKeyMapper(128)); + cacheCfg.setBackups(0); + cacheCfg.setQueryIndexEnabled(false); + cacheCfg.setAtomicityMode(TRANSACTIONAL); + + CacheConfiguration metaCacheCfg = defaultCacheConfiguration(); + + metaCacheCfg.setName("replicated"); + metaCacheCfg.setCacheMode(REPLICATED); + metaCacheCfg.setWriteSynchronizationMode(GridCacheWriteSynchronizationMode.FULL_SYNC); + metaCacheCfg.setQueryIndexEnabled(false); + metaCacheCfg.setAtomicityMode(TRANSACTIONAL); + + IgniteConfiguration cfg = new IgniteConfiguration(); + + cfg.setGridName("ggfs-grid"); + + TcpDiscoverySpi discoSpi = new TcpDiscoverySpi(); + + discoSpi.setIpFinder(new TcpDiscoveryVmIpFinder(true)); + + cfg.setDiscoverySpi(discoSpi); + cfg.setCacheConfiguration(metaCacheCfg, cacheCfg); + cfg.setGgfsConfiguration(ggfsCfg); + + cfg.setLocalHost("127.0.0.1"); + cfg.setRestEnabled(false); + + grid = (GridEx)G.start(cfg); + + ggfs = (GridGgfsImpl)grid.fileSystem("ggfs"); + } + + /** + * Startup secondary file system. + * + * @throws Exception If failed. + */ + private void startUpSecondary() throws Exception { + IgniteFsConfiguration ggfsCfg = new IgniteFsConfiguration(); + + ggfsCfg.setDataCacheName("partitioned"); + ggfsCfg.setMetaCacheName("replicated"); + ggfsCfg.setName("ggfs-secondary"); + ggfsCfg.setBlockSize(512 * 1024); + ggfsCfg.setDefaultMode(PRIMARY); + ggfsCfg.setIpcEndpointConfiguration(new HashMap<String, String>() {{ + put("type", "tcp"); + put("port", "11500"); + }}); + + CacheConfiguration cacheCfg = defaultCacheConfiguration(); + + cacheCfg.setName("partitioned"); + cacheCfg.setCacheMode(PARTITIONED); + cacheCfg.setDistributionMode(GridCacheDistributionMode.PARTITIONED_ONLY); + cacheCfg.setWriteSynchronizationMode(GridCacheWriteSynchronizationMode.FULL_SYNC); + cacheCfg.setAffinityMapper(new IgniteFsGroupDataBlocksKeyMapper(128)); + cacheCfg.setBackups(0); + cacheCfg.setQueryIndexEnabled(false); + cacheCfg.setAtomicityMode(TRANSACTIONAL); + + CacheConfiguration metaCacheCfg = defaultCacheConfiguration(); + + metaCacheCfg.setName("replicated"); + metaCacheCfg.setCacheMode(REPLICATED); + metaCacheCfg.setWriteSynchronizationMode(GridCacheWriteSynchronizationMode.FULL_SYNC); + metaCacheCfg.setQueryIndexEnabled(false); + metaCacheCfg.setAtomicityMode(TRANSACTIONAL); + + IgniteConfiguration cfg = new IgniteConfiguration(); + + cfg.setGridName("ggfs-grid-secondary"); + + TcpDiscoverySpi discoSpi = new TcpDiscoverySpi(); + + discoSpi.setIpFinder(new TcpDiscoveryVmIpFinder(true)); + + cfg.setDiscoverySpi(discoSpi); + cfg.setCacheConfiguration(metaCacheCfg, cacheCfg); + cfg.setGgfsConfiguration(ggfsCfg); + + cfg.setLocalHost("127.0.0.1"); + cfg.setRestEnabled(false); + + ggfsSecondary = (GridGgfsImpl)G.start(cfg).fileSystem("ggfs-secondary"); + } + + /** + * Set GGFS modes for particular paths. + * + * @param modes Modes. + */ + @SafeVarargs + final void pathModes(IgniteBiTuple<String, IgniteFsMode>... modes) { + assert modes != null; + + pathModes = new LinkedHashMap<>(modes.length, 1.0f); + + for (IgniteBiTuple<String, IgniteFsMode> mode : modes) + pathModes.put(mode.getKey(), mode.getValue()); + } + + /** + * Test predefined path modes for PRIMARY mode. + * + * @throws Exception If failed. + */ + public void testDefaultFoldersPrimary() throws Exception { + setSecondaryFs = true; + + mode = DUAL_ASYNC; + + startUp(); + + checkMode("/gridgain/primary", PRIMARY); + checkMode("/gridgain/primary/", PRIMARY); + checkMode("/gridgain/primary/subfolder", PRIMARY); + checkMode("/gridgain/primary/folder/file.txt", PRIMARY); + checkMode("/gridgain/primaryx", DUAL_ASYNC); + checkMode("/gridgain/primaryx/", DUAL_ASYNC); + checkMode("/gridgain/primaryx/subfolder", DUAL_ASYNC); + checkMode("/gridgain/primaryx/folder/file.txt", DUAL_ASYNC); + } + + /** + * Test predefined path modes for all modes except of PRIMARY mode. + * + * @throws Exception If failed. + */ + public void testDefaultFoldersNonPrimary() throws Exception { + setSecondaryFs = true; + + mode = PRIMARY; + + startUp(); + + checkMode("/gridgain/proxy", PROXY); + checkMode("/gridgain/proxy/", PROXY); + checkMode("/gridgain/proxy/subfolder", PROXY); + checkMode("/gridgain/proxy/folder/file.txt", PROXY); + checkMode("/gridgain/proxyx", PRIMARY); + checkMode("/gridgain/proxyx/", PRIMARY); + checkMode("/gridgain/proxyx/subfolder", PRIMARY); + checkMode("/gridgain/proxyx/folder/file.txt", PRIMARY); + + checkMode("/userdir/gridgain/proxy", PRIMARY); + checkMode("/userdir/gridgain/proxy/", PRIMARY); + checkMode("/userdir/gridgain/proxy/subfolder", PRIMARY); + checkMode("/userdir/gridgain/proxy/folder/file.txt", PRIMARY); + + checkMode("/gridgain/sync", DUAL_SYNC); + checkMode("/gridgain/sync/", DUAL_SYNC); + checkMode("/gridgain/sync/subfolder", DUAL_SYNC); + checkMode("/gridgain/sync/folder/file.txt", DUAL_SYNC); + checkMode("/gridgain/syncx", PRIMARY); + checkMode("/gridgain/syncx/", PRIMARY); + checkMode("/gridgain/syncx/subfolder", PRIMARY); + checkMode("/gridgain/syncx/folder/file.txt", PRIMARY); + + checkMode("/userdir/gridgain/sync", PRIMARY); + checkMode("/userdir/gridgain/sync/", PRIMARY); + checkMode("/userdir/gridgain/sync/subfolder", PRIMARY); + checkMode("/userdir/gridgain/sync/folder/file.txt", PRIMARY); + + checkMode("/gridgain/async", DUAL_ASYNC); + checkMode("/gridgain/async/", DUAL_ASYNC); + checkMode("/gridgain/async/subfolder", DUAL_ASYNC); + checkMode("/gridgain/async/folder/file.txt", DUAL_ASYNC); + checkMode("/gridgain/asyncx", PRIMARY); + checkMode("/gridgain/asyncx/", PRIMARY); + checkMode("/gridgain/asyncx/subfolder", PRIMARY); + checkMode("/gridgain/asyncx/folder/file.txt", PRIMARY); + + checkMode("/userdir/gridgain/async", PRIMARY); + checkMode("/userdir/gridgain/async/", PRIMARY); + checkMode("/userdir/gridgain/async/subfolder", PRIMARY); + checkMode("/userdir/gridgain/async/folder/file.txt", PRIMARY); + } + + /** + * Ensure that in case secondary file system URI is not provided, all predefined have no special mappings. This test + * doesn't make sense for PRIMARY mode since in case URI is not provided DUAL_* modes automatically transforms to + * PRIMARY and for PROXY mode we will have an exception during startup. + * + * @throws Exception If failed. + */ + public void testDefaultsNoSecondaryUriNonPrimary() throws Exception { + startUp(); + + checkMode("/gridgain/proxy", PRIMARY); + checkMode("/gridgain/proxy/", PRIMARY); + checkMode("/gridgain/proxy/subfolder", PRIMARY); + checkMode("/gridgain/proxy/folder/file.txt", PRIMARY); + + checkMode("/gridgain/sync", PRIMARY); + checkMode("/gridgain/sync/", PRIMARY); + checkMode("/gridgain/sync/subfolder", PRIMARY); + checkMode("/gridgain/sync/folder/file.txt", PRIMARY); + + checkMode("/gridgain/async", PRIMARY); + checkMode("/gridgain/async/", PRIMARY); + checkMode("/gridgain/async/subfolder", PRIMARY); + checkMode("/gridgain/async/folder/file.txt", PRIMARY); + } + + /** + * Ensure that it is impossible to override mappings for /gridgain/* folders. + * + * @throws Exception If failed. + */ + public void testDefaultFoldersOverride() throws Exception { + setSecondaryFs = true; + + mode = DUAL_ASYNC; + + pathModes(F.t("/gridgain/primary", PROXY), F.t("/gridgain/proxy", DUAL_SYNC), + F.t("/gridgain/sync", DUAL_ASYNC), F.t("/gridgain/async", PRIMARY)); + + startUp(); + + checkMode("/gridgain/primary", PRIMARY); + checkMode("/gridgain/proxy", PROXY); + checkMode("/gridgain/sync", DUAL_SYNC); + checkMode("/gridgain/async", DUAL_ASYNC); + } + + /** + * Ensure that DUAL_ASYNC mode is set by default. + * + * @throws Exception If failed. + */ + public void testModeDefaultIsNotSet() throws Exception { + setSecondaryFs = true; + + startUp(); + + checkMode("/dir", DUAL_ASYNC); + } + + /** + * Ensure that when mode is set, it is correctly resolved. + * + * @throws Exception If failed. + */ + public void testModeDefaultIsSet() throws Exception { + mode = DUAL_SYNC; + + setSecondaryFs = true; + + startUp(); + + checkMode("/dir", DUAL_SYNC); + } + + /** + * Ensure that Grid doesn't start in case default mode is SECONDARY and secondary FS URI is not provided. + * + * @throws Exception If failed. + */ + public void testModeSecondaryNoUri() throws Exception { + mode = PROXY; + + String errMsg = null; + + try { + startUp(); + } + catch (IgniteCheckedException e) { + errMsg = e.getCause().getMessage(); + } + + assertTrue(errMsg.startsWith( + "Grid configuration parameter invalid: secondaryFileSystem cannot be null when mode is SECONDARY")); + } + + /** + * Ensure that modes are resolved correctly when path modes are set. + * + * @throws Exception If failed. + */ + public void testPathMode() throws Exception { + pathModes(F.t("/dir1", PROXY), F.t("/dir2", DUAL_SYNC), + F.t("/dir3", PRIMARY), F.t("/dir4", PRIMARY)); + + mode = DUAL_ASYNC; + + setSecondaryFs = true; + + startUp(); + + checkMode("/dir", DUAL_ASYNC); + checkMode("/dir1", PROXY); + checkMode("/dir2", DUAL_SYNC); + + checkMode("/dir3", PRIMARY); + checkMode("/somedir/dir3", DUAL_ASYNC); + + checkMode("/dir4", PRIMARY); + checkMode("/dir4/subdir", PRIMARY); + checkMode("/somedir/dir4", DUAL_ASYNC); + checkMode("/somedir/dir4/subdir", DUAL_ASYNC); + } + + /** + * Ensure that path modes switch to PRIMARY in case secondary FS config is not provided. + * + * @throws Exception If failed. + */ + public void testPathModeSwitchToPrimary() throws Exception { + mode = DUAL_SYNC; + + pathModes(F.t("/dir1", PRIMARY), F.t("/dir2", DUAL_SYNC)); + + startUp(); + + checkMode("/dir", PRIMARY); + checkMode("/dir1", PRIMARY); + checkMode("/dir2", PRIMARY); + } + + /** + * Ensure that Grid doesn't start in case path mode is SECONDARY and secondary FS config path is not provided. + * + * @throws Exception If failed. + */ + public void testPathModeSecondaryNoCfg() throws Exception { + pathModes(F.t("dir", PROXY)); + + String errMsg = null; + + try { + startUp(); + } + catch (IgniteCheckedException e) { + errMsg = e.getCause().getMessage(); + } + + assertTrue(errMsg.startsWith( + "Grid configuration parameter invalid: secondaryFileSystem cannot be null when mode is SECONDARY")); + } + + /** + * Ensure that data is not propagated to the secondary GGFS in PRIMARY mode. + * + * @throws Exception If failed. + */ + public void testPropagationPrimary() throws Exception { + mode = PRIMARY; + + checkPropagation(); + } + + /** + * Ensure that data is propagated to the secondary GGFS in DUAL_SYNC mode. + * + * @throws Exception If failed. + */ + public void testPropagationDualSync() throws Exception { + mode = DUAL_SYNC; + + checkPropagation(); + } + + /** + * Ensure that data is propagated to the secondary GGFS in DUAL_SYNC mode. + * + * @throws Exception If failed. + */ + public void _testPropagationDualAsync() throws Exception { + mode = DUAL_ASYNC; + + checkPropagation(); + } + + /** + * Resolve GGFS mode for the given path and compare it with expected one. + * + * @param pathStr Path ot resolve. + * @param expMode Expected mode. + * @throws Exception If failed. + */ + private void checkMode(String pathStr, IgniteFsMode expMode) throws Exception { + assert ggfs != null; + + IgniteFsPath path = new IgniteFsPath(pathStr); + + GridGgfsModeResolver rslvr = ggfs.modeResolver(); + + IgniteFsMode mode = rslvr.resolveMode(path); + + assertEquals(expMode, mode); + } + + /** + * Check propagation of various operations to secondary file system. + * + * @throws Exception If failed. + */ + private void checkPropagation() throws Exception { + byte[] testData1 = new byte[] {0, 1, 2, 3, 4, 5, 6, 7}; + byte[] testData2 = new byte[] {8, 9, 10, 11}; + byte[] testData = Arrays.copyOf(testData1, testData1.length + testData2.length); + + U.arrayCopy(testData2, 0, testData, testData1.length, testData2.length); + + setSecondaryFs = true; + + startUp(); + + boolean primaryNotUsed = mode == PROXY; + boolean secondaryUsed = mode != PRIMARY; + + IgniteFsPath dir = new IgniteFsPath("/dir"); + IgniteFsPath file = new IgniteFsPath("/dir/file"); + + // Create new directory. + ggfs.mkdirs(dir); + + // Create new file. + IgniteFsOutputStream os = ggfs.create(file, 1024, true, null, 0, 2048, null); + + os.write(testData1); + + os.close(); + + // Re-open it and append. + os = ggfs.append(file, 1024, false, null); + + os.write(testData2); + + os.close(); + + // Check file content. + IgniteFsInputStream is = ggfs.open(file); + + assertEquals(testData.length, is.length()); + + byte[] data = new byte[testData.length]; + + is.read(data, 0, testData.length); + + is.close(); + + assert Arrays.equals(testData, data); + + if (secondaryUsed) { + assert ggfsSecondary.exists(dir); + assert ggfsSecondary.exists(file); + + // In ASYNC mode we wait at most 2 seconds for background writer to finish. + for (int i = 0; i < 20; i++) { + IgniteFsInputStream isSecondary = null; + + try { + isSecondary = ggfsSecondary.open(file); + + if (isSecondary.length() == testData.length) + break; + else + U.sleep(100); + } + finally { + U.closeQuiet(isSecondary); + } + } + + IgniteFsInputStream isSecondary = ggfsSecondary.open(file); + + assertEquals(testData.length, isSecondary.length()); + + isSecondary.read(data, 0, testData.length); + + assert Arrays.equals(testData, data); + } + else { + assert !ggfsSecondary.exists(dir); + assert !ggfsSecondary.exists(file); + } + + int cacheSize = grid.cachex("partitioned").size(); + + if (primaryNotUsed) + assert cacheSize == 0; + else + assert cacheSize != 0; + + // Now delete all. + ggfs.delete(dir, true); + + assert !ggfs.exists(dir); + assert !ggfs.exists(file); + + assert !ggfsSecondary.exists(dir); + assert !ggfsSecondary.exists(file); + } +}
