ignite-718 - minor + test
Project: http://git-wip-us.apache.org/repos/asf/incubator-ignite/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-ignite/commit/37f8fbe9 Tree: http://git-wip-us.apache.org/repos/asf/incubator-ignite/tree/37f8fbe9 Diff: http://git-wip-us.apache.org/repos/asf/incubator-ignite/diff/37f8fbe9 Branch: refs/heads/ignite-718 Commit: 37f8fbe97d4705d087331e3ba3c9a0595072979d Parents: 6a53351 Author: S.Vladykin <svlady...@gridgain.com> Authored: Sat Apr 11 20:32:13 2015 +0300 Committer: S.Vladykin <svlady...@gridgain.com> Committed: Sat Apr 11 20:32:13 2015 +0300 ---------------------------------------------------------------------- .../apache/ignite/IgniteSystemProperties.java | 7 ++ .../query/h2/twostep/GridMergeIndex.java | 12 ++- .../cache/IgniteCacheLargeResultSelfTest.java | 104 +++++++++++++++++++ 3 files changed, 118 insertions(+), 5 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/37f8fbe9/modules/core/src/main/java/org/apache/ignite/IgniteSystemProperties.java ---------------------------------------------------------------------- diff --git a/modules/core/src/main/java/org/apache/ignite/IgniteSystemProperties.java b/modules/core/src/main/java/org/apache/ignite/IgniteSystemProperties.java index c4b30d6..ca7857c 100644 --- a/modules/core/src/main/java/org/apache/ignite/IgniteSystemProperties.java +++ b/modules/core/src/main/java/org/apache/ignite/IgniteSystemProperties.java @@ -337,6 +337,13 @@ public final class IgniteSystemProperties { public static final String IGNITE_JCACHE_DEFAULT_ISOLATED = "IGNITE_CACHE_CLIENT"; /** + * Property controlling maximum number of SQL result rows which can be fetched into a merge table. + * If there are less rows than this threshold then multiple passes throw a table will be possible, + * otherwise only one pass (e.g. only result streaming is possible). + */ + public static final String IGNITE_SQL_MERGE_TABLE_MAX_SIZE = "IGNITE_SQL_MERGE_TABLE_MAX_SIZE"; + + /** * Enforces singleton. */ private IgniteSystemProperties() { http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/37f8fbe9/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/twostep/GridMergeIndex.java ---------------------------------------------------------------------- diff --git a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/twostep/GridMergeIndex.java b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/twostep/GridMergeIndex.java index 5138013..407341e 100644 --- a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/twostep/GridMergeIndex.java +++ b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/twostep/GridMergeIndex.java @@ -30,15 +30,17 @@ import java.util.*; import java.util.concurrent.*; import java.util.concurrent.atomic.*; +import static org.apache.ignite.IgniteSystemProperties.*; + /** * Merge index. */ public abstract class GridMergeIndex extends BaseIndex { /** */ - private static final int MAX_FETCH_SIZE = 100000; // TODO configure + private static final int MAX_FETCH_SIZE = getInteger(IGNITE_SQL_MERGE_TABLE_MAX_SIZE, 10_000); /** All rows number. */ - private final AtomicInteger rowsCnt = new AtomicInteger(0); + private final AtomicInteger expectedRowsCnt = new AtomicInteger(0); /** Remaining rows per source node ID. */ private final ConcurrentMap<UUID, Counter> remainingRows = new ConcurrentHashMap8<>(); @@ -74,7 +76,7 @@ public abstract class GridMergeIndex extends BaseIndex { /** {@inheritDoc} */ @Override public long getRowCount(Session session) { - return rowsCnt.get(); + return expectedRowsCnt.get(); } /** {@inheritDoc} */ @@ -114,7 +116,7 @@ public abstract class GridMergeIndex extends BaseIndex { assert !cnt.initialized : "Counter is already initialized."; cnt.addAndGet(allRows); - rowsCnt.addAndGet(allRows); + expectedRowsCnt.addAndGet(allRows); // We need this separate flag to handle case when the first source contains only one page // and it will signal that all remaining counters are zero and fetch is finished. @@ -167,7 +169,7 @@ public abstract class GridMergeIndex extends BaseIndex { * @return {@code true} If we have fetched all the remote rows. */ public boolean fetchedAll() { - return fetchedCnt == rowsCnt.get(); + return fetchedCnt == expectedRowsCnt.get(); } /** http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/37f8fbe9/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/IgniteCacheLargeResultSelfTest.java ---------------------------------------------------------------------- diff --git a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/IgniteCacheLargeResultSelfTest.java b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/IgniteCacheLargeResultSelfTest.java new file mode 100644 index 0000000..3f88ad6 --- /dev/null +++ b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/IgniteCacheLargeResultSelfTest.java @@ -0,0 +1,104 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ignite.internal.processors.cache; + +import org.apache.ignite.*; +import org.apache.ignite.cache.*; +import org.apache.ignite.cache.query.*; +import org.apache.ignite.configuration.*; +import org.apache.ignite.spi.discovery.tcp.*; +import org.apache.ignite.spi.discovery.tcp.ipfinder.*; +import org.apache.ignite.spi.discovery.tcp.ipfinder.vm.*; +import org.apache.ignite.testframework.junits.common.*; + +import java.util.*; + +import static org.apache.ignite.cache.CacheAtomicityMode.*; +import static org.apache.ignite.cache.CacheMode.*; + +/** + */ +public class IgniteCacheLargeResultSelfTest extends GridCommonAbstractTest { + /** */ + private static TcpDiscoveryIpFinder ipFinder = new TcpDiscoveryVmIpFinder(true); + + /** {@inheritDoc} */ + @Override protected IgniteConfiguration getConfiguration(String gridName) throws Exception { + IgniteConfiguration cfg = super.getConfiguration(gridName); + + TcpDiscoverySpi disco = new TcpDiscoverySpi(); + + disco.setIpFinder(ipFinder); + + cfg.setDiscoverySpi(disco); + + CacheConfiguration<?,?> cacheCfg = defaultCacheConfiguration(); + + cacheCfg.setCacheMode(PARTITIONED); + cacheCfg.setAtomicityMode(TRANSACTIONAL); + cacheCfg.setWriteSynchronizationMode(CacheWriteSynchronizationMode.FULL_SYNC); + cacheCfg.setSwapEnabled(false); + cacheCfg.setBackups(1); + cacheCfg.setIndexedTypes( + Integer.class, Integer.class + ); + + cfg.setCacheConfiguration(cacheCfg); + + return cfg; + } + + /** {@inheritDoc} */ + @Override protected void beforeTestsStarted() throws Exception { + startGridsMultiThreaded(3); + } + + /** {@inheritDoc} */ + @Override protected void afterTestsStopped() throws Exception { + stopAllGrids(); + } + + /** + */ + public void testLargeResult() { + // Fill cache. + IgniteCache<Integer,Integer> c = ignite(0).cache(null); + + for (int i = 0; i < 50_000; i++) // default max merge table size is 10000 + c.put(i, i); + + try(QueryCursor<List<?>> res = c.query( + new SqlFieldsQuery("select _val from Integer where _key between ? and ?") + .setArgs(10_000, 40_000))){ + + int cnt = 0; + + for (List<?> row : res) { + cnt++; + + int val = (Integer)row.get(0); + + assertTrue(val >= 10_000 && val <= 40_000); + } + + assertEquals(30_001, cnt); // Streaming of a large result works well. + } + + // Currently we have no ways to do multiple passes through a merge table. + } +}
