# ignite-537 merge from sprint-3
Project: http://git-wip-us.apache.org/repos/asf/incubator-ignite/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-ignite/commit/347f3adb Tree: http://git-wip-us.apache.org/repos/asf/incubator-ignite/tree/347f3adb Diff: http://git-wip-us.apache.org/repos/asf/incubator-ignite/diff/347f3adb Branch: refs/heads/ignite-721 Commit: 347f3adba92ba9ca70f9509fc18105f3f5c0d743 Parents: eb85f06 60640de Author: sboikov <sboi...@gridgain.com> Authored: Fri Apr 10 15:10:20 2015 +0300 Committer: sboikov <sboi...@gridgain.com> Committed: Fri Apr 10 15:10:20 2015 +0300 ---------------------------------------------------------------------- assembly/release-hadoop.xml | 4 - bin/ignitevisorcmd.bat | 39 +-- .../examples/MessagingExamplesSelfTest.java | 2 +- .../java8/examples/BasicExamplesSelfTest.java | 3 + .../java8/examples/CacheExamplesSelfTest.java | 4 + .../examples/CheckpointExamplesSelfTest.java | 3 + .../examples/ClusterGroupExampleSelfTest.java | 3 +- .../examples/ContinuationExamplesSelfTest.java | 3 + .../ContinuousMapperExamplesSelfTest.java | 4 +- .../DeploymentExamplesMultiNodeSelfTest.java | 2 + .../examples/DeploymentExamplesSelfTest.java | 2 + .../HibernateL2CacheExampleSelfTest.java | 3 + .../java8/examples/IgfsExamplesSelfTest.java | 3 + .../examples/LifecycleExamplesSelfTest.java | 3 + .../MemcacheRestExamplesMultiNodeSelfTest.java | 2 + .../examples/MemcacheRestExamplesSelfTest.java | 2 + .../examples/MessagingExamplesSelfTest.java | 4 +- .../examples/MonteCarloExamplesSelfTest.java | 3 + .../examples/SpringBeanExamplesSelfTest.java | 3 + .../java8/examples/TaskExamplesSelfTest.java | 2 + .../client/ClientReconnectionSelfTest.java | 67 +++--- .../ignite/codegen/MessageCodeGenerator.java | 1 - modules/core/pom.xml | 2 +- .../java/org/apache/ignite/IgniteCache.java | 18 ++ .../java/org/apache/ignite/IgniteState.java | 2 +- .../configuration/CacheConfiguration.java | 69 +++--- .../configuration/IgniteConfiguration.java | 35 +-- .../internal/ComputeTaskInternalFuture.java | 2 +- .../ignite/internal/GridJobExecuteRequest.java | 10 +- .../ignite/internal/GridJobExecuteResponse.java | 4 +- .../apache/ignite/internal/GridTaskMessage.java | 30 --- .../ignite/internal/GridTaskSessionRequest.java | 6 +- .../org/apache/ignite/internal/IgnitionEx.java | 4 +- .../client/GridClientConfiguration.java | 14 +- .../GridClientConnectionManagerAdapter.java | 2 +- .../router/GridTcpRouterConfiguration.java | 6 +- .../internal/cluster/ClusterGroupAdapter.java | 150 ++++++++++-- .../internal/managers/GridManagerAdapter.java | 4 +- .../discovery/GridDiscoveryManager.java | 32 ++- .../eventstorage/GridEventStorageManager.java | 4 +- .../processors/cache/CacheMetricsImpl.java | 7 +- .../processors/cache/CacheMetricsSnapshot.java | 223 +++++++++++++++++- .../processors/cache/GridCacheAdapter.java | 12 +- .../processors/cache/GridCacheContext.java | 64 ++++- .../cache/GridCacheEvictionManager.java | 6 +- .../cache/GridCacheExplicitLockSpan.java | 2 +- .../processors/cache/GridCacheMapEntry.java | 6 +- .../processors/cache/GridCacheMvcc.java | 2 +- .../cache/GridCacheMvccCandidate.java | 50 +++- .../processors/cache/GridCacheMvccManager.java | 44 +++- .../processors/cache/GridCacheProcessor.java | 52 ++-- .../processors/cache/IgniteCacheProxy.java | 26 ++ .../distributed/GridDistributedCacheEntry.java | 12 +- .../distributed/dht/GridDhtCacheEntry.java | 36 ++- .../distributed/dht/GridDhtLockFuture.java | 15 +- .../dht/GridDhtTransactionalCacheAdapter.java | 14 +- .../distributed/dht/GridDhtTxFinishFuture.java | 22 +- .../cache/distributed/dht/GridDhtTxLocal.java | 2 + .../distributed/dht/GridDhtTxLocalAdapter.java | 30 ++- .../distributed/dht/GridDhtTxPrepareFuture.java | 28 ++- .../dht/atomic/GridDhtAtomicCache.java | 6 +- .../dht/colocated/GridDhtColocatedCache.java | 35 ++- .../colocated/GridDhtColocatedLockFuture.java | 46 +++- .../GridDhtPartitionsExchangeFuture.java | 39 ++- .../distributed/near/GridNearAtomicCache.java | 2 +- .../distributed/near/GridNearCacheEntry.java | 10 +- .../distributed/near/GridNearLockFuture.java | 13 +- .../near/GridNearTransactionalCache.java | 2 +- .../near/GridNearTxFinishFuture.java | 8 +- .../cache/distributed/near/GridNearTxLocal.java | 7 + .../near/GridNearTxPrepareFuture.java | 122 ++++++---- .../near/GridNearTxPrepareRequest.java | 64 +++-- .../local/atomic/GridLocalAtomicCache.java | 6 +- .../cache/query/GridCacheQueryAdapter.java | 2 +- .../continuous/CacheContinuousQueryManager.java | 2 +- .../cache/transactions/IgniteTxAdapter.java | 3 +- .../cache/transactions/IgniteTxHandler.java | 120 +++++++--- .../transactions/IgniteTxLocalAdapter.java | 82 ++++--- .../cache/transactions/IgniteTxManager.java | 7 + .../internal/processors/igfs/IgfsAsyncImpl.java | 4 +- .../ignite/internal/processors/igfs/IgfsEx.java | 4 +- .../processors/igfs/IgfsFileWorker.java | 180 -------------- .../processors/igfs/IgfsFileWorkerBatch.java | 225 +++++++----------- .../processors/igfs/IgfsFileWorkerTask.java | 32 --- .../internal/processors/igfs/IgfsImpl.java | 150 +++++++----- .../internal/processors/igfs/IgfsProcessor.java | 2 +- .../processors/rest/GridRestProcessor.java | 33 +-- .../security/GridSecurityProcessor.java | 13 +- .../processors/security/SecurityContext.java | 8 +- .../security/os/GridOsSecurityProcessor.java | 11 +- .../processors/task/GridTaskProcessor.java | 9 +- .../internal/util/future/GridFutureAdapter.java | 20 +- .../ignite/internal/util/worker/GridWorker.java | 6 +- .../visor/cache/VisorCacheClearTask.java | 3 + .../cache/VisorCacheQueryConfiguration.java | 9 - .../internal/visor/log/VisorLogSearchTask.java | 12 +- .../node/VisorNodeEventsCollectorTask.java | 3 + .../node/VisorSegmentationConfiguration.java | 4 +- .../internal/visor/util/VisorTaskUtils.java | 3 + .../plugin/security/AuthenticationContext.java | 12 +- .../security/GridSecurityCredentials.java | 184 --------------- .../GridSecurityCredentialsBasicProvider.java | 44 ---- .../GridSecurityCredentialsProvider.java | 44 ---- .../plugin/security/GridSecurityException.java | 58 ----- .../plugin/security/GridSecurityPermission.java | 69 ------ .../security/GridSecurityPermissionSet.java | 66 ------ .../plugin/security/GridSecuritySubject.java | 62 ----- .../security/GridSecuritySubjectType.java | 48 ---- .../plugin/security/SecurityCredentials.java | 191 +++++++++++++++ .../SecurityCredentialsBasicProvider.java | 44 ++++ .../security/SecurityCredentialsProvider.java | 44 ++++ .../plugin/security/SecurityException.java | 61 +++++ .../plugin/security/SecurityPermission.java | 69 ++++++ .../plugin/security/SecurityPermissionSet.java | 66 ++++++ .../ignite/plugin/security/SecuritySubject.java | 62 +++++ .../plugin/security/SecuritySubjectType.java | 48 ++++ .../segmentation/GridSegmentationPolicy.java | 47 ---- .../segmentation/GridSegmentationResolver.java | 64 ----- .../plugin/segmentation/SegmentationPolicy.java | 49 ++++ .../segmentation/SegmentationResolver.java | 65 +++++ .../org/apache/ignite/spi/IgniteSpiAdapter.java | 4 +- .../org/apache/ignite/spi/IgniteSpiContext.java | 4 +- .../spi/discovery/DiscoveryMetricsProvider.java | 10 + .../DiscoverySpiNodeAuthenticator.java | 2 +- .../discovery/tcp/TcpClientDiscoverySpi.java | 35 ++- .../spi/discovery/tcp/TcpDiscoverySpi.java | 57 ++++- .../tcp/internal/TcpDiscoveryNode.java | 76 +++++- .../messages/TcpDiscoveryHeartbeatMessage.java | 65 +++++ .../startup/cmdline/CommandLineStartup.java | 7 +- .../internal/GridLifecycleAwareSelfTest.java | 2 +- .../ignite/internal/GridProjectionSelfTest.java | 82 +++++++ .../ignite/internal/GridStartStopSelfTest.java | 8 +- .../CacheMetricsForClusterGroupSelfTest.java | 236 +++++++++++++++++++ .../cache/GridCacheBasicApiAbstractTest.java | 4 +- ...CacheOffHeapMultiThreadedUpdateSelfTest.java | 42 ++-- .../GridCacheOffHeapTieredAbstractSelfTest.java | 4 +- .../cache/GridCacheReloadSelfTest.java | 3 +- .../GridCacheReturnValueTransferSelfTest.java | 15 +- .../IgniteTxMultiThreadedAbstractTest.java | 5 +- ...GridCacheQueueMultiNodeAbstractSelfTest.java | 5 +- .../GridCacheSetAbstractSelfTest.java | 10 +- .../GridCacheSetFailoverAbstractSelfTest.java | 6 +- .../distributed/GridCacheEventAbstractTest.java | 16 +- .../GridCacheNodeFailureAbstractTest.java | 15 +- .../dht/IgniteCacheLockFailoverSelfTest.java | 156 ++++++++++++ .../dht/IgniteCacheMultiTxLockSelfTest.java | 219 +++++++++++++++++ ...licatedNearOnlyMultiNodeFullApiSelfTest.java | 2 +- .../continuous/GridEventConsumeSelfTest.java | 7 +- .../processors/igfs/IgfsAbstractSelfTest.java | 24 +- .../tostring/GridToStringBuilderSelfTest.java | 5 +- .../GridP2PMissedResourceCacheSizeSelfTest.java | 2 +- .../tcp/TcpClientDiscoverySelfTest.java | 5 +- .../testframework/GridSpiTestContext.java | 4 +- .../testframework/junits/GridAbstractTest.java | 6 +- .../junits/spi/GridSpiAbstractTest.java | 72 +----- .../IgniteCacheMetricsSelfTestSuite.java | 4 + .../ignite/testsuites/IgniteCacheTestSuite.java | 3 + .../IgniteSpiDiscoverySelfTestSuite.java | 2 + ...rrentLinkedHashMapMultiThreadedSelfTest.java | 4 +- .../HadoopDefaultMapReducePlannerSelfTest.java | 2 +- .../GridCacheAbstractFieldsQuerySelfTest.java | 10 +- .../processors/cache/GridCacheSwapSelfTest.java | 5 +- ...hePartitionedQueryMultiThreadedSelfTest.java | 2 +- .../cache/IgniteCacheQueryIndexSelfTest.java | 3 +- .../IgniteCachePartitionedQuerySelfTest.java | 5 +- .../IgniteCacheReplicatedQuerySelfTest.java | 18 +- .../processors/cache/GridCacheJtaSelfTest.java | 6 +- .../http/jetty/GridJettyRestHandler.java | 2 +- .../p2p/GridP2PUserVersionChangeSelfTest.java | 8 +- .../ignite/tools/classgen/ClassesGenerator.java | 2 +- .../ignite/visor/commands/VisorConsole.scala | 19 ++ 171 files changed, 3389 insertions(+), 1834 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/347f3adb/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/TcpClientDiscoverySpi.java ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/347f3adb/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/TcpDiscoverySpi.java ---------------------------------------------------------------------- diff --cc modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/TcpDiscoverySpi.java index bc0fec5,4b43fec..7bd1a97 --- a/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/TcpDiscoverySpi.java +++ b/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/TcpDiscoverySpi.java @@@ -5302,310 -5186,11 +5324,319 @@@ public class TcpDiscoverySpi extends Tc } /** + * @param node Node created event. + * @return Class loader for custom event unmarshalling. + */ + @Nullable protected ClassLoader customMessageClassLoader(TcpDiscoveryNode node) { + assert ignite != null; + + if (!ignite.configuration().isPeerClassLoadingEnabled()) + return null; + + if (node.id().equals(getLocalNodeId())) + return locLdr; + + DiscoveryDeploymentClassLoader ldr = p2pLdrs.get(node.id()); + + if (ldr == null) + ldr = F.addIfAbsent(p2pLdrs, node.id(), new DiscoveryDeploymentClassLoader(node)); + + return ldr; + } + + /** + * @param joiningNode Joining node. + * @param nodeId Remote node provided data. + * @return Class loader for exchange data unmarshalling. + */ + @Nullable protected ClassLoader exchangeClassLoader(TcpDiscoveryNode joiningNode, UUID nodeId) { + assert joiningNode != null; + assert ignite != null; + + if (!ignite.configuration().isPeerClassLoadingEnabled()) + return null; + + if (nodeId.equals(getLocalNodeId())) + return locLdr; + + TcpDiscoveryNode node; + + if (joiningNode.id().equals(nodeId)) + node = joiningNode; + else { + node = ring.node(nodeId); + + if (node == null) { + if (log.isDebugEnabled()) + log.debug("Node provided exchange data left, will use local class loader " + + "for exchange data [nodeId=" + nodeId + ']'); + + return locLdr; + } + } + + if (node.isClient()) // Do not support loading from client nodes. + return locLdr; + + DiscoveryDeploymentClassLoader ldr = p2pLdrs.get(nodeId); + + if (ldr == null) + ldr = F.addIfAbsent(p2pLdrs, nodeId, new DiscoveryDeploymentClassLoader(node)); + + return ldr; + } + + /** + * @param nodeId Node ID. + * @return Marshalled exchange data. + * @throws IgniteSpiException If failed. + */ + private Map<Integer, byte[]> collectExchangeData(UUID nodeId) throws IgniteSpiException { + Map<Integer, Object> data = exchange.collect(nodeId); + + Map<Integer, byte[]> data0 = U.newHashMap(data.size()); + + for (Map.Entry<Integer, Object> entry : data.entrySet()) { + try { + byte[] bytes = marsh.marshal(entry.getValue()); + + data0.put(entry.getKey(), bytes); + } + catch (IgniteCheckedException e) { + U.error(log, "Failed to marshal discovery data " + + "[comp=" + entry.getKey() + ", data=" + entry.getValue() + ']', e); + + throw new IgniteSpiException("Failed to marshal discovery data.", e); + } + } + + return data0; + } + + /** + * + */ + private class DiscoveryDeploymentClassLoader extends ClassLoader { + /** */ + private final UUID nodeId; + + /** */ + private volatile TcpDiscoveryNode node; + + /** */ + private Socket sock; + + /** */ + private final ReadWriteLock lock = new ReentrantReadWriteLock(); + + /** + * @param node Node. + */ + public DiscoveryDeploymentClassLoader(TcpDiscoveryNode node) { + assert !node.isClient(); + assert !node.id().equals(getLocalNodeId()); + + this.node = node; + + nodeId = node.id(); + } + + /** + * @return Target node ID. + */ + UUID nodeId() { + return nodeId; + } + + /** + * Node left callback. + */ + void onNodeLeft() { + lock.writeLock().lock(); + + try { + if (sock != null) { + if (log.isDebugEnabled()) + log.debug("Closing deployment class loader connection on node left [node=" + nodeId + ']'); + + U.closeQuiet(sock); + + sock = null; + } + + node = null; + } + finally { + lock.writeLock().unlock(); + } + } + + /** + * Closes connection if there is no class loading in progress. + */ + void closeConnectionIfNotUsed() { + if (lock.writeLock().tryLock()) { + try { + if (sock != null) { + if (log.isDebugEnabled()) + log.debug("Closing idle deployment class loader connection [node=" + nodeId + ']'); + + U.closeQuiet(sock); + + sock = null; + } + } + finally { + lock.writeLock().unlock(); + } + } + } + + /** {@inheritDoc} */ + @Override protected Class<?> findClass(String name) throws ClassNotFoundException { + if (node == null) + throw new ClassNotFoundException("Failed to load class, peer node left " + + "[cls=" +name + ", node=" + nodeId + ']'); + + lock.readLock().lock(); + + try { + TcpDiscoveryGetClassResponse res = requestClass(name); + + if (res == null) + throw new ClassNotFoundException("Failed to load class, can not connect to peer node " + + "[cls=" + name + ", node=" + nodeId + ']'); + + if (res.error() != null) + throw new ClassNotFoundException(res.error()); + + assert res.classBytes() != null; + + return defineClass(name, res.classBytes(), 0, res.classBytes().length); + } + finally { + lock.readLock().unlock(); + } + } + + /** + * @param name Class name. + * @return Class response or {@code null} if failed to connect. + */ + @Nullable private synchronized TcpDiscoveryGetClassResponse requestClass(String name) { + TcpDiscoveryGetClassRequest msg = new TcpDiscoveryGetClassRequest(getLocalNodeId(), name); + + for (int i = 0; i < reconCnt; i++) { + if (sock == null) { + TcpDiscoveryNode node0 = node; + + if (node0 == null) + return null; // Node left. + + sock = connect(node0); + + if (sock == null) + break; + } + + try { + return request(sock, msg); + } + catch (IOException | IgniteCheckedException e) { + U.closeQuiet(sock); + + sock = null; + } + } + + node = null; // Consider node failed. + + p2pLdrs.remove(nodeId, this); + + return null; + } + + /** + * @param sock Socket. + * @param msg Message. + * @return Response. + * @throws IOException If request failed. + * @throws IgniteCheckedException If request failed. + */ + private TcpDiscoveryGetClassResponse request(Socket sock, TcpDiscoveryGetClassRequest msg) + throws IOException, IgniteCheckedException + { + long tstamp = U.currentTimeMillis(); + + writeToSocket(sock, msg); + + stats.onMessageSent(msg, U.currentTimeMillis() - tstamp); + + TcpDiscoveryGetClassResponse res = readMessage(sock, null, netTimeout); + + stats.onMessageReceived(res); + + return res; + } + + /** + * @param node Node. + * @return Socket or {@code null} if failed to connect. + */ + private Socket connect(TcpDiscoveryNode node) { + Socket sock = null; + + for (InetSocketAddress addr : getNodeAddresses(node, U.sameMacs(locNode, node))) { + sock = connect(addr); + + if (sock != null) + break; + } + + return sock; + } + + /** + * @param addr Address. + * @return Socket or {@code null} if failed to connect. + */ + private Socket connect(InetSocketAddress addr) { + TcpDiscoveryHandshakeRequest req = new TcpDiscoveryHandshakeRequest(getLocalNodeId()); + + for (int i = 0; i < reconCnt; i++) { + Socket sock = null; + + long tstamp = U.currentTimeMillis(); + + try { + sock = openSocket(addr); + + writeToSocket(sock, req); + + TcpDiscoveryHandshakeResponse res = readMessage(sock, null, netTimeout); + + if (!res.creatorNodeId().equals(nodeId)) + return null; + + stats.onClientSocketInitialized(U.currentTimeMillis() - tstamp); + + return sock; + } + catch (IOException | IgniteCheckedException e) { + U.closeQuiet(sock); + } + } + + return null; + } + } ++ ++ /** + * @param msg Message. + * @param nodeId Node ID. + */ + private static void removeMetrics(TcpDiscoveryHeartbeatMessage msg, UUID nodeId) { + msg.removeMetrics(nodeId); + msg.removeCacheMetrics(nodeId); + } } http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/347f3adb/modules/core/src/test/java/org/apache/ignite/internal/processors/continuous/GridEventConsumeSelfTest.java ----------------------------------------------------------------------
