[
https://issues.apache.org/jira/browse/GEODE-8696?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel&focusedCommentId=17243548#comment-17243548
]
ASF GitHub Bot commented on GEODE-8696:
---------------------------------------
kirklund commented on a change in pull request #5728:
URL: https://github.com/apache/geode/pull/5728#discussion_r535683589
##########
File path:
geode-core/src/distributedTest/java/org/apache/geode/management/JMXMBeanLocatorRestartTest.java
##########
@@ -0,0 +1,317 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
contributor license
+ * agreements. See the NOTICE file distributed with this work for additional
information regarding
+ * copyright ownership. The ASF licenses this file to You under the Apache
License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the
License. You may obtain a
+ * copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
distributed under the License
+ * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
KIND, either express
+ * or implied. See the License for the specific language governing permissions
and limitations under
+ * the License.
+ */
+package org.apache.geode.management;
+
+import static java.lang.management.ManagementFactory.getPlatformMBeanServer;
+import static java.util.Arrays.asList;
+import static java.util.concurrent.TimeUnit.MINUTES;
+import static javax.management.ObjectName.getInstance;
+import static org.apache.geode.cache.Region.SEPARATOR;
+import static
org.apache.geode.distributed.ConfigurationProperties.HTTP_SERVICE_PORT;
+import static org.apache.geode.distributed.ConfigurationProperties.JMX_MANAGER;
+import static
org.apache.geode.distributed.ConfigurationProperties.JMX_MANAGER_PORT;
+import static
org.apache.geode.distributed.ConfigurationProperties.JMX_MANAGER_START;
+import static org.apache.geode.distributed.ConfigurationProperties.LOCATORS;
+import static org.apache.geode.distributed.ConfigurationProperties.LOG_FILE;
+import static
org.apache.geode.distributed.ConfigurationProperties.MAX_WAIT_TIME_RECONNECT;
+import static
org.apache.geode.distributed.ConfigurationProperties.MEMBER_TIMEOUT;
+import static
org.apache.geode.distributed.internal.membership.api.MembershipManagerHelper.crashDistributedSystem;
+import static
org.apache.geode.internal.AvailablePortHelper.getRandomAvailableTCPPorts;
+import static org.apache.geode.test.awaitility.GeodeAwaitility.getTimeout;
+import static org.apache.geode.test.dunit.Disconnect.disconnectAllFromDS;
+import static org.apache.geode.test.dunit.IgnoredException.addIgnoredException;
+import static org.apache.geode.test.dunit.Invoke.invokeInEveryVM;
+import static org.apache.geode.test.dunit.VM.getVM;
+import static org.apache.geode.test.dunit.VM.getVMId;
+import static org.apache.geode.util.internal.GeodeGlossary.GEMFIRE_PREFIX;
+import static org.assertj.core.api.Assertions.assertThat;
+import static org.mockito.Mockito.mock;
+
+import java.io.File;
+import java.io.Serializable;
+import java.util.HashSet;
+import java.util.Set;
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.atomic.AtomicReference;
+
+import javax.management.MalformedObjectNameException;
+import javax.management.ObjectName;
+
+import org.awaitility.core.ConditionFactory;
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Rule;
+import org.junit.Test;
+
+import org.apache.geode.CancelException;
+import org.apache.geode.ForcedDisconnectException;
+import org.apache.geode.alerting.internal.spi.AlertingIOException;
+import org.apache.geode.cache.CacheClosedException;
+import org.apache.geode.distributed.DistributedSystemDisconnectedException;
+import org.apache.geode.distributed.LocatorLauncher;
+import org.apache.geode.distributed.ServerLauncher;
+import org.apache.geode.distributed.internal.InternalLocator;
+import
org.apache.geode.distributed.internal.membership.api.MemberDisconnectedException;
+import org.apache.geode.test.awaitility.GeodeAwaitility;
+import org.apache.geode.test.dunit.VM;
+import org.apache.geode.test.dunit.rules.DistributedErrorCollector;
+import org.apache.geode.test.dunit.rules.DistributedRestoreSystemProperties;
+import org.apache.geode.test.dunit.rules.DistributedRule;
+import org.apache.geode.test.junit.rules.GfshCommandRule;
+import
org.apache.geode.test.junit.rules.serializable.SerializableTemporaryFolder;
+
+public class JMXMBeanLocatorRestartTest implements Serializable {
+
+ private static final long TIMEOUT_MILLIS = getTimeout().toMillis();
+ private static final LocatorLauncher DUMMY_LOCATOR =
mock(LocatorLauncher.class);
+ private static final ServerLauncher DUMMY_SERVER =
mock(ServerLauncher.class);
+
+ private static final AtomicReference<CountDownLatch> BEFORE =
+ new AtomicReference<>(new CountDownLatch(0));
+ private static final AtomicReference<CountDownLatch> AFTER =
+ new AtomicReference<>(new CountDownLatch(0));
+
+ private static final AtomicReference<LocatorLauncher> LOCATOR =
+ new AtomicReference<>(DUMMY_LOCATOR);
+ private static final AtomicReference<ServerLauncher> SERVER =
+ new AtomicReference<>(DUMMY_SERVER);
+
+ private VM locatorVM;
+ private VM serverVM;
+
+ private String locatorName;
+ private String serverName;
+ private String locators;
+ private int locatorPort;
+ private int locatorJmxPort;
+ private String regionName;
+ private Set<ObjectName> mxbeansOnServer;
+ private Set<ObjectName> mxbeansOnLocator;
+
+ @Rule
+ public DistributedRule distributedRule = new DistributedRule();
+ @Rule
+ public SerializableTemporaryFolder temporaryFolder = new
SerializableTemporaryFolder();
+ @Rule
+ public DistributedErrorCollector errorCollector = new
DistributedErrorCollector();
+ @Rule
+ public transient GfshCommandRule gfsh = new GfshCommandRule();
+ @Rule
+ public DistributedRestoreSystemProperties restoreProps = new
DistributedRestoreSystemProperties();
+
+ @Before
+ public void setUp() throws Exception {
+ locatorVM = getVM(1);
+ serverVM = getVM(0);
+
+ for (VM vm : asList(locatorVM, serverVM)) {
+ vm.invoke(() -> System.setProperty(GEMFIRE_PREFIX +
"standard-output-always-on", "true"));
+ }
+
+ locatorName = "locator1";
+ serverName = "server1";
+ File locator1Dir = temporaryFolder.newFolder(locatorName);
+ File serverDir = temporaryFolder.newFolder(serverName);
+
+ int[] port = getRandomAvailableTCPPorts(4);
+ locatorPort = port[0];
+ locatorJmxPort = port[2];
+ locators = "localhost[" + locatorPort + "]";
+
+ locatorVM.invoke(() -> {
+ startLocator(locatorName, locator1Dir, locatorPort, locatorJmxPort,
locators);
+ });
+
+ serverVM.invoke(() -> startServer(serverName, serverDir, locators));
+
+ gfsh.connectAndVerify(locatorJmxPort, GfshCommandRule.PortType.jmxManager);
+
+ regionName = "region1";
+ String createRegionCommand = "create region --type=REPLICATE --name=" +
SEPARATOR + regionName;
+ gfsh.executeAndAssertThat(createRegionCommand).statusIsSuccess();
+
+ addIgnoredException(AlertingIOException.class);
+ addIgnoredException(CacheClosedException.class);
+ addIgnoredException(CancelException.class);
+ addIgnoredException(DistributedSystemDisconnectedException.class);
+ addIgnoredException(ForcedDisconnectException.class);
+ addIgnoredException(MemberDisconnectedException.class);
+ addIgnoredException("Possible loss of quorum");
+
+ mxbeansOnServer = serverVM.invoke(() -> {
+ await().untilAsserted(() -> {
+
assertThat(getPlatformMBeanServer().queryNames(getInstance("GemFire:*"), null))
+ .as("GemFire mbeans on server1")
+ .containsAll(expectedServerMXBeans(serverName, regionName));
+ });
+ return getPlatformMBeanServer().queryNames(getInstance("GemFire:*"),
null);
+ });
+
+ mxbeansOnLocator = locatorVM.invoke(() -> {
+ await().untilAsserted(() -> {
+
assertThat(getPlatformMBeanServer().queryNames(getInstance("GemFire:*"), null))
+ .as("GemFire mbeans on locator1")
+ .containsAll(expectedServerMXBeans(serverName, regionName))
+ .containsAll(expectedLocatorMXBeans(locatorName))
+ .containsAll(expectedDistributedMXBeans(regionName));
+ });
+
+ return getPlatformMBeanServer().queryNames(getInstance("GemFire:*"),
null);
+ });
+ }
+
+ @After
+ public void tearDown() {
+ invokeInEveryVM(() -> {
+ BEFORE.get().countDown();
Review comment:
I copied too much code from JMXMBeanReconnectDUnitTest which focuses on
restart of servers.
JMXMBeanLocatorRestartTest is new and focuses on restart of locator. I need
to do more cleanup. Thanks!
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
us...@infra.apache.org
> Startup of JMX Manager may hang during crash of other members
> -------------------------------------------------------------
>
> Key: GEODE-8696
> URL: https://issues.apache.org/jira/browse/GEODE-8696
> Project: Geode
> Issue Type: Bug
> Components: jmx, management
> Reporter: Kirk Lund
> Assignee: Kirk Lund
> Priority: Major
> Labels: GeodeOperationAPI, pull-request-available
>
> The fix for GEODE-7400 removed final from the executorService field to
> introduce a Supplier<ExecutorService>. I think this hang was caused by adding
> synchronized to FederatingManager.executeTask instead of making
> executorService a volatile field.
> _vm_3_thr_3_client2_host2_21145_ hung while synchronized on
> *0x00000000f6316520*:
> {noformat}
> "vm_3_thr_3_client2_host2_21145" #56 daemon prio=5 os_prio=0
> tid=0x00007f1854002000 nid=0x5326 waiting on condition [0x00007f18520e2000]
> java.lang.Thread.State: WAITING (parking)
> at sun.misc.Unsafe.park(Native Method)
> - parking to wait for <0x00000000f6364030> (a
> java.util.concurrent.FutureTask)
> at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
> at java.util.concurrent.FutureTask.awaitDone(FutureTask.java:429)
> at java.util.concurrent.FutureTask.get(FutureTask.java:191)
> at
> java.util.concurrent.AbstractExecutorService.invokeAll(AbstractExecutorService.java:244)
> at
> org.apache.geode.management.internal.FederatingManager.startManagingActivity(FederatingManager.java:256)
> at
> org.apache.geode.management.internal.FederatingManager.startManager(FederatingManager.java:121)
> - locked <0x00000000f6316520> (a
> org.apache.geode.management.internal.FederatingManager)
> at
> org.apache.geode.management.internal.SystemManagementService.startManager(SystemManagementService.java:373)
> - locked <0x00000000fed91a70> (a java.util.HashMap)
> at
> org.apache.geode.management.internal.beans.ManagementAdapter.handleCacheCreation(ManagementAdapter.java:197)
> at
> org.apache.geode.management.internal.beans.ManagementListener.handleEvent(ManagementListener.java:127)
> at
> org.apache.geode.distributed.internal.InternalDistributedSystem.notifyResourceEventListeners(InternalDistributedSystem.java:2089)
> at
> org.apache.geode.distributed.internal.InternalDistributedSystem.handleResourceEvent(InternalDistributedSystem.java:643)
> at
> org.apache.geode.internal.cache.GemFireCacheImpl.initialize(GemFireCacheImpl.java:1363)
> at
> org.apache.geode.internal.cache.InternalCacheBuilder.create(InternalCacheBuilder.java:191)
> - locked <0x00000000e11065f0> (a java.lang.Class for
> org.apache.geode.internal.cache.GemFireCacheImpl)
> - locked <0x00000000e1101220> (a java.lang.Class for
> org.apache.geode.internal.cache.InternalCacheBuilder)
> at
> org.apache.geode.internal.cache.InternalCacheBuilder.create(InternalCacheBuilder.java:158)
> - locked <0x00000000e1101220> (a java.lang.Class for
> org.apache.geode.internal.cache.InternalCacheBuilder)
> at org.apache.geode.cache.CacheFactory.create(CacheFactory.java:142)
> at
> hydra.CacheVersionHelper.configureAndCreateCache(CacheVersionHelper.java:51)
> at hydra.CacheHelper.createCacheWithHttpService(CacheHelper.java:127)
> - locked <0x00000000fecab060> (a java.lang.Class for
> hydra.CacheHelper)
> at hydra.CacheHelper.createCache(CacheHelper.java:87)
> at
> splitBrain.NetworkPartitionTest.initialize(NetworkPartitionTest.java:293)
> at
> splitBrain.NetworkPartitionTest.initializeInstance(NetworkPartitionTest.java:228)
> - locked <0x00000000e14a2db0> (a java.lang.Class for
> splitBrain.NetworkPartitionTest)
> at
> splitBrain.NetworkPartitionTest.HydraTask_initialize(NetworkPartitionTest.java:203)
> - locked <0x00000000e14a2db0> (a java.lang.Class for
> splitBrain.NetworkPartitionTest)
> at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
> at
> sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
> at
> sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
> at java.lang.reflect.Method.invoke(Method.java:498)
> at hydra.MethExecutor.execute(MethExecutor.java:173)
> at hydra.MethExecutor.execute(MethExecutor.java:141)
> at hydra.TestTask.execute(TestTask.java:197)
> at hydra.RemoteTestModule$1.run(RemoteTestModule.java:213)
> {noformat}
> _vm_3_thr_3_client2_host2_21145_ is waiting for _FederatingManager1_ to
> complete:
> {noformat}
> "FederatingManager1" #66 daemon prio=5 os_prio=0 tid=0x00007f1874327800
> nid=0x5331 waiting on condition [0x00007f18515d9000]
> java.lang.Thread.State: TIMED_WAITING (parking)
> at sun.misc.Unsafe.park(Native Method)
> - parking to wait for <0x00000000f63b2590> (a
> java.util.concurrent.CountDownLatch$Sync)
> at
> java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
> at
> java.util.concurrent.locks.AbstractQueuedSynchronizer.doAcquireSharedNanos(AbstractQueuedSynchronizer.java:1037)
> at
> java.util.concurrent.locks.AbstractQueuedSynchronizer.tryAcquireSharedNanos(AbstractQueuedSynchronizer.java:1328)
> at java.util.concurrent.CountDownLatch.await(CountDownLatch.java:277)
> at
> org.apache.geode.internal.util.concurrent.StoppableCountDownLatch.await(StoppableCountDownLatch.java:72)
> at
> org.apache.geode.distributed.internal.ReplyProcessor21.basicWait(ReplyProcessor21.java:731)
> at
> org.apache.geode.distributed.internal.ReplyProcessor21.waitForReplies(ReplyProcessor21.java:639)
> at
> org.apache.geode.distributed.internal.ReplyProcessor21.waitForReplies(ReplyProcessor21.java:620)
> at
> org.apache.geode.distributed.internal.ReplyProcessor21.waitForReplies(ReplyProcessor21.java:534)
> at
> org.apache.geode.internal.cache.StateFlushOperation.flush(StateFlushOperation.java:244)
> at
> org.apache.geode.internal.cache.InitialImageOperation.getFromOne(InitialImageOperation.java:432)
> at
> org.apache.geode.internal.cache.DistributedRegion.getInitialImageAndRecovery(DistributedRegion.java:1236)
> at
> org.apache.geode.internal.cache.DistributedRegion.initialize(DistributedRegion.java:1082)
> at
> org.apache.geode.internal.cache.GemFireCacheImpl.createVMRegion(GemFireCacheImpl.java:2971)
> at
> org.apache.geode.internal.cache.InternalCacheForClientAccess.createInternalRegion(InternalCacheForClientAccess.java:255)
> at
> org.apache.geode.management.internal.FederatingManager.addMemberArtifacts(FederatingManager.java:448)
> - locked <0x00000000fec1a480> (a
> org.apache.geode.distributed.internal.membership.InternalDistributedMember)
> at
> org.apache.geode.management.internal.FederatingManager$GIITask.call(FederatingManager.java:560)
> at
> org.apache.geode.management.internal.FederatingManager$GIITask.call(FederatingManager.java:550)
> at java.util.concurrent.FutureTask.run(FutureTask.java:266)
> at
> java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
> at
> java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
> at java.lang.Thread.run(Thread.java:748)
> {noformat}
> _DM-MemberEventInvoker_ is preventing _FederatingManager1_ from completing
> its call to {{ReplyProcessor21.basicWait}} while waiting to lock
> *0x00000000f6316520* which is held by _vm_3_thr_3_client2_host2_21145_:
> {noformat}
> "DM-MemberEventInvoker" #28 daemon prio=5 os_prio=0 tid=0x00007f1860356000
> nid=0x5313 waiting for monitor entry [0x00007f18531f2000]
> java.lang.Thread.State: BLOCKED (on object monitor)
> at
> org.apache.geode.management.internal.FederatingManager.executeTask(FederatingManager.java:230)
> - waiting to lock <0x00000000f6316520> (a
> org.apache.geode.management.internal.FederatingManager)
> at
> org.apache.geode.management.internal.FederatingManager.removeMember(FederatingManager.java:160)
> at
> org.apache.geode.management.internal.ManagementMembershipListener.memberDeparted(ManagementMembershipListener.java:57)
> at
> org.apache.geode.distributed.internal.ClusterDistributionManager$MemberCrashedEvent.handleEvent(ClusterDistributionManager.java:2519)
> at
> org.apache.geode.distributed.internal.ClusterDistributionManager$MemberEvent.handleEvent(ClusterDistributionManager.java:2424)
> at
> org.apache.geode.distributed.internal.ClusterDistributionManager$MemberEvent.handleEvent(ClusterDistributionManager.java:2413)
> at
> org.apache.geode.distributed.internal.ClusterDistributionManager.handleMemberEvent(ClusterDistributionManager.java:1401)
> at
> org.apache.geode.distributed.internal.ClusterDistributionManager.access$200(ClusterDistributionManager.java:108)
> at
> org.apache.geode.distributed.internal.ClusterDistributionManager$MemberEventInvoker.run(ClusterDistributionManager.java:1433)
> at java.lang.Thread.run(Thread.java:748)
> {noformat}
--
This message was sent by Atlassian Jira
(v8.3.4#803005)
