Hello guys,
I am having 5 nodes solr cloud setup with single shard. The solr version is 5.2.1. server1 (http://146.XXX.com:4001/solr/contracts_shard1_replica4)is the leader. A document with id '43e14a86cbdd422880cac22d9a15d3c0' was not replicated 3 nodes. Log shows that the "{add=[43e14a86cbdd422880cac22d9a15d3c0 (1573510697298427904)]}" request is received only by leader and server5 node. The server2, server3 & server4 node did not receive the request and hence the document is missing in these nodes. Search "43e14a86cbdd422880cac22d9a15d3c0 " C:\solrIssue\solr_server1.log INFO - 2017-07-21 05:54:59.853; [contracts shard1 core_node2 contracts_shard1_replica4] org.apache.solr.update.processor.LogUpdateProcessor; [contracts_shard1_replica4] webapp=/solr path=/update params= {wt=javabin&version=2} {deleteByQuery=id:(9467353f398448788c261aa347d75b8b 93332ab7f7ff4141a371713871ab65ad 8568e0eab8364bfc89c876aadfa01022 43e14a86cbdd422880cac22d9a15d3c0 a0af8cb24ef94d25b9691eee1f7024ca 8ad... INFO - 2017-07-21 05:54:59.853; [contracts shard1 core_node2 contracts_shard1_replica4] org.apache.solr.update.processor.LogUpdateProcessor; [contracts_shard1_replica4] webapp=/solr path=/update params= {wt=javabin&version=2} {deleteByQuery=id:(9467353f398448788c261aa347d75b8b 93332ab7f7ff4141a371713871ab65ad 8568e0eab8364bfc89c876aadfa01022 43e14a86cbdd422880cac22d9a15d3c0 a0af8cb24ef94d25b9691eee1f7024ca 8ad... INFO - 2017-07-21 05:59:23.845; [contracts shard1 core_node2 contracts_shard1_replica4] org.apache.solr.update.processor.LogUpdateProcessor; [contracts_shard1_replica4] webapp=/solr path=/update params= {wt=javabin&version=2} {add=[43e14a86cbdd422880cac22d9a15d3c0 (1573510697298427904)]} 0 26582 C:\solrIssue\solr_server2\solr.log.1 INFO - 2017-07-21 05:54:59.595; [contracts shard1 core_node4 contracts_shard1_replica5] org.apache.solr.update.processor.LogUpdateProcessor; [contracts_shard1_replica5] webapp=/solr path=/update params= {update.distrib=FROMLEADER&_version_=-1573510446380482560&distrib.from=http://146.XXX.com:4001/solr/contracts_shard1_replica4/&wt=javabin&version=2} {deleteByQuery=id:(9467353f398448788c261aa347d75b8b 93332ab7f7ff4141a371713871ab65ad 8568e0eab8364bfc89c876aadfa01022 43e14a86cbdd422880cac22d9a15d3c0 a0af8cb24ef94d25b9691eee1f7024ca 8ad... INFO - 2017-07-21 05:54:59.595; [contracts shard1 core_node4 contracts_shard1_replica5] org.apache.solr.update.processor.LogUpdateProcessor; [contracts_shard1_replica5] webapp=/solr path=/update params= {update.distrib=FROMLEADER&_version_=-1573510446380482560&distrib.from=http://146.XXX.com:4001/solr/contracts_shard1_replica4/&wt=javabin&version=2} {deleteByQuery=id:(9467353f398448788c261aa347d75b8b 93332ab7f7ff4141a371713871ab65ad 8568e0eab8364bfc89c876aadfa01022 43e14a86cbdd422880cac22d9a15d3c0 a0af8cb24ef94d25b9691eee1f7024ca 8ad... C:\solrIssue\solr_server3.log INFO - 2017-07-21 05:54:59.844; [contracts shard1 core_node1 contracts_shard1_replica3] org.apache.solr.update.processor.LogUpdateProcessor; [contracts_shard1_replica3] webapp=/solr path=/update params= {update.distrib=FROMLEADER&_version_=-1573510446380482560&distrib.from=http://146.XXX.com:4001/solr/contracts_shard1_replica4/&wt=javabin&version=2} {deleteByQuery=id:(9467353f398448788c261aa347d75b8b 93332ab7f7ff4141a371713871ab65ad 8568e0eab8364bfc89c876aadfa01022 43e14a86cbdd422880cac22d9a15d3c0 a0af8cb24ef94d25b9691eee1f7024ca 8ad... INFO - 2017-07-21 05:54:59.844; [contracts shard1 core_node1 contracts_shard1_replica3] org.apache.solr.update.processor.LogUpdateProcessor; [contracts_shard1_replica3] webapp=/solr path=/update params= {update.distrib=FROMLEADER&_version_=-1573510446380482560&distrib.from=http://146.XXX.com:4001/solr/contracts_shard1_replica4/&wt=javabin&version=2} {deleteByQuery=id:(9467353f398448788c261aa347d75b8b 93332ab7f7ff4141a371713871ab65ad 8568e0eab8364bfc89c876aadfa01022 43e14a86cbdd422880cac22d9a15d3c0 a0af8cb24ef94d25b9691eee1f7024ca 8ad... C:\solrIssue\solr_server4\solr.log.1 INFO - 2017-07-21 05:54:59.734; [contracts shard1 core_node3 contracts_shard1_replica1] org.apache.solr.update.processor.LogUpdateProcessor; [contracts_shard1_replica1] webapp=/solr path=/update params= {update.distrib=FROMLEADER&_version_=-1573510446380482560&distrib.from=http://146.XXX.com:4001/solr/contracts_shard1_replica4/&wt=javabin&version=2} {deleteByQuery=id:(9467353f398448788c261aa347d75b8b 93332ab7f7ff4141a371713871ab65ad 8568e0eab8364bfc89c876aadfa01022 43e14a86cbdd422880cac22d9a15d3c0 a0af8cb24ef94d25b9691eee1f7024ca 8ad... INFO - 2017-07-21 05:54:59.734; [contracts shard1 core_node3 contracts_shard1_replica1] org.apache.solr.update.processor.LogUpdateProcessor; [contracts_shard1_replica1] webapp=/solr path=/update params= {update.distrib=FROMLEADER&_version_=-1573510446380482560&distrib.from=http://146.XXX.com:4001/solr/contracts_shard1_replica4/&wt=javabin&version=2} {deleteByQuery=id:(9467353f398448788c261aa347d75b8b 93332ab7f7ff4141a371713871ab65ad 8568e0eab8364bfc89c876aadfa01022 43e14a86cbdd422880cac22d9a15d3c0 a0af8cb24ef94d25b9691eee1f7024ca 8ad... C:\solrIssue\solr_server5\solr.log.1 INFO - 2017-07-21 05:54:59.851; [contracts shard1 core_node5 contracts_shard1_replica2] org.apache.solr.update.processor.LogUpdateProcessor; [contracts_shard1_replica2] webapp=/solr path=/update params= {update.distrib=FROMLEADER&_version_=-1573510446380482560&distrib.from=http://146.XXX.com:4001/solr/contracts_shard1_replica4/&wt=javabin&version=2} {deleteByQuery=id:(9467353f398448788c261aa347d75b8b 93332ab7f7ff4141a371713871ab65ad 8568e0eab8364bfc89c876aadfa01022 43e14a86cbdd422880cac22d9a15d3c0 a0af8cb24ef94d25b9691eee1f7024ca 8ad... INFO - 2017-07-21 05:54:59.851; [contracts shard1 core_node5 contracts_shard1_replica2] org.apache.solr.update.processor.LogUpdateProcessor; [contracts_shard1_replica2] webapp=/solr path=/update params= {update.distrib=FROMLEADER&_version_=-1573510446380482560&distrib.from=http://146.XXX.com:4001/solr/contracts_shard1_replica4/&wt=javabin&version=2} {deleteByQuery=id:(9467353f398448788c261aa347d75b8b 93332ab7f7ff4141a371713871ab65ad 8568e0eab8364bfc89c876aadfa01022 43e14a86cbdd422880cac22d9a15d3c0 a0af8cb24ef94d25b9691eee1f7024ca 8ad... INFO - 2017-07-21 05:59:23.843; [contracts shard1 core_node5 contracts_shard1_replica2] org.apache.solr.update.processor.LogUpdateProcessor; [contracts_shard1_replica2] webapp=/solr path=/update params= {update.distrib=FROMLEADER&distrib.from=http://146.XXX.com:4001/solr/contracts_shard1_replica4/&wt=javabin&version=2} {add=[43e14a86cbdd422880cac22d9a15d3c0 (1573510697298427904)]} 0 16241 There are 2 more documents which are not replicated and having similar logs entries. Can you please help me to under stand why the replication not happen on these nodes? What can be the possible reasons and how to troubleshoot the issue if reoccurs? There is no error indicating failure of replication or nodes. However there are some errors "java.util.concurrent.TimeoutException: Idle timeout expired: 50000/50000 ms". ERROR - 2017-07-21 05:59:35.181; [contracts shard1 core_node3 contracts_shard1_replica1] org.apache.solr.common.SolrException; null:java.io.IOException: java.util.concurrent.TimeoutException: Idle timeout expired: 50000/50000 ms at org.eclipse.jetty.util.SharedBlockingCallback$Blocker.block (SharedBlockingCallback.java:234) at org.eclipse.jetty.server.HttpInputOverHTTP.blockForContent (HttpInputOverHTTP.java:66) at org.eclipse.jetty.server.HttpInput$1.waitForContent (HttpInput.java:476) at org.eclipse.jetty.server.HttpInput.read(HttpInput.java:121) at org.apache.solr.common.util.FastInputStream.readWrappedStream (FastInputStream.java:80) at org.apache.solr.common.util.FastInputStream.refill (FastInputStream.java:89) at org.apache.solr.common.util.FastInputStream.readByte (FastInputStream.java:207) at org.apache.solr.common.util.JavaBinCodec.readVal (JavaBinCodec.java:188) at org.apache.solr.common.util.JavaBinCodec.readSolrInputDocument (JavaBinCodec.java:424) at org.apache.solr.common.util.JavaBinCodec.readVal (JavaBinCodec.java:245) at org.apache.solr.common.util.JavaBinCodec.readSolrInputDocument (JavaBinCodec.java:414) at org.apache.solr.common.util.JavaBinCodec.readVal (JavaBinCodec.java:245) at org.apache.solr.common.util.JavaBinCodec.readMapEntry (JavaBinCodec.java:542) at org.apache.solr.common.util.JavaBinCodec.readVal (JavaBinCodec.java:249) at org.apache.solr.client.solrj.request.JavaBinUpdateRequestCodec $1.readOuterMostDocIterator(JavaBinUpdateRequestCodec.java:147) at org.apache.solr.client.solrj.request.JavaBinUpdateRequestCodec $1.readIterator(JavaBinUpdateRequestCodec.java:135) at org.apache.solr.common.util.JavaBinCodec.readVal (JavaBinCodec.java:241) at org.apache.solr.client.solrj.request.JavaBinUpdateRequestCodec $1.readNamedList(JavaBinUpdateRequestCodec.java:121) at org.apache.solr.common.util.JavaBinCodec.readVal (JavaBinCodec.java:206) at org.apache.solr.common.util.JavaBinCodec.unmarshal (JavaBinCodec.java:126) at org.apache.solr.client.solrj.request.JavaBinUpdateRequestCodec.unmarshal (JavaBinUpdateRequestCodec.java:186) at org.apache.solr.handler.loader.JavabinLoader.parseAndLoadDocs (JavabinLoader.java:111) at org.apache.solr.handler.loader.JavabinLoader.load (JavabinLoader.java:58) at org.apache.solr.handler.UpdateRequestHandler$1.load (UpdateRequestHandler.java:98) at org.apache.solr.handler.ContentStreamHandlerBase.handleRequestBody (ContentStreamHandlerBase.java:74) at org.apache.solr.handler.RequestHandlerBase.handleRequest (RequestHandlerBase.java:143) at org.apache.solr.core.SolrCore.execute(SolrCore.java:2064) at org.apache.solr.servlet.HttpSolrCall.execute (HttpSolrCall.java:654) at org.apache.solr.servlet.HttpSolrCall.call (HttpSolrCall.java:450) at org.apache.solr.servlet.SolrDispatchFilter.doFilter (SolrDispatchFilter.java:227) at org.apache.solr.servlet.SolrDispatchFilter.doFilter (SolrDispatchFilter.java:196) at org.eclipse.jetty.servlet.ServletHandler $CachedChain.doFilter(ServletHandler.java:1652) at org.eclipse.jetty.servlet.ServletHandler.doHandle (ServletHandler.java:585) at org.eclipse.jetty.server.handler.ScopedHandler.handle (ScopedHandler.java:143) at org.eclipse.jetty.security.SecurityHandler.handle (SecurityHandler.java:577) at org.eclipse.jetty.server.session.SessionHandler.doHandle (SessionHandler.java:223) at org.eclipse.jetty.server.handler.ContextHandler.doHandle (ContextHandler.java:1127) at org.eclipse.jetty.servlet.ServletHandler.doScope (ServletHandler.java:515) at org.eclipse.jetty.server.session.SessionHandler.doScope (SessionHandler.java:185) at org.eclipse.jetty.server.handler.ContextHandler.doScope (ContextHandler.java:1061) at org.eclipse.jetty.server.handler.ScopedHandler.handle (ScopedHandler.java:141) at org.eclipse.jetty.server.handler.ContextHandlerCollection.handle (ContextHandlerCollection.java:215) at org.eclipse.jetty.server.handler.HandlerCollection.handle (HandlerCollection.java:110) at org.eclipse.jetty.server.handler.HandlerWrapper.handle (HandlerWrapper.java:97) at org.eclipse.jetty.server.Server.handle(Server.java:497) at org.eclipse.jetty.server.HttpChannel.handle (HttpChannel.java:310) at org.eclipse.jetty.server.HttpConnection.onFillable (HttpConnection.java:257) at org.eclipse.jetty.io.AbstractConnection$2.run (AbstractConnection.java:540) at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob (QueuedThreadPool.java:635) at org.eclipse.jetty.util.thread.QueuedThreadPool$3.run (QueuedThreadPool.java:555) at java.lang.Thread.run(Thread.java:785) Caused by: java.util.concurrent.TimeoutException: Idle timeout expired: 50000/50000 ms at org.eclipse.jetty.io.IdleTimeout.checkIdleTimeout (IdleTimeout.java:161) at org.eclipse.jetty.io.IdleTimeout$1.run(IdleTimeout.java:50) at java.util.concurrent.Executors$RunnableAdapter.call (Executors.java:511) at java.util.concurrent.FutureTask.run(FutureTask.java:267) at java.util.concurrent.ScheduledThreadPoolExecutor $ScheduledFutureTask.access$201(ScheduledThreadPoolExecutor.java:180) at java.util.concurrent.ScheduledThreadPoolExecutor $ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:293) at java.util.concurrent.ThreadPoolExecutor.runWorker (ThreadPoolExecutor.java:1143) at java.util.concurrent.ThreadPoolExecutor$Worker.run (ThreadPoolExecutor.java:618) ... 1 more Thanks in advance. Regards, Sanjay
