Environment:
* solr 7.4.1
* all cores are vanilla cores with "loadOnStartUp" set to false, and
"transient" set to true
* we have about 75 cores with "transientCacheSize" set to 32
Issue: we have core corruption from time to time (2-3 core corruption a day)
How to reproduce:
* Set the "transientCacheSize" to 1
* Ingest high load to core1 only (no issue at this time)
* Continue ingest high load to core1 and start ingest load to core2
simultaneously (core2 immediately corrupted) (stack trace is attached below)
Please advise how to resolve this issue?
Thank you so much!
--Michael
stack trace:
2018-08-16 23:02:31.212 ERROR (qtp225472281-4098) [
x:aggregator-core-be43376de27b1675562841f64c498] o.a.s.u.SolrIndexWriter Error
closing IndexWriter
java.nio.file.NoSuchFileException:
/opt/solr/volumes/data1/4cf838d4b9e4675-core-897/index/_2_Lucene50_0.pos
at
sun.nio.fs.UnixException.translateToIOException(UnixException.java:86)
~[?:1.8.0_162]
at
sun.nio.fs.UnixException.rethrowAsIOException(UnixException.java:102)
~[?:1.8.0_162]
at
sun.nio.fs.UnixException.rethrowAsIOException(UnixException.java:107)
~[?:1.8.0_162]
at
sun.nio.fs.UnixFileAttributeViews$Basic.readAttributes(UnixFileAttributeViews.java:55)
~[?:1.8.0_162]
at
sun.nio.fs.UnixFileSystemProvider.readAttributes(UnixFileSystemProvider.java:144)
~[?:1.8.0_162]
at
sun.nio.fs.LinuxFileSystemProvider.readAttributes(LinuxFileSystemProvider.java:99)
~[?:1.8.0_162]
at java.nio.file.Files.readAttributes(Files.java:1737) ~[?:1.8.0_162]
at java.nio.file.Files.size(Files.java:2332) ~[?:1.8.0_162]
at org.apache.lucene.store.FSDirectory.fileLength(FSDirectory.java:243)
~[lucene-core-7.4.0.jar:7.4.0 9060ac689c270b02143f375de0348b7f626adebc -
jpountz - 2018-06-18 16:51:45]
at
org.apache.lucene.store.NRTCachingDirectory.fileLength(NRTCachingDirectory.java:128)
~[lucene-core-7.4.0.jar:7.4.0 9060ac689c270b02143f375de0348b7f626adebc -
jpountz - 2018-06-18 16:51:45]
at
org.apache.lucene.index.SegmentCommitInfo.sizeInBytes(SegmentCommitInfo.java:217)
~[lucene-core-7.4.0.jar:7.4.0 9060ac689c270b02143f375de0348b7f626adebc -
jpountz - 2018-06-18 16:51:45]
at org.apache.lucene.index.MergePolicy.size(MergePolicy.java:558)
~[lucene-core-7.4.0.jar:7.4.0 9060ac689c270b02143f375de0348b7f626adebc -
jpountz - 2018-06-18 16:51:45]
at
org.apache.lucene.index.TieredMergePolicy.getSegmentSizes(TieredMergePolicy.java:279)
~[lucene-core-7.4.0.jar:7.4.0 9060ac689c270b02143f375de0348b7f626adebc -
jpountz - 2018-06-18 16:51:45]
at
org.apache.lucene.index.TieredMergePolicy.findMerges(TieredMergePolicy.java:300)
~[lucene-core-7.4.0.jar:7.4.0 9060ac689c270b02143f375de0348b7f626adebc -
jpountz - 2018-06-18 16:51:45]
at
org.apache.lucene.index.IndexWriter.updatePendingMerges(IndexWriter.java:2199)
~[lucene-core-7.4.0.jar:7.4.0 9060ac689c270b02143f375de0348b7f626adebc -
jpountz - 2018-06-18 16:51:45]
at
org.apache.lucene.index.IndexWriter.maybeMerge(IndexWriter.java:2162)
~[lucene-core-7.4.0.jar:7.4.0 9060ac689c270b02143f375de0348b7f626adebc -
jpountz - 2018-06-18 16:51:45]
at org.apache.lucene.index.IndexWriter.flush(IndexWriter.java:3571)
~[lucene-core-7.4.0.jar:7.4.0 9060ac689c270b02143f375de0348b7f626adebc -
jpountz - 2018-06-18 16:51:45]
at org.apache.lucene.index.IndexWriter.shutdown(IndexWriter.java:1028)
~[lucene-core-7.4.0.jar:7.4.0 9060ac689c270b02143f375de0348b7f626adebc -
jpountz - 2018-06-18 16:51:45]
at org.apache.lucene.index.IndexWriter.close(IndexWriter.java:1071)
~[lucene-core-7.4.0.jar:7.4.0 9060ac689c270b02143f375de0348b7f626adebc -
jpountz - 2018-06-18 16:51:45]
at
org.apache.solr.update.SolrIndexWriter.close(SolrIndexWriter.java:286)
[solr-core-7.4.0.jar:7.4.0 9060ac689c270b02143f375de0348b7f626adebc - jpountz -
2018-06-18 16:55:13]
at
org.apache.solr.update.DirectUpdateHandler2.closeWriter(DirectUpdateHandler2.java:917)
[solr-core-7.4.0.jar:7.4.0 9060ac689c270b02143f375de0348b7f626adebc - jpountz
- 2018-06-18 16:55:13]
at
org.apache.solr.update.DefaultSolrCoreState.closeIndexWriter(DefaultSolrCoreState.java:105)
[solr-core-7.4.0.jar:7.4.0 9060ac689c270b02143f375de0348b7f626adebc - jpountz
- 2018-06-18 16:55:13]
at
org.apache.solr.update.DefaultSolrCoreState.close(DefaultSolrCoreState.java:399)
[solr-core-7.4.0.jar:7.4.0 9060ac689c270b02143f375de0348b7f626adebc - jpountz
- 2018-06-18 16:55:13]
at
org.apache.solr.update.SolrCoreState.decrefSolrCoreState(SolrCoreState.java:83)
[solr-core-7.4.0.jar:7.4.0 9060ac689c270b02143f375de0348b7f626adebc - jpountz -
2018-06-18 16:55:13]
at org.apache.solr.core.SolrCore.close(SolrCore.java:1572)
[solr-core-7.4.0.jar:7.4.0 9060ac689c270b02143f375de0348b7f626adebc - jpountz -
2018-06-18 16:55:13]
at
com.vmware.ops.data.solr.handler.AggregateIngestionHandler.handleRequestBody(AggregateIngestionHandler.java:163)
[ops-data-solr-1.0.0-SNAPSHOT.jar:?]
at
org.apache.solr.handler.RequestHandlerBase.handleRequest(RequestHandlerBase.java:199)
[solr-core-7.4.0.jar:7.4.0 9060ac689c270b02143f375de0348b7f626adebc - jpountz
- 2018-06-18 16:55:13]
at org.apache.solr.core.SolrCore.execute(SolrCore.java:2539)
[solr-core-7.4.0.jar:7.4.0 9060ac689c270b02143f375de0348b7f626adebc - jpountz -
2018-06-18 16:55:13]
at org.apache.solr.servlet.HttpSolrCall.execute(HttpSolrCall.java:709)
[solr-core-7.4.0.jar:7.4.0 9060ac689c270b02143f375de0348b7f626adebc - jpountz -
2018-06-18 16:55:13]
at org.apache.solr.servlet.HttpSolrCall.call(HttpSolrCall.java:515)
[solr-core-7.4.0.jar:7.4.0 9060ac689c270b02143f375de0348b7f626adebc - jpountz -
2018-06-18 16:55:13]
at
org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:377)
[solr-core-7.4.0.jar:7.4.0 9060ac689c270b02143f375de0348b7f626adebc - jpountz
- 2018-06-18 16:55:13]
at
org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:323)
[solr-core-7.4.0.jar:7.4.0 9060ac689c270b02143f375de0348b7f626adebc - jpountz
- 2018-06-18 16:55:13]
at
org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1634)
[jetty-servlet-9.4.10.v20180503.jar:9.4.10.v20180503]
at
org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:533)
[jetty-servlet-9.4.10.v20180503.jar:9.4.10.v20180503]
at
org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:146)
[jetty-server-9.4.10.v20180503.jar:9.4.10.v20180503]
at
org.eclipse.jetty.security.SecurityHandler.handle(SecurityHandler.java:548)
[jetty-security-9.4.10.v20180503.jar:9.4.10.v20180503]
at
org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:132)
[jetty-server-9.4.10.v20180503.jar:9.4.10.v20180503]
at
org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:257)
[jetty-server-9.4.10.v20180503.jar:9.4.10.v20180503]
at
org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:1595)
[jetty-server-9.4.10.v20180503.jar:9.4.10.v20180503]
at
org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:255)
[jetty-server-9.4.10.v20180503.jar:9.4.10.v20180503]
at
org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1253)
[jetty-server-9.4.10.v20180503.jar:9.4.10.v20180503]
at
org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:203)
[jetty-server-9.4.10.v20180503.jar:9.4.10.v20180503]
at
org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:473)
[jetty-servlet-9.4.10.v20180503.jar:9.4.10.v20180503]
at
org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:1564)
[jetty-server-9.4.10.v20180503.jar:9.4.10.v20180503]
at
org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:201)
[jetty-server-9.4.10.v20180503.jar:9.4.10.v20180503]
at
org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1155)
[jetty-server-9.4.10.v20180503.jar:9.4.10.v20180503]
at
org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:144)
[jetty-server-9.4.10.v20180503.jar:9.4.10.v20180503]
at
org.eclipse.jetty.server.handler.ContextHandlerCollection.handle(ContextHandlerCollection.java:219)
[jetty-server-9.4.10.v20180503.jar:9.4.10.v20180503]
at
org.eclipse.jetty.server.handler.HandlerCollection.handle(HandlerCollection.java:126)
[jetty-server-9.4.10.v20180503.jar:9.4.10.v20180503]
at
org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:132)
[jetty-server-9.4.10.v20180503.jar:9.4.10.v20180503]
at org.eclipse.jetty.server.Server.handle(Server.java:531)
[jetty-server-9.4.10.v20180503.jar:9.4.10.v20180503]
at org.eclipse.jetty.server.HttpChannel.handle(HttpChannel.java:352)
[jetty-server-9.4.10.v20180503.jar:9.4.10.v20180503]
at
org.eclipse.jetty.server.HttpConnection.onFillable(HttpConnection.java:260)
[jetty-server-9.4.10.v20180503.jar:9.4.10.v20180503]
at
org.eclipse.jetty.io.AbstractConnection$ReadCallback.succeeded(AbstractConnection.java:281)
[jetty-io-9.4.10.v20180503.jar:9.4.10.v20180503]
at org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:102)
[jetty-io-9.4.10.v20180503.jar:9.4.10.v20180503]
at org.eclipse.jetty.io.ChannelEndPoint$2.run(ChannelEndPoint.java:118)
[jetty-io-9.4.10.v20180503.jar:9.4.10.v20180503]
at
org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.runTask(EatWhatYouKill.java:333)
[jetty-util-9.4.10.v20180503.jar:9.4.10.v20180503]
at
org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:310)
[jetty-util-9.4.10.v20180503.jar:9.4.10.v20180503]
at
org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:168)
[jetty-util-9.4.10.v20180503.jar:9.4.10.v20180503]
at
org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.run(EatWhatYouKill.java:126)
[jetty-util-9.4.10.v20180503.jar:9.4.10.v20180503]
at
org.eclipse.jetty.util.thread.ReservedThreadExecutor$ReservedThread.run(ReservedThreadExecutor.java:366)
[jetty-util-9.4.10.v20180503.jar:9.4.10.v20180503]
at
org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:760)
[jetty-util-9.4.10.v20180503.jar:9.4.10.v20180503]
at
org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:678)
[jetty-util-9.4.10.v20180503.jar:9.4.10.v20180503]
at java.lang.Thread.run(Thread.java:748) [?:1.8.0_162]
Suppressed: java.nio.file.NoSuchFileException:
/opt/solr/volumes/data1/4cf838d4b9e4675-core-897/index/_2_Lucene50_0.pos
at
sun.nio.fs.UnixException.translateToIOException(UnixException.java:86)
~[?:1.8.0_162]
at
sun.nio.fs.UnixException.rethrowAsIOException(UnixException.java:102)
~[?:1.8.0_162]
at
sun.nio.fs.UnixException.rethrowAsIOException(UnixException.java:107)
~[?:1.8.0_162]
at
sun.nio.fs.UnixFileSystemProvider.implDelete(UnixFileSystemProvider.java:244)
~[?:1.8.0_162]
at
sun.nio.fs.AbstractFileSystemProvider.delete(AbstractFileSystemProvider.java:103)
~[?:1.8.0_162]
at java.nio.file.Files.delete(Files.java:1126) ~[?:1.8.0_162]
at
org.apache.lucene.store.FSDirectory.privateDeleteFile(FSDirectory.java:371)
~[lucene-core-7.4.0.jar:7.4.0 9060ac689c270b02143f375de0348b7f626adebc -
jpountz - 2018-06-18 16:51:45]
at
org.apache.lucene.store.FSDirectory.deleteFile(FSDirectory.java:340)
~[lucene-core-7.4.0.jar:7.4.0 9060ac689c270b02143f375de0348b7f626adebc -
jpountz - 2018-06-18 16:51:45]
at
org.apache.lucene.store.NRTCachingDirectory.deleteFile(NRTCachingDirectory.java:119)
~[lucene-core-7.4.0.jar:7.4.0 9060ac689c270b02143f375de0348b7f626adebc -
jpountz - 2018-06-18 16:51:45]
at
org.apache.lucene.store.LockValidatingDirectoryWrapper.deleteFile(LockValidatingDirectoryWrapper.java:38)
~[lucene-core-7.4.0.jar:7.4.0 9060ac689c270b02143f375de0348b7f626adebc -
jpountz - 2018-06-18 16:51:45]
at
org.apache.lucene.index.IndexFileDeleter.deleteFile(IndexFileDeleter.java:696)
~[lucene-core-7.4.0.jar:7.4.0 9060ac689c270b02143f375de0348b7f626adebc -
jpountz - 2018-06-18 16:51:45]
at
org.apache.lucene.index.IndexFileDeleter.deleteFiles(IndexFileDeleter.java:690)
~[lucene-core-7.4.0.jar:7.4.0 9060ac689c270b02143f375de0348b7f626adebc -
jpountz - 2018-06-18 16:51:45]
at
org.apache.lucene.index.IndexFileDeleter.decRef(IndexFileDeleter.java:589)
~[lucene-core-7.4.0.jar:7.4.0 9060ac689c270b02143f375de0348b7f626adebc -
jpountz - 2018-06-18 16:51:45]
at
org.apache.lucene.index.IndexFileDeleter.checkpoint(IndexFileDeleter.java:531)
~[lucene-core-7.4.0.jar:7.4.0 9060ac689c270b02143f375de0348b7f626adebc -
jpountz - 2018-06-18 16:51:45]
at
org.apache.lucene.index.IndexWriter.rollbackInternalNoCommit(IndexWriter.java:2331)
~[lucene-core-7.4.0.jar:7.4.0 9060ac689c270b02143f375de0348b7f626adebc -
jpountz - 2018-06-18 16:51:45]
at
org.apache.lucene.index.IndexWriter.rollbackInternal(IndexWriter.java:2274)
~[lucene-core-7.4.0.jar:7.4.0 9060ac689c270b02143f375de0348b7f626adebc -
jpountz - 2018-06-18 16:51:45]
at
org.apache.lucene.index.IndexWriter.shutdown(IndexWriter.java:1035)
~[lucene-core-7.4.0.jar:7.4.0 9060ac689c270b02143f375de0348b7f626adebc -
jpountz - 2018-06-18 16:51:45]
at
org.apache.lucene.index.IndexWriter.close(IndexWriter.java:1071)
~[lucene-core-7.4.0.jar:7.4.0 9060ac689c270b02143f375de0348b7f626adebc -
jpountz - 2018-06-18 16:51:45]
at
org.apache.solr.update.SolrIndexWriter.close(SolrIndexWriter.java:286)
[solr-core-7.4.0.jar:7.4.0 9060ac689c270b02143f375de0348b7f626adebc - jpountz -
2018-06-18 16:55:13]
at
org.apache.solr.update.DirectUpdateHandler2.closeWriter(DirectUpdateHandler2.java:917)
[solr-core-7.4.0.jar:7.4.0 9060ac689c270b02143f375de0348b7f626adebc - jpountz
- 2018-06-18 16:55:13]
at
org.apache.solr.update.DefaultSolrCoreState.closeIndexWriter(DefaultSolrCoreState.java:105)
[solr-core-7.4.0.jar:7.4.0 9060ac689c270b02143f375de0348b7f626adebc - jpountz
- 2018-06-18 16:55:13]
at
org.apache.solr.update.DefaultSolrCoreState.close(DefaultSolrCoreState.java:399)
[solr-core-7.4.0.jar:7.4.0 9060ac689c270b02143f375de0348b7f626adebc - jpountz
- 2018-06-18 16:55:13]
at
org.apache.solr.update.SolrCoreState.decrefSolrCoreState(SolrCoreState.java:83)
[solr-core-7.4.0.jar:7.4.0 9060ac689c270b02143f375de0348b7f626adebc - jpountz -
2018-06-18 16:55:13]
at org.apache.solr.core.SolrCore.close(SolrCore.java:1572)
[solr-core-7.4.0.jar:7.4.0 9060ac689c270b02143f375de0348b7f626adebc - jpountz -
2018-06-18 16:55:13]
at
com.vmware.ops.data.solr.handler.AggregateIngestionHandler.handleRequestBody(AggregateIngestionHandler.java:163)
[ops-data-solr-1.0.0-SNAPSHOT.jar:?]
at
org.apache.solr.handler.RequestHandlerBase.handleRequest(RequestHandlerBase.java:199)
[solr-core-7.4.0.jar:7.4.0 9060ac689c270b02143f375de0348b7f626adebc - jpountz
- 2018-06-18 16:55:13]
at org.apache.solr.core.SolrCore.execute(SolrCore.java:2539)
[solr-core-7.4.0.jar:7.4.0 9060ac689c270b02143f375de0348b7f626adebc - jpountz -
2018-06-18 16:55:13]
at
org.apache.solr.servlet.HttpSolrCall.execute(HttpSolrCall.java:709)
[solr-core-7.4.0.jar:7.4.0 9060ac689c270b02143f375de0348b7f626adebc - jpountz -
2018-06-18 16:55:13]
at
org.apache.solr.servlet.HttpSolrCall.call(HttpSolrCall.java:515)
[solr-core-7.4.0.jar:7.4.0 9060ac689c270b02143f375de0348b7f626adebc - jpountz -
2018-06-18 16:55:13]
at
org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:377)
[solr-core-7.4.0.jar:7.4.0 9060ac689c270b02143f375de0348b7f626adebc - jpountz
- 2018-06-18 16:55:13]
at
org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:323)
[solr-core-7.4.0.jar:7.4.0 9060ac689c270b02143f375de0348b7f626adebc - jpountz
- 2018-06-18 16:55:13]
at
org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1634)
[jetty-servlet-9.4.10.v20180503.jar:9.4.10.v20180503]
at
org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:533)
[jetty-servlet-9.4.10.v20180503.jar:9.4.10.v20180503]
at
org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:146)
[jetty-server-9.4.10.v20180503.jar:9.4.10.v20180503]
at
org.eclipse.jetty.security.SecurityHandler.handle(SecurityHandler.java:548)
[jetty-security-9.4.10.v20180503.jar:9.4.10.v20180503]
at
org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:132)
[jetty-server-9.4.10.v20180503.jar:9.4.10.v20180503]
at
org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:257)
[jetty-server-9.4.10.v20180503.jar:9.4.10.v20180503]
at
org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:1595)
[jetty-server-9.4.10.v20180503.jar:9.4.10.v20180503]
at
org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:255)
[jetty-server-9.4.10.v20180503.jar:9.4.10.v20180503]
at
org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1253)
[jetty-server-9.4.10.v20180503.jar:9.4.10.v20180503]
at
org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:203)
[jetty-server-9.4.10.v20180503.jar:9.4.10.v20180503]
at
org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:473)
[jetty-servlet-9.4.10.v20180503.jar:9.4.10.v20180503]
at
org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:1564)
[jetty-server-9.4.10.v20180503.jar:9.4.10.v20180503]
at
org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:201)
[jetty-server-9.4.10.v20180503.jar:9.4.10.v20180503]
at
org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1155)
[jetty-server-9.4.10.v20180503.jar:9.4.10.v20180503]
at
org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:144)
[jetty-server-9.4.10.v20180503.jar:9.4.10.v20180503]
at
org.eclipse.jetty.server.handler.ContextHandlerCollection.handle(ContextHandlerCollection.java:219)
[jetty-server-9.4.10.v20180503.jar:9.4.10.v20180503]
at
org.eclipse.jetty.server.handler.HandlerCollection.handle(HandlerCollection.java:126)
[jetty-server-9.4.10.v20180503.jar:9.4.10.v20180503]
at
org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:132)
[jetty-server-9.4.10.v20180503.jar:9.4.10.v20180503]
at org.eclipse.jetty.server.Server.handle(Server.java:531)
[jetty-server-9.4.10.v20180503.jar:9.4.10.v20180503]
at
org.eclipse.jetty.server.HttpChannel.handle(HttpChannel.java:352)
[jetty-server-9.4.10.v20180503.jar:9.4.10.v20180503]
at
org.eclipse.jetty.server.HttpConnection.onFillable(HttpConnection.java:260)
[jetty-server-9.4.10.v20180503.jar:9.4.10.v20180503]
at
org.eclipse.jetty.io.AbstractConnection$ReadCallback.succeeded(AbstractConnection.java:281)
[jetty-io-9.4.10.v20180503.jar:9.4.10.v20180503]
at
org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:102)
[jetty-io-9.4.10.v20180503.jar:9.4.10.v20180503]
at
org.eclipse.jetty.io.ChannelEndPoint$2.run(ChannelEndPoint.java:118)
[jetty-io-9.4.10.v20180503.jar:9.4.10.v20180503]
at
org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.runTask(EatWhatYouKill.java:333)
[jetty-util-9.4.10.v20180503.jar:9.4.10.v20180503]
at
org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:310)
[jetty-util-9.4.10.v20180503.jar:9.4.10.v20180503]
at
org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:168)
[jetty-util-9.4.10.v20180503.jar:9.4.10.v20180503]
at
org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.run(EatWhatYouKill.java:126)
[jetty-util-9.4.10.v20180503.jar:9.4.10.v20180503]
at
org.eclipse.jetty.util.thread.ReservedThreadExecutor$ReservedThread.run(ReservedThreadExecutor.java:366)
[jetty-util-9.4.10.v20180503.jar:9.4.10.v20180503]
at
org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:760)
[jetty-util-9.4.10.v20180503.jar:9.4.10.v20180503]
at
org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:678)
[jetty-util-9.4.10.v20180503.jar:9.4.10.v20180503]
at java.lang.Thread.run(Thread.java:748) [?:1.8.0_162]
