Tried changing to use /solr/admin/cores instead as a test - still see the same issue, though much less frequent.
------------------------------------------------------------ Nathan Neulinger nn...@neulinger.org Neulinger Consulting (573) 612-1412 On Sat, Jul 26, 2014 at 6:15 PM, Nathan Neulinger <nn...@neulinger.org> wrote: > Recently deployed haproxy in front of my solr instances, and seeing a > large number of exceptions in the logs now... Example below. I can pound > the server with requests against /solr/admin/ping via curl, with no obvious > issue, but the haproxy checks appear to be aggravating something. > > Solr 4.8.0 w/ solr cloud, 2 nodes, 3 zk, linux x86_64 > > It seems like when the issue occurs, I get a set of the errors all in a > burst (below), never just one. > > Suggestions? > > -- Nathan > > ------------------------------------------------------------ > Nathan Neulinger nn...@neulinger.org > Neulinger Consulting (573) 612-1412 > > > > 2014-07-26 23:04:36,506 ERROR qtp1532385072-4864 > [g.apache.solr.servlet.SolrDispatchFilter] > - null:org.eclipse.jetty.io.EofException > at org.eclipse.jetty.http.HttpGenerator.flushBuffer( > HttpGenerator.java:914) > at org.eclipse.jetty.http.AbstractGenerator.flush( > AbstractGenerator.java:443) > at org.eclipse.jetty.server.HttpOutput.flush(HttpOutput.java:100) > at org.eclipse.jetty.server.AbstractHttpConnection$Output. > flush(AbstractHttpConnection.java:1094) > at sun.nio.cs.StreamEncoder.implFlush(StreamEncoder.java:297) > at sun.nio.cs.StreamEncoder.flush(StreamEncoder.java:141) > at java.io.OutputStreamWriter.flush(OutputStreamWriter.java:229) > at org.apache.solr.util.FastWriter.flush(FastWriter.java:137) > at org.apache.solr.servlet.SolrDispatchFilter.writeResponse( > SolrDispatchFilter.java:763) > at org.apache.solr.servlet.SolrDispatchFilter.doFilter( > SolrDispatchFilter.java:431) > at org.apache.solr.servlet.SolrDispatchFilter.doFilter( > SolrDispatchFilter.java:339) > at org.apache.solr.servlet.SolrDispatchFilter.doFilter( > SolrDispatchFilter.java:207) > at org.eclipse.jetty.servlet.ServletHandler$CachedChain. > doFilter(ServletHandler.java:1419) > at org.eclipse.jetty.servlet.ServletHandler.doHandle( > ServletHandler.java:455) > at org.eclipse.jetty.server.handler.ScopedHandler.handle( > ScopedHandler.java:137) > at org.eclipse.jetty.security.SecurityHandler.handle( > SecurityHandler.java:557) > at org.eclipse.jetty.server.session.SessionHandler. > doHandle(SessionHandler.java:231) > at org.eclipse.jetty.server.handler.ContextHandler. > doHandle(ContextHandler.java:1075) > at org.eclipse.jetty.servlet.ServletHandler.doScope( > ServletHandler.java:384) > at org.eclipse.jetty.server.session.SessionHandler. > doScope(SessionHandler.java:193) > at org.eclipse.jetty.server.handler.ContextHandler. > doScope(ContextHandler.java:1009) > at org.eclipse.jetty.server.handler.ScopedHandler.handle( > ScopedHandler.java:135) > at org.eclipse.jetty.server.handler.ContextHandlerCollection. > handle(ContextHandlerCollection.java:255) > at org.eclipse.jetty.server.handler.HandlerCollection. > handle(HandlerCollection.java:154) > at org.eclipse.jetty.server.handler.HandlerWrapper.handle( > HandlerWrapper.java:116) > at org.eclipse.jetty.server.Server.handle(Server.java:368) > at org.eclipse.jetty.server.AbstractHttpConnection.handleRequest( > AbstractHttpConnection.java:489) > at org.eclipse.jetty.server.BlockingHttpConnection.handleRequest( > BlockingHttpConnection.java:53) > at org.eclipse.jetty.server.AbstractHttpConnection.headerComplete( > AbstractHttpConnection.java:942) > at org.eclipse.jetty.server.AbstractHttpConnection$ > RequestHandler.headerComplete(AbstractHttpConnection.java:1004) > at org.eclipse.jetty.http.HttpParser.parseNext( > HttpParser.java:640) > at org.eclipse.jetty.http.HttpParser.parseAvailable( > HttpParser.java:235) > at org.eclipse.jetty.server.BlockingHttpConnection.handle( > BlockingHttpConnection.java:72) > at org.eclipse.jetty.server.bio.SocketConnector$ > ConnectorEndPoint.run(SocketConnector.java:264) > at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob( > QueuedThreadPool.java:608) > at org.eclipse.jetty.util.thread.QueuedThreadPool$3.run( > QueuedThreadPool.java:543) > at java.lang.Thread.run(Thread.java:745) > Caused by: java.net.SocketException: Connection reset > at java.net.SocketOutputStream.socketWrite( > SocketOutputStream.java:118) > at java.net.SocketOutputStream.write(SocketOutputStream.java:159) > at org.eclipse.jetty.io.ByteArrayBuffer.writeTo( > ByteArrayBuffer.java:375) > at org.eclipse.jetty.io.bio.StreamEndPoint.flush( > StreamEndPoint.java:164) > at org.eclipse.jetty.io.bio.StreamEndPoint.flush( > StreamEndPoint.java:194) > at org.eclipse.jetty.http.HttpGenerator.flushBuffer( > HttpGenerator.java:838) > ... 36 more > > 2014-07-26 23:04:36,513 ERROR qtp1532385072-4864 > [g.apache.solr.servlet.SolrDispatchFilter] > - null:org.eclipse.jetty.io.EofException > at org.eclipse.jetty.http.HttpGenerator.flushBuffer( > HttpGenerator.java:914) > at org.eclipse.jetty.http.AbstractGenerator.flush( > AbstractGenerator.java:443) > at org.eclipse.jetty.server.HttpOutput.flush(HttpOutput.java:100) > at org.eclipse.jetty.server.AbstractHttpConnection$Output. > flush(AbstractHttpConnection.java:1094) > at sun.nio.cs.StreamEncoder.implFlush(StreamEncoder.java:297) > at sun.nio.cs.StreamEncoder.flush(StreamEncoder.java:141) > at java.io.OutputStreamWriter.flush(OutputStreamWriter.java:229) > at org.apache.solr.util.FastWriter.flush(FastWriter.java:137) > at org.apache.solr.servlet.SolrDispatchFilter.writeResponse( > SolrDispatchFilter.java:763) > at org.apache.solr.servlet.SolrDispatchFilter.doFilter( > SolrDispatchFilter.java:431) > at org.apache.solr.servlet.SolrDispatchFilter.doFilter( > SolrDispatchFilter.java:339) > at org.apache.solr.servlet.SolrDispatchFilter.doFilter( > SolrDispatchFilter.java:207) > at org.eclipse.jetty.servlet.ServletHandler$CachedChain. > doFilter(ServletHandler.java:1419) > at org.eclipse.jetty.servlet.ServletHandler.doHandle( > ServletHandler.java:455) > at org.eclipse.jetty.server.handler.ScopedHandler.handle( > ScopedHandler.java:137) > at org.eclipse.jetty.security.SecurityHandler.handle( > SecurityHandler.java:557) > at org.eclipse.jetty.server.session.SessionHandler. > doHandle(SessionHandler.java:231) > at org.eclipse.jetty.server.handler.ContextHandler. > doHandle(ContextHandler.java:1075) > at org.eclipse.jetty.servlet.ServletHandler.doScope( > ServletHandler.java:384) > at org.eclipse.jetty.server.session.SessionHandler. > doScope(SessionHandler.java:193) > at org.eclipse.jetty.server.handler.ContextHandler. > doScope(ContextHandler.java:1009) > at org.eclipse.jetty.server.handler.ScopedHandler.handle( > ScopedHandler.java:135) > at org.eclipse.jetty.server.handler.ContextHandlerCollection. > handle(ContextHandlerCollection.java:255) > at org.eclipse.jetty.server.handler.HandlerCollection. > handle(HandlerCollection.java:154) > at org.eclipse.jetty.server.handler.HandlerWrapper.handle( > HandlerWrapper.java:116) > at org.eclipse.jetty.server.Server.handle(Server.java:368) > at org.eclipse.jetty.server.AbstractHttpConnection.handleRequest( > AbstractHttpConnection.java:489) > at org.eclipse.jetty.server.BlockingHttpConnection.handleRequest( > BlockingHttpConnection.java:53) > at org.eclipse.jetty.server.AbstractHttpConnection.headerComplete( > AbstractHttpConnection.java:942) > at org.eclipse.jetty.server.AbstractHttpConnection$ > RequestHandler.headerComplete(AbstractHttpConnection.java:1004) > at org.eclipse.jetty.http.HttpParser.parseNext( > HttpParser.java:640) > at org.eclipse.jetty.http.HttpParser.parseAvailable( > HttpParser.java:235) > at org.eclipse.jetty.server.BlockingHttpConnection.handle( > BlockingHttpConnection.java:72) > at org.eclipse.jetty.server.bio.SocketConnector$ > ConnectorEndPoint.run(SocketConnector.java:264) > at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob( > QueuedThreadPool.java:608) > at org.eclipse.jetty.util.thread.QueuedThreadPool$3.run( > QueuedThreadPool.java:543) > at java.lang.Thread.run(Thread.java:745) > Caused by: java.net.SocketException: Connection reset > at java.net.SocketOutputStream.socketWrite( > SocketOutputStream.java:118) > at java.net.SocketOutputStream.write(SocketOutputStream.java:159) > at org.eclipse.jetty.io.ByteArrayBuffer.writeTo( > ByteArrayBuffer.java:375) > at org.eclipse.jetty.io.bio.StreamEndPoint.flush( > StreamEndPoint.java:164) > at org.eclipse.jetty.io.bio.StreamEndPoint.flush( > StreamEndPoint.java:194) > at org.eclipse.jetty.http.HttpGenerator.flushBuffer( > HttpGenerator.java:838) > ... 36 more > > 2014-07-26 23:04:36,514 WARN qtp1532385072-4864 > [org.eclipse.jetty.server.Response > ] - Committed before 500 {msg=Connection > reset,trace=org.eclipse.jetty.io.EofException > at org.eclipse.jetty.http.HttpGenerator.flushBuffer( > HttpGenerator.java:914) > at org.eclipse.jetty.http.AbstractGenerator.flush( > AbstractGenerator.java:443) > at org.eclipse.jetty.server.HttpOutput.flush(HttpOutput.java:100) > at org.eclipse.jetty.server.AbstractHttpConnection$Output. > flush(AbstractHttpConnection.java:1094) > at sun.nio.cs.StreamEncoder.implFlush(StreamEncoder.java:297) > at sun.nio.cs.StreamEncoder.flush(StreamEncoder.java:141) > at java.io.OutputStreamWriter.flush(OutputStreamWriter.java:229) > at org.apache.solr.util.FastWriter.flush(FastWriter.java:137) > at org.apache.solr.servlet.SolrDispatchFilter.writeResponse( > SolrDispatchFilter.java:763) > at org.apache.solr.servlet.SolrDispatchFilter.doFilter( > SolrDispatchFilter.java:431) > at org.apache.solr.servlet.SolrDispatchFilter.doFilter( > SolrDispatchFilter.java:339) > at org.apache.solr.servlet.SolrDispatchFilter.doFilter( > SolrDispatchFilter.java:207) > at org.eclipse.jetty.servlet.ServletHandler$CachedChain. > doFilter(ServletHandler.java:1419) > at org.eclipse.jetty.servlet.ServletHandler.doHandle( > ServletHandler.java:455) > at org.eclipse.jetty.server.handler.ScopedHandler.handle( > ScopedHandler.java:137) > at org.eclipse.jetty.security.SecurityHandler.handle( > SecurityHandler.java:557) > at org.eclipse.jetty.server.session.SessionHandler. > doHandle(SessionHandler.java:231) > at org.eclipse.jetty.server.handler.ContextHandler. > doHandle(ContextHandler.java:1075) > at org.eclipse.jetty.servlet.ServletHandler.doScope( > ServletHandler.java:384) > at org.eclipse.jetty.server.session.SessionHandler. > doScope(SessionHandler.java:193) > at org.eclipse.jetty.server.handler.ContextHandler. > doScope(ContextHandler.java:1009) > at org.eclipse.jetty.server.handler.ScopedHandler.handle( > ScopedHandler.java:135) > at org.eclipse.jetty.server.handler.ContextHandlerCollection. > handle(ContextHandlerCollection.java:255) > at org.eclipse.jetty.server.handler.HandlerCollection. > handle(HandlerCollection.java:154) > at org.eclipse.jetty.server.handler.HandlerWrapper.handle( > HandlerWrapper.java:116) > at org.eclipse.jetty.server.Server.handle(Server.java:368) > at org.eclipse.jetty.server.AbstractHttpConnection.handleRequest( > AbstractHttpConnection.java:489) > at org.eclipse.jetty.server.BlockingHttpConnection.handleRequest( > BlockingHttpConnection.java:53) > at org.eclipse.jetty.server.AbstractHttpConnection.headerComplete( > AbstractHttpConnection.java:942) > at org.eclipse.jetty.server.AbstractHttpConnection$ > RequestHandler.headerComplete(AbstractHttpConnection.java:1004) > at org.eclipse.jetty.http.HttpParser.parseNext( > HttpParser.java:640) > at org.eclipse.jetty.http.HttpParser.parseAvailable( > HttpParser.java:235) > at org.eclipse.jetty.server.BlockingHttpConnection.handle( > BlockingHttpConnection.java:72) > at org.eclipse.jetty.server.bio.SocketConnector$ > ConnectorEndPoint.run(SocketConnector.java:264) > at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob( > QueuedThreadPool.java:608) > at org.eclipse.jetty.util.thread.QueuedThreadPool$3.run( > QueuedThreadPool.java:543) > at java.lang.Thread.run(Thread.java:745) > Caused by: java.net.SocketException: Connection reset > at java.net.SocketOutputStream.socketWrite( > SocketOutputStream.java:118) > at java.net.SocketOutputStream.write(SocketOutputStream.java:159) > at org.eclipse.jetty.io.ByteArrayBuffer.writeTo( > ByteArrayBuffer.java:375) > at org.eclipse.jetty.io.bio.StreamEndPoint.flush( > StreamEndPoint.java:164) > at org.eclipse.jetty.io.bio.StreamEndPoint.flush( > StreamEndPoint.java:194) > at org.eclipse.jetty.http.HttpGenerator.flushBuffer( > HttpGenerator.java:838) > ... 36 more > ,code=500} > 2014-07-26 23:04:36,517 ERROR qtp1532385072-4864 > [g.apache.solr.servlet.SolrDispatchFilter] > - null:java.lang.IllegalStateException: Committed > at org.eclipse.jetty.server.Response.resetBuffer(Response. > java:1144) > at org.eclipse.jetty.server.Response.sendError(Response.java:314) > at org.apache.solr.servlet.SolrDispatchFilter.sendError( > SolrDispatchFilter.java:818) > at org.apache.solr.servlet.SolrDispatchFilter.doFilter( > SolrDispatchFilter.java:439) > at org.apache.solr.servlet.SolrDispatchFilter.doFilter( > SolrDispatchFilter.java:339) > at org.apache.solr.servlet.SolrDispatchFilter.doFilter( > SolrDispatchFilter.java:207) > at org.eclipse.jetty.servlet.ServletHandler$CachedChain. > doFilter(ServletHandler.java:1419) > at org.eclipse.jetty.servlet.ServletHandler.doHandle( > ServletHandler.java:455) > at org.eclipse.jetty.server.handler.ScopedHandler.handle( > ScopedHandler.java:137) > at org.eclipse.jetty.security.SecurityHandler.handle( > SecurityHandler.java:557) > at org.eclipse.jetty.server.session.SessionHandler. > doHandle(SessionHandler.java:231) > at org.eclipse.jetty.server.handler.ContextHandler. > doHandle(ContextHandler.java:1075) > at org.eclipse.jetty.servlet.ServletHandler.doScope( > ServletHandler.java:384) > at org.eclipse.jetty.server.session.SessionHandler. > doScope(SessionHandler.java:193) > at org.eclipse.jetty.server.handler.ContextHandler. > doScope(ContextHandler.java:1009) > at org.eclipse.jetty.server.handler.ScopedHandler.handle( > ScopedHandler.java:135) > at org.eclipse.jetty.server.handler.ContextHandlerCollection. > handle(ContextHandlerCollection.java:255) > at org.eclipse.jetty.server.handler.HandlerCollection. > handle(HandlerCollection.java:154) > at org.eclipse.jetty.server.handler.HandlerWrapper.handle( > HandlerWrapper.java:116) > at org.eclipse.jetty.server.Server.handle(Server.java:368) > at org.eclipse.jetty.server.AbstractHttpConnection.handleRequest( > AbstractHttpConnection.java:489) > at org.eclipse.jetty.server.BlockingHttpConnection.handleRequest( > BlockingHttpConnection.java:53) > at org.eclipse.jetty.server.AbstractHttpConnection.headerComplete( > AbstractHttpConnection.java:942) > at org.eclipse.jetty.server.AbstractHttpConnection$ > RequestHandler.headerComplete(AbstractHttpConnection.java:1004) > at org.eclipse.jetty.http.HttpParser.parseNext( > HttpParser.java:640) > at org.eclipse.jetty.http.HttpParser.parseAvailable( > HttpParser.java:235) > at org.eclipse.jetty.server.BlockingHttpConnection.handle( > BlockingHttpConnection.java:72) > at org.eclipse.jetty.server.bio.SocketConnector$ > ConnectorEndPoint.run(SocketConnector.java:264) > at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob( > QueuedThreadPool.java:608) > at org.eclipse.jetty.util.thread.QueuedThreadPool$3.run( > QueuedThreadPool.java:543) > at java.lang.Thread.run(Thread.java:745) > > 2014-07-26 23:04:36,518 WARN qtp1532385072-4864 > [org.eclipse.jetty.server.Response > ] - Committed before 500 > {msg=Committed,trace=java.lang.IllegalStateException: > Committed > at org.eclipse.jetty.server.Response.resetBuffer(Response. > java:1144) > at org.eclipse.jetty.server.Response.sendError(Response.java:314) > at org.apache.solr.servlet.SolrDispatchFilter.sendError( > SolrDispatchFilter.java:818) > at org.apache.solr.servlet.SolrDispatchFilter.doFilter( > SolrDispatchFilter.java:439) > at org.apache.solr.servlet.SolrDispatchFilter.doFilter( > SolrDispatchFilter.java:339) > at org.apache.solr.servlet.SolrDispatchFilter.doFilter( > SolrDispatchFilter.java:207) > at org.eclipse.jetty.servlet.ServletHandler$CachedChain. > doFilter(ServletHandler.java:1419) > at org.eclipse.jetty.servlet.ServletHandler.doHandle( > ServletHandler.java:455) > at org.eclipse.jetty.server.handler.ScopedHandler.handle( > ScopedHandler.java:137) > at org.eclipse.jetty.security.SecurityHandler.handle( > SecurityHandler.java:557) > at org.eclipse.jetty.server.session.SessionHandler. > doHandle(SessionHandler.java:231) > at org.eclipse.jetty.server.handler.ContextHandler. > doHandle(ContextHandler.java:1075) > at org.eclipse.jetty.servlet.ServletHandler.doScope( > ServletHandler.java:384) > at org.eclipse.jetty.server.session.SessionHandler. > doScope(SessionHandler.java:193) > at org.eclipse.jetty.server.handler.ContextHandler. > doScope(ContextHandler.java:1009) > at org.eclipse.jetty.server.handler.ScopedHandler.handle( > ScopedHandler.java:135) > at org.eclipse.jetty.server.handler.ContextHandlerCollection. > handle(ContextHandlerCollection.java:255) > at org.eclipse.jetty.server.handler.HandlerCollection. > handle(HandlerCollection.java:154) > at org.eclipse.jetty.server.handler.HandlerWrapper.handle( > HandlerWrapper.java:116) > at org.eclipse.jetty.server.Server.handle(Server.java:368) > at org.eclipse.jetty.server.AbstractHttpConnection.handleRequest( > AbstractHttpConnection.java:489) > at org.eclipse.jetty.server.BlockingHttpConnection.handleRequest( > BlockingHttpConnection.java:53) > at org.eclipse.jetty.server.AbstractHttpConnection.headerComplete( > AbstractHttpConnection.java:942) > at org.eclipse.jetty.server.AbstractHttpConnection$ > RequestHandler.headerComplete(AbstractHttpConnection.java:1004) > at org.eclipse.jetty.http.HttpParser.parseNext( > HttpParser.java:640) > at org.eclipse.jetty.http.HttpParser.parseAvailable( > HttpParser.java:235) > at org.eclipse.jetty.server.BlockingHttpConnection.handle( > BlockingHttpConnection.java:72) > at org.eclipse.jetty.server.bio.SocketConnector$ > ConnectorEndPoint.run(SocketConnector.java:264) > at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob( > QueuedThreadPool.java:608) > at org.eclipse.jetty.util.thread.QueuedThreadPool$3.run( > QueuedThreadPool.java:543) > at java.lang.Thread.run(Thread.java:745) > ,code=500} > 2014-07-26 23:04:36,518 WARN qtp1532385072-4864 > [org.eclipse.jetty.servlet.ServletHandler] - /solr/admin/ping > java.lang.IllegalStateException: Committed > at org.eclipse.jetty.server.Response.resetBuffer(Response. > java:1144) > at org.eclipse.jetty.server.Response.sendError(Response.java:314) > at org.apache.solr.servlet.SolrDispatchFilter.sendError( > SolrDispatchFilter.java:818) > at org.apache.solr.servlet.SolrDispatchFilter.doFilter( > SolrDispatchFilter.java:439) > at org.apache.solr.servlet.SolrDispatchFilter.doFilter( > SolrDispatchFilter.java:207) > at org.eclipse.jetty.servlet.ServletHandler$CachedChain. > doFilter(ServletHandler.java:1419) > at org.eclipse.jetty.servlet.ServletHandler.doHandle( > ServletHandler.java:455) > at org.eclipse.jetty.server.handler.ScopedHandler.handle( > ScopedHandler.java:137) > at org.eclipse.jetty.security.SecurityHandler.handle( > SecurityHandler.java:557) > at org.eclipse.jetty.server.session.SessionHandler. > doHandle(SessionHandler.java:231) > at org.eclipse.jetty.server.handler.ContextHandler. > doHandle(ContextHandler.java:1075) > at org.eclipse.jetty.servlet.ServletHandler.doScope( > ServletHandler.java:384) > at org.eclipse.jetty.server.session.SessionHandler. > doScope(SessionHandler.java:193) > at org.eclipse.jetty.server.handler.ContextHandler. > doScope(ContextHandler.java:1009) > at org.eclipse.jetty.server.handler.ScopedHandler.handle( > ScopedHandler.java:135) > at org.eclipse.jetty.server.handler.ContextHandlerCollection. > handle(ContextHandlerCollection.java:255) > at org.eclipse.jetty.server.handler.HandlerCollection. > handle(HandlerCollection.java:154) > at org.eclipse.jetty.server.handler.HandlerWrapper.handle( > HandlerWrapper.java:116) > at org.eclipse.jetty.server.Server.handle(Server.java:368) > at org.eclipse.jetty.server.AbstractHttpConnection.handleRequest( > AbstractHttpConnection.java:489) > at org.eclipse.jetty.server.BlockingHttpConnection.handleRequest( > BlockingHttpConnection.java:53) > at org.eclipse.jetty.server.AbstractHttpConnection.headerComplete( > AbstractHttpConnection.java:942) > at org.eclipse.jetty.server.AbstractHttpConnection$ > RequestHandler.headerComplete(AbstractHttpConnection.java:1004) > at org.eclipse.jetty.http.HttpParser.parseNext( > HttpParser.java:640) > at org.eclipse.jetty.http.HttpParser.parseAvailable( > HttpParser.java:235) > at org.eclipse.jetty.server.BlockingHttpConnection.handle( > BlockingHttpConnection.java:72) > at org.eclipse.jetty.server.bio.SocketConnector$ > ConnectorEndPoint.run(SocketConnector.java:264) > at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob( > QueuedThreadPool.java:608) > at org.eclipse.jetty.util.thread.QueuedThreadPool$3.run( > QueuedThreadPool.java:543) > at java.lang.Thread.run(Thread.java:745) >