Hello,
I have set up a SolrCloud with 3 Solr nodes (v7.7) and 3 external Zookeepers
(3.4.14). However, whenever I enable SSL I get these errors in the logs that
for some reason Solr (or maybe ZK) still wants to communicate via http. I am
able to pull up admin Console via https but when I click on Cloud/Nodes to look
at metrics that’s when I see this message in the logs.
In the documentation it’s showing to pass “-name urlScheme -val https” to
Zookeeper also but external zkServer.sh doesn’t have that option.
What am I doing wrong? I am pulling my hair out, trying to get this going.
JKS has Key and a SAN certificate that includes all of the node names. JKS is
loaded on all Solr nodes.
This is running inside of docker contains and I have passed the following
variables:
- SOLR_MODE=solrcloud
- ZK_HOST=mynode1_fqdn:2181,mynode2_fqdn:2181,mynode2_fqdn:2181
- SOLR_SSL_ENABLED=true
- SOLR_SSL_KEY_STORE=/opt/solr/etc/solr-ssl.keystore.jks
- SOLR_SSL_KEY_STORE_PASSWORD=secret
- SOLR_SSL_TRUST_STORE=/opt/solr/etc/solr-ssl.keystore.jks
- SOLR_SSL_TRUST_STORE_PASSWORD=secret
- SOLR_SSL_NEED_CLIENT_AUTH=false
- SOLR_SSL_WANT_CLIENT_AUTH=true
- SOLR_SSL_CHECK_PEER_NAME=false
- SOLR_URL_SCHEME=https
2019-05-16 22:15:44.999 WARN (qtp841262455-21) [ ]
o.a.s.h.a.AdminHandlersProxy Exception when fetching result from node
mynode1_fqdn:8983_solr
java.util.concurrent.ExecutionException:
org.apache.solr.client.solrj.SolrServerException: IOException occured when
talking to server at: http://mynode1_fqdn:8983/solr
at java.util.concurrent.FutureTask.report(FutureTask.java:122)
~[?:?]
at java.util.concurrent.FutureTask.get(FutureTask.java:205)
~[?:?]
at
org.apache.solr.handler.admin.AdminHandlersProxy.maybeProxyToNodes(AdminHandlersProxy.java:102)
[solr-core-7.7.1.jar:7.7.1 5bf96d32f88eb8a2f5e775339885cd6ba84a3b58 - ishan -
2019-02-23 02:39:07]
at
org.apache.solr.handler.admin.SystemInfoHandler.handleRequestBody(SystemInfoHandler.java:136)
[solr-core-7.7.1.jar:7.7.1 5bf96d32f88eb8a2f5e775339885cd6ba84a3b58 - ishan -
2019-02-23 02:39:07]
at
org.apache.solr.handler.RequestHandlerBase.handleRequest(RequestHandlerBase.java:199)
[solr-core-7.7.1.jar:7.7.1 5bf96d32f88eb8a2f5e775339885cd6ba84a3b58 - ishan -
2019-02-23 02:39:07]
at
org.apache.solr.handler.admin.InfoHandler.handle(InfoHandler.java:91)
[solr-core-7.7.1.jar:7.7.1 5bf96d32f88eb8a2f5e775339885cd6ba84a3b58 - ishan -
2019-02-23 02:39:07]
at
org.apache.solr.handler.admin.InfoHandler.handleRequestBody(InfoHandler.java:81)
[solr-core-7.7.1.jar:7.7.1 5bf96d32f88eb8a2f5e775339885cd6ba84a3b58 - ishan -
2019-02-23 02:39:07]
at
org.apache.solr.handler.RequestHandlerBase.handleRequest(RequestHandlerBase.java:199)
[solr-core-7.7.1.jar:7.7.1 5bf96d32f88eb8a2f5e775339885cd6ba84a3b58 - ishan -
2019-02-23 02:39:07]
at
org.apache.solr.servlet.HttpSolrCall.handleAdmin(HttpSolrCall.java:735)
[solr-core-7.7.1.jar:7.7.1 5bf96d32f88eb8a2f5e775339885cd6ba84a3b58 - ishan -
2019-02-23 02:39:07]
at
org.apache.solr.servlet.HttpSolrCall.handleAdminRequest(HttpSolrCall.java:716)
[solr-core-7.7.1.jar:7.7.1 5bf96d32f88eb8a2f5e775339885cd6ba84a3b58 - ishan -
2019-02-23 02:39:07]
at
org.apache.solr.servlet.HttpSolrCall.call(HttpSolrCall.java:496)
[solr-core-7.7.1.jar:7.7.1 5bf96d32f88eb8a2f5e775339885cd6ba84a3b58 - ishan -
2019-02-23 02:39:07]
at
org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:395)
[solr-core-7.7.1.jar:7.7.1 5bf96d32f88eb8a2f5e775339885cd6ba84a3b58 - ishan -
2019-02-23 02:39:07]
at
org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:341)
[solr-core-7.7.1.jar:7.7.1 5bf96d32f88eb8a2f5e775339885cd6ba84a3b58 - ishan -
2019-02-23 02:39:07]
at
org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1602)
[jetty-servlet-9.4.14.v20181114.jar:9.4.14.v20181114]
at
org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:540)
[jetty-servlet-9.4.14.v20181114.jar:9.4.14.v20181114]
at
org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:146)
[jetty-server-9.4.14.v20181114.jar:9.4.14.v20181114]
at
org.eclipse.jetty.security.SecurityHandler.handle(SecurityHandler.java:548)
[jetty-security-9.4.14.v20181114.jar:9.4.14.v20181114]
at
org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:132)
[jetty-server-9.4.14.v20181114.jar:9.4.14.v20181114]
at
org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:257)
[jetty-server-9.4.14.v20181114.jar:9.4.14.v20181114]
at
org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:1588)
[jetty-server-9.4.14.v20181114.jar:9.4.14.v20181114]
at
org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:255)
[jetty-server-9.4.14.v20181114.jar:9.4.14.v20181114]
at
org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1345)
[jetty-server-9.4.14.v20181114.jar:9.4.14.v20181114]
at
org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:203)
[jetty-server-9.4.14.v20181114.jar:9.4.14.v20181114]
at
org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:480)
[jetty-servlet-9.4.14.v20181114.jar:9.4.14.v20181114]
at
org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:1557)
[jetty-server-9.4.14.v20181114.jar:9.4.14.v20181114]
at
org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:201)
[jetty-server-9.4.14.v20181114.jar:9.4.14.v20181114]
at
org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1247)
[jetty-server-9.4.14.v20181114.jar:9.4.14.v20181114]
at
org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:144)
[jetty-server-9.4.14.v20181114.jar:9.4.14.v20181114]
at
org.eclipse.jetty.server.handler.ContextHandlerCollection.handle(ContextHandlerCollection.java:220)
[jetty-server-9.4.14.v20181114.jar:9.4.14.v20181114]
at
org.eclipse.jetty.server.handler.HandlerCollection.handle(HandlerCollection.java:126)
[jetty-server-9.4.14.v20181114.jar:9.4.14.v20181114]
at
org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:132)
[jetty-server-9.4.14.v20181114.jar:9.4.14.v20181114]
at
org.eclipse.jetty.rewrite.handler.RewriteHandler.handle(RewriteHandler.java:335)
[jetty-rewrite-9.4.14.v20181114.jar:9.4.14.v20181114]
at
org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:132)
[jetty-server-9.4.14.v20181114.jar:9.4.14.v20181114]
at org.eclipse.jetty.server.Server.handle(Server.java:502)
[jetty-server-9.4.14.v20181114.jar:9.4.14.v20181114]
at
org.eclipse.jetty.server.HttpChannel.handle(HttpChannel.java:364)
[jetty-server-9.4.14.v20181114.jar:9.4.14.v20181114]
at
org.eclipse.jetty.server.HttpConnection.onFillable(HttpConnection.java:260)
[jetty-server-9.4.14.v20181114.jar:9.4.14.v20181114]
at
org.eclipse.jetty.io.AbstractConnection$ReadCallback.succeeded(AbstractConnection.java:305)
[jetty-io-9.4.14.v20181114.jar:9.4.14.v20181114]
at
org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:103)
[jetty-io-9.4.14.v20181114.jar:9.4.14.v20181114]
at
org.eclipse.jetty.io.ssl.SslConnection$DecryptedEndPoint.onFillable(SslConnection.java:411)
[jetty-io-9.4.14.v20181114.jar:9.4.14.v20181114]
at
org.eclipse.jetty.io.ssl.SslConnection.onFillable(SslConnection.java:305)
[jetty-io-9.4.14.v20181114.jar:9.4.14.v20181114]
at
org.eclipse.jetty.io.ssl.SslConnection$2.succeeded(SslConnection.java:159)
[jetty-io-9.4.14.v20181114.jar:9.4.14.v20181114]
at
org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:103)
[jetty-io-9.4.14.v20181114.jar:9.4.14.v20181114]
at
org.eclipse.jetty.io.ChannelEndPoint$2.run(ChannelEndPoint.java:118)
[jetty-io-9.4.14.v20181114.jar:9.4.14.v20181114]
at
org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.runTask(EatWhatYouKill.java:333)
[jetty-util-9.4.14.v20181114.jar:9.4.14.v20181114]
at
org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:310)
[jetty-util-9.4.14.v20181114.jar:9.4.14.v20181114]
at
org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:168)
[jetty-util-9.4.14.v20181114.jar:9.4.14.v20181114]
at
org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.run(EatWhatYouKill.java:126)
[jetty-util-9.4.14.v20181114.jar:9.4.14.v20181114]
at
org.eclipse.jetty.util.thread.ReservedThreadExecutor$ReservedThread.run(ReservedThreadExecutor.java:366)
[jetty-util-9.4.14.v20181114.jar:9.4.14.v20181114]
at
org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:765)
[jetty-util-9.4.14.v20181114.jar:9.4.14.v20181114]
at
org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:683)
[jetty-util-9.4.14.v20181114.jar:9.4.14.v20181114]
at java.lang.Thread.run(Thread.java:834) [?:?]
Caused by: org.apache.solr.client.solrj.SolrServerException: IOException
occured when talking to server at: http://mynode1_fqdn:8983/solr
at
org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:657)
~[solr-solrj-7.7.1.jar:7.7.1 5bf96d32f88eb8a2f5e775339885cd6ba84a3b58 - ishan
- 2019-02-23 02:39:09]
at
org.apache.solr.client.solrj.impl.HttpSolrClient.lambda$httpUriRequest$0(HttpSolrClient.java:299)
~[solr-solrj-7.7.1.jar:7.7.1 5bf96d32f88eb8a2f5e775339885cd6ba84a3b58 - ishan
- 2019-02-23 02:39:09]
at java.util.concurrent.FutureTask.run(FutureTask.java:264)
~[?:?]
at
org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:209)
~[solr-solrj-7.7.1.jar:7.7.1 5bf96d32f88eb8a2f5e775339885cd6ba84a3b58 - ishan
- 2019-02-23 02:39:09]
at
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128)
~[?:?]
at
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
~[?:?]
... 1 more
Caused by: org.apache.http.client.ClientProtocolException
at
org.apache.http.impl.client.InternalHttpClient.doExecute(InternalHttpClient.java:187)
~[httpclient-4.5.6.jar:4.5.6]
at
org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:83)
~[httpclient-4.5.6.jar:4.5.6]
at
org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:56)
~[httpclient-4.5.6.jar:4.5.6]
at
org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:542)
~[solr-solrj-7.7.1.jar:7.7.1 5bf96d32f88eb8a2f5e775339885cd6ba84a3b58 - ishan
- 2019-02-23 02:39:09]
at
org.apache.solr.client.solrj.impl.HttpSolrClient.lambda$httpUriRequest$0(HttpSolrClient.java:299)
~[solr-solrj-7.7.1.jar:7.7.1 5bf96d32f88eb8a2f5e775339885cd6ba84a3b58 - ishan
- 2019-02-23 02:39:09]
at java.util.concurrent.FutureTask.run(FutureTask.java:264)
~[?:?]
at
org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:209)
~[solr-solrj-7.7.1.jar:7.7.1 5bf96d32f88eb8a2f5e775339885cd6ba84a3b58 - ishan
- 2019-02-23 02:39:09]
at
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128)
~[?:?]
at
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
~[?:?]
... 1 more
Caused by: org.apache.http.ProtocolException: The server failed to respond with
a valid HTTP response
at
org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:149)
~[httpclient-4.5.6.jar:4.5.6]
at
org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:56)
~[httpclient-4.5.6.jar:4.5.6]
at
org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:259)
~[httpcore-4.4.10.jar:4.4.10]
at
org.apache.http.impl.DefaultBHttpClientConnection.receiveResponseHeader(DefaultBHttpClientConnection.java:163)
~[httpcore-4.4.10.jar:4.4.10]
at
org.apache.http.impl.conn.CPoolProxy.receiveResponseHeader(CPoolProxy.java:165)
~[httpclient-4.5.6.jar:4.5.6]
at
org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:273)
~[httpcore-4.4.10.jar:4.4.10]
at
org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:125)
~[httpcore-4.4.10.jar:4.4.10]
at
org.apache.http.impl.execchain.MainClientExec.execute(MainClientExec.java:272)
~[httpclient-4.5.6.jar:4.5.6]
at
org.apache.http.impl.execchain.ProtocolExec.execute(ProtocolExec.java:185)
~[httpclient-4.5.6.jar:4.5.6]
at
org.apache.http.impl.execchain.RetryExec.execute(RetryExec.java:89)
~[httpclient-4.5.6.jar:4.5.6]
at
org.apache.http.impl.execchain.RedirectExec.execute(RedirectExec.java:110)
~[httpclient-4.5.6.jar:4.5.6]
at
org.apache.http.impl.client.InternalHttpClient.doExecute(InternalHttpClient.java:185)
~[httpclient-4.5.6.jar:4.5.6]
at
org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:83)
~[httpclient-4.5.6.jar:4.5.6]
at
org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:56)
~[httpclient-4.5.6.jar:4.5.6]
at
org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:542)
~[solr-solrj-7.7.1.jar:7.7.1 5bf96d32f88eb8a2f5e775339885cd6ba84a3b58 - ishan
- 2019-02-23 02:39:09]
Tony Sargent
Unix Middle Tier Admin, Web Operations
801.442.5579 | tony.sarg...@imail.org<mailto:tony.sarg...@imail.org>
