Just to be clear... can you confirm whether or not your servers are I/O bound?
Have you verified that your total index size on each server is no greater than the amount of OS system memory available for file caching? I mean, the fact that you are "heavy sharded" on each node does not mean there will be a reduction in the amount of system memory needed for file caching of the Lucene index. 100 / 4 * 2.8GB = 70 GB of RAM needed on each server. -- Jack Krupansky On Thu, Jan 8, 2015 at 10:57 AM, Andrew Butkus < andrew.but...@c6-intelligence.com> wrote: > Hi Shawn, > > Thank you for your reply > > >The part about memory usage is not clear. That 4GB and 16GB could refer > to the operating system view of memory, or the view of memory within the > JVM. I'm curious about how much total >RAM each machine has, how large the > Java heap is, and what the total size of the indexes that live on each > machine is. > > >Even if they are individually very small, 500 million documents will > result in a very large index, so I'm guessing that you don't have enough > RAM on each server for your index size. > > >What can happen with a highly sharded index that is too large for > available RAM: Index data for the initial queries gets read from the OS > disk cache, but as those queries run, the information >required for the > shards that come later in the distributed query gets pushed out of the disk > cache, so Solr must actually read the disk to do those later queries. > Disks are slow, so if the machine >has to actually read from the disk, Solr > will be slow. > > >http://wiki.apache.org/solr/SolrPerformanceProblems#RAM > > > We have 4gb usage (because the shards are split by 100 each shard is > approx. 2.8gb on disk), we have allocated 14gb min and 16gb max of ram to > solr, so it has plenty to use (the ram in the dashboard never goes above > about 8gb - so still plenty ). > > I've managed to reproduce the issue with shards= parameter, and think I > have proven the disk cache issue to not be the problem > > I'm simply querying the same shard, on the same server, multiple times (so > the shards index should always be in memory and never loaded from disk)? > > All but the last query are low ms ... > > { > "responseHeader":{ > "status":0, > "QTime":50190, > "params":{ > "sort":"id ASC", > "shards":" > 1.1.1.18:8985/solr/Collection,1.1.1.18:8985/solr/Collection,1.1.1.18:8985/solr/Collection,1.1.1.18:8985/solr/Collection,1.1.1.18:8985/solr/Collection,1.1.1.18:8985/solr/Collection,1.1.1.18:8985/solr/Collection,1.1.1.18:8985/solr/Collection,1.1.1.18:8985/solr/Collection,1.1.1.18:8985/solr/Collection,1.1.1.18:8985/solr/Collection,1.1.1.18:8985/solr/Collection,1.1.1.18:8985/solr/Collection,1.1.1.18:8985/solr/Collection,1.1.1.18:8985/solr/Collection,1.1.1.18:8985/solr/Collection,1.1.1.18:8985/solr/Collection,1.1.1.18:8985/solr/Collection,1.1.1.18:8985/solr/Collection,1.1.1.18:8985/solr/Collection,1.1.1.18:8985/solr/Collection,1.1.1.18:8985/solr/Collection,1.1.1.18:8985/solr/Collection,1.1.1.18:8985/solr/Collection,1.1.1.18:8985/solr/Collection,1.1.1.18:8985/solr/Collection,1.1.1.18:8985/solr/Collection,1.1.1.18:8985/solr/Collection,1.1.1.18:8985/solr/Collection,1.1.1.18:8985/solr/Collection,1.1.1.18:8985/solr/Collection,1.1.1.18:8985/solr/Collection,1.1.1.18:8985/solr/Collection,1.1.1.18:8985/solr/Collection,1.1.1.18:8985/solr/Collection,1.1.1.18:8985/solr/Collection,1.1.1.18:8985/solr/Collection,1.1.1.18:8985/solr/Collection,1.1.1.18:8985/solr/Collection > ", > "indent":"true", > "q":"spec_country:\"United Kingdom\"", > "shards.info":"true", > "distrib":"false", > "cursorMark":"*", > "wt":"json", > "rows":"10"}}, > "shards.info":{ > "1.1.1.18:8985/solr/Collection":{ > "numFound":242731, > "maxScore":null, > "shardAddress":"http://1.1.1.18:8985/solr/Collection";, > "time":24}, > "1.1.1.18:8985/solr/Collection":{ > "numFound":242731, > "maxScore":null, > "shardAddress":"http://1.1.1.18:8985/solr/Collection";, > "time":35}, > "1.1.1.18:8985/solr/Collection":{ > "numFound":242731, > "maxScore":null, > "shardAddress":"http://1.1.1.18:8985/solr/Collection";, > "time":35}, > "1.1.1.18:8985/solr/Collection":{ > "numFound":242731, > "maxScore":null, > "shardAddress":"http://1.1.1.18:8985/solr/Collection";, > "time":52}, > "1.1.1.18:8985/solr/Collection":{ > "numFound":242731, > "maxScore":null, > "shardAddress":"http://1.1.1.18:8985/solr/Collection";, > "time":55}, > "1.1.1.18:8985/solr/Collection":{ > "numFound":242731, > "maxScore":null, > "shardAddress":"http://1.1.1.18:8985/solr/Collection";, > "time":59}, > "1.1.1.18:8985/solr/Collection":{ > "numFound":242731, > "maxScore":null, > "shardAddress":"http://1.1.1.18:8985/solr/Collection";, > "time":58}, > "1.1.1.18:8985/solr/Collection":{ > "numFound":242731, > "maxScore":null, > "shardAddress":"http://1.1.1.18:8985/solr/Collection";, > "time":75}, > "1.1.1.18:8985/solr/Collection":{ > "numFound":242731, > "maxScore":null, > "shardAddress":"http://1.1.1.18:8985/solr/Collection";, > "time":79}, > "1.1.1.18:8985/solr/Collection":{ > "numFound":242731, > "maxScore":null, > "shardAddress":"http://1.1.1.18:8985/solr/Collection";, > "time":78}, > "1.1.1.18:8985/solr/Collection":{ > "numFound":242731, > "maxScore":null, > "shardAddress":"http://1.1.1.18:8985/solr/Collection";, > "time":79}, > "1.1.1.18:8985/solr/Collection":{ > "numFound":242731, > "maxScore":null, > "shardAddress":"http://1.1.1.18:8985/solr/Collection";, > "time":80}, > "1.1.1.18:8985/solr/Collection":{ > "numFound":242731, > "maxScore":null, > "shardAddress":"http://1.1.1.18:8985/solr/Collection";, > "time":101}, > "1.1.1.18:8985/solr/Collection":{ > "numFound":242731, > "maxScore":null, > "shardAddress":"http://1.1.1.18:8985/solr/Collection";, > "time":100}, > "1.1.1.18:8985/solr/Collection":{ > "numFound":242731, > "maxScore":null, > "shardAddress":"http://1.1.1.18:8985/solr/Collection";, > "time":106}, > "1.1.1.18:8985/solr/Collection":{ > "numFound":242731, > "maxScore":null, > "shardAddress":"http://1.1.1.18:8985/solr/Collection";, > "time":114}, > "1.1.1.18:8985/solr/Collection":{ > "numFound":242731, > "maxScore":null, > "shardAddress":"http://1.1.1.18:8985/solr/Collection";, > "time":112}, > "1.1.1.18:8985/solr/Collection":{ > "numFound":242731, > "maxScore":null, > "shardAddress":"http://1.1.1.18:8985/solr/Collection";, > "time":123}, > "1.1.1.18:8985/solr/Collection":{ > "numFound":242731, > "maxScore":null, > "shardAddress":"http://1.1.1.18:8985/solr/Collection";, > "time":135}, > "1.1.1.18:8985/solr/Collection":{ > "numFound":242731, > "maxScore":null, > "shardAddress":"http://1.1.1.18:8985/solr/Collection";, > "time":135}, > "1.1.1.18:8985/solr/Collection":{ > "numFound":242731, > "maxScore":null, > "shardAddress":"http://1.1.1.18:8985/solr/Collection";, > "time":136}, > "1.1.1.18:8985/solr/Collection":{ > "numFound":242731, > "maxScore":null, > "shardAddress":"http://1.1.1.18:8985/solr/Collection";, > "time":135}, > "1.1.1.18:8985/solr/Collection":{ > "numFound":242731, > "maxScore":null, > "shardAddress":"http://1.1.1.18:8985/solr/Collection";, > "time":136}, > "1.1.1.18:8985/solr/Collection":{ > "numFound":242731, > "maxScore":null, > "shardAddress":"http://1.1.1.18:8985/solr/Collection";, > "time":137}, > "1.1.1.18:8985/solr/Collection":{ > "numFound":242731, > "maxScore":null, > "shardAddress":"http://1.1.1.18:8985/solr/Collection";, > "time":136}, > "1.1.1.18:8985/solr/Collection":{ > "numFound":242731, > "maxScore":null, > "shardAddress":"http://1.1.1.18:8985/solr/Collection";, > "time":135}, > "1.1.1.18:8985/solr/Collection":{ > "numFound":242731, > "maxScore":null, > "shardAddress":"http://1.1.1.18:8985/solr/Collection";, > "time":137}, > "1.1.1.18:8985/solr/Collection":{ > "numFound":242731, > "maxScore":null, > "shardAddress":"http://1.1.1.18:8985/solr/Collection";, > "time":135}, > "1.1.1.18:8985/solr/Collection":{ > "numFound":242731, > "maxScore":null, > "shardAddress":"http://1.1.1.18:8985/solr/Collection";, > "time":136}, > "1.1.1.18:8985/solr/Collection":{ > "numFound":242731, > "maxScore":null, > "shardAddress":"http://1.1.1.18:8985/solr/Collection";, > "time":137}, > "1.1.1.18:8985/solr/Collection":{ > "numFound":242731, > "maxScore":null, > "shardAddress":"http://1.1.1.18:8985/solr/Collection";, > "time":137}, > "1.1.1.18:8985/solr/Collection":{ > "numFound":242731, > "maxScore":null, > "shardAddress":"http://1.1.1.18:8985/solr/Collection";, > "time":141}, > "1.1.1.18:8985/solr/Collection":{ > "numFound":242731, > "maxScore":null, > "shardAddress":"http://1.1.1.18:8985/solr/Collection";, > "time":145}, > "1.1.1.18:8985/solr/Collection":{ > "numFound":242731, > "maxScore":null, > "shardAddress":"http://1.1.1.18:8985/solr/Collection";, > "time":148}, > "1.1.1.18:8985/solr/Collection":{ > "numFound":242731, > "maxScore":null, > "shardAddress":"http://1.1.1.18:8985/solr/Collection";, > "time":151}, > "1.1.1.18:8985/solr/Collection":{ > "numFound":242731, > "maxScore":null, > "shardAddress":"http://1.1.1.18:8985/solr/Collection";, > "time":152}, > "1.1.1.18:8985/solr/Collection":{ > "numFound":242731, > "maxScore":null, > "shardAddress":"http://1.1.1.18:8985/solr/Collection";, > "time":155}, > "1.1.1.18:8985/solr/Collection":{ > "numFound":242731, > "maxScore":null, > "shardAddress":"http://1.1.1.18:8985/solr/Collection";, > "time":161}, > "1.1.1.18:8985/solr/Collection":{ > "numFound":242731, > "maxScore":null, > "shardAddress":"http://1.1.1.18:8985/solr/Collection";, > "time":50180}}, > "response":{"numFound":9466129,"start":0,"docs":[ > { > } > "nextCursorMark":"AoE2NjM1MzU0ODg1OTk2NTAzMTUyMjg1OA=="} >
