Hi Shawn, Thank you for your reply
>The part about memory usage is not clear. That 4GB and 16GB could refer to >the operating system view of memory, or the view of memory within the JVM. >I'm curious about how much total >RAM each machine has, how large the Java >heap is, and what the total size of the indexes that live on each machine is. >Even if they are individually very small, 500 million documents will result in >a very large index, so I'm guessing that you don't have enough RAM on each >server for your index size. >What can happen with a highly sharded index that is too large for available >RAM: Index data for the initial queries gets read from the OS disk cache, but >as those queries run, the information >required for the shards that come later >in the distributed query gets pushed out of the disk cache, so Solr must >actually read the disk to do those later queries. Disks are slow, so if the >machine >has to actually read from the disk, Solr will be slow. >http://wiki.apache.org/solr/SolrPerformanceProblems#RAM We have 4gb usage (because the shards are split by 100 each shard is approx. 2.8gb on disk), we have allocated 14gb min and 16gb max of ram to solr, so it has plenty to use (the ram in the dashboard never goes above about 8gb - so still plenty ). I've managed to reproduce the issue with shards= parameter, and think I have proven the disk cache issue to not be the problem I'm simply querying the same shard, on the same server, multiple times (so the shards index should always be in memory and never loaded from disk)? All but the last query are low ms ... { "responseHeader":{ "status":0, "QTime":50190, "params":{ "sort":"id ASC", "shards":"1.1.1.18:8985/solr/Collection,1.1.1.18:8985/solr/Collection,1.1.1.18:8985/solr/Collection,1.1.1.18:8985/solr/Collection,1.1.1.18:8985/solr/Collection,1.1.1.18:8985/solr/Collection,1.1.1.18:8985/solr/Collection,1.1.1.18:8985/solr/Collection,1.1.1.18:8985/solr/Collection,1.1.1.18:8985/solr/Collection,1.1.1.18:8985/solr/Collection,1.1.1.18:8985/solr/Collection,1.1.1.18:8985/solr/Collection,1.1.1.18:8985/solr/Collection,1.1.1.18:8985/solr/Collection,1.1.1.18:8985/solr/Collection,1.1.1.18:8985/solr/Collection,1.1.1.18:8985/solr/Collection,1.1.1.18:8985/solr/Collection,1.1.1.18:8985/solr/Collection,1.1.1.18:8985/solr/Collection,1.1.1.18:8985/solr/Collection,1.1.1.18:8985/solr/Collection,1.1.1.18:8985/solr/Collection,1.1.1.18:8985/solr/Collection,1.1.1.18:8985/solr/Collection,1.1.1.18:8985/solr/Collection,1.1.1.18:8985/solr/Collection,1.1.1.18:8985/solr/Collection,1.1.1.18:8985/solr/Collection,1.1.1.18:8985/solr/Collection,1.1.1.18:8985/solr/Collection,1.1.1.18:8985/solr/Collection,1.1.1.18:8985/solr/Collection,1.1.1.18:8985/solr/Collection,1.1.1.18:8985/solr/Collection,1.1.1.18:8985/solr/Collection,1.1.1.18:8985/solr/Collection,1.1.1.18:8985/solr/Collection", "indent":"true", "q":"spec_country:\"United Kingdom\"", "shards.info":"true", "distrib":"false", "cursorMark":"*", "wt":"json", "rows":"10"}}, "shards.info":{ "1.1.1.18:8985/solr/Collection":{ "numFound":242731, "maxScore":null, "shardAddress":"http://1.1.1.18:8985/solr/Collection";, "time":24}, "1.1.1.18:8985/solr/Collection":{ "numFound":242731, "maxScore":null, "shardAddress":"http://1.1.1.18:8985/solr/Collection";, "time":35}, "1.1.1.18:8985/solr/Collection":{ "numFound":242731, "maxScore":null, "shardAddress":"http://1.1.1.18:8985/solr/Collection";, "time":35}, "1.1.1.18:8985/solr/Collection":{ "numFound":242731, "maxScore":null, "shardAddress":"http://1.1.1.18:8985/solr/Collection";, "time":52}, "1.1.1.18:8985/solr/Collection":{ "numFound":242731, "maxScore":null, "shardAddress":"http://1.1.1.18:8985/solr/Collection";, "time":55}, "1.1.1.18:8985/solr/Collection":{ "numFound":242731, "maxScore":null, "shardAddress":"http://1.1.1.18:8985/solr/Collection";, "time":59}, "1.1.1.18:8985/solr/Collection":{ "numFound":242731, "maxScore":null, "shardAddress":"http://1.1.1.18:8985/solr/Collection";, "time":58}, "1.1.1.18:8985/solr/Collection":{ "numFound":242731, "maxScore":null, "shardAddress":"http://1.1.1.18:8985/solr/Collection";, "time":75}, "1.1.1.18:8985/solr/Collection":{ "numFound":242731, "maxScore":null, "shardAddress":"http://1.1.1.18:8985/solr/Collection";, "time":79}, "1.1.1.18:8985/solr/Collection":{ "numFound":242731, "maxScore":null, "shardAddress":"http://1.1.1.18:8985/solr/Collection";, "time":78}, "1.1.1.18:8985/solr/Collection":{ "numFound":242731, "maxScore":null, "shardAddress":"http://1.1.1.18:8985/solr/Collection";, "time":79}, "1.1.1.18:8985/solr/Collection":{ "numFound":242731, "maxScore":null, "shardAddress":"http://1.1.1.18:8985/solr/Collection";, "time":80}, "1.1.1.18:8985/solr/Collection":{ "numFound":242731, "maxScore":null, "shardAddress":"http://1.1.1.18:8985/solr/Collection";, "time":101}, "1.1.1.18:8985/solr/Collection":{ "numFound":242731, "maxScore":null, "shardAddress":"http://1.1.1.18:8985/solr/Collection";, "time":100}, "1.1.1.18:8985/solr/Collection":{ "numFound":242731, "maxScore":null, "shardAddress":"http://1.1.1.18:8985/solr/Collection";, "time":106}, "1.1.1.18:8985/solr/Collection":{ "numFound":242731, "maxScore":null, "shardAddress":"http://1.1.1.18:8985/solr/Collection";, "time":114}, "1.1.1.18:8985/solr/Collection":{ "numFound":242731, "maxScore":null, "shardAddress":"http://1.1.1.18:8985/solr/Collection";, "time":112}, "1.1.1.18:8985/solr/Collection":{ "numFound":242731, "maxScore":null, "shardAddress":"http://1.1.1.18:8985/solr/Collection";, "time":123}, "1.1.1.18:8985/solr/Collection":{ "numFound":242731, "maxScore":null, "shardAddress":"http://1.1.1.18:8985/solr/Collection";, "time":135}, "1.1.1.18:8985/solr/Collection":{ "numFound":242731, "maxScore":null, "shardAddress":"http://1.1.1.18:8985/solr/Collection";, "time":135}, "1.1.1.18:8985/solr/Collection":{ "numFound":242731, "maxScore":null, "shardAddress":"http://1.1.1.18:8985/solr/Collection";, "time":136}, "1.1.1.18:8985/solr/Collection":{ "numFound":242731, "maxScore":null, "shardAddress":"http://1.1.1.18:8985/solr/Collection";, "time":135}, "1.1.1.18:8985/solr/Collection":{ "numFound":242731, "maxScore":null, "shardAddress":"http://1.1.1.18:8985/solr/Collection";, "time":136}, "1.1.1.18:8985/solr/Collection":{ "numFound":242731, "maxScore":null, "shardAddress":"http://1.1.1.18:8985/solr/Collection";, "time":137}, "1.1.1.18:8985/solr/Collection":{ "numFound":242731, "maxScore":null, "shardAddress":"http://1.1.1.18:8985/solr/Collection";, "time":136}, "1.1.1.18:8985/solr/Collection":{ "numFound":242731, "maxScore":null, "shardAddress":"http://1.1.1.18:8985/solr/Collection";, "time":135}, "1.1.1.18:8985/solr/Collection":{ "numFound":242731, "maxScore":null, "shardAddress":"http://1.1.1.18:8985/solr/Collection";, "time":137}, "1.1.1.18:8985/solr/Collection":{ "numFound":242731, "maxScore":null, "shardAddress":"http://1.1.1.18:8985/solr/Collection";, "time":135}, "1.1.1.18:8985/solr/Collection":{ "numFound":242731, "maxScore":null, "shardAddress":"http://1.1.1.18:8985/solr/Collection";, "time":136}, "1.1.1.18:8985/solr/Collection":{ "numFound":242731, "maxScore":null, "shardAddress":"http://1.1.1.18:8985/solr/Collection";, "time":137}, "1.1.1.18:8985/solr/Collection":{ "numFound":242731, "maxScore":null, "shardAddress":"http://1.1.1.18:8985/solr/Collection";, "time":137}, "1.1.1.18:8985/solr/Collection":{ "numFound":242731, "maxScore":null, "shardAddress":"http://1.1.1.18:8985/solr/Collection";, "time":141}, "1.1.1.18:8985/solr/Collection":{ "numFound":242731, "maxScore":null, "shardAddress":"http://1.1.1.18:8985/solr/Collection";, "time":145}, "1.1.1.18:8985/solr/Collection":{ "numFound":242731, "maxScore":null, "shardAddress":"http://1.1.1.18:8985/solr/Collection";, "time":148}, "1.1.1.18:8985/solr/Collection":{ "numFound":242731, "maxScore":null, "shardAddress":"http://1.1.1.18:8985/solr/Collection";, "time":151}, "1.1.1.18:8985/solr/Collection":{ "numFound":242731, "maxScore":null, "shardAddress":"http://1.1.1.18:8985/solr/Collection";, "time":152}, "1.1.1.18:8985/solr/Collection":{ "numFound":242731, "maxScore":null, "shardAddress":"http://1.1.1.18:8985/solr/Collection";, "time":155}, "1.1.1.18:8985/solr/Collection":{ "numFound":242731, "maxScore":null, "shardAddress":"http://1.1.1.18:8985/solr/Collection";, "time":161}, "1.1.1.18:8985/solr/Collection":{ "numFound":242731, "maxScore":null, "shardAddress":"http://1.1.1.18:8985/solr/Collection";, "time":50180}}, "response":{"numFound":9466129,"start":0,"docs":[ { } "nextCursorMark":"AoE2NjM1MzU0ODg1OTk2NTAzMTUyMjg1OA=="}
