Hi, we have 8 solr servers, split 4x4 across 2 data centers. We have a collection of around ½ billion documents, split over 100 shards, each is replicated 4 times on separate nodes (evenly distributed across both data centers).
The problem we have is that when we use cursormark (and also when we don't use cursormark the pattern below is the same but just shorter in time) the time it takes to query each shard gets progressively longer when distrib=true , I have tried to query shards directly (with shards=) and select my own shards to query to see if it was a bandwidth bottleneck and the performance is normal / fine - when using pre-defined shards. Does anyone know why the shards become progressively slower when distrib=true? Or any suggestions on how I can fix, or how to debug the problem further? I have monitored the performance of CPU and it never goes above 10% on each server, so its not cpu, also the memory usage is about 4gb out of 16gb so its not a memory issue either. I have tried all shard shuffling strategies incase it was a bottleneck at a server being over used but as above, the cpu never goes above 10%, and when I use shards= there are never any querytime bottlenecks. Around http://2.2.213:8985/solr/Collection_shard16_replica1/|http://1.1.1.16:8985/solr/Collection_shard16_replica2/|http://1.1.1.17:8985/solr/Collection_shard16_replica3/|http://2.2.216:8985/solr/Collection_shard16_replica4/":{ "numFound":242899, "maxScore":null, "shardAddress":"http://1.1.1.17:8985/solr/Collection_shard16_replica3";, "time":134}, The timings get progressively worse, (there is a pattern, the time it takes to run queries on shards increasingly gets worse after about the first 60 entries - even though the earlier ones took a few milliseconds) Here is my trace output { "responseHeader":{ "status":0, "QTime":50093, "params":{ "shard.shuffling.strategy":"query", "sort":"id ASC", "indent":"true", "q":"spec_country:\"United Kingdom\"", "shards.info":"true", "distrib":"true", "cursorMark":"*", "wt":"json", "rows":"0"}}, "shards.info":{ "http://1.1.1.17:8985/solr/Collection_shard78_replica3/|http://2.2.216:8985/solr/Collection_shard78_replica4/|http://2.2.213:8985/solr/Collection_shard78_replica1/|http://1.1.1.16:8985/solr/Collection_shard78_replica2/":{ "numFound":243009, "maxScore":null, "shardAddress":"http://1.1.1.16:8985/solr/Collection_shard78_replica2";, "time":24}, "http://2.2.213:8985/solr/Collection_shard24_replica1/|http://1.1.1.16:8985/solr/Collection_shard24_replica2/|http://1.1.1.17:8985/solr/Collection_shard24_replica3/|http://2.2.216:8985/solr/Collection_shard24_replica4/":{ "numFound":242309, "maxScore":null, "shardAddress":"http://1.1.1.16:8985/solr/Collection_shard24_replica2";, "time":23}, "http://1.1.1.17:8985/solr/Collection_shard70_replica3/|http://2.2.213:8985/solr/Collection_shard70_replica1/|http://1.1.1.16:8985/solr/Collection_shard70_replica2/|http://2.2.216:8985/solr/Collection_shard70_replica4/":{ "numFound":242727, "maxScore":null, "shardAddress":"http://1.1.1.16:8985/solr/Collection_shard70_replica2";, "time":23}, "http://2.2.216:8985/solr/Collection_shard76_replica4/|http://1.1.1.17:8985/solr/Collection_shard76_replica3/|http://2.2.213:8985/solr/Collection_shard76_replica1/|http://1.1.1.16:8985/solr/Collection_shard76_replica2/":{ "numFound":243324, "maxScore":null, "shardAddress":"http://2.2.216:8985/solr/Collection_shard76_replica4";, "time":26}, "http://2.2.214:8985/solr/Collection_shard29_replica1/|http://1.1.1.18:8985/solr/Collection_shard29_replica2/|http://1.1.1.15:8985/solr/Collection_shard29_replica3/|http://2.2.215:8985/solr/Collection_shard29_replica4/":{ "numFound":242559, "maxScore":null, "shardAddress":"http://2.2.214:8985/solr/Collection_shard29_replica1";, "time":25}, "http://1.1.1.17:8985/solr/Collection_shard74_replica3/|http://2.2.213:8985/solr/Collection_shard74_replica1/|http://2.2.216:8985/solr/Collection_shard74_replica4/|http://1.1.1.16:8985/solr/Collection_shard74_replica2/":{ "numFound":242419, "maxScore":null, "shardAddress":"http://2.2.216:8985/solr/Collection_shard74_replica4";, "time":24}, "http://1.1.1.18:8985/solr/Collection_shard33_replica2/|http://1.1.1.15:8985/solr/Collection_shard33_replica3/|http://2.2.214:8985/solr/Collection_shard33_replica1/|http://2.2.215:8985/solr/Collection_shard33_replica4/":{ "numFound":242571, "maxScore":null, "shardAddress":"http://2.2.214:8985/solr/Collection_shard33_replica1";, "time":25}, "http://1.1.1.18:8985/solr/Collection_shard77_replica2/|http://2.2.215:8985/solr/Collection_shard77_replica4/|http://1.1.1.15:8985/solr/Collection_shard77_replica3/|http://2.2.214:8985/solr/Collection_shard77_replica1/":{ "numFound":242901, "maxScore":null, "shardAddress":"http://2.2.215:8985/solr/Collection_shard77_replica4";, "time":27}, "http://1.1.1.18:8985/solr/Collection_shard75_replica2/|http://2.2.215:8985/solr/Collection_shard75_replica4/|http://1.1.1.15:8985/solr/Collection_shard75_replica3/|http://2.2.214:8985/solr/Collection_shard75_replica1/":{ "numFound":242731, "maxScore":null, "shardAddress":"http://1.1.1.15:8985/solr/Collection_shard75_replica3";, "time":29}, "http://2.2.213:8985/solr/Collection_shard72_replica1/|http://1.1.1.17:8985/solr/Collection_shard72_replica3/|http://2.2.216:8985/solr/Collection_shard72_replica4/|http://1.1.1.16:8985/solr/Collection_shard72_replica2/":{ "numFound":243030, "maxScore":null, "shardAddress":"http://2.2.216:8985/solr/Collection_shard72_replica4";, "time":28}, "http://2.2.213:8985/solr/Collection_shard26_replica1/|http://1.1.1.17:8985/solr/Collection_shard26_replica3/|http://1.1.1.16:8985/solr/Collection_shard26_replica2/|http://2.2.216:8985/solr/Collection_shard26_replica4/":{ "numFound":243083, "maxScore":null, "shardAddress":"http://1.1.1.16:8985/solr/Collection_shard26_replica2";, "time":29}, "http://1.1.1.18:8985/solr/Collection_shard25_replica2/|http://1.1.1.15:8985/solr/Collection_shard25_replica3/|http://2.2.214:8985/solr/Collection_shard25_replica1/|http://2.2.215:8985/solr/Collection_shard25_replica4/":{ "numFound":243050, "maxScore":null, "shardAddress":"http://2.2.215:8985/solr/Collection_shard25_replica4";, "time":29}, "http://2.2.215:8985/solr/Collection_shard73_replica4/|http://1.1.1.18:8985/solr/Collection_shard73_replica2/|http://1.1.1.15:8985/solr/Collection_shard73_replica3/|http://2.2.214:8985/solr/Collection_shard73_replica1/":{ "numFound":243667, "maxScore":null, "shardAddress":"http://2.2.215:8985/solr/Collection_shard73_replica4";, "time":29}, "http://2.2.215:8985/solr/Collection_shard71_replica4/|http://1.1.1.18:8985/solr/Collection_shard71_replica2/|http://1.1.1.15:8985/solr/Collection_shard71_replica3/|http://2.2.214:8985/solr/Collection_shard71_replica1/":{ "numFound":242827, "maxScore":null, "shardAddress":"http://1.1.1.18:8985/solr/Collection_shard71_replica2";, "time":34}, "http://2.2.213:8985/solr/Collection_shard32_replica1/|http://1.1.1.17:8985/solr/Collection_shard32_replica3/|http://1.1.1.16:8985/solr/Collection_shard32_replica2/|http://2.2.216:8985/solr/Collection_shard32_replica4/":{ "numFound":242320, "maxScore":null, "shardAddress":"http://2.2.216:8985/solr/Collection_shard32_replica4";, "time":33}, "http://2.2.215:8985/solr/Collection_shard69_replica4/|http://1.1.1.18:8985/solr/Collection_shard69_replica2/|http://1.1.1.15:8985/solr/Collection_shard69_replica3/|http://2.2.214:8985/solr/Collection_shard69_replica1/":{ "numFound":243481, "maxScore":null, "shardAddress":"http://2.2.214:8985/solr/Collection_shard69_replica1";, "time":33}, "http://1.1.1.18:8985/solr/Collection_shard31_replica2/|http://1.1.1.15:8985/solr/Collection_shard31_replica3/|http://2.2.214:8985/solr/Collection_shard31_replica1/|http://2.2.215:8985/solr/Collection_shard31_replica4/":{ "numFound":242878, "maxScore":null, "shardAddress":"http://2.2.215:8985/solr/Collection_shard31_replica4";, "time":34}, "http://2.2.214:8985/solr/Collection_shard59_replica1/|http://2.2.215:8985/solr/Collection_shard59_replica4/|http://1.1.1.15:8985/solr/Collection_shard59_replica3/|http://1.1.1.18:8985/solr/Collection_shard59_replica2/":{ "numFound":243374, "maxScore":null, "shardAddress":"http://1.1.1.15:8985/solr/Collection_shard59_replica3";, "time":20}, "http://1.1.1.18:8985/solr/Collection_shard23_replica2/|http://1.1.1.15:8985/solr/Collection_shard23_replica3/|http://2.2.214:8985/solr/Collection_shard23_replica1/|http://2.2.215:8985/solr/Collection_shard23_replica4/":{ "numFound":243482, "maxScore":null, "shardAddress":"http://2.2.214:8985/solr/Collection_shard23_replica1";, "time":22}, "http://1.1.1.17:8985/solr/Collection_shard68_replica3/|http://2.2.213:8985/solr/Collection_shard68_replica1/|http://1.1.1.16:8985/solr/Collection_shard68_replica2/|http://2.2.216:8985/solr/Collection_shard68_replica4/":{ "numFound":242898, "maxScore":null, "shardAddress":"http://1.1.1.17:8985/solr/Collection_shard68_replica3";, "time":35}, "http://1.1.1.17:8985/solr/Collection_shard60_replica3/|http://2.2.213:8985/solr/Collection_shard60_replica1/|http://2.2.216:8985/solr/Collection_shard60_replica4/|http://1.1.1.16:8985/solr/Collection_shard60_replica2/":{ "numFound":243392, "maxScore":null, "shardAddress":"http://1.1.1.16:8985/solr/Collection_shard60_replica2";, "time":23}, "http://1.1.1.18:8985/solr/Collection_shard15_replica2/|http://1.1.1.15:8985/solr/Collection_shard15_replica3/|http://2.2.214:8985/solr/Collection_shard15_replica1/|http://2.2.215:8985/solr/Collection_shard15_replica4/":{ "numFound":242558, "maxScore":null, "shardAddress":"http://1.1.1.15:8985/solr/Collection_shard15_replica3";, "time":26}, "http://1.1.1.18:8985/solr/Collection_shard21_replica2/|http://1.1.1.15:8985/solr/Collection_shard21_replica3/|http://2.2.214:8985/solr/Collection_shard21_replica1/|http://2.2.215:8985/solr/Collection_shard21_replica4/":{ "numFound":242492, "maxScore":null, "shardAddress":"http://1.1.1.15:8985/solr/Collection_shard21_replica3";, "time":25}, "http://1.1.1.17:8985/solr/Collection_shard64_replica3/|http://2.2.216:8985/solr/Collection_shard64_replica4/|http://2.2.213:8985/solr/Collection_shard64_replica1/|http://1.1.1.16:8985/solr/Collection_shard64_replica2/":{ "numFound":243205, "maxScore":null, "shardAddress":"http://2.2.216:8985/solr/Collection_shard64_replica4";, "time":25}, "http://1.1.1.17:8985/solr/Collection_shard94_replica3/|http://2.2.213:8985/solr/Collection_shard94_replica1/|http://2.2.216:8985/solr/Collection_shard94_replica4/|http://1.1.1.16:8985/solr/Collection_shard94_replica2/":{ "numFound":242995, "maxScore":null, "shardAddress":"http://1.1.1.16:8985/solr/Collection_shard94_replica2";, "time":21}, "http://1.1.1.17:8985/solr/Collection_shard98_replica3/|http://2.2.216:8985/solr/Collection_shard98_replica4/|http://2.2.213:8985/solr/Collection_shard98_replica1/|http://1.1.1.16:8985/solr/Collection_shard98_replica2/":{ "numFound":243837, "maxScore":null, "shardAddress":"http://1.1.1.16:8985/solr/Collection_shard98_replica2";, "time":18}, "http://2.2.213:8985/solr/Collection_shard28_replica1/|http://1.1.1.16:8985/solr/Collection_shard28_replica2/|http://1.1.1.17:8985/solr/Collection_shard28_replica3/|http://2.2.216:8985/solr/Collection_shard28_replica4/":{ "numFound":242576, "maxScore":null, "shardAddress":"http://1.1.1.17:8985/solr/Collection_shard28_replica3";, "time":42}, "http://2.2.213:8985/solr/Collection_shard34_replica1/|http://1.1.1.16:8985/solr/Collection_shard34_replica2/|http://1.1.1.17:8985/solr/Collection_shard34_replica3/|http://2.2.216:8985/solr/Collection_shard34_replica4/":{ "numFound":243883, "maxScore":null, "shardAddress":"http://1.1.1.17:8985/solr/Collection_shard34_replica3";, "time":41}, "http://2.2.215:8985/solr/Collection_shard61_replica4/|http://1.1.1.18:8985/solr/Collection_shard61_replica2/|http://1.1.1.15:8985/solr/Collection_shard61_replica3/|http://2.2.214:8985/solr/Collection_shard61_replica1/":{ "numFound":244110, "maxScore":null, "shardAddress":"http://2.2.215:8985/solr/Collection_shard61_replica4";, "time":29}, "http://1.1.1.18:8985/solr/Collection_shard17_replica2/|http://2.2.214:8985/solr/Collection_shard17_replica1/|http://1.1.1.15:8985/solr/Collection_shard17_replica3/|http://2.2.215:8985/solr/Collection_shard17_replica4/":{ "numFound":243370, "maxScore":null, "shardAddress":"http://1.1.1.15:8985/solr/Collection_shard17_replica3";, "time":42}, "http://2.2.213:8985/solr/Collection_shard22_replica1/|http://1.1.1.17:8985/solr/Collection_shard22_replica3/|http://1.1.1.16:8985/solr/Collection_shard22_replica2/|http://2.2.216:8985/solr/Collection_shard22_replica4/":{ "numFound":242412, "maxScore":null, "shardAddress":"http://2.2.213:8985/solr/Collection_shard22_replica1";, "time":29}, "http://2.2.213:8985/solr/Collection_shard20_replica1/|http://1.1.1.16:8985/solr/Collection_shard20_replica2/|http://1.1.1.17:8985/solr/Collection_shard20_replica3/|http://2.2.216:8985/solr/Collection_shard20_replica4/":{ "numFound":243885, "maxScore":null, "shardAddress":"http://2.2.213:8985/solr/Collection_shard20_replica1";, "time":30}, "http://2.2.215:8985/solr/Collection_shard47_replica4/|http://1.1.1.18:8985/solr/Collection_shard47_replica2/|http://1.1.1.15:8985/solr/Collection_shard47_replica3/|http://2.2.214:8985/solr/Collection_shard47_replica1/":{ "numFound":243191, "maxScore":null, "shardAddress":"http://1.1.1.18:8985/solr/Collection_shard47_replica2";, "time":22}, "http://1.1.1.18:8985/solr/Collection_shard19_replica2/|http://1.1.1.15:8985/solr/Collection_shard19_replica3/|http://2.2.214:8985/solr/Collection_shard19_replica1/|http://2.2.215:8985/solr/Collection_shard19_replica4/":{ "numFound":242392, "maxScore":null, "shardAddress":"http://1.1.1.18:8985/solr/Collection_shard19_replica2";, "time":44}, "http://2.2.215:8985/solr/Collection_shard97_replica4/|http://1.1.1.15:8985/solr/Collection_shard97_replica3/|http://1.1.1.18:8985/solr/Collection_shard97_replica2/|http://2.2.214:8985/solr/Collection_shard97_replica1/":{ "numFound":243004, "maxScore":null, "shardAddress":"http://2.2.214:8985/solr/Collection_shard97_replica1";, "time":23}, "http://1.1.1.17:8985/solr/Collection_shard2_replica3/|http://2.2.213:8985/solr/Collection_shard2_replica1/|http://1.1.1.16:8985/solr/Collection_shard2_replica2/|http://2.2.216:8985/solr/Collection_shard2_replica4/":{ "numFound":242394, "maxScore":null, "shardAddress":"http://1.1.1.16:8985/solr/Collection_shard2_replica2";, "time":18}, "http://2.2.215:8985/solr/Collection_shard67_replica4/|http://1.1.1.18:8985/solr/Collection_shard67_replica2/|http://1.1.1.15:8985/solr/Collection_shard67_replica3/|http://2.2.214:8985/solr/Collection_shard67_replica1/":{ "numFound":242803, "maxScore":null, "shardAddress":"http://1.1.1.18:8985/solr/Collection_shard67_replica2";, "time":45}, "http://1.1.1.18:8985/solr/Collection_shard13_replica2/|http://1.1.1.15:8985/solr/Collection_shard13_replica3/|http://2.2.214:8985/solr/Collection_shard13_replica1/|http://2.2.215:8985/solr/Collection_shard13_replica4/":{ "numFound":242987, "maxScore":null, "shardAddress":"http://1.1.1.18:8985/solr/Collection_shard13_replica2";, "time":34}, "http://1.1.1.15:8985/solr/Collection_shard93_replica3/|http://1.1.1.18:8985/solr/Collection_shard93_replica2/|http://2.2.215:8985/solr/Collection_shard93_replica4/|http://2.2.214:8985/solr/Collection_shard93_replica1/":{ "numFound":242626, "maxScore":null, "shardAddress":"http://1.1.1.18:8985/solr/Collection_shard93_replica2";, "time":30}, "http://1.1.1.17:8985/solr/Collection_shard66_replica3/|http://2.2.216:8985/solr/Collection_shard66_replica4/|http://1.1.1.16:8985/solr/Collection_shard66_replica2/|http://2.2.213:8985/solr/Collection_shard66_replica1/":{ "numFound":242834, "maxScore":null, "shardAddress":"http://2.2.213:8985/solr/Collection_shard66_replica1";, "time":47}, "http://2.2.213:8985/solr/Collection_shard18_replica1/|http://1.1.1.17:8985/solr/Collection_shard18_replica3/|http://1.1.1.16:8985/solr/Collection_shard18_replica2/|http://2.2.216:8985/solr/Collection_shard18_replica4/":{ "numFound":242419, "maxScore":null, "shardAddress":"http://1.1.1.17:8985/solr/Collection_shard18_replica3";, "time":47}, "http://2.2.215:8985/solr/Collection_shard49_replica4/|http://1.1.1.18:8985/solr/Collection_shard49_replica2/|http://1.1.1.15:8985/solr/Collection_shard49_replica3/|http://2.2.214:8985/solr/Collection_shard49_replica1/":{ "numFound":242427, "maxScore":null, "shardAddress":"http://1.1.1.18:8985/solr/Collection_shard49_replica2";, "time":31}, "http://1.1.1.18:8985/solr/Collection_shard1_replica2/|http://1.1.1.15:8985/solr/Collection_shard1_replica3/|http://2.2.214:8985/solr/Collection_shard1_replica1/|http://2.2.215:8985/solr/Collection_shard1_replica4/":{ "numFound":242058, "maxScore":null, "shardAddress":"http://2.2.215:8985/solr/Collection_shard1_replica4";, "time":22}, "http://1.1.1.18:8985/solr/Collection_shard57_replica2/|http://2.2.215:8985/solr/Collection_shard57_replica4/|http://1.1.1.15:8985/solr/Collection_shard57_replica3/|http://2.2.214:8985/solr/Collection_shard57_replica1/":{ "numFound":242880, "maxScore":null, "shardAddress":"http://2.2.215:8985/solr/Collection_shard57_replica4";, "time":36}, "http://1.1.1.18:8985/solr/Collection_shard99_replica2/|http://2.2.214:8985/solr/Collection_shard99_replica1/|http://1.1.1.15:8985/solr/Collection_shard99_replica3/|http://2.2.215:8985/solr/Collection_shard99_replica4/":{ "numFound":242298, "maxScore":null, "shardAddress":"http://1.1.1.15:8985/solr/Collection_shard99_replica3";, "time":29}, "http://1.1.1.18:8985/solr/Collection_shard11_replica2/|http://2.2.214:8985/solr/Collection_shard11_replica1/|http://1.1.1.15:8985/solr/Collection_shard11_replica3/|http://2.2.215:8985/solr/Collection_shard11_replica4/":{ "numFound":241778, "maxScore":null, "shardAddress":"http://1.1.1.15:8985/solr/Collection_shard11_replica3";, "time":26}, "http://2.2.213:8985/solr/Collection_shard92_replica1/|http://1.1.1.17:8985/solr/Collection_shard92_replica3/|http://1.1.1.16:8985/solr/Collection_shard92_replica2/|http://2.2.216:8985/solr/Collection_shard92_replica4/":{ "numFound":243558, "maxScore":null, "shardAddress":"http://2.2.216:8985/solr/Collection_shard92_replica4";, "time":26}, "http://1.1.1.17:8985/solr/Collection_shard90_replica3/|http://2.2.213:8985/solr/Collection_shard90_replica1/|http://2.2.216:8985/solr/Collection_shard90_replica4/|http://1.1.1.16:8985/solr/Collection_shard90_replica2/":{ "numFound":243305, "maxScore":null, "shardAddress":"http://2.2.216:8985/solr/Collection_shard90_replica4";, "time":27}, "http://1.1.1.18:8985/solr/Collection_shard95_replica2/|http://2.2.215:8985/solr/Collection_shard95_replica4/|http://1.1.1.15:8985/solr/Collection_shard95_replica3/|http://2.2.214:8985/solr/Collection_shard95_replica1/":{ "numFound":243205, "maxScore":null, "shardAddress":"http://2.2.215:8985/solr/Collection_shard95_replica4";, "time":37}, "http://2.2.213:8985/solr/Collection_shard4_replica1/|http://1.1.1.16:8985/solr/Collection_shard4_replica2/|http://1.1.1.17:8985/solr/Collection_shard4_replica3/|http://2.2.216:8985/solr/Collection_shard4_replica4/":{ "numFound":242872, "maxScore":null, "shardAddress":"http://2.2.213:8985/solr/Collection_shard4_replica1";, "time":31}, "http://1.1.1.15:8985/solr/Collection_shard51_replica3/|http://2.2.215:8985/solr/Collection_shard51_replica4/|http://1.1.1.18:8985/solr/Collection_shard51_replica2/|http://2.2.214:8985/solr/Collection_shard51_replica1/":{ "numFound":243346, "maxScore":null, "shardAddress":"http://2.2.215:8985/solr/Collection_shard51_replica4";, "time":31}, "http://2.2.213:8985/solr/Collection_shard10_replica1/|http://1.1.1.16:8985/solr/Collection_shard10_replica2/|http://1.1.1.17:8985/solr/Collection_shard10_replica3/|http://2.2.216:8985/solr/Collection_shard10_replica4/":{ "numFound":242891, "maxScore":null, "shardAddress":"http://1.1.1.16:8985/solr/Collection_shard10_replica2";, "time":32}, "http://1.1.1.18:8985/solr/Collection_shard91_replica2/|http://2.2.215:8985/solr/Collection_shard91_replica4/|http://2.2.214:8985/solr/Collection_shard91_replica1/|http://1.1.1.15:8985/solr/Collection_shard91_replica3/":{ "numFound":243100, "maxScore":null, "shardAddress":"http://1.1.1.18:8985/solr/Collection_shard91_replica2";, "time":34}, "http://1.1.1.17:8985/solr/Collection_shard46_replica3/|http://2.2.213:8985/solr/Collection_shard46_replica1/|http://2.2.216:8985/solr/Collection_shard46_replica4/|http://1.1.1.16:8985/solr/Collection_shard46_replica2/":{ "numFound":243261, "maxScore":null, "shardAddress":"http://2.2.213:8985/solr/Collection_shard46_replica1";, "time":42}, "http://2.2.213:8985/solr/Collection_shard8_replica1/|http://1.1.1.16:8985/solr/Collection_shard8_replica2/|http://1.1.1.17:8985/solr/Collection_shard8_replica3/|http://2.2.216:8985/solr/Collection_shard8_replica4/":{ "numFound":243243, "maxScore":null, "shardAddress":"http://2.2.213:8985/solr/Collection_shard8_replica1";, "time":34}, "http://2.2.213:8985/solr/Collection_shard36_replica1/|http://1.1.1.17:8985/solr/Collection_shard36_replica3/|http://1.1.1.16:8985/solr/Collection_shard36_replica2/|http://2.2.216:8985/solr/Collection_shard36_replica4/":{ "numFound":242983, "maxScore":null, "shardAddress":"http://1.1.1.16:8985/solr/Collection_shard36_replica2";, "time":20}, "http://1.1.1.18:8985/solr/Collection_shard87_replica2/|http://1.1.1.15:8985/solr/Collection_shard87_replica3/|http://2.2.215:8985/solr/Collection_shard87_replica4/|http://2.2.214:8985/solr/Collection_shard87_replica1/":{ "numFound":243257, "maxScore":null, "shardAddress":"http://1.1.1.15:8985/solr/Collection_shard87_replica3";, "time":21}, "http://1.1.1.15:8985/solr/Collection_shard65_replica3/|http://2.2.215:8985/solr/Collection_shard65_replica4/|http://2.2.214:8985/solr/Collection_shard65_replica1/|http://1.1.1.18:8985/solr/Collection_shard65_replica2/":{ "numFound":242687, "maxScore":null, "shardAddress":"http://1.1.1.18:8985/solr/Collection_shard65_replica2";, "time":62}, "http://1.1.1.17:8985/solr/Collection_shard52_replica3/|http://2.2.213:8985/solr/Collection_shard52_replica1/|http://2.2.216:8985/solr/Collection_shard52_replica4/|http://1.1.1.16:8985/solr/Collection_shard52_replica2/":{ "numFound":243207, "maxScore":null, "shardAddress":"http://2.2.216:8985/solr/Collection_shard52_replica4";, "time":36}, "http://1.1.1.17:8985/solr/Collection_shard54_replica3/|http://2.2.213:8985/solr/Collection_shard54_replica1/|http://1.1.1.16:8985/solr/Collection_shard54_replica2/|http://2.2.216:8985/solr/Collection_shard54_replica4/":{ "numFound":243025, "maxScore":null, "shardAddress":"http://2.2.213:8985/solr/Collection_shard54_replica1";, "time":25}, "http://1.1.1.18:8985/solr/Collection_shard55_replica2/|http://2.2.215:8985/solr/Collection_shard55_replica4/|http://1.1.1.15:8985/solr/Collection_shard55_replica3/|http://2.2.214:8985/solr/Collection_shard55_replica1/":{ "numFound":242210, "maxScore":null, "shardAddress":"http://2.2.214:8985/solr/Collection_shard55_replica1";, "time":26}, "http://1.1.1.18:8985/solr/Collection_shard9_replica2/|http://2.2.214:8985/solr/Collection_shard9_replica1/|http://1.1.1.15:8985/solr/Collection_shard9_replica3/|http://2.2.215:8985/solr/Collection_shard9_replica4/":{ "numFound":242553, "maxScore":null, "shardAddress":"http://1.1.1.18:8985/solr/Collection_shard9_replica2";, "time":26}, "http://2.2.213:8985/solr/Collection_shard6_replica1/|http://1.1.1.16:8985/solr/Collection_shard6_replica2/|http://1.1.1.17:8985/solr/Collection_shard6_replica3/|http://2.2.216:8985/solr/Collection_shard6_replica4/":{ "numFound":243807, "maxScore":null, "shardAddress":"http://2.2.213:8985/solr/Collection_shard6_replica1";, "time":25}, "http://1.1.1.15:8985/solr/Collection_shard81_replica3/|http://1.1.1.18:8985/solr/Collection_shard81_replica2/|http://2.2.215:8985/solr/Collection_shard81_replica4/|http://2.2.214:8985/solr/Collection_shard81_replica1/":{ "numFound":242842, "maxScore":null, "shardAddress":"http://2.2.215:8985/solr/Collection_shard81_replica4";, "time":25}, "http://2.2.216:8985/solr/Collection_shard42_replica4/|http://2.2.213:8985/solr/Collection_shard42_replica1/|http://1.1.1.17:8985/solr/Collection_shard42_replica3/|http://1.1.1.16:8985/solr/Collection_shard42_replica2/":{ "numFound":243083, "maxScore":null, "shardAddress":"http://2.2.216:8985/solr/Collection_shard42_replica4";, "time":26}, "http://1.1.1.18:8985/solr/Collection_shard7_replica2/|http://1.1.1.15:8985/solr/Collection_shard7_replica3/|http://2.2.214:8985/solr/Collection_shard7_replica1/|http://2.2.215:8985/solr/Collection_shard7_replica4/":{ "numFound":242131, "maxScore":null, "shardAddress":"http://2.2.215:8985/solr/Collection_shard7_replica4";, "time":28}, "http://1.1.1.18:8985/solr/Collection_shard5_replica2/|http://2.2.214:8985/solr/Collection_shard5_replica1/|http://1.1.1.15:8985/solr/Collection_shard5_replica3/|http://2.2.215:8985/solr/Collection_shard5_replica4/":{ "numFound":242824, "maxScore":null, "shardAddress":"http://1.1.1.18:8985/solr/Collection_shard5_replica2";, "time":44}, "http://2.2.214:8985/solr/Collection_shard37_replica1/|http://1.1.1.18:8985/solr/Collection_shard37_replica2/|http://1.1.1.15:8985/solr/Collection_shard37_replica3/|http://2.2.215:8985/solr/Collection_shard37_replica4/":{ "numFound":243118, "maxScore":null, "shardAddress":"http://1.1.1.18:8985/solr/Collection_shard37_replica2";, "time":29}, "http://2.2.215:8985/solr/Collection_shard43_replica4/|http://1.1.1.18:8985/solr/Collection_shard43_replica2/|http://1.1.1.15:8985/solr/Collection_shard43_replica3/|http://2.2.214:8985/solr/Collection_shard43_replica1/":{ "numFound":242412, "maxScore":null, "shardAddress":"http://1.1.1.15:8985/solr/Collection_shard43_replica3";, "time":25}, "http://1.1.1.17:8985/solr/Collection_shard86_replica3/|http://2.2.213:8985/solr/Collection_shard86_replica1/|http://2.2.216:8985/solr/Collection_shard86_replica4/|http://1.1.1.16:8985/solr/Collection_shard86_replica2/":{ "numFound":242448, "maxScore":null, "shardAddress":"http://2.2.216:8985/solr/Collection_shard86_replica4";, "time":30}, "http://1.1.1.18:8985/solr/Collection_shard85_replica2/|http://1.1.1.15:8985/solr/Collection_shard85_replica3/|http://2.2.215:8985/solr/Collection_shard85_replica4/|http://2.2.214:8985/solr/Collection_shard85_replica1/":{ "numFound":242110, "maxScore":null, "shardAddress":"http://1.1.1.18:8985/solr/Collection_shard85_replica2";, "time":31}, "http://1.1.1.18:8985/solr/Collection_shard39_replica2/|http://1.1.1.15:8985/solr/Collection_shard39_replica3/|http://2.2.214:8985/solr/Collection_shard39_replica1/|http://2.2.215:8985/solr/Collection_shard39_replica4/":{ "numFound":243459, "maxScore":null, "shardAddress":"http://1.1.1.18:8985/solr/Collection_shard39_replica2";, "time":32}, "http://2.2.213:8985/solr/Collection_shard40_replica1/|http://1.1.1.16:8985/solr/Collection_shard40_replica2/|http://1.1.1.17:8985/solr/Collection_shard40_replica3/|http://2.2.216:8985/solr/Collection_shard40_replica4/":{ "numFound":243297, "maxScore":null, "shardAddress":"http://2.2.216:8985/solr/Collection_shard40_replica4";, "time":33}, "http://1.1.1.18:8985/solr/Collection_shard53_replica2/|http://2.2.215:8985/solr/Collection_shard53_replica4/|http://1.1.1.15:8985/solr/Collection_shard53_replica3/|http://2.2.214:8985/solr/Collection_shard53_replica1/":{ "numFound":242564, "maxScore":null, "shardAddress":"http://2.2.215:8985/solr/Collection_shard53_replica4";, "time":35}, "http://1.1.1.18:8985/solr/Collection_shard89_replica2/|http://1.1.1.15:8985/solr/Collection_shard89_replica3/|http://2.2.214:8985/solr/Collection_shard89_replica1/|http://2.2.215:8985/solr/Collection_shard89_replica4/":{ "numFound":243331, "maxScore":null, "shardAddress":"http://2.2.215:8985/solr/Collection_shard89_replica4";, "time":34}, "http://2.2.214:8985/solr/Collection_shard35_replica1/|http://1.1.1.18:8985/solr/Collection_shard35_replica2/|http://1.1.1.15:8985/solr/Collection_shard35_replica3/|http://2.2.215:8985/solr/Collection_shard35_replica4/":{ "numFound":242939, "maxScore":null, "shardAddress":"http://1.1.1.18:8985/solr/Collection_shard35_replica2";, "time":39}, "http://1.1.1.17:8985/solr/Collection_shard44_replica3/|http://2.2.213:8985/solr/Collection_shard44_replica1/|http://2.2.216:8985/solr/Collection_shard44_replica4/|http://1.1.1.16:8985/solr/Collection_shard44_replica2/":{ "numFound":242072, "maxScore":null, "shardAddress":"http://1.1.1.16:8985/solr/Collection_shard44_replica2";, "time":20}, "http://2.2.213:8985/solr/Collection_shard30_replica1/|http://1.1.1.17:8985/solr/Collection_shard30_replica3/|http://1.1.1.16:8985/solr/Collection_shard30_replica2/|http://2.2.216:8985/solr/Collection_shard30_replica4/":{ "numFound":242416, "maxScore":null, "shardAddress":"http://1.1.1.17:8985/solr/Collection_shard30_replica3";, "time":84}, "http://2.2.215:8985/solr/Collection_shard41_replica4/|http://1.1.1.18:8985/solr/Collection_shard41_replica2/|http://2.2.214:8985/solr/Collection_shard41_replica1/|http://1.1.1.15:8985/solr/Collection_shard41_replica3/":{ "numFound":243127, "maxScore":null, "shardAddress":"http://1.1.1.18:8985/solr/Collection_shard41_replica2";, "time":39}, "http://2.2.215:8985/solr/Collection_shard45_replica4/|http://1.1.1.18:8985/solr/Collection_shard45_replica2/|http://1.1.1.15:8985/solr/Collection_shard45_replica3/|http://2.2.214:8985/solr/Collection_shard45_replica1/":{ "numFound":242839, "maxScore":null, "shardAddress":"http://1.1.1.18:8985/solr/Collection_shard45_replica2";, "time":43}, "http://2.2.213:8985/solr/Collection_shard14_replica1/|http://1.1.1.16:8985/solr/Collection_shard14_replica2/|http://1.1.1.17:8985/solr/Collection_shard14_replica3/|http://2.2.216:8985/solr/Collection_shard14_replica4/":{ "numFound":242888, "maxScore":null, "shardAddress":"http://1.1.1.17:8985/solr/Collection_shard14_replica3";, "time":74}, "http://2.2.216:8985/solr/Collection_shard62_replica4/|http://1.1.1.17:8985/solr/Collection_shard62_replica3/|http://2.2.213:8985/solr/Collection_shard62_replica1/|http://1.1.1.16:8985/solr/Collection_shard62_replica2/":{ "numFound":243208, "maxScore":null, "shardAddress":"http://1.1.1.17:8985/solr/Collection_shard62_replica3";, "time":76}, "http://1.1.1.17:8985/solr/Collection_shard50_replica3/|http://2.2.216:8985/solr/Collection_shard50_replica4/|http://2.2.213:8985/solr/Collection_shard50_replica1/|http://1.1.1.16:8985/solr/Collection_shard50_replica2/":{ "numFound":242274, "maxScore":null, "shardAddress":"http://1.1.1.17:8985/solr/Collection_shard50_replica3";, "time":58}, "http://1.1.1.17:8985/solr/Collection_shard56_replica3/|http://2.2.216:8985/solr/Collection_shard56_replica4/|http://2.2.213:8985/solr/Collection_shard56_replica1/|http://1.1.1.16:8985/solr/Collection_shard56_replica2/":{ "numFound":242606, "maxScore":null, "shardAddress":"http://1.1.1.17:8985/solr/Collection_shard56_replica3";, "time":59}, "http://2.2.213:8985/solr/Collection_shard38_replica1/|http://1.1.1.16:8985/solr/Collection_shard38_replica2/|http://2.2.216:8985/solr/Collection_shard38_replica4/|http://1.1.1.17:8985/solr/Collection_shard38_replica3/":{ "numFound":242346, "maxScore":null, "shardAddress":"http://1.1.1.17:8985/solr/Collection_shard38_replica3";, "time":61}, "http://1.1.1.17:8985/solr/Collection_shard88_replica3/|http://2.2.216:8985/solr/Collection_shard88_replica4/|http://2.2.213:8985/solr/Collection_shard88_replica1/|http://1.1.1.16:8985/solr/Collection_shard88_replica2/":{ "numFound":242530, "maxScore":null, "shardAddress":"http://1.1.1.17:8985/solr/Collection_shard88_replica3";, "time":61}, "http://1.1.1.17:8985/solr/Collection_shard80_replica3/|http://2.2.216:8985/solr/Collection_shard80_replica4/|http://2.2.213:8985/solr/Collection_shard80_replica1/|http://1.1.1.16:8985/solr/Collection_shard80_replica2/":{ "numFound":241641, "maxScore":null, "shardAddress":"http://1.1.1.17:8985/solr/Collection_shard80_replica3";, "time":61}, "http://1.1.1.17:8985/solr/Collection_shard82_replica3/|http://2.2.213:8985/solr/Collection_shard82_replica1/|http://2.2.216:8985/solr/Collection_shard82_replica4/|http://1.1.1.16:8985/solr/Collection_shard82_replica2/":{ "numFound":243561, "maxScore":null, "shardAddress":"http://1.1.1.17:8985/solr/Collection_shard82_replica3";, "time":65}, "http://1.1.1.17:8985/solr/Collection_shard84_replica3/|http://2.2.213:8985/solr/Collection_shard84_replica1/|http://2.2.216:8985/solr/Collection_shard84_replica4/|http://1.1.1.16:8985/solr/Collection_shard84_replica2/":{ "numFound":242343, "maxScore":null, "shardAddress":"http://1.1.1.17:8985/solr/Collection_shard84_replica3";, "time":68}, "http://2.2.213:8985/solr/Collection_shard16_replica1/|http://1.1.1.16:8985/solr/Collection_shard16_replica2/|http://1.1.1.17:8985/solr/Collection_shard16_replica3/|http://2.2.216:8985/solr/Collection_shard16_replica4/":{ "numFound":242899, "maxScore":null, "shardAddress":"http://1.1.1.17:8985/solr/Collection_shard16_replica3";, "time":134}, "http://2.2.215:8985/solr/Collection_shard63_replica4/|http://1.1.1.18:8985/solr/Collection_shard63_replica2/|http://1.1.1.15:8985/solr/Collection_shard63_replica3/|http://2.2.214:8985/solr/Collection_shard63_replica1/":{ "numFound":242351, "maxScore":null, "shardAddress":"http://1.1.1.18:8985/solr/Collection_shard63_replica2";, "time":183}, "http://2.2.213:8985/solr/Collection_shard12_replica1/|http://1.1.1.17:8985/solr/Collection_shard12_replica3/|http://1.1.1.16:8985/solr/Collection_shard12_replica2/|http://2.2.216:8985/solr/Collection_shard12_replica4/":{ "numFound":243015, "maxScore":null, "shardAddress":"http://1.1.1.16:8985/solr/Collection_shard12_replica2";, "time":345}, "http://2.2.216:8985/solr/Collection_shard100_replica4/|http://2.2.213:8985/solr/Collection_shard100_replica1/|http://1.1.1.17:8985/solr/Collection_shard100_replica3/|http://1.1.1.16:8985/solr/Collection_shard100_replica2/":{ "numFound":242767, "maxScore":null, "shardAddress":"http://2.2.216:8985/solr/Collection_shard100_replica4";, "time":446}, "http://2.2.216:8985/solr/Collection_shard96_replica4/|http://2.2.213:8985/solr/Collection_shard96_replica1/|http://1.1.1.17:8985/solr/Collection_shard96_replica3/|http://1.1.1.16:8985/solr/Collection_shard96_replica2/":{ "numFound":243190, "maxScore":null, "shardAddress":"http://2.2.213:8985/solr/Collection_shard96_replica1";, "time":1274}, "http://1.1.1.17:8985/solr/Collection_shard58_replica3/|http://2.2.216:8985/solr/Collection_shard58_replica4/|http://2.2.213:8985/solr/Collection_shard58_replica1/|http://1.1.1.16:8985/solr/Collection_shard58_replica2/":{ "numFound":242296, "maxScore":null, "shardAddress":"http://2.2.213:8985/solr/Collection_shard58_replica1";, "time":1312}, "http://2.2.214:8985/solr/Collection_shard3_replica1/|http://1.1.1.18:8985/solr/Collection_shard3_replica2/|http://1.1.1.15:8985/solr/Collection_shard3_replica3/|http://2.2.215:8985/solr/Collection_shard3_replica4/":{ "numFound":243326, "maxScore":null, "shardAddress":"http://1.1.1.15:8985/solr/Collection_shard3_replica3";, "time":1343}, "http://1.1.1.17:8985/solr/Collection_shard48_replica3/|http://2.2.216:8985/solr/Collection_shard48_replica4/|http://2.2.213:8985/solr/Collection_shard48_replica1/|http://1.1.1.16:8985/solr/Collection_shard48_replica2/":{ "numFound":243498, "maxScore":null, "shardAddress":"http://2.2.213:8985/solr/Collection_shard48_replica1";, "time":1382}, "http://1.1.1.18:8985/solr/Collection_shard27_replica2/|http://1.1.1.15:8985/solr/Collection_shard27_replica3/|http://2.2.214:8985/solr/Collection_shard27_replica1/|http://2.2.215:8985/solr/Collection_shard27_replica4/":{ "numFound":242363, "maxScore":null, "shardAddress":"http://1.1.1.15:8985/solr/Collection_shard27_replica3";, "time":1504}, "http://2.2.215:8985/solr/Collection_shard83_replica4/|http://1.1.1.18:8985/solr/Collection_shard83_replica2/|http://1.1.1.15:8985/solr/Collection_shard83_replica3/|http://2.2.214:8985/solr/Collection_shard83_replica1/":{ "numFound":242315, "maxScore":null, "shardAddress":"http://1.1.1.15:8985/solr/Collection_shard83_replica3";, "time":1475}, "http://2.2.215:8985/solr/Collection_shard79_replica4/|http://1.1.1.18:8985/solr/Collection_shard79_replica2/|http://1.1.1.15:8985/solr/Collection_shard79_replica3/|http://2.2.214:8985/solr/Collection_shard79_replica1/":{ "numFound":241799, "maxScore":null, "shardAddress":"http://1.1.1.18:8985/solr/Collection_shard79_replica2";, "time":50046}}, "response":{"numFound":24286184,"start":0,"docs":[] }, "nextCursorMark":"*"}
