Distributed Search Test
Hello,
I have been performing some simple distributed search tests and don't
understand why distributed search seems to work in some circumstances but
not others.
In my setup I have compiled the example server using the solr trunk
downloaded on Aug 22nd. I am running a sample server instance on 2 separate
hosts (localhost and "fred"). I've added a portion of the sample docs
[a-n]*.xml to the local host solr server, and added the other portion,
[m-z]*.xml sample docs to host fred.
Assuming that I have setup things correctly, I would expect to receive a see
non zero length SolrDocumentList for any distributed search that matches
syntax in the example docs.
Specifically when I test the contents of each server separately ( using the
included TestCase ) the tests pass. This confirms that each server has
different documents. However when I do the distributed tests, it seems the
tests pass or fail based on the initial URL passed in the
createNewSolrServer(String URL). I realize a real junit should be self
contained, unlike this one.
junit test testDistrbutedSearch() passes, while testDistrbutedSearch2()
fails. Why?
My understanding is that each host should send a query to all shards and
collate the responses, and return them to the client. Is this true?
Ron
Here is my TestCase;
package org.apache.solr.client.solrj.ron;
import junit.framework.TestCase;
import org.apache.solr.client.solrj.SolrQuery;
import org.apache.solr.client.solrj.SolrServer;
import org.apache.solr.client.solrj.impl.CommonsHttpSolrServer;
import org.apache.solr.client.solrj.response.QueryResponse;
import org.apache.solr.client.solrj.response.SolrPingResponse;
import org.apache.solr.common.SolrDocumentList;
import org.apache.solr.common.params.ShardParams;
public class SolrExampleDistributedTest extends TestCase {
int port = 8983;
static final String context = "/solr";
static String SOLR_SHARD1 = "localhost:8983/solr";
static String SOLR_SHARD2 = "fred:8983/solr";
static String SOLR_SHARDS = SOLR_SHARD1 + "," + SOLR_SHARD2;
static String HTTP_PREFIX = "http://";;
static String SOLR_URL1 = HTTP_PREFIX + SOLR_SHARD1;
static String SOLR_URL2 = HTTP_PREFIX + SOLR_SHARD2;
static String QUERY1 = "Samsung";
static String QUERY2 = "solr";
@Override
public void setUp() throws Exception {
super.setUp();
}
public SolrExampleDistributedTest(String name) {
super(name);
}
@Override
public void tearDown() throws Exception {
super.tearDown();
}
protected SolrServer createNewSolrServer(String url) {
try {
CommonsHttpSolrServer s = new CommonsHttpSolrServer(url);
s.setConnectionTimeout(100); // 1/10th sec
s.setDefaultMaxConnectionsPerHost(100);
s.setMaxTotalConnections(100);
return s;
} catch (Exception ex) {
throw new RuntimeException(ex);
}
}
public void testLocalhost() {
try {
SolrServer server = createNewSolrServer(SOLR_URL1);
SolrQuery query = new SolrQuery();
query.setQuery(QUERY1);
QueryResponse qr = server.query(query);
SolrDocumentList sdl = qr.getResults();
assertTrue(sdl.getNumFound() > 0);
query = new SolrQuery();
query.setQuery(QUERY2);
qr = server.query(query);
sdl = qr.getResults();
assertTrue(sdl.getNumFound() == 0);
} catch (Exception ex) {
ex.printStackTrace();
fail();
}
}
public void testRemoteHost() {
try {
SolrServer server = createNewSolrServer(SOLR_URL2);
SolrQuery query = new SolrQuery();
query.setQuery(QUERY1);
QueryResponse qr = server.query(query);
SolrDocumentList sdl = qr.getResults();
assertTrue(sdl.getNumFound() == 0);
query = new SolrQuery();
query.setQuery(QUERY2);
qr = server.query(query);
sdl = qr.getResults();
assertTrue(sdl.getNumFound() > 0);
} catch (Exception ex) {
// expected
ex.printStackTrace();
fail();
}
}
public void testDistrbutedSearch() {
try {
SolrServer server = createNewSolrServer(SOLR_URL1);
SolrQuery query = new SolrQuery();
query.setQuery(QUERY1);
query.setParam(ShardParams.SHARDS, SOLR_SHARDS);
QueryResponse qr = server.query(query);
SolrDocumentList sdl = qr.getResults();
assertTrue(sdl.getNumFound() > 0);
SolrQuery query2 = new SolrQuery();
query2.setQuery(QUERY2);
query2.setParam(ShardParams.SHARDS, SOLR_SHARDS);
QueryResponse qr2 = server.query(query);
SolrDocumentList sdl2 = qr2.getResults();
assertTrue(sdl.getNumFound() >
Re: Distributed Search Test
Yonik,
Thanks for your reply. I'm not sure if I understand completely. Do you
mean that each solr server should be given a different shard list and not a
list containing all shards?
So in my case:
1) host fred should be given a shard list containing only locahost,
2) localhost should be given a shard list of fred
I'll give it a try.
Thanks again
Ron
On Wed, Aug 27, 2008 at 12:21 PM, Yonik Seeley <[EMAIL PROTECTED]> wrote:
> It fails because you are using "localhost" as part of a shard name.
> When you send the request to "fred" it uses the "fred" shard and the
> "localhost" shard (which is the same as fred!)
>
> -Yonik
>
> On Wed, Aug 27, 2008 at 12:07 PM, Ronald Aubin <[EMAIL PROTECTED]>
> wrote:
> > Hello,
> >I have been performing some simple distributed search tests and don't
> > understand why distributed search seems to work in some circumstances but
> > not others.
> >
> > In my setup I have compiled the example server using the solr trunk
> > downloaded on Aug 22nd. I am running a sample server instance on 2
> separate
> > hosts (localhost and "fred"). I've added a portion of the sample docs
> > [a-n]*.xml to the local host solr server, and added the other portion,
> > [m-z]*.xml sample docs to host fred.
> >
> > Assuming that I have setup things correctly, I would expect to receive a
> see
> > non zero length SolrDocumentList for any distributed search that matches
> > syntax in the example docs.
> >
> > Specifically when I test the contents of each server separately ( using
> the
> > included TestCase ) the tests pass. This confirms that each server has
> > different documents. However when I do the distributed tests, it seems
> the
> > tests pass or fail based on the initial URL passed in the
> > createNewSolrServer(String URL). I realize a real junit should be self
> > contained, unlike this one.
> >
> > junit test testDistrbutedSearch() passes, while testDistrbutedSearch2()
> > fails. Why?
> >
> > My understanding is that each host should send a query to all shards and
> > collate the responses, and return them to the client. Is this true?
> >
> > Ron
> >
> >
> > Here is my TestCase;
> >
> > package org.apache.solr.client.solrj.ron;
> >
> > import junit.framework.TestCase;
> >
> > import org.apache.solr.client.solrj.SolrQuery;
> > import org.apache.solr.client.solrj.SolrServer;
> > import org.apache.solr.client.solrj.impl.CommonsHttpSolrServer;
> > import org.apache.solr.client.solrj.response.QueryResponse;
> > import org.apache.solr.client.solrj.response.SolrPingResponse;
> > import org.apache.solr.common.SolrDocumentList;
> > import org.apache.solr.common.params.ShardParams;
> >
> > public class SolrExampleDistributedTest extends TestCase {
> >
> >int port = 8983;
> >static final String context = "/solr";
> >
> >static String SOLR_SHARD1 = "localhost:8983/solr";
> >static String SOLR_SHARD2 = "fred:8983/solr";
> >static String SOLR_SHARDS = SOLR_SHARD1 + "," + SOLR_SHARD2;
> >static String HTTP_PREFIX = "http://";;
> >static String SOLR_URL1 = HTTP_PREFIX + SOLR_SHARD1;
> >static String SOLR_URL2 = HTTP_PREFIX + SOLR_SHARD2;
> >static String QUERY1 = "Samsung";
> >static String QUERY2 = "solr";
> >
> >@Override
> >public void setUp() throws Exception {
> >super.setUp();
> >
> >}
> >
> >public SolrExampleDistributedTest(String name) {
> >super(name);
> >}
> >
> >@Override
> >public void tearDown() throws Exception {
> >super.tearDown();
> >}
> >
> >protected SolrServer createNewSolrServer(String url) {
> >try {
> >
> >CommonsHttpSolrServer s = new CommonsHttpSolrServer(url);
> >s.setConnectionTimeout(100); // 1/10th sec
> >s.setDefaultMaxConnectionsPerHost(100);
> >s.setMaxTotalConnections(100);
> >return s;
> >} catch (Exception ex) {
> >throw new RuntimeException(ex);
> >}
> >}
> >
> >public void testLocalhost() {
> >try {
> >SolrServer server = createNewSolrServer(SOLR_URL1);
> >
> >SolrQuery query = new SolrQuery();
> >query.setQuery(QUERY1);
> >
Re: Distributed Search Test
Yonik,
I now perfectly understand. Thanks for your help. All my tests now
work.
Ron
On Wed, Aug 27, 2008 at 12:21 PM, Yonik Seeley <[EMAIL PROTECTED]> wrote:
> It fails because you are using "localhost" as part of a shard name.
> When you send the request to "fred" it uses the "fred" shard and the
> "localhost" shard (which is the same as fred!)
>
> -Yonik
>
> On Wed, Aug 27, 2008 at 12:07 PM, Ronald Aubin <[EMAIL PROTECTED]>
> wrote:
> > Hello,
> >I have been performing some simple distributed search tests and don't
> > understand why distributed search seems to work in some circumstances but
> > not others.
> >
> > In my setup I have compiled the example server using the solr trunk
> > downloaded on Aug 22nd. I am running a sample server instance on 2
> separate
> > hosts (localhost and "fred"). I've added a portion of the sample docs
> > [a-n]*.xml to the local host solr server, and added the other portion,
> > [m-z]*.xml sample docs to host fred.
> >
> > Assuming that I have setup things correctly, I would expect to receive a
> see
> > non zero length SolrDocumentList for any distributed search that matches
> > syntax in the example docs.
> >
> > Specifically when I test the contents of each server separately ( using
> the
> > included TestCase ) the tests pass. This confirms that each server has
> > different documents. However when I do the distributed tests, it seems
> the
> > tests pass or fail based on the initial URL passed in the
> > createNewSolrServer(String URL). I realize a real junit should be self
> > contained, unlike this one.
> >
> > junit test testDistrbutedSearch() passes, while testDistrbutedSearch2()
> > fails. Why?
> >
> > My understanding is that each host should send a query to all shards and
> > collate the responses, and return them to the client. Is this true?
> >
> > Ron
> >
> >
> > Here is my TestCase;
> >
> > package org.apache.solr.client.solrj.ron;
> >
> > import junit.framework.TestCase;
> >
> > import org.apache.solr.client.solrj.SolrQuery;
> > import org.apache.solr.client.solrj.SolrServer;
> > import org.apache.solr.client.solrj.impl.CommonsHttpSolrServer;
> > import org.apache.solr.client.solrj.response.QueryResponse;
> > import org.apache.solr.client.solrj.response.SolrPingResponse;
> > import org.apache.solr.common.SolrDocumentList;
> > import org.apache.solr.common.params.ShardParams;
> >
> > public class SolrExampleDistributedTest extends TestCase {
> >
> >int port = 8983;
> >static final String context = "/solr";
> >
> >static String SOLR_SHARD1 = "localhost:8983/solr";
> >static String SOLR_SHARD2 = "fred:8983/solr";
> >static String SOLR_SHARDS = SOLR_SHARD1 + "," + SOLR_SHARD2;
> >static String HTTP_PREFIX = "http://";;
> >static String SOLR_URL1 = HTTP_PREFIX + SOLR_SHARD1;
> >static String SOLR_URL2 = HTTP_PREFIX + SOLR_SHARD2;
> >static String QUERY1 = "Samsung";
> >static String QUERY2 = "solr";
> >
> >@Override
> >public void setUp() throws Exception {
> >super.setUp();
> >
> >}
> >
> >public SolrExampleDistributedTest(String name) {
> >super(name);
> >}
> >
> >@Override
> >public void tearDown() throws Exception {
> >super.tearDown();
> >}
> >
> >protected SolrServer createNewSolrServer(String url) {
> >try {
> >
> >CommonsHttpSolrServer s = new CommonsHttpSolrServer(url);
> >s.setConnectionTimeout(100); // 1/10th sec
> >s.setDefaultMaxConnectionsPerHost(100);
> >s.setMaxTotalConnections(100);
> >return s;
> >} catch (Exception ex) {
> >throw new RuntimeException(ex);
> >}
> >}
> >
> >public void testLocalhost() {
> >try {
> >SolrServer server = createNewSolrServer(SOLR_URL1);
> >
> >SolrQuery query = new SolrQuery();
> >query.setQuery(QUERY1);
> >QueryResponse qr = server.query(query);
> >SolrDocumentList sdl = qr.getResults();
> >assertTrue(sdl.getNumFound() > 0);
> >
> >query = new SolrQuery();
> >query.setQuery(QUERY2);
> >qr = server.query(query
