Index: src/test/org/apache/lucene/index/stresstests/SharedState.java =================================================================== --- src/test/org/apache/lucene/index/stresstests/SharedState.java (revision 0) +++ src/test/org/apache/lucene/index/stresstests/SharedState.java (revision 0) @@ -0,0 +1,91 @@ +package org.apache.lucene.index.stresstests; + +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import java.util.HashSet; +import java.util.concurrent.BlockingDeque; +import java.util.concurrent.LinkedBlockingDeque; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.concurrent.atomic.AtomicLong; + +/** + * State shared by several IndexWriterClientThread to verify expectations, + * as to which terms should be found in the index. + */ +public class SharedState { + + final BlockingDeque stringsInIndex = new LinkedBlockingDeque(); + final BlockingDeque stringsOutOfIndex = new LinkedBlockingDeque(); + private final AtomicLong indexWriterActionCount = new AtomicLong(); + private final AtomicLong searchingActionCount = new AtomicLong(); + private final AtomicInteger errors = new AtomicInteger(0); + private volatile boolean quit = false; + + public SharedState(int initialDictionarySize) { + HashSet strings = new HashSet(); + for (int i = 1; i <= initialDictionarySize; i++) { + strings.add(String.valueOf(i)); + } + stringsOutOfIndex.addAll(strings); + } + + public boolean needToQuit() { + return errors.get()!=0 || quit; + } + + public void incrementIndexWriterTaskCount(long delta) { + long count = indexWriterActionCount.addAndGet(delta); + if (count % 500 == 0) { + System.out.println("IndexWriter tasks done: " + count); + } + } + + public void incrementIndexSearchesCount(long delta) { + long count = searchingActionCount.addAndGet(delta); + if (count % 500 == 0) { + System.out.println("Search tasks done: " + count); + } + } + + public String getStringToAddToIndex() throws InterruptedException { + return stringsOutOfIndex.take(); + } + + public void quit() { + quit = true; + } + + public void addStringWrittenToIndex(String termToAdd) { + stringsInIndex.add(termToAdd); + } + + public void errorManage(String message) { + errors.incrementAndGet(); + System.err.println(message); + } + + public void errorManage(Exception e){ + errors.incrementAndGet(); + e.printStackTrace(); + } + + public int errorCount() { + return errors.get(); + } + +} \ No newline at end of file Property changes on: src/test/org/apache/lucene/index/stresstests/SharedState.java ___________________________________________________________________ Added: svn:keywords + Id Revision Added: svn:eol-style + LF Index: src/test/org/apache/lucene/index/stresstests/TestIndexWriterConcurrent.java =================================================================== --- src/test/org/apache/lucene/index/stresstests/TestIndexWriterConcurrent.java (revision 0) +++ src/test/org/apache/lucene/index/stresstests/TestIndexWriterConcurrent.java (revision 0) @@ -0,0 +1,107 @@ +package org.apache.lucene.index.stresstests; + +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import java.io.IOException; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.TimeUnit; + +import junit.framework.Assert; +import junit.framework.TestCase; + +import org.apache.lucene.analysis.Analyzer; +import org.apache.lucene.analysis.standard.StandardAnalyzer; +import org.apache.lucene.index.ConcurrentMergeScheduler; +import org.apache.lucene.index.IndexWriter; +import org.apache.lucene.index.MergeScheduler; +import org.apache.lucene.index.SerialMergeScheduler; +import org.apache.lucene.index.IndexWriter.MaxFieldLength; +import org.apache.lucene.store.Directory; +import org.apache.lucene.store.RAMDirectory; +import org.apache.lucene.util.Version; + +/** + * IndexWriterConcurrenTest is similar to org.apache.lucene.index.TestStressIndexing + * but in addition to running several writing threads and searching threads it asserts + * the correct terms are found in the index after write and commit, or not found when + * delete and commit. + */ +public class TestIndexWriterConcurrent extends TestCase { + + /** The number of seconds to run the stresstest */ + private static final int STRESS_TIME = 600; + + /** Concurrent Threads in tests */ + private static final int THREADS = 40; + + private static final Analyzer analyzer = new StandardAnalyzer(Version.LUCENE_CURRENT); + private static final MergeScheduler serialMergeScheduler = new SerialMergeScheduler(); + private static final MergeScheduler concurrentMergeScheduler = new ConcurrentMergeScheduler(); + + /** + * test using a RAMDirectory while using a SerialMergeScheduler + */ + public void testWriterOnRAMAndSerialMerge() throws IOException, InterruptedException { + Directory dir = new RAMDirectory(); + stressConcurrentIndexWriterUsage(dir, serialMergeScheduler); + dir.close(); + } + + /** + * test using a RAMDirectory while using a ConcurrentMergeScheduler + */ + public void testWriterOnRAMAndConcurrentMerge() throws IOException, InterruptedException { + Directory dir = new RAMDirectory(); + stressConcurrentIndexWriterUsage(dir, concurrentMergeScheduler); + dir.close(); + } + + /** + * Runs the stress-test on a Directory, using several threads + * and waiting for a timeout or for the first failure. + */ + public void stressConcurrentIndexWriterUsage(Directory dir, MergeScheduler mergeScheduler) throws IOException, InterruptedException { + SharedState state = new SharedState(100000); + initializeDirectory(dir); + IndexWriter iw = openWriter(dir, mergeScheduler); + ExecutorService e = Executors.newFixedThreadPool(THREADS); + for (int i = 0; i < THREADS; i++) { + e.execute(new IndexWriterClientThread(iw, state, dir)); + } + e.shutdown(); + e.awaitTermination(STRESS_TIME, TimeUnit.SECONDS); + state.quit(); + e.awaitTermination(10, TimeUnit.SECONDS); + iw.close(); + Assert.assertEquals(0, state.errorCount()); + } + + private static void initializeDirectory(Directory directory) throws IOException { + IndexWriter iwriter = new IndexWriter(directory, analyzer, true, MaxFieldLength.UNLIMITED); + iwriter.commit(); + iwriter.close(); + } + + public static IndexWriter openWriter(Directory directory, MergeScheduler mergeScheduler) throws IOException { + IndexWriter iwriter = new IndexWriter(directory, analyzer, false, MaxFieldLength.UNLIMITED); + iwriter.setMergeScheduler(mergeScheduler); + return iwriter; + } + +} Property changes on: src/test/org/apache/lucene/index/stresstests/TestIndexWriterConcurrent.java ___________________________________________________________________ Added: svn:keywords + Id Revision Added: svn:eol-style + LF Index: src/test/org/apache/lucene/index/stresstests/IndexWriterClientThread.java =================================================================== --- src/test/org/apache/lucene/index/stresstests/IndexWriterClientThread.java (revision 0) +++ src/test/org/apache/lucene/index/stresstests/IndexWriterClientThread.java (revision 0) @@ -0,0 +1,108 @@ +package org.apache.lucene.index.stresstests; + +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import java.io.IOException; +import org.apache.lucene.document.Document; +import org.apache.lucene.document.Field; +import org.apache.lucene.document.Field.Index; +import org.apache.lucene.document.Field.Store; +import org.apache.lucene.index.CorruptIndexException; +import org.apache.lucene.index.IndexReader; +import org.apache.lucene.index.IndexWriter; +import org.apache.lucene.index.Term; +import org.apache.lucene.store.Directory; + +/** + * IndexWriterClientThread exercises the provided Directory and IndexWriter + * to add and delete terms the index. + */ +public class IndexWriterClientThread implements Runnable { + + /** constants **/ + private static final String FIELDNAME = "mainField"; + private static final Term term = new Term(FIELDNAME); + + /** final threadsafe resources set by constructor **/ + private final IndexWriter iw; + private final SharedState state; + private final Directory dir; + + /** local only state **/ + private IndexReader ir; + + IndexWriterClientThread(IndexWriter iw, SharedState state, Directory dir) { + this.iw = iw; + this.state = state; + this.dir = dir; + } + + @Override + public void run() { + while (!state.needToQuit()) { + try { + runtest(); + } catch (Exception e) { + state.errorManage(e); + } + } + try { + getIndexReader().close(); + } catch (IOException e) { + } + } + + private void runtest() throws InterruptedException, CorruptIndexException, IOException { + String termToAdd = state.getStringToAddToIndex(); + addTermToIndex(termToAdd); + verifyTermExists(termToAdd); + state.addStringWrittenToIndex(termToAdd); + } + + private void verifyTermExists(String termToAdd) throws CorruptIndexException, IOException { + IndexReader ir = getIndexReader(); + Term t = term.createTerm(termToAdd); + int count = ir.docFreq(t); + if (count!=1) { + state.errorManage("Unexpected term count of " + count + " for term: '" + termToAdd + "'"); + } + state.incrementIndexSearchesCount(1); + } + + private void addTermToIndex(String termToAdd) throws CorruptIndexException, IOException { + Document doc = new Document(); + Field f = new Field(FIELDNAME, termToAdd, Store.YES, Index.NOT_ANALYZED_NO_NORMS); + doc.add(f); + iw.addDocument(doc); + iw.commit(); + state.incrementIndexWriterTaskCount(1); + } + + private IndexReader getIndexReader() throws CorruptIndexException, IOException { + if (ir == null) { + ir = IndexReader.open(dir, true); + } + else { + IndexReader newIr = ir.reopen(); + ir.close(); + ir = newIr; + } + return ir; + } + +} Property changes on: src/test/org/apache/lucene/index/stresstests/IndexWriterClientThread.java ___________________________________________________________________ Added: svn:keywords + Id Revision Added: svn:eol-style + LF