dnhatn commented on a change in pull request #140:
URL: https://github.com/apache/lucene/pull/140#discussion_r649215227
##########
File path:
lucene/test-framework/src/java/org/apache/lucene/index/BaseTermVectorsFormatTestCase.java
##########
@@ -667,43 +673,94 @@ public void testRandom() throws IOException {
dir.close();
}
- public void testMerge() throws IOException {
+ private void doTestMerge(Sort indexSort, boolean allowDeletes) throws
IOException {
final RandomDocumentFactory docFactory = new RandomDocumentFactory(5, 20);
final int numDocs = atLeast(100);
- final int numDeletes = random().nextInt(numDocs);
- final Set<Integer> deletes = new HashSet<>();
- while (deletes.size() < numDeletes) {
- deletes.add(random().nextInt(numDocs));
- }
for (Options options : validOptions()) {
- final RandomDocument[] docs = new RandomDocument[numDocs];
+ Map<String, RandomDocument> docs = new HashMap<>();
for (int i = 0; i < numDocs; ++i) {
- docs[i] = docFactory.newDocument(TestUtil.nextInt(random(), 1, 3),
atLeast(10), options);
+ docs.put(
+ Integer.toString(i),
+ docFactory.newDocument(TestUtil.nextInt(random(), 1, 3),
atLeast(10), options));
}
final Directory dir = newDirectory();
- final RandomIndexWriter writer = new RandomIndexWriter(random(), dir);
- for (int i = 0; i < numDocs; ++i) {
- writer.addDocument(addId(docs[i].toDocument(), "" + i));
+ final IndexWriterConfig iwc = newIndexWriterConfig();
+ if (indexSort != null) {
+ iwc.setIndexSort(indexSort);
+ }
+ final RandomIndexWriter writer = new RandomIndexWriter(random(), dir,
iwc);
+ List<String> liveDocIDs = new ArrayList<>();
+ List<String> ids = new ArrayList<>(docs.keySet());
+ Collections.shuffle(ids, random());
+ Runnable verifyTermVectors =
+ () -> {
+ try (DirectoryReader reader =
maybeWrapWithMergingReader(writer.getReader())) {
+ for (String id : liveDocIDs) {
+ final int docID = docID(reader, id);
+ assertEquals(docs.get(id), reader.getTermVectors(docID));
+ }
+ } catch (IOException e) {
+ throw new UncheckedIOException(e);
+ }
+ };
+ for (String id : ids) {
+ final Document doc = addId(docs.get(id).toDocument(), id);
+ if (indexSort != null) {
+ for (SortField sortField : indexSort.getSort()) {
+ doc.add(
+ new NumericDocValuesField(
+ sortField.getField(), TestUtil.nextInt(random(), 0,
1024)));
+ }
+ }
+ if (random().nextInt(100) < 5) {
+ // add via foreign writer
+ IndexWriterConfig otherIwc = newIndexWriterConfig();
+ if (indexSort != null) {
+ otherIwc.setIndexSort(indexSort);
+ }
+ try (Directory otherDir = newDirectory();
+ RandomIndexWriter otherIw = new RandomIndexWriter(random(),
otherDir, otherIwc)) {
+ otherIw.addDocument(doc);
+ try (DirectoryReader otherReader = otherIw.getReader()) {
+ TestUtil.addIndexesSlowly(writer.w, otherReader);
+ }
+ }
+ } else {
+ writer.addDocument(doc);
+ }
+ liveDocIDs.add(id);
+ if (allowDeletes && random().nextInt(100) < 5) {
Review comment:
If we delete every iteration, then we will delete every newly added
document and we won't have any docs at the end. I creased the probability of
this from 5% to 20% (see
https://github.com/apache/lucene/pull/140/commits/65a1ecb38786601e73ff1dd78b6fdb22b0f1ff48).
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
us...@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscr...@lucene.apache.org
For additional commands, e-mail: issues-h...@lucene.apache.org
