dnhatn commented on a change in pull request #1361: LUCENE-8118: Throw
exception if DWPT grows beyond it's maximum ram limit
URL: https://github.com/apache/lucene-solr/pull/1361#discussion_r395423835
##########
File path: lucene/core/src/test/org/apache/lucene/index/TestIndexWriter.java
##########
@@ -3941,4 +3943,108 @@ public void testRandomOperationsWithSoftDeletes()
throws Exception {
}
}
}
+
+ public void testAbortDocWhenHardLimitExceeds() throws IOException {
+ Directory dir = newDirectory();
+ IndexWriterConfig indexWriterConfig = new IndexWriterConfig();
+ IndexWriter w = new IndexWriter(dir, indexWriterConfig);
+ indexWriterConfig.setRAMPerThreadHardLimitMB(1);
+ w.addDocument(new Document()); // no problem
+ w.flush();
+ assertEquals(1, w.getFlushCount());
+ final long seed = random().nextLong();
+ int numDocs = 1;
+ try (LineFileDocs docs = new LineFileDocs(new Random(seed))) {
+ while (w.getFlushCount() == 1) {
+ w.addDocument(docs.nextDoc());
+ numDocs++;
+ }
+ }
+ try (LineFileDocs docs = new LineFileDocs(new Random(seed))) {
+ if (random().nextBoolean()) {
+ w.addDocument(new Document());
+ numDocs++;
+ }
+ w.addDocuments(() -> new Iterator<>() {
+ @Override
+ public boolean hasNext() {
+ return true;
+ }
+
+ @Override
+ public Iterable<? extends IndexableField> next() {
+ try {
+ return docs.nextDoc();
+ } catch (IOException e) {
+ throw new UncheckedIOException(e);
+ }
+ }
+ });
+ } catch (IllegalArgumentException e) {
+ assertEquals("RAM used by a single DocumentsWriterPerThread can't
exceed: 1MB", e.getMessage());
+ assertNull(w.getTragicException());
+ }
+ w.addDocument(new Document());
+ numDocs++;
+ try (LineFileDocs docs = new LineFileDocs(new Random(seed))) {
+ final int currentNumDocs = numDocs;
+ for (int i = 0; i < currentNumDocs; i++) {
+ w.addDocument(docs.nextDoc());
+ numDocs++;
+ }
+ }
+ w.addDocument(new Document());
+ numDocs++;
+ w.forceMergeDeletes(true);
+ w.commit();
+
+ try (IndexReader reader = DirectoryReader.open(dir)) {
+ assertEquals(numDocs, reader.numDocs());
+ }
+
+
+ w.close();
+ dir.close();
+ }
+
+ @Nightly
+ public void testAddDocumentsMassive() throws Exception {
+ Directory dir = newFSDirectory(createTempDir("addDocumentsMassive"));
+ IndexWriter w = new IndexWriter(dir, new IndexWriterConfig());
+ StringBuilder sb = new StringBuilder();
+ // same doc with lots of duplicated terms to put pressure on
FreqProxTermsWriter instead of terms dict.
+ for (int i = 0; i < 1024; i++) {
+ sb.append(i);
+ sb.append(' ');
+ }
+ final AtomicInteger count = new AtomicInteger(0);
+ Document doc = new Document();
+ try {
+ doc.add(new TextField("text", sb.toString(), Field.Store.NO));
+ w.addDocument(doc);
+ w.addDocuments((Iterable<Document>) () -> new Iterator<>() {
+
+ @Override
+ public boolean hasNext() {
+ return count.get() < 100_000_000;
+ }
+
+ @Override
+ public Document next() {
+ if (!hasNext()) {
+ throw new AssertionError();
+ }
+ count.incrementAndGet();
+ return doc;
+ }
+ });
+ } catch (IllegalArgumentException ex) {
+ assertEquals("RAM used by a single DocumentsWriterPerThread can exceed:
1945MB", ex.getMessage());
Review comment:
can -> can't
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
us...@infra.apache.org
With regards,
Apache Git Services
---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscr...@lucene.apache.org
For additional commands, e-mail: issues-h...@lucene.apache.org
