dweiss commented on a change in pull request #128:
URL: https://github.com/apache/lucene/pull/128#discussion_r634091331
##########
File path: lucene/core/src/java/org/apache/lucene/index/CheckIndex.java
##########
@@ -701,104 +765,196 @@ public Status checkIndex(List<String> onlySegments)
throws IOException {
if (reader.hasDeletions()) {
if (reader.numDocs() != info.info.maxDoc() - info.getDelCount()) {
throw new RuntimeException(
- "delete count mismatch: info="
+ segmentId
+ + "delete count mismatch: info="
+ (info.info.maxDoc() - info.getDelCount())
+ " vs reader="
+ reader.numDocs());
}
if ((info.info.maxDoc() - reader.numDocs()) > reader.maxDoc()) {
throw new RuntimeException(
- "too many deleted docs: maxDoc()="
+ segmentId
+ + "too many deleted docs: maxDoc()="
+ reader.maxDoc()
+ " vs del count="
+ (info.info.maxDoc() - reader.numDocs()));
}
if (info.info.maxDoc() - reader.numDocs() != info.getDelCount()) {
throw new RuntimeException(
- "delete count mismatch: info="
+ segmentId
+ + "delete count mismatch: info="
+ info.getDelCount()
+ " vs reader="
+ (info.info.maxDoc() - reader.numDocs()));
}
} else {
if (info.getDelCount() != 0) {
throw new RuntimeException(
- "delete count mismatch: info="
+ segmentId
+ + "delete count mismatch: info="
+ info.getDelCount()
+ " vs reader="
+ (info.info.maxDoc() - reader.numDocs()));
}
}
if (checksumsOnly == false) {
+ // This redundant assignment is done to make compiler happy
+ SegmentReader finalReader = reader;
+
// Test Livedocs
- segInfoStat.liveDocStatus = testLiveDocs(reader, infoStream,
failFast);
+ CompletableFuture<Void> testliveDocs =
+ runAysncSegmentPartCheck(
+ executorService,
+ () -> testLiveDocs(finalReader, infoStream, segmentId),
+ liveDocStatus -> segInfoStat.liveDocStatus = liveDocStatus);
// Test Fieldinfos
- segInfoStat.fieldInfoStatus = testFieldInfos(reader, infoStream,
failFast);
+ CompletableFuture<Void> testFieldInfos =
+ runAysncSegmentPartCheck(
+ executorService,
+ () -> testFieldInfos(finalReader, infoStream, segmentId),
+ fieldInfoStatus -> segInfoStat.fieldInfoStatus =
fieldInfoStatus);
// Test Field Norms
- segInfoStat.fieldNormStatus = testFieldNorms(reader, infoStream,
failFast);
+ CompletableFuture<Void> testFieldNorms =
+ runAysncSegmentPartCheck(
+ executorService,
+ () -> testFieldNorms(finalReader, infoStream, segmentId),
+ fieldNormStatus -> segInfoStat.fieldNormStatus =
fieldNormStatus);
// Test the Term Index
- segInfoStat.termIndexStatus =
- testPostings(reader, infoStream, verbose, doSlowChecks,
failFast);
+ CompletableFuture<Void> testTermIndex =
+ runAysncSegmentPartCheck(
+ executorService,
+ () -> testPostings(finalReader, infoStream, segmentId,
verbose, doSlowChecks),
+ termIndexStatus -> segInfoStat.termIndexStatus =
termIndexStatus);
// Test Stored Fields
- segInfoStat.storedFieldStatus = testStoredFields(reader, infoStream,
failFast);
+ CompletableFuture<Void> testStoredFields =
+ runAysncSegmentPartCheck(
+ executorService,
+ () -> testStoredFields(finalReader, infoStream, segmentId),
+ storedFieldStatus -> segInfoStat.storedFieldStatus =
storedFieldStatus);
// Test Term Vectors
- segInfoStat.termVectorStatus =
- testTermVectors(reader, infoStream, verbose, doSlowChecks,
failFast);
+ CompletableFuture<Void> testTermVectors =
+ runAysncSegmentPartCheck(
+ executorService,
+ () -> testTermVectors(finalReader, infoStream, segmentId,
verbose, doSlowChecks),
+ termVectorStatus -> segInfoStat.termVectorStatus =
termVectorStatus);
// Test Docvalues
- segInfoStat.docValuesStatus = testDocValues(reader, infoStream,
failFast);
+ CompletableFuture<Void> testDocValues =
+ runAysncSegmentPartCheck(
+ executorService,
+ () -> testDocValues(finalReader, infoStream, segmentId),
+ docValuesStatus -> segInfoStat.docValuesStatus =
docValuesStatus);
// Test PointValues
- segInfoStat.pointsStatus = testPoints(reader, infoStream, failFast);
+ CompletableFuture<Void> testPointvalues =
+ runAysncSegmentPartCheck(
+ executorService,
+ () -> testPoints(finalReader, infoStream, segmentId),
+ pointsStatus -> segInfoStat.pointsStatus = pointsStatus);
// Test VectorValues
- segInfoStat.vectorValuesStatus = testVectors(reader, infoStream,
failFast);
+ CompletableFuture<Void> testVectors =
+ runAysncSegmentPartCheck(
+ executorService,
+ () -> testVectors(finalReader, infoStream, segmentId),
+ vectorValuesStatus -> segInfoStat.vectorValuesStatus =
vectorValuesStatus);
// Test index sort
- segInfoStat.indexSortStatus = testSort(reader, indexSort,
infoStream, failFast);
+ CompletableFuture<Void> testSort =
+ runAysncSegmentPartCheck(
+ executorService,
+ () -> testSort(finalReader, indexSort, infoStream,
segmentId),
+ indexSortStatus -> segInfoStat.indexSortStatus =
indexSortStatus);
+
+ CompletableFuture<Void> testSoftDeletes = null;
+ final String softDeletesField =
reader.getFieldInfos().getSoftDeletesField();
+ if (softDeletesField != null) {
+ testSoftDeletes =
+ runAysncSegmentPartCheck(
+ executorService,
+ () ->
+ checkSoftDeletes(
+ softDeletesField, info, finalReader, infoStream,
segmentId),
+ softDeletesStatus -> segInfoStat.softDeletesStatus =
softDeletesStatus);
+ }
// Rethrow the first exception we encountered
// This will cause stats for failed segments to be incremented
properly
+ testliveDocs.join();
if (segInfoStat.liveDocStatus.error != null) {
- throw new RuntimeException("Live docs test failed");
- } else if (segInfoStat.fieldInfoStatus.error != null) {
- throw new RuntimeException("Field Info test failed");
- } else if (segInfoStat.fieldNormStatus.error != null) {
- throw new RuntimeException("Field Norm test failed");
- } else if (segInfoStat.termIndexStatus.error != null) {
- throw new RuntimeException("Term Index test failed");
- } else if (segInfoStat.storedFieldStatus.error != null) {
- throw new RuntimeException("Stored Field test failed");
- } else if (segInfoStat.termVectorStatus.error != null) {
- throw new RuntimeException("Term Vector test failed");
- } else if (segInfoStat.docValuesStatus.error != null) {
- throw new RuntimeException("DocValues test failed");
- } else if (segInfoStat.pointsStatus.error != null) {
- throw new RuntimeException("Points test failed");
+ throw new RuntimeException(segmentId + "Live docs test failed");
+ }
+
+ testFieldInfos.join();
+ if (segInfoStat.fieldInfoStatus.error != null) {
+ throw new RuntimeException(segmentId + "Field Info test failed");
Review comment:
I think the cause should be added to those runtime exceptions.
##########
File path: lucene/core/src/java/org/apache/lucene/index/CheckIndex.java
##########
@@ -701,104 +765,196 @@ public Status checkIndex(List<String> onlySegments)
throws IOException {
if (reader.hasDeletions()) {
if (reader.numDocs() != info.info.maxDoc() - info.getDelCount()) {
throw new RuntimeException(
- "delete count mismatch: info="
+ segmentId
+ + "delete count mismatch: info="
+ (info.info.maxDoc() - info.getDelCount())
+ " vs reader="
+ reader.numDocs());
}
if ((info.info.maxDoc() - reader.numDocs()) > reader.maxDoc()) {
throw new RuntimeException(
- "too many deleted docs: maxDoc()="
+ segmentId
+ + "too many deleted docs: maxDoc()="
+ reader.maxDoc()
+ " vs del count="
+ (info.info.maxDoc() - reader.numDocs()));
}
if (info.info.maxDoc() - reader.numDocs() != info.getDelCount()) {
throw new RuntimeException(
- "delete count mismatch: info="
+ segmentId
+ + "delete count mismatch: info="
+ info.getDelCount()
+ " vs reader="
+ (info.info.maxDoc() - reader.numDocs()));
}
} else {
if (info.getDelCount() != 0) {
throw new RuntimeException(
- "delete count mismatch: info="
+ segmentId
+ + "delete count mismatch: info="
+ info.getDelCount()
+ " vs reader="
+ (info.info.maxDoc() - reader.numDocs()));
}
}
if (checksumsOnly == false) {
+ // This redundant assignment is done to make compiler happy
+ SegmentReader finalReader = reader;
+
// Test Livedocs
- segInfoStat.liveDocStatus = testLiveDocs(reader, infoStream,
failFast);
+ CompletableFuture<Void> testliveDocs =
+ runAysncSegmentPartCheck(
Review comment:
Typo in aysnc.
##########
File path: lucene/core/src/java/org/apache/lucene/index/CheckIndex.java
##########
@@ -488,8 +519,35 @@ public Status checkIndex() throws IOException {
* quite a long time to run.
*/
public Status checkIndex(List<String> onlySegments) throws IOException {
+ ExecutorService executorService =
+ Executors.newFixedThreadPool(threadCount, new
NamedThreadFactory("async-check-index"));
+ try {
+ return checkIndex(onlySegments, executorService);
+ } finally {
+ executorService.shutdown();
+ try {
+ executorService.awaitTermination(5, TimeUnit.SECONDS);
+ } catch (
+ @SuppressWarnings("unused")
Review comment:
Why ignore? Seems like something went wrong there - log it at least?
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
us...@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscr...@lucene.apache.org
For additional commands, e-mail: issues-h...@lucene.apache.org
