mikemccand commented on a change in pull request #128:
URL: https://github.com/apache/lucene/pull/128#discussion_r638788325
##########
File path: lucene/core/src/java/org/apache/lucene/index/CheckIndex.java
##########
@@ -305,7 +317,7 @@ private FieldNormStatus() {}
public long totVectors = 0;
/** Exception thrown during term vector test (null on success) */
- public Throwable error = null;
Review comment:
Thank you for fixing all of these!
##########
File path: lucene/core/src/java/org/apache/lucene/index/CheckIndex.java
##########
@@ -731,74 +810,173 @@ public Status checkIndex(List<String> onlySegments)
throws IOException {
}
if (checksumsOnly == false) {
+ // This redundant assignment is done to make compiler happy
+ SegmentReader finalReader = reader;
+
// Test Livedocs
- segInfoStat.liveDocStatus = testLiveDocs(reader, infoStream,
failFast);
+ CompletableFuture<Void> testliveDocs =
+ runAsyncSegmentPartCheck(
+ executorService,
+ () -> testLiveDocs(finalReader, infoStream, segmentId),
+ liveDocStatus -> segInfoStat.liveDocStatus = liveDocStatus);
// Test Fieldinfos
- segInfoStat.fieldInfoStatus = testFieldInfos(reader, infoStream,
failFast);
+ CompletableFuture<Void> testFieldInfos =
+ runAsyncSegmentPartCheck(
+ executorService,
+ () -> testFieldInfos(finalReader, infoStream, segmentId),
+ fieldInfoStatus -> segInfoStat.fieldInfoStatus =
fieldInfoStatus);
// Test Field Norms
- segInfoStat.fieldNormStatus = testFieldNorms(reader, infoStream,
failFast);
+ CompletableFuture<Void> testFieldNorms =
+ runAsyncSegmentPartCheck(
+ executorService,
+ () -> testFieldNorms(finalReader, infoStream, segmentId),
+ fieldNormStatus -> segInfoStat.fieldNormStatus =
fieldNormStatus);
// Test the Term Index
- segInfoStat.termIndexStatus =
- testPostings(reader, infoStream, verbose, doSlowChecks,
failFast);
+ CompletableFuture<Void> testTermIndex =
+ runAsyncSegmentPartCheck(
+ executorService,
+ () -> testPostings(finalReader, infoStream, segmentId,
verbose, doSlowChecks),
+ termIndexStatus -> segInfoStat.termIndexStatus =
termIndexStatus);
// Test Stored Fields
- segInfoStat.storedFieldStatus = testStoredFields(reader, infoStream,
failFast);
+ CompletableFuture<Void> testStoredFields =
+ runAsyncSegmentPartCheck(
+ executorService,
+ () -> testStoredFields(finalReader, infoStream, segmentId),
+ storedFieldStatus -> segInfoStat.storedFieldStatus =
storedFieldStatus);
// Test Term Vectors
- segInfoStat.termVectorStatus =
- testTermVectors(reader, infoStream, verbose, doSlowChecks,
failFast);
+ CompletableFuture<Void> testTermVectors =
+ runAsyncSegmentPartCheck(
+ executorService,
+ () -> testTermVectors(finalReader, infoStream, segmentId,
verbose, doSlowChecks),
+ termVectorStatus -> segInfoStat.termVectorStatus =
termVectorStatus);
// Test Docvalues
- segInfoStat.docValuesStatus = testDocValues(reader, infoStream,
failFast);
+ CompletableFuture<Void> testDocValues =
+ runAsyncSegmentPartCheck(
+ executorService,
+ () -> testDocValues(finalReader, infoStream, segmentId),
+ docValuesStatus -> segInfoStat.docValuesStatus =
docValuesStatus);
// Test PointValues
- segInfoStat.pointsStatus = testPoints(reader, infoStream, failFast);
+ CompletableFuture<Void> testPointvalues =
+ runAsyncSegmentPartCheck(
+ executorService,
+ () -> testPoints(finalReader, infoStream, segmentId),
+ pointsStatus -> segInfoStat.pointsStatus = pointsStatus);
// Test VectorValues
- segInfoStat.vectorValuesStatus = testVectors(reader, infoStream,
failFast);
+ CompletableFuture<Void> testVectors =
+ runAsyncSegmentPartCheck(
+ executorService,
+ () -> testVectors(finalReader, infoStream, segmentId),
+ vectorValuesStatus -> segInfoStat.vectorValuesStatus =
vectorValuesStatus);
// Test index sort
- segInfoStat.indexSortStatus = testSort(reader, indexSort,
infoStream, failFast);
+ CompletableFuture<Void> testSort =
+ runAsyncSegmentPartCheck(
+ executorService,
+ () -> testSort(finalReader, indexSort, infoStream,
segmentId),
+ indexSortStatus -> segInfoStat.indexSortStatus =
indexSortStatus);
+
+ CompletableFuture<Void> testSoftDeletes = null;
+ final String softDeletesField =
reader.getFieldInfos().getSoftDeletesField();
+ if (softDeletesField != null) {
+ testSoftDeletes =
+ runAsyncSegmentPartCheck(
+ executorService,
+ () ->
+ checkSoftDeletes(
+ softDeletesField, info, finalReader, infoStream,
segmentId),
+ softDeletesStatus -> segInfoStat.softDeletesStatus =
softDeletesStatus);
+ }
// Rethrow the first exception we encountered
// This will cause stats for failed segments to be incremented
properly
+ testliveDocs.join();
if (segInfoStat.liveDocStatus.error != null) {
- throw new RuntimeException("Live docs test failed");
- } else if (segInfoStat.fieldInfoStatus.error != null) {
- throw new RuntimeException("Field Info test failed");
- } else if (segInfoStat.fieldNormStatus.error != null) {
- throw new RuntimeException("Field Norm test failed");
- } else if (segInfoStat.termIndexStatus.error != null) {
- throw new RuntimeException("Term Index test failed");
- } else if (segInfoStat.storedFieldStatus.error != null) {
- throw new RuntimeException("Stored Field test failed");
- } else if (segInfoStat.termVectorStatus.error != null) {
- throw new RuntimeException("Term Vector test failed");
- } else if (segInfoStat.docValuesStatus.error != null) {
- throw new RuntimeException("DocValues test failed");
- } else if (segInfoStat.pointsStatus.error != null) {
- throw new RuntimeException("Points test failed");
+ throw new CheckIndexException(
+ segmentId, "", "Live docs test failed",
segInfoStat.liveDocStatus.error);
+ }
+
+ testFieldInfos.join();
+ if (segInfoStat.fieldInfoStatus.error != null) {
+ throw new CheckIndexException(
+ segmentId, "", "Field Info test failed",
segInfoStat.fieldInfoStatus.error);
+ }
+
+ testFieldNorms.join();
+ if (segInfoStat.fieldNormStatus.error != null) {
+ throw new CheckIndexException(
+ segmentId, "", "Field Norm test failed",
segInfoStat.fieldNormStatus.error);
+ }
+
+ testTermIndex.join();
+ if (segInfoStat.termIndexStatus.error != null) {
+ throw new CheckIndexException(
+ segmentId, "", "Term Index test failed",
segInfoStat.termIndexStatus.error);
+ }
+
+ testStoredFields.join();
+ if (segInfoStat.storedFieldStatus.error != null) {
+ throw new CheckIndexException(
+ segmentId, "", "Stored Field test failed",
segInfoStat.storedFieldStatus.error);
+ }
+
+ testTermVectors.join();
+ if (segInfoStat.termVectorStatus.error != null) {
+ throw new CheckIndexException(
+ segmentId, "", "Term Vector test failed",
segInfoStat.termVectorStatus.error);
Review comment:
I think instead of empty string for `partId` param we can pass e.g.
`"field infos"`, "term vectors"`, etc.
##########
File path: lucene/core/src/java/org/apache/lucene/index/CheckIndex.java
##########
@@ -1058,16 +1261,14 @@ public Status checkIndex(List<String> onlySegments)
throws IOException {
msg(
infoStream,
+ segmentId + partId,
String.format(
Locale.ROOT,
"OK [%d fields] [took %.3f sec]",
status.totFields,
nsToSec(System.nanoTime() - startNS)));
} catch (Throwable e) {
- if (failFast) {
- throw IOUtils.rethrowAlways(e);
- }
- msg(infoStream, "ERROR [" + String.valueOf(e.getMessage()) + "]");
+ msg(infoStream, segmentId + partId, "ERROR [" +
String.valueOf(e.getMessage()) + "]");
Review comment:
Hmm if `partId` is a simple string like `"term vectors"` but then
sometimes empty string, maybe we need to make a helper method to carefully join
them? E.g. if `partId` is `null` or `""` then we leave it off, but if it is
not then we maybe print something like `_7 [term vectors]: ...`?
##########
File path: lucene/core/src/java/org/apache/lucene/index/CheckIndex.java
##########
@@ -700,29 +771,37 @@ public Status checkIndex(List<String> onlySegments)
throws IOException {
if (reader.hasDeletions()) {
if (reader.numDocs() != info.info.maxDoc() - info.getDelCount()) {
- throw new RuntimeException(
+ throw new CheckIndexException(
+ segmentId,
+ "",
Review comment:
You could maybe pass `"live docs"` as the `partId` here?
##########
File path: lucene/core/src/java/org/apache/lucene/index/CheckIndex.java
##########
@@ -731,74 +810,173 @@ public Status checkIndex(List<String> onlySegments)
throws IOException {
}
if (checksumsOnly == false) {
+ // This redundant assignment is done to make compiler happy
+ SegmentReader finalReader = reader;
+
// Test Livedocs
- segInfoStat.liveDocStatus = testLiveDocs(reader, infoStream,
failFast);
+ CompletableFuture<Void> testliveDocs =
+ runAsyncSegmentPartCheck(
+ executorService,
+ () -> testLiveDocs(finalReader, infoStream, segmentId),
+ liveDocStatus -> segInfoStat.liveDocStatus = liveDocStatus);
// Test Fieldinfos
- segInfoStat.fieldInfoStatus = testFieldInfos(reader, infoStream,
failFast);
+ CompletableFuture<Void> testFieldInfos =
+ runAsyncSegmentPartCheck(
+ executorService,
+ () -> testFieldInfos(finalReader, infoStream, segmentId),
+ fieldInfoStatus -> segInfoStat.fieldInfoStatus =
fieldInfoStatus);
// Test Field Norms
- segInfoStat.fieldNormStatus = testFieldNorms(reader, infoStream,
failFast);
+ CompletableFuture<Void> testFieldNorms =
+ runAsyncSegmentPartCheck(
+ executorService,
+ () -> testFieldNorms(finalReader, infoStream, segmentId),
+ fieldNormStatus -> segInfoStat.fieldNormStatus =
fieldNormStatus);
// Test the Term Index
- segInfoStat.termIndexStatus =
- testPostings(reader, infoStream, verbose, doSlowChecks,
failFast);
+ CompletableFuture<Void> testTermIndex =
+ runAsyncSegmentPartCheck(
+ executorService,
+ () -> testPostings(finalReader, infoStream, segmentId,
verbose, doSlowChecks),
+ termIndexStatus -> segInfoStat.termIndexStatus =
termIndexStatus);
// Test Stored Fields
- segInfoStat.storedFieldStatus = testStoredFields(reader, infoStream,
failFast);
+ CompletableFuture<Void> testStoredFields =
+ runAsyncSegmentPartCheck(
+ executorService,
+ () -> testStoredFields(finalReader, infoStream, segmentId),
+ storedFieldStatus -> segInfoStat.storedFieldStatus =
storedFieldStatus);
// Test Term Vectors
- segInfoStat.termVectorStatus =
- testTermVectors(reader, infoStream, verbose, doSlowChecks,
failFast);
+ CompletableFuture<Void> testTermVectors =
+ runAsyncSegmentPartCheck(
+ executorService,
+ () -> testTermVectors(finalReader, infoStream, segmentId,
verbose, doSlowChecks),
+ termVectorStatus -> segInfoStat.termVectorStatus =
termVectorStatus);
// Test Docvalues
- segInfoStat.docValuesStatus = testDocValues(reader, infoStream,
failFast);
+ CompletableFuture<Void> testDocValues =
+ runAsyncSegmentPartCheck(
+ executorService,
+ () -> testDocValues(finalReader, infoStream, segmentId),
+ docValuesStatus -> segInfoStat.docValuesStatus =
docValuesStatus);
// Test PointValues
- segInfoStat.pointsStatus = testPoints(reader, infoStream, failFast);
+ CompletableFuture<Void> testPointvalues =
+ runAsyncSegmentPartCheck(
+ executorService,
+ () -> testPoints(finalReader, infoStream, segmentId),
+ pointsStatus -> segInfoStat.pointsStatus = pointsStatus);
// Test VectorValues
- segInfoStat.vectorValuesStatus = testVectors(reader, infoStream,
failFast);
+ CompletableFuture<Void> testVectors =
+ runAsyncSegmentPartCheck(
+ executorService,
+ () -> testVectors(finalReader, infoStream, segmentId),
+ vectorValuesStatus -> segInfoStat.vectorValuesStatus =
vectorValuesStatus);
// Test index sort
- segInfoStat.indexSortStatus = testSort(reader, indexSort,
infoStream, failFast);
+ CompletableFuture<Void> testSort =
+ runAsyncSegmentPartCheck(
+ executorService,
+ () -> testSort(finalReader, indexSort, infoStream,
segmentId),
+ indexSortStatus -> segInfoStat.indexSortStatus =
indexSortStatus);
+
+ CompletableFuture<Void> testSoftDeletes = null;
+ final String softDeletesField =
reader.getFieldInfos().getSoftDeletesField();
+ if (softDeletesField != null) {
+ testSoftDeletes =
+ runAsyncSegmentPartCheck(
+ executorService,
+ () ->
+ checkSoftDeletes(
+ softDeletesField, info, finalReader, infoStream,
segmentId),
+ softDeletesStatus -> segInfoStat.softDeletesStatus =
softDeletesStatus);
+ }
// Rethrow the first exception we encountered
// This will cause stats for failed segments to be incremented
properly
+ testliveDocs.join();
if (segInfoStat.liveDocStatus.error != null) {
- throw new RuntimeException("Live docs test failed");
- } else if (segInfoStat.fieldInfoStatus.error != null) {
- throw new RuntimeException("Field Info test failed");
- } else if (segInfoStat.fieldNormStatus.error != null) {
- throw new RuntimeException("Field Norm test failed");
- } else if (segInfoStat.termIndexStatus.error != null) {
- throw new RuntimeException("Term Index test failed");
- } else if (segInfoStat.storedFieldStatus.error != null) {
- throw new RuntimeException("Stored Field test failed");
- } else if (segInfoStat.termVectorStatus.error != null) {
- throw new RuntimeException("Term Vector test failed");
- } else if (segInfoStat.docValuesStatus.error != null) {
- throw new RuntimeException("DocValues test failed");
- } else if (segInfoStat.pointsStatus.error != null) {
- throw new RuntimeException("Points test failed");
+ throw new CheckIndexException(
+ segmentId, "", "Live docs test failed",
segInfoStat.liveDocStatus.error);
+ }
+
+ testFieldInfos.join();
+ if (segInfoStat.fieldInfoStatus.error != null) {
+ throw new CheckIndexException(
+ segmentId, "", "Field Info test failed",
segInfoStat.fieldInfoStatus.error);
+ }
+
+ testFieldNorms.join();
+ if (segInfoStat.fieldNormStatus.error != null) {
+ throw new CheckIndexException(
+ segmentId, "", "Field Norm test failed",
segInfoStat.fieldNormStatus.error);
+ }
+
+ testTermIndex.join();
+ if (segInfoStat.termIndexStatus.error != null) {
+ throw new CheckIndexException(
+ segmentId, "", "Term Index test failed",
segInfoStat.termIndexStatus.error);
+ }
+
+ testStoredFields.join();
+ if (segInfoStat.storedFieldStatus.error != null) {
+ throw new CheckIndexException(
+ segmentId, "", "Stored Field test failed",
segInfoStat.storedFieldStatus.error);
+ }
+
+ testTermVectors.join();
+ if (segInfoStat.termVectorStatus.error != null) {
+ throw new CheckIndexException(
+ segmentId, "", "Term Vector test failed",
segInfoStat.termVectorStatus.error);
+ }
+
+ testDocValues.join();
+ if (segInfoStat.docValuesStatus.error != null) {
+ throw new CheckIndexException(
+ segmentId, "", "DocValues test failed",
segInfoStat.docValuesStatus.error);
+ }
+
+ testPointvalues.join();
+ if (segInfoStat.pointsStatus.error != null) {
+ throw new CheckIndexException(
+ segmentId, "", "Points test failed",
segInfoStat.pointsStatus.error);
+ }
+
+ testVectors.join();
+ if (segInfoStat.vectorValuesStatus.error != null) {
+ throw new CheckIndexException(
+ segmentId, "", "Vectors test failed",
segInfoStat.vectorValuesStatus.error);
Review comment:
This `partId` can be `"vectors"`.
##########
File path: lucene/core/src/java/org/apache/lucene/index/CheckIndex.java
##########
@@ -488,8 +503,35 @@ public Status checkIndex() throws IOException {
* quite a long time to run.
*/
public Status checkIndex(List<String> onlySegments) throws IOException {
Review comment:
Maybe post the `CheckIndex` output on a real-ish (multi-segment) index
as an example? Actually I will do that on the nightly benchmark (English
Wikipedia) index!
##########
File path: lucene/core/src/java/org/apache/lucene/index/CheckIndex.java
##########
@@ -731,74 +810,173 @@ public Status checkIndex(List<String> onlySegments)
throws IOException {
}
if (checksumsOnly == false) {
+ // This redundant assignment is done to make compiler happy
+ SegmentReader finalReader = reader;
+
// Test Livedocs
- segInfoStat.liveDocStatus = testLiveDocs(reader, infoStream,
failFast);
+ CompletableFuture<Void> testliveDocs =
+ runAsyncSegmentPartCheck(
+ executorService,
+ () -> testLiveDocs(finalReader, infoStream, segmentId),
+ liveDocStatus -> segInfoStat.liveDocStatus = liveDocStatus);
// Test Fieldinfos
- segInfoStat.fieldInfoStatus = testFieldInfos(reader, infoStream,
failFast);
+ CompletableFuture<Void> testFieldInfos =
+ runAsyncSegmentPartCheck(
+ executorService,
+ () -> testFieldInfos(finalReader, infoStream, segmentId),
+ fieldInfoStatus -> segInfoStat.fieldInfoStatus =
fieldInfoStatus);
// Test Field Norms
- segInfoStat.fieldNormStatus = testFieldNorms(reader, infoStream,
failFast);
+ CompletableFuture<Void> testFieldNorms =
+ runAsyncSegmentPartCheck(
+ executorService,
+ () -> testFieldNorms(finalReader, infoStream, segmentId),
+ fieldNormStatus -> segInfoStat.fieldNormStatus =
fieldNormStatus);
// Test the Term Index
- segInfoStat.termIndexStatus =
- testPostings(reader, infoStream, verbose, doSlowChecks,
failFast);
+ CompletableFuture<Void> testTermIndex =
+ runAsyncSegmentPartCheck(
+ executorService,
+ () -> testPostings(finalReader, infoStream, segmentId,
verbose, doSlowChecks),
+ termIndexStatus -> segInfoStat.termIndexStatus =
termIndexStatus);
// Test Stored Fields
- segInfoStat.storedFieldStatus = testStoredFields(reader, infoStream,
failFast);
+ CompletableFuture<Void> testStoredFields =
+ runAsyncSegmentPartCheck(
+ executorService,
+ () -> testStoredFields(finalReader, infoStream, segmentId),
+ storedFieldStatus -> segInfoStat.storedFieldStatus =
storedFieldStatus);
// Test Term Vectors
- segInfoStat.termVectorStatus =
- testTermVectors(reader, infoStream, verbose, doSlowChecks,
failFast);
+ CompletableFuture<Void> testTermVectors =
+ runAsyncSegmentPartCheck(
+ executorService,
+ () -> testTermVectors(finalReader, infoStream, segmentId,
verbose, doSlowChecks),
+ termVectorStatus -> segInfoStat.termVectorStatus =
termVectorStatus);
// Test Docvalues
- segInfoStat.docValuesStatus = testDocValues(reader, infoStream,
failFast);
+ CompletableFuture<Void> testDocValues =
+ runAsyncSegmentPartCheck(
+ executorService,
+ () -> testDocValues(finalReader, infoStream, segmentId),
+ docValuesStatus -> segInfoStat.docValuesStatus =
docValuesStatus);
// Test PointValues
- segInfoStat.pointsStatus = testPoints(reader, infoStream, failFast);
+ CompletableFuture<Void> testPointvalues =
+ runAsyncSegmentPartCheck(
+ executorService,
+ () -> testPoints(finalReader, infoStream, segmentId),
+ pointsStatus -> segInfoStat.pointsStatus = pointsStatus);
// Test VectorValues
- segInfoStat.vectorValuesStatus = testVectors(reader, infoStream,
failFast);
+ CompletableFuture<Void> testVectors =
+ runAsyncSegmentPartCheck(
+ executorService,
+ () -> testVectors(finalReader, infoStream, segmentId),
+ vectorValuesStatus -> segInfoStat.vectorValuesStatus =
vectorValuesStatus);
// Test index sort
- segInfoStat.indexSortStatus = testSort(reader, indexSort,
infoStream, failFast);
+ CompletableFuture<Void> testSort =
+ runAsyncSegmentPartCheck(
+ executorService,
+ () -> testSort(finalReader, indexSort, infoStream,
segmentId),
+ indexSortStatus -> segInfoStat.indexSortStatus =
indexSortStatus);
+
+ CompletableFuture<Void> testSoftDeletes = null;
+ final String softDeletesField =
reader.getFieldInfos().getSoftDeletesField();
+ if (softDeletesField != null) {
+ testSoftDeletes =
+ runAsyncSegmentPartCheck(
+ executorService,
+ () ->
+ checkSoftDeletes(
+ softDeletesField, info, finalReader, infoStream,
segmentId),
+ softDeletesStatus -> segInfoStat.softDeletesStatus =
softDeletesStatus);
+ }
// Rethrow the first exception we encountered
// This will cause stats for failed segments to be incremented
properly
+ testliveDocs.join();
if (segInfoStat.liveDocStatus.error != null) {
- throw new RuntimeException("Live docs test failed");
- } else if (segInfoStat.fieldInfoStatus.error != null) {
- throw new RuntimeException("Field Info test failed");
- } else if (segInfoStat.fieldNormStatus.error != null) {
- throw new RuntimeException("Field Norm test failed");
- } else if (segInfoStat.termIndexStatus.error != null) {
- throw new RuntimeException("Term Index test failed");
- } else if (segInfoStat.storedFieldStatus.error != null) {
- throw new RuntimeException("Stored Field test failed");
- } else if (segInfoStat.termVectorStatus.error != null) {
- throw new RuntimeException("Term Vector test failed");
- } else if (segInfoStat.docValuesStatus.error != null) {
- throw new RuntimeException("DocValues test failed");
- } else if (segInfoStat.pointsStatus.error != null) {
- throw new RuntimeException("Points test failed");
+ throw new CheckIndexException(
+ segmentId, "", "Live docs test failed",
segInfoStat.liveDocStatus.error);
+ }
+
+ testFieldInfos.join();
Review comment:
OK, here is where we could always
e.g.`System.out.print(segInfoState.fieldInfoMessages.toString())` or so?
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
us...@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscr...@lucene.apache.org
For additional commands, e-mail: issues-h...@lucene.apache.org
