benchaplin commented on code in PR #13984:
URL: https://github.com/apache/lucene/pull/13984#discussion_r1840492668
##########
lucene/core/src/java/org/apache/lucene/index/CheckIndex.java:
##########
@@ -2746,6 +2781,106 @@ public static Status.VectorValuesStatus testVectors(
return status;
}
+ /** Test the HNSW graph. */
+ public static Status.HnswGraphsStatus testHnswGraphs(
+ CodecReader reader, PrintStream infoStream, boolean failFast) throws
IOException {
+ if (infoStream != null) {
+ infoStream.print(" test: hnsw graphs.........");
+ }
+ long startNS = System.nanoTime();
+ Status.HnswGraphsStatus status = new Status.HnswGraphsStatus();
+ KnnVectorsReader vectorsReader = reader.getVectorReader();
+ FieldInfos fieldInfos = reader.getFieldInfos();
+
+ try {
+ if (fieldInfos.hasVectorValues()) {
+ for (FieldInfo fieldInfo : fieldInfos) {
+ if (fieldInfo.hasVectorValues()) {
+ if (vectorsReader instanceof
PerFieldKnnVectorsFormat.FieldsReader) {
+ KnnVectorsReader fieldReader =
+ ((PerFieldKnnVectorsFormat.FieldsReader) vectorsReader)
+ .getFieldReader(fieldInfo.name);
+ if (fieldReader instanceof HnswGraphProvider) {
+ HnswGraph hnswGraph = ((HnswGraphProvider)
fieldReader).getGraph(fieldInfo.name);
+ testHnswGraph(hnswGraph, fieldInfo.name, status);
+ }
+ }
+ }
+ }
+ }
+ StringJoiner hnswGraphResultJoiner = new StringJoiner(", ");
+ for (Map.Entry<String, Status.HnswGraphStatus> hnswGraphStatus :
+ status.hnswGraphsStatusByField.entrySet()) {
+ hnswGraphResultJoiner.add(
+ String.format(
+ Locale.ROOT,
+ "(field name: %s, levels: %d, total nodes: %d)",
+ hnswGraphStatus.getKey(),
+ hnswGraphStatus.getValue().numLevels,
+ hnswGraphStatus.getValue().totalNumNodes));
+ }
+ msg(
+ infoStream,
+ String.format(
+ Locale.ROOT,
+ "OK [%d fields%s] [took %.3f sec]",
+ status.hnswGraphsStatusByField.size(),
+ hnswGraphResultJoiner.toString().isEmpty() ? "" : ": " +
hnswGraphResultJoiner,
+ nsToSec(System.nanoTime() - startNS)));
+ } catch (Throwable e) {
+ if (failFast) {
+ throw IOUtils.rethrowAlways(e);
+ }
+ msg(infoStream, "ERROR: " + e);
+ status.error = e;
+ if (infoStream != null) {
+ e.printStackTrace(infoStream);
+ }
+ }
+
+ return status;
+ }
+
+ private static void testHnswGraph(
+ HnswGraph hnswGraph, String fieldName, Status.HnswGraphsStatus status)
+ throws IOException, CheckIndexException {
+ if (hnswGraph != null) {
+ status.hnswGraphsStatusByField.put(fieldName, new
Status.HnswGraphStatus());
+ final int numLevels = hnswGraph.numLevels();
+ // Perform tests on each level of the HNSW graph
+ for (int level = numLevels - 1; level >= 0; level--) {
+ HnswGraph.NodesIterator nodesIterator =
hnswGraph.getNodesOnLevel(level);
+ while (nodesIterator.hasNext()) {
+ int node = nodesIterator.nextInt();
+ hnswGraph.seek(level, node);
+ int nbr, lastNeighbor = -1, firstNeighbor = -1;
+ while ((nbr = hnswGraph.nextNeighbor()) != NO_MORE_DOCS) {
+ if (firstNeighbor == -1) {
Review Comment:
Sounds good, I'll add a TODO (and add those checks in a followup PR). A
couple questions first:
- Doesn't (neighbors are in order) and (neighbors are not repeated) verify
uniqueness?
- Also, why wouldn't we assert neighbors are on this level for level 0?
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: issues-unsubscr...@lucene.apache.org
For queries about this service, please contact Infrastructure at:
us...@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscr...@lucene.apache.org
For additional commands, e-mail: issues-h...@lucene.apache.org
