This is an automated email from the ASF dual-hosted git repository.
Jackie-Jiang pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/pinot.git
The following commit(s) were added to refs/heads/master by this push:
new 1216618848e Fix FSDirectory resource leak when IndexWriter
construction fails in HNSW vector index (#18964)
1216618848e is described below
commit 1216618848ed4fc69e4aa4b514a8479823b87a26
Author: Akanksha kedia <[email protected]>
AuthorDate: Fri Jul 17 03:11:51 2026 +0530
Fix FSDirectory resource leak when IndexWriter construction fails in HNSW
vector index (#18964)
---
.../realtime/impl/vector/MutableVectorIndex.java | 33 ++++++++++++++++++----
.../impl/vector/HnswVectorIndexCreator.java | 17 +++++++++--
2 files changed, 42 insertions(+), 8 deletions(-)
diff --git
a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/realtime/impl/vector/MutableVectorIndex.java
b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/realtime/impl/vector/MutableVectorIndex.java
index 39c05e532f7..b62ea4b81d9 100644
---
a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/realtime/impl/vector/MutableVectorIndex.java
+++
b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/realtime/impl/vector/MutableVectorIndex.java
@@ -91,21 +91,42 @@ public class MutableVectorIndex implements
VectorIndexReader, MutableIndex, Vect
_commitDocs = Long.parseLong(
vectorIndexConfig.getProperties().getOrDefault("commitDocs",
String.valueOf(DEFAULT_COMMIT_DOCS)));
_vectorSimilarityFunction =
VectorIndexUtils.toSimilarityFunction(vectorIndexConfig.getVectorDistanceFunction());
+ // Use local variables so that resources opened before a failure can be
closed before rethrowing,
+ // preventing file-descriptor and temp-directory leaks.
+ File indexDir = new File(FileUtils.getTempDirectory(), segmentName);
+ FSDirectory indexDirectory = null;
+ IndexWriter indexWriter = null;
try {
// segment generation is always in V1 and later we convert (as part of
post creation processing)
// to V3 if segmentVersion is set to V3 in SegmentGeneratorConfig.
- _indexDir = new File(FileUtils.getTempDirectory(), segmentName);
- _indexDirectory = FSDirectory.open(
- new File(_indexDir, _vectorColumn +
V1Constants.Indexes.VECTOR_V912_HNSW_INDEX_FILE_EXTENSION).toPath());
+ indexDirectory = FSDirectory.open(
+ new File(indexDir, _vectorColumn +
V1Constants.Indexes.VECTOR_V912_HNSW_INDEX_FILE_EXTENSION).toPath());
LOGGER.info("Creating mutable HNSW index for segment: {}, column: {} at
path: {} with {}", segmentName,
- vectorColumn, _indexDir.getAbsolutePath(),
vectorIndexConfig.getProperties());
- _indexWriter = new IndexWriter(_indexDirectory,
VectorIndexUtils.getIndexWriterConfig(vectorIndexConfig));
- _indexWriter.commit();
+ vectorColumn, indexDir.getAbsolutePath(),
vectorIndexConfig.getProperties());
+ indexWriter = new IndexWriter(indexDirectory,
VectorIndexUtils.getIndexWriterConfig(vectorIndexConfig));
+ indexWriter.commit();
_lastCommitTime = System.currentTimeMillis();
} catch (Exception e) {
+ if (indexWriter != null) {
+ try {
+ indexWriter.close();
+ } catch (IOException closeEx) {
+ e.addSuppressed(closeEx);
+ }
+ } else if (indexDirectory != null) {
+ try {
+ indexDirectory.close();
+ } catch (IOException closeEx) {
+ e.addSuppressed(closeEx);
+ }
+ }
+ FileUtils.deleteQuietly(indexDir);
throw new RuntimeException(
"Caught exception while instantiating the LuceneTextIndexCreator for
column: " + vectorColumn, e);
}
+ _indexDir = indexDir;
+ _indexDirectory = indexDirectory;
+ _indexWriter = indexWriter;
}
@Override
diff --git
a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/creator/impl/vector/HnswVectorIndexCreator.java
b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/creator/impl/vector/HnswVectorIndexCreator.java
index 9c34995060e..dd58ccc47e9 100644
---
a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/creator/impl/vector/HnswVectorIndexCreator.java
+++
b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/creator/impl/vector/HnswVectorIndexCreator.java
@@ -65,20 +65,33 @@ public class HnswVectorIndexCreator implements
VectorIndexCreator {
_vectorSimilarityFunction =
VectorIndexUtils.toSimilarityFunction(vectorIndexConfig.getVectorDistanceFunction());
_storeInSegmentFile = vectorIndexConfig.isStoreInSegmentFile();
_segmentIndexDir = segmentIndexDir;
+ // Use local variables so that, if IndexWriter construction fails after
FSDirectory is already
+ // open, we can close the directory before rethrowing — preventing a
file-descriptor leak.
+ Directory indexDirectory = null;
+ IndexWriter indexWriter;
try {
// segment generation is always in V1 and later we convert (as part of
post creation processing)
// to V3 if segmentVersion is set to V3 in SegmentGeneratorConfig.
File indexFile = new File(segmentIndexDir, _vectorColumn
+ V1Constants.Indexes.VECTOR_V912_HNSW_INDEX_FILE_EXTENSION);
_hnswIndexDir = indexFile;
- _indexDirectory = FSDirectory.open(indexFile.toPath());
+ indexDirectory = FSDirectory.open(indexFile.toPath());
LOGGER.info("Creating HNSW index for column: {} at path: {} with {} for
segment: {}", column,
indexFile.getAbsolutePath(), vectorIndexConfig.getProperties(),
segmentIndexDir.getAbsolutePath());
- _indexWriter = new IndexWriter(_indexDirectory,
VectorIndexUtils.getIndexWriterConfig(vectorIndexConfig));
+ indexWriter = new IndexWriter(indexDirectory,
VectorIndexUtils.getIndexWriterConfig(vectorIndexConfig));
} catch (Exception e) {
+ if (indexDirectory != null) {
+ try {
+ indexDirectory.close();
+ } catch (IOException closeEx) {
+ e.addSuppressed(closeEx);
+ }
+ }
throw new RuntimeException(
"Caught exception while instantiating the HnswVectorIndexCreator for
column: " + column, e);
}
+ _indexDirectory = indexDirectory;
+ _indexWriter = indexWriter;
}
@Override
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
