This is an automated email from the ASF dual-hosted git repository.
JingsongLi pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/paimon.git
The following commit(s) were added to refs/heads/master by this push:
new 8b16141ab1 [lumina] Release the vector file when FileBackedDataset
fails to construct (#9481)
8b16141ab1 is described below
commit 8b16141ab13cf42b67c570c81325a333a96ae759
Author: ZIHAN DAI <[email protected]>
AuthorDate: Mon Sep 14 10:43:48 2026 +1000
[lumina] Release the vector file when FileBackedDataset fails to construct
(#9481)
---
.../index/LuminaVectorGlobalIndexWriter.java | 28 +++++----
.../index/LuminaFileBackedDatasetCloseTest.java | 69 ++++++++++++++++++++++
2 files changed, 87 insertions(+), 10 deletions(-)
diff --git
a/paimon-lumina/src/main/java/org/apache/paimon/lumina/index/LuminaVectorGlobalIndexWriter.java
b/paimon-lumina/src/main/java/org/apache/paimon/lumina/index/LuminaVectorGlobalIndexWriter.java
index 9d9ebeaad8..0613405f46 100644
---
a/paimon-lumina/src/main/java/org/apache/paimon/lumina/index/LuminaVectorGlobalIndexWriter.java
+++
b/paimon-lumina/src/main/java/org/apache/paimon/lumina/index/LuminaVectorGlobalIndexWriter.java
@@ -28,6 +28,7 @@ import org.apache.paimon.types.ArrayType;
import org.apache.paimon.types.DataType;
import org.apache.paimon.types.FloatType;
import org.apache.paimon.types.VectorType;
+import org.apache.paimon.utils.IOUtils;
import org.aliyun.lumina.LuminaDataset;
import org.aliyun.lumina.LuminaFileOutput;
@@ -427,16 +428,23 @@ public class LuminaVectorGlobalIndexWriter implements
GlobalIndexSingleColumnWri
FileBackedDataset(File file, int dim, long totalCount, String phase,
int bufferSize)
throws IOException {
this.raf = new RandomAccessFile(file, "r");
- this.channel = raf.getChannel();
- this.dim = dim;
- this.totalCount = totalCount;
- this.recordSizeInBytes = checkedRecordSize(dim, bufferSize);
- this.cursor = 0;
- this.readBuf = ByteBuffer.allocateDirect(bufferSize);
- this.readBuf.order(ByteOrder.nativeOrder());
- this.readBuf.limit(0); // empty initially
- this.phase = phase;
- this.lastLoggedPercent = -1;
+ try {
+ this.channel = raf.getChannel();
+ this.dim = dim;
+ this.totalCount = totalCount;
+ this.recordSizeInBytes = checkedRecordSize(dim, bufferSize);
+ this.cursor = 0;
+ this.readBuf = ByteBuffer.allocateDirect(bufferSize);
+ this.readBuf.order(ByteOrder.nativeOrder());
+ this.readBuf.limit(0); // empty initially
+ this.phase = phase;
+ this.lastLoggedPercent = -1;
+ } catch (RuntimeException | Error e) {
+ // a caller's try-with-resources never sees an object whose
construction failed,
+ // so this is the only chance to release the file handle
+ IOUtils.closeQuietly(raf);
+ throw e;
+ }
}
@Override
diff --git
a/paimon-lumina/src/test/java/org/apache/paimon/lumina/index/LuminaFileBackedDatasetCloseTest.java
b/paimon-lumina/src/test/java/org/apache/paimon/lumina/index/LuminaFileBackedDatasetCloseTest.java
new file mode 100644
index 0000000000..a51874fc7c
--- /dev/null
+++
b/paimon-lumina/src/test/java/org/apache/paimon/lumina/index/LuminaFileBackedDatasetCloseTest.java
@@ -0,0 +1,69 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.paimon.lumina.index;
+
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.io.TempDir;
+
+import java.io.File;
+import java.io.FileOutputStream;
+import java.nio.file.Path;
+
+import static org.assertj.core.api.Assertions.assertThat;
+import static org.assertj.core.api.Assertions.assertThatThrownBy;
+import static org.junit.jupiter.api.Assumptions.assumeTrue;
+
+/**
+ * Tests that {@link LuminaVectorGlobalIndexWriter.FileBackedDataset} releases
the file it opened
+ * when the rest of its construction fails.
+ */
+public class LuminaFileBackedDatasetCloseTest {
+
+ @TempDir Path tempDir;
+
+ /**
+ * checkedRecordSize rejects a dimension whose record does not fit the
read buffer, and it runs
+ * after the file is open. A caller's try-with-resources never sees an
object whose constructor
+ * threw, so nothing would close that handle.
+ */
+ @Test
+ public void testFailedConstructionReleasesTheFile() throws Exception {
+ File fdDir = new File("/dev/fd");
+ assumeTrue(fdDir.isDirectory() && fdDir.list() != null, "needs
/dev/fd");
+
+ File file = new File(tempDir.toFile(), "vectors.bin");
+ try (FileOutputStream out = new FileOutputStream(file)) {
+ out.write(new byte[64]);
+ }
+
+ int before = fdDir.list().length;
+ for (int i = 0; i < 200; i++) {
+ assertThatThrownBy(
+ () ->
+ new
LuminaVectorGlobalIndexWriter.FileBackedDataset(
+ file, Integer.MAX_VALUE / 4, 1L,
"test", 4096))
+ .isInstanceOf(IllegalStateException.class);
+ }
+ int after = fdDir.list().length;
+
+ assertThat(after - before)
+ .as("200 failed constructions must not strand 200 descriptors")
+ .isLessThan(50);
+ }
+}
