[pinot] branch master updated: Support Off Heap for Native Text Indices (#10842)

[atri]

This is an automated email from the ASF dual-hosted git repository.

atri pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/pinot.git


The following commit(s) were added to refs/heads/master by this push:
     new f8766129f0 Support Off Heap for Native Text Indices (#10842)
f8766129f0 is described below

commit f8766129f06f29894b7e7c2dc1fed3b36af75d16
Author: Atri Sharma <atri.j...@gmail.com>
AuthorDate: Tue Jun 6 11:11:53 2023 +0530

    Support Off Heap for Native Text Indices (#10842)
    
    This commit allows native text indices to be loaded off heap.
---
 .../local/segment/index/readers/text/NativeTextIndexReader.java    | 7 ++++++-
 1 file changed, 6 insertions(+), 1 deletion(-)

diff --git 
a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/index/readers/text/NativeTextIndexReader.java
 
b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/index/readers/text/NativeTextIndexReader.java
index 8f1c2e6a73..3650e3531f 100644
--- 
a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/index/readers/text/NativeTextIndexReader.java
+++ 
b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/index/readers/text/NativeTextIndexReader.java
@@ -52,7 +52,11 @@ public class NativeTextIndexReader implements 
TextIndexReader {
   public NativeTextIndexReader(String column, File indexDir) {
     _column = column;
     try {
-      _buffer = PinotDataBuffer.loadBigEndianFile(getTextIndexFile(indexDir));
+      String desc = "Native text index buffer: " + column;
+      File indexFile = getTextIndexFile(indexDir);
+      //TODO: Pass the load mode in (Direct, MMap)
+      _buffer =
+          PinotDataBuffer.mapFile(indexFile, /* readOnly */ true, 0, 
indexFile.length(), ByteOrder.BIG_ENDIAN, desc);
       populateIndexes();
     } catch (Exception e) {
       LOGGER.error("Failed to instantiate Lucene text index reader for column 
{}, exception {}", column,
@@ -115,5 +119,6 @@ public class NativeTextIndexReader implements 
TextIndexReader {
   @Override
   public void close()
       throws IOException {
+    _buffer.close();
   }
 }


---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscr...@pinot.apache.org
For additional commands, e-mail: commits-h...@pinot.apache.org