Copilot commented on code in PR #17060:
URL: https://github.com/apache/pinot/pull/17060#discussion_r2457279798
##########
pinot-tools/src/main/resources/examples/minions/batch/baseballStats/baseballStats_offline_table_config_raw_inverted_index.json:
##########
@@ -0,0 +1,49 @@
+{
+ "tableName": "baseballStats",
+ "tableType": "OFFLINE",
+ "segmentsConfig": {
+ "segmentPushType": "APPEND",
+ "segmentAssignmentStrategy": "BalanceNumSegmentAssignmentStrategy",
+ "replication": "1"
+ },
+ "tenants": {
+ },
+ "tableIndexConfig": {
+ "loadMode": "HEAP",
+ "invertedIndexColumns": [
+ "teamID"
Review Comment:
Missing comma separator between array elements. The JSON is malformed and
will fail to parse.
```suggestion
"teamID",
```
##########
pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/index/readers/RawValueBitmapInvertedIndexReader.java:
##########
@@ -0,0 +1,162 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.pinot.segment.local.segment.index.readers;
+
+import java.io.IOException;
+import org.apache.pinot.segment.spi.index.reader.Dictionary;
+import org.apache.pinot.segment.spi.index.reader.InvertedIndexReader;
+import org.apache.pinot.segment.spi.memory.PinotDataBuffer;
+import org.apache.pinot.spi.data.FieldSpec.DataType;
+import org.roaringbitmap.buffer.ImmutableRoaringBitmap;
+import org.roaringbitmap.buffer.MutableRoaringBitmap;
+
+/**
+ * Reader for raw value bitmap inverted index.
+ * File format:
+ * <ul>
+ * <li>Header (36 bytes):</li>
+ * <ul>
+ * <li>version (4 bytes)</li>
+ * <li>cardinality (4 bytes)</li>
+ * <li>maxLength (4 bytes) - for string/bytes data types</li>
+ * <li>dictionaryOffset (8 bytes)</li>
+ * <li>dictionaryLength (8 bytes)</li>
+ * <li>invertedIndexOffset (8 bytes)</li>
+ * <li>invertedIndexLength (8 bytes)</li>
+ * </ul>
+ * <li>Dictionary data</li>
+ * <li>Inverted index data</li>
+ * </ul>
+ */
+public class RawValueBitmapInvertedIndexReader implements
InvertedIndexReader<ImmutableRoaringBitmap> {
+ private static final int VERSION = 1;
+ private static final int HEADER_LENGTH = 36;
+
+ private final PinotDataBuffer _dataBuffer;
+ private final DataType _dataType;
+ private final Dictionary _dictionary;
+ private final BitmapInvertedIndexReader _invertedIndexReader;
+ private boolean _isClosed;
+
+ public RawValueBitmapInvertedIndexReader(PinotDataBuffer dataBuffer,
DataType dataType) throws IOException {
+ _dataBuffer = dataBuffer;
+ _dataType = dataType;
+
+ // Read header
+ int version = _dataBuffer.getInt(0);
+ if (version != VERSION) {
+ throw new IllegalStateException("Unsupported version: " + version);
+ }
+ int cardinality = _dataBuffer.getInt(4);
+ int maxLength = _dataBuffer.getInt(8);
+ long dictionaryOffset = _dataBuffer.getLong(12);
+ long dictionaryLength = _dataBuffer.getLong(20);
+ long invertedIndexOffset = _dataBuffer.getLong(28);
+ long invertedIndexLength = _dataBuffer.getLong(36);
+
+ // Initialize dictionary
+ PinotDataBuffer dictionaryBuffer = _dataBuffer.view(dictionaryOffset,
dictionaryOffset + dictionaryLength);
+ switch (_dataType.getStoredType()) {
+ case INT:
+ _dictionary = new IntDictionary(dictionaryBuffer, cardinality);
+ break;
+ case LONG:
+ _dictionary = new LongDictionary(dictionaryBuffer, cardinality);
+ break;
+ case FLOAT:
+ _dictionary = new FloatDictionary(dictionaryBuffer, cardinality);
+ break;
+ case DOUBLE:
+ _dictionary = new DoubleDictionary(dictionaryBuffer, cardinality);
+ break;
+ case STRING:
+ _dictionary = new StringDictionary(dictionaryBuffer, cardinality,
maxLength);
+ break;
+ case BYTES:
+ _dictionary = new BytesDictionary(dictionaryBuffer, cardinality,
maxLength);
+ break;
+ default:
+ throw new IllegalStateException("Unsupported data type: " + _dataType);
+ }
+
+ // Initialize inverted index
+ PinotDataBuffer invertedIndexBuffer = _dataBuffer.view(invertedIndexOffset,
+ invertedIndexOffset + invertedIndexLength);
+ _invertedIndexReader = new BitmapInvertedIndexReader(invertedIndexBuffer,
cardinality);
+ }
+
+ @Override
+ public ImmutableRoaringBitmap getDocIds(int dictId) {
+ return _invertedIndexReader.getDocIds(dictId);
+ }
+
+ /**
+ * Returns the document IDs for the given raw INT value.
+ */
+ public ImmutableRoaringBitmap getDocIdsForInt(int value) {
+ int dictId = _dictionary.indexOf(value);
+ return dictId >= 0 ? getDocIds(dictId) : new MutableRoaringBitmap();
Review Comment:
Creating new MutableRoaringBitmap instances for non-existent values is
inefficient. Consider using a shared empty ImmutableRoaringBitmap singleton or
caching empty bitmaps to avoid unnecessary object allocation.
##########
pinot-segment-spi/src/main/java/org/apache/pinot/segment/spi/index/creator/RawValueBasedInvertedIndexCreator.java:
##########
@@ -59,4 +59,12 @@ public interface RawValueBasedInvertedIndexCreator extends
InvertedIndexCreator
* For multi-value column, adds the double values for the next document.
*/
void add(double[] values, int length);
+
+ default void add(String value) { }
+
+ default void add(String[] value, int length) { }
+
+ default void add(byte[] value) { }
+
Review Comment:
These new default methods lack javadoc comments explaining their purpose,
parameters, and behavior. Following the pattern of existing methods in this
interface, these should have documentation.
```suggestion
/**
* For single-value column, adds the String value for the next document.
*
* @param value String value to add for the next document
*/
default void add(String value) { }
/**
* For multi-value column, adds the String values for the next document.
*
* @param value Array of String values to add for the next document
* @param length Number of values in the array
*/
default void add(String[] value, int length) { }
/**
* For single-value column, adds the byte array value for the next
document.
*
* @param value Byte array value to add for the next document
*/
default void add(byte[] value) { }
/**
* For multi-value column, adds the byte array values for the next
document.
*
* @param value Array of byte array values to add for the next document
* @param length Number of values in the array
*/
```
##########
pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/index/readers/RawValueBitmapInvertedIndexReader.java:
##########
@@ -0,0 +1,162 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.pinot.segment.local.segment.index.readers;
+
+import java.io.IOException;
+import org.apache.pinot.segment.spi.index.reader.Dictionary;
+import org.apache.pinot.segment.spi.index.reader.InvertedIndexReader;
+import org.apache.pinot.segment.spi.memory.PinotDataBuffer;
+import org.apache.pinot.spi.data.FieldSpec.DataType;
+import org.roaringbitmap.buffer.ImmutableRoaringBitmap;
+import org.roaringbitmap.buffer.MutableRoaringBitmap;
+
+/**
+ * Reader for raw value bitmap inverted index.
+ * File format:
+ * <ul>
+ * <li>Header (36 bytes):</li>
+ * <ul>
+ * <li>version (4 bytes)</li>
+ * <li>cardinality (4 bytes)</li>
+ * <li>maxLength (4 bytes) - for string/bytes data types</li>
+ * <li>dictionaryOffset (8 bytes)</li>
+ * <li>dictionaryLength (8 bytes)</li>
+ * <li>invertedIndexOffset (8 bytes)</li>
+ * <li>invertedIndexLength (8 bytes)</li>
+ * </ul>
+ * <li>Dictionary data</li>
+ * <li>Inverted index data</li>
+ * </ul>
+ */
+public class RawValueBitmapInvertedIndexReader implements
InvertedIndexReader<ImmutableRoaringBitmap> {
+ private static final int VERSION = 1;
+ private static final int HEADER_LENGTH = 36;
+
+ private final PinotDataBuffer _dataBuffer;
+ private final DataType _dataType;
+ private final Dictionary _dictionary;
+ private final BitmapInvertedIndexReader _invertedIndexReader;
+ private boolean _isClosed;
+
+ public RawValueBitmapInvertedIndexReader(PinotDataBuffer dataBuffer,
DataType dataType) throws IOException {
+ _dataBuffer = dataBuffer;
+ _dataType = dataType;
+
+ // Read header
+ int version = _dataBuffer.getInt(0);
+ if (version != VERSION) {
+ throw new IllegalStateException("Unsupported version: " + version);
+ }
+ int cardinality = _dataBuffer.getInt(4);
+ int maxLength = _dataBuffer.getInt(8);
+ long dictionaryOffset = _dataBuffer.getLong(12);
+ long dictionaryLength = _dataBuffer.getLong(20);
+ long invertedIndexOffset = _dataBuffer.getLong(28);
+ long invertedIndexLength = _dataBuffer.getLong(36);
Review Comment:
Incorrect offset for reading invertedIndexLength. According to the header
format, this field starts at byte 28 (after dictionaryOffset at 12 +
dictionaryLength at 20), but it's being read from offset 36. The offset should
be 28.
```suggestion
long invertedIndexLength = _dataBuffer.getLong(28);
```
##########
pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/index/loader/invertedindex/InvertedIndexHandler.java:
##########
@@ -135,35 +136,86 @@ private void
createInvertedIndexForColumn(SegmentDirectory.Writer segmentWriter,
// Create new inverted index for the column.
LOGGER.info("Creating new inverted index for segment: {}, column: {}",
segmentName, columnName);
int numDocs = columnMetadata.getTotalDocs();
-
IndexCreationContext.Common context = IndexCreationContext.builder()
.withIndexDir(indexDir)
.withColumnMetadata(columnMetadata)
.withTableNameWithType(_tableConfig.getTableName())
.withContinueOnError(_tableConfig.getIngestionConfig() != null
&& _tableConfig.getIngestionConfig().isContinueOnError())
.build();
-
- try (DictionaryBasedInvertedIndexCreator creator =
StandardIndexes.inverted()
- .createIndexCreator(context, IndexConfig.ENABLED)) {
- IndexReaderFactory<ForwardIndexReader> readerFactory =
StandardIndexes.forward().getReaderFactory();
- try (ForwardIndexReader forwardIndexReader =
readerFactory.createIndexReader(segmentWriter,
- _fieldIndexConfigs.get(columnMetadata.getColumnName()),
columnMetadata);
+ if (columnMetadata.hasDictionary()) {
+ // Dictionary-based inverted index
+
+
+ try (DictionaryBasedInvertedIndexCreator creator =
StandardIndexes.inverted()
+ .createIndexCreator(context, IndexConfig.ENABLED)) {
+
+ try (
+ ForwardIndexReader forwardIndexReader = StandardIndexes.forward()
+ .getReaderFactory()
+ .createIndexReader(segmentWriter,
_fieldIndexConfigs.get(columnName), columnMetadata);
+ ForwardIndexReaderContext readerContext =
forwardIndexReader.createContext()) {
+ if (columnMetadata.isSingleValue()) {
+ // Single-value column.
+ for (int i = 0; i < numDocs; i++) {
+ creator.add(forwardIndexReader.getDictId(i, readerContext));
+ }
+ } else {
+ // Multi-value column.
+ int[] dictIds = new
int[columnMetadata.getMaxNumberOfMultiValues()];
+ for (int i = 0; i < numDocs; i++) {
+ int length = forwardIndexReader.getDictIdMV(i, dictIds,
readerContext);
+ creator.add(dictIds, length);
+ }
+ }
+ creator.seal();
+ }
+ }
+ } else {
+ // Raw value based inverted index
+ IndexReaderFactory<ForwardIndexReader> readerFactory =
StandardIndexes.forward()
+ .getReaderFactory();
+ try (RawValueBitmapInvertedIndexCreator creator = new
RawValueBitmapInvertedIndexCreator(context);
+ ForwardIndexReader forwardIndexReader = readerFactory
+ .createIndexReader(segmentWriter,
_fieldIndexConfigs.get(columnName), columnMetadata);
ForwardIndexReaderContext readerContext =
forwardIndexReader.createContext()) {
if (columnMetadata.isSingleValue()) {
- // Single-value column.
- for (int i = 0; i < numDocs; i++) {
- creator.add(forwardIndexReader.getDictId(i, readerContext));
+ // Single-value column
+ switch (columnMetadata.getDataType()) {
+ case INT:
+ for (int i = 0; i < numDocs; i++) {
+ creator.add(forwardIndexReader.getInt(i, readerContext));
+ }
+ break;
+ case LONG:
+ for (int i = 0; i < numDocs; i++) {
+ creator.add(forwardIndexReader.getLong(i, readerContext));
+ }
+ break;
+ case FLOAT:
+ for (int i = 0; i < numDocs; i++) {
+ creator.add(forwardIndexReader.getFloat(i, readerContext));
+ }
+ break;
+ case DOUBLE:
+ for (int i = 0; i < numDocs; i++) {
+ creator.add(forwardIndexReader.getDouble(i, readerContext));
+ }
+ break;
+ case STRING:
+ for (int i = 0; i < numDocs; i++) {
+ creator.add(forwardIndexReader.getString(i, readerContext));
+ }
+ break;
+ default:
+ throw new IllegalStateException(
+ "Unsupported data type for raw inverted index: " +
columnMetadata.getDataType());
}
} else {
- // Multi-value column.
- int[] dictIds = new int[columnMetadata.getMaxNumberOfMultiValues()];
- for (int i = 0; i < numDocs; i++) {
- int length = forwardIndexReader.getDictIdMV(i, dictIds,
readerContext);
- creator.add(dictIds, length);
- }
+ // Multi-value columns not supported for raw inverted index
+ throw new IllegalStateException("Raw inverted index not supported
for multi-value columns: " + columnName);
}
- creator.seal();
+ creator.close();
Review Comment:
Calling close() explicitly inside a try-with-resources block is redundant
and can cause double-close errors. The try-with-resources statement will
automatically call close() when exiting the block. Remove this explicit close()
call.
```suggestion
```
##########
pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/index/readers/RawValueBitmapInvertedIndexReader.java:
##########
@@ -0,0 +1,162 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.pinot.segment.local.segment.index.readers;
+
+import java.io.IOException;
+import org.apache.pinot.segment.spi.index.reader.Dictionary;
+import org.apache.pinot.segment.spi.index.reader.InvertedIndexReader;
+import org.apache.pinot.segment.spi.memory.PinotDataBuffer;
+import org.apache.pinot.spi.data.FieldSpec.DataType;
+import org.roaringbitmap.buffer.ImmutableRoaringBitmap;
+import org.roaringbitmap.buffer.MutableRoaringBitmap;
+
+/**
+ * Reader for raw value bitmap inverted index.
+ * File format:
+ * <ul>
+ * <li>Header (36 bytes):</li>
+ * <ul>
+ * <li>version (4 bytes)</li>
+ * <li>cardinality (4 bytes)</li>
+ * <li>maxLength (4 bytes) - for string/bytes data types</li>
+ * <li>dictionaryOffset (8 bytes)</li>
+ * <li>dictionaryLength (8 bytes)</li>
+ * <li>invertedIndexOffset (8 bytes)</li>
+ * <li>invertedIndexLength (8 bytes)</li>
+ * </ul>
+ * <li>Dictionary data</li>
+ * <li>Inverted index data</li>
+ * </ul>
+ */
+public class RawValueBitmapInvertedIndexReader implements
InvertedIndexReader<ImmutableRoaringBitmap> {
+ private static final int VERSION = 1;
+ private static final int HEADER_LENGTH = 36;
+
+ private final PinotDataBuffer _dataBuffer;
+ private final DataType _dataType;
+ private final Dictionary _dictionary;
+ private final BitmapInvertedIndexReader _invertedIndexReader;
+ private boolean _isClosed;
+
+ public RawValueBitmapInvertedIndexReader(PinotDataBuffer dataBuffer,
DataType dataType) throws IOException {
+ _dataBuffer = dataBuffer;
+ _dataType = dataType;
+
+ // Read header
+ int version = _dataBuffer.getInt(0);
+ if (version != VERSION) {
+ throw new IllegalStateException("Unsupported version: " + version);
+ }
+ int cardinality = _dataBuffer.getInt(4);
+ int maxLength = _dataBuffer.getInt(8);
+ long dictionaryOffset = _dataBuffer.getLong(12);
+ long dictionaryLength = _dataBuffer.getLong(20);
+ long invertedIndexOffset = _dataBuffer.getLong(28);
+ long invertedIndexLength = _dataBuffer.getLong(36);
+
+ // Initialize dictionary
+ PinotDataBuffer dictionaryBuffer = _dataBuffer.view(dictionaryOffset,
dictionaryOffset + dictionaryLength);
+ switch (_dataType.getStoredType()) {
+ case INT:
+ _dictionary = new IntDictionary(dictionaryBuffer, cardinality);
+ break;
+ case LONG:
+ _dictionary = new LongDictionary(dictionaryBuffer, cardinality);
+ break;
+ case FLOAT:
+ _dictionary = new FloatDictionary(dictionaryBuffer, cardinality);
+ break;
+ case DOUBLE:
+ _dictionary = new DoubleDictionary(dictionaryBuffer, cardinality);
+ break;
+ case STRING:
+ _dictionary = new StringDictionary(dictionaryBuffer, cardinality,
maxLength);
+ break;
+ case BYTES:
+ _dictionary = new BytesDictionary(dictionaryBuffer, cardinality,
maxLength);
+ break;
+ default:
+ throw new IllegalStateException("Unsupported data type: " + _dataType);
+ }
+
+ // Initialize inverted index
+ PinotDataBuffer invertedIndexBuffer = _dataBuffer.view(invertedIndexOffset,
+ invertedIndexOffset + invertedIndexLength);
+ _invertedIndexReader = new BitmapInvertedIndexReader(invertedIndexBuffer,
cardinality);
+ }
+
+ @Override
+ public ImmutableRoaringBitmap getDocIds(int dictId) {
+ return _invertedIndexReader.getDocIds(dictId);
+ }
+
+ /**
+ * Returns the document IDs for the given raw INT value.
+ */
+ public ImmutableRoaringBitmap getDocIdsForInt(int value) {
+ int dictId = _dictionary.indexOf(value);
+ return dictId >= 0 ? getDocIds(dictId) : new MutableRoaringBitmap();
+ }
+
+ /**
+ * Returns the document IDs for the given raw LONG value.
+ */
+ public ImmutableRoaringBitmap getDocIdsForLong(long value) {
+ int dictId = _dictionary.indexOf(value);
+ return dictId >= 0 ? getDocIds(dictId) : new MutableRoaringBitmap();
+ }
+
+ /**
+ * Returns the document IDs for the given raw FLOAT value.
+ */
+ public ImmutableRoaringBitmap getDocIdsForFloat(float value) {
+ int dictId = _dictionary.indexOf(value);
+ return dictId >= 0 ? getDocIds(dictId) : new MutableRoaringBitmap();
+ }
+
+ /**
+ * Returns the document IDs for the given raw DOUBLE value.
+ */
+ public ImmutableRoaringBitmap getDocIdsForDouble(double value) {
+ int dictId = _dictionary.indexOf(value);
+ return dictId >= 0 ? getDocIds(dictId) : new MutableRoaringBitmap();
Review Comment:
Creating new MutableRoaringBitmap instances for non-existent values is
inefficient. Consider using a shared empty ImmutableRoaringBitmap singleton or
caching empty bitmaps to avoid unnecessary object allocation.
##########
pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/creator/impl/inv/RawValueBitmapInvertedIndexCreator.java:
##########
@@ -0,0 +1,334 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.pinot.segment.local.segment.creator.impl.inv;
+
+import java.io.File;
+import java.io.FileOutputStream;
+import java.io.IOException;
+import java.io.RandomAccessFile;
+import java.nio.ByteBuffer;
+import java.nio.ByteOrder;
+import java.nio.channels.FileChannel;
+import java.util.TreeMap;
+import javax.annotation.Nullable;
+import org.apache.commons.io.FileUtils;
+import
org.apache.pinot.segment.local.segment.creator.impl.SegmentDictionaryCreator;
+import org.apache.pinot.segment.spi.V1Constants;
+import org.apache.pinot.segment.spi.creator.IndexCreationContext;
+import
org.apache.pinot.segment.spi.index.creator.RawValueBasedInvertedIndexCreator;
+import org.apache.pinot.segment.spi.memory.PinotDataBuffer;
+import org.apache.pinot.spi.data.DimensionFieldSpec;
+import org.apache.pinot.spi.data.FieldSpec;
+import org.apache.pinot.spi.data.FieldSpec.DataType;
+import org.roaringbitmap.buffer.MutableRoaringBitmap;
+
+/**
+ * Creator for raw value bitmap inverted index.
+ * File format:
+ * <ul>
+ * <li>Header (44 bytes):</li>
+ * <ul>
+ * <li>version (4 bytes)</li>
+ * <li>cardinality (4 bytes)</li>
+ * <li>maxLength (4 bytes) - for string/bytes data types</li>
+ * <li>dictionaryOffset (8 bytes)</li>
+ * <li>dictionaryLength (8 bytes)</li>
+ * <li>invertedIndexOffset (8 bytes)</li>
+ * <li>invertedIndexLength (8 bytes)</li>
+ * </ul>
+ * <li>Dictionary data</li>
+ * <li>Inverted index data</li>
+ * </ul>
+ */
+public class RawValueBitmapInvertedIndexCreator implements
RawValueBasedInvertedIndexCreator {
+ private static final int VERSION = 1;
+ private static final int HEADER_LENGTH = 44;
Review Comment:
Header length mismatch with reader. The creator defines HEADER_LENGTH as 44
bytes but the reader (RawValueBitmapInvertedIndexReader) defines it as 36
bytes. These constants must match for proper serialization/deserialization.
##########
pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/index/readers/RawValueBitmapInvertedIndexReader.java:
##########
@@ -0,0 +1,162 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.pinot.segment.local.segment.index.readers;
+
+import java.io.IOException;
+import org.apache.pinot.segment.spi.index.reader.Dictionary;
+import org.apache.pinot.segment.spi.index.reader.InvertedIndexReader;
+import org.apache.pinot.segment.spi.memory.PinotDataBuffer;
+import org.apache.pinot.spi.data.FieldSpec.DataType;
+import org.roaringbitmap.buffer.ImmutableRoaringBitmap;
+import org.roaringbitmap.buffer.MutableRoaringBitmap;
+
+/**
+ * Reader for raw value bitmap inverted index.
+ * File format:
+ * <ul>
+ * <li>Header (36 bytes):</li>
+ * <ul>
+ * <li>version (4 bytes)</li>
+ * <li>cardinality (4 bytes)</li>
+ * <li>maxLength (4 bytes) - for string/bytes data types</li>
+ * <li>dictionaryOffset (8 bytes)</li>
+ * <li>dictionaryLength (8 bytes)</li>
+ * <li>invertedIndexOffset (8 bytes)</li>
+ * <li>invertedIndexLength (8 bytes)</li>
+ * </ul>
+ * <li>Dictionary data</li>
+ * <li>Inverted index data</li>
+ * </ul>
+ */
+public class RawValueBitmapInvertedIndexReader implements
InvertedIndexReader<ImmutableRoaringBitmap> {
+ private static final int VERSION = 1;
+ private static final int HEADER_LENGTH = 36;
+
+ private final PinotDataBuffer _dataBuffer;
+ private final DataType _dataType;
+ private final Dictionary _dictionary;
+ private final BitmapInvertedIndexReader _invertedIndexReader;
+ private boolean _isClosed;
+
+ public RawValueBitmapInvertedIndexReader(PinotDataBuffer dataBuffer,
DataType dataType) throws IOException {
+ _dataBuffer = dataBuffer;
+ _dataType = dataType;
+
+ // Read header
+ int version = _dataBuffer.getInt(0);
+ if (version != VERSION) {
+ throw new IllegalStateException("Unsupported version: " + version);
+ }
+ int cardinality = _dataBuffer.getInt(4);
+ int maxLength = _dataBuffer.getInt(8);
+ long dictionaryOffset = _dataBuffer.getLong(12);
+ long dictionaryLength = _dataBuffer.getLong(20);
+ long invertedIndexOffset = _dataBuffer.getLong(28);
+ long invertedIndexLength = _dataBuffer.getLong(36);
+
+ // Initialize dictionary
+ PinotDataBuffer dictionaryBuffer = _dataBuffer.view(dictionaryOffset,
dictionaryOffset + dictionaryLength);
+ switch (_dataType.getStoredType()) {
+ case INT:
+ _dictionary = new IntDictionary(dictionaryBuffer, cardinality);
+ break;
+ case LONG:
+ _dictionary = new LongDictionary(dictionaryBuffer, cardinality);
+ break;
+ case FLOAT:
+ _dictionary = new FloatDictionary(dictionaryBuffer, cardinality);
+ break;
+ case DOUBLE:
+ _dictionary = new DoubleDictionary(dictionaryBuffer, cardinality);
+ break;
+ case STRING:
+ _dictionary = new StringDictionary(dictionaryBuffer, cardinality,
maxLength);
+ break;
+ case BYTES:
+ _dictionary = new BytesDictionary(dictionaryBuffer, cardinality,
maxLength);
+ break;
+ default:
+ throw new IllegalStateException("Unsupported data type: " + _dataType);
+ }
+
+ // Initialize inverted index
+ PinotDataBuffer invertedIndexBuffer = _dataBuffer.view(invertedIndexOffset,
+ invertedIndexOffset + invertedIndexLength);
+ _invertedIndexReader = new BitmapInvertedIndexReader(invertedIndexBuffer,
cardinality);
+ }
+
+ @Override
+ public ImmutableRoaringBitmap getDocIds(int dictId) {
+ return _invertedIndexReader.getDocIds(dictId);
+ }
+
+ /**
+ * Returns the document IDs for the given raw INT value.
+ */
+ public ImmutableRoaringBitmap getDocIdsForInt(int value) {
+ int dictId = _dictionary.indexOf(value);
+ return dictId >= 0 ? getDocIds(dictId) : new MutableRoaringBitmap();
+ }
+
+ /**
+ * Returns the document IDs for the given raw LONG value.
+ */
+ public ImmutableRoaringBitmap getDocIdsForLong(long value) {
+ int dictId = _dictionary.indexOf(value);
+ return dictId >= 0 ? getDocIds(dictId) : new MutableRoaringBitmap();
Review Comment:
Creating new MutableRoaringBitmap instances for non-existent values is
inefficient. Consider using a shared empty ImmutableRoaringBitmap singleton or
caching empty bitmaps to avoid unnecessary object allocation.
##########
pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/index/readers/RawValueBitmapInvertedIndexReader.java:
##########
@@ -0,0 +1,162 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.pinot.segment.local.segment.index.readers;
+
+import java.io.IOException;
+import org.apache.pinot.segment.spi.index.reader.Dictionary;
+import org.apache.pinot.segment.spi.index.reader.InvertedIndexReader;
+import org.apache.pinot.segment.spi.memory.PinotDataBuffer;
+import org.apache.pinot.spi.data.FieldSpec.DataType;
+import org.roaringbitmap.buffer.ImmutableRoaringBitmap;
+import org.roaringbitmap.buffer.MutableRoaringBitmap;
+
+/**
+ * Reader for raw value bitmap inverted index.
+ * File format:
+ * <ul>
+ * <li>Header (36 bytes):</li>
+ * <ul>
+ * <li>version (4 bytes)</li>
+ * <li>cardinality (4 bytes)</li>
+ * <li>maxLength (4 bytes) - for string/bytes data types</li>
+ * <li>dictionaryOffset (8 bytes)</li>
+ * <li>dictionaryLength (8 bytes)</li>
+ * <li>invertedIndexOffset (8 bytes)</li>
+ * <li>invertedIndexLength (8 bytes)</li>
+ * </ul>
+ * <li>Dictionary data</li>
+ * <li>Inverted index data</li>
+ * </ul>
+ */
+public class RawValueBitmapInvertedIndexReader implements
InvertedIndexReader<ImmutableRoaringBitmap> {
+ private static final int VERSION = 1;
+ private static final int HEADER_LENGTH = 36;
+
+ private final PinotDataBuffer _dataBuffer;
+ private final DataType _dataType;
+ private final Dictionary _dictionary;
+ private final BitmapInvertedIndexReader _invertedIndexReader;
+ private boolean _isClosed;
+
+ public RawValueBitmapInvertedIndexReader(PinotDataBuffer dataBuffer,
DataType dataType) throws IOException {
+ _dataBuffer = dataBuffer;
+ _dataType = dataType;
+
+ // Read header
+ int version = _dataBuffer.getInt(0);
+ if (version != VERSION) {
+ throw new IllegalStateException("Unsupported version: " + version);
+ }
+ int cardinality = _dataBuffer.getInt(4);
+ int maxLength = _dataBuffer.getInt(8);
+ long dictionaryOffset = _dataBuffer.getLong(12);
+ long dictionaryLength = _dataBuffer.getLong(20);
+ long invertedIndexOffset = _dataBuffer.getLong(28);
+ long invertedIndexLength = _dataBuffer.getLong(36);
+
+ // Initialize dictionary
+ PinotDataBuffer dictionaryBuffer = _dataBuffer.view(dictionaryOffset,
dictionaryOffset + dictionaryLength);
+ switch (_dataType.getStoredType()) {
+ case INT:
+ _dictionary = new IntDictionary(dictionaryBuffer, cardinality);
+ break;
+ case LONG:
+ _dictionary = new LongDictionary(dictionaryBuffer, cardinality);
+ break;
+ case FLOAT:
+ _dictionary = new FloatDictionary(dictionaryBuffer, cardinality);
+ break;
+ case DOUBLE:
+ _dictionary = new DoubleDictionary(dictionaryBuffer, cardinality);
+ break;
+ case STRING:
+ _dictionary = new StringDictionary(dictionaryBuffer, cardinality,
maxLength);
+ break;
+ case BYTES:
+ _dictionary = new BytesDictionary(dictionaryBuffer, cardinality,
maxLength);
+ break;
+ default:
+ throw new IllegalStateException("Unsupported data type: " + _dataType);
+ }
+
+ // Initialize inverted index
+ PinotDataBuffer invertedIndexBuffer = _dataBuffer.view(invertedIndexOffset,
+ invertedIndexOffset + invertedIndexLength);
+ _invertedIndexReader = new BitmapInvertedIndexReader(invertedIndexBuffer,
cardinality);
+ }
+
+ @Override
+ public ImmutableRoaringBitmap getDocIds(int dictId) {
+ return _invertedIndexReader.getDocIds(dictId);
+ }
+
+ /**
+ * Returns the document IDs for the given raw INT value.
+ */
+ public ImmutableRoaringBitmap getDocIdsForInt(int value) {
+ int dictId = _dictionary.indexOf(value);
+ return dictId >= 0 ? getDocIds(dictId) : new MutableRoaringBitmap();
+ }
+
+ /**
+ * Returns the document IDs for the given raw LONG value.
+ */
+ public ImmutableRoaringBitmap getDocIdsForLong(long value) {
+ int dictId = _dictionary.indexOf(value);
+ return dictId >= 0 ? getDocIds(dictId) : new MutableRoaringBitmap();
+ }
+
+ /**
+ * Returns the document IDs for the given raw FLOAT value.
+ */
+ public ImmutableRoaringBitmap getDocIdsForFloat(float value) {
+ int dictId = _dictionary.indexOf(value);
+ return dictId >= 0 ? getDocIds(dictId) : new MutableRoaringBitmap();
Review Comment:
Creating new MutableRoaringBitmap instances for non-existent values is
inefficient. Consider using a shared empty ImmutableRoaringBitmap singleton or
caching empty bitmaps to avoid unnecessary object allocation.
##########
pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/creator/impl/inv/RawValueBitmapInvertedIndexCreator.java:
##########
@@ -0,0 +1,334 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.pinot.segment.local.segment.creator.impl.inv;
+
+import java.io.File;
+import java.io.FileOutputStream;
+import java.io.IOException;
+import java.io.RandomAccessFile;
+import java.nio.ByteBuffer;
+import java.nio.ByteOrder;
+import java.nio.channels.FileChannel;
+import java.util.TreeMap;
+import javax.annotation.Nullable;
+import org.apache.commons.io.FileUtils;
+import
org.apache.pinot.segment.local.segment.creator.impl.SegmentDictionaryCreator;
+import org.apache.pinot.segment.spi.V1Constants;
+import org.apache.pinot.segment.spi.creator.IndexCreationContext;
+import
org.apache.pinot.segment.spi.index.creator.RawValueBasedInvertedIndexCreator;
+import org.apache.pinot.segment.spi.memory.PinotDataBuffer;
+import org.apache.pinot.spi.data.DimensionFieldSpec;
+import org.apache.pinot.spi.data.FieldSpec;
+import org.apache.pinot.spi.data.FieldSpec.DataType;
+import org.roaringbitmap.buffer.MutableRoaringBitmap;
+
+/**
+ * Creator for raw value bitmap inverted index.
+ * File format:
+ * <ul>
+ * <li>Header (44 bytes):</li>
+ * <ul>
+ * <li>version (4 bytes)</li>
+ * <li>cardinality (4 bytes)</li>
+ * <li>maxLength (4 bytes) - for string/bytes data types</li>
+ * <li>dictionaryOffset (8 bytes)</li>
+ * <li>dictionaryLength (8 bytes)</li>
+ * <li>invertedIndexOffset (8 bytes)</li>
+ * <li>invertedIndexLength (8 bytes)</li>
+ * </ul>
+ * <li>Dictionary data</li>
+ * <li>Inverted index data</li>
+ * </ul>
+ */
+public class RawValueBitmapInvertedIndexCreator implements
RawValueBasedInvertedIndexCreator {
+ private static final int VERSION = 1;
+ private static final int HEADER_LENGTH = 44;
+
+ private final File _indexFile;
+ private final DataType _dataType;
+ private final String _columnName;
+ private final TreeMap<Object, MutableRoaringBitmap> _valueToDocIds;
+ private SegmentDictionaryCreator _dictionaryCreator;
+ private int _maxLength; // Only used for STRING and BYTES
+
+ private int _nextDocId = 0;
+ private boolean _isClosed = false;
+
+ public RawValueBitmapInvertedIndexCreator(IndexCreationContext context)
+ throws IOException {
+ this(context.getFieldSpec().getDataType().getStoredType(),
context.getFieldSpec().getName(),
+ context.getIndexDir());
+ }
+
+ public RawValueBitmapInvertedIndexCreator(DataType dataType, String
columnName, File indexDir) {
+ _dataType = dataType;
+ _columnName = columnName;
+ _indexFile = new File(indexDir, columnName +
V1Constants.Indexes.BITMAP_INVERTED_INDEX_FILE_EXTENSION);
+ _valueToDocIds = new TreeMap<>();
+ }
+
+ @Override
+ public void add(int value) {
+ addValue(value);
+ }
+
+ @Override
+ public void add(int[] values, int length) {
+ Integer[] boxedValues = new Integer[length];
+ for (int i = 0; i < length; i++) {
+ boxedValues[i] = values[i];
+ }
+ addValues(boxedValues, length);
+ }
+
+ @Override
+ public void add(long value) {
+ addValue(value);
+ }
+
+ @Override
+ public void add(long[] values, int length) {
+ Long[] boxedValues = new Long[length];
+ for (int i = 0; i < length; i++) {
+ boxedValues[i] = values[i];
+ }
+ addValues(boxedValues, length);
+ }
+
+ @Override
+ public void add(float value) {
+ addValue(value);
+ }
+
+ @Override
+ public void add(float[] values, int length) {
+ Float[] boxedValues = new Float[length];
+ for (int i = 0; i < length; i++) {
+ boxedValues[i] = values[i];
+ }
+ addValues(boxedValues, length);
+ }
+
+ @Override
+ public void add(double value) {
+ addValue(value);
+ }
+
+ @Override
+ public void add(double[] values, int length) {
+ Double[] boxedValues = new Double[length];
+ for (int i = 0; i < length; i++) {
+ boxedValues[i] = values[i];
+ }
+ addValues(boxedValues, length);
+ }
+
+ @Override
+ public void add(String value) {
+ if (value != null) {
+ _maxLength = Math.max(_maxLength, value.length());
+ addValue(value);
+ }
+ }
+
+ @Override
+ public void add(String[] values, int length) {
+ for (int i = 0; i < length; i++) {
+ if (values[i] != null) {
+ _maxLength = Math.max(_maxLength, values[i].length());
+ }
+ }
+ addValues(values, length);
+ }
+
+ @Override
+ public void add(byte[] value) {
+ if (value != null) {
+ _maxLength = Math.max(_maxLength, value.length);
+ addValue(value);
+ }
+ }
+
+ @Override
+ public void add(byte[][] values, int length) {
+ for (int i = 0; i < length; i++) {
+ if (values[i] != null) {
+ _maxLength = Math.max(_maxLength, values[i].length);
+ }
+ }
+ addValues(values, length);
+ }
+ @Override
+ public void add(Object value, int dictId)
+ throws IOException {
+ }
+
+ @Override
+ public void add(Object[] values, @Nullable int[] dictIds)
+ throws IOException {
+ }
+
+ private void addValue(Object value) {
+ if (value != null) {
+ MutableRoaringBitmap bitmap = _valueToDocIds.get(value);
+ if (bitmap == null) {
+ bitmap = new MutableRoaringBitmap();
+ _valueToDocIds.put(value, bitmap);
+ }
+ bitmap.add(_nextDocId);
+ }
+ _nextDocId++;
+ }
+
+ private void addValues(Object[] values, int length) {
+ for (int i = 0; i < length; i++) {
+ if (values[i] != null) {
+ MutableRoaringBitmap bitmap = _valueToDocIds.get(values[i]);
+ if (bitmap == null) {
+ bitmap = new MutableRoaringBitmap();
+ _valueToDocIds.put(values[i], bitmap);
+ }
+ bitmap.add(_nextDocId);
+ }
+ }
+ _nextDocId++;
+ }
+
+ @Override
+ public void seal()
+ throws IOException {
+ if (_isClosed) {
+ return;
+ }
+
+ // Create dictionary from unique values
+ boolean useVarLengthDictionary = _dataType == DataType.STRING;
+ FieldSpec fieldSpec = new DimensionFieldSpec(_columnName, _dataType, true);
+ _dictionaryCreator = new SegmentDictionaryCreator(fieldSpec, new
File(_indexFile.getParent()),
+ useVarLengthDictionary);
+
+ // Convert values to type-specific array
+ Object[] rawValues = _valueToDocIds.keySet().toArray();
+ int numValues = rawValues.length;
+ Object typedValues;
+ switch (_dataType.getStoredType()) {
+ case INT:
+ int[] intValues = new int[numValues];
+ for (int i = 0; i < numValues; i++) {
+ intValues[i] = (Integer) rawValues[i];
+ }
+ typedValues = intValues;
+ break;
+ case LONG:
+ long[] longValues = new long[numValues];
+ for (int i = 0; i < numValues; i++) {
+ longValues[i] = (Long) rawValues[i];
+ }
+ typedValues = longValues;
+ break;
+ case FLOAT:
+ float[] floatValues = new float[numValues];
+ for (int i = 0; i < numValues; i++) {
+ floatValues[i] = (Float) rawValues[i];
+ }
+ typedValues = floatValues;
+ break;
+ case DOUBLE:
+ double[] doubleValues = new double[numValues];
+ for (int i = 0; i < numValues; i++) {
+ doubleValues[i] = (Double) rawValues[i];
+ }
+ typedValues = doubleValues;
+ break;
+ case STRING:
+ String[] stringValues = new String[numValues];
+ for (int i = 0; i < numValues; i++) {
+ stringValues[i] = (String) rawValues[i];
+ }
+ typedValues = stringValues;
+ break;
+ case BYTES:
+ byte[][] bytesValues = new byte[numValues][];
+ for (int i = 0; i < numValues; i++) {
+ bytesValues[i] = (byte[]) rawValues[i];
+ }
+ typedValues = bytesValues;
+ break;
+ default:
+ throw new IllegalStateException("Unsupported data type: " + _dataType);
+ }
+ _dictionaryCreator.build(typedValues);
+
+ // Write header placeholder
+ ByteBuffer headerBuffer =
ByteBuffer.allocate(HEADER_LENGTH).order(ByteOrder.BIG_ENDIAN);
+ headerBuffer.putInt(VERSION);
+ headerBuffer.putInt(numValues);
+ headerBuffer.putInt(_maxLength);
+ headerBuffer.putLong(0L); // dictionaryOffset placeholder
+ headerBuffer.putLong(0L); // dictionaryLength placeholder
+ headerBuffer.putLong(0L); // invertedIndexOffset placeholder
+ headerBuffer.putLong(0L); // invertedIndexLength placeholder
+ try (FileOutputStream outputStream = new FileOutputStream(_indexFile)) {
+ outputStream.write(headerBuffer.array());
+ }
+
+ // Write dictionary
+ long dictionaryOffset = HEADER_LENGTH;
+ File dictionaryFile = _dictionaryCreator.getDictionaryFile();
+ long dictionaryLength = dictionaryFile.length();
+ FileUtils.writeByteArrayToFile(_indexFile,
FileUtils.readFileToByteArray(dictionaryFile), true);
+
+ // Write inverted index
+ long invertedIndexOffset = dictionaryOffset + dictionaryLength;
+ try (FileChannel channel = new RandomAccessFile(_indexFile,
"rw").getChannel()) {
Review Comment:
Resource leak: RandomAccessFile is not properly closed. When getting the
channel from RandomAccessFile, the file itself remains open. Should use
try-with-resources for the RandomAccessFile as well.
```suggestion
try (RandomAccessFile raf = new RandomAccessFile(_indexFile, "rw")) {
FileChannel channel = raf.getChannel();
```
##########
pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/index/readers/RawValueBitmapInvertedIndexReader.java:
##########
@@ -0,0 +1,162 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.pinot.segment.local.segment.index.readers;
+
+import java.io.IOException;
+import org.apache.pinot.segment.spi.index.reader.Dictionary;
+import org.apache.pinot.segment.spi.index.reader.InvertedIndexReader;
+import org.apache.pinot.segment.spi.memory.PinotDataBuffer;
+import org.apache.pinot.spi.data.FieldSpec.DataType;
+import org.roaringbitmap.buffer.ImmutableRoaringBitmap;
+import org.roaringbitmap.buffer.MutableRoaringBitmap;
+
+/**
+ * Reader for raw value bitmap inverted index.
+ * File format:
+ * <ul>
+ * <li>Header (36 bytes):</li>
+ * <ul>
+ * <li>version (4 bytes)</li>
+ * <li>cardinality (4 bytes)</li>
+ * <li>maxLength (4 bytes) - for string/bytes data types</li>
+ * <li>dictionaryOffset (8 bytes)</li>
+ * <li>dictionaryLength (8 bytes)</li>
+ * <li>invertedIndexOffset (8 bytes)</li>
+ * <li>invertedIndexLength (8 bytes)</li>
+ * </ul>
+ * <li>Dictionary data</li>
+ * <li>Inverted index data</li>
+ * </ul>
+ */
+public class RawValueBitmapInvertedIndexReader implements
InvertedIndexReader<ImmutableRoaringBitmap> {
+ private static final int VERSION = 1;
+ private static final int HEADER_LENGTH = 36;
+
+ private final PinotDataBuffer _dataBuffer;
+ private final DataType _dataType;
+ private final Dictionary _dictionary;
+ private final BitmapInvertedIndexReader _invertedIndexReader;
+ private boolean _isClosed;
+
+ public RawValueBitmapInvertedIndexReader(PinotDataBuffer dataBuffer,
DataType dataType) throws IOException {
+ _dataBuffer = dataBuffer;
+ _dataType = dataType;
+
+ // Read header
+ int version = _dataBuffer.getInt(0);
+ if (version != VERSION) {
+ throw new IllegalStateException("Unsupported version: " + version);
+ }
+ int cardinality = _dataBuffer.getInt(4);
+ int maxLength = _dataBuffer.getInt(8);
+ long dictionaryOffset = _dataBuffer.getLong(12);
+ long dictionaryLength = _dataBuffer.getLong(20);
+ long invertedIndexOffset = _dataBuffer.getLong(28);
+ long invertedIndexLength = _dataBuffer.getLong(36);
+
+ // Initialize dictionary
+ PinotDataBuffer dictionaryBuffer = _dataBuffer.view(dictionaryOffset,
dictionaryOffset + dictionaryLength);
+ switch (_dataType.getStoredType()) {
+ case INT:
+ _dictionary = new IntDictionary(dictionaryBuffer, cardinality);
+ break;
+ case LONG:
+ _dictionary = new LongDictionary(dictionaryBuffer, cardinality);
+ break;
+ case FLOAT:
+ _dictionary = new FloatDictionary(dictionaryBuffer, cardinality);
+ break;
+ case DOUBLE:
+ _dictionary = new DoubleDictionary(dictionaryBuffer, cardinality);
+ break;
+ case STRING:
+ _dictionary = new StringDictionary(dictionaryBuffer, cardinality,
maxLength);
+ break;
+ case BYTES:
+ _dictionary = new BytesDictionary(dictionaryBuffer, cardinality,
maxLength);
+ break;
+ default:
+ throw new IllegalStateException("Unsupported data type: " + _dataType);
+ }
+
+ // Initialize inverted index
+ PinotDataBuffer invertedIndexBuffer = _dataBuffer.view(invertedIndexOffset,
+ invertedIndexOffset + invertedIndexLength);
+ _invertedIndexReader = new BitmapInvertedIndexReader(invertedIndexBuffer,
cardinality);
+ }
+
+ @Override
+ public ImmutableRoaringBitmap getDocIds(int dictId) {
+ return _invertedIndexReader.getDocIds(dictId);
+ }
+
+ /**
+ * Returns the document IDs for the given raw INT value.
+ */
+ public ImmutableRoaringBitmap getDocIdsForInt(int value) {
+ int dictId = _dictionary.indexOf(value);
+ return dictId >= 0 ? getDocIds(dictId) : new MutableRoaringBitmap();
+ }
+
+ /**
+ * Returns the document IDs for the given raw LONG value.
+ */
+ public ImmutableRoaringBitmap getDocIdsForLong(long value) {
+ int dictId = _dictionary.indexOf(value);
+ return dictId >= 0 ? getDocIds(dictId) : new MutableRoaringBitmap();
+ }
+
+ /**
+ * Returns the document IDs for the given raw FLOAT value.
+ */
+ public ImmutableRoaringBitmap getDocIdsForFloat(float value) {
+ int dictId = _dictionary.indexOf(value);
+ return dictId >= 0 ? getDocIds(dictId) : new MutableRoaringBitmap();
+ }
+
+ /**
+ * Returns the document IDs for the given raw DOUBLE value.
+ */
+ public ImmutableRoaringBitmap getDocIdsForDouble(double value) {
+ int dictId = _dictionary.indexOf(value);
+ return dictId >= 0 ? getDocIds(dictId) : new MutableRoaringBitmap();
+ }
+
+ /**
+ * Returns the document IDs for the given raw STRING value.
+ */
+ public ImmutableRoaringBitmap getDocIdsForString(String value) {
+ int dictId = _dictionary.indexOf(value);
+ return dictId >= 0 ? getDocIds(dictId) : new MutableRoaringBitmap();
Review Comment:
Creating new MutableRoaringBitmap instances for non-existent values is
inefficient. Consider using a shared empty ImmutableRoaringBitmap singleton or
caching empty bitmaps to avoid unnecessary object allocation.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
