This is an automated email from the ASF dual-hosted git repository.
vvivekiyer pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/pinot.git
The following commit(s) were added to refs/heads/master by this push:
new dbb6e30298 Fix potential int overflow issue for memory allocation of
indices in segment-local module (#13717)
dbb6e30298 is described below
commit dbb6e30298e4983b5fa206d29aa5fe7a5c186cba
Author: Sajjad Moradi <moradi.saj...@gmail.com>
AuthorDate: Wed Aug 7 09:05:05 2024 -0700
Fix potential int overflow issue for memory allocation of indices in
segment-local module (#13717)
* Add proper error message for handling large forward index size
* Fix all potential INT overflow issues in segment local module
* fix linter issues
---
.../impl/FixedByteSingleValueMultiColumnReaderWriter.java | 6 +++---
.../local/io/writer/impl/FixedByteChunkForwardIndexWriter.java | 2 +-
.../realtime/impl/dictionary/BaseOffHeapMutableDictionary.java | 2 +-
.../realtime/impl/forward/FixedByteSVMutableForwardIndex.java | 2 +-
.../local/segment/creator/impl/inv/RangeIndexCreator.java | 4 ++--
.../index/readers/forward/BaseChunkForwardIndexReader.java | 4 ++--
.../index/readers/forward/FixedBitMVForwardIndexReader.java | 2 +-
.../readers/forward/VarByteChunkSVForwardIndexReader.java | 10 ++++++----
.../segment/index/readers/sorted/SortedIndexReaderImpl.java | 2 +-
.../local/startree/v2/builder/OffHeapSingleTreeBuilder.java | 10 +++++-----
10 files changed, 23 insertions(+), 21 deletions(-)
diff --git
a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/io/readerwriter/impl/FixedByteSingleValueMultiColumnReaderWriter.java
b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/io/readerwriter/impl/FixedByteSingleValueMultiColumnReaderWriter.java
index 2084a45373..601b09a251 100644
---
a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/io/readerwriter/impl/FixedByteSingleValueMultiColumnReaderWriter.java
+++
b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/io/readerwriter/impl/FixedByteSingleValueMultiColumnReaderWriter.java
@@ -43,10 +43,10 @@ public class FixedByteSingleValueMultiColumnReaderWriter
implements Closeable {
private final int[] _columnSizesInBytes;
private final PinotDataBufferMemoryManager _memoryManager;
- private String _allocationContext;
+ private final String _allocationContext;
private final long _chunkSizeInBytes;
private int _capacityInRows;
- private int _numColumns;
+ private final int _numColumns;
/**
* Constructor for the class.
@@ -74,7 +74,7 @@ public class FixedByteSingleValueMultiColumnReaderWriter
implements Closeable {
}
_numColumns = _columnSizesInBytes.length;
- _chunkSizeInBytes = rowSizeInBytes * numRowsPerChunk;
+ _chunkSizeInBytes = (long) rowSizeInBytes * numRowsPerChunk;
}
public int getInt(int row, int column) {
diff --git
a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/io/writer/impl/FixedByteChunkForwardIndexWriter.java
b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/io/writer/impl/FixedByteChunkForwardIndexWriter.java
index 11a250791b..8b517a84f9 100644
---
a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/io/writer/impl/FixedByteChunkForwardIndexWriter.java
+++
b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/io/writer/impl/FixedByteChunkForwardIndexWriter.java
@@ -49,7 +49,7 @@ public class FixedByteChunkForwardIndexWriter extends
BaseChunkForwardIndexWrite
int numDocsPerChunk, int sizeOfEntry, int writerVersion)
throws IOException {
super(file, compressionType, totalDocs,
normalizeDocsPerChunk(writerVersion, numDocsPerChunk),
- (sizeOfEntry * normalizeDocsPerChunk(writerVersion, numDocsPerChunk)),
sizeOfEntry, writerVersion, true);
+ (long) sizeOfEntry * normalizeDocsPerChunk(writerVersion,
numDocsPerChunk), sizeOfEntry, writerVersion, true);
_chunkDataOffset = 0;
}
diff --git
a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/realtime/impl/dictionary/BaseOffHeapMutableDictionary.java
b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/realtime/impl/dictionary/BaseOffHeapMutableDictionary.java
index ae50758e42..3fca1354bd 100644
---
a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/realtime/impl/dictionary/BaseOffHeapMutableDictionary.java
+++
b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/realtime/impl/dictionary/BaseOffHeapMutableDictionary.java
@@ -420,7 +420,7 @@ public abstract class BaseOffHeapMutableDictionary
implements MutableDictionary
ValueToDictId valueToDictId = _valueToDict;
long size = 0;
for (IntBuffer iBuf : valueToDictId._iBufList) {
- size = size + iBuf.capacity() * Integer.BYTES;
+ size = size + (long) iBuf.capacity() * Integer.BYTES;
}
return size;
}
diff --git
a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/realtime/impl/forward/FixedByteSVMutableForwardIndex.java
b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/realtime/impl/forward/FixedByteSVMutableForwardIndex.java
index 7ceab2a941..8c25bb0706 100644
---
a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/realtime/impl/forward/FixedByteSVMutableForwardIndex.java
+++
b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/realtime/impl/forward/FixedByteSVMutableForwardIndex.java
@@ -81,7 +81,7 @@ public class FixedByteSVMutableForwardIndex implements
MutableForwardIndex {
_valueSizeInBytes = storedType.size();
}
_numRowsPerChunk = numRowsPerChunk;
- _chunkSizeInBytes = numRowsPerChunk * _valueSizeInBytes;
+ _chunkSizeInBytes = (long) numRowsPerChunk * _valueSizeInBytes;
_memoryManager = memoryManager;
_allocationContext = allocationContext;
addBuffer();
diff --git
a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/creator/impl/inv/RangeIndexCreator.java
b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/creator/impl/inv/RangeIndexCreator.java
index e1c5479214..a97c45fec6 100644
---
a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/creator/impl/inv/RangeIndexCreator.java
+++
b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/creator/impl/inv/RangeIndexCreator.java
@@ -347,7 +347,7 @@ public final class RangeIndexCreator implements
CombinedInvertedIndexCreator {
Number rangeStart = _numberValueBuffer.get(range.getLeft());
writeNumberToHeader(header, rangeStart);
}
- bytesWritten += ranges.size() * _valueType.size(); // Range start values
+ bytesWritten += (long) ranges.size() * _valueType.size(); // Range start
values
Number lastRangeEnd = _numberValueBuffer.get(ranges.get(ranges.size() -
1).getRight());
writeNumberToHeader(header, lastRangeEnd);
@@ -355,7 +355,7 @@ public final class RangeIndexCreator implements
CombinedInvertedIndexCreator {
//compute the offset where the bitmap for the first range would be
written
//bitmap start offset for each range, one extra to make it easy to get
the length for last one.
- long bitmapOffsetHeaderSize = (ranges.size() + 1) * Long.BYTES;
+ long bitmapOffsetHeaderSize = (long) (ranges.size() + 1) * Long.BYTES;
long bitmapOffset = bytesWritten + bitmapOffsetHeaderSize;
header.writeLong(bitmapOffset);
diff --git
a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/index/readers/forward/BaseChunkForwardIndexReader.java
b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/index/readers/forward/BaseChunkForwardIndexReader.java
index 745bd18fde..80ed58a416 100644
---
a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/index/readers/forward/BaseChunkForwardIndexReader.java
+++
b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/index/readers/forward/BaseChunkForwardIndexReader.java
@@ -241,10 +241,10 @@ public abstract class BaseChunkForwardIndexReader
implements ForwardIndexReader<
protected long getChunkPositionAndRecordRanges(int chunkId, List<ByteRange>
ranges) {
if (_headerEntryChunkOffsetSize == Integer.BYTES) {
- ranges.add(new ByteRange(_dataHeaderStart + chunkId *
_headerEntryChunkOffsetSize, Integer.BYTES));
+ ranges.add(new ByteRange(_dataHeaderStart + (long) chunkId *
_headerEntryChunkOffsetSize, Integer.BYTES));
return _dataHeader.getInt(chunkId * _headerEntryChunkOffsetSize);
} else {
- ranges.add(new ByteRange(_dataHeaderStart + chunkId *
_headerEntryChunkOffsetSize, Long.BYTES));
+ ranges.add(new ByteRange(_dataHeaderStart + (long) chunkId *
_headerEntryChunkOffsetSize, Long.BYTES));
return _dataHeader.getLong(chunkId * _headerEntryChunkOffsetSize);
}
}
diff --git
a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/index/readers/forward/FixedBitMVForwardIndexReader.java
b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/index/readers/forward/FixedBitMVForwardIndexReader.java
index 983de9c6b9..ddb9436c76 100644
---
a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/index/readers/forward/FixedBitMVForwardIndexReader.java
+++
b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/index/readers/forward/FixedBitMVForwardIndexReader.java
@@ -68,7 +68,7 @@ public final class FixedBitMVForwardIndexReader implements
ForwardIndexReader<Fi
_numValues = numValues;
_numDocsPerChunk = (int) (Math.ceil((float) PREFERRED_NUM_VALUES_PER_CHUNK
/ (numValues / numDocs)));
int numChunks = (numDocs + _numDocsPerChunk - 1) / _numDocsPerChunk;
- long endOffset = numChunks * Integer.BYTES;
+ long endOffset = (long) numChunks * Integer.BYTES;
_bitmapReaderStartOffset = endOffset;
_chunkOffsetReader = new FixedByteValueReaderWriter(dataBuffer.view(0L,
endOffset));
int bitmapSize = (numValues + Byte.SIZE - 1) / Byte.SIZE;
diff --git
a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/index/readers/forward/VarByteChunkSVForwardIndexReader.java
b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/index/readers/forward/VarByteChunkSVForwardIndexReader.java
index 585d48d7ea..651f7bcdcb 100644
---
a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/index/readers/forward/VarByteChunkSVForwardIndexReader.java
+++
b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/index/readers/forward/VarByteChunkSVForwardIndexReader.java
@@ -99,7 +99,8 @@ public final class VarByteChunkSVForwardIndexReader extends
BaseChunkForwardInde
// These offsets are offset in the data buffer
long chunkStartOffset = getChunkPosition(chunkId);
- long valueStartOffset = chunkStartOffset +
_dataBuffer.getInt(chunkStartOffset + chunkRowId * ROW_OFFSET_SIZE);
+ long valueStartOffset =
+ chunkStartOffset + _dataBuffer.getInt(chunkStartOffset + (long)
chunkRowId * ROW_OFFSET_SIZE);
long valueEndOffset = getValueEndOffset(chunkId, chunkRowId,
chunkStartOffset);
int length = (int) (valueEndOffset - valueStartOffset);
@@ -152,7 +153,8 @@ public final class VarByteChunkSVForwardIndexReader extends
BaseChunkForwardInde
// These offsets are offset in the data buffer
long chunkStartOffset = getChunkPosition(chunkId);
- long valueStartOffset = chunkStartOffset +
_dataBuffer.getInt(chunkStartOffset + chunkRowId * ROW_OFFSET_SIZE);
+ long valueStartOffset =
+ chunkStartOffset + _dataBuffer.getInt(chunkStartOffset + (long)
chunkRowId * ROW_OFFSET_SIZE);
long valueEndOffset = getValueEndOffset(chunkId, chunkRowId,
chunkStartOffset);
byte[] bytes = new byte[(int) (valueEndOffset - valueStartOffset)];
@@ -188,7 +190,7 @@ public final class VarByteChunkSVForwardIndexReader extends
BaseChunkForwardInde
// Last row in the last chunk
return _dataBuffer.size();
} else {
- int valueEndOffsetInChunk = _dataBuffer.getInt(chunkStartOffset +
(chunkRowId + 1) * ROW_OFFSET_SIZE);
+ int valueEndOffsetInChunk = _dataBuffer.getInt(chunkStartOffset +
(long) (chunkRowId + 1) * ROW_OFFSET_SIZE);
if (valueEndOffsetInChunk == 0) {
// Last row in the last chunk (chunk is incomplete, which stores 0
as the offset for the absent rows)
return _dataBuffer.size();
@@ -201,7 +203,7 @@ public final class VarByteChunkSVForwardIndexReader extends
BaseChunkForwardInde
// Last row in the chunk
return getChunkPosition(chunkId + 1);
} else {
- return chunkStartOffset + _dataBuffer.getInt(chunkStartOffset +
(chunkRowId + 1) * ROW_OFFSET_SIZE);
+ return chunkStartOffset + _dataBuffer.getInt(chunkStartOffset + (long)
(chunkRowId + 1) * ROW_OFFSET_SIZE);
}
}
}
diff --git
a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/index/readers/sorted/SortedIndexReaderImpl.java
b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/index/readers/sorted/SortedIndexReaderImpl.java
index e508b0d013..014d85bcd4 100644
---
a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/index/readers/sorted/SortedIndexReaderImpl.java
+++
b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/index/readers/sorted/SortedIndexReaderImpl.java
@@ -36,7 +36,7 @@ public class SortedIndexReaderImpl implements
SortedIndexReader<SortedIndexReade
public SortedIndexReaderImpl(PinotDataBuffer dataBuffer, int cardinality) {
// 2 values per dictionary id
- Preconditions.checkState(dataBuffer.size() == 2 * cardinality *
Integer.BYTES);
+ Preconditions.checkState(dataBuffer.size() == 2L * cardinality *
Integer.BYTES);
_reader = new FixedByteValueReaderWriter(dataBuffer);
_cardinality = cardinality;
}
diff --git
a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/startree/v2/builder/OffHeapSingleTreeBuilder.java
b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/startree/v2/builder/OffHeapSingleTreeBuilder.java
index 49a72eedf6..258b5e77ec 100644
---
a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/startree/v2/builder/OffHeapSingleTreeBuilder.java
+++
b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/startree/v2/builder/OffHeapSingleTreeBuilder.java
@@ -172,7 +172,7 @@ public class OffHeapSingleTreeBuilder extends
BaseSingleTreeBuilder {
int getDimensionValue(int docId, int dimensionId)
throws IOException {
ensureBufferReadable(docId);
- return _starTreeRecordBuffer.getInt(_starTreeRecordOffsets.get(docId) +
dimensionId * Integer.BYTES);
+ return _starTreeRecordBuffer.getInt(_starTreeRecordOffsets.get(docId) +
(long) dimensionId * Integer.BYTES);
}
private void ensureBufferReadable(int docId)
@@ -219,8 +219,8 @@ public class OffHeapSingleTreeBuilder extends
BaseSingleTreeBuilder {
long offset1 = (long) sortedDocIds[i1] * _numDimensions *
Integer.BYTES;
long offset2 = (long) sortedDocIds[i2] * _numDimensions *
Integer.BYTES;
for (int i = 0; i < _numDimensions; i++) {
- int dimension1 = dataBuffer.getInt(offset1 + i * Integer.BYTES);
- int dimension2 = dataBuffer.getInt(offset2 + i * Integer.BYTES);
+ int dimension1 = dataBuffer.getInt(offset1 + (long) i *
Integer.BYTES);
+ int dimension2 = dataBuffer.getInt(offset2 + (long) i *
Integer.BYTES);
if (dimension1 != dimension2) {
return dimension1 - dimension2;
}
@@ -282,8 +282,8 @@ public class OffHeapSingleTreeBuilder extends
BaseSingleTreeBuilder {
long offset1 = _starTreeRecordOffsets.get(sortedDocIds[i1]);
long offset2 = _starTreeRecordOffsets.get(sortedDocIds[i2]);
for (int i = dimensionId + 1; i < _numDimensions; i++) {
- int dimension1 = _starTreeRecordBuffer.getInt(offset1 + i *
Integer.BYTES);
- int dimension2 = _starTreeRecordBuffer.getInt(offset2 + i *
Integer.BYTES);
+ int dimension1 = _starTreeRecordBuffer.getInt(offset1 + (long) i *
Integer.BYTES);
+ int dimension2 = _starTreeRecordBuffer.getInt(offset2 + (long) i *
Integer.BYTES);
if (dimension1 != dimension2) {
return dimension1 - dimension2;
}
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscr...@pinot.apache.org
For additional commands, e-mail: commits-h...@pinot.apache.org
