pvary commented on code in PR #11661:
URL: https://github.com/apache/iceberg/pull/11661#discussion_r2113600021
##########
arrow/src/main/java/org/apache/iceberg/arrow/vectorized/parquet/VectorizedParquetDefinitionLevelReader.java:
##########
@@ -45,122 +46,217 @@ public VectorizedParquetDefinitionLevelReader(
super(bitWidth, maxDefLevel, readLength, setArrowValidityVector);
}
- abstract class NumericBaseReader {
- public void nextBatch(
+ abstract class CommonReader {
+ private void nextBatch(
final FieldVector vector,
final int startOffset,
final int typeWidth,
final int numValsToRead,
NullabilityHolder nullabilityHolder,
- ValuesAsBytesReader valuesReader) {
- int bufferIdx = startOffset;
+ ValuesReader valuesReader,
+ Dictionary dict) {
+ int idx = startOffset;
int left = numValsToRead;
while (left > 0) {
if (currentCount == 0) {
readNextGroup();
}
int numValues = Math.min(left, currentCount);
+
+ byte[] byteArray = null;
+ if (typeWidth > -1) {
+ byteArray = new byte[typeWidth];
+ }
+ ArrowBuf validityBuffer = vector.getValidityBuffer();
+
switch (mode) {
case RLE:
- setNextNValuesInVector(
- typeWidth, nullabilityHolder, valuesReader, bufferIdx, vector,
numValues);
- bufferIdx += numValues;
+ if (valuesReader instanceof ValuesAsBytesReader) {
+ nextRleBatch(
+ vector,
+ typeWidth,
+ nullabilityHolder,
+ (ValuesAsBytesReader) valuesReader,
+ idx,
+ numValues,
+ byteArray);
+ } else if (valuesReader instanceof
VectorizedDictionaryEncodedParquetValuesReader) {
+ nextRleDictEncodedBatch(
+ vector,
+ typeWidth,
+ nullabilityHolder,
+ (VectorizedDictionaryEncodedParquetValuesReader)
valuesReader,
+ dict,
+ idx,
+ numValues,
+ validityBuffer);
+ }
+ idx += numValues;
break;
case PACKED:
- for (int i = 0; i < numValues; ++i) {
- if (packedValuesBuffer[packedValuesBufferIdx++] == maxDefLevel) {
- nextVal(vector, bufferIdx * typeWidth, valuesReader, mode);
- nullabilityHolder.setNotNull(bufferIdx);
- if (setArrowValidityVector) {
- BitVectorHelper.setBit(vector.getValidityBuffer(),
bufferIdx);
- }
- } else {
- setNull(nullabilityHolder, bufferIdx,
vector.getValidityBuffer());
- }
- bufferIdx++;
+ if (valuesReader instanceof ValuesAsBytesReader) {
Review Comment:
What if we create a reader function, like this:
```
public interface ReaderFunction {
void nextBatch(Mode mode, int idx, byte[] byteArray, ArrowBuf
validityBuffer);
}
```
Then the we can have:
```
private void nextBatch(
final FieldVector vector,
final int startOffset,
final int typeWidth,
final int numValsToRead, ReaderFunction consumer) {
[..]
consumer.nextBatch(mode, idx, byteArray, validityBuffer);
[..]
}
```
And we can put the reader type specific stuff to the function implementation
(most probably could be inlined)
WDYT?
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: issues-unsubscr...@iceberg.apache.org
For queries about this service, please contact Infrastructure at:
us...@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscr...@iceberg.apache.org
For additional commands, e-mail: issues-h...@iceberg.apache.org
