pvary commented on code in PR #11661:
URL: https://github.com/apache/iceberg/pull/11661#discussion_r2120824584
##########
arrow/src/main/java/org/apache/iceberg/arrow/vectorized/parquet/VectorizedParquetDefinitionLevelReader.java:
##########
@@ -45,122 +46,217 @@ public VectorizedParquetDefinitionLevelReader(
super(bitWidth, maxDefLevel, readLength, setArrowValidityVector);
}
- abstract class NumericBaseReader {
- public void nextBatch(
+ abstract class CommonReader {
+ private void nextBatch(
final FieldVector vector,
final int startOffset,
final int typeWidth,
final int numValsToRead,
NullabilityHolder nullabilityHolder,
- ValuesAsBytesReader valuesReader) {
- int bufferIdx = startOffset;
+ ValuesReader valuesReader,
+ Dictionary dict) {
+ int idx = startOffset;
int left = numValsToRead;
while (left > 0) {
if (currentCount == 0) {
readNextGroup();
}
int numValues = Math.min(left, currentCount);
+
+ byte[] byteArray = null;
+ if (typeWidth > -1) {
+ byteArray = new byte[typeWidth];
+ }
+ ArrowBuf validityBuffer = vector.getValidityBuffer();
+
switch (mode) {
case RLE:
- setNextNValuesInVector(
- typeWidth, nullabilityHolder, valuesReader, bufferIdx, vector,
numValues);
- bufferIdx += numValues;
+ if (valuesReader instanceof ValuesAsBytesReader) {
+ nextRleBatch(
+ vector,
+ typeWidth,
+ nullabilityHolder,
+ (ValuesAsBytesReader) valuesReader,
+ idx,
+ numValues,
+ byteArray);
+ } else if (valuesReader instanceof
VectorizedDictionaryEncodedParquetValuesReader) {
+ nextRleDictEncodedBatch(
+ vector,
+ typeWidth,
+ nullabilityHolder,
+ (VectorizedDictionaryEncodedParquetValuesReader)
valuesReader,
+ dict,
+ idx,
+ numValues,
+ validityBuffer);
+ }
+ idx += numValues;
break;
case PACKED:
- for (int i = 0; i < numValues; ++i) {
- if (packedValuesBuffer[packedValuesBufferIdx++] == maxDefLevel) {
- nextVal(vector, bufferIdx * typeWidth, valuesReader, mode);
- nullabilityHolder.setNotNull(bufferIdx);
- if (setArrowValidityVector) {
- BitVectorHelper.setBit(vector.getValidityBuffer(),
bufferIdx);
- }
- } else {
- setNull(nullabilityHolder, bufferIdx,
vector.getValidityBuffer());
- }
- bufferIdx++;
+ if (valuesReader instanceof ValuesAsBytesReader) {
Review Comment:
Thanks for implementing this!
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: issues-unsubscr...@iceberg.apache.org
For queries about this service, please contact Infrastructure at:
us...@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscr...@iceberg.apache.org
For additional commands, e-mail: issues-h...@iceberg.apache.org
