This is an automated email from the ASF dual-hosted git repository.
xiangfu pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/pinot.git
The following commit(s) were added to refs/heads/master by this push:
new 51335a2db1 Adding zstd examples (#11808)
51335a2db1 is described below
commit 51335a2db1720ac18da8818a4b565b557a0c667b
Author: Xiang Fu <xiangfu.1...@gmail.com>
AuthorDate: Mon Oct 16 12:08:01 2023 -0700
Adding zstd examples (#11808)
---
.../parquet/ParquetNativeRecordReaderFullTest.java | 2 ++
.../inputformat/parquet/ParquetRecordReaderTest.java | 12 +++++++-----
.../src/test/resources/airlineStats.zstd.parquet | Bin 0 -> 891041 bytes
.../src/test/resources/baseballStats.zstd.parquet | Bin 0 -> 1609345 bytes
4 files changed, 9 insertions(+), 5 deletions(-)
diff --git
a/pinot-plugins/pinot-input-format/pinot-parquet/src/test/java/org/apache/pinot/plugin/inputformat/parquet/ParquetNativeRecordReaderFullTest.java
b/pinot-plugins/pinot-input-format/pinot-parquet/src/test/java/org/apache/pinot/plugin/inputformat/parquet/ParquetNativeRecordReaderFullTest.java
index 93edf424dd..f0438a0508 100644
---
a/pinot-plugins/pinot-input-format/pinot-parquet/src/test/java/org/apache/pinot/plugin/inputformat/parquet/ParquetNativeRecordReaderFullTest.java
+++
b/pinot-plugins/pinot-input-format/pinot-parquet/src/test/java/org/apache/pinot/plugin/inputformat/parquet/ParquetNativeRecordReaderFullTest.java
@@ -30,6 +30,8 @@ public class ParquetNativeRecordReaderFullTest {
@Test
protected void testReadDataSet1()
throws Exception {
+ testParquetFile("airlineStats.zstd.parquet");
+ testParquetFile("baseballStats.zstd.parquet");
testParquetFile("test-data/gzip-nation.impala.parquet");
//testParquetFile("test-data/nation.dict.parquet");
testParquetFile("test-data/nation.impala.parquet");
diff --git
a/pinot-plugins/pinot-input-format/pinot-parquet/src/test/java/org/apache/pinot/plugin/inputformat/parquet/ParquetRecordReaderTest.java
b/pinot-plugins/pinot-input-format/pinot-parquet/src/test/java/org/apache/pinot/plugin/inputformat/parquet/ParquetRecordReaderTest.java
index 9c861237e1..8133a18850 100644
---
a/pinot-plugins/pinot-input-format/pinot-parquet/src/test/java/org/apache/pinot/plugin/inputformat/parquet/ParquetRecordReaderTest.java
+++
b/pinot-plugins/pinot-input-format/pinot-parquet/src/test/java/org/apache/pinot/plugin/inputformat/parquet/ParquetRecordReaderTest.java
@@ -39,10 +39,6 @@ import org.testng.annotations.Test;
public class ParquetRecordReaderTest extends AbstractRecordReaderTest {
- private final File _testParquetFileWithInt96AndDecimal =
- new
File(getClass().getClassLoader().getResource("test-file-with-int96-and-decimal.snappy.parquet").getFile());
-
- private static final int NUM_RECORDS_TEST_PARQUET_WITH_INT96 = 1965;
@Override
protected RecordReader createRecordReader(File file)
@@ -99,8 +95,13 @@ public class ParquetRecordReaderTest extends
AbstractRecordReaderTest {
public void testParquetNativeRecordReader()
throws IOException {
ParquetNativeRecordReader nativeRecordReader = new
ParquetNativeRecordReader();
- nativeRecordReader.init(_testParquetFileWithInt96AndDecimal,
ImmutableSet.of(), new ParquetRecordReaderConfig());
+ nativeRecordReader.init(
+ new
File(getClass().getClassLoader().getResource("test-file-with-int96-and-decimal.snappy.parquet").getFile()),
+ ImmutableSet.of(), new ParquetRecordReaderConfig());
testReadParquetFile(nativeRecordReader, 1965);
+ nativeRecordReader.init(new
File(getClass().getClassLoader().getResource("airlineStats.zstd.parquet").getFile()),
+ ImmutableSet.of(), new ParquetRecordReaderConfig());
+ testReadParquetFile(nativeRecordReader, 19492);
nativeRecordReader.init(_dataFile, ImmutableSet.of(), new
ParquetRecordReaderConfig());
testReadParquetFile(nativeRecordReader, SAMPLE_RECORDS_SIZE);
}
@@ -129,6 +130,7 @@ public class ParquetRecordReaderTest extends
AbstractRecordReaderTest {
testComparison(new
File(getClass().getClassLoader().getResource("test-comparison.gz.parquet").getFile()),
363667);
testComparison(new
File(getClass().getClassLoader().getResource("test-comparison.snappy.parquet").getFile()),
2870);
testComparison(new
File(getClass().getClassLoader().getResource("baseballStats.snappy.parquet").getFile()),
97889);
+ testComparison(new
File(getClass().getClassLoader().getResource("baseballStats.zstd.parquet").getFile()),
97889);
testComparison(new
File(getClass().getClassLoader().getResource("githubEvents.snappy.parquet").getFile()),
10000);
testComparison(new
File(getClass().getClassLoader().getResource("starbucksStores.snappy.parquet").getFile()),
6443);
testComparison(new
File(getClass().getClassLoader().getResource("airlineStats.snappy.parquet").getFile()),
19492);
diff --git
a/pinot-plugins/pinot-input-format/pinot-parquet/src/test/resources/airlineStats.zstd.parquet
b/pinot-plugins/pinot-input-format/pinot-parquet/src/test/resources/airlineStats.zstd.parquet
new file mode 100644
index 0000000000..3ec4bee99a
Binary files /dev/null and
b/pinot-plugins/pinot-input-format/pinot-parquet/src/test/resources/airlineStats.zstd.parquet
differ
diff --git
a/pinot-plugins/pinot-input-format/pinot-parquet/src/test/resources/baseballStats.zstd.parquet
b/pinot-plugins/pinot-input-format/pinot-parquet/src/test/resources/baseballStats.zstd.parquet
new file mode 100644
index 0000000000..a14ec8a455
Binary files /dev/null and
b/pinot-plugins/pinot-input-format/pinot-parquet/src/test/resources/baseballStats.zstd.parquet
differ
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscr...@pinot.apache.org
For additional commands, e-mail: commits-h...@pinot.apache.org
