alexanderbianchi commented on code in PR #3514:
URL: https://github.com/apache/iceberg-rust/pull/3514#discussion_r4538820115
##########
crates/iceberg/src/arrow/reader/mod.rs:
##########
@@ -168,16 +104,90 @@ impl ArrowReaderBuilder {
/// Reads data from Parquet files
#[derive(Clone)]
pub struct ArrowReader {
- batch_size: Option<usize>,
+ options: ArrowReaderOptions,
file_io: FileIO,
delete_file_loader: CachingDeleteFileLoader,
- /// the maximum number of data files that can be fetched at the same time
- concurrency_limit_data_files: usize,
-
- row_group_filtering_enabled: bool,
- row_selection_enabled: bool,
- bloom_filter_enabled: bool,
- parquet_read_options: ParquetReadOptions,
runtime: Runtime,
}
+
+#[cfg(test)]
+mod tests {
+ use std::time::Duration;
+
+ use futures::{TryStreamExt, stream};
+
+ use super::{ArrowReaderBuilder, ArrowReaderOptions, ParquetReadOptions};
+ use crate::io::FileIO;
+ use crate::test_utils::test_runtime;
+ use crate::util::available_parallelism;
+
+ #[tokio::test]
+ async fn deserialized_zero_concurrency_completes_reads() {
Review Comment:
Checking that the stream completes tests the behavior we care about. Our
current test reads an empty task stream, so there’s no file I/O, but it can
still catch the zero-concurrency hang.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
