iceberg/arrow/reader/
mod.rs1use crate::arrow::caching_delete_file_loader::CachingDeleteFileLoader;
21use crate::io::FileIO;
22use crate::runtime::Runtime;
23use crate::util::available_parallelism;
24
25const DEFAULT_RANGE_COALESCE_BYTES: u64 = 1024 * 1024;
28
29const DEFAULT_RANGE_FETCH_CONCURRENCY: usize = 10;
32
33const DEFAULT_METADATA_SIZE_HINT: usize = 512 * 1024;
36
37mod file_reader;
38mod options;
39mod pipeline;
40mod positional_deletes;
41mod predicate_visitor;
42mod projection;
43mod row_filter;
44mod row_lineage;
45pub use file_reader::ArrowFileReader;
46pub use options::{ArrowReaderOptions, ParquetReadOptions};
47use predicate_visitor::{CollectFieldIdVisitor, PredicateConverter};
48use projection::{
49 add_fallback_field_ids_to_arrow_schema, apply_name_mapping_to_arrow_schema,
50 coerce_fixed_size_lists, find_leaf_by_field_id,
51};
52
53pub struct ArrowReaderBuilder {
55 options: ArrowReaderOptions,
56 file_io: FileIO,
57 runtime: Runtime,
58}
59
60impl ArrowReaderBuilder {
61 pub fn new(file_io: FileIO, runtime: Runtime) -> Self {
63 ArrowReaderBuilder {
64 options: ArrowReaderOptions::default(),
65 file_io,
66 runtime,
67 }
68 }
69
70 pub fn with_options(mut self, options: ArrowReaderOptions) -> Self {
72 self.options = options;
73 self
74 }
75
76 pub fn with_parquet_options(mut self, options: ParquetReadOptions) -> Self {
78 self.options = self.options.with_parquet_options(options);
79 self
80 }
81
82 pub fn build(self) -> ArrowReader {
84 let options = self
85 .options
86 .with_data_file_concurrency_limit_if_unset(available_parallelism().get().max(1));
87 let concurrency_limit_data_files = options
88 .data_file_concurrency_limit()
89 .expect("reader concurrency was set above")
90 .max(1);
91 ArrowReader {
92 options,
93 file_io: self.file_io.clone(),
94 delete_file_loader: CachingDeleteFileLoader::new(
95 self.file_io.clone(),
96 concurrency_limit_data_files,
97 self.runtime.clone(),
98 ),
99 runtime: self.runtime.clone(),
100 }
101 }
102}
103
104#[derive(Clone)]
106pub struct ArrowReader {
107 options: ArrowReaderOptions,
108 file_io: FileIO,
109 delete_file_loader: CachingDeleteFileLoader,
110
111 runtime: Runtime,
112}
113
114#[cfg(test)]
115mod tests {
116 use std::time::Duration;
117
118 use futures::{TryStreamExt, stream};
119
120 use super::{ArrowReaderBuilder, ArrowReaderOptions, ParquetReadOptions};
121 use crate::io::FileIO;
122 use crate::test_utils::test_runtime;
123 use crate::util::available_parallelism;
124
125 #[tokio::test]
126 async fn deserialized_zero_concurrency_completes_reads() {
127 let options: ArrowReaderOptions =
128 serde_json::from_str(r#"{"data_file_concurrency_limit":0}"#).unwrap();
129 let reader = ArrowReaderBuilder::new(FileIO::new_with_memory(), test_runtime())
130 .with_options(options)
131 .build();
132 let batches = tokio::time::timeout(
133 Duration::from_secs(1),
134 reader
135 .read(Box::pin(stream::empty()))
136 .unwrap()
137 .stream()
138 .try_collect::<Vec<_>>(),
139 )
140 .await
141 .expect("zero concurrency must not stall the task stream")
142 .unwrap();
143 assert!(batches.is_empty());
144 }
145
146 #[test]
147 fn default_options_restore_local_reader_defaults() {
148 let reader = ArrowReaderBuilder::new(FileIO::new_with_memory(), test_runtime())
149 .with_options(
150 ArrowReaderOptions::builder()
151 .with_batch_size(1024)
152 .with_data_file_concurrency_limit(3)
153 .with_row_group_filtering_enabled(false)
154 .with_row_selection_enabled(true)
155 .with_bloom_filter_enabled(true)
156 .with_parquet_options(
157 ParquetReadOptions::builder()
158 .with_range_fetch_concurrency(99)
159 .build(),
160 )
161 .build(),
162 )
163 .with_options(ArrowReaderOptions::default())
164 .build();
165 assert_eq!(
166 reader.options,
167 ArrowReaderOptions::builder()
168 .with_data_file_concurrency_limit(available_parallelism().get())
169 .build()
170 );
171 }
172
173 #[test]
174 fn explicit_options_configure_reader_settings() {
175 let parquet = ParquetReadOptions::builder()
176 .with_metadata_size_hint(4096)
177 .with_range_coalesce_bytes(8192)
178 .with_range_fetch_concurrency(2)
179 .build();
180 let options = ArrowReaderOptions::builder()
181 .with_batch_size(17)
182 .with_data_file_concurrency_limit(3)
183 .with_row_group_filtering_enabled(false)
184 .with_row_selection_enabled(true)
185 .with_bloom_filter_enabled(true)
186 .with_parquet_options(parquet)
187 .build();
188 let reader = ArrowReaderBuilder::new(FileIO::new_with_memory(), test_runtime())
189 .with_options(options.clone())
190 .build();
191 assert_eq!(reader.options, options);
192 }
193}