iceberg/arrow/reader/options.rs
1// Licensed to the Apache Software Foundation (ASF) under one
2// or more contributor license agreements. See the NOTICE file
3// distributed with this work for additional information
4// regarding copyright ownership. The ASF licenses this file
5// to you under the Apache License, Version 2.0 (the
6// "License"); you may not use this file except in compliance
7// with the License. You may obtain a copy of the License at
8//
9// http://www.apache.org/licenses/LICENSE-2.0
10//
11// Unless required by applicable law or agreed to in writing,
12// software distributed under the License is distributed on an
13// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14// KIND, either express or implied. See the License for the
15// specific language governing permissions and limitations
16// under the License.
17
18use serde::{Deserialize, Serialize};
19use typed_builder::TypedBuilder;
20
21use super::{
22 DEFAULT_METADATA_SIZE_HINT, DEFAULT_RANGE_COALESCE_BYTES, DEFAULT_RANGE_FETCH_CONCURRENCY,
23};
24
25/// Configures Arrow reads independently of file planning and local I/O resources.
26///
27/// Serializable alongside planned file tasks so readers can be constructed later.
28/// In self-describing map formats such as JSON, field names are part of the
29/// serialized format: missing fields use defaults and unknown fields are ignored.
30/// Compatibility across different crate versions is not otherwise guaranteed.
31///
32/// ```no_run
33/// # use iceberg::{Result, arrow::{ArrowReaderOptions, ParquetReadOptions}, table::Table};
34/// # async fn read(table: &Table) -> Result<()> {
35/// let options = ArrowReaderOptions::builder()
36/// .with_batch_size(1024)
37/// .with_parquet_options(
38/// ParquetReadOptions::builder()
39/// .with_metadata_size_hint(512 * 1024)
40/// .build(),
41/// )
42/// .build();
43/// let stream = table.scan().build()?.to_arrow(options).await?;
44/// # Ok(())
45/// # }
46/// ```
47#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, TypedBuilder)]
48#[serde(default)]
49#[builder(field_defaults(default, setter(prefix = "with_")))]
50pub struct ArrowReaderOptions {
51 /// Desired batch size, or the reader default when unset.
52 #[builder(default = None, setter(strip_option))]
53 batch_size: Option<usize>,
54 /// Maximum in-flight data files, or the construction path's default when unset.
55 #[builder(default = None, setter(strip_option))]
56 data_file_concurrency_limit: Option<usize>,
57 /// Whether to prune row groups using their statistics.
58 #[builder(default = true)]
59 row_group_filtering_enabled: bool,
60 /// Whether to use page indexes for row selection.
61 row_selection_enabled: bool,
62 /// Whether to use bloom filters for row-group pruning.
63 bloom_filter_enabled: bool,
64 /// I/O tuning applied when reading Parquet files.
65 parquet_options: ParquetReadOptions,
66}
67
68impl Default for ArrowReaderOptions {
69 fn default() -> Self {
70 Self::builder().build()
71 }
72}
73
74impl ArrowReaderOptions {
75 /// Supplies a fallback limit without replacing an explicit reader limit.
76 pub(super) fn with_data_file_concurrency_limit_if_unset(mut self, limit: usize) -> Self {
77 self.data_file_concurrency_limit.get_or_insert(limit);
78 self
79 }
80
81 /// Returns the configured batch size.
82 pub fn batch_size(&self) -> Option<usize> {
83 self.batch_size
84 }
85
86 /// Returns the configured data file concurrency limit.
87 ///
88 /// When unset, reader construction uses execution-host parallelism.
89 /// Zero is treated as one.
90 pub fn data_file_concurrency_limit(&self) -> Option<usize> {
91 self.data_file_concurrency_limit
92 }
93
94 /// Returns whether row-group statistics filtering is enabled.
95 ///
96 /// Enabled by default: statistics can skip groups that cannot match the
97 /// predicate, generally improving performance with little overhead.
98 pub fn row_group_filtering_enabled(&self) -> bool {
99 self.row_group_filtering_enabled
100 }
101
102 /// Returns whether page-index row selection is enabled.
103 ///
104 /// Disabled by default: parsing page indexes can cost more than the reads
105 /// it avoids. Evaluate it against the table's sorting, partitioning, row-group
106 /// size, and page size before enabling it.
107 pub fn row_selection_enabled(&self) -> bool {
108 self.row_selection_enabled
109 }
110
111 /// Returns whether bloom-filter pruning is enabled.
112 ///
113 /// Disabled by default: bloom filters require extra reads per relevant
114 /// column and row group. They can eliminate groups for equality and IN predicates.
115 pub fn bloom_filter_enabled(&self) -> bool {
116 self.bloom_filter_enabled
117 }
118
119 /// Returns the configured parquet options.
120 pub fn parquet_options(&self) -> &ParquetReadOptions {
121 &self.parquet_options
122 }
123
124 pub(super) fn with_parquet_options(mut self, value: ParquetReadOptions) -> Self {
125 self.parquet_options = value;
126 self
127 }
128}
129
130/// Options for tuning Parquet file I/O.
131#[derive(Clone, Debug, PartialEq, Serialize, Deserialize, TypedBuilder)]
132#[serde(default)]
133#[builder(field_defaults(setter(prefix = "with_")))]
134pub struct ParquetReadOptions {
135 /// Number of bytes to prefetch for parsing the Parquet metadata.
136 ///
137 /// This hint can help reduce the number of fetch requests. For more details see the
138 /// [ParquetMetaDataReader documentation](https://docs.rs/parquet/latest/parquet/file/metadata/struct.ParquetMetaDataReader.html#method.with_prefetch_hint).
139 ///
140 /// Defaults to 512 KiB, matching DataFusion's default `ParquetOptions::metadata_size_hint`.
141 #[builder(default = DEFAULT_METADATA_SIZE_HINT)]
142 metadata_size_hint: usize,
143 /// Gap threshold for merging nearby byte ranges into a single request.
144 /// Ranges with gaps smaller than this value will be coalesced.
145 ///
146 /// Defaults to 1 MiB, matching object_store's `OBJECT_STORE_COALESCE_DEFAULT`.
147 #[builder(default = DEFAULT_RANGE_COALESCE_BYTES)]
148 range_coalesce_bytes: u64,
149 /// Maximum number of merged byte ranges to fetch concurrently.
150 ///
151 /// Defaults to 10, matching object_store's `OBJECT_STORE_COALESCE_PARALLEL`.
152 #[builder(default = DEFAULT_RANGE_FETCH_CONCURRENCY)]
153 range_fetch_concurrency: usize,
154 #[builder(default = true)]
155 preload_column_index: bool,
156 #[builder(default = true)]
157 preload_offset_index: bool,
158 /// Stored page-index flag; the reader derives preloading per data file from
159 /// the predicate and delete files, overriding this value.
160 #[builder(default = false)]
161 preload_page_index: bool,
162}
163
164impl Default for ParquetReadOptions {
165 fn default() -> Self {
166 Self::builder().build()
167 }
168}
169
170impl ParquetReadOptions {
171 pub(super) fn with_preload_page_index(mut self, enabled: bool) -> Self {
172 self.preload_page_index = enabled;
173 self
174 }
175
176 /// Returns whether to preload the column index when reading Parquet metadata.
177 pub fn preload_column_index(&self) -> bool {
178 self.preload_column_index
179 }
180
181 /// Returns whether to preload the offset index when reading Parquet metadata.
182 pub fn preload_offset_index(&self) -> bool {
183 self.preload_offset_index
184 }
185
186 /// Returns the stored page-index flag.
187 ///
188 /// The reader ignores this value: page-index preloading is derived per data
189 /// file from the predicate and delete files.
190 pub fn preload_page_index(&self) -> bool {
191 self.preload_page_index
192 }
193
194 /// Returns the configured metadata size hint.
195 pub fn metadata_size_hint(&self) -> usize {
196 self.metadata_size_hint
197 }
198
199 /// Returns the byte-gap threshold for coalescing read requests.
200 pub fn range_coalesce_bytes(&self) -> u64 {
201 self.range_coalesce_bytes
202 }
203
204 /// Returns the maximum number of concurrent range requests.
205 pub fn range_fetch_concurrency(&self) -> usize {
206 self.range_fetch_concurrency
207 }
208}
209
210#[cfg(test)]
211mod tests {
212 use rstest::rstest;
213
214 use super::*;
215
216 #[rstest]
217 #[case::unset(None, 7)]
218 #[case::explicit(Some(3), 3)]
219 #[case::zero(Some(0), 0)]
220 fn default_data_file_concurrency_preserves_explicit_options(
221 #[case] configured: Option<usize>,
222 #[case] expected: usize,
223 ) {
224 // Include unrelated settings to check that resolving concurrency preserves them.
225 let options: ArrowReaderOptions = serde_json::from_value(serde_json::json!({
226 "batch_size": 17,
227 "data_file_concurrency_limit": configured,
228 "row_selection_enabled": true,
229 }))
230 .unwrap();
231 // Resolve the reader fallback; explicit zero is normalized later by the reader.
232 let options = options.with_data_file_concurrency_limit_if_unset(7);
233 let expected = ArrowReaderOptions::builder()
234 .with_batch_size(17)
235 .with_data_file_concurrency_limit(expected)
236 .with_row_selection_enabled(true)
237 .build();
238 assert_eq!(options, expected);
239 }
240
241 #[test]
242 fn missing_options_use_defaults() {
243 let decoded: ArrowReaderOptions = serde_json::from_str(r#"{"batch_size":17}"#).unwrap();
244 assert_eq!(
245 decoded,
246 ArrowReaderOptions::builder().with_batch_size(17).build()
247 );
248 }
249
250 #[test]
251 fn unknown_options_are_ignored() {
252 let decoded: ArrowReaderOptions =
253 serde_json::from_str(r#"{"future_option":true}"#).unwrap();
254 assert_eq!(decoded, ArrowReaderOptions::default());
255 }
256
257 #[test]
258 fn unset_concurrency_serializes_as_null() {
259 assert_eq!(
260 serde_json::to_value(ArrowReaderOptions::default()).unwrap()["data_file_concurrency_limit"],
261 serde_json::Value::Null,
262 );
263 }
264
265 #[test]
266 fn parquet_serialization_preserves_preload_settings() {
267 let options = ParquetReadOptions {
268 preload_column_index: false,
269 preload_offset_index: false,
270 preload_page_index: true,
271 ..ParquetReadOptions::default()
272 };
273 let encoded = serde_json::to_value(&options).unwrap();
274 assert_eq!(
275 encoded,
276 serde_json::json!({
277 "metadata_size_hint": DEFAULT_METADATA_SIZE_HINT,
278 "range_coalesce_bytes": DEFAULT_RANGE_COALESCE_BYTES,
279 "range_fetch_concurrency": DEFAULT_RANGE_FETCH_CONCURRENCY,
280 "preload_column_index": false,
281 "preload_offset_index": false,
282 "preload_page_index": true,
283 }),
284 );
285 let decoded: ParquetReadOptions = serde_json::from_value(encoded).unwrap();
286 assert_eq!(decoded, options);
287 }
288
289 #[test]
290 fn partial_parquet_options_preserve_defaults() {
291 let options: ArrowReaderOptions =
292 serde_json::from_str(r#"{"parquet_options":{"range_fetch_concurrency":2}}"#).unwrap();
293 assert_eq!(
294 options,
295 ArrowReaderOptions::builder()
296 .with_parquet_options(
297 ParquetReadOptions::builder()
298 .with_range_fetch_concurrency(2)
299 .build(),
300 )
301 .build()
302 );
303 }
304}