Skip to main content

iceberg/arrow/reader/
options.rs

1// Licensed to the Apache Software Foundation (ASF) under one
2// or more contributor license agreements.  See the NOTICE file
3// distributed with this work for additional information
4// regarding copyright ownership.  The ASF licenses this file
5// to you under the Apache License, Version 2.0 (the
6// "License"); you may not use this file except in compliance
7// with the License.  You may obtain a copy of the License at
8//
9//   http://www.apache.org/licenses/LICENSE-2.0
10//
11// Unless required by applicable law or agreed to in writing,
12// software distributed under the License is distributed on an
13// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14// KIND, either express or implied.  See the License for the
15// specific language governing permissions and limitations
16// under the License.
17
18use serde::{Deserialize, Serialize};
19use typed_builder::TypedBuilder;
20
21use super::{
22    DEFAULT_METADATA_SIZE_HINT, DEFAULT_RANGE_COALESCE_BYTES, DEFAULT_RANGE_FETCH_CONCURRENCY,
23};
24
25/// Configures Arrow reads independently of file planning and local I/O resources.
26///
27/// Serializable alongside planned file tasks so readers can be constructed later.
28/// In self-describing map formats such as JSON, field names are part of the
29/// serialized format: missing fields use defaults and unknown fields are ignored.
30/// Compatibility across different crate versions is not otherwise guaranteed.
31///
32/// ```no_run
33/// # use iceberg::{Result, arrow::{ArrowReaderOptions, ParquetReadOptions}, table::Table};
34/// # async fn read(table: &Table) -> Result<()> {
35/// let options = ArrowReaderOptions::builder()
36///     .with_batch_size(1024)
37///     .with_parquet_options(
38///         ParquetReadOptions::builder()
39///             .with_metadata_size_hint(512 * 1024)
40///             .build(),
41///     )
42///     .build();
43/// let stream = table.scan().build()?.to_arrow(options).await?;
44/// # Ok(())
45/// # }
46/// ```
47#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, TypedBuilder)]
48#[serde(default)]
49#[builder(field_defaults(default, setter(prefix = "with_")))]
50pub struct ArrowReaderOptions {
51    /// Desired batch size, or the reader default when unset.
52    #[builder(default = None, setter(strip_option))]
53    batch_size: Option<usize>,
54    /// Maximum in-flight data files, or the construction path's default when unset.
55    #[builder(default = None, setter(strip_option))]
56    data_file_concurrency_limit: Option<usize>,
57    /// Whether to prune row groups using their statistics.
58    #[builder(default = true)]
59    row_group_filtering_enabled: bool,
60    /// Whether to use page indexes for row selection.
61    row_selection_enabled: bool,
62    /// Whether to use bloom filters for row-group pruning.
63    bloom_filter_enabled: bool,
64    /// I/O tuning applied when reading Parquet files.
65    parquet_options: ParquetReadOptions,
66}
67
68impl Default for ArrowReaderOptions {
69    fn default() -> Self {
70        Self::builder().build()
71    }
72}
73
74impl ArrowReaderOptions {
75    /// Supplies a fallback limit without replacing an explicit reader limit.
76    pub(super) fn with_data_file_concurrency_limit_if_unset(mut self, limit: usize) -> Self {
77        self.data_file_concurrency_limit.get_or_insert(limit);
78        self
79    }
80
81    /// Returns the configured batch size.
82    pub fn batch_size(&self) -> Option<usize> {
83        self.batch_size
84    }
85
86    /// Returns the configured data file concurrency limit.
87    ///
88    /// When unset, reader construction uses execution-host parallelism.
89    /// Zero is treated as one.
90    pub fn data_file_concurrency_limit(&self) -> Option<usize> {
91        self.data_file_concurrency_limit
92    }
93
94    /// Returns whether row-group statistics filtering is enabled.
95    ///
96    /// Enabled by default: statistics can skip groups that cannot match the
97    /// predicate, generally improving performance with little overhead.
98    pub fn row_group_filtering_enabled(&self) -> bool {
99        self.row_group_filtering_enabled
100    }
101
102    /// Returns whether page-index row selection is enabled.
103    ///
104    /// Disabled by default: parsing page indexes can cost more than the reads
105    /// it avoids. Evaluate it against the table's sorting, partitioning, row-group
106    /// size, and page size before enabling it.
107    pub fn row_selection_enabled(&self) -> bool {
108        self.row_selection_enabled
109    }
110
111    /// Returns whether bloom-filter pruning is enabled.
112    ///
113    /// Disabled by default: bloom filters require extra reads per relevant
114    /// column and row group. They can eliminate groups for equality and IN predicates.
115    pub fn bloom_filter_enabled(&self) -> bool {
116        self.bloom_filter_enabled
117    }
118
119    /// Returns the configured parquet options.
120    pub fn parquet_options(&self) -> &ParquetReadOptions {
121        &self.parquet_options
122    }
123
124    pub(super) fn with_parquet_options(mut self, value: ParquetReadOptions) -> Self {
125        self.parquet_options = value;
126        self
127    }
128}
129
130/// Options for tuning Parquet file I/O.
131#[derive(Clone, Debug, PartialEq, Serialize, Deserialize, TypedBuilder)]
132#[serde(default)]
133#[builder(field_defaults(setter(prefix = "with_")))]
134pub struct ParquetReadOptions {
135    /// Number of bytes to prefetch for parsing the Parquet metadata.
136    ///
137    /// This hint can help reduce the number of fetch requests. For more details see the
138    /// [ParquetMetaDataReader documentation](https://docs.rs/parquet/latest/parquet/file/metadata/struct.ParquetMetaDataReader.html#method.with_prefetch_hint).
139    ///
140    /// Defaults to 512 KiB, matching DataFusion's default `ParquetOptions::metadata_size_hint`.
141    #[builder(default = DEFAULT_METADATA_SIZE_HINT)]
142    metadata_size_hint: usize,
143    /// Gap threshold for merging nearby byte ranges into a single request.
144    /// Ranges with gaps smaller than this value will be coalesced.
145    ///
146    /// Defaults to 1 MiB, matching object_store's `OBJECT_STORE_COALESCE_DEFAULT`.
147    #[builder(default = DEFAULT_RANGE_COALESCE_BYTES)]
148    range_coalesce_bytes: u64,
149    /// Maximum number of merged byte ranges to fetch concurrently.
150    ///
151    /// Defaults to 10, matching object_store's `OBJECT_STORE_COALESCE_PARALLEL`.
152    #[builder(default = DEFAULT_RANGE_FETCH_CONCURRENCY)]
153    range_fetch_concurrency: usize,
154    #[builder(default = true)]
155    preload_column_index: bool,
156    #[builder(default = true)]
157    preload_offset_index: bool,
158    /// Stored page-index flag; the reader derives preloading per data file from
159    /// the predicate and delete files, overriding this value.
160    #[builder(default = false)]
161    preload_page_index: bool,
162}
163
164impl Default for ParquetReadOptions {
165    fn default() -> Self {
166        Self::builder().build()
167    }
168}
169
170impl ParquetReadOptions {
171    pub(super) fn with_preload_page_index(mut self, enabled: bool) -> Self {
172        self.preload_page_index = enabled;
173        self
174    }
175
176    /// Returns whether to preload the column index when reading Parquet metadata.
177    pub fn preload_column_index(&self) -> bool {
178        self.preload_column_index
179    }
180
181    /// Returns whether to preload the offset index when reading Parquet metadata.
182    pub fn preload_offset_index(&self) -> bool {
183        self.preload_offset_index
184    }
185
186    /// Returns the stored page-index flag.
187    ///
188    /// The reader ignores this value: page-index preloading is derived per data
189    /// file from the predicate and delete files.
190    pub fn preload_page_index(&self) -> bool {
191        self.preload_page_index
192    }
193
194    /// Returns the configured metadata size hint.
195    pub fn metadata_size_hint(&self) -> usize {
196        self.metadata_size_hint
197    }
198
199    /// Returns the byte-gap threshold for coalescing read requests.
200    pub fn range_coalesce_bytes(&self) -> u64 {
201        self.range_coalesce_bytes
202    }
203
204    /// Returns the maximum number of concurrent range requests.
205    pub fn range_fetch_concurrency(&self) -> usize {
206        self.range_fetch_concurrency
207    }
208}
209
210#[cfg(test)]
211mod tests {
212    use rstest::rstest;
213
214    use super::*;
215
216    #[rstest]
217    #[case::unset(None, 7)]
218    #[case::explicit(Some(3), 3)]
219    #[case::zero(Some(0), 0)]
220    fn default_data_file_concurrency_preserves_explicit_options(
221        #[case] configured: Option<usize>,
222        #[case] expected: usize,
223    ) {
224        // Include unrelated settings to check that resolving concurrency preserves them.
225        let options: ArrowReaderOptions = serde_json::from_value(serde_json::json!({
226            "batch_size": 17,
227            "data_file_concurrency_limit": configured,
228            "row_selection_enabled": true,
229        }))
230        .unwrap();
231        // Resolve the reader fallback; explicit zero is normalized later by the reader.
232        let options = options.with_data_file_concurrency_limit_if_unset(7);
233        let expected = ArrowReaderOptions::builder()
234            .with_batch_size(17)
235            .with_data_file_concurrency_limit(expected)
236            .with_row_selection_enabled(true)
237            .build();
238        assert_eq!(options, expected);
239    }
240
241    #[test]
242    fn missing_options_use_defaults() {
243        let decoded: ArrowReaderOptions = serde_json::from_str(r#"{"batch_size":17}"#).unwrap();
244        assert_eq!(
245            decoded,
246            ArrowReaderOptions::builder().with_batch_size(17).build()
247        );
248    }
249
250    #[test]
251    fn unknown_options_are_ignored() {
252        let decoded: ArrowReaderOptions =
253            serde_json::from_str(r#"{"future_option":true}"#).unwrap();
254        assert_eq!(decoded, ArrowReaderOptions::default());
255    }
256
257    #[test]
258    fn unset_concurrency_serializes_as_null() {
259        assert_eq!(
260            serde_json::to_value(ArrowReaderOptions::default()).unwrap()["data_file_concurrency_limit"],
261            serde_json::Value::Null,
262        );
263    }
264
265    #[test]
266    fn parquet_serialization_preserves_preload_settings() {
267        let options = ParquetReadOptions {
268            preload_column_index: false,
269            preload_offset_index: false,
270            preload_page_index: true,
271            ..ParquetReadOptions::default()
272        };
273        let encoded = serde_json::to_value(&options).unwrap();
274        assert_eq!(
275            encoded,
276            serde_json::json!({
277                "metadata_size_hint": DEFAULT_METADATA_SIZE_HINT,
278                "range_coalesce_bytes": DEFAULT_RANGE_COALESCE_BYTES,
279                "range_fetch_concurrency": DEFAULT_RANGE_FETCH_CONCURRENCY,
280                "preload_column_index": false,
281                "preload_offset_index": false,
282                "preload_page_index": true,
283            }),
284        );
285        let decoded: ParquetReadOptions = serde_json::from_value(encoded).unwrap();
286        assert_eq!(decoded, options);
287    }
288
289    #[test]
290    fn partial_parquet_options_preserve_defaults() {
291        let options: ArrowReaderOptions =
292            serde_json::from_str(r#"{"parquet_options":{"range_fetch_concurrency":2}}"#).unwrap();
293        assert_eq!(
294            options,
295            ArrowReaderOptions::builder()
296                .with_parquet_options(
297                    ParquetReadOptions::builder()
298                        .with_range_fetch_concurrency(2)
299                        .build(),
300                )
301                .build()
302        );
303    }
304}