Skip to main content

iceberg/arrow/reader/
mod.rs

1// Licensed to the Apache Software Foundation (ASF) under one
2// or more contributor license agreements.  See the NOTICE file
3// distributed with this work for additional information
4// regarding copyright ownership.  The ASF licenses this file
5// to you under the Apache License, Version 2.0 (the
6// "License"); you may not use this file except in compliance
7// with the License.  You may obtain a copy of the License at
8//
9//   http://www.apache.org/licenses/LICENSE-2.0
10//
11// Unless required by applicable law or agreed to in writing,
12// software distributed under the License is distributed on an
13// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14// KIND, either express or implied.  See the License for the
15// specific language governing permissions and limitations
16// under the License.
17
18//! Parquet file data reader
19
20use crate::arrow::caching_delete_file_loader::CachingDeleteFileLoader;
21use crate::io::FileIO;
22use crate::runtime::Runtime;
23use crate::util::available_parallelism;
24
25/// Default gap between byte ranges below which they are coalesced into a
26/// single request. Matches object_store's `OBJECT_STORE_COALESCE_DEFAULT`.
27const DEFAULT_RANGE_COALESCE_BYTES: u64 = 1024 * 1024;
28
29/// Default maximum number of coalesced byte ranges fetched concurrently.
30/// Matches object_store's `OBJECT_STORE_COALESCE_PARALLEL`.
31const DEFAULT_RANGE_FETCH_CONCURRENCY: usize = 10;
32
33/// Default number of bytes to prefetch when parsing Parquet footer metadata.
34/// Matches DataFusion's default `ParquetOptions::metadata_size_hint`.
35const DEFAULT_METADATA_SIZE_HINT: usize = 512 * 1024;
36
37mod file_reader;
38mod options;
39mod pipeline;
40mod positional_deletes;
41mod predicate_visitor;
42mod projection;
43mod row_filter;
44mod row_lineage;
45pub use file_reader::ArrowFileReader;
46pub use options::{ArrowReaderOptions, ParquetReadOptions};
47use predicate_visitor::{CollectFieldIdVisitor, PredicateConverter};
48use projection::{
49    add_fallback_field_ids_to_arrow_schema, apply_name_mapping_to_arrow_schema,
50    coerce_fixed_size_lists, find_leaf_by_field_id,
51};
52
53/// Builder to create ArrowReader
54pub struct ArrowReaderBuilder {
55    options: ArrowReaderOptions,
56    file_io: FileIO,
57    runtime: Runtime,
58}
59
60impl ArrowReaderBuilder {
61    /// Create a new ArrowReaderBuilder
62    pub fn new(file_io: FileIO, runtime: Runtime) -> Self {
63        ArrowReaderBuilder {
64            options: ArrowReaderOptions::default(),
65            file_io,
66            runtime,
67        }
68    }
69
70    /// Replaces all reader settings while preserving local I/O and runtime resources.
71    pub fn with_options(mut self, options: ArrowReaderOptions) -> Self {
72        self.options = options;
73        self
74    }
75
76    /// Replaces Parquet I/O settings while preserving other reader settings.
77    pub fn with_parquet_options(mut self, options: ParquetReadOptions) -> Self {
78        self.options = self.options.with_parquet_options(options);
79        self
80    }
81
82    /// Build the ArrowReader.
83    pub fn build(self) -> ArrowReader {
84        let options = self
85            .options
86            .with_data_file_concurrency_limit_if_unset(available_parallelism().get().max(1));
87        let concurrency_limit_data_files = options
88            .data_file_concurrency_limit()
89            .expect("reader concurrency was set above")
90            .max(1);
91        ArrowReader {
92            options,
93            file_io: self.file_io.clone(),
94            delete_file_loader: CachingDeleteFileLoader::new(
95                self.file_io.clone(),
96                concurrency_limit_data_files,
97                self.runtime.clone(),
98            ),
99            runtime: self.runtime.clone(),
100        }
101    }
102}
103
104/// Reads data from Parquet files
105#[derive(Clone)]
106pub struct ArrowReader {
107    options: ArrowReaderOptions,
108    file_io: FileIO,
109    delete_file_loader: CachingDeleteFileLoader,
110
111    runtime: Runtime,
112}
113
114#[cfg(test)]
115mod tests {
116    use std::time::Duration;
117
118    use futures::{TryStreamExt, stream};
119
120    use super::{ArrowReaderBuilder, ArrowReaderOptions, ParquetReadOptions};
121    use crate::io::FileIO;
122    use crate::test_utils::test_runtime;
123    use crate::util::available_parallelism;
124
125    #[tokio::test]
126    async fn deserialized_zero_concurrency_completes_reads() {
127        let options: ArrowReaderOptions =
128            serde_json::from_str(r#"{"data_file_concurrency_limit":0}"#).unwrap();
129        let reader = ArrowReaderBuilder::new(FileIO::new_with_memory(), test_runtime())
130            .with_options(options)
131            .build();
132        let batches = tokio::time::timeout(
133            Duration::from_secs(1),
134            reader
135                .read(Box::pin(stream::empty()))
136                .unwrap()
137                .stream()
138                .try_collect::<Vec<_>>(),
139        )
140        .await
141        .expect("zero concurrency must not stall the task stream")
142        .unwrap();
143        assert!(batches.is_empty());
144    }
145
146    #[test]
147    fn default_options_restore_local_reader_defaults() {
148        let reader = ArrowReaderBuilder::new(FileIO::new_with_memory(), test_runtime())
149            .with_options(
150                ArrowReaderOptions::builder()
151                    .with_batch_size(1024)
152                    .with_data_file_concurrency_limit(3)
153                    .with_row_group_filtering_enabled(false)
154                    .with_row_selection_enabled(true)
155                    .with_bloom_filter_enabled(true)
156                    .with_parquet_options(
157                        ParquetReadOptions::builder()
158                            .with_range_fetch_concurrency(99)
159                            .build(),
160                    )
161                    .build(),
162            )
163            .with_options(ArrowReaderOptions::default())
164            .build();
165        assert_eq!(
166            reader.options,
167            ArrowReaderOptions::builder()
168                .with_data_file_concurrency_limit(available_parallelism().get())
169                .build()
170        );
171    }
172
173    #[test]
174    fn explicit_options_configure_reader_settings() {
175        let parquet = ParquetReadOptions::builder()
176            .with_metadata_size_hint(4096)
177            .with_range_coalesce_bytes(8192)
178            .with_range_fetch_concurrency(2)
179            .build();
180        let options = ArrowReaderOptions::builder()
181            .with_batch_size(17)
182            .with_data_file_concurrency_limit(3)
183            .with_row_group_filtering_enabled(false)
184            .with_row_selection_enabled(true)
185            .with_bloom_filter_enabled(true)
186            .with_parquet_options(parquet)
187            .build();
188        let reader = ArrowReaderBuilder::new(FileIO::new_with_memory(), test_runtime())
189            .with_options(options.clone())
190            .build();
191        assert_eq!(reader.options, options);
192    }
193}