Skip to main content

iceberg/arrow/reader/
pipeline.rs

1// Licensed to the Apache Software Foundation (ASF) under one
2// or more contributor license agreements.  See the NOTICE file
3// distributed with this work for additional information
4// regarding copyright ownership.  The ASF licenses this file
5// to you under the Apache License, Version 2.0 (the
6// "License"); you may not use this file except in compliance
7// with the License.  You may obtain a copy of the License at
8//
9//   http://www.apache.org/licenses/LICENSE-2.0
10//
11// Unless required by applicable law or agreed to in writing,
12// software distributed under the License is distributed on an
13// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14// KIND, either express or implied.  See the License for the
15// specific language governing permissions and limitations
16// under the License.
17
18//! The main `ArrowReader` pipeline: reading a stream of `FileScanTask`s,
19//! opening Parquet files and resolving schemas, then wiring projection,
20//! predicates, row-group / row selection, and delete handling into a stream
21//! of transformed Arrow `RecordBatch`es.
22
23use std::borrow::Cow;
24use std::collections::HashMap;
25use std::sync::Arc;
26use std::sync::atomic::AtomicU64;
27
28use arrow_schema::{DataType, Field};
29use futures::{StreamExt, TryStreamExt};
30use parquet::arrow::arrow_reader::{ArrowReaderMetadata, ArrowReaderOptions};
31use parquet::arrow::{
32    PARQUET_FIELD_ID_META_KEY, ParquetRecordBatchStreamBuilder, ProjectionMask, RowNumber,
33};
34use parquet::encryption::decrypt::FileDecryptionProperties;
35
36use super::row_lineage::synthesize_row_id_column;
37use super::{
38    ArrowFileReader, ArrowReader, ParquetReadOptions, add_fallback_field_ids_to_arrow_schema,
39    apply_name_mapping_to_arrow_schema, coerce_fixed_size_lists, find_leaf_by_field_id,
40};
41use crate::arrow::build_partition_constant;
42use crate::arrow::caching_delete_file_loader::CachingDeleteFileLoader;
43use crate::arrow::int96::coerce_int96_timestamps;
44use crate::arrow::record_batch_transformer::RecordBatchTransformerBuilder;
45use crate::arrow::scan_metrics::{CountingFileRead, ScanMetrics, ScanResult};
46use crate::encryption::StandardKeyMetadata;
47use crate::error::{Result, invalid_data};
48use crate::expr::BoundPredicate;
49use crate::expr::visitors::bloom_filter_evaluator::{
50    BloomFilterEvaluator, ColumnBloomFilter, collect_bloom_filter_field_ids,
51};
52use crate::io::{FileIO, FileMetadata, FileRead};
53use crate::metadata_columns::{
54    RESERVED_COL_NAME_LAST_UPDATED_SEQUENCE_NUMBER, RESERVED_COL_NAME_POS,
55    RESERVED_COL_NAME_ROW_ID, RESERVED_FIELD_ID_FILE,
56    RESERVED_FIELD_ID_LAST_UPDATED_SEQUENCE_NUMBER, RESERVED_FIELD_ID_PARTITION,
57    RESERVED_FIELD_ID_POS, RESERVED_FIELD_ID_ROW_ID, RESERVED_FIELD_ID_SPEC_ID, is_metadata_field,
58};
59use crate::runtime::Runtime;
60use crate::scan::{ArrowRecordBatchStream, FileScanTask, FileScanTaskStream};
61use crate::spec::{Datum, PartitionSpec, Struct};
62use crate::{Error, ErrorKind};
63
64impl ArrowReader {
65    /// Take a stream of FileScanTasks and reads all the files.
66    /// Returns a [`ScanResult`] containing the record batch stream and scan metrics.
67    pub fn read(self, tasks: FileScanTaskStream) -> Result<ScanResult> {
68        let concurrency_limit_data_files = self
69            .options
70            .data_file_concurrency_limit()
71            .expect("reader construction sets concurrency")
72            .max(1);
73        let scan_metrics = ScanMetrics::new();
74
75        let task_reader = FileScanTaskReader {
76            batch_size: self.options.batch_size(),
77            file_io: self.file_io,
78            delete_file_loader: self
79                .delete_file_loader
80                .with_scan_metrics(scan_metrics.clone()),
81            row_group_filtering_enabled: self.options.row_group_filtering_enabled(),
82            row_selection_enabled: self.options.row_selection_enabled(),
83            bloom_filter_enabled: self.options.bloom_filter_enabled(),
84            parquet_read_options: self.options.parquet_options().clone(),
85            scan_metrics: scan_metrics.clone(),
86            runtime: self.runtime,
87        };
88
89        // Fast-path for single concurrency to avoid overhead of try_flatten_unordered
90        let stream: ArrowRecordBatchStream = if concurrency_limit_data_files == 1 {
91            Box::pin(
92                tasks
93                    .and_then(move |task| task_reader.clone().process(task))
94                    .map_err(|err| {
95                        Error::new(ErrorKind::Unexpected, "file scan task generate failed")
96                            .with_source(err)
97                    })
98                    .try_flatten(),
99            )
100        } else {
101            Box::pin(
102                tasks
103                    .map_ok(move |task| task_reader.clone().process(task))
104                    .map_err(|err| {
105                        Error::new(ErrorKind::Unexpected, "file scan task generate failed")
106                            .with_source(err)
107                    })
108                    .try_buffer_unordered(concurrency_limit_data_files)
109                    .try_flatten_unordered(concurrency_limit_data_files),
110            )
111        };
112
113        Ok(ScanResult::new(stream, scan_metrics))
114    }
115}
116
117// Metadata columns synthesized without reading any data column, so a projection of only
118// these (including the empty projection, e.g. `COUNT(*)`) can be pruned to zero data
119// columns. Narrower than `is_metadata_field`, which also matches `_deleted` -- excluded here
120// because it has no synthesis handler.
121const PRUNABLE_METADATA_FIELDS: &[i32] = &[
122    RESERVED_FIELD_ID_FILE,
123    RESERVED_FIELD_ID_SPEC_ID,
124    RESERVED_FIELD_ID_PARTITION,
125    RESERVED_FIELD_ID_POS,
126    RESERVED_FIELD_ID_ROW_ID,
127    RESERVED_FIELD_ID_LAST_UPDATED_SEQUENCE_NUMBER,
128];
129
130/// Per-scan state for processing [`FileScanTask`]s. Created once per
131/// [`ArrowReader::read`] call and cloned per task.
132#[derive(Clone)]
133struct FileScanTaskReader {
134    batch_size: Option<usize>,
135    file_io: FileIO,
136    delete_file_loader: CachingDeleteFileLoader,
137    row_group_filtering_enabled: bool,
138    row_selection_enabled: bool,
139    bloom_filter_enabled: bool,
140    parquet_read_options: ParquetReadOptions,
141    scan_metrics: ScanMetrics,
142    runtime: Runtime,
143}
144
145// Borrow an existing selection, or enumerate all groups for a whole-file scan.
146fn candidate_row_groups(
147    selected_row_groups: Option<&[usize]>,
148    num_row_groups: usize,
149) -> Cow<'_, [usize]> {
150    match selected_row_groups {
151        Some(indices) => Cow::Borrowed(indices),
152        None => Cow::Owned((0..num_row_groups).collect()),
153    }
154}
155
156impl FileScanTaskReader {
157    async fn process(self, task: FileScanTask) -> Result<ArrowRecordBatchStream> {
158        // A known-empty data file contributes no rows, regardless of predicates or deletes.
159        if task.record_count() == Some(0) {
160            return Ok(Box::pin(futures::stream::empty()));
161        }
162
163        let should_load_page_index = (self.row_selection_enabled && task.predicate().is_some())
164            || !task.deletes().is_empty();
165
166        let delete_filter_rx = self
167            .delete_file_loader
168            .load_deletes(task.deletes(), task.schema_ref());
169
170        // `try_flatten_unordered` stops polling the buffered opens once it holds
171        // `concurrency_limit_data_files` streams, so spawn the open (#3276).
172        // Footer and page-index decode also run on the IO runtime.
173        // Aborts on drop.
174        let path = task.data_file_path().to_string();
175        let file_io = self.file_io.clone();
176        let file_size_in_bytes = task.file_size_in_bytes();
177        let bytes_read = Arc::clone(self.scan_metrics.bytes_read_counter());
178        let key_metadata = task.key_metadata().map(|km| km.to_vec());
179        let parquet_read_options = self
180            .parquet_read_options
181            .clone()
182            .with_preload_page_index(should_load_page_index);
183        let opening = self
184            .runtime
185            .io()
186            .spawn(async move {
187                ArrowReader::open_parquet_file(
188                    &path,
189                    &file_io,
190                    file_size_in_bytes,
191                    parquet_read_options,
192                    &bytes_read,
193                    key_metadata.as_deref(),
194                )
195                .await
196            })
197            .abort_on_drop();
198        let (parquet_file_reader, arrow_metadata) = opening.await??;
199
200        // Check if Parquet file has embedded field IDs
201        // Corresponds to Java's ParquetSchemaUtil.hasIds()
202        // Reference: parquet/src/main/java/org/apache/iceberg/parquet/ParquetSchemaUtil.java:118
203        let missing_field_ids = arrow_metadata
204            .schema()
205            .fields()
206            .iter()
207            .next()
208            .is_some_and(|f| f.metadata().get(PARQUET_FIELD_ID_META_KEY).is_none());
209
210        // Position-based fallback applies only when the file has no embedded field IDs
211        // AND no name mapping is available. With a name mapping, field IDs are assigned
212        // to the Arrow schema below, and projection/predicate planning must use them
213        // (see #2403).
214        let use_position_fallback = missing_field_ids && task.name_mapping().is_none();
215
216        let project_pos = task.project_field_ids().contains(&RESERVED_FIELD_ID_POS);
217        let project_row_id = task.project_field_ids().contains(&RESERVED_FIELD_ID_ROW_ID);
218
219        // The RowNumber virtual column materializes `_pos`. It is also the per-row
220        // positional fallback for `_row_id` (`first_row_id + pos`), so add it whenever
221        // `_row_id` is synthesized. A null `first_row_id` nulls the whole `_row_id`
222        // column, so nothing is synthesized and the column is not needed.
223        let need_row_number = project_pos || (project_row_id && task.first_row_id().is_some());
224
225        let field_ids = task.project_field_ids();
226        // Vacuously true for an empty projection (e.g. `COUNT(*)`), so no `is_empty()` guard.
227        let no_data_columns_projected = field_ids
228            .iter()
229            .all(|id| PRUNABLE_METADATA_FIELDS.contains(id));
230
231        let install_row_number = need_row_number || no_data_columns_projected;
232
233        let arrow_metadata = Self::configure_arrow_reader_metadata(
234            arrow_metadata,
235            &task,
236            missing_field_ids,
237            install_row_number,
238        )?;
239
240        // Build the stream reader, reusing the already-opened file reader
241        let mut record_batch_stream_builder =
242            ParquetRecordBatchStreamBuilder::new_with_metadata(parquet_file_reader, arrow_metadata);
243
244        // Whether the file physically carries the `_last_updated_sequence_number` column
245        // (some engines, e.g. Iceberg Java on rewrite, write it per-row), resolved by its
246        // embedded field id against the Parquet schema.
247        let project_last_updated_seq = task
248            .project_field_ids()
249            .contains(&RESERVED_FIELD_ID_LAST_UPDATED_SEQUENCE_NUMBER);
250
251        // Parquet leaf index of the physically-stored column, resolved by its embedded
252        // reserved field id. `find_leaf_by_field_id` tolerates id-less leaves (e.g. a
253        // Variant column's internal metadata/value leaves, which the spec requires to have
254        // no id), so an unprojected variant alongside a metadata column with correct ID does
255        // not hide it.
256        let phys_last_updated_seq_leaf = if project_last_updated_seq {
257            find_leaf_by_field_id(
258                record_batch_stream_builder.parquet_schema(),
259                RESERVED_FIELD_ID_LAST_UPDATED_SEQUENCE_NUMBER,
260            )
261        } else {
262            None
263        };
264
265        // Present by name but not by the embedded id (only meaningful when no by-id column
266        // was found). An unthreadable shape we reject rather than coalesce incorrectly.
267        let last_updated_seq_present_by_name_only = project_last_updated_seq
268            && phys_last_updated_seq_leaf.is_none()
269            && record_batch_stream_builder
270                .schema()
271                .fields()
272                .iter()
273                .any(|f| f.name() == RESERVED_COL_NAME_LAST_UPDATED_SEQUENCE_NUMBER);
274
275        // Read the physical column only when first_row_id is set (with a data sequence
276        // number to fall back to). A null first_row_id drops the leaf and nulls the whole
277        // column below, discarding any per-row values the file carries -- matching Java
278        // (`ValueReaders.lastUpdated` nulls when the base row id is null).
279        let coalesce_last_updated_seq_leaf = phys_last_updated_seq_leaf
280            .filter(|_| task.first_row_id().is_some() && task.data_sequence_number().is_some());
281
282        let phys_row_id_leaf = if project_row_id {
283            find_leaf_by_field_id(
284                record_batch_stream_builder.parquet_schema(),
285                RESERVED_FIELD_ID_ROW_ID,
286            )
287        } else {
288            None
289        };
290
291        // A column named `_row_id` that carries no embedded field id, in a file that DOES
292        // use embedded ids -- an unthreadable physically-stored `_row_id`, rejected below.
293        // Gated on `!use_position_fallback`: under positional fallback every column lacks an
294        // embedded id and synthetic ids are assigned by position, so a user column that
295        // happens to be named `_row_id` is real data, not a reserved metadata column.
296        let row_id_present_by_name_only = project_row_id
297            && !use_position_fallback
298            && phys_row_id_leaf.is_none()
299            && record_batch_stream_builder
300                .schema()
301                .fields()
302                .iter()
303                .any(|f| f.name() == RESERVED_COL_NAME_ROW_ID);
304
305        // Read the physical column only when first_row_id is set. A null first_row_id
306        // nulls the whole column below (matching Java `ValueReaders.rowIds`).
307        let coalesce_row_id_leaf = phys_row_id_leaf.filter(|_| task.first_row_id().is_some());
308
309        // Filter out metadata fields for Parquet projection (they don't exist in files)
310        let project_field_ids_without_metadata: Vec<i32> = task
311            .project_field_ids()
312            .iter()
313            .filter(|&&id| !is_metadata_field(id))
314            .copied()
315            .collect();
316
317        // Create projection mask based on field IDs
318        // - If file has embedded IDs: field-ID-based projection
319        // - If name mapping applied: field-ID-based projection using the IDs the name
320        //   mapping assigned to the Arrow schema
321        // - Otherwise: position-based fallback projection
322        let mut projection_mask = ArrowReader::get_arrow_projection_mask(
323            &project_field_ids_without_metadata,
324            task.schema(),
325            record_batch_stream_builder.parquet_schema(),
326            record_batch_stream_builder.schema(),
327            use_position_fallback, // Whether to use position-based (true) or field-ID-based (false) projection
328        )?;
329
330        // A projection with no data columns leaves `project_field_ids_without_metadata`
331        // empty, which `get_arrow_projection_mask` maps to "read all columns". Downgrade that
332        // to "read no data columns": `install_row_number` put the RowNumber virtual column on
333        // every such projection as a row-count source independent of the data columns, so the
334        // count survives with zero data columns read.
335        //
336        // This runs BEFORE the union so any physical metadata leaf is added onto a `none`
337        // base, pruning the read to just that leaf (`union` with an `all` base stays `all`).
338        if project_field_ids_without_metadata.is_empty() && install_row_number {
339            projection_mask =
340                ProjectionMask::none(record_batch_stream_builder.parquet_schema().num_columns());
341        }
342
343        // Union in the physical leaves of any metadata columns we will coalesce. Their
344        // reserved field ids are not in the task schema, so they can't be requested through
345        // `get_arrow_projection_mask` (which resolves ids against the task schema); add
346        // their Parquet leaves directly.
347        for leaf in [coalesce_last_updated_seq_leaf, coalesce_row_id_leaf]
348            .into_iter()
349            .flatten()
350        {
351            let phys_mask =
352                ProjectionMask::leaves(record_batch_stream_builder.parquet_schema(), vec![leaf]);
353            projection_mask.union(&phys_mask);
354        }
355
356        record_batch_stream_builder =
357            record_batch_stream_builder.with_projection(projection_mask.clone());
358
359        // RecordBatchTransformer performs any transformations required on the RecordBatches
360        // that come back from the file, such as type promotion, default column insertion,
361        // column re-ordering, partition constants, and virtual field addition (like _file)
362        let mut record_batch_transformer_builder =
363            RecordBatchTransformerBuilder::new(task.schema_ref(), task.project_field_ids());
364
365        // Add the _file metadata column if it's in the projected fields
366        if task.project_field_ids().contains(&RESERVED_FIELD_ID_FILE) {
367            let file_datum = Datum::string(task.data_file_path().to_string());
368            record_batch_transformer_builder =
369                record_batch_transformer_builder.with_constant(RESERVED_FIELD_ID_FILE, file_datum);
370        }
371
372        if task
373            .project_field_ids()
374            .contains(&RESERVED_FIELD_ID_SPEC_ID)
375        {
376            let partition_spec = task
377                .partition_spec()
378                .ok_or_else(|| Error::new(ErrorKind::Unexpected, "Partition spec is missing"))?;
379
380            let spec_id_datum = Datum::int(partition_spec.spec_id());
381            record_batch_transformer_builder = record_batch_transformer_builder
382                .with_constant(RESERVED_FIELD_ID_SPEC_ID, spec_id_datum);
383        }
384
385        if project_last_updated_seq {
386            // Materialize the column, gated on the data file's `first_row_id`. Java gates
387            // it this way (`ValueReaders.lastUpdated` returns nulls when the base row id is
388            // null); the spec itself only says the column is assigned the manifest entry's
389            // sequence number on read.
390            record_batch_transformer_builder =
391                match (task.first_row_id(), task.data_sequence_number()) {
392                    (Some(_), Some(seq)) => {
393                        let datum = Datum::long(seq);
394                        if coalesce_last_updated_seq_leaf.is_some() {
395                            // The file physically carries the column: read the per-row value,
396                            // falling back to the data sequence number only where null.
397                            record_batch_transformer_builder.with_coalesced_last_updated_seq_column(
398                                RESERVED_FIELD_ID_LAST_UPDATED_SEQUENCE_NUMBER,
399                                datum,
400                            )
401                        } else if last_updated_seq_present_by_name_only {
402                            // Present by name but without the embedded field id (name mapping /
403                            // positional fallback). The transformer keys the source column by
404                            // field id, so we can't thread it; no real writer produces this, so
405                            // reject loudly rather than silently overwrite with the constant.
406                            // Arm-local by design: only this arm reads the physical column, so
407                            // only here can a name-only column defeat us. The `(None, _)` arm
408                            // nulls the column without reading it, so it needs no such guard.
409                            return Err(Error::new(
410                                ErrorKind::FeatureUnsupported,
411                                "Reading a physically-stored _last_updated_sequence_number column \
412                             without an embedded field id is not supported",
413                            ));
414                        } else {
415                            // Column absent: derive it from the data sequence number.
416                            record_batch_transformer_builder.with_constant(
417                                RESERVED_FIELD_ID_LAST_UPDATED_SEQUENCE_NUMBER,
418                                datum,
419                            )
420                        }
421                    }
422                    // Null first_row_id (v1/v2, or a pre-upgrade v3 snapshot): the column is null.
423                    (None, _) => record_batch_transformer_builder.with_null_metadata_column(
424                        RESERVED_FIELD_ID_LAST_UPDATED_SEQUENCE_NUMBER,
425                    )?,
426                    // first_row_id present but no data sequence number: after manifest
427                    // inheritance a committed entry always has one, so this is a malformed
428                    // manifest rather than a legitimate null.
429                    (Some(_), None) => {
430                        return Err(invalid_data!(
431                            "Data file {} has a first_row_id but no data sequence number",
432                            task.data_file_path()
433                        ));
434                    }
435                };
436        }
437
438        if project_row_id {
439            // A name-only physical `_row_id` can't be threaded (synthesis keys the leaf by
440            // its reserved field id). Reject only when `first_row_id` is set; otherwise the
441            // leaf is never read and `_row_id` is nulled out downstream (matching Java
442            // `ValueReaders.rowIds`), so a pre-v3 file with a user column named `_row_id`
443            // reads back as null rather than erroring.
444            if task.first_row_id().is_some() && row_id_present_by_name_only {
445                return Err(Error::new(
446                    ErrorKind::FeatureUnsupported,
447                    "Reading a physically-stored _row_id column without an embedded field id \
448                     is not supported",
449                ));
450            }
451
452            // `_row_id` is synthesized downstream over the record-batch stream (see
453            // `row_lineage::synthesize_row_id_column`); the transformer only passes the
454            // resulting column through, like `_pos`.
455            record_batch_transformer_builder =
456                record_batch_transformer_builder.with_virtual_field(RESERVED_FIELD_ID_ROW_ID);
457        }
458
459        if let (Some(partition_spec), Some(partition_data)) =
460            (task.partition_spec(), task.partition())
461        {
462            record_batch_transformer_builder = record_batch_transformer_builder
463                .with_partition(Arc::clone(partition_spec), partition_data.clone())?;
464        }
465
466        if project_pos {
467            record_batch_transformer_builder =
468                record_batch_transformer_builder.with_virtual_field(RESERVED_FIELD_ID_POS);
469        }
470
471        // Add the _partition metadata struct column if it's in the projected fields.
472        // Computed lazily here at read time from the unified partition type + task's spec + data.
473        if task
474            .project_field_ids()
475            .contains(&RESERVED_FIELD_ID_PARTITION)
476            && let Some(unified_type) = task.unified_partition_type()
477        {
478            let (spec, partition_data) = match (task.partition_spec(), task.partition()) {
479                (Some(spec), Some(data)) => (Arc::clone(spec), data.clone()),
480                // A missing spec/data is only acceptable when there are no partition
481                // fields to fill (unpartitioned table). If the unified type has fields
482                // but we lack a spec or data, the task is inconsistent and we cannot
483                // build the _partition column.
484                _ if unified_type.fields().is_empty() => {
485                    (Arc::new(PartitionSpec::unpartition_spec()), Struct::empty())
486                }
487                _ => {
488                    return Err(Error::new(
489                        ErrorKind::Unexpected,
490                        "cannot build _partition column: unified partition type has fields \
491                         but the scan task is missing its partition spec or data",
492                    ));
493                }
494            };
495            let constant = build_partition_constant(unified_type, &spec, &partition_data)?;
496            record_batch_transformer_builder =
497                record_batch_transformer_builder.with_partition_constant(constant);
498        }
499
500        let mut record_batch_transformer = record_batch_transformer_builder.build();
501
502        if let Some(batch_size) = self.batch_size {
503            record_batch_stream_builder = record_batch_stream_builder.with_batch_size(batch_size);
504        }
505
506        let delete_filter = delete_filter_rx.await.unwrap()?;
507        let delete_predicate = delete_filter.build_equality_delete_predicate(&task).await?;
508
509        // In addition to the optional predicate supplied in the `FileScanTask`,
510        // we also have an optional predicate resulting from equality delete files.
511        // If both are present, we logical-AND them together to form a single filter
512        // predicate that we can pass to the `RecordBatchStreamBuilder`.
513        let final_predicate = match (task.predicate(), delete_predicate) {
514            (None, None) => None,
515            (Some(predicate), None) => Some(predicate.clone()),
516            (None, Some(ref predicate)) => Some(predicate.clone()),
517            (Some(filter_predicate), Some(delete_predicate)) => {
518                Some(filter_predicate.clone().and(delete_predicate))
519            }
520        };
521
522        // There are three possible sources for potential lists of selected RowGroup indices,
523        // and two for `RowSelection`s.
524        // Selected RowGroup index lists can come from three sources:
525        //   * When task.start and task.length specify a byte range (file splitting);
526        //   * When there are equality delete files that are applicable;
527        //   * When there is a scan predicate and row_group_filtering_enabled = true.
528        // `RowSelection`s can be created in either or both of the following cases:
529        //   * When there are positional delete files that are applicable;
530        //   * When there is a scan predicate and row_selection_enabled = true
531        // Note that row group filtering from predicates only happens when
532        // there is a scan predicate AND row_group_filtering_enabled = true,
533        // but we perform row selection filtering if there are applicable
534        // equality delete files OR (there is a scan predicate AND row_selection_enabled),
535        // since the only implemented method of applying positional deletes is
536        // by using a `RowSelection`.
537        let mut selected_row_group_indices = None;
538        let mut row_selection = None;
539
540        // Filter row groups based on byte range from task.start and task.length.
541        // If both start and length are 0, read the entire file (backwards compatibility).
542        if task.start() != 0 || task.length() != 0 {
543            let byte_range_filtered_row_groups = ArrowReader::filter_row_groups_by_byte_range(
544                record_batch_stream_builder.metadata(),
545                task.start(),
546                task.length(),
547            )?;
548            selected_row_group_indices = Some(byte_range_filtered_row_groups);
549        }
550
551        if let Some(predicate) = final_predicate {
552            // Normalize NOT out of the predicate once for every consumer below. The
553            // conservative row-group and page-index evaluators cannot soundly negate
554            // a NOT node (negating a "might match" bound can prune matching rows, and
555            // the page-index evaluator rejects NOT outright). A FileScanTask built or
556            // deserialized outside TableScan carries an un-normalized predicate, so
557            // rewrite it here. Matches TableScan::with_filter.
558            let predicate = predicate.rewrite_not();
559
560            let (iceberg_field_ids, field_id_map) = ArrowReader::build_field_id_set_and_map(
561                record_batch_stream_builder.parquet_schema(),
562                record_batch_stream_builder.schema(),
563                &predicate,
564                use_position_fallback,
565            )?;
566
567            let row_filter = ArrowReader::get_row_filter(
568                &predicate,
569                record_batch_stream_builder.parquet_schema(),
570                &iceberg_field_ids,
571                &field_id_map,
572            )?;
573            record_batch_stream_builder = record_batch_stream_builder.with_row_filter(row_filter);
574
575            if self.row_group_filtering_enabled {
576                let candidates = candidate_row_groups(
577                    selected_row_group_indices.as_deref(),
578                    record_batch_stream_builder.metadata().num_row_groups(),
579                );
580
581                selected_row_group_indices = Some(ArrowReader::get_selected_row_group_indices(
582                    &predicate,
583                    record_batch_stream_builder.metadata(),
584                    &field_id_map,
585                    task.schema(),
586                    candidates.as_ref(),
587                )?);
588            }
589
590            if self.bloom_filter_enabled {
591                let candidates = candidate_row_groups(
592                    selected_row_group_indices.as_deref(),
593                    record_batch_stream_builder.metadata().num_row_groups(),
594                );
595
596                // Spawned for the same reason as the open (#3276).
597                let bloom_predicate = predicate.clone();
598                let bloom_candidates = candidates.as_ref().to_vec();
599                let bloom_field_ids = field_id_map.clone();
600                let mut builder = record_batch_stream_builder;
601                let (builder, bloom_filtered) = self
602                    .runtime
603                    .io()
604                    .spawn(async move {
605                        let filtered = Self::filter_row_groups_by_bloom_filter(
606                            &bloom_predicate,
607                            &mut builder,
608                            &bloom_candidates,
609                            &bloom_field_ids,
610                        )
611                        .await;
612                        (builder, filtered)
613                    })
614                    .abort_on_drop()
615                    .await?;
616                record_batch_stream_builder = builder;
617                let bloom_filtered = bloom_filtered?;
618
619                if bloom_filtered.len() < candidates.len() {
620                    selected_row_group_indices = Some(bloom_filtered);
621                }
622            }
623
624            if self.row_selection_enabled {
625                row_selection = ArrowReader::get_row_selection_for_filter_predicate(
626                    &predicate,
627                    record_batch_stream_builder.metadata(),
628                    &selected_row_group_indices,
629                    &field_id_map,
630                    task.schema(),
631                )?;
632            }
633        }
634
635        let positional_delete_indexes = delete_filter.get_delete_vector(&task);
636
637        if let Some(positional_delete_indexes) = positional_delete_indexes {
638            let delete_row_selection = {
639                let positional_delete_indexes = positional_delete_indexes.lock().unwrap();
640
641                ArrowReader::build_deletes_row_selection(
642                    record_batch_stream_builder.metadata().row_groups(),
643                    &selected_row_group_indices,
644                    &positional_delete_indexes,
645                )
646            }?;
647
648            // merge the row selection from the delete files with the row selection
649            // from the filter predicate, if there is one from the filter predicate
650            row_selection = match row_selection {
651                None => Some(delete_row_selection),
652                Some(filter_row_selection) => {
653                    Some(filter_row_selection.intersection(&delete_row_selection))
654                }
655            };
656        }
657
658        if let Some(row_selection) = row_selection {
659            record_batch_stream_builder =
660                record_batch_stream_builder.with_row_selection(row_selection);
661        }
662
663        if let Some(selected_row_group_indices) = selected_row_group_indices {
664            record_batch_stream_builder =
665                record_batch_stream_builder.with_row_groups(selected_row_group_indices);
666        }
667
668        // Build the batch stream and send all the RecordBatches that it generates
669        // to the requester. When `_row_id` is projected, synthesize it over the raw parquet
670        // batches (using the reader-produced `_pos` position) before the transformer, which
671        // then passes it through as a virtual field.
672        let first_row_id = task.first_row_id();
673        let record_batch_stream = record_batch_stream_builder.build()?.map(move |batch| {
674            let mut batch = batch.map_err(|err| -> Error { err.into() })?;
675            if project_row_id {
676                batch = synthesize_row_id_column(batch, first_row_id)?;
677            }
678            // Process the record batch (type promotion, column reordering, virtual fields, etc.)
679            record_batch_transformer.process_record_batch(batch)
680        });
681
682        Ok(Box::pin(record_batch_stream) as ArrowRecordBatchStream)
683    }
684
685    /// Applies all task-specific schema and virtual-column options, rebuilding the
686    /// Arrow reader metadata at most once.
687    fn configure_arrow_reader_metadata(
688        arrow_metadata: ArrowReaderMetadata,
689        task: &FileScanTask,
690        missing_field_ids: bool,
691        install_row_number: bool,
692    ) -> Result<ArrowReaderMetadata> {
693        // Schema resolution follows the Iceberg Column Projection rule:
694        // "Columns in Iceberg data files are selected by field id."
695        // https://iceberg.apache.org/spec/#column-projection
696        //
697        // This mirrors Java's ReadConf strategy: use embedded IDs with pruneColumns();
698        // otherwise, use applyNameMapping() followed by pruneColumns() when configured, or
699        // addFallbackIds() followed by pruneColumnsFallback() for position-based fallback.
700        // The fast path returns early without materializing an owned schema.
701        //
702        // Rewrite FixedSizeList hints to List before name mapping / fallback IDs so
703        // parquet-rs uses the 3-level LIST reader. Same placement as #3467's list-view rewrite.
704        let without_fixed_size_lists = coerce_fixed_size_lists(arrow_metadata.schema());
705        let file_schema = without_fixed_size_lists
706            .as_ref()
707            .unwrap_or(arrow_metadata.schema());
708        let arrow_schema = if missing_field_ids {
709            let schema = if let Some(name_mapping) = task.name_mapping() {
710                apply_name_mapping_to_arrow_schema(Arc::clone(file_schema), name_mapping)?
711            } else {
712                add_fallback_field_ids_to_arrow_schema(file_schema)
713            };
714            // Coerce INT96 timestamp columns before building the stream reader to avoid
715            // i64 overflow in arrow-rs. Apply this after assigning any missing field IDs
716            // so the final schema contains both changes.
717            coerce_int96_timestamps(&schema, task.schema()).unwrap_or(schema)
718        } else if let Some(coerced) = coerce_int96_timestamps(file_schema, task.schema()) {
719            coerced
720        } else if without_fixed_size_lists.is_some() || install_row_number {
721            Arc::clone(file_schema)
722        } else {
723            return Ok(arrow_metadata);
724        };
725
726        let mut options = ArrowReaderOptions::new().with_schema(Arc::clone(&arrow_schema));
727        if install_row_number {
728            let row_number_field = Arc::new(
729                Field::new(RESERVED_COL_NAME_POS, DataType::Int64, false)
730                    .with_metadata(HashMap::from([(
731                        PARQUET_FIELD_ID_META_KEY.to_string(),
732                        RESERVED_FIELD_ID_POS.to_string(),
733                    )]))
734                    .with_extension_type(RowNumber),
735            );
736            options = options.with_virtual_columns(vec![row_number_field])?;
737        }
738
739        ArrowReaderMetadata::try_new(Arc::clone(arrow_metadata.metadata()), options).map_err(|e| {
740            Error::new(
741                ErrorKind::Unexpected,
742                format!(
743                    "Failed to create ArrowReaderMetadata with the configured reader options \
744                     (missing_field_ids: {missing_field_ids}, \
745                      install_row_number: {install_row_number}, schema: {arrow_schema})"
746                ),
747            )
748            .with_source(e)
749        })
750    }
751
752    /// Reads bloom filters for relevant columns and evaluates the predicate
753    /// against them to filter out row groups that definitely don't match.
754    async fn filter_row_groups_by_bloom_filter(
755        predicate: &BoundPredicate,
756        builder: &mut ParquetRecordBatchStreamBuilder<ArrowFileReader>,
757        candidate_row_groups: &[usize],
758        field_id_map: &HashMap<i32, usize>,
759    ) -> Result<Vec<usize>> {
760        // Only collect field IDs from eq/in predicates — the only types
761        // bloom filters can help with. Skip columns not in the parquet schema.
762        let bloom_filter_field_ids: Vec<i32> = collect_bloom_filter_field_ids(predicate)?
763            .into_iter()
764            .filter(|id| field_id_map.contains_key(id))
765            .collect();
766
767        if bloom_filter_field_ids.is_empty() {
768            return Ok(candidate_row_groups.to_vec());
769        }
770
771        let mut result = Vec::with_capacity(candidate_row_groups.len());
772
773        for &rg_idx in candidate_row_groups {
774            let mut bloom_filters: HashMap<i32, ColumnBloomFilter> = HashMap::new();
775
776            for &field_id in &bloom_filter_field_ids {
777                let col_idx = field_id_map[&field_id];
778                let col_meta = builder.metadata().row_group(rg_idx).column(col_idx);
779
780                // Only attempt to load if this column chunk actually has a bloom filter
781                if col_meta.bloom_filter_offset().is_none() {
782                    continue;
783                }
784
785                let physical_type = col_meta.column_type();
786                let type_length = col_meta.column_descr().type_length();
787
788                match builder
789                    .get_row_group_column_bloom_filter(rg_idx, col_idx)
790                    .await
791                {
792                    Ok(Some(sbbf)) => {
793                        bloom_filters.insert(
794                            field_id,
795                            ColumnBloomFilter::new(sbbf, physical_type, type_length),
796                        );
797                    }
798                    Ok(None) => {}
799                    Err(e) => {
800                        // Left absent from the map, so the evaluator treats the column
801                        // as might-match and the row group survives.
802                        tracing::debug!(
803                            "Bloom filter for field {field_id} in row group {rg_idx} could not be read: {e}"
804                        );
805                    }
806                }
807            }
808
809            match BloomFilterEvaluator::eval(predicate, &bloom_filters) {
810                Ok(true) => result.push(rg_idx),
811                Ok(false) => { /* Row group pruned by bloom filter */ }
812                Err(e) => {
813                    tracing::debug!(
814                        "Bloom filter evaluation failed for row group {rg_idx}, including it: {e}"
815                    );
816                    result.push(rg_idx);
817                }
818            }
819        }
820
821        Ok(result)
822    }
823}
824
825impl ArrowReader {
826    /// Opens a Parquet file and loads its metadata, wrapping the reader with
827    /// [`CountingFileRead`] so all I/O is accumulated into `bytes_read`.
828    pub(crate) async fn open_parquet_file(
829        data_file_path: &str,
830        file_io: &FileIO,
831        file_size_in_bytes: u64,
832        parquet_read_options: ParquetReadOptions,
833        bytes_read: &Arc<AtomicU64>,
834        key_metadata: Option<&[u8]>,
835    ) -> Result<(ArrowFileReader, ArrowReaderMetadata)> {
836        let parquet_file = file_io.new_input(data_file_path)?;
837        let counting_reader =
838            CountingFileRead::new(parquet_file.reader().await?, Arc::clone(bytes_read));
839        Self::build_parquet_reader(
840            Box::new(counting_reader),
841            file_size_in_bytes,
842            parquet_read_options,
843            key_metadata,
844        )
845        .await
846    }
847
848    async fn build_parquet_reader(
849        parquet_reader: Box<dyn FileRead>,
850        file_size_in_bytes: u64,
851        parquet_read_options: ParquetReadOptions,
852        key_metadata: Option<&[u8]>,
853    ) -> Result<(ArrowFileReader, ArrowReaderMetadata)> {
854        let mut reader = ArrowFileReader::new(
855            FileMetadata {
856                size: file_size_in_bytes,
857            },
858            parquet_reader,
859        )
860        .with_parquet_read_options(parquet_read_options);
861
862        let arrow_reader_options = Self::build_arrow_reader_options(key_metadata)?;
863
864        let arrow_metadata = ArrowReaderMetadata::load_async(&mut reader, arrow_reader_options)
865            .await
866            .map_err(|e| {
867                Error::new(ErrorKind::Unexpected, "Failed to load Parquet metadata").with_source(e)
868            })?;
869
870        Ok((reader, arrow_metadata))
871    }
872
873    /// Builds `ArrowReaderOptions`, adding `FileDecryptionProperties` when
874    /// key metadata is present for Parquet Modular Encryption.
875    fn build_arrow_reader_options(key_metadata: Option<&[u8]>) -> Result<ArrowReaderOptions> {
876        match key_metadata {
877            Some(km) => {
878                let standard_key_metadata = StandardKeyMetadata::decode(km)?;
879                let mut builder = FileDecryptionProperties::builder(
880                    standard_key_metadata.encryption_key().as_bytes().to_vec(),
881                );
882                if let Some(aad) = standard_key_metadata.aad_prefix() {
883                    builder = builder.with_aad_prefix(aad.to_vec());
884                }
885                let decryption_properties = builder.build().map_err(|e| {
886                    Error::new(
887                        ErrorKind::Unexpected,
888                        "Failed to build Parquet file decryption properties",
889                    )
890                    .with_source(e)
891                })?;
892                Ok(
893                    ArrowReaderOptions::new()
894                        .with_file_decryption_properties(decryption_properties),
895                )
896            }
897            None => Ok(ArrowReaderOptions::default()),
898        }
899    }
900}
901
902#[cfg(test)]
903mod tests {
904    use std::collections::HashMap;
905    use std::fs::File;
906    use std::sync::Arc;
907
908    use arrow_array::cast::AsArray;
909    use arrow_array::{Array, ArrayRef, Int32Array, Int64Array, RecordBatch, StringArray};
910    use arrow_cast::cast;
911    use arrow_schema::{DataType, Field, Schema as ArrowSchema};
912    use futures::TryStreamExt;
913    use parquet::arrow::{ArrowWriter, PARQUET_FIELD_ID_META_KEY};
914    use parquet::basic::Compression;
915    use parquet::file::properties::WriterProperties;
916    use tempfile::TempDir;
917
918    use crate::Runtime;
919    use crate::arrow::test_utils::write_encrypted_parquet;
920    use crate::arrow::{ArrowReaderBuilder, ArrowReaderOptions as ReaderOptions};
921    use crate::io::FileIO;
922    use crate::metadata_columns::{
923        RESERVED_COL_NAME_POS, RESERVED_COL_NAME_ROW_ID, RESERVED_FIELD_ID_FILE,
924        RESERVED_FIELD_ID_POS, RESERVED_FIELD_ID_ROW_ID,
925    };
926    use crate::scan::{FileScanTask, FileScanTaskDeleteFile, FileScanTaskStream};
927    use crate::spec::{DataFileFormat, NestedField, PrimitiveType, Schema, SchemaRef, Type};
928
929    fn record_count_task(path: &str, file_size: u64, record_count: Option<u64>) -> FileScanTask {
930        let schema = Schema::builder()
931            .with_fields(vec![
932                NestedField::required(1, "id", Type::Primitive(PrimitiveType::Int)).into(),
933            ])
934            .build()
935            .unwrap();
936        FileScanTask::builder()
937            .with_data_file_path(path.to_string())
938            .with_data_file_format(DataFileFormat::Parquet)
939            .with_file_size_in_bytes(file_size)
940            .with_start(0)
941            .with_length(file_size)
942            .with_record_count(record_count)
943            .with_schema(Arc::new(schema))
944            .with_project_field_ids(vec![1])
945            .with_case_sensitive(true)
946            .build()
947            .unwrap()
948    }
949
950    async fn read_tasks(
951        tasks: Vec<FileScanTask>,
952        concurrency: usize,
953    ) -> crate::Result<Vec<RecordBatch>> {
954        let tasks = futures::stream::iter(tasks.into_iter().map(Ok));
955        ArrowReaderBuilder::new(FileIO::new_with_fs(), Runtime::current())
956            .with_options(
957                ReaderOptions::builder()
958                    .with_data_file_concurrency_limit(concurrency)
959                    .build(),
960            )
961            .build()
962            .read(Box::pin(tasks))?
963            .stream()
964            .try_collect()
965            .await
966    }
967
968    #[tokio::test]
969    async fn test_read_zero_record_files_without_opening_them() {
970        let temp_dir = TempDir::new().unwrap();
971        let missing_path = temp_dir
972            .path()
973            .join("missing.parquet")
974            .display()
975            .to_string();
976
977        for concurrency in [1, 2] {
978            // The record count, not the byte size, determines whether the file is empty.
979            let tasks = vec![
980                record_count_task(&missing_path, 0, Some(0)),
981                record_count_task(&missing_path, 128, Some(0)),
982            ];
983            let batches = read_tasks(tasks, concurrency).await.unwrap();
984            assert!(batches.is_empty());
985        }
986    }
987
988    #[tokio::test]
989    async fn test_read_zero_size_files_with_nonzero_or_unknown_record_count() {
990        let temp_dir = TempDir::new().unwrap();
991        let path = temp_dir.path().join("empty.parquet").display().to_string();
992        File::create(&path).unwrap();
993
994        for concurrency in [1, 2] {
995            for record_count in [Some(1), None] {
996                let task = record_count_task(&path, 0, record_count);
997                let error = read_tasks(vec![task], concurrency).await.unwrap_err();
998                assert!(
999                    error
1000                        .to_string()
1001                        .contains("file size of 0 is less than footer"),
1002                    "{error}"
1003                );
1004            }
1005        }
1006    }
1007
1008    #[tokio::test]
1009    async fn test_read_nonempty_files_alongside_zero_record_files() {
1010        use arrow_array::types::Int32Type;
1011
1012        let temp_dir = TempDir::new().unwrap();
1013        let dir = temp_dir.path().to_str().unwrap();
1014        let path = write_plain_parquet(dir, "data.parquet", vec![], vec![]);
1015        let file_size = std::fs::metadata(&path).unwrap().len();
1016        let missing_path = format!("{dir}/missing.parquet");
1017
1018        for concurrency in [1, 2] {
1019            for record_count in [Some(3), None] {
1020                let empty_task = record_count_task(&missing_path, 0, Some(0));
1021                let data_task = record_count_task(&path, file_size, record_count);
1022                let tasks = vec![empty_task.clone(), data_task, empty_task];
1023                let batches = read_tasks(tasks, concurrency).await.unwrap();
1024
1025                assert_eq!(batches.len(), 1);
1026                let ids = batches[0].column(0).as_primitive::<Int32Type>();
1027                assert_eq!(ids.values(), &[1, 2, 3]);
1028            }
1029        }
1030    }
1031
1032    // INT96 encoding: [nanos_low_u32, nanos_high_u32, julian_day_u32]
1033    // Julian day 2_440_588 = Unix epoch (1970-01-01)
1034    const UNIX_EPOCH_JULIAN: i64 = 2_440_588;
1035    const MICROS_PER_DAY: i64 = 86_400_000_000;
1036    // Noon on 3333-01-01 (Julian day 2_953_529) — outside the i64 nanosecond range (~1677-2262).
1037    const INT96_TEST_NANOS_WITHIN_DAY: u64 = 43_200_000_000_000;
1038    const INT96_TEST_JULIAN_DAY: u32 = 2_953_529;
1039
1040    fn make_int96_test_value() -> (parquet::data_type::Int96, i64) {
1041        let mut val = parquet::data_type::Int96::new();
1042        val.set_data(
1043            (INT96_TEST_NANOS_WITHIN_DAY & 0xFFFFFFFF) as u32,
1044            (INT96_TEST_NANOS_WITHIN_DAY >> 32) as u32,
1045            INT96_TEST_JULIAN_DAY,
1046        );
1047        let expected_micros = (INT96_TEST_JULIAN_DAY as i64 - UNIX_EPOCH_JULIAN) * MICROS_PER_DAY
1048            + (INT96_TEST_NANOS_WITHIN_DAY / 1_000) as i64;
1049        (val, expected_micros)
1050    }
1051
1052    async fn read_int96_batches(
1053        file_path: &str,
1054        schema: SchemaRef,
1055        project_field_ids: Vec<i32>,
1056    ) -> Vec<RecordBatch> {
1057        let file_io = FileIO::new_with_fs();
1058        let reader = ArrowReaderBuilder::new(file_io, Runtime::current()).build();
1059
1060        let file_size = std::fs::metadata(file_path).unwrap().len();
1061        let task = FileScanTask::builder()
1062            .with_file_size_in_bytes(file_size)
1063            .with_start(0)
1064            .with_length(file_size)
1065            .with_data_file_path(file_path.to_string())
1066            .with_data_file_format(DataFileFormat::Parquet)
1067            .with_schema(schema)
1068            .with_project_field_ids(project_field_ids)
1069            .with_case_sensitive(false)
1070            .build()
1071            .unwrap();
1072
1073        let tasks = Box::pin(futures::stream::iter(vec![Ok(task)])) as FileScanTaskStream;
1074        reader
1075            .read(tasks)
1076            .unwrap()
1077            .stream()
1078            .try_collect()
1079            .await
1080            .unwrap()
1081    }
1082
1083    // ArrowWriter cannot write INT96, so we use SerializedFileWriter directly.
1084    fn write_int96_parquet_file(
1085        table_location: &str,
1086        filename: &str,
1087        with_field_ids: bool,
1088    ) -> (String, Vec<i64>) {
1089        use parquet::basic::{Repetition, Type as PhysicalType};
1090        use parquet::data_type::{Int32Type, Int96, Int96Type};
1091        use parquet::file::writer::SerializedFileWriter;
1092        use parquet::schema::types::Type as SchemaType;
1093
1094        let file_path = format!("{table_location}/{filename}");
1095
1096        let mut ts_builder = SchemaType::primitive_type_builder("ts", PhysicalType::INT96)
1097            .with_repetition(Repetition::OPTIONAL);
1098        let mut id_builder = SchemaType::primitive_type_builder("id", PhysicalType::INT32)
1099            .with_repetition(Repetition::REQUIRED);
1100
1101        if with_field_ids {
1102            ts_builder = ts_builder.with_id(Some(1));
1103            id_builder = id_builder.with_id(Some(2));
1104        }
1105
1106        let schema = SchemaType::group_type_builder("schema")
1107            .with_fields(vec![
1108                Arc::new(ts_builder.build().unwrap()),
1109                Arc::new(id_builder.build().unwrap()),
1110            ])
1111            .build()
1112            .unwrap();
1113
1114        // Dates outside the i64 nanosecond range (~1677-2262) overflow without coercion.
1115        const NOON_NANOS: u64 = INT96_TEST_NANOS_WITHIN_DAY;
1116        const JULIAN_3333: u32 = INT96_TEST_JULIAN_DAY;
1117        const JULIAN_2100: u32 = 2_488_070;
1118
1119        let test_data: Vec<(u32, u32, u32, i64)> = vec![
1120            // 3333-01-01 00:00:00
1121            (
1122                0,
1123                0,
1124                JULIAN_3333,
1125                (JULIAN_3333 as i64 - UNIX_EPOCH_JULIAN) * MICROS_PER_DAY,
1126            ),
1127            // 3333-01-01 12:00:00
1128            (
1129                (NOON_NANOS & 0xFFFFFFFF) as u32,
1130                (NOON_NANOS >> 32) as u32,
1131                JULIAN_3333,
1132                (JULIAN_3333 as i64 - UNIX_EPOCH_JULIAN) * MICROS_PER_DAY
1133                    + (NOON_NANOS / 1_000) as i64,
1134            ),
1135            // 2100-01-01 00:00:00
1136            (
1137                0,
1138                0,
1139                JULIAN_2100,
1140                (JULIAN_2100 as i64 - UNIX_EPOCH_JULIAN) * MICROS_PER_DAY,
1141            ),
1142        ];
1143
1144        let int96_values: Vec<Int96> = test_data
1145            .iter()
1146            .map(|(lo, hi, day, _)| {
1147                let mut v = Int96::new();
1148                v.set_data(*lo, *hi, *day);
1149                v
1150            })
1151            .collect();
1152
1153        let id_values: Vec<i32> = (0..test_data.len() as i32).collect();
1154        let expected_micros: Vec<i64> = test_data.iter().map(|(_, _, _, m)| *m).collect();
1155
1156        let file = File::create(&file_path).unwrap();
1157        let mut writer =
1158            SerializedFileWriter::new(file, Arc::new(schema), Default::default()).unwrap();
1159
1160        let mut row_group = writer.next_row_group().unwrap();
1161        {
1162            // def=1: ts is OPTIONAL and present. No repetition levels (top-level columns).
1163            let mut col = row_group.next_column().unwrap().unwrap();
1164            col.typed::<Int96Type>()
1165                .write_batch(&int96_values, Some(&vec![1; test_data.len()]), None)
1166                .unwrap();
1167            col.close().unwrap();
1168        }
1169        {
1170            let mut col = row_group.next_column().unwrap().unwrap();
1171            col.typed::<Int32Type>()
1172                .write_batch(&id_values, None, None)
1173                .unwrap();
1174            col.close().unwrap();
1175        }
1176        row_group.close().unwrap();
1177        writer.close().unwrap();
1178
1179        (file_path, expected_micros)
1180    }
1181
1182    async fn assert_int96_read_matches(
1183        file_path: &str,
1184        schema: SchemaRef,
1185        project_field_ids: Vec<i32>,
1186        expected_micros: &[i64],
1187    ) {
1188        use arrow_array::TimestampMicrosecondArray;
1189
1190        let batches = read_int96_batches(file_path, schema, project_field_ids).await;
1191
1192        assert_eq!(batches.len(), 1);
1193        let ts_array = batches[0]
1194            .column(0)
1195            .as_any()
1196            .downcast_ref::<TimestampMicrosecondArray>()
1197            .expect("Expected TimestampMicrosecondArray");
1198
1199        for (i, expected) in expected_micros.iter().enumerate() {
1200            assert_eq!(
1201                ts_array.value(i),
1202                *expected,
1203                "Row {i}: got {}, expected {expected}",
1204                ts_array.value(i)
1205            );
1206        }
1207    }
1208
1209    /// Writes a single-column Parquet file encrypted with `encryption_key`, then reads it
1210    /// back through `ArrowReader` and asserts the round-tripped values. The key length
1211    /// selects the AES-GCM variant in arrow-rs (16 -> AES-128, 32 -> AES-256).
1212    async fn assert_encrypted_parquet_roundtrip(encryption_key: &[u8]) {
1213        let aad_prefix = b"aad_prefix";
1214
1215        let schema = Arc::new(
1216            Schema::builder()
1217                .with_schema_id(1)
1218                .with_fields(vec![
1219                    NestedField::required(1, "id", Type::Primitive(PrimitiveType::Int)).into(),
1220                ])
1221                .build()
1222                .unwrap(),
1223        );
1224
1225        let arrow_schema = Arc::new(ArrowSchema::new(vec![
1226            Field::new("id", DataType::Int32, false).with_metadata(HashMap::from([(
1227                PARQUET_FIELD_ID_META_KEY.to_string(),
1228                "1".to_string(),
1229            )])),
1230        ]));
1231
1232        let tmp_dir = TempDir::new().unwrap();
1233        let table_location = tmp_dir.path().to_str().unwrap().to_string();
1234        let file_io = FileIO::new_with_fs();
1235
1236        let id_data = Arc::new(Int32Array::from(vec![10, 20, 30])) as ArrayRef;
1237        let batch = RecordBatch::try_new(arrow_schema.clone(), vec![id_data]).unwrap();
1238
1239        let file_path = format!("{table_location}/encrypted.parquet");
1240        write_encrypted_parquet(&file_path, &batch, encryption_key, Some(aad_prefix));
1241
1242        let key_metadata = crate::encryption::StandardKeyMetadata::try_new(encryption_key)
1243            .unwrap()
1244            .with_aad_prefix(aad_prefix)
1245            .encode()
1246            .unwrap();
1247
1248        let reader = ArrowReaderBuilder::new(file_io, Runtime::current()).build();
1249
1250        let task = FileScanTask::builder()
1251            .with_file_size_in_bytes(std::fs::metadata(&file_path).unwrap().len())
1252            .with_start(0)
1253            .with_length(0)
1254            .with_data_file_path(file_path)
1255            .with_data_file_format(DataFileFormat::Parquet)
1256            .with_schema(schema)
1257            .with_project_field_ids(vec![1])
1258            .with_case_sensitive(false)
1259            .with_key_metadata(Some(key_metadata))
1260            .build()
1261            .unwrap();
1262
1263        let tasks = Box::pin(futures::stream::iter(vec![Ok(task)])) as FileScanTaskStream;
1264        let batches: Vec<RecordBatch> = reader
1265            .read(tasks)
1266            .unwrap()
1267            .stream()
1268            .try_collect()
1269            .await
1270            .unwrap();
1271
1272        assert_eq!(batches.len(), 1);
1273        let ids = batches[0]
1274            .column(0)
1275            .as_any()
1276            .downcast_ref::<Int32Array>()
1277            .unwrap();
1278        assert_eq!(ids.values(), &[10, 20, 30]);
1279    }
1280
1281    #[tokio::test]
1282    async fn test_read_encrypted_parquet_aes_128() {
1283        assert_encrypted_parquet_roundtrip(b"0123456789abcdef").await;
1284    }
1285
1286    #[tokio::test]
1287    async fn test_read_encrypted_parquet_aes_256() {
1288        assert_encrypted_parquet_roundtrip(b"0123456789abcdef0123456789abcdef").await;
1289    }
1290
1291    #[tokio::test]
1292    async fn test_read_encrypted_parquet_without_key_metadata_fails() {
1293        let encryption_key = b"0123456789abcdef";
1294
1295        let schema = Arc::new(
1296            Schema::builder()
1297                .with_schema_id(1)
1298                .with_fields(vec![
1299                    NestedField::required(1, "id", Type::Primitive(PrimitiveType::Int)).into(),
1300                ])
1301                .build()
1302                .unwrap(),
1303        );
1304
1305        let arrow_schema = Arc::new(ArrowSchema::new(vec![
1306            Field::new("id", DataType::Int32, false).with_metadata(HashMap::from([(
1307                PARQUET_FIELD_ID_META_KEY.to_string(),
1308                "1".to_string(),
1309            )])),
1310        ]));
1311
1312        let tmp_dir = TempDir::new().unwrap();
1313        let table_location = tmp_dir.path().to_str().unwrap().to_string();
1314        let file_io = FileIO::new_with_fs();
1315
1316        let id_data = Arc::new(Int32Array::from(vec![1, 2, 3])) as ArrayRef;
1317        let batch = RecordBatch::try_new(arrow_schema.clone(), vec![id_data]).unwrap();
1318
1319        let file_path = format!("{table_location}/encrypted_no_key.parquet");
1320        write_encrypted_parquet(&file_path, &batch, encryption_key, None);
1321
1322        let reader = ArrowReaderBuilder::new(file_io, Runtime::current()).build();
1323
1324        let task = FileScanTask::builder()
1325            .with_file_size_in_bytes(std::fs::metadata(&file_path).unwrap().len())
1326            .with_start(0)
1327            .with_length(0)
1328            .with_data_file_path(file_path)
1329            .with_data_file_format(DataFileFormat::Parquet)
1330            .with_schema(schema)
1331            .with_project_field_ids(vec![1])
1332            .with_case_sensitive(false)
1333            .build()
1334            .unwrap();
1335
1336        let tasks = Box::pin(futures::stream::iter(vec![Ok(task)])) as FileScanTaskStream;
1337        let result: Result<Vec<RecordBatch>, _> =
1338            reader.read(tasks).unwrap().stream().try_collect().await;
1339
1340        let err = result.unwrap_err();
1341        assert_eq!(err.kind(), crate::ErrorKind::Unexpected);
1342        let err_str = format!("{err}");
1343        assert!(
1344            err_str.contains("encrypted footer"),
1345            "Expected error about encrypted footer, got: {err_str}"
1346        );
1347        assert!(
1348            err_str.contains("decryption properties were not provided"),
1349            "Expected error about missing decryption properties, got: {err_str}"
1350        );
1351    }
1352
1353    /// Writes a plain (unencrypted) single-column Int32 "id" parquet file with the
1354    /// given extra Arrow fields/columns appended, returning the file path.
1355    fn write_plain_parquet(
1356        dir: &str,
1357        name: &str,
1358        extra_fields: Vec<Field>,
1359        extra_columns: Vec<ArrayRef>,
1360    ) -> String {
1361        let props = WriterProperties::builder()
1362            .set_compression(Compression::SNAPPY)
1363            .build();
1364        write_parquet_with(dir, name, extra_fields, extra_columns, props)
1365    }
1366
1367    fn write_parquet_with(
1368        dir: &str,
1369        name: &str,
1370        extra_fields: Vec<Field>,
1371        extra_columns: Vec<ArrayRef>,
1372        props: WriterProperties,
1373    ) -> String {
1374        let mut fields =
1375            vec![
1376                Field::new("id", DataType::Int32, false).with_metadata(HashMap::from([(
1377                    PARQUET_FIELD_ID_META_KEY.to_string(),
1378                    "1".to_string(),
1379                )])),
1380            ];
1381        fields.extend(extra_fields);
1382        let arrow_schema = Arc::new(ArrowSchema::new(fields));
1383
1384        let mut columns: Vec<ArrayRef> = vec![Arc::new(Int32Array::from(vec![1, 2, 3]))];
1385        columns.extend(extra_columns);
1386        let batch = RecordBatch::try_new(arrow_schema.clone(), columns).unwrap();
1387
1388        let file_path = format!("{dir}/{name}");
1389        let file = File::create(&file_path).unwrap();
1390        let mut writer = ArrowWriter::try_new(file, arrow_schema, Some(props)).unwrap();
1391        writer.write(&batch).unwrap();
1392        writer.close().unwrap();
1393        file_path
1394    }
1395
1396    fn last_updated_seq_task(
1397        file_path: String,
1398        first_row_id: Option<i64>,
1399        data_sequence_number: Option<i64>,
1400    ) -> FileScanTask {
1401        use crate::metadata_columns::RESERVED_FIELD_ID_LAST_UPDATED_SEQUENCE_NUMBER;
1402
1403        let schema = Arc::new(
1404            Schema::builder()
1405                .with_schema_id(1)
1406                .with_fields(vec![
1407                    NestedField::required(1, "id", Type::Primitive(PrimitiveType::Int)).into(),
1408                ])
1409                .build()
1410                .unwrap(),
1411        );
1412
1413        FileScanTask::builder()
1414            .with_file_size_in_bytes(std::fs::metadata(&file_path).unwrap().len())
1415            .with_start(0)
1416            .with_length(0)
1417            .with_data_file_path(file_path)
1418            .with_data_file_format(DataFileFormat::Parquet)
1419            .with_schema(schema)
1420            .with_project_field_ids(vec![1, RESERVED_FIELD_ID_LAST_UPDATED_SEQUENCE_NUMBER])
1421            .with_first_row_id(first_row_id)
1422            .with_data_sequence_number(data_sequence_number)
1423            .with_case_sensitive(false)
1424            .build()
1425            .unwrap()
1426    }
1427
1428    /// Asserts the logical per-row values of the `_last_updated_sequence_number`
1429    /// column across all batches, independent of the physical (run-end) encoding.
1430    fn assert_last_updated_seq_column(batches: &[RecordBatch], expected: &[Option<i64>]) {
1431        use arrow_array::cast::AsArray;
1432        use arrow_cast::cast;
1433        use arrow_schema::DataType;
1434
1435        use crate::metadata_columns::RESERVED_COL_NAME_LAST_UPDATED_SEQUENCE_NUMBER;
1436
1437        let mut actual = Vec::new();
1438        for batch in batches {
1439            let col = batch
1440                .column_by_name(RESERVED_COL_NAME_LAST_UPDATED_SEQUENCE_NUMBER)
1441                .expect("_last_updated_sequence_number column should be present");
1442            let logical = cast(col, &DataType::Int64).unwrap();
1443            let values = logical.as_primitive::<arrow_array::types::Int64Type>();
1444            for i in 0..values.len() {
1445                actual.push((!values.is_null(i)).then(|| values.value(i)));
1446            }
1447        }
1448        assert_eq!(actual, expected);
1449    }
1450
1451    #[tokio::test]
1452    async fn test_last_updated_sequence_number_null_when_no_first_row_id() {
1453        let tmp_dir = TempDir::new().unwrap();
1454        let dir = tmp_dir.path().to_str().unwrap();
1455        let file_path = write_plain_parquet(dir, "no_first_row_id.parquet", vec![], vec![]);
1456
1457        // A file with a null first_row_id (v1/v2, or a pre-upgrade v3 snapshot) produces
1458        // a null _last_updated_sequence_number column, even though it has a data
1459        // sequence number; the spec gates both lineage columns on first_row_id.
1460        let task = last_updated_seq_task(file_path, None, Some(9));
1461
1462        let reader = ArrowReaderBuilder::new(FileIO::new_with_fs(), Runtime::current()).build();
1463        let tasks = Box::pin(futures::stream::iter(vec![Ok(task)])) as FileScanTaskStream;
1464        let batches: Vec<RecordBatch> = reader
1465            .read(tasks)
1466            .unwrap()
1467            .stream()
1468            .try_collect()
1469            .await
1470            .unwrap();
1471
1472        assert_last_updated_seq_column(&batches, &[None, None, None]);
1473    }
1474
1475    #[tokio::test]
1476    async fn test_last_updated_sequence_number_error_when_no_data_seq() {
1477        let tmp_dir = TempDir::new().unwrap();
1478        let dir = tmp_dir.path().to_str().unwrap();
1479        let file_path = write_plain_parquet(dir, "no_data_seq.parquet", vec![], vec![]);
1480
1481        // first_row_id present but data_sequence_number absent: after manifest
1482        // inheritance a committed entry always has one, so this is a malformed
1483        // manifest and must error rather than fabricate or null the column.
1484        let task = last_updated_seq_task(file_path, Some(42), None);
1485
1486        let reader = ArrowReaderBuilder::new(FileIO::new_with_fs(), Runtime::current()).build();
1487        let tasks = Box::pin(futures::stream::iter(vec![Ok(task)])) as FileScanTaskStream;
1488        let result: Result<Vec<RecordBatch>, _> =
1489            reader.read(tasks).unwrap().stream().try_collect().await;
1490
1491        let err = result.unwrap_err();
1492        assert_eq!(err.kind(), crate::ErrorKind::DataInvalid);
1493        assert!(
1494            format!("{err}").contains("no data sequence number"),
1495            "unexpected error: {err}"
1496        );
1497    }
1498
1499    #[tokio::test]
1500    async fn test_last_updated_sequence_number_derived_from_data_seq() {
1501        let tmp_dir = TempDir::new().unwrap();
1502        let dir = tmp_dir.path().to_str().unwrap();
1503        let file_path = write_plain_parquet(dir, "with_first_row_id.parquet", vec![], vec![]);
1504
1505        // Non-null first_row_id + data sequence number -> the derived value (the data
1506        // sequence number) for every row. This is the only value-producing arm.
1507        let task = last_updated_seq_task(file_path, Some(42), Some(7));
1508
1509        let reader = ArrowReaderBuilder::new(FileIO::new_with_fs(), Runtime::current()).build();
1510        let tasks = Box::pin(futures::stream::iter(vec![Ok(task)])) as FileScanTaskStream;
1511        let batches: Vec<RecordBatch> = reader
1512            .read(tasks)
1513            .unwrap()
1514            .stream()
1515            .try_collect()
1516            .await
1517            .unwrap();
1518
1519        assert_last_updated_seq_column(&batches, &[Some(7), Some(7), Some(7)]);
1520    }
1521
1522    #[tokio::test]
1523    async fn test_last_updated_sequence_number_mixed_files_share_schema() {
1524        use arrow_select::concat::concat_batches;
1525
1526        let tmp_dir = TempDir::new().unwrap();
1527        let dir = tmp_dir.path().to_str().unwrap();
1528
1529        // Three files in one scan exercising all three column paths, which must all
1530        // produce the SAME Arrow type (run-end-encoded) or concatenation fails:
1531        //   - constant: first_row_id set, no physical column -> derived constant
1532        //   - null gate: no first_row_id -> null column
1533        //   - coalesce: first_row_id set, physical column present -> per-row + fallback
1534        let constant = last_updated_seq_task(
1535            write_plain_parquet(dir, "constant.parquet", vec![], vec![]),
1536            Some(42),
1537            Some(7),
1538        );
1539        let nulled = last_updated_seq_task(
1540            write_plain_parquet(dir, "nulled.parquet", vec![], vec![]),
1541            None,
1542            Some(7),
1543        );
1544        let coalesced = last_updated_seq_task(
1545            write_plain_parquet(
1546                dir,
1547                "coalesced.parquet",
1548                vec![physical_last_updated_seq_field()],
1549                vec![Arc::new(Int64Array::from(vec![Some(5), None, Some(8)])) as ArrayRef],
1550            ),
1551            Some(50),
1552            Some(7),
1553        );
1554
1555        let reader = ArrowReaderBuilder::new(FileIO::new_with_fs(), Runtime::current()).build();
1556        let tasks = Box::pin(futures::stream::iter(vec![
1557            Ok(constant),
1558            Ok(nulled),
1559            Ok(coalesced),
1560        ])) as FileScanTaskStream;
1561        let batches: Vec<RecordBatch> = reader
1562            .read(tasks)
1563            .unwrap()
1564            .stream()
1565            .try_collect()
1566            .await
1567            .unwrap();
1568
1569        assert_eq!(batches.len(), 3);
1570        // Identical schema across all three paths -> concat succeeds.
1571        let schema = batches[0].schema();
1572        concat_batches(&schema, &batches)
1573            .expect("constant, null and coalesce files must share one column type");
1574    }
1575
1576    /// A parquet field carrying the embedded `_last_updated_sequence_number` field id.
1577    fn physical_last_updated_seq_field() -> Field {
1578        use crate::metadata_columns::{
1579            RESERVED_COL_NAME_LAST_UPDATED_SEQUENCE_NUMBER,
1580            RESERVED_FIELD_ID_LAST_UPDATED_SEQUENCE_NUMBER,
1581        };
1582        Field::new(
1583            RESERVED_COL_NAME_LAST_UPDATED_SEQUENCE_NUMBER,
1584            DataType::Int64,
1585            true,
1586        )
1587        .with_metadata(HashMap::from([(
1588            PARQUET_FIELD_ID_META_KEY.to_string(),
1589            RESERVED_FIELD_ID_LAST_UPDATED_SEQUENCE_NUMBER.to_string(),
1590        )]))
1591    }
1592
1593    #[tokio::test]
1594    async fn test_last_updated_sequence_number_physical_column_coalesced() {
1595        let tmp_dir = TempDir::new().unwrap();
1596        let dir = tmp_dir.path().to_str().unwrap();
1597        // A file that physically carries the column, as Iceberg Java writes when
1598        // carrying rows forward across a rewrite: some rows have a stored value, some
1599        // are null (added/modified rows, inherited on read).
1600        let seq_col = Arc::new(Int64Array::from(vec![Some(5), None, Some(8)])) as ArrayRef;
1601        let file_path = write_plain_parquet(
1602            dir,
1603            "with_seq.parquet",
1604            vec![physical_last_updated_seq_field()],
1605            vec![seq_col],
1606        );
1607
1608        let task = last_updated_seq_task(file_path, Some(100), Some(9));
1609
1610        let reader = ArrowReaderBuilder::new(FileIO::new_with_fs(), Runtime::current()).build();
1611        let tasks = Box::pin(futures::stream::iter(vec![Ok(task)])) as FileScanTaskStream;
1612        let batches: Vec<RecordBatch> = reader
1613            .read(tasks)
1614            .unwrap()
1615            .stream()
1616            .try_collect()
1617            .await
1618            .unwrap();
1619
1620        // Per-row value where non-null; the data sequence number (9) where null.
1621        assert_last_updated_seq_column(&batches, &[Some(5), Some(9), Some(8)]);
1622    }
1623
1624    #[tokio::test]
1625    async fn test_last_updated_sequence_number_coalesced_with_pos_column() {
1626        use crate::metadata_columns::{
1627            RESERVED_COL_NAME_POS, RESERVED_FIELD_ID_LAST_UPDATED_SEQUENCE_NUMBER,
1628            RESERVED_FIELD_ID_POS,
1629        };
1630
1631        let tmp_dir = TempDir::new().unwrap();
1632        let dir = tmp_dir.path().to_str().unwrap();
1633        let seq_col = Arc::new(Int64Array::from(vec![Some(5), None, Some(8)])) as ArrayRef;
1634        let file_path = write_plain_parquet(
1635            dir,
1636            "with_seq_and_pos.parquet",
1637            vec![physical_last_updated_seq_field()],
1638            vec![seq_col],
1639        );
1640
1641        // Co-project `_pos` (a virtual column appended to the Arrow output schema) with the
1642        // physical coalesce column. This guards that the physical column's index is
1643        // resolved in the Parquet schema, not the Arrow schema (whose indices shift once
1644        // virtual columns are appended).
1645        let schema = Arc::new(
1646            Schema::builder()
1647                .with_schema_id(1)
1648                .with_fields(vec![
1649                    NestedField::required(1, "id", Type::Primitive(PrimitiveType::Int)).into(),
1650                ])
1651                .build()
1652                .unwrap(),
1653        );
1654        let task = FileScanTask::builder()
1655            .with_file_size_in_bytes(std::fs::metadata(&file_path).unwrap().len())
1656            .with_start(0)
1657            .with_length(0)
1658            .with_data_file_path(file_path)
1659            .with_data_file_format(DataFileFormat::Parquet)
1660            .with_schema(schema)
1661            .with_project_field_ids(vec![
1662                1,
1663                RESERVED_FIELD_ID_POS,
1664                RESERVED_FIELD_ID_LAST_UPDATED_SEQUENCE_NUMBER,
1665            ])
1666            .with_first_row_id(Some(100))
1667            .with_data_sequence_number(Some(9))
1668            .with_case_sensitive(false)
1669            .build()
1670            .unwrap();
1671
1672        let reader = ArrowReaderBuilder::new(FileIO::new_with_fs(), Runtime::current()).build();
1673        let tasks = Box::pin(futures::stream::iter(vec![Ok(task)])) as FileScanTaskStream;
1674        let batches: Vec<RecordBatch> = reader
1675            .read(tasks)
1676            .unwrap()
1677            .stream()
1678            .try_collect()
1679            .await
1680            .unwrap();
1681
1682        // The seq column still coalesces correctly...
1683        assert_last_updated_seq_column(&batches, &[Some(5), Some(9), Some(8)]);
1684        // ...and `_pos` is the row position, unaffected by the physical-column union.
1685        let pos_col = batches[0]
1686            .column_by_name(RESERVED_COL_NAME_POS)
1687            .expect("_pos column should be present")
1688            .as_primitive::<arrow_array::types::Int64Type>();
1689        assert_eq!(pos_col.values(), &[0, 1, 2]);
1690    }
1691
1692    #[tokio::test]
1693    async fn test_last_updated_sequence_number_physical_column_nulled_without_first_row_id() {
1694        let tmp_dir = TempDir::new().unwrap();
1695        let dir = tmp_dir.path().to_str().unwrap();
1696        let seq_col = Arc::new(Int64Array::from(vec![Some(5), None, Some(8)])) as ArrayRef;
1697        let file_path = write_plain_parquet(
1698            dir,
1699            "with_seq_no_first_row_id.parquet",
1700            vec![physical_last_updated_seq_field()],
1701            vec![seq_col],
1702        );
1703
1704        // Null first_row_id: the whole column is null even though the file physically
1705        // carries per-row values -- the gate wins, and the physical column is not read.
1706        let task = last_updated_seq_task(file_path, None, Some(9));
1707
1708        let reader = ArrowReaderBuilder::new(FileIO::new_with_fs(), Runtime::current()).build();
1709        let tasks = Box::pin(futures::stream::iter(vec![Ok(task)])) as FileScanTaskStream;
1710        let batches: Vec<RecordBatch> = reader
1711            .read(tasks)
1712            .unwrap()
1713            .stream()
1714            .try_collect()
1715            .await
1716            .unwrap();
1717
1718        assert_last_updated_seq_column(&batches, &[None, None, None]);
1719    }
1720
1721    #[tokio::test]
1722    async fn test_last_updated_sequence_number_present_by_name_without_id_unsupported() {
1723        let tmp_dir = TempDir::new().unwrap();
1724        let dir = tmp_dir.path().to_str().unwrap();
1725        // Column present by name but WITHOUT the embedded field id (e.g. name mapping /
1726        // positional fallback). The transformer keys the source column by field id, so
1727        // this shape can't be threaded and is rejected loudly.
1728        let seq_field = Field::new(
1729            crate::metadata_columns::RESERVED_COL_NAME_LAST_UPDATED_SEQUENCE_NUMBER,
1730            DataType::Int64,
1731            true,
1732        );
1733        let seq_col = Arc::new(Int64Array::from(vec![Some(5), None, Some(8)])) as ArrayRef;
1734        let file_path =
1735            write_plain_parquet(dir, "with_seq_by_name.parquet", vec![seq_field], vec![
1736                seq_col,
1737            ]);
1738
1739        let task = last_updated_seq_task(file_path, Some(100), Some(9));
1740
1741        let reader = ArrowReaderBuilder::new(FileIO::new_with_fs(), Runtime::current()).build();
1742        let tasks = Box::pin(futures::stream::iter(vec![Ok(task)])) as FileScanTaskStream;
1743        let result: Result<Vec<RecordBatch>, _> =
1744            reader.read(tasks).unwrap().stream().try_collect().await;
1745
1746        let err = result.unwrap_err();
1747        assert_eq!(err.kind(), crate::ErrorKind::FeatureUnsupported);
1748        assert!(
1749            format!("{err}").contains("without an embedded field id"),
1750            "unexpected error: {err}"
1751        );
1752    }
1753
1754    #[tokio::test]
1755    async fn test_last_updated_sequence_number_physical_column_first_row_id_without_data_seq() {
1756        let tmp_dir = TempDir::new().unwrap();
1757        let dir = tmp_dir.path().to_str().unwrap();
1758        let seq_col = Arc::new(Int64Array::from(vec![Some(5), None, Some(8)])) as ArrayRef;
1759        let file_path = write_plain_parquet(
1760            dir,
1761            "with_seq_no_data_seq.parquet",
1762            vec![physical_last_updated_seq_field()],
1763            vec![seq_col],
1764        );
1765
1766        // first_row_id set but no data sequence number: after manifest inheritance a
1767        // committed entry always has one, so this is a malformed manifest, rejected loudly
1768        // rather than nulled.
1769        let task = last_updated_seq_task(file_path, Some(100), None);
1770
1771        let reader = ArrowReaderBuilder::new(FileIO::new_with_fs(), Runtime::current()).build();
1772        let tasks = Box::pin(futures::stream::iter(vec![Ok(task)])) as FileScanTaskStream;
1773        let result: Result<Vec<RecordBatch>, _> =
1774            reader.read(tasks).unwrap().stream().try_collect().await;
1775
1776        let err = result.unwrap_err();
1777        assert_eq!(err.kind(), crate::ErrorKind::DataInvalid);
1778        assert!(
1779            format!("{err}").contains("no data sequence number"),
1780            "unexpected error: {err}"
1781        );
1782    }
1783
1784    /// A scan task projecting `id` + `_row_id`, with the given `first_row_id`.
1785    fn row_id_task(file_path: String, first_row_id: Option<i64>) -> FileScanTask {
1786        row_id_task_with_options(file_path, first_row_id, 0, 0, vec![])
1787    }
1788
1789    fn row_id_task_with_options(
1790        file_path: String,
1791        first_row_id: Option<i64>,
1792        start: u64,
1793        length: u64,
1794        deletes: Vec<FileScanTaskDeleteFile>,
1795    ) -> FileScanTask {
1796        let schema = Arc::new(
1797            Schema::builder()
1798                .with_schema_id(1)
1799                .with_fields(vec![
1800                    NestedField::required(1, "id", Type::Primitive(PrimitiveType::Int)).into(),
1801                ])
1802                .build()
1803                .unwrap(),
1804        );
1805
1806        FileScanTask::builder()
1807            .with_file_size_in_bytes(std::fs::metadata(&file_path).unwrap().len())
1808            .with_start(start)
1809            .with_length(length)
1810            .with_data_file_path(file_path)
1811            .with_data_file_format(DataFileFormat::Parquet)
1812            .with_schema(schema)
1813            .with_project_field_ids(vec![1, RESERVED_FIELD_ID_ROW_ID])
1814            .with_first_row_id(first_row_id)
1815            .with_deletes(deletes)
1816            .with_case_sensitive(false)
1817            .build()
1818            .unwrap()
1819    }
1820
1821    /// Asserts the logical per-row values of the `_row_id` column across all batches,
1822    /// independent of the physical (run-end) encoding.
1823    fn assert_row_id_column(batches: &[RecordBatch], expected: &[Option<i64>]) {
1824        use arrow_array::cast::AsArray;
1825        use arrow_cast::cast;
1826        use arrow_schema::DataType;
1827
1828        let mut actual = Vec::new();
1829        for batch in batches {
1830            let col = batch
1831                .column_by_name(RESERVED_COL_NAME_ROW_ID)
1832                .expect("_row_id column should be present");
1833            let logical = cast(col, &DataType::Int64).unwrap();
1834            let values = logical.as_primitive::<arrow_array::types::Int64Type>();
1835            for i in 0..values.len() {
1836                actual.push((!values.is_null(i)).then(|| values.value(i)));
1837            }
1838        }
1839        assert_eq!(actual, expected);
1840    }
1841
1842    /// A parquet field carrying the embedded `_row_id` field id.
1843    fn physical_row_id_field() -> Field {
1844        Field::new(RESERVED_COL_NAME_ROW_ID, DataType::Int64, true).with_metadata(HashMap::from([
1845            (
1846                PARQUET_FIELD_ID_META_KEY.to_string(),
1847                RESERVED_FIELD_ID_ROW_ID.to_string(),
1848            ),
1849        ]))
1850    }
1851
1852    #[tokio::test]
1853    async fn test_row_id_synthesized_from_first_row_id_and_pos() {
1854        let tmp_dir = TempDir::new().unwrap();
1855        let dir = tmp_dir.path().to_str().unwrap();
1856        let file_path = write_plain_parquet(dir, "row_id_synth.parquet", vec![], vec![]);
1857
1858        // No physical column: every row is first_row_id + pos.
1859        let task = row_id_task(file_path, Some(100));
1860
1861        let reader = ArrowReaderBuilder::new(FileIO::new_with_fs(), Runtime::current()).build();
1862        let tasks = Box::pin(futures::stream::iter(vec![Ok(task)])) as FileScanTaskStream;
1863        let batches: Vec<RecordBatch> = reader
1864            .read(tasks)
1865            .unwrap()
1866            .stream()
1867            .try_collect()
1868            .await
1869            .unwrap();
1870
1871        assert_row_id_column(&batches, &[Some(100), Some(101), Some(102)]);
1872    }
1873
1874    #[tokio::test]
1875    async fn test_row_id_physical_column_coalesced() {
1876        let tmp_dir = TempDir::new().unwrap();
1877        let dir = tmp_dir.path().to_str().unwrap();
1878        // A file that physically carries `_row_id`, as written when carrying rows forward
1879        // across a rewrite: some rows have a stored value, some are null.
1880        let id_col = Arc::new(Int64Array::from(vec![Some(5), None, Some(8)])) as ArrayRef;
1881        let file_path = write_plain_parquet(
1882            dir,
1883            "row_id_phys.parquet",
1884            vec![physical_row_id_field()],
1885            vec![id_col],
1886        );
1887
1888        let task = row_id_task(file_path, Some(100));
1889
1890        let reader = ArrowReaderBuilder::new(FileIO::new_with_fs(), Runtime::current()).build();
1891        let tasks = Box::pin(futures::stream::iter(vec![Ok(task)])) as FileScanTaskStream;
1892        let batches: Vec<RecordBatch> = reader
1893            .read(tasks)
1894            .unwrap()
1895            .stream()
1896            .try_collect()
1897            .await
1898            .unwrap();
1899
1900        // Per-row value where non-null; first_row_id + pos (101) where null.
1901        assert_row_id_column(&batches, &[Some(5), Some(101), Some(8)]);
1902    }
1903
1904    #[tokio::test]
1905    async fn test_row_id_only_synthesis_reads_no_data_columns() {
1906        // The common v3 case: a new-row file with `first_row_id` set and NO physically
1907        // stored `_row_id`, projecting only `_row_id`. `_row_id` synthesis installs the
1908        // RowNumber virtual column (via `need_row_number`), so the row count comes from it
1909        // -- the scan must read no data columns, not fall back to reading everything.
1910        let tmp_dir = TempDir::new().unwrap();
1911        let dir = tmp_dir.path().to_str().unwrap();
1912
1913        let meta_only = metadata_projection_task_with_first_row_id(
1914            write_parquet_with_wide_column(dir, "row_id_only.parquet", vec![], vec![]),
1915            id_and_wide_schema(),
1916            vec![RESERVED_FIELD_ID_ROW_ID],
1917            Some(100),
1918        );
1919        let (batches, meta_only_bytes) = scan_task(meta_only).await;
1920
1921        assert_eq!(batches[0].num_columns(), 1);
1922        assert_row_id_column(&batches, &[Some(100), Some(101), Some(102)]);
1923
1924        // A scan that also projects the wide data column must read materially more.
1925        let with_data = metadata_projection_task_with_first_row_id(
1926            write_parquet_with_wide_column(dir, "row_id_only_ref.parquet", vec![], vec![]),
1927            id_and_wide_schema(),
1928            vec![2, RESERVED_FIELD_ID_ROW_ID],
1929            Some(100),
1930        );
1931        let (_, with_data_bytes) = scan_task(with_data).await;
1932
1933        assert!(
1934            meta_only_bytes < with_data_bytes,
1935            "_row_id-only synthesis should read fewer bytes than a scan of the wide column: \
1936             {meta_only_bytes} vs {with_data_bytes}"
1937        );
1938    }
1939
1940    #[tokio::test]
1941    async fn test_row_id_only_null_first_row_id_reads_no_data_columns() {
1942        // A null `first_row_id` (v1/v2, or a pre-upgrade v3 snapshot) nulls the whole
1943        // `_row_id` column, so nothing is synthesized -- but the column still has a length.
1944        // The RowNumber counter installed for the metadata-only projection supplies it, so
1945        // the scan reads no data columns instead of reading everything just for the count.
1946        let tmp_dir = TempDir::new().unwrap();
1947        let dir = tmp_dir.path().to_str().unwrap();
1948
1949        let meta_only = metadata_projection_task_with_first_row_id(
1950            write_parquet_with_wide_column(dir, "row_id_null.parquet", vec![], vec![]),
1951            id_and_wide_schema(),
1952            vec![RESERVED_FIELD_ID_ROW_ID],
1953            None,
1954        );
1955        let (batches, meta_only_bytes) = scan_task(meta_only).await;
1956
1957        assert_eq!(batches[0].num_columns(), 1);
1958        assert_row_id_column(&batches, &[None, None, None]);
1959
1960        let with_data = metadata_projection_task_with_first_row_id(
1961            write_parquet_with_wide_column(dir, "row_id_null_ref.parquet", vec![], vec![]),
1962            id_and_wide_schema(),
1963            vec![2, RESERVED_FIELD_ID_ROW_ID],
1964            None,
1965        );
1966        let (_, with_data_bytes) = scan_task(with_data).await;
1967
1968        assert!(
1969            meta_only_bytes < with_data_bytes,
1970            "_row_id-only scan with a null first_row_id should read fewer bytes than a scan \
1971             of the wide column: {meta_only_bytes} vs {with_data_bytes}"
1972        );
1973    }
1974
1975    #[tokio::test]
1976    async fn test_last_updated_seq_only_reads_no_data_columns() {
1977        use crate::metadata_columns::RESERVED_FIELD_ID_LAST_UPDATED_SEQUENCE_NUMBER;
1978
1979        // `SELECT _last_updated_sequence_number` derives a per-row constant (the data
1980        // sequence number) with no physical column to read. The metadata-only RowNumber
1981        // counter supplies the row count, so the scan prunes the data columns rather than
1982        // reading them all just to size the constant.
1983        let tmp_dir = TempDir::new().unwrap();
1984        let dir = tmp_dir.path().to_str().unwrap();
1985
1986        let lusn_only_task = |file_path: String, project_field_ids: Vec<i32>| {
1987            FileScanTask::builder()
1988                .with_file_size_in_bytes(std::fs::metadata(&file_path).unwrap().len())
1989                .with_start(0)
1990                .with_length(0)
1991                .with_data_file_path(file_path)
1992                .with_data_file_format(DataFileFormat::Parquet)
1993                .with_schema(id_and_wide_schema())
1994                .with_project_field_ids(project_field_ids)
1995                .with_first_row_id(Some(42))
1996                .with_data_sequence_number(Some(7))
1997                .with_case_sensitive(false)
1998                .build()
1999                .unwrap()
2000        };
2001
2002        let meta_only = lusn_only_task(
2003            write_parquet_with_wide_column(dir, "lusn_only.parquet", vec![], vec![]),
2004            vec![RESERVED_FIELD_ID_LAST_UPDATED_SEQUENCE_NUMBER],
2005        );
2006        let (batches, meta_only_bytes) = scan_task(meta_only).await;
2007
2008        assert_eq!(batches[0].num_columns(), 1);
2009        assert_last_updated_seq_column(&batches, &[Some(7), Some(7), Some(7)]);
2010
2011        let with_data = lusn_only_task(
2012            write_parquet_with_wide_column(dir, "lusn_only_ref.parquet", vec![], vec![]),
2013            vec![2, RESERVED_FIELD_ID_LAST_UPDATED_SEQUENCE_NUMBER],
2014        );
2015        let (_, with_data_bytes) = scan_task(with_data).await;
2016
2017        assert!(
2018            meta_only_bytes < with_data_bytes,
2019            "_last_updated_sequence_number-only scan should read fewer bytes than a scan of \
2020             the wide column: {meta_only_bytes} vs {with_data_bytes}"
2021        );
2022    }
2023
2024    #[tokio::test]
2025    async fn test_partition_only_reads_no_data_columns() {
2026        use arrow_array::StructArray;
2027
2028        use crate::metadata_columns::{RESERVED_COL_NAME_PARTITION, RESERVED_FIELD_ID_PARTITION};
2029        use crate::spec::{Literal, PartitionSpec, Struct, Transform};
2030
2031        // `SELECT _partition` materializes a struct constant from the task's partition
2032        // metadata, with no physical column to read. It is the only struct-constant metadata
2033        // column; the RowNumber counter sizes it, so the scan prunes the data columns.
2034        let tmp_dir = TempDir::new().unwrap();
2035        let dir = tmp_dir.path().to_str().unwrap();
2036        let schema = id_and_wide_schema();
2037        let spec = Arc::new(
2038            PartitionSpec::builder(schema.clone())
2039                .with_spec_id(7)
2040                .add_partition_field("id", "id", Transform::Identity)
2041                .unwrap()
2042                .build()
2043                .unwrap(),
2044        );
2045        let unified_type = Arc::new(spec.partition_type(&schema).unwrap());
2046        let partition_data = Struct::from_iter(vec![Some(Literal::int(42))]);
2047
2048        let partition_task = |file_path: String, project_field_ids: Vec<i32>| {
2049            FileScanTask::builder()
2050                .with_file_size_in_bytes(std::fs::metadata(&file_path).unwrap().len())
2051                .with_start(0)
2052                .with_length(0)
2053                .with_data_file_path(file_path)
2054                .with_data_file_format(DataFileFormat::Parquet)
2055                .with_schema(schema.clone())
2056                .with_project_field_ids(project_field_ids)
2057                .with_partition_spec(Some(spec.clone()))
2058                .with_partition(Some(partition_data.clone()))
2059                .with_unified_partition_type(Some(unified_type.clone()))
2060                .with_case_sensitive(false)
2061                .build()
2062                .unwrap()
2063        };
2064
2065        let meta_only = partition_task(
2066            write_parquet_with_wide_column(dir, "partition_only.parquet", vec![], vec![]),
2067            vec![RESERVED_FIELD_ID_PARTITION],
2068        );
2069        let (batches, meta_only_bytes) = scan_task(meta_only).await;
2070
2071        assert_eq!(batches[0].num_columns(), 1);
2072        let partition_col = batches[0]
2073            .column_by_name(RESERVED_COL_NAME_PARTITION)
2074            .expect("_partition column should be present")
2075            .as_any()
2076            .downcast_ref::<StructArray>()
2077            .unwrap();
2078        assert_eq!(partition_col.len(), 3);
2079        let inner = partition_col
2080            .column(0)
2081            .as_any()
2082            .downcast_ref::<Int32Array>()
2083            .unwrap();
2084        assert_eq!(inner.values(), &[42, 42, 42]);
2085
2086        let with_data = partition_task(
2087            write_parquet_with_wide_column(dir, "partition_only_ref.parquet", vec![], vec![]),
2088            vec![2, RESERVED_FIELD_ID_PARTITION],
2089        );
2090        let (_, with_data_bytes) = scan_task(with_data).await;
2091
2092        assert!(
2093            meta_only_bytes < with_data_bytes,
2094            "_partition-only scan should read fewer bytes than a scan of the wide column: \
2095             {meta_only_bytes} vs {with_data_bytes}"
2096        );
2097    }
2098
2099    #[tokio::test]
2100    async fn test_row_id_resolves_alongside_id_less_leaf() {
2101        // A file with an id-less leaf (mimicking a Variant column's internal metadata/value
2102        // leaves, which the spec requires to have no field id) plus a physical `_row_id`
2103        // that carries its embedded id. The reserved id must still resolve -- an
2104        // all-or-nothing field map would bail on the id-less leaf and wrongly reject the file.
2105        let tmp_dir = TempDir::new().unwrap();
2106        let dir = tmp_dir.path().to_str().unwrap();
2107        let idless_field = Field::new("variant_internal", DataType::Utf8, true);
2108        let idless_col = Arc::new(StringArray::from(vec!["a", "b", "c"])) as ArrayRef;
2109        let row_id_col = Arc::new(Int64Array::from(vec![Some(5), None, Some(8)])) as ArrayRef;
2110        let file_path = write_plain_parquet(
2111            dir,
2112            "row_id_with_idless_leaf.parquet",
2113            vec![idless_field, physical_row_id_field()],
2114            vec![idless_col, row_id_col],
2115        );
2116
2117        let task = row_id_task(file_path, Some(100));
2118
2119        let reader = ArrowReaderBuilder::new(FileIO::new_with_fs(), Runtime::current()).build();
2120        let tasks = Box::pin(futures::stream::iter(vec![Ok(task)])) as FileScanTaskStream;
2121        let batches: Vec<RecordBatch> = reader
2122            .read(tasks)
2123            .unwrap()
2124            .stream()
2125            .try_collect()
2126            .await
2127            .unwrap();
2128
2129        // Physical value where non-null; first_row_id + pos (101) where null.
2130        assert_row_id_column(&batches, &[Some(5), Some(101), Some(8)]);
2131    }
2132
2133    #[tokio::test]
2134    async fn test_row_id_and_last_updated_seq_co_projected() {
2135        use crate::metadata_columns::RESERVED_FIELD_ID_LAST_UPDATED_SEQUENCE_NUMBER;
2136
2137        // Both lineage columns projected together over a file carrying both physical
2138        // leaves. Each must materialize independently -- neither leaf's mask clobbers the
2139        // other, and the two synthesized columns keep their own values.
2140        let tmp_dir = TempDir::new().unwrap();
2141        let dir = tmp_dir.path().to_str().unwrap();
2142        let row_id_col = Arc::new(Int64Array::from(vec![Some(5), None, Some(8)])) as ArrayRef;
2143        let seq_col = Arc::new(Int64Array::from(vec![Some(50), None, Some(70)])) as ArrayRef;
2144        let file_path = write_plain_parquet(
2145            dir,
2146            "row_id_and_seq.parquet",
2147            vec![physical_row_id_field(), physical_last_updated_seq_field()],
2148            vec![row_id_col, seq_col],
2149        );
2150
2151        let schema = Arc::new(
2152            Schema::builder()
2153                .with_schema_id(1)
2154                .with_fields(vec![
2155                    NestedField::required(1, "id", Type::Primitive(PrimitiveType::Int)).into(),
2156                ])
2157                .build()
2158                .unwrap(),
2159        );
2160        let task = FileScanTask::builder()
2161            .with_file_size_in_bytes(std::fs::metadata(&file_path).unwrap().len())
2162            .with_start(0)
2163            .with_length(0)
2164            .with_data_file_path(file_path)
2165            .with_data_file_format(DataFileFormat::Parquet)
2166            .with_schema(schema)
2167            .with_project_field_ids(vec![
2168                1,
2169                RESERVED_FIELD_ID_ROW_ID,
2170                RESERVED_FIELD_ID_LAST_UPDATED_SEQUENCE_NUMBER,
2171            ])
2172            .with_first_row_id(Some(100))
2173            .with_data_sequence_number(Some(9))
2174            .with_case_sensitive(false)
2175            .build()
2176            .unwrap();
2177
2178        let reader = ArrowReaderBuilder::new(FileIO::new_with_fs(), Runtime::current()).build();
2179        let tasks = Box::pin(futures::stream::iter(vec![Ok(task)])) as FileScanTaskStream;
2180        let batches: Vec<RecordBatch> = reader
2181            .read(tasks)
2182            .unwrap()
2183            .stream()
2184            .try_collect()
2185            .await
2186            .unwrap();
2187
2188        // _row_id: physical value where non-null, else first_row_id + pos (101).
2189        assert_row_id_column(&batches, &[Some(5), Some(101), Some(8)]);
2190        // _last_updated_sequence_number: physical value where non-null, else data seq (9).
2191        assert_last_updated_seq_column(&batches, &[Some(50), Some(9), Some(70)]);
2192    }
2193
2194    #[tokio::test]
2195    async fn test_row_id_null_when_no_first_row_id() {
2196        let tmp_dir = TempDir::new().unwrap();
2197        let dir = tmp_dir.path().to_str().unwrap();
2198        // Physically carries `_row_id`, but the file has a null first_row_id.
2199        let id_col = Arc::new(Int64Array::from(vec![Some(5), Some(6), Some(7)])) as ArrayRef;
2200        let file_path = write_plain_parquet(
2201            dir,
2202            "row_id_no_first.parquet",
2203            vec![physical_row_id_field()],
2204            vec![id_col],
2205        );
2206
2207        // Null first_row_id: the whole column is null; the physical values are not read.
2208        let task = row_id_task(file_path, None);
2209
2210        let reader = ArrowReaderBuilder::new(FileIO::new_with_fs(), Runtime::current()).build();
2211        let tasks = Box::pin(futures::stream::iter(vec![Ok(task)])) as FileScanTaskStream;
2212        let batches: Vec<RecordBatch> = reader
2213            .read(tasks)
2214            .unwrap()
2215            .stream()
2216            .try_collect()
2217            .await
2218            .unwrap();
2219
2220        assert_row_id_column(&batches, &[None, None, None]);
2221    }
2222
2223    #[tokio::test]
2224    async fn test_row_id_with_pos_column() {
2225        use crate::metadata_columns::RESERVED_COL_NAME_POS;
2226
2227        let tmp_dir = TempDir::new().unwrap();
2228        let dir = tmp_dir.path().to_str().unwrap();
2229        let id_col = Arc::new(Int64Array::from(vec![Some(5), None, Some(8)])) as ArrayRef;
2230        let file_path = write_plain_parquet(
2231            dir,
2232            "row_id_and_pos.parquet",
2233            vec![physical_row_id_field()],
2234            vec![id_col],
2235        );
2236
2237        // Co-project `_pos` and `_row_id`. `_row_id` synthesis consumes the position, and
2238        // `_pos` is also emitted -- the RowNumber column must be added once and the two
2239        // must not interfere.
2240        let schema = Arc::new(
2241            Schema::builder()
2242                .with_schema_id(1)
2243                .with_fields(vec![
2244                    NestedField::required(1, "id", Type::Primitive(PrimitiveType::Int)).into(),
2245                ])
2246                .build()
2247                .unwrap(),
2248        );
2249        let task = FileScanTask::builder()
2250            .with_file_size_in_bytes(std::fs::metadata(&file_path).unwrap().len())
2251            .with_start(0)
2252            .with_length(0)
2253            .with_data_file_path(file_path)
2254            .with_data_file_format(DataFileFormat::Parquet)
2255            .with_schema(schema)
2256            .with_project_field_ids(vec![1, RESERVED_FIELD_ID_POS, RESERVED_FIELD_ID_ROW_ID])
2257            .with_first_row_id(Some(100))
2258            .with_case_sensitive(false)
2259            .build()
2260            .unwrap();
2261
2262        let reader = ArrowReaderBuilder::new(FileIO::new_with_fs(), Runtime::current()).build();
2263        let tasks = Box::pin(futures::stream::iter(vec![Ok(task)])) as FileScanTaskStream;
2264        let batches: Vec<RecordBatch> = reader
2265            .read(tasks)
2266            .unwrap()
2267            .stream()
2268            .try_collect()
2269            .await
2270            .unwrap();
2271
2272        // `_row_id` coalesces correctly...
2273        assert_row_id_column(&batches, &[Some(5), Some(101), Some(8)]);
2274        // ...and `_pos` is the row position, not double-counted.
2275        let pos_col = batches[0]
2276            .column_by_name(RESERVED_COL_NAME_POS)
2277            .expect("_pos column should be present")
2278            .as_primitive::<arrow_array::types::Int64Type>();
2279        assert_eq!(pos_col.values(), &[0, 1, 2]);
2280    }
2281
2282    #[tokio::test]
2283    async fn test_row_id_mixed_files_share_schema() {
2284        use arrow_select::concat::concat_batches;
2285
2286        let tmp_dir = TempDir::new().unwrap();
2287        let dir = tmp_dir.path().to_str().unwrap();
2288
2289        // Three files in one scan exercising all three column paths, which must all
2290        // produce the SAME Arrow type (plain Int64) or concatenation fails:
2291        //   - synthesis: first_row_id set, no physical column -> first_row_id + pos
2292        //   - null gate: no first_row_id -> null column
2293        //   - coalesce: first_row_id set, physical column present -> per-row + fallback
2294        let synth = row_id_task(
2295            write_plain_parquet(dir, "row_id_synth2.parquet", vec![], vec![]),
2296            Some(42),
2297        );
2298        let nulled = row_id_task(
2299            write_plain_parquet(dir, "row_id_null2.parquet", vec![], vec![]),
2300            None,
2301        );
2302        let coalesced = row_id_task(
2303            write_plain_parquet(
2304                dir,
2305                "row_id_coalesced2.parquet",
2306                vec![physical_row_id_field()],
2307                vec![Arc::new(Int64Array::from(vec![Some(5), None, Some(8)])) as ArrayRef],
2308            ),
2309            Some(50),
2310        );
2311
2312        let reader = ArrowReaderBuilder::new(FileIO::new_with_fs(), Runtime::current()).build();
2313        let tasks = Box::pin(futures::stream::iter(vec![
2314            Ok(synth),
2315            Ok(nulled),
2316            Ok(coalesced),
2317        ])) as FileScanTaskStream;
2318        let batches: Vec<RecordBatch> = reader
2319            .read(tasks)
2320            .unwrap()
2321            .stream()
2322            .try_collect()
2323            .await
2324            .unwrap();
2325
2326        assert_eq!(batches.len(), 3);
2327        let schema = batches[0].schema();
2328        concat_batches(&schema, &batches)
2329            .expect("synthesis, null and coalesce files must share one column type");
2330    }
2331
2332    #[tokio::test]
2333    async fn test_row_id_present_by_name_without_id_unsupported() {
2334        let tmp_dir = TempDir::new().unwrap();
2335        let dir = tmp_dir.path().to_str().unwrap();
2336        // Column present by name but WITHOUT the embedded field id. The transformer keys
2337        // the source column by field id, so this shape can't be threaded and is rejected.
2338        let id_field = Field::new(RESERVED_COL_NAME_ROW_ID, DataType::Int64, true);
2339        let id_col = Arc::new(Int64Array::from(vec![Some(5), None, Some(8)])) as ArrayRef;
2340        let file_path =
2341            write_plain_parquet(dir, "row_id_by_name.parquet", vec![id_field], vec![id_col]);
2342
2343        let task = row_id_task(file_path, Some(100));
2344
2345        let reader = ArrowReaderBuilder::new(FileIO::new_with_fs(), Runtime::current()).build();
2346        let tasks = Box::pin(futures::stream::iter(vec![Ok(task)])) as FileScanTaskStream;
2347        let result: Result<Vec<RecordBatch>, _> =
2348            reader.read(tasks).unwrap().stream().try_collect().await;
2349
2350        let err = result.unwrap_err();
2351        assert_eq!(err.kind(), crate::ErrorKind::FeatureUnsupported);
2352        assert!(
2353            format!("{err}").contains("without an embedded field id"),
2354            "unexpected error: {err}"
2355        );
2356    }
2357
2358    #[tokio::test]
2359    async fn test_row_id_present_by_name_without_id_nulls_when_no_lineage() {
2360        let tmp_dir = TempDir::new().unwrap();
2361        let dir = tmp_dir.path().to_str().unwrap();
2362        // Same name-only shape as the reject test above, but the file carries no row lineage
2363        // (`first_row_id = None`), as a migrated pre-v3 file with a user column named
2364        // `_row_id` would. The physical leaf is never read, so `_row_id` is nulled out
2365        // rather than rejected (matching Java `ValueReaders.rowIds`).
2366        let id_field = Field::new(RESERVED_COL_NAME_ROW_ID, DataType::Int64, true);
2367        let id_col = Arc::new(Int64Array::from(vec![Some(5), None, Some(8)])) as ArrayRef;
2368        let file_path = write_plain_parquet(
2369            dir,
2370            "row_id_by_name_no_lineage.parquet",
2371            vec![id_field],
2372            vec![id_col],
2373        );
2374
2375        let task = row_id_task(file_path, None);
2376
2377        let reader = ArrowReaderBuilder::new(FileIO::new_with_fs(), Runtime::current()).build();
2378        let tasks = Box::pin(futures::stream::iter(vec![Ok(task)])) as FileScanTaskStream;
2379        let batches: Vec<RecordBatch> = reader
2380            .read(tasks)
2381            .unwrap()
2382            .stream()
2383            .try_collect()
2384            .await
2385            .unwrap();
2386
2387        assert_row_id_column(&batches, &[None, None, None]);
2388    }
2389
2390    /// Builds a `row_id_task` (see above) that additionally carries a bound predicate,
2391    /// so a `RowSelection` is applied when the reader has row selection enabled.
2392    fn row_id_task_with_predicate(
2393        file_path: String,
2394        first_row_id: Option<i64>,
2395        extra_project_field_ids: Vec<i32>,
2396        predicate: crate::expr::Predicate,
2397    ) -> FileScanTask {
2398        use crate::expr::Bind;
2399
2400        let schema = Arc::new(
2401            Schema::builder()
2402                .with_schema_id(1)
2403                .with_fields(vec![
2404                    NestedField::required(1, "id", Type::Primitive(PrimitiveType::Int)).into(),
2405                ])
2406                .build()
2407                .unwrap(),
2408        );
2409        let bound = predicate.bind(Arc::clone(&schema), false).unwrap();
2410
2411        let mut project_field_ids = vec![1];
2412        project_field_ids.extend(extra_project_field_ids);
2413        project_field_ids.push(RESERVED_FIELD_ID_ROW_ID);
2414
2415        FileScanTask::builder()
2416            .with_file_size_in_bytes(std::fs::metadata(&file_path).unwrap().len())
2417            .with_start(0)
2418            .with_length(0)
2419            .with_data_file_path(file_path)
2420            .with_data_file_format(DataFileFormat::Parquet)
2421            .with_schema(schema)
2422            .with_project_field_ids(project_field_ids)
2423            .with_predicate(Some(bound))
2424            .with_first_row_id(first_row_id)
2425            .with_case_sensitive(false)
2426            .build()
2427            .unwrap()
2428    }
2429
2430    #[tokio::test]
2431    async fn test_row_id_stable_under_row_selection() {
2432        use crate::expr::Reference;
2433        use crate::spec::Datum;
2434
2435        let tmp_dir = TempDir::new().unwrap();
2436        let dir = tmp_dir.path().to_str().unwrap();
2437        // id = [1, 2, 3]; drop the middle physical row via a predicate + row selection.
2438        let file_path = write_plain_parquet(dir, "row_id_selection.parquet", vec![], vec![]);
2439
2440        let task = row_id_task_with_predicate(
2441            file_path,
2442            Some(100),
2443            vec![],
2444            Reference::new("id").not_equal_to(Datum::int(2)),
2445        );
2446
2447        // Row selection must be enabled for the predicate to produce a RowSelection.
2448        let reader = ArrowReaderBuilder::new(FileIO::new_with_fs(), Runtime::current())
2449            .with_options(
2450                ReaderOptions::builder()
2451                    .with_row_selection_enabled(true)
2452                    .build(),
2453            )
2454            .build();
2455        let tasks = Box::pin(futures::stream::iter(vec![Ok(task)])) as FileScanTaskStream;
2456        let batches: Vec<RecordBatch> = reader
2457            .read(tasks)
2458            .unwrap()
2459            .stream()
2460            .try_collect()
2461            .await
2462            .unwrap();
2463
2464        // The survivors are physical rows 0 and 2, so their _row_id is first_row_id + the
2465        // PHYSICAL position: [100, 102]. A dense output index would wrongly give [100, 101].
2466        assert_row_id_column(&batches, &[Some(100), Some(102)]);
2467    }
2468
2469    #[tokio::test]
2470    async fn test_row_id_coalesce_stable_under_row_selection() {
2471        use crate::expr::Reference;
2472        use crate::spec::Datum;
2473
2474        let tmp_dir = TempDir::new().unwrap();
2475        let dir = tmp_dir.path().to_str().unwrap();
2476        // Physical _row_id = [Some(5), None, Some(8)] over id = [1, 2, 3]. Dropping the
2477        // middle row must keep the physical column and the RowNumber fallback row-aligned.
2478        let id_col = Arc::new(Int64Array::from(vec![Some(5), None, Some(8)])) as ArrayRef;
2479        let file_path = write_plain_parquet(
2480            dir,
2481            "row_id_coalesce_selection.parquet",
2482            vec![physical_row_id_field()],
2483            vec![id_col],
2484        );
2485
2486        let task = row_id_task_with_predicate(
2487            file_path,
2488            Some(100),
2489            vec![],
2490            Reference::new("id").not_equal_to(Datum::int(2)),
2491        );
2492
2493        let reader = ArrowReaderBuilder::new(FileIO::new_with_fs(), Runtime::current())
2494            .with_options(
2495                ReaderOptions::builder()
2496                    .with_row_selection_enabled(true)
2497                    .build(),
2498            )
2499            .build();
2500        let tasks = Box::pin(futures::stream::iter(vec![Ok(task)])) as FileScanTaskStream;
2501        let batches: Vec<RecordBatch> = reader
2502            .read(tasks)
2503            .unwrap()
2504            .stream()
2505            .try_collect()
2506            .await
2507            .unwrap();
2508
2509        // Rows 0 and 2 survive: their stored values (5, 8) pass through. The dropped
2510        // row's null (which would have fallen back to 100 + 1) is gone -- proving the
2511        // physical column and the positional fallback are filtered by the same selection.
2512        assert_row_id_column(&batches, &[Some(5), Some(8)]);
2513    }
2514
2515    #[tokio::test]
2516    async fn test_row_id_global_across_row_groups() {
2517        let tmp_dir = TempDir::new().unwrap();
2518        let dir = tmp_dir.path().to_str().unwrap();
2519
2520        // 5 rows written with max_row_group_size = 2 -> 3 row groups. `_pos` must be the
2521        // GLOBAL file position, so `_row_id` continues across row-group boundaries rather
2522        // than restarting per group (which would silently duplicate ids).
2523        let file_path = format!("{dir}/row_id_multi_rg.parquet");
2524        let field = Field::new("id", DataType::Int32, false).with_metadata(HashMap::from([(
2525            PARQUET_FIELD_ID_META_KEY.to_string(),
2526            "1".to_string(),
2527        )]));
2528        let arrow_schema = Arc::new(ArrowSchema::new(vec![field]));
2529        let batch = RecordBatch::try_new(arrow_schema.clone(), vec![Arc::new(Int32Array::from(
2530            vec![1, 2, 3, 4, 5],
2531        ))])
2532        .unwrap();
2533        let props = WriterProperties::builder()
2534            .set_compression(Compression::SNAPPY)
2535            .set_max_row_group_row_count(Some(2))
2536            .build();
2537        let file = File::create(&file_path).unwrap();
2538        let mut writer = ArrowWriter::try_new(file, arrow_schema, Some(props)).unwrap();
2539        writer.write(&batch).unwrap();
2540        writer.close().unwrap();
2541
2542        let task = row_id_task(file_path, Some(0));
2543        let reader = ArrowReaderBuilder::new(FileIO::new_with_fs(), Runtime::current()).build();
2544        let tasks = Box::pin(futures::stream::iter(vec![Ok(task)])) as FileScanTaskStream;
2545        let batches: Vec<RecordBatch> = reader
2546            .read(tasks)
2547            .unwrap()
2548            .stream()
2549            .try_collect()
2550            .await
2551            .unwrap();
2552
2553        assert_row_id_column(&batches, &[Some(0), Some(1), Some(2), Some(3), Some(4)]);
2554    }
2555
2556    #[tokio::test]
2557    async fn test_row_id_global_when_first_row_group_pruned() {
2558        use parquet::file::reader::{FileReader, SerializedFileReader};
2559
2560        let tmp_dir = TempDir::new().unwrap();
2561        let dir = tmp_dir.path().to_str().unwrap();
2562
2563        // 6 rows with max_row_group_size = 2 -> 3 row groups. A byte-range split that prunes
2564        // row group 0 reaches the reader via `with_row_groups()` -- a different path than the
2565        // `RowSelection` cases above. The survivors must keep their GLOBAL positions (starting
2566        // at 2), so a per-group RowNumber restart would surface as duplicate ids here.
2567        let file_path = format!("{dir}/row_id_prune_rg0.parquet");
2568        let field = Field::new("id", DataType::Int32, false).with_metadata(HashMap::from([(
2569            PARQUET_FIELD_ID_META_KEY.to_string(),
2570            "1".to_string(),
2571        )]));
2572        let arrow_schema = Arc::new(ArrowSchema::new(vec![field]));
2573        let batch = RecordBatch::try_new(arrow_schema.clone(), vec![Arc::new(Int32Array::from(
2574            vec![1, 2, 3, 4, 5, 6],
2575        ))])
2576        .unwrap();
2577        let props = WriterProperties::builder()
2578            .set_compression(Compression::SNAPPY)
2579            .set_max_row_group_row_count(Some(2))
2580            .build();
2581        let file = File::create(&file_path).unwrap();
2582        let mut writer = ArrowWriter::try_new(file, arrow_schema, Some(props)).unwrap();
2583        writer.write(&batch).unwrap();
2584        writer.close().unwrap();
2585
2586        // A byte range starting just past row group 0 prunes it (its midpoint falls below
2587        // `start`) while keeping groups 1 and 2 (physical rows 2..6).
2588        let metadata = SerializedFileReader::new(File::open(&file_path).unwrap())
2589            .unwrap()
2590            .metadata()
2591            .clone();
2592        assert_eq!(metadata.num_row_groups(), 3);
2593        let start = 4 + metadata.row_group(0).compressed_size() as u64;
2594        let file_size = std::fs::metadata(&file_path).unwrap().len();
2595
2596        let task = row_id_task_with_options(file_path, Some(100), start, file_size - start, vec![]);
2597
2598        let reader = ArrowReaderBuilder::new(FileIO::new_with_fs(), Runtime::current()).build();
2599        let tasks = Box::pin(futures::stream::iter(vec![Ok(task)])) as FileScanTaskStream;
2600        let batches: Vec<RecordBatch> = reader
2601            .read(tasks)
2602            .unwrap()
2603            .stream()
2604            .try_collect()
2605            .await
2606            .unwrap();
2607
2608        // Groups 1 and 2 survive at physical positions 2..6 -> _row_id 102..106, not a
2609        // per-group restart at 100.
2610        assert_row_id_column(&batches, &[Some(102), Some(103), Some(104), Some(105)]);
2611    }
2612
2613    /// Writes a positional delete file (`file_path` + `pos` reserved columns) marking the
2614    /// given `positions` of `data_file_path` as deleted.
2615    fn write_positional_delete(
2616        dir: &str,
2617        name: &str,
2618        data_file_path: &str,
2619        positions: &[i64],
2620    ) -> String {
2621        use arrow_array::StringArray;
2622
2623        let file_path_field =
2624            Field::new("file_path", DataType::Utf8, false).with_metadata(HashMap::from([(
2625                PARQUET_FIELD_ID_META_KEY.to_string(),
2626                "2147483546".to_string(),
2627            )]));
2628        let pos_field = Field::new("pos", DataType::Int64, false).with_metadata(HashMap::from([(
2629            PARQUET_FIELD_ID_META_KEY.to_string(),
2630            "2147483545".to_string(),
2631        )]));
2632        let arrow_schema = Arc::new(ArrowSchema::new(vec![file_path_field, pos_field]));
2633        let batch = RecordBatch::try_new(arrow_schema.clone(), vec![
2634            Arc::new(StringArray::from(vec![data_file_path; positions.len()])),
2635            Arc::new(Int64Array::from(positions.to_vec())),
2636        ])
2637        .unwrap();
2638
2639        let path = format!("{dir}/{name}");
2640        let file = File::create(&path).unwrap();
2641        let props = WriterProperties::builder()
2642            .set_compression(Compression::SNAPPY)
2643            .build();
2644        let mut writer = ArrowWriter::try_new(file, arrow_schema, Some(props)).unwrap();
2645        writer.write(&batch).unwrap();
2646        writer.close().unwrap();
2647        path
2648    }
2649
2650    #[tokio::test]
2651    async fn test_row_id_survives_positional_delete() {
2652        use crate::spec::DataContentType;
2653
2654        let tmp_dir = TempDir::new().unwrap();
2655        let dir = tmp_dir.path().to_str().unwrap();
2656        // id = [1, 2, 3]; a positional delete drops the middle physical row (pos = 1).
2657        let data_path = write_plain_parquet(dir, "row_id_posdel_data.parquet", vec![], vec![]);
2658        let del_path = write_positional_delete(dir, "row_id_posdel.parquet", &data_path, &[1]);
2659
2660        let delete = FileScanTaskDeleteFile::builder()
2661            .with_file_path(del_path.clone())
2662            .with_file_size_in_bytes(std::fs::metadata(&del_path).unwrap().len())
2663            .with_file_type(DataContentType::PositionDeletes)
2664            .with_file_format(DataFileFormat::Parquet)
2665            .with_partition_spec_id(0)
2666            .build()
2667            .unwrap();
2668        let task = row_id_task_with_options(data_path, Some(100), 0, 0, vec![delete]);
2669
2670        let reader = ArrowReaderBuilder::new(FileIO::new_with_fs(), Runtime::current()).build();
2671        let tasks = Box::pin(futures::stream::iter(vec![Ok(task)])) as FileScanTaskStream;
2672        let batches: Vec<RecordBatch> = reader
2673            .read(tasks)
2674            .unwrap()
2675            .stream()
2676            .try_collect()
2677            .await
2678            .unwrap();
2679
2680        // Survivors keep their ABSOLUTE positions -- [100, 102], not renumbered [100, 101].
2681        // Positional-delete selection reaches the reader via a different path than predicate
2682        // selection, so this covers it explicitly.
2683        assert_row_id_column(&batches, &[Some(100), Some(102)]);
2684    }
2685
2686    #[tokio::test]
2687    async fn test_row_id_name_collision_under_positional_fallback() {
2688        let tmp_dir = TempDir::new().unwrap();
2689        let dir = tmp_dir.path().to_str().unwrap();
2690
2691        // A file with NO embedded field ids (positional fallback) whose columns include one
2692        // literally named `_row_id` (user data). Projecting `_row_id` must NOT be rejected as
2693        // an unthreadable physical metadata column -- under fallback the reserved column is
2694        // synthesized and the same-named user column is just data.
2695        let file_path = format!("{dir}/fallback_row_id_name.parquet");
2696        let arrow_schema = Arc::new(ArrowSchema::new(vec![
2697            Field::new("id", DataType::Int32, false),
2698            Field::new(RESERVED_COL_NAME_ROW_ID, DataType::Int64, true),
2699        ]));
2700        let batch = RecordBatch::try_new(arrow_schema.clone(), vec![
2701            Arc::new(Int32Array::from(vec![1, 2, 3])),
2702            Arc::new(Int64Array::from(vec![7i64, 8, 9])),
2703        ])
2704        .unwrap();
2705        let props = WriterProperties::builder()
2706            .set_compression(Compression::SNAPPY)
2707            .build();
2708        let file = File::create(&file_path).unwrap();
2709        let mut writer = ArrowWriter::try_new(file, arrow_schema, Some(props)).unwrap();
2710        writer.write(&batch).unwrap();
2711        writer.close().unwrap();
2712
2713        let task = row_id_task(file_path, Some(100));
2714        let reader = ArrowReaderBuilder::new(FileIO::new_with_fs(), Runtime::current()).build();
2715        let tasks = Box::pin(futures::stream::iter(vec![Ok(task)])) as FileScanTaskStream;
2716        let batches: Vec<RecordBatch> = reader
2717            .read(tasks)
2718            .unwrap()
2719            .stream()
2720            .try_collect()
2721            .await
2722            .unwrap();
2723
2724        // Not rejected; `_row_id` is synthesized as first_row_id + pos.
2725        assert_row_id_column(&batches, &[Some(100), Some(101), Some(102)]);
2726    }
2727
2728    #[tokio::test]
2729    async fn test_read_encrypted_parquet_with_wrong_key_fails() {
2730        let encryption_key = b"0123456789abcdef";
2731        let wrong_key = b"fedcba9876543210";
2732
2733        let schema = Arc::new(
2734            Schema::builder()
2735                .with_schema_id(1)
2736                .with_fields(vec![
2737                    NestedField::required(1, "id", Type::Primitive(PrimitiveType::Int)).into(),
2738                ])
2739                .build()
2740                .unwrap(),
2741        );
2742
2743        let arrow_schema = Arc::new(ArrowSchema::new(vec![
2744            Field::new("id", DataType::Int32, false).with_metadata(HashMap::from([(
2745                PARQUET_FIELD_ID_META_KEY.to_string(),
2746                "1".to_string(),
2747            )])),
2748        ]));
2749
2750        let tmp_dir = TempDir::new().unwrap();
2751        let table_location = tmp_dir.path().to_str().unwrap().to_string();
2752        let file_io = FileIO::new_with_fs();
2753
2754        let id_data = Arc::new(Int32Array::from(vec![1, 2, 3])) as ArrayRef;
2755        let batch = RecordBatch::try_new(arrow_schema.clone(), vec![id_data]).unwrap();
2756
2757        let file_path = format!("{table_location}/encrypted_wrong_key.parquet");
2758        write_encrypted_parquet(&file_path, &batch, encryption_key, None);
2759
2760        let wrong_key_metadata = crate::encryption::StandardKeyMetadata::try_new(wrong_key)
2761            .unwrap()
2762            .encode()
2763            .unwrap();
2764
2765        let reader = ArrowReaderBuilder::new(file_io, Runtime::current()).build();
2766
2767        let task = FileScanTask::builder()
2768            .with_file_size_in_bytes(std::fs::metadata(&file_path).unwrap().len())
2769            .with_start(0)
2770            .with_length(0)
2771            .with_data_file_path(file_path)
2772            .with_data_file_format(DataFileFormat::Parquet)
2773            .with_schema(schema)
2774            .with_project_field_ids(vec![1])
2775            .with_case_sensitive(false)
2776            .with_key_metadata(Some(wrong_key_metadata))
2777            .build()
2778            .unwrap();
2779
2780        let tasks = Box::pin(futures::stream::iter(vec![Ok(task)])) as FileScanTaskStream;
2781        let result: Result<Vec<RecordBatch>, _> =
2782            reader.read(tasks).unwrap().stream().try_collect().await;
2783
2784        let err = result.unwrap_err();
2785        assert_eq!(err.kind(), crate::ErrorKind::Unexpected);
2786        let err_str = format!("{err}");
2787        assert!(
2788            err_str.contains("unable to decrypt parquet footer"),
2789            "Expected error about decryption failure, got: {err_str}"
2790        );
2791    }
2792
2793    /// Test that concurrency=1 reads all files correctly and in deterministic order.
2794    /// This verifies the fast-path optimization for single concurrency.
2795    #[tokio::test]
2796    async fn test_read_with_concurrency_one() {
2797        use arrow_array::Int32Array;
2798
2799        let schema = Arc::new(
2800            Schema::builder()
2801                .with_schema_id(1)
2802                .with_fields(vec![
2803                    NestedField::required(1, "id", Type::Primitive(PrimitiveType::Int)).into(),
2804                    NestedField::required(2, "file_num", Type::Primitive(PrimitiveType::Int))
2805                        .into(),
2806                ])
2807                .build()
2808                .unwrap(),
2809        );
2810
2811        let arrow_schema = Arc::new(ArrowSchema::new(vec![
2812            Field::new("id", DataType::Int32, false).with_metadata(HashMap::from([(
2813                PARQUET_FIELD_ID_META_KEY.to_string(),
2814                "1".to_string(),
2815            )])),
2816            Field::new("file_num", DataType::Int32, false).with_metadata(HashMap::from([(
2817                PARQUET_FIELD_ID_META_KEY.to_string(),
2818                "2".to_string(),
2819            )])),
2820        ]));
2821
2822        let tmp_dir = TempDir::new().unwrap();
2823        let table_location = tmp_dir.path().to_str().unwrap().to_string();
2824        let file_io = FileIO::new_with_fs();
2825
2826        // Create 3 parquet files with different data
2827        let props = WriterProperties::builder()
2828            .set_compression(Compression::SNAPPY)
2829            .build();
2830
2831        for file_num in 0..3 {
2832            let id_data = Arc::new(Int32Array::from_iter_values(
2833                file_num * 10..(file_num + 1) * 10,
2834            )) as ArrayRef;
2835            let file_num_data = Arc::new(Int32Array::from(vec![file_num; 10])) as ArrayRef;
2836
2837            let to_write =
2838                RecordBatch::try_new(arrow_schema.clone(), vec![id_data, file_num_data]).unwrap();
2839
2840            let file = File::create(format!("{table_location}/file_{file_num}.parquet")).unwrap();
2841            let mut writer =
2842                ArrowWriter::try_new(file, to_write.schema(), Some(props.clone())).unwrap();
2843            writer.write(&to_write).expect("Writing batch");
2844            writer.close().unwrap();
2845        }
2846
2847        // Read with concurrency=1 (fast-path)
2848        let reader = ArrowReaderBuilder::new(file_io, Runtime::current())
2849            .with_options(
2850                ReaderOptions::builder()
2851                    .with_data_file_concurrency_limit(1)
2852                    .build(),
2853            )
2854            .build();
2855
2856        // Create tasks in a specific order: file_0, file_1, file_2
2857        let tasks = vec![
2858            Ok(FileScanTask::builder()
2859                .with_file_size_in_bytes(
2860                    std::fs::metadata(format!("{table_location}/file_0.parquet"))
2861                        .unwrap()
2862                        .len(),
2863                )
2864                .with_start(0)
2865                .with_length(0)
2866                .with_data_file_path(format!("{table_location}/file_0.parquet"))
2867                .with_data_file_format(DataFileFormat::Parquet)
2868                .with_schema(schema.clone())
2869                .with_project_field_ids(vec![1, 2])
2870                .with_case_sensitive(false)
2871                .build()
2872                .unwrap()),
2873            Ok(FileScanTask::builder()
2874                .with_file_size_in_bytes(
2875                    std::fs::metadata(format!("{table_location}/file_1.parquet"))
2876                        .unwrap()
2877                        .len(),
2878                )
2879                .with_start(0)
2880                .with_length(0)
2881                .with_data_file_path(format!("{table_location}/file_1.parquet"))
2882                .with_data_file_format(DataFileFormat::Parquet)
2883                .with_schema(schema.clone())
2884                .with_project_field_ids(vec![1, 2])
2885                .with_case_sensitive(false)
2886                .build()
2887                .unwrap()),
2888            Ok(FileScanTask::builder()
2889                .with_file_size_in_bytes(
2890                    std::fs::metadata(format!("{table_location}/file_2.parquet"))
2891                        .unwrap()
2892                        .len(),
2893                )
2894                .with_start(0)
2895                .with_length(0)
2896                .with_data_file_path(format!("{table_location}/file_2.parquet"))
2897                .with_data_file_format(DataFileFormat::Parquet)
2898                .with_schema(schema.clone())
2899                .with_project_field_ids(vec![1, 2])
2900                .with_case_sensitive(false)
2901                .build()
2902                .unwrap()),
2903        ];
2904
2905        let tasks_stream = Box::pin(futures::stream::iter(tasks)) as FileScanTaskStream;
2906
2907        let result = reader
2908            .read(tasks_stream)
2909            .unwrap()
2910            .stream()
2911            .try_collect::<Vec<RecordBatch>>()
2912            .await
2913            .unwrap();
2914
2915        // Verify we got all 30 rows (10 from each file)
2916        let total_rows: usize = result.iter().map(|b| b.num_rows()).sum();
2917        assert_eq!(total_rows, 30, "Should have 30 total rows");
2918
2919        // Collect all ids and file_nums to verify data
2920        let mut all_ids = Vec::new();
2921        let mut all_file_nums = Vec::new();
2922
2923        for batch in &result {
2924            let id_col = batch
2925                .column(0)
2926                .as_primitive::<arrow_array::types::Int32Type>();
2927            let file_num_col = batch
2928                .column(1)
2929                .as_primitive::<arrow_array::types::Int32Type>();
2930
2931            for i in 0..batch.num_rows() {
2932                all_ids.push(id_col.value(i));
2933                all_file_nums.push(file_num_col.value(i));
2934            }
2935        }
2936
2937        assert_eq!(all_ids.len(), 30);
2938        assert_eq!(all_file_nums.len(), 30);
2939
2940        // With concurrency=1 and sequential processing, files should be processed in order
2941        // file_0: ids 0-9, file_num=0
2942        // file_1: ids 10-19, file_num=1
2943        // file_2: ids 20-29, file_num=2
2944        for i in 0..10 {
2945            assert_eq!(all_file_nums[i], 0, "First 10 rows should be from file_0");
2946            assert_eq!(all_ids[i], i as i32, "IDs should be 0-9");
2947        }
2948        for i in 10..20 {
2949            assert_eq!(all_file_nums[i], 1, "Next 10 rows should be from file_1");
2950            assert_eq!(all_ids[i], i as i32, "IDs should be 10-19");
2951        }
2952        for i in 20..30 {
2953            assert_eq!(all_file_nums[i], 2, "Last 10 rows should be from file_2");
2954            assert_eq!(all_ids[i], i as i32, "IDs should be 20-29");
2955        }
2956    }
2957
2958    #[tokio::test]
2959    async fn test_read_int96_timestamps_with_field_ids() {
2960        let schema = Arc::new(
2961            Schema::builder()
2962                .with_schema_id(1)
2963                .with_fields(vec![
2964                    NestedField::optional(1, "ts", Type::Primitive(PrimitiveType::Timestamp))
2965                        .into(),
2966                    NestedField::required(2, "id", Type::Primitive(PrimitiveType::Int)).into(),
2967                ])
2968                .build()
2969                .unwrap(),
2970        );
2971
2972        let tmp_dir = TempDir::new().unwrap();
2973        let table_location = tmp_dir.path().to_str().unwrap().to_string();
2974        let (file_path, expected_micros) =
2975            write_int96_parquet_file(&table_location, "with_ids.parquet", true);
2976
2977        assert_int96_read_matches(&file_path, schema, vec![1, 2], &expected_micros).await;
2978    }
2979
2980    #[tokio::test]
2981    async fn test_read_int96_timestamps_without_field_ids() {
2982        let schema = Arc::new(
2983            Schema::builder()
2984                .with_schema_id(1)
2985                .with_fields(vec![
2986                    NestedField::optional(1, "ts", Type::Primitive(PrimitiveType::Timestamp))
2987                        .into(),
2988                    NestedField::required(2, "id", Type::Primitive(PrimitiveType::Int)).into(),
2989                ])
2990                .build()
2991                .unwrap(),
2992        );
2993
2994        let tmp_dir = TempDir::new().unwrap();
2995        let table_location = tmp_dir.path().to_str().unwrap().to_string();
2996        let (file_path, expected_micros) =
2997            write_int96_parquet_file(&table_location, "no_ids.parquet", false);
2998
2999        assert_int96_read_matches(&file_path, schema, vec![1, 2], &expected_micros).await;
3000    }
3001
3002    #[tokio::test]
3003    async fn test_read_int96_timestamps_with_fallback_ids_and_pos() {
3004        use arrow_array::TimestampMicrosecondArray;
3005
3006        // Regression test for the combined path this refactor introduced: a field-id-less
3007        // file (positional fallback IDs) with an INT96 column and a `_pos` projection.
3008        // All three transforms -- field-ID assignment, INT96 coercion, and the row-number
3009        // virtual column -- apply in the single ArrowReaderMetadata rebuild.
3010        let schema = Arc::new(
3011            Schema::builder()
3012                .with_schema_id(1)
3013                .with_fields(vec![
3014                    NestedField::optional(1, "ts", Type::Primitive(PrimitiveType::Timestamp))
3015                        .into(),
3016                    NestedField::required(2, "id", Type::Primitive(PrimitiveType::Int)).into(),
3017                ])
3018                .build()
3019                .unwrap(),
3020        );
3021
3022        let tmp_dir = TempDir::new().unwrap();
3023        let table_location = tmp_dir.path().to_str().unwrap().to_string();
3024        let (file_path, expected_micros) =
3025            write_int96_parquet_file(&table_location, "no_ids_with_pos.parquet", false);
3026
3027        let batches =
3028            read_int96_batches(&file_path, schema, vec![1, 2, RESERVED_FIELD_ID_POS]).await;
3029
3030        assert_eq!(batches.len(), 1);
3031        // The INT96 timestamps are coerced to micros...
3032        let ts_col = batches[0]
3033            .column_by_name("ts")
3034            .expect("ts column should be present")
3035            .as_any()
3036            .downcast_ref::<TimestampMicrosecondArray>()
3037            .expect("Expected TimestampMicrosecondArray");
3038        for (i, expected) in expected_micros.iter().enumerate() {
3039            assert_eq!(ts_col.value(i), *expected, "Row {i}");
3040        }
3041        // ...and `_pos`, materialized by the RowNumber virtual column, counts rows 0,1,2.
3042        let pos_col = batches[0]
3043            .column_by_name(RESERVED_COL_NAME_POS)
3044            .expect("_pos column should be present")
3045            .as_primitive::<arrow_array::types::Int64Type>();
3046        assert_eq!(pos_col.values(), &[0, 1, 2]);
3047    }
3048
3049    #[tokio::test]
3050    async fn test_read_int96_timestamps_with_field_ids_and_pos() {
3051        use arrow_array::TimestampMicrosecondArray;
3052
3053        let schema = Arc::new(
3054            Schema::builder()
3055                .with_schema_id(1)
3056                .with_fields(vec![
3057                    NestedField::optional(1, "ts", Type::Primitive(PrimitiveType::Timestamp))
3058                        .into(),
3059                    NestedField::required(2, "id", Type::Primitive(PrimitiveType::Int)).into(),
3060                ])
3061                .build()
3062                .unwrap(),
3063        );
3064
3065        let tmp_dir = TempDir::new().unwrap();
3066        let table_location = tmp_dir.path().to_str().unwrap().to_string();
3067        let (file_path, expected_micros) =
3068            write_int96_parquet_file(&table_location, "with_ids_with_pos.parquet", true);
3069
3070        let batches =
3071            read_int96_batches(&file_path, schema, vec![1, 2, RESERVED_FIELD_ID_POS]).await;
3072
3073        assert_eq!(batches.len(), 1);
3074        let ts_col = batches[0]
3075            .column_by_name("ts")
3076            .expect("ts column should be present")
3077            .as_any()
3078            .downcast_ref::<TimestampMicrosecondArray>()
3079            .expect("Expected TimestampMicrosecondArray");
3080
3081        for (i, expected) in expected_micros.iter().enumerate() {
3082            assert_eq!(ts_col.value(i), *expected, "Row {i}");
3083        }
3084
3085        let pos_col = batches[0]
3086            .column_by_name(RESERVED_COL_NAME_POS)
3087            .expect("_pos column should be present")
3088            .as_primitive::<arrow_array::types::Int64Type>();
3089
3090        assert_eq!(pos_col.values(), &[0, 1, 2]);
3091    }
3092
3093    #[tokio::test]
3094    async fn test_read_int96_timestamps_in_struct() {
3095        use arrow_array::{StructArray, TimestampMicrosecondArray};
3096        use parquet::basic::{Repetition, Type as PhysicalType};
3097        use parquet::data_type::Int96Type;
3098        use parquet::file::writer::SerializedFileWriter;
3099        use parquet::schema::types::Type as SchemaType;
3100
3101        let tmp_dir = TempDir::new().unwrap();
3102        let table_location = tmp_dir.path().to_str().unwrap().to_string();
3103        let file_path = format!("{table_location}/struct_int96.parquet");
3104
3105        let ts_type = SchemaType::primitive_type_builder("ts", PhysicalType::INT96)
3106            .with_repetition(Repetition::OPTIONAL)
3107            .with_id(Some(2))
3108            .build()
3109            .unwrap();
3110
3111        let struct_type = SchemaType::group_type_builder("data")
3112            .with_repetition(Repetition::REQUIRED)
3113            .with_id(Some(1))
3114            .with_fields(vec![Arc::new(ts_type)])
3115            .build()
3116            .unwrap();
3117
3118        let parquet_schema = SchemaType::group_type_builder("schema")
3119            .with_fields(vec![Arc::new(struct_type)])
3120            .build()
3121            .unwrap();
3122
3123        let (int96_val, expected_micros) = make_int96_test_value();
3124
3125        let file = File::create(&file_path).unwrap();
3126        let mut writer =
3127            SerializedFileWriter::new(file, Arc::new(parquet_schema), Default::default()).unwrap();
3128
3129        // def=1: struct is REQUIRED so no level, ts is OPTIONAL and present (1).
3130        // No repetition levels needed (no repeated groups).
3131        let mut row_group = writer.next_row_group().unwrap();
3132        {
3133            let mut col = row_group.next_column().unwrap().unwrap();
3134            col.typed::<Int96Type>()
3135                .write_batch(&[int96_val], Some(&[1]), None)
3136                .unwrap();
3137            col.close().unwrap();
3138        }
3139        row_group.close().unwrap();
3140        writer.close().unwrap();
3141
3142        let iceberg_schema = Arc::new(
3143            Schema::builder()
3144                .with_schema_id(1)
3145                .with_fields(vec![
3146                    NestedField::required(
3147                        1,
3148                        "data",
3149                        Type::Struct(crate::spec::StructType::new(vec![
3150                            NestedField::optional(
3151                                2,
3152                                "ts",
3153                                Type::Primitive(PrimitiveType::Timestamp),
3154                            )
3155                            .into(),
3156                        ])),
3157                    )
3158                    .into(),
3159                ])
3160                .build()
3161                .unwrap(),
3162        );
3163
3164        let batches = read_int96_batches(&file_path, iceberg_schema, vec![1]).await;
3165
3166        assert_eq!(batches.len(), 1);
3167        let struct_array = batches[0]
3168            .column(0)
3169            .as_any()
3170            .downcast_ref::<StructArray>()
3171            .expect("Expected StructArray");
3172        let ts_array = struct_array
3173            .column(0)
3174            .as_any()
3175            .downcast_ref::<TimestampMicrosecondArray>()
3176            .expect("Expected TimestampMicrosecondArray inside struct");
3177
3178        assert_eq!(
3179            ts_array.value(0),
3180            expected_micros,
3181            "INT96 in struct: got {}, expected {expected_micros}",
3182            ts_array.value(0)
3183        );
3184    }
3185
3186    #[tokio::test]
3187    async fn test_read_int96_timestamps_in_list() {
3188        use arrow_array::{ListArray, TimestampMicrosecondArray};
3189        use parquet::basic::{Repetition, Type as PhysicalType};
3190        use parquet::data_type::Int96Type;
3191        use parquet::file::writer::SerializedFileWriter;
3192        use parquet::schema::types::Type as SchemaType;
3193
3194        let tmp_dir = TempDir::new().unwrap();
3195        let table_location = tmp_dir.path().to_str().unwrap().to_string();
3196        let file_path = format!("{table_location}/list_int96.parquet");
3197
3198        // 3-level LIST encoding:
3199        //   optional group timestamps (LIST) {
3200        //     repeated group list {
3201        //       optional int96 element;
3202        //     }
3203        //   }
3204        let element_type = SchemaType::primitive_type_builder("element", PhysicalType::INT96)
3205            .with_repetition(Repetition::OPTIONAL)
3206            .with_id(Some(2))
3207            .build()
3208            .unwrap();
3209
3210        let list_group = SchemaType::group_type_builder("list")
3211            .with_repetition(Repetition::REPEATED)
3212            .with_fields(vec![Arc::new(element_type)])
3213            .build()
3214            .unwrap();
3215
3216        let list_type = SchemaType::group_type_builder("timestamps")
3217            .with_repetition(Repetition::OPTIONAL)
3218            .with_id(Some(1))
3219            .with_logical_type(Some(parquet::basic::LogicalType::List))
3220            .with_fields(vec![Arc::new(list_group)])
3221            .build()
3222            .unwrap();
3223
3224        let parquet_schema = SchemaType::group_type_builder("schema")
3225            .with_fields(vec![Arc::new(list_type)])
3226            .build()
3227            .unwrap();
3228
3229        let (int96_val, expected_micros) = make_int96_test_value();
3230
3231        let file = File::create(&file_path).unwrap();
3232        let mut writer =
3233            SerializedFileWriter::new(file, Arc::new(parquet_schema), Default::default()).unwrap();
3234
3235        // Write a single row with a list containing one INT96 element.
3236        // def=3: list present (1) + repeated group (2) + element present (3)
3237        // rep=0: start of a new list
3238        let mut row_group = writer.next_row_group().unwrap();
3239        {
3240            let mut col = row_group.next_column().unwrap().unwrap();
3241            col.typed::<Int96Type>()
3242                .write_batch(&[int96_val], Some(&[3]), Some(&[0]))
3243                .unwrap();
3244            col.close().unwrap();
3245        }
3246        row_group.close().unwrap();
3247        writer.close().unwrap();
3248
3249        let iceberg_schema = Arc::new(
3250            Schema::builder()
3251                .with_schema_id(1)
3252                .with_fields(vec![
3253                    NestedField::optional(
3254                        1,
3255                        "timestamps",
3256                        Type::List(crate::spec::ListType {
3257                            element_field: NestedField::optional(
3258                                2,
3259                                "element",
3260                                Type::Primitive(PrimitiveType::Timestamp),
3261                            )
3262                            .into(),
3263                        }),
3264                    )
3265                    .into(),
3266                ])
3267                .build()
3268                .unwrap(),
3269        );
3270
3271        let batches = read_int96_batches(&file_path, iceberg_schema, vec![1]).await;
3272
3273        assert_eq!(batches.len(), 1);
3274        let list_array = batches[0]
3275            .column(0)
3276            .as_any()
3277            .downcast_ref::<ListArray>()
3278            .expect("Expected ListArray");
3279        let ts_array = list_array
3280            .values()
3281            .as_any()
3282            .downcast_ref::<TimestampMicrosecondArray>()
3283            .expect("Expected TimestampMicrosecondArray inside list");
3284
3285        assert_eq!(
3286            ts_array.value(0),
3287            expected_micros,
3288            "INT96 in list: got {}, expected {expected_micros}",
3289            ts_array.value(0)
3290        );
3291    }
3292
3293    #[tokio::test]
3294    async fn test_read_int96_timestamps_in_map() {
3295        use arrow_array::{MapArray, TimestampMicrosecondArray};
3296        use parquet::basic::{Repetition, Type as PhysicalType};
3297        use parquet::data_type::{ByteArrayType, Int96Type};
3298        use parquet::file::writer::SerializedFileWriter;
3299        use parquet::schema::types::Type as SchemaType;
3300
3301        let tmp_dir = TempDir::new().unwrap();
3302        let table_location = tmp_dir.path().to_str().unwrap().to_string();
3303        let file_path = format!("{table_location}/map_int96.parquet");
3304
3305        // MAP encoding:
3306        //   optional group ts_map (MAP) {
3307        //     repeated group key_value {
3308        //       required binary key (UTF8);
3309        //       optional int96 value;
3310        //     }
3311        //   }
3312        let key_type = SchemaType::primitive_type_builder("key", PhysicalType::BYTE_ARRAY)
3313            .with_repetition(Repetition::REQUIRED)
3314            .with_logical_type(Some(parquet::basic::LogicalType::String))
3315            .with_id(Some(2))
3316            .build()
3317            .unwrap();
3318
3319        let value_type = SchemaType::primitive_type_builder("value", PhysicalType::INT96)
3320            .with_repetition(Repetition::OPTIONAL)
3321            .with_id(Some(3))
3322            .build()
3323            .unwrap();
3324
3325        let key_value_group = SchemaType::group_type_builder("key_value")
3326            .with_repetition(Repetition::REPEATED)
3327            .with_fields(vec![Arc::new(key_type), Arc::new(value_type)])
3328            .build()
3329            .unwrap();
3330
3331        let map_type = SchemaType::group_type_builder("ts_map")
3332            .with_repetition(Repetition::OPTIONAL)
3333            .with_id(Some(1))
3334            .with_logical_type(Some(parquet::basic::LogicalType::Map))
3335            .with_fields(vec![Arc::new(key_value_group)])
3336            .build()
3337            .unwrap();
3338
3339        let parquet_schema = SchemaType::group_type_builder("schema")
3340            .with_fields(vec![Arc::new(map_type)])
3341            .build()
3342            .unwrap();
3343
3344        let (int96_val, expected_micros) = make_int96_test_value();
3345
3346        let file = File::create(&file_path).unwrap();
3347        let mut writer =
3348            SerializedFileWriter::new(file, Arc::new(parquet_schema), Default::default()).unwrap();
3349
3350        // Write a single row with a map containing one key-value pair.
3351        // rep=0 for both columns: start of a new map.
3352        // key def=2: map present (1) + key_value entry present (2), key is REQUIRED.
3353        // value def=3: map present (1) + key_value entry present (2) + value present (3).
3354        let mut row_group = writer.next_row_group().unwrap();
3355        {
3356            let mut col = row_group.next_column().unwrap().unwrap();
3357            col.typed::<ByteArrayType>()
3358                .write_batch(
3359                    &[parquet::data_type::ByteArray::from("event_time")],
3360                    Some(&[2]),
3361                    Some(&[0]),
3362                )
3363                .unwrap();
3364            col.close().unwrap();
3365        }
3366        {
3367            let mut col = row_group.next_column().unwrap().unwrap();
3368            col.typed::<Int96Type>()
3369                .write_batch(&[int96_val], Some(&[3]), Some(&[0]))
3370                .unwrap();
3371            col.close().unwrap();
3372        }
3373        row_group.close().unwrap();
3374        writer.close().unwrap();
3375
3376        let iceberg_schema = Arc::new(
3377            Schema::builder()
3378                .with_schema_id(1)
3379                .with_fields(vec![
3380                    NestedField::optional(
3381                        1,
3382                        "ts_map",
3383                        Type::Map(crate::spec::MapType {
3384                            key_field: NestedField::required(
3385                                2,
3386                                "key",
3387                                Type::Primitive(PrimitiveType::String),
3388                            )
3389                            .into(),
3390                            value_field: NestedField::optional(
3391                                3,
3392                                "value",
3393                                Type::Primitive(PrimitiveType::Timestamp),
3394                            )
3395                            .into(),
3396                        }),
3397                    )
3398                    .into(),
3399                ])
3400                .build()
3401                .unwrap(),
3402        );
3403
3404        let batches = read_int96_batches(&file_path, iceberg_schema, vec![1]).await;
3405
3406        assert_eq!(batches.len(), 1);
3407        let map_array = batches[0]
3408            .column(0)
3409            .as_any()
3410            .downcast_ref::<MapArray>()
3411            .expect("Expected MapArray");
3412        let ts_array = map_array
3413            .values()
3414            .as_any()
3415            .downcast_ref::<TimestampMicrosecondArray>()
3416            .expect("Expected TimestampMicrosecondArray as map values");
3417
3418        assert_eq!(
3419            ts_array.value(0),
3420            expected_micros,
3421            "INT96 in map: got {}, expected {expected_micros}",
3422            ts_array.value(0)
3423        );
3424    }
3425
3426    /// Writes `id` (Int32) plus a wide string column (field id 2) whose bytes dominate
3427    /// the file, so that reading it is visible in `bytes_read`.
3428    ///
3429    /// `extra_fields`/`extra_columns` (e.g. a physical metadata leaf) are appended after
3430    /// the `id` and wide columns, mirroring `write_plain_parquet`'s shape.
3431    fn write_parquet_with_wide_column(
3432        dir: &str,
3433        name: &str,
3434        extra_fields: Vec<Field>,
3435        extra_columns: Vec<ArrayRef>,
3436    ) -> String {
3437        let wide_field =
3438            Field::new("wide", DataType::Utf8, false).with_metadata(HashMap::from([(
3439                PARQUET_FIELD_ID_META_KEY.to_string(),
3440                "2".to_string(),
3441            )]));
3442        // Varied bytes so the column chunk does not compress away under SNAPPY, keeping
3443        // the `bytes_read` difference between projecting it and not unambiguous.
3444        let wide_values: Vec<String> = (0..3)
3445            .map(|i| {
3446                (0..2048)
3447                    .map(|j| ((i * 2048 + j) % 251) as u8 as char)
3448                    .collect()
3449            })
3450            .collect();
3451
3452        let mut fields = vec![wide_field];
3453        fields.extend(extra_fields);
3454        let mut columns: Vec<ArrayRef> = vec![Arc::new(StringArray::from(wide_values))];
3455        columns.extend(extra_columns);
3456        write_plain_parquet(dir, name, fields, columns)
3457    }
3458
3459    /// Schema with `id` (field 1, Int) and `wide` (field 2, String), matching
3460    /// `write_parquet_with_wide_column`.
3461    fn id_and_wide_schema() -> SchemaRef {
3462        Arc::new(
3463            Schema::builder()
3464                .with_schema_id(1)
3465                .with_fields(vec![
3466                    NestedField::required(1, "id", Type::Primitive(PrimitiveType::Int)).into(),
3467                    NestedField::required(2, "wide", Type::Primitive(PrimitiveType::String)).into(),
3468                ])
3469                .build()
3470                .unwrap(),
3471        )
3472    }
3473
3474    /// Builds a scan task over `file_path` projecting `project_field_ids`.
3475    fn metadata_projection_task(
3476        file_path: String,
3477        schema: SchemaRef,
3478        project_field_ids: Vec<i32>,
3479    ) -> FileScanTask {
3480        metadata_projection_task_with_first_row_id(file_path, schema, project_field_ids, None)
3481    }
3482
3483    fn metadata_projection_task_with_first_row_id(
3484        file_path: String,
3485        schema: SchemaRef,
3486        project_field_ids: Vec<i32>,
3487        first_row_id: Option<i64>,
3488    ) -> FileScanTask {
3489        FileScanTask::builder()
3490            .with_file_size_in_bytes(std::fs::metadata(&file_path).unwrap().len())
3491            .with_start(0)
3492            .with_length(0)
3493            .with_data_file_path(file_path)
3494            .with_data_file_format(DataFileFormat::Parquet)
3495            .with_schema(schema)
3496            .with_project_field_ids(project_field_ids)
3497            .with_first_row_id(first_row_id)
3498            .with_case_sensitive(false)
3499            .build()
3500            .unwrap()
3501    }
3502
3503    /// Runs a single-task scan and returns the batches plus the bytes read from storage.
3504    async fn scan_task(task: FileScanTask) -> (Vec<RecordBatch>, u64) {
3505        let reader = ArrowReaderBuilder::new(FileIO::new_with_fs(), Runtime::current()).build();
3506        let tasks = Box::pin(futures::stream::iter(vec![Ok(task)])) as FileScanTaskStream;
3507        let scan = reader.read(tasks).unwrap();
3508        let metrics = scan.metrics().clone();
3509        let batches = scan.stream().try_collect().await.unwrap();
3510        (batches, metrics.bytes_read())
3511    }
3512
3513    #[tokio::test]
3514    async fn test_pos_only_projection_reads_no_data_columns() {
3515        let tmp_dir = TempDir::new().unwrap();
3516        let dir = tmp_dir.path().to_str().unwrap();
3517
3518        let pos_only = metadata_projection_task(
3519            write_parquet_with_wide_column(dir, "pos_only.parquet", vec![], vec![]),
3520            id_and_wide_schema(),
3521            vec![RESERVED_FIELD_ID_POS],
3522        );
3523        let (batches, pos_only_bytes) = scan_task(pos_only).await;
3524
3525        // Only `_pos` is materialized -- no data columns.
3526        assert_eq!(batches[0].num_columns(), 1);
3527        let pos_col = batches[0]
3528            .column_by_name(RESERVED_COL_NAME_POS)
3529            .expect("_pos column should be present")
3530            .as_primitive::<arrow_array::types::Int64Type>();
3531        assert_eq!(pos_col.values(), &[0, 1, 2]);
3532
3533        // A scan of the same-shaped file that also projects the wide data column must read
3534        // materially more, proving the wide column chunk was not fetched above.
3535        let with_data = metadata_projection_task(
3536            write_parquet_with_wide_column(dir, "pos_only_ref.parquet", vec![], vec![]),
3537            id_and_wide_schema(),
3538            vec![2, RESERVED_FIELD_ID_POS],
3539        );
3540        let (_, with_data_bytes) = scan_task(with_data).await;
3541
3542        assert!(
3543            pos_only_bytes < with_data_bytes,
3544            "_pos-only scan should read fewer bytes than a scan of the wide column: \
3545             {pos_only_bytes} vs {with_data_bytes}"
3546        );
3547    }
3548
3549    #[tokio::test]
3550    async fn test_empty_projection_reads_no_data_columns() {
3551        let tmp_dir = TempDir::new().unwrap();
3552        let dir = tmp_dir.path().to_str().unwrap();
3553
3554        let empty = metadata_projection_task(
3555            write_parquet_with_wide_column(dir, "empty.parquet", vec![], vec![]),
3556            id_and_wide_schema(),
3557            vec![],
3558        );
3559        let (batches, empty_bytes) = scan_task(empty).await;
3560
3561        // No data columns come out, but the fixture's 3 rows are still counted.
3562        assert!(batches.iter().all(|b| b.num_columns() == 0));
3563        assert_eq!(batches.iter().map(|b| b.num_rows()).sum::<usize>(), 3);
3564
3565        let with_data = metadata_projection_task(
3566            write_parquet_with_wide_column(dir, "empty_ref.parquet", vec![], vec![]),
3567            id_and_wide_schema(),
3568            vec![2],
3569        );
3570        let (_, with_data_bytes) = scan_task(with_data).await;
3571
3572        assert!(
3573            empty_bytes < with_data_bytes,
3574            "empty projection should read fewer bytes than the wide column: \
3575             {empty_bytes} vs {with_data_bytes}"
3576        );
3577    }
3578
3579    #[tokio::test]
3580    async fn test_empty_projection_counts_rows_after_positional_delete() {
3581        use crate::spec::DataContentType;
3582
3583        let tmp_dir = TempDir::new().unwrap();
3584        let dir = tmp_dir.path().to_str().unwrap();
3585        let data_path = write_plain_parquet(dir, "empty_posdel_data.parquet", vec![], vec![]);
3586        let del_path = write_positional_delete(dir, "empty_posdel.parquet", &data_path, &[1]);
3587        let delete = FileScanTaskDeleteFile::builder()
3588            .with_file_path(del_path.clone())
3589            .with_file_size_in_bytes(std::fs::metadata(&del_path).unwrap().len())
3590            .with_file_type(DataContentType::PositionDeletes)
3591            .with_file_format(DataFileFormat::Parquet)
3592            .with_partition_spec_id(0)
3593            .build()
3594            .unwrap();
3595        let task = FileScanTask::builder()
3596            .with_file_size_in_bytes(std::fs::metadata(&data_path).unwrap().len())
3597            .with_start(0)
3598            .with_length(0)
3599            .with_data_file_path(data_path)
3600            .with_data_file_format(DataFileFormat::Parquet)
3601            .with_schema(id_and_wide_schema())
3602            .with_project_field_ids(vec![])
3603            .with_deletes(vec![delete])
3604            .with_case_sensitive(false)
3605            .build()
3606            .unwrap();
3607
3608        let (batches, _) = scan_task(task).await;
3609
3610        assert_eq!(batches.iter().map(|b| b.num_rows()).sum::<usize>(), 2);
3611    }
3612
3613    #[tokio::test]
3614    async fn test_empty_projection_counts_rows_after_predicate() {
3615        use crate::expr::{Bind, Reference};
3616        use crate::spec::Datum;
3617
3618        let tmp_dir = TempDir::new().unwrap();
3619        let dir = tmp_dir.path().to_str().unwrap();
3620        let file_path = write_plain_parquet(dir, "empty_predicate.parquet", vec![], vec![]);
3621        let schema = id_and_wide_schema();
3622        // `id` is filtered on but not projected, so the row filter reads it on its own mask.
3623        let bound = Reference::new("id")
3624            .not_equal_to(Datum::int(2))
3625            .bind(Arc::clone(&schema), false)
3626            .unwrap();
3627        let task = FileScanTask::builder()
3628            .with_file_size_in_bytes(std::fs::metadata(&file_path).unwrap().len())
3629            .with_start(0)
3630            .with_length(0)
3631            .with_data_file_path(file_path)
3632            .with_data_file_format(DataFileFormat::Parquet)
3633            .with_schema(schema)
3634            .with_project_field_ids(vec![])
3635            .with_predicate(Some(bound))
3636            .with_case_sensitive(false)
3637            .build()
3638            .unwrap();
3639
3640        let (batches, _) = scan_task(task).await;
3641
3642        assert_eq!(batches.iter().map(|b| b.num_rows()).sum::<usize>(), 2);
3643    }
3644
3645    #[tokio::test]
3646    async fn test_pos_only_projection_keeps_absolute_pos_under_predicate() {
3647        use crate::expr::{Bind, Reference};
3648        use crate::spec::Datum;
3649
3650        let tmp_dir = TempDir::new().unwrap();
3651        let dir = tmp_dir.path().to_str().unwrap();
3652        // id = [1, 2, 3]; drop the middle physical row via a predicate + row selection.
3653        let file_path = write_plain_parquet(dir, "pos_only_predicate.parquet", vec![], vec![]);
3654
3655        let schema = Arc::new(
3656            Schema::builder()
3657                .with_schema_id(1)
3658                .with_fields(vec![
3659                    NestedField::required(1, "id", Type::Primitive(PrimitiveType::Int)).into(),
3660                ])
3661                .build()
3662                .unwrap(),
3663        );
3664        let bound = Reference::new("id")
3665            .not_equal_to(Datum::int(2))
3666            .bind(Arc::clone(&schema), false)
3667            .unwrap();
3668        let task = FileScanTask::builder()
3669            .with_file_size_in_bytes(std::fs::metadata(&file_path).unwrap().len())
3670            .with_start(0)
3671            .with_length(0)
3672            .with_data_file_path(file_path)
3673            .with_data_file_format(DataFileFormat::Parquet)
3674            .with_schema(schema)
3675            .with_project_field_ids(vec![RESERVED_FIELD_ID_POS])
3676            .with_predicate(Some(bound))
3677            .with_case_sensitive(false)
3678            .build()
3679            .unwrap();
3680
3681        // Row selection must be enabled for the predicate to filter rows. The row filter
3682        // reads `id` for its own evaluation even though `id` is not projected; the surviving
3683        // rows must keep their ABSOLUTE positions (0 and 2), not renumbered (0 and 1).
3684        let reader = ArrowReaderBuilder::new(FileIO::new_with_fs(), Runtime::current())
3685            .with_options(
3686                ReaderOptions::builder()
3687                    .with_row_selection_enabled(true)
3688                    .build(),
3689            )
3690            .build();
3691        let tasks = Box::pin(futures::stream::iter(vec![Ok(task)])) as FileScanTaskStream;
3692        let batches: Vec<RecordBatch> = reader
3693            .read(tasks)
3694            .unwrap()
3695            .stream()
3696            .try_collect()
3697            .await
3698            .unwrap();
3699
3700        let pos: Vec<i64> = batches
3701            .iter()
3702            .flat_map(|b| {
3703                b.column_by_name(RESERVED_COL_NAME_POS)
3704                    .expect("_pos column should be present")
3705                    .as_primitive::<arrow_array::types::Int64Type>()
3706                    .values()
3707                    .to_vec()
3708            })
3709            .collect();
3710        assert_eq!(pos, vec![0, 2]);
3711    }
3712
3713    #[tokio::test]
3714    async fn test_pos_and_file_projection() {
3715        use crate::metadata_columns::RESERVED_COL_NAME_FILE;
3716
3717        let tmp_dir = TempDir::new().unwrap();
3718        let dir = tmp_dir.path().to_str().unwrap();
3719        // The motivating row-lineage shape: a synthesized position column (mask -> none)
3720        // alongside a materialized per-file constant.
3721        let file_path = write_parquet_with_wide_column(dir, "pos_and_file.parquet", vec![], vec![]);
3722        let task = metadata_projection_task(file_path.clone(), id_and_wide_schema(), vec![
3723            RESERVED_FIELD_ID_POS,
3724            RESERVED_FIELD_ID_FILE,
3725        ]);
3726        let (batches, _) = scan_task(task).await;
3727
3728        // Both metadata columns materialize; no data column is read.
3729        assert_eq!(batches[0].num_columns(), 2);
3730        let pos_col = batches[0]
3731            .column_by_name(RESERVED_COL_NAME_POS)
3732            .expect("_pos column should be present")
3733            .as_primitive::<arrow_array::types::Int64Type>();
3734        assert_eq!(pos_col.values(), &[0, 1, 2]);
3735        let file_col = batches[0]
3736            .column_by_name(RESERVED_COL_NAME_FILE)
3737            .expect("_file column should be present");
3738        let file_col = cast(file_col, &DataType::Utf8).unwrap();
3739        let file_col = file_col.as_any().downcast_ref::<StringArray>().unwrap();
3740        assert_eq!(file_col.value(0), file_path);
3741    }
3742
3743    #[tokio::test]
3744    async fn test_pos_and_physical_seq_projection_reads_only_the_leaf() {
3745        use crate::metadata_columns::{
3746            RESERVED_COL_NAME_LAST_UPDATED_SEQUENCE_NUMBER,
3747            RESERVED_FIELD_ID_LAST_UPDATED_SEQUENCE_NUMBER,
3748        };
3749
3750        // A v3 rewrite that carried rows forward stores `_last_updated_sequence_number`
3751        // per-row. Projecting only `_pos` + the sequence column must read just that one
3752        // physical leaf, not every data column.
3753        let tmp_dir = TempDir::new().unwrap();
3754        let dir = tmp_dir.path().to_str().unwrap();
3755
3756        // File: id (1), wide data column (2), physical _last_updated_sequence_number.
3757        let write = |name: &str| {
3758            write_parquet_with_wide_column(
3759                dir,
3760                name,
3761                vec![physical_last_updated_seq_field()],
3762                vec![Arc::new(Int64Array::from(vec![Some(5), None, Some(8)])) as ArrayRef],
3763            )
3764        };
3765
3766        let seq_task = |path: String, ids: Vec<i32>| {
3767            FileScanTask::builder()
3768                .with_file_size_in_bytes(std::fs::metadata(&path).unwrap().len())
3769                .with_start(0)
3770                .with_length(0)
3771                .with_data_file_path(path)
3772                .with_data_file_format(DataFileFormat::Parquet)
3773                .with_schema(id_and_wide_schema())
3774                .with_project_field_ids(ids)
3775                .with_first_row_id(Some(100))
3776                .with_data_sequence_number(Some(9))
3777                .with_case_sensitive(false)
3778                .build()
3779                .unwrap()
3780        };
3781
3782        let meta_only = seq_task(write("pos_seq.parquet"), vec![
3783            RESERVED_FIELD_ID_POS,
3784            RESERVED_FIELD_ID_LAST_UPDATED_SEQUENCE_NUMBER,
3785        ]);
3786        let (batches, meta_only_bytes) = scan_task(meta_only).await;
3787
3788        // `_pos` and the coalesced sequence column materialize; the wide column does not.
3789        let pos_col = batches[0]
3790            .column_by_name(RESERVED_COL_NAME_POS)
3791            .expect("_pos column should be present")
3792            .as_primitive::<arrow_array::types::Int64Type>();
3793        assert_eq!(pos_col.values(), &[0, 1, 2]);
3794        let seq_col = batches[0]
3795            .column_by_name(RESERVED_COL_NAME_LAST_UPDATED_SEQUENCE_NUMBER)
3796            .expect("_last_updated_sequence_number column should be present");
3797        let seq_col = cast(seq_col, &DataType::Int64).unwrap();
3798        let seq_col = seq_col.as_any().downcast_ref::<Int64Array>().unwrap();
3799        // Per-row stored value where non-null, else the data sequence number (9).
3800        assert_eq!(seq_col.value(0), 5);
3801        assert_eq!(seq_col.value(1), 9);
3802        assert_eq!(seq_col.value(2), 8);
3803        assert!(batches[0].column_by_name("wide").is_none());
3804
3805        // A scan that also projects the wide data column must read materially more,
3806        // proving the metadata-only scan pruned to just the sequence leaf.
3807        let with_data = seq_task(write("pos_seq_ref.parquet"), vec![
3808            2,
3809            RESERVED_FIELD_ID_POS,
3810            RESERVED_FIELD_ID_LAST_UPDATED_SEQUENCE_NUMBER,
3811        ]);
3812        let (_, with_data_bytes) = scan_task(with_data).await;
3813
3814        assert!(
3815            meta_only_bytes < with_data_bytes,
3816            "_pos + physical sequence scan should read fewer bytes than one that also \
3817             reads the wide column: {meta_only_bytes} vs {with_data_bytes}"
3818        );
3819    }
3820
3821    #[tokio::test]
3822    async fn test_seq_only_projection_reads_only_the_leaf() {
3823        use crate::metadata_columns::{
3824            RESERVED_COL_NAME_LAST_UPDATED_SEQUENCE_NUMBER,
3825            RESERVED_FIELD_ID_LAST_UPDATED_SEQUENCE_NUMBER,
3826        };
3827
3828        // `_last_updated_sequence_number` alone (no `_pos`, no data column). This is a
3829        // metadata-only projection, so RowNumber drives the `none()` downgrade and the
3830        // physical leaf is unioned back in, pruning the read to just that leaf.
3831        let tmp_dir = TempDir::new().unwrap();
3832        let dir = tmp_dir.path().to_str().unwrap();
3833        let write = |name: &str| {
3834            write_parquet_with_wide_column(
3835                dir,
3836                name,
3837                vec![physical_last_updated_seq_field()],
3838                vec![Arc::new(Int64Array::from(vec![Some(5), None, Some(8)])) as ArrayRef],
3839            )
3840        };
3841        let seq_task = |path: String, ids: Vec<i32>| {
3842            FileScanTask::builder()
3843                .with_file_size_in_bytes(std::fs::metadata(&path).unwrap().len())
3844                .with_start(0)
3845                .with_length(0)
3846                .with_data_file_path(path)
3847                .with_data_file_format(DataFileFormat::Parquet)
3848                .with_schema(id_and_wide_schema())
3849                .with_project_field_ids(ids)
3850                .with_first_row_id(Some(100))
3851                .with_data_sequence_number(Some(9))
3852                .with_case_sensitive(false)
3853                .build()
3854                .unwrap()
3855        };
3856
3857        let meta_only = seq_task(write("seq_only.parquet"), vec![
3858            RESERVED_FIELD_ID_LAST_UPDATED_SEQUENCE_NUMBER,
3859        ]);
3860        let (batches, meta_only_bytes) = scan_task(meta_only).await;
3861
3862        let seq_col = batches[0]
3863            .column_by_name(RESERVED_COL_NAME_LAST_UPDATED_SEQUENCE_NUMBER)
3864            .expect("_last_updated_sequence_number column should be present");
3865        let seq_col = cast(seq_col, &DataType::Int64).unwrap();
3866        let seq_col = seq_col.as_any().downcast_ref::<Int64Array>().unwrap();
3867        assert_eq!(seq_col.value(0), 5);
3868        assert_eq!(seq_col.value(1), 9);
3869        assert_eq!(seq_col.value(2), 8);
3870        let total_rows: usize = batches.iter().map(|b| b.num_rows()).sum();
3871        assert_eq!(total_rows, 3);
3872        assert!(batches[0].column_by_name("wide").is_none());
3873
3874        let with_data = seq_task(write("seq_only_ref.parquet"), vec![
3875            2,
3876            RESERVED_FIELD_ID_LAST_UPDATED_SEQUENCE_NUMBER,
3877        ]);
3878        let (_, with_data_bytes) = scan_task(with_data).await;
3879
3880        assert!(
3881            meta_only_bytes < with_data_bytes,
3882            "seq-only scan should read fewer bytes than one that also reads the wide \
3883             column: {meta_only_bytes} vs {with_data_bytes}"
3884        );
3885    }
3886
3887    #[tokio::test]
3888    async fn test_seq_only_null_first_row_id_reads_no_data_columns() {
3889        use crate::metadata_columns::{
3890            RESERVED_COL_NAME_LAST_UPDATED_SEQUENCE_NUMBER,
3891            RESERVED_FIELD_ID_LAST_UPDATED_SEQUENCE_NUMBER,
3892        };
3893
3894        // Seq-only projection with a null first_row_id: the column is nulled and the
3895        // physical leaf is NOT read (the gated `coalesce_last_updated_seq_leaf` is None).
3896        // This is a metadata-only projection, so RowNumber supplies the row count and the
3897        // data columns are pruned; the row count is still 3 and the values all null.
3898        let tmp_dir = TempDir::new().unwrap();
3899        let dir = tmp_dir.path().to_str().unwrap();
3900        let seq_task = |name: &str, ids: Vec<i32>| {
3901            let file_path = write_parquet_with_wide_column(
3902                dir,
3903                name,
3904                vec![physical_last_updated_seq_field()],
3905                vec![Arc::new(Int64Array::from(vec![Some(5), Some(6), Some(7)])) as ArrayRef],
3906            );
3907            FileScanTask::builder()
3908                .with_file_size_in_bytes(std::fs::metadata(&file_path).unwrap().len())
3909                .with_start(0)
3910                .with_length(0)
3911                .with_data_file_path(file_path)
3912                .with_data_file_format(DataFileFormat::Parquet)
3913                .with_schema(id_and_wide_schema())
3914                .with_project_field_ids(ids)
3915                .with_first_row_id(None)
3916                .with_data_sequence_number(Some(9))
3917                .with_case_sensitive(false)
3918                .build()
3919                .unwrap()
3920        };
3921
3922        let meta_only = seq_task("seq_only_null_first.parquet", vec![
3923            RESERVED_FIELD_ID_LAST_UPDATED_SEQUENCE_NUMBER,
3924        ]);
3925        let (batches, meta_only_bytes) = scan_task(meta_only).await;
3926
3927        assert_eq!(batches[0].num_columns(), 1);
3928        let total_rows: usize = batches.iter().map(|b| b.num_rows()).sum();
3929        assert_eq!(total_rows, 3);
3930        let seq_col = batches[0]
3931            .column_by_name(RESERVED_COL_NAME_LAST_UPDATED_SEQUENCE_NUMBER)
3932            .expect("_last_updated_sequence_number column should be present");
3933        let seq_col = cast(seq_col, &DataType::Int64).unwrap();
3934        let seq_col = seq_col.as_any().downcast_ref::<Int64Array>().unwrap();
3935        assert!((0..3).all(|i| seq_col.is_null(i)));
3936
3937        let with_data = seq_task("seq_only_null_first_ref.parquet", vec![
3938            2,
3939            RESERVED_FIELD_ID_LAST_UPDATED_SEQUENCE_NUMBER,
3940        ]);
3941        let (_, with_data_bytes) = scan_task(with_data).await;
3942
3943        assert!(
3944            meta_only_bytes < with_data_bytes,
3945            "seq-only scan with a null first_row_id should read fewer bytes than a scan of \
3946             the wide column: {meta_only_bytes} vs {with_data_bytes}"
3947        );
3948    }
3949
3950    #[tokio::test]
3951    async fn test_file_only_reads_no_data_columns() {
3952        use crate::metadata_columns::RESERVED_COL_NAME_FILE;
3953
3954        // `SELECT _file` materializes a per-file string constant with no physical column to
3955        // read. The metadata-only RowNumber counter sizes it, so the scan prunes the data
3956        // columns rather than reading them all just to recover the row count.
3957        let tmp_dir = TempDir::new().unwrap();
3958        let dir = tmp_dir.path().to_str().unwrap();
3959
3960        let file_path = write_parquet_with_wide_column(dir, "file_only.parquet", vec![], vec![]);
3961        let meta_only = metadata_projection_task(file_path.clone(), id_and_wide_schema(), vec![
3962            RESERVED_FIELD_ID_FILE,
3963        ]);
3964        let (batches, meta_only_bytes) = scan_task(meta_only).await;
3965
3966        assert_eq!(batches[0].num_columns(), 1);
3967        let total_rows: usize = batches.iter().map(|b| b.num_rows()).sum();
3968        assert_eq!(total_rows, 3);
3969        let file_col = batches[0]
3970            .column_by_name(RESERVED_COL_NAME_FILE)
3971            .expect("_file column should be present");
3972        let file_col = cast(file_col, &DataType::Utf8).unwrap();
3973        let file_col = file_col.as_any().downcast_ref::<StringArray>().unwrap();
3974        assert_eq!(file_col.value(0), file_path);
3975
3976        let with_data = metadata_projection_task(
3977            write_parquet_with_wide_column(dir, "file_only_ref.parquet", vec![], vec![]),
3978            id_and_wide_schema(),
3979            vec![2, RESERVED_FIELD_ID_FILE],
3980        );
3981        let (_, with_data_bytes) = scan_task(with_data).await;
3982
3983        assert!(
3984            meta_only_bytes < with_data_bytes,
3985            "_file-only scan should read fewer bytes than a scan of the wide column: \
3986             {meta_only_bytes} vs {with_data_bytes}"
3987        );
3988    }
3989
3990    #[tokio::test]
3991    async fn test_spec_id_only_reads_no_data_columns() {
3992        use crate::metadata_columns::{RESERVED_COL_NAME_SPEC_ID, RESERVED_FIELD_ID_SPEC_ID};
3993        use crate::spec::{Literal, PartitionSpec, Struct, Transform};
3994
3995        // `SELECT _spec_id` materializes an int constant from the task's partition spec id,
3996        // with no physical column to read. The RowNumber counter sizes it, so the scan
3997        // prunes the data columns rather than reading them all to recover the row count.
3998        let tmp_dir = TempDir::new().unwrap();
3999        let dir = tmp_dir.path().to_str().unwrap();
4000        let schema = id_and_wide_schema();
4001        let spec = Arc::new(
4002            PartitionSpec::builder(schema.clone())
4003                .with_spec_id(7)
4004                .add_partition_field("id", "id", Transform::Identity)
4005                .unwrap()
4006                .build()
4007                .unwrap(),
4008        );
4009
4010        let spec_id_task = |file_path: String, project_field_ids: Vec<i32>| {
4011            FileScanTask::builder()
4012                .with_file_size_in_bytes(std::fs::metadata(&file_path).unwrap().len())
4013                .with_start(0)
4014                .with_length(0)
4015                .with_data_file_path(file_path)
4016                .with_data_file_format(DataFileFormat::Parquet)
4017                .with_schema(schema.clone())
4018                .with_project_field_ids(project_field_ids)
4019                .with_partition(Some(Struct::from_iter([Some(Literal::int(42))])))
4020                .with_partition_spec(Some(spec.clone()))
4021                .with_case_sensitive(false)
4022                .build()
4023                .unwrap()
4024        };
4025
4026        let meta_only = spec_id_task(
4027            write_parquet_with_wide_column(dir, "spec_id_only.parquet", vec![], vec![]),
4028            vec![RESERVED_FIELD_ID_SPEC_ID],
4029        );
4030        let (batches, meta_only_bytes) = scan_task(meta_only).await;
4031
4032        assert_eq!(batches[0].num_columns(), 1);
4033        let total_rows: usize = batches.iter().map(|b| b.num_rows()).sum();
4034        assert_eq!(total_rows, 3);
4035        let spec_id_col = batches[0]
4036            .column_by_name(RESERVED_COL_NAME_SPEC_ID)
4037            .expect("_spec_id column should be present");
4038        let spec_id_col = cast(spec_id_col, &DataType::Int32).unwrap();
4039        let spec_id_col = spec_id_col.as_any().downcast_ref::<Int32Array>().unwrap();
4040        assert_eq!(spec_id_col.values(), &[7, 7, 7]);
4041
4042        let with_data = spec_id_task(
4043            write_parquet_with_wide_column(dir, "spec_id_only_ref.parquet", vec![], vec![]),
4044            vec![2, RESERVED_FIELD_ID_SPEC_ID],
4045        );
4046        let (_, with_data_bytes) = scan_task(with_data).await;
4047
4048        assert!(
4049            meta_only_bytes < with_data_bytes,
4050            "_spec_id-only scan should read fewer bytes than a scan of the wide column: \
4051             {meta_only_bytes} vs {with_data_bytes}"
4052        );
4053    }
4054
4055    #[tokio::test]
4056    async fn test_empty_projection_preserves_row_count() {
4057        let tmp_dir = TempDir::new().unwrap();
4058        let dir = tmp_dir.path().to_str().unwrap();
4059        let file_path = write_plain_parquet(dir, "empty_projection.parquet", vec![], vec![]);
4060        let schema = Arc::new(
4061            Schema::builder()
4062                .with_schema_id(1)
4063                .with_fields(vec![
4064                    NestedField::required(1, "id", Type::Primitive(PrimitiveType::Int)).into(),
4065                ])
4066                .build()
4067                .unwrap(),
4068        );
4069        let task = metadata_projection_task(file_path, schema, vec![]);
4070        let (batches, _) = scan_task(task).await;
4071
4072        // A bare COUNT(*)-style empty projection must still report the row count.
4073        let total_rows: usize = batches.iter().map(|b| b.num_rows()).sum();
4074        assert_eq!(total_rows, 3);
4075    }
4076
4077    /// The reads a data file needs must finish even when nothing polls the scan,
4078    /// and a dropped scan must stop reading (#3276).
4079    mod open_progress {
4080        use std::ops::Range;
4081        use std::sync::Mutex;
4082
4083        use async_trait::async_trait;
4084        use bytes::Bytes;
4085        use futures::StreamExt;
4086        use futures::stream::BoxStream;
4087        use parquet::arrow::arrow_reader::ParquetRecordBatchReaderBuilder;
4088        use serde::{Deserialize, Serialize};
4089
4090        use super::*;
4091        use crate::Result;
4092        use crate::arrow::ParquetReadOptions;
4093        use crate::arrow::scan_metrics::ScanMetrics;
4094        use crate::expr::{Bind, Predicate, Reference};
4095        use crate::io::{
4096            FileIOBuilder, FileMetadata, FileRead, FileWrite, InputFile, MemoryStorage, OutputFile,
4097            Storage, StorageConfig, StorageFactory,
4098        };
4099        use crate::scan::ArrowRecordBatchStream;
4100        use crate::spec::Datum;
4101
4102        const CONCURRENCY: usize = 2;
4103        /// Smaller than a footer, so one open takes more than one request.
4104        const METADATA_SIZE_HINT: usize = 64;
4105        /// Bytes one open reads: the hint, then the footer and its indexes.
4106        /// The footer stores the writer's `created_by` version, so a `parquet`
4107        /// upgrade can change this.
4108        const OPEN_BYTES: u64 = 547;
4109        /// Scheduler turns, far more than a request needs. A bound only because
4110        /// `#[tokio::test]` runs the test and the spawned reads on one thread.
4111        const YIELDS: usize = 100;
4112
4113        #[derive(Debug, Default)]
4114        struct Reads {
4115            last_start: Option<u64>,
4116            started: usize,
4117            finished: usize,
4118        }
4119
4120        /// What the storage was asked for, and how much of it came back.
4121        /// `ScanMetrics` counts a read when the request goes out, so it cannot
4122        /// tell a read that hangs from one that answers.
4123        #[derive(Debug, Clone, Default)]
4124        struct ReadLog(Arc<Mutex<Reads>>);
4125
4126        impl ReadLog {
4127            /// Where the newest read began, which says which read it is.
4128            fn last_start(&self) -> Option<u64> {
4129                self.0.lock().unwrap().last_start
4130            }
4131
4132            fn in_flight(&self) -> usize {
4133                let reads = self.0.lock().unwrap();
4134                reads.started - reads.finished
4135            }
4136        }
4137
4138        /// In-memory files whose reads take a scheduler turn to answer.
4139        #[derive(Debug, Clone, Default, Serialize, Deserialize)]
4140        struct DeferredStorage {
4141            #[serde(skip)]
4142            files: MemoryStorage,
4143            #[serde(skip)]
4144            log: ReadLog,
4145        }
4146
4147        struct DeferredRead {
4148            contents: Bytes,
4149            log: ReadLog,
4150        }
4151
4152        #[async_trait]
4153        impl FileRead for DeferredRead {
4154            async fn read(&self, range: Range<u64>) -> Result<Bytes> {
4155                {
4156                    let mut reads = self.log.0.lock().unwrap();
4157                    reads.last_start = Some(range.start);
4158                    reads.started += 1;
4159                }
4160                // The yield stands in for a request in flight: it finishes only
4161                // if something still drives the read.
4162                tokio::task::yield_now().await;
4163                self.log.0.lock().unwrap().finished += 1;
4164                Ok(self
4165                    .contents
4166                    .slice(range.start as usize..range.end as usize))
4167            }
4168        }
4169
4170        #[async_trait]
4171        #[typetag::serde]
4172        impl Storage for DeferredStorage {
4173            async fn reader(&self, path: &str) -> Result<Box<dyn FileRead>> {
4174                Ok(Box::new(DeferredRead {
4175                    contents: self.files.read(path).await?,
4176                    log: self.log.clone(),
4177                }))
4178            }
4179
4180            fn new_input(&self, path: &str) -> Result<InputFile> {
4181                Ok(InputFile::new(Arc::new(self.clone()), path.to_string()))
4182            }
4183
4184            async fn exists(&self, path: &str) -> Result<bool> {
4185                self.files.exists(path).await
4186            }
4187
4188            async fn metadata(&self, path: &str) -> Result<FileMetadata> {
4189                self.files.metadata(path).await
4190            }
4191
4192            async fn read(&self, path: &str) -> Result<Bytes> {
4193                self.files.read(path).await
4194            }
4195
4196            async fn write(&self, path: &str, bs: Bytes) -> Result<()> {
4197                self.files.write(path, bs).await
4198            }
4199
4200            async fn writer(&self, path: &str) -> Result<Box<dyn FileWrite>> {
4201                self.files.writer(path).await
4202            }
4203
4204            async fn delete(&self, path: &str) -> Result<()> {
4205                self.files.delete(path).await
4206            }
4207
4208            async fn delete_prefix(&self, path: &str) -> Result<()> {
4209                self.files.delete_prefix(path).await
4210            }
4211
4212            async fn delete_stream(&self, paths: BoxStream<'static, String>) -> Result<()> {
4213                self.files.delete_stream(paths).await
4214            }
4215
4216            fn new_output(&self, path: &str) -> Result<OutputFile> {
4217                Ok(OutputFile::new(Arc::new(self.clone()), path.to_string()))
4218            }
4219        }
4220
4221        #[derive(Debug, Default, Serialize, Deserialize)]
4222        struct DeferredStorageFactory {
4223            #[serde(skip)]
4224            storage: DeferredStorage,
4225        }
4226
4227        #[typetag::serde]
4228        impl StorageFactory for DeferredStorageFactory {
4229            fn build(&self, _config: &StorageConfig) -> Result<Arc<dyn Storage>> {
4230                Ok(Arc::new(self.storage.clone()))
4231            }
4232        }
4233
4234        /// A scan of `files` copies of `contents`, the metrics that count the bytes
4235        /// it has read, and the log of what the storage saw.
4236        async fn deferred_scan_of(
4237            contents: Bytes,
4238            files: usize,
4239            predicate: Option<Predicate>,
4240        ) -> (ScanMetrics, ReadLog, ArrowRecordBatchStream) {
4241            let schema = Arc::new(
4242                Schema::builder()
4243                    .with_schema_id(1)
4244                    .with_fields(vec![
4245                        NestedField::required(1, "id", Type::Primitive(PrimitiveType::Int)).into(),
4246                    ])
4247                    .build()
4248                    .unwrap(),
4249            );
4250            let bound = predicate.map(|p| p.bind(Arc::clone(&schema), false).unwrap());
4251
4252            let storage = DeferredStorage::default();
4253            let log = storage.log.clone();
4254            let mut tasks = Vec::with_capacity(files);
4255            for file_num in 0..files {
4256                let path = format!("memory:/data/file_{file_num}.parquet");
4257                storage.write(&path, contents.clone()).await.unwrap();
4258                tasks.push(
4259                    FileScanTask::builder()
4260                        .with_data_file_path(path)
4261                        .with_file_size_in_bytes(contents.len() as u64)
4262                        .with_start(0)
4263                        .with_length(0)
4264                        .with_data_file_format(DataFileFormat::Parquet)
4265                        .with_schema(Arc::clone(&schema))
4266                        .with_project_field_ids(vec![1])
4267                        .with_predicate(bound.clone())
4268                        .with_case_sensitive(false)
4269                        .build()
4270                        .unwrap(),
4271                );
4272            }
4273
4274            let file_io = FileIOBuilder::new(Arc::new(DeferredStorageFactory { storage })).build();
4275
4276            let reader = ArrowReaderBuilder::new(file_io, Runtime::current())
4277                .with_options(
4278                    ReaderOptions::builder()
4279                        .with_data_file_concurrency_limit(CONCURRENCY)
4280                        .with_bloom_filter_enabled(bound.is_some())
4281                        .with_parquet_options(
4282                            ParquetReadOptions::builder()
4283                                .with_metadata_size_hint(METADATA_SIZE_HINT)
4284                                .build(),
4285                        )
4286                        .build(),
4287                )
4288                .build();
4289            let tasks =
4290                Box::pin(futures::stream::iter(tasks.into_iter().map(Ok))) as FileScanTaskStream;
4291            let result = reader.read(tasks).unwrap();
4292            (result.metrics().clone(), log, result.stream())
4293        }
4294
4295        /// `CONCURRENCY` identical files and no predicate, so the open is the only read.
4296        async fn deferred_scan() -> (ScanMetrics, ArrowRecordBatchStream) {
4297            let tmp_dir = TempDir::new().unwrap();
4298            let on_disk = write_plain_parquet(
4299                tmp_dir.path().to_str().unwrap(),
4300                "data.parquet",
4301                vec![],
4302                vec![],
4303            );
4304            let contents = Bytes::from(std::fs::read(on_disk).unwrap());
4305            let (metrics, _, scan) = deferred_scan_of(contents, CONCURRENCY, None).await;
4306            (metrics, scan)
4307        }
4308
4309        /// Gives the scheduler turns until `done` is true, or `YIELDS` turns pass.
4310        async fn spin(done: impl Fn() -> bool) {
4311            for _ in 0..YIELDS {
4312                if done() {
4313                    return;
4314                }
4315                tokio::task::yield_now().await;
4316            }
4317        }
4318
4319        /// An open must finish its requests even when the scan is never polled again.
4320        #[tokio::test]
4321        async fn test_open_proceeds_without_the_scan_being_polled() {
4322            let (metrics, scan) = deferred_scan().await;
4323            let mut scan = Box::pin(scan);
4324            let every_open = OPEN_BYTES * CONCURRENCY as u64;
4325
4326            // One poll starts an open per concurrency slot. Nothing polls the
4327            // scan after this, so only a spawned open can read any further.
4328            let _ = futures::poll!(scan.next());
4329            spin(|| metrics.bytes_read() >= every_open).await;
4330            assert_eq!(
4331                metrics.bytes_read(),
4332                every_open,
4333                "the opens stopped where the scan stopped polling"
4334            );
4335        }
4336
4337        /// A dropped scan must stop opening files.
4338        #[tokio::test]
4339        async fn test_dropped_scan_stops_opening_files() {
4340            let (metrics, scan) = deferred_scan().await;
4341            let mut scan = Box::pin(scan);
4342            let every_open = OPEN_BYTES * CONCURRENCY as u64;
4343
4344            // Drop while the opens are part way through, or there is nothing to abort.
4345            let _ = futures::poll!(scan.next());
4346            spin(|| metrics.bytes_read() > 0).await;
4347            let in_flight = metrics.bytes_read();
4348            assert!(in_flight > 0, "no open reached the storage");
4349            assert!(in_flight < every_open, "the opens finished before the drop");
4350
4351            drop(scan);
4352            spin(|| false).await;
4353            assert_eq!(
4354                metrics.bytes_read(),
4355                in_flight,
4356                "a dropped scan read {} more bytes",
4357                metrics.bytes_read() - in_flight
4358            );
4359        }
4360
4361        /// The bloom filter fetch is the next read after the open, and runs in the
4362        /// same buffered future, so it must finish on its own too.
4363        #[tokio::test]
4364        async fn test_bloom_filter_fetch_proceeds_without_the_scan_being_polled() {
4365            let tmp_dir = TempDir::new().unwrap();
4366            let props = WriterProperties::builder()
4367                .set_compression(Compression::SNAPPY)
4368                .set_bloom_filter_enabled(true)
4369                .build();
4370            let on_disk = write_parquet_with(
4371                tmp_dir.path().to_str().unwrap(),
4372                "bloom.parquet",
4373                vec![],
4374                vec![],
4375                props,
4376            );
4377            let bloom_offset =
4378                ParquetRecordBatchReaderBuilder::try_new(File::open(&on_disk).unwrap())
4379                    .unwrap()
4380                    .metadata()
4381                    .row_group(0)
4382                    .column(0)
4383                    .bloom_filter_offset()
4384                    .expect("the writer put a bloom filter on `id`") as u64;
4385
4386            let contents = Bytes::from(std::fs::read(&on_disk).unwrap());
4387            let predicate = Reference::new("id").equal_to(Datum::int(2));
4388            let (_, log, scan) = deferred_scan_of(contents, 1, Some(predicate)).await;
4389            let mut scan = Box::pin(scan);
4390
4391            // Poll only until the open is done and the bloom filter fetch is the
4392            // read in flight. Any later poll would drive that fetch itself.
4393            let at_the_bloom_filter = || log.last_start() == Some(bloom_offset);
4394            for _ in 0..YIELDS {
4395                if at_the_bloom_filter() {
4396                    break;
4397                }
4398                let _ = futures::poll!(scan.next());
4399                tokio::task::yield_now().await;
4400            }
4401            assert!(at_the_bloom_filter(), "the bloom filter was never read");
4402
4403            // Nothing polls the scan from here, so only a spawned fetch can finish.
4404            spin(|| log.in_flight() == 0).await;
4405            assert_eq!(
4406                log.in_flight(),
4407                0,
4408                "a read was left in flight where the scan stopped polling"
4409            );
4410        }
4411    }
4412}