Skip to main content

iceberg/scan/
mod.rs

1// Licensed to the Apache Software Foundation (ASF) under one
2// or more contributor license agreements.  See the NOTICE file
3// distributed with this work for additional information
4// regarding copyright ownership.  The ASF licenses this file
5// to you under the Apache License, Version 2.0 (the
6// "License"); you may not use this file except in compliance
7// with the License.  You may obtain a copy of the License at
8//
9//   http://www.apache.org/licenses/LICENSE-2.0
10//
11// Unless required by applicable law or agreed to in writing,
12// software distributed under the License is distributed on an
13// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14// KIND, either express or implied.  See the License for the
15// specific language governing permissions and limitations
16// under the License.
17
18//! Table scan api.
19
20mod cache;
21use cache::*;
22mod context;
23use context::*;
24mod task;
25
26use std::collections::HashMap;
27use std::future::Future;
28use std::sync::Arc;
29
30use arrow_array::RecordBatch;
31use futures::channel::mpsc::{Sender, channel};
32use futures::stream::BoxStream;
33use futures::{SinkExt, StreamExt, TryStreamExt};
34pub use task::*;
35
36use crate::arrow::{ArrowReaderBuilder, ArrowReaderOptions};
37pub use crate::arrow::{ScanMetrics, ScanResult};
38use crate::delete_file_index::DeleteFileIndex;
39use crate::error::invalid_data;
40use crate::expr::visitors::inclusive_metrics_evaluator::InclusiveMetricsEvaluator;
41use crate::expr::{Bind, BoundPredicate, Predicate};
42use crate::io::FileIO;
43use crate::metadata_columns::{
44    RESERVED_FIELD_ID_PARTITION, get_metadata_field_id, is_metadata_column_name,
45};
46use crate::runtime::Runtime;
47use crate::spec::{DataContentType, Schema, SchemaRef, SnapshotRef, SortOrderRef, StructType};
48use crate::table::Table;
49use crate::util::available_parallelism;
50use crate::util::snapshot::snapshot_id_as_of_time;
51use crate::{Error, ErrorKind, Result};
52
53/// A stream of arrow [`RecordBatch`]es.
54pub type ArrowRecordBatchStream = BoxStream<'static, Result<RecordBatch>>;
55
56/// Resolves a column name to its field ID, honouring the scan's case sensitivity.
57fn resolve_field_id(
58    schema: &Schema,
59    column_name: &str,
60    case_sensitive: bool,
61) -> Result<Option<i32>> {
62    if case_sensitive {
63        Ok(schema.field_id_by_name(column_name))
64    } else {
65        let field = schema.field_by_name_case_insensitive_checked(column_name)?;
66        Ok(field.map(|field| field.id))
67    }
68}
69
70/// Resolves a column name to its field ID, failing if no field matches.
71fn require_field_id(schema: &Schema, column_name: &str, case_sensitive: bool) -> Result<i32> {
72    resolve_field_id(schema, column_name, case_sensitive)?
73        .ok_or_else(|| invalid_data!("Column {column_name} not found in table. Schema: {schema}"))
74}
75
76fn collect_scan_field_ids(
77    schema: &Schema,
78    column_names: Option<&[String]>,
79    case_sensitive: bool,
80) -> Result<Vec<i32>> {
81    let Some(column_names) = column_names else {
82        return Ok(schema.as_struct().fields().iter().map(|f| f.id).collect());
83    };
84
85    column_names
86        .iter()
87        .map(|column_name| {
88            if is_metadata_column_name(column_name) {
89                return get_metadata_field_id(column_name);
90            }
91
92            let field_id = require_field_id(schema, column_name, case_sensitive)?;
93
94            schema
95                .as_struct()
96                .field_by_id(field_id)
97                .ok_or_else(|| {
98                    Error::new(
99                        ErrorKind::FeatureUnsupported,
100                        format!(
101                            "Column {column_name} is not a direct child of schema but a nested field, which is not supported now. Schema: {schema}"
102                        ),
103                    )
104                })?;
105
106            Ok(field_id)
107        })
108        .collect()
109}
110
111/// Resolves the columns to keep statistics for into their field ids and types.
112fn resolve_column_stats_columns(
113    schema: &Schema,
114    column_names: Option<&[String]>,
115    case_sensitive: bool,
116) -> Result<Option<ColumnStatsColumns>> {
117    column_names
118        .map(|column_names| {
119            column_names
120                .iter()
121                .map(|column_name| {
122                    if is_metadata_column_name(column_name) {
123                        return Err(invalid_data!(
124                            "Metadata column {column_name} has no column statistics"
125                        ));
126                    }
127                    let field_id = require_field_id(schema, column_name, case_sensitive)?;
128                    let column_type = schema
129                        .field_by_id(field_id)
130                        .and_then(|field| field.field_type.as_primitive_type())
131                        .ok_or_else(|| {
132                            invalid_data!(
133                                "Column {column_name} is not a primitive field and has no column statistics"
134                            )
135                        })?;
136                    Ok((field_id, column_type.clone()))
137                })
138                .collect()
139        })
140        .transpose()
141}
142
143fn bind_scan_predicate(
144    schema: &SchemaRef,
145    predicate: Option<&Predicate>,
146    case_sensitive: bool,
147) -> Result<Option<Arc<BoundPredicate>>> {
148    predicate
149        .map(|predicate| predicate.bind(schema.clone(), case_sensitive))
150        .transpose()
151        .map(|predicate| predicate.map(Arc::new))
152}
153
154fn projected_partition_type(
155    table: &Table,
156    schema: &Schema,
157    field_ids: &[i32],
158) -> Result<Option<Arc<StructType>>> {
159    if !field_ids.contains(&RESERVED_FIELD_ID_PARTITION) {
160        return Ok(None);
161    }
162
163    table
164        .metadata()
165        .unified_partition_type(schema)
166        .map(Arc::new)
167        .map(Some)
168}
169
170/// Selects a snapshot by ID or timestamp.
171enum SnapshotSelection {
172    SnapshotId(i64),
173    AsOfTime(i64),
174}
175
176/// Builder to create table scan.
177pub struct TableScanBuilder<'a> {
178    table: &'a Table,
179    // Defaults to none which means select all columns
180    column_names: Option<Vec<String>>,
181    column_stats_names: Option<Vec<String>>,
182    snapshot_selection: Option<SnapshotSelection>,
183    case_sensitive: bool,
184    filter: Option<Predicate>,
185    concurrency_limit_manifest_entries: usize,
186    concurrency_limit_manifest_files: usize,
187}
188
189impl<'a> TableScanBuilder<'a> {
190    pub(crate) fn new(table: &'a Table) -> Self {
191        let num_cpus = available_parallelism().get();
192
193        Self {
194            table,
195            column_names: None,
196            column_stats_names: None,
197            snapshot_selection: None,
198            case_sensitive: true,
199            filter: None,
200            concurrency_limit_manifest_entries: num_cpus,
201            concurrency_limit_manifest_files: num_cpus,
202        }
203    }
204
205    /// Sets the scan's case sensitivity
206    pub fn with_case_sensitive(mut self, case_sensitive: bool) -> Self {
207        self.case_sensitive = case_sensitive;
208        self
209    }
210
211    /// Specifies a predicate to use as a filter
212    pub fn with_filter(mut self, predicate: Predicate) -> Self {
213        // calls rewrite_not to remove Not nodes, which must be absent
214        // when applying the manifest evaluator
215        self.filter = Some(predicate.rewrite_not());
216        self
217    }
218
219    /// Select all columns.
220    pub fn select_all(mut self) -> Self {
221        self.column_names = None;
222        self
223    }
224
225    /// Select empty columns.
226    pub fn select_empty(mut self) -> Self {
227        self.column_names = Some(vec![]);
228        self
229    }
230
231    /// Select some columns of the table.
232    pub fn select(mut self, column_names: impl IntoIterator<Item = impl ToString>) -> Self {
233        self.column_names = Some(
234            column_names
235                .into_iter()
236                .map(|item| item.to_string())
237                .collect(),
238        );
239        self
240    }
241
242    /// Retains the data file's column statistics for the named columns on each planned
243    /// [`FileScanTask`], exposed as [`FileScanTask::column_stats`].
244    ///
245    /// Names resolve against the scan's schema with the scan's case sensitivity and must name
246    /// primitive fields, such as `s.f` for a field of struct `s`. By default, tasks carry no
247    /// statistics.
248    ///
249    /// [`build`](Self::build) fails if a name matches no field, matches more than one field when
250    /// the scan is case-insensitive, or names a struct, list, map or metadata column.
251    ///
252    /// ```
253    /// # use futures::TryStreamExt;
254    /// # use iceberg::table::Table;
255    /// # async fn example(table: &Table) -> iceberg::Result<()> {
256    /// let scan = table.scan().with_column_stats(["x"]).build()?;
257    /// let mut tasks = scan.plan_files().await?;
258    /// while let Some(task) = tasks.try_next().await? {
259    ///     let x = task.schema().field_id_by_name("x").unwrap();
260    ///     let lower_bound = task
261    ///         .column_stats()
262    ///         .and_then(|stats| stats.lower_bounds().get(&x));
263    /// }
264    /// # Ok(())
265    /// # }
266    /// ```
267    pub fn with_column_stats(
268        mut self,
269        column_names: impl IntoIterator<Item = impl ToString>,
270    ) -> Self {
271        self.column_stats_names = Some(
272            column_names
273                .into_iter()
274                .map(|item| item.to_string())
275                .collect(),
276        );
277        self
278    }
279
280    /// Set the snapshot to scan. The scan projects the schema that snapshot was
281    /// written with.
282    ///
283    /// The last call to this or [`Self::as_of_time`] takes precedence.
284    /// If neither is set, the scan reads the current snapshot with the table's
285    /// current schema.
286    pub fn snapshot_id(mut self, snapshot_id: i64) -> Self {
287        self.snapshot_selection = Some(SnapshotSelection::SnapshotId(snapshot_id));
288        self
289    }
290
291    /// Select a snapshot at or before a timestamp in milliseconds since the Unix epoch.
292    ///
293    /// Uses the table's main history, and projects the schema the selected snapshot
294    /// was written with. The last call to this or [`Self::snapshot_id`] takes precedence.
295    ///
296    /// ```
297    /// # use iceberg::TableIdent;
298    /// # use iceberg::io::FileIO;
299    /// # use iceberg::table::StaticTable;
300    /// # #[tokio::main]
301    /// # async fn main() -> iceberg::Result<()> {
302    /// # let location = concat!(env!("CARGO_MANIFEST_DIR"), "/testdata/example_table_metadata_v2.json");
303    /// # let ident = TableIdent::from_strs(["ns", "t"])?;
304    /// # let table = StaticTable::from_metadata_file(location, ident, FileIO::new_with_fs()).await?;
305    /// // The snapshot log has entries at 1515100955770 and 1555100955770.
306    /// let scan = table.scan().as_of_time(1_555_100_955_769).build()?;
307    /// assert_eq!(scan.snapshot().unwrap().snapshot_id(), 3051729675574597004);
308    /// # Ok(())
309    /// # }
310    /// ```
311    pub fn as_of_time(mut self, timestamp_ms: i64) -> Self {
312        self.snapshot_selection = Some(SnapshotSelection::AsOfTime(timestamp_ms));
313        self
314    }
315
316    /// Sets manifest file and manifest entry concurrency for file planning.
317    pub fn with_concurrency_limit(mut self, limit: usize) -> Self {
318        self.concurrency_limit_manifest_files = limit;
319        self.concurrency_limit_manifest_entries = limit;
320        self
321    }
322
323    /// Sets the manifest entry concurrency limit for this scan
324    pub fn with_manifest_entry_concurrency_limit(mut self, limit: usize) -> Self {
325        self.concurrency_limit_manifest_entries = limit;
326        self
327    }
328
329    /// Build the table scan.
330    pub fn build(self) -> Result<TableScan> {
331        let selected_snapshot = match self.snapshot_selection {
332            Some(SnapshotSelection::SnapshotId(id)) => {
333                Some((id, format!("Snapshot with id {id} not found")))
334            }
335            Some(SnapshotSelection::AsOfTime(timestamp_ms)) => {
336                let id = snapshot_id_as_of_time(&self.table.metadata_ref(), timestamp_ms)?;
337                // History outlives expired snapshots; name the timestamp that resolved to `id`.
338                Some((
339                    id,
340                    format!(
341                        "Snapshot with id {id} resolved from timestamp {timestamp_ms} ms not found; it may have expired"
342                    ),
343                ))
344            }
345            None => None,
346        };
347        let is_time_travel = selected_snapshot.is_some();
348        let snapshot = match selected_snapshot {
349            Some((snapshot_id, not_found_message)) => self
350                .table
351                .metadata()
352                .snapshot_by_id(snapshot_id)
353                .ok_or_else(|| invalid_data!(not_found_message))?
354                .clone(),
355            None => {
356                let Some(current_snapshot_id) = self.table.metadata().current_snapshot() else {
357                    return Ok(TableScan {
358                        column_names: self.column_names,
359                        file_io: self.table.file_io().clone(),
360                        plan_context: None,
361                        concurrency_limit_manifest_entries: self.concurrency_limit_manifest_entries,
362                        concurrency_limit_manifest_files: self.concurrency_limit_manifest_files,
363                        runtime: self.table.runtime().clone(),
364                    });
365                };
366                current_snapshot_id.clone()
367            }
368        };
369
370        // A scan of the current state uses the table's current schema; time travel uses the
371        // schema its snapshot was written with.
372        let schema = if is_time_travel {
373            snapshot.schema(self.table.metadata())?
374        } else {
375            self.table.metadata().current_schema().clone()
376        };
377        let field_ids =
378            collect_scan_field_ids(&schema, self.column_names.as_deref(), self.case_sensitive)?;
379        let column_stats_columns = resolve_column_stats_columns(
380            &schema,
381            self.column_stats_names.as_deref(),
382            self.case_sensitive,
383        )?;
384        let snapshot_bound_predicate =
385            bind_scan_predicate(&schema, self.filter.as_ref(), self.case_sensitive)?;
386        let name_mapping = self
387            .table
388            .metadata()
389            .table_properties()
390            .default_name_mapping()?
391            .map(Arc::new);
392        let unified_partition_type = projected_partition_type(self.table, &schema, &field_ids)?;
393
394        // Precompute the table's sort orders once, keyed by id, so each manifest-file
395        // context carries only this narrow map instead of the full table metadata.
396        let sort_orders = Arc::new(
397            self.table
398                .metadata()
399                .sort_orders_iter()
400                .map(|order| (order.order_id, order.clone()))
401                .collect::<HashMap<i64, SortOrderRef>>(),
402        );
403
404        let plan_context = PlanContext {
405            snapshot,
406            table_metadata: self.table.metadata_ref(),
407            snapshot_schema: schema,
408            case_sensitive: self.case_sensitive,
409            predicate: self.filter.map(Arc::new),
410            snapshot_bound_predicate,
411            object_cache: self.table.object_cache(),
412            field_ids: Arc::new(field_ids),
413            name_mapping,
414            partition_filter_cache: Arc::new(PartitionFilterCache::new()),
415            manifest_evaluator_cache: Arc::new(ManifestEvaluatorCache::new()),
416            expression_evaluator_cache: Arc::new(ExpressionEvaluatorCache::new()),
417            unified_partition_type,
418            sort_orders,
419            column_stats_columns,
420        };
421
422        Ok(TableScan {
423            column_names: self.column_names,
424            file_io: self.table.file_io().clone(),
425            plan_context: Some(plan_context),
426            concurrency_limit_manifest_entries: self.concurrency_limit_manifest_entries,
427            concurrency_limit_manifest_files: self.concurrency_limit_manifest_files,
428            runtime: self.table.runtime().clone(),
429        })
430    }
431}
432
433/// Table scan.
434#[derive(Debug)]
435pub struct TableScan {
436    /// A [PlanContext], if this table has at least one snapshot, otherwise None.
437    ///
438    /// If this is None, then the scan contains no rows.
439    plan_context: Option<PlanContext>,
440    file_io: FileIO,
441    column_names: Option<Vec<String>>,
442    /// The maximum number of manifest files that will be
443    /// retrieved from [`FileIO`] concurrently
444    concurrency_limit_manifest_files: usize,
445
446    /// The maximum number of [`ManifestEntry`]s that will
447    /// be processed in parallel
448    concurrency_limit_manifest_entries: usize,
449
450    runtime: Runtime,
451}
452
453impl TableScan {
454    /// Returns a stream of [`FileScanTask`]s.
455    pub async fn plan_files(&self) -> Result<FileScanTaskStream> {
456        self.plan_data_files(|ctx| ctx.into_file_scan_task()).await
457    }
458
459    pub(crate) async fn plan_cow_rewrite_files(
460        &self,
461    ) -> Result<BoxStream<'static, Result<crate::cow_rewrite::CowRewriteFile>>> {
462        self.plan_data_files(|ctx| ctx.into_cow_rewrite_file())
463            .await
464    }
465
466    async fn plan_data_files<T, F, Fut>(
467        &self,
468        build_result: F,
469    ) -> Result<BoxStream<'static, Result<T>>>
470    where
471        T: Send + 'static,
472        F: Fn(ManifestEntryContext) -> Fut + Copy + Send + Sync + 'static,
473        Fut: Future<Output = Result<T>> + Send + 'static,
474    {
475        let Some(plan_context) = self.plan_context.as_ref() else {
476            return Ok(Box::pin(futures::stream::empty()));
477        };
478
479        let concurrency_limit_manifest_files = self.concurrency_limit_manifest_files;
480        let concurrency_limit_manifest_entries = self.concurrency_limit_manifest_entries;
481
482        // used to stream ManifestEntryContexts between stages of the file plan operation
483        let (manifest_entry_data_ctx_tx, manifest_entry_data_ctx_rx) =
484            channel(concurrency_limit_manifest_files);
485        let (manifest_entry_delete_ctx_tx, manifest_entry_delete_ctx_rx) =
486            channel(concurrency_limit_manifest_files);
487
488        // used to stream the planned data file results back to the caller
489        let (planned_file_tx, planned_file_rx) = channel(concurrency_limit_manifest_entries);
490
491        let (delete_file_idx, delete_file_tx) = DeleteFileIndex::new(self.runtime.clone());
492
493        let manifest_list = plan_context.get_manifest_list().await?;
494
495        // get the [`ManifestFile`]s from the [`ManifestList`], filtering out any
496        // whose partitions cannot match this
497        // scan's filter
498        let manifest_file_contexts = plan_context.build_manifest_file_contexts(
499            manifest_list,
500            manifest_entry_data_ctx_tx,
501            delete_file_idx.clone(),
502            manifest_entry_delete_ctx_tx,
503        )?;
504
505        let mut channel_for_manifest_error = planned_file_tx.clone();
506        let mut channel_for_data_manifest_entry_error = planned_file_tx.clone();
507        let mut channel_for_delete_manifest_entry_error = planned_file_tx.clone();
508
509        let rt = self.runtime.clone();
510
511        // Concurrently load all [`Manifest`]s and stream their [`ManifestEntry`]s
512        rt.io().spawn(async move {
513            let result = futures::stream::iter(manifest_file_contexts)
514                .try_for_each_concurrent(concurrency_limit_manifest_files, |ctx| async move {
515                    ctx.fetch_manifest_and_stream_manifest_entries().await
516                })
517                .await;
518
519            if let Err(error) = result {
520                let _ = channel_for_manifest_error.send(Err(error)).await;
521            }
522        });
523
524        // Process the delete file [`ManifestEntry`] stream in parallel
525        {
526            let rt = rt.clone();
527            let rt_inner = rt.clone();
528            rt.cpu().spawn(async move {
529                let result = manifest_entry_delete_ctx_rx
530                    .map(|me_ctx| Ok((me_ctx, delete_file_tx.clone())))
531                    .try_for_each_concurrent(
532                        concurrency_limit_manifest_entries,
533                        |(manifest_entry_context, tx)| {
534                            let rt_inner = rt_inner.clone();
535                            async move {
536                                rt_inner
537                                    .cpu()
538                                    .spawn(async move {
539                                        Self::process_delete_manifest_entry(
540                                            manifest_entry_context,
541                                            tx,
542                                        )
543                                        .await
544                                    })
545                                    .await?
546                            }
547                        },
548                    )
549                    .await;
550
551                if let Err(error) = result {
552                    let _ = channel_for_delete_manifest_entry_error
553                        .send(Err(error))
554                        .await;
555                }
556            });
557        }
558
559        // Process the data file [`ManifestEntry`] stream in parallel
560        {
561            let rt_inner = rt.clone();
562            rt.cpu().spawn(async move {
563                let result = manifest_entry_data_ctx_rx
564                    .map(|me_ctx| Ok((me_ctx, planned_file_tx.clone())))
565                    .try_for_each_concurrent(
566                        concurrency_limit_manifest_entries,
567                        |(manifest_entry_context, tx)| {
568                            let rt_inner = rt_inner.clone();
569                            async move {
570                                rt_inner
571                                    .cpu()
572                                    .spawn(async move {
573                                        Self::process_data_manifest_entry(
574                                            manifest_entry_context,
575                                            tx,
576                                            build_result,
577                                        )
578                                        .await
579                                    })
580                                    .await?
581                            }
582                        },
583                    )
584                    .await;
585
586                if let Err(error) = result {
587                    let _ = channel_for_data_manifest_entry_error.send(Err(error)).await;
588                }
589            });
590        }
591
592        Ok(planned_file_rx.boxed())
593    }
594
595    /// Returns an [`ArrowRecordBatchStream`] using the supplied reader settings.
596    ///
597    /// Reader settings are independent of file planning. Unset data-file concurrency
598    /// uses execution-host parallelism; zero is treated as one.
599    /// [`TableScanBuilder::with_concurrency_limit`] only controls manifest planning.
600    ///
601    /// ```no_run
602    /// # use iceberg::{Result, arrow::{ArrowReaderOptions, ParquetReadOptions}, table::Table};
603    /// # async fn read(table: &Table) -> Result<()> {
604    /// let options = ArrowReaderOptions::builder()
605    ///     .with_batch_size(1024)
606    ///     .with_row_selection_enabled(true)
607    ///     .with_parquet_options(
608    ///         ParquetReadOptions::builder()
609    ///             .with_metadata_size_hint(512 * 1024)
610    ///             .build(),
611    ///     )
612    ///     .build();
613    /// let stream = table.scan().build()?.to_arrow(options).await?;
614    /// # Ok(())
615    /// # }
616    /// ```
617    ///
618    /// When upgrading, move reader-setting calls from `TableScanBuilder` to
619    /// `ArrowReaderOptions::builder()` and pass the result to `to_arrow(options)`.
620    /// Use `to_arrow(Default::default())` for default settings. To preserve a
621    /// data-file concurrency bound previously set through the scan, set
622    /// `with_data_file_concurrency_limit(limit)` on the reader options explicitly.
623    ///
624    /// Locally constructed readers accept the same options through
625    /// [`ArrowReaderBuilder::with_options`], including Parquet I/O tuning.
626    /// This replaces the complete configuration. Configure individual settings
627    /// through the options builders; per-field setters on `ArrowReaderBuilder`
628    /// have been removed. [`ArrowReaderBuilder::with_parquet_options`] replaces
629    /// only the nested Parquet configuration.
630    pub async fn to_arrow(&self, options: ArrowReaderOptions) -> Result<ArrowRecordBatchStream> {
631        ArrowReaderBuilder::new(self.file_io.clone(), self.runtime.clone())
632            .with_options(options)
633            .build()
634            .read(self.plan_files().await?)
635            .map(|result| result.stream())
636    }
637
638    /// Returns a reference to the column names of the table scan.
639    pub fn column_names(&self) -> Option<&[String]> {
640        self.column_names.as_deref()
641    }
642
643    /// Returns a reference to the snapshot of the table scan.
644    pub fn snapshot(&self) -> Option<&SnapshotRef> {
645        self.plan_context.as_ref().map(|x| &x.snapshot)
646    }
647
648    async fn process_data_manifest_entry<T, F, Fut>(
649        manifest_entry_context: ManifestEntryContext,
650        mut planned_file_tx: Sender<Result<T>>,
651        build_result: F,
652    ) -> Result<()>
653    where
654        F: Fn(ManifestEntryContext) -> Fut + Send + Sync + 'static,
655        Fut: Future<Output = Result<T>> + Send + 'static,
656    {
657        // abort the plan if we encounter a manifest entry for a delete file
658        if manifest_entry_context.manifest_entry.content_type() != DataContentType::Data {
659            return Err(Error::new(
660                ErrorKind::FeatureUnsupported,
661                "Encountered an entry for a delete file in a data file manifest",
662            ));
663        }
664
665        if let Some(ref bound_predicates) = manifest_entry_context.bound_predicates {
666            let BoundPredicates {
667                snapshot_bound_predicate,
668                partition_bound_predicate,
669            } = bound_predicates.as_ref();
670
671            let expression_evaluator_cache =
672                manifest_entry_context.expression_evaluator_cache.as_ref();
673
674            let expression_evaluator = expression_evaluator_cache.get(
675                manifest_entry_context.partition_spec_id,
676                partition_bound_predicate,
677            )?;
678
679            // skip any data file whose partition data indicates that it can't contain
680            // any data that matches this scan's filter
681            if !expression_evaluator.eval(manifest_entry_context.manifest_entry.data_file())? {
682                return Ok(());
683            }
684
685            // skip any data file whose metrics don't match this scan's filter
686            if !InclusiveMetricsEvaluator::eval(
687                snapshot_bound_predicate,
688                manifest_entry_context.manifest_entry.data_file(),
689                false,
690            )? {
691                return Ok(());
692            }
693        }
694
695        // congratulations! the manifest entry has made its way through the
696        // entire plan without getting filtered out. Build the planned file before
697        // sending so delete-file lookup preserves the original scan timing.
698        planned_file_tx
699            .send(Ok(build_result(manifest_entry_context).await?))
700            .await?;
701
702        Ok(())
703    }
704
705    async fn process_delete_manifest_entry(
706        manifest_entry_context: ManifestEntryContext,
707        mut delete_file_ctx_tx: Sender<DeleteFileContext>,
708    ) -> Result<()> {
709        // abort the plan if we encounter a manifest entry that is not for a delete file
710        if manifest_entry_context.manifest_entry.content_type() == DataContentType::Data {
711            return Err(Error::new(
712                ErrorKind::FeatureUnsupported,
713                "Encountered an entry for a data file in a delete manifest",
714            ));
715        }
716
717        if let Some(ref bound_predicates) = manifest_entry_context.bound_predicates {
718            let expression_evaluator_cache =
719                manifest_entry_context.expression_evaluator_cache.as_ref();
720
721            let expression_evaluator = expression_evaluator_cache.get(
722                manifest_entry_context.partition_spec_id,
723                &bound_predicates.partition_bound_predicate,
724            )?;
725
726            // skip any data file whose partition data indicates that it can't contain
727            // any data that matches this scan's filter
728            if !expression_evaluator.eval(manifest_entry_context.manifest_entry.data_file())? {
729                return Ok(());
730            }
731        }
732
733        delete_file_ctx_tx
734            .send(DeleteFileContext {
735                manifest_entry: manifest_entry_context.manifest_entry.clone(),
736                partition_spec_id: manifest_entry_context.partition_spec_id,
737            })
738            .await?;
739
740        Ok(())
741    }
742}
743
744pub(crate) struct BoundPredicates {
745    partition_bound_predicate: BoundPredicate,
746    snapshot_bound_predicate: BoundPredicate,
747}
748
749#[cfg(test)]
750mod tests {
751    //! shared tests for the table scan API
752
753    use std::collections::HashMap;
754    use std::fs;
755    use std::fs::File;
756    use std::sync::Arc;
757
758    use arrow_array::cast::AsArray;
759    use arrow_array::types::Int32Type;
760    use arrow_array::{
761        Array, ArrayRef, BooleanArray, Float64Array, Int32Array, Int64Array, RecordBatch, RunArray,
762        StringArray,
763    };
764    use futures::{StreamExt, TryStreamExt, stream};
765    use parquet::arrow::ArrowWriter;
766    use uuid::Uuid;
767
768    use super::{resolve_column_stats_columns, resolve_field_id};
769    use crate::arrow::{ArrowReaderBuilder, ArrowReaderOptions, ParquetReadOptions};
770    use crate::expr::{BoundPredicate, Reference};
771    use crate::io::FileIO;
772    use crate::metadata_columns::{
773        RESERVED_COL_NAME_FILE, RESERVED_COL_NAME_LAST_UPDATED_SEQUENCE_NUMBER,
774        RESERVED_COL_NAME_POS, RESERVED_COL_NAME_SPEC_ID, RESERVED_FIELD_ID_POS,
775    };
776    use crate::scan::{FileScanTask, FileScanTaskColumnStats, FileScanTaskDeleteFile, TableScan};
777    use crate::spec::{
778        DataContentType, DataFileBuilder, DataFileFormat, Datum, ListType, Literal, MAIN_BRANCH,
779        ManifestEntry, ManifestListWriter, ManifestStatus, ManifestWriterBuilder, MappedField,
780        NameMapping, NestedField, NullOrder, Operation, PartitionSpec, PrimitiveType, Schema,
781        Snapshot, SortDirection, SortField, SortOrder, Struct, StructType, Summary,
782        TableMetadataBuilder, TableProperties, Transform, Type, UnboundPartitionSpec,
783    };
784    use crate::table::Table;
785    use crate::test_utils::scan::{
786        ROWS_PER_FIXTURE_FILE, TableTestFixture, assert_last_updated_seq_all,
787    };
788    use crate::test_utils::test_runtime;
789    use crate::util::snapshot::snapshot_id_as_of_time;
790    use crate::{ErrorKind, Result, Runtime, TableIdent};
791
792    #[tokio::test]
793    async fn test_table_scan_columns() {
794        let table = TableTestFixture::new().table;
795
796        let table_scan = table.scan().select(["x", "y"]).build().unwrap();
797        assert_eq!(
798            Some(vec!["x".to_string(), "y".to_string()]),
799            table_scan.column_names
800        );
801
802        let table_scan = table
803            .scan()
804            .select(["x", "y"])
805            .select(["z"])
806            .build()
807            .unwrap();
808        assert_eq!(Some(vec!["z".to_string()]), table_scan.column_names);
809    }
810
811    #[tokio::test]
812    async fn test_select_all() {
813        let table = TableTestFixture::new().table;
814
815        let table_scan = table.scan().select_all().build().unwrap();
816        assert!(table_scan.column_names.is_none());
817    }
818
819    #[test]
820    fn test_select_no_exist_column() {
821        let table = TableTestFixture::new().table;
822
823        let table_scan = table.scan().select(["x", "y", "z", "a", "b"]).build();
824        assert!(table_scan.is_err());
825    }
826
827    #[test]
828    fn test_case_sensitive_scan_rejects_mismatched_column_case() {
829        let table = TableTestFixture::new().table;
830
831        // Case sensitivity defaults to true, so "X" must not resolve to "x".
832        assert!(table.scan().select(["X"]).build().is_err());
833        assert!(
834            table
835                .scan()
836                .with_filter(Reference::new("X").greater_than(Datum::long(1)))
837                .build()
838                .is_err()
839        );
840    }
841
842    /// A schema with a struct `s`, a list `l` and a long `n`, whose field ids are 1 to 5.
843    fn nested_schema() -> Schema {
844        Schema::builder()
845            .with_fields(vec![
846                NestedField::required(
847                    1,
848                    "s",
849                    Type::Struct(StructType::new(vec![
850                        NestedField::optional(2, "f", PrimitiveType::Long).into(),
851                    ])),
852                )
853                .into(),
854                NestedField::required(
855                    3,
856                    "l",
857                    Type::List(ListType::new(
858                        NestedField::list_element(4, PrimitiveType::Long, true).into(),
859                    )),
860                )
861                .into(),
862                NestedField::required(5, "n", PrimitiveType::Long).into(),
863            ])
864            .build()
865            .unwrap()
866    }
867
868    fn column_names(names: &[&str]) -> Vec<String> {
869        names.iter().map(|name| name.to_string()).collect()
870    }
871
872    #[test]
873    fn test_resolve_column_stats_columns_resolves_primitive_names() {
874        let schema = nested_schema();
875        let long = PrimitiveType::Long;
876
877        assert_eq!(
878            resolve_column_stats_columns(&schema, Some(&column_names(&["s.f", "n"])), true)
879                .unwrap(),
880            Some(Arc::<[_]>::from([(2, long.clone()), (5, long.clone())]))
881        );
882        assert_eq!(
883            resolve_column_stats_columns(&schema, Some(&column_names(&["N"])), false).unwrap(),
884            Some(Arc::<[_]>::from([(5, long)]))
885        );
886        assert_eq!(
887            resolve_column_stats_columns(&schema, None, true).unwrap(),
888            None
889        );
890    }
891
892    #[test]
893    fn test_resolve_column_stats_columns_rejects_names_without_statistics() {
894        let schema = nested_schema();
895        let cases = [
896            (
897                "s",
898                "Column s is not a primitive field and has no column statistics",
899            ),
900            (
901                "l",
902                "Column l is not a primitive field and has no column statistics",
903            ),
904            (
905                RESERVED_COL_NAME_FILE,
906                "Metadata column _file has no column statistics",
907            ),
908            ("missing", "Column missing not found in table"),
909            ("N", "Column N not found in table"),
910        ];
911
912        for (name, expected) in cases {
913            let err = resolve_column_stats_columns(&schema, Some(&column_names(&[name])), true)
914                .unwrap_err();
915            assert_eq!(err.kind(), ErrorKind::DataInvalid, "{name}");
916            assert!(err.message().starts_with(expected), "{name}: {err}");
917        }
918    }
919
920    #[test]
921    fn test_resolve_field_id_case_insensitive_rejects_ambiguous_name() {
922        let schema = Schema::builder()
923            .with_fields(vec![
924                NestedField::optional(1, "id", PrimitiveType::Int).into(),
925                NestedField::optional(2, "ID", PrimitiveType::Int).into(),
926                NestedField::optional(3, "data", PrimitiveType::Int).into(),
927            ])
928            .build()
929            .unwrap();
930
931        assert_eq!(resolve_field_id(&schema, "id", true).unwrap(), Some(1));
932        assert_eq!(resolve_field_id(&schema, "DATA", false).unwrap(), Some(3));
933        let error = resolve_field_id(&schema, "Id", false).unwrap_err();
934        assert_eq!(error.kind(), ErrorKind::DataInvalid);
935        assert!(error.message().contains("case-insensitively"), "{error}");
936    }
937
938    #[test]
939    fn test_case_insensitive_scan_with_ambiguous_schema() {
940        let fixture = TableTestFixture::new();
941        let mut metadata = fixture.table.metadata().clone();
942        let schema = Schema::builder()
943            .with_schema_id(metadata.current_schema_id())
944            .with_fields(vec![
945                NestedField::optional(1, "id", PrimitiveType::Int).into(),
946                NestedField::optional(2, "ID", PrimitiveType::Int).into(),
947                NestedField::optional(3, "data", PrimitiveType::Int).into(),
948            ])
949            .build()
950            .unwrap();
951        metadata
952            .schemas
953            .insert(metadata.current_schema_id(), Arc::new(schema));
954        let table = fixture.table.with_metadata(Arc::new(metadata));
955
956        table
957            .scan()
958            .with_case_sensitive(false)
959            .select(["DATA"])
960            .build()
961            .unwrap();
962        let error = table
963            .scan()
964            .with_case_sensitive(false)
965            .select(["Id"])
966            .build()
967            .unwrap_err();
968        assert_eq!(error.kind(), ErrorKind::DataInvalid);
969        assert!(error.message().contains("case-insensitively"), "{error}");
970    }
971
972    #[tokio::test]
973    async fn test_case_insensitive_scan_resolves_mismatched_column_case() {
974        let mut fixture = TableTestFixture::new();
975        fixture.setup_manifest_files().await;
976
977        // The schema declares lowercase "x" and "z"; a case-insensitive scan must
978        // resolve the upper-cased names in both the projection and the filter.
979        let table_scan = fixture
980            .table
981            .scan()
982            .with_case_sensitive(false)
983            .select(["X", "Z"])
984            .with_filter(Reference::new("Y").greater_than(Datum::long(1)))
985            .build()
986            .unwrap();
987
988        let batches: Vec<_> = table_scan
989            .to_arrow(ArrowReaderOptions::default())
990            .await
991            .unwrap()
992            .try_collect()
993            .await
994            .unwrap();
995
996        assert_eq!(batches[0].num_columns(), 2);
997        assert_eq!(
998            batches[0]
999                .column_by_name("x")
1000                .unwrap()
1001                .as_any()
1002                .downcast_ref::<Int64Array>()
1003                .unwrap()
1004                .value(0),
1005            1
1006        );
1007        assert_eq!(
1008            batches[0]
1009                .column_by_name("z")
1010                .unwrap()
1011                .as_any()
1012                .downcast_ref::<Int64Array>()
1013                .unwrap()
1014                .value(0),
1015            3
1016        );
1017    }
1018
1019    #[tokio::test]
1020    async fn test_table_scan_default_snapshot_id() {
1021        let table = TableTestFixture::new().table;
1022
1023        let table_scan = table.scan().build().unwrap();
1024        assert_eq!(
1025            table.metadata().current_snapshot().unwrap().snapshot_id(),
1026            table_scan.snapshot().unwrap().snapshot_id()
1027        );
1028    }
1029
1030    #[test]
1031    fn test_table_scan_non_exist_snapshot_id() {
1032        let table = TableTestFixture::new().table;
1033
1034        let table_scan = table.scan().snapshot_id(1024).build();
1035        assert!(table_scan.is_err());
1036    }
1037
1038    #[tokio::test]
1039    async fn test_table_scan_with_snapshot_id() {
1040        let table = TableTestFixture::new().table;
1041
1042        let table_scan = table
1043            .scan()
1044            .snapshot_id(3051729675574597004)
1045            .build()
1046            .unwrap();
1047        assert_eq!(
1048            table_scan.snapshot().unwrap().snapshot_id(),
1049            3051729675574597004
1050        );
1051    }
1052
1053    #[test]
1054    fn test_table_scan_as_of_time_after_rollback() {
1055        for elapsed_ms in [0, 1000] {
1056            let table = TableTestFixture::new().table;
1057            let mut metadata = table.metadata().clone();
1058            let original = metadata.history()[0].snapshot_id;
1059            let previous = metadata.history()[1].clone();
1060            let rollback_time = previous.timestamp_ms + elapsed_ms;
1061            metadata.snapshot_log.push(crate::spec::SnapshotLog {
1062                timestamp_ms: rollback_time,
1063                snapshot_id: original,
1064            });
1065            metadata.current_snapshot_id = Some(original);
1066            metadata.refs.get_mut(MAIN_BRANCH).unwrap().snapshot_id = original;
1067            let table = table.with_metadata(Arc::new(metadata));
1068            let scan = table.scan().as_of_time(rollback_time).build().unwrap();
1069            // A rollback at the same timestamp keeps the earlier history entry.
1070            let expected = if elapsed_ms == 0 {
1071                previous.snapshot_id
1072            } else {
1073                original
1074            };
1075            assert_eq!(scan.snapshot().unwrap().snapshot_id(), expected);
1076        }
1077    }
1078
1079    #[test]
1080    fn test_table_scan_as_of_time_rejects_empty_history() {
1081        let table = TableTestFixture::new_empty().table;
1082        assert!(table.scan().build().unwrap().snapshot().is_none());
1083        let err = table.scan().as_of_time(0).build().unwrap_err();
1084        assert_eq!(err.kind(), ErrorKind::DataInvalid);
1085        assert!(err.message().contains("No snapshot history"));
1086    }
1087
1088    #[test]
1089    fn test_table_scan_as_of_time_rejects_before_history() {
1090        let table = TableTestFixture::new().table;
1091        let before = table.metadata().history()[0].timestamp_ms - 1;
1092        let err = table.scan().as_of_time(before).build().unwrap_err();
1093        assert_eq!(err.kind(), ErrorKind::DataInvalid);
1094        assert!(err.message().contains(&before.to_string()));
1095    }
1096
1097    #[test]
1098    fn test_table_scan_as_of_time_rejects_all_qualifying_snapshots_expired() {
1099        let table = TableTestFixture::new_with_deep_history().table;
1100        let mut metadata = table.metadata().clone();
1101        let entry = metadata.history()[3].clone();
1102        let expired_ids: Vec<_> = metadata
1103            .history()
1104            .iter()
1105            .filter(|log| log.timestamp_ms <= entry.timestamp_ms)
1106            .map(|log| log.snapshot_id)
1107            .collect();
1108        assert_eq!(expired_ids.len(), 4);
1109        for snapshot_id in expired_ids {
1110            metadata.snapshots.remove(&snapshot_id);
1111        }
1112        let table = table.with_metadata(Arc::new(metadata));
1113        // History resolution succeeds even though every qualifying snapshot expired.
1114        assert_eq!(
1115            snapshot_id_as_of_time(&table.metadata_ref(), entry.timestamp_ms).unwrap(),
1116            entry.snapshot_id
1117        );
1118        assert!(table.scan().build().unwrap().snapshot().is_some());
1119        let err = table
1120            .scan()
1121            .as_of_time(entry.timestamp_ms)
1122            .build()
1123            .unwrap_err();
1124        assert_eq!(err.kind(), ErrorKind::DataInvalid);
1125        assert_eq!(
1126            err.message(),
1127            format!(
1128                "Snapshot with id {} resolved from timestamp {} ms not found; it may have expired",
1129                entry.snapshot_id, entry.timestamp_ms
1130            )
1131        );
1132    }
1133
1134    #[test]
1135    fn test_table_scan_last_snapshot_selection_wins() {
1136        let table = TableTestFixture::new().table;
1137        let first = &table.metadata().history()[0];
1138        let last = &table.metadata().history()[1];
1139        let cases = [
1140            (
1141                table
1142                    .scan()
1143                    .snapshot_id(last.snapshot_id)
1144                    .as_of_time(first.timestamp_ms),
1145                first.snapshot_id,
1146            ),
1147            (
1148                table
1149                    .scan()
1150                    .as_of_time(first.timestamp_ms)
1151                    .snapshot_id(last.snapshot_id),
1152                last.snapshot_id,
1153            ),
1154            // An overwritten selector is not resolved or validated.
1155            (
1156                table.scan().snapshot_id(-1).as_of_time(first.timestamp_ms),
1157                first.snapshot_id,
1158            ),
1159            (
1160                table
1161                    .scan()
1162                    .as_of_time(i64::MIN)
1163                    .snapshot_id(last.snapshot_id),
1164                last.snapshot_id,
1165            ),
1166            (
1167                table
1168                    .scan()
1169                    .snapshot_id(first.snapshot_id)
1170                    .snapshot_id(last.snapshot_id),
1171                last.snapshot_id,
1172            ),
1173            (
1174                table
1175                    .scan()
1176                    .as_of_time(i64::MAX)
1177                    .as_of_time(first.timestamp_ms),
1178                first.snapshot_id,
1179            ),
1180        ];
1181        for (builder, expected) in cases {
1182            let scan = builder.build().unwrap();
1183            assert_eq!(scan.snapshot().unwrap().snapshot_id(), expected);
1184        }
1185    }
1186
1187    #[test]
1188    fn test_table_scan_invalid_last_snapshot_selection() {
1189        let table = TableTestFixture::new().table;
1190        let first = &table.metadata().history()[0];
1191        let cases = [
1192            (
1193                table
1194                    .scan()
1195                    .snapshot_id(first.snapshot_id)
1196                    .as_of_time(first.timestamp_ms - 1),
1197                "No snapshot history",
1198            ),
1199            (
1200                table.scan().as_of_time(first.timestamp_ms).snapshot_id(-1),
1201                "Snapshot with id -1 not found",
1202            ),
1203        ];
1204        for (builder, expected) in cases {
1205            let err = builder.build().unwrap_err();
1206            assert_eq!(err.kind(), ErrorKind::DataInvalid);
1207            assert!(err.message().contains(expected));
1208        }
1209    }
1210
1211    #[test]
1212    fn test_table_scan_as_of_time_uses_snapshot_schema() {
1213        for select_current in [false, true] {
1214            let table = TableTestFixture::new().table;
1215            let mut metadata = table.metadata().clone();
1216            let entry = metadata.history()[usize::from(select_current)].clone();
1217            // Simulate a schema-only update after this snapshot: schema 0 has x only,
1218            // while the current table schema also contains y and other columns.
1219            Arc::make_mut(metadata.snapshots.get_mut(&entry.snapshot_id).unwrap()).schema_id =
1220                Some(0);
1221            let table = table.with_metadata(Arc::new(metadata));
1222            let scan = table
1223                .scan()
1224                .as_of_time(entry.timestamp_ms)
1225                .select(["x"])
1226                .with_filter(Reference::new("x").greater_than(Datum::long(0)))
1227                .build()
1228                .unwrap();
1229            let context = scan.plan_context.as_ref().unwrap();
1230            assert_eq!(context.snapshot_schema.schema_id(), 0);
1231            assert!(context.snapshot_bound_predicate.is_some());
1232            assert!(
1233                table
1234                    .scan()
1235                    .as_of_time(entry.timestamp_ms)
1236                    .select(["y"])
1237                    .build()
1238                    .is_err()
1239            );
1240            assert!(
1241                table
1242                    .scan()
1243                    .as_of_time(entry.timestamp_ms)
1244                    .with_filter(Reference::new("y").greater_than(Datum::long(0)))
1245                    .build()
1246                    .is_err()
1247            );
1248        }
1249    }
1250
1251    #[test]
1252    fn test_table_scan_as_of_time_schema_compatibility() {
1253        let table = TableTestFixture::new().table;
1254        let entry = table.metadata().history()[0].clone();
1255        // Older snapshots may omit schema-id and use the current-schema fallback.
1256        assert!(
1257            table
1258                .metadata()
1259                .snapshot_by_id(entry.snapshot_id)
1260                .unwrap()
1261                .schema_id()
1262                .is_none()
1263        );
1264        let scan = table.scan().as_of_time(entry.timestamp_ms).build().unwrap();
1265        assert_eq!(
1266            scan.plan_context.as_ref().unwrap().snapshot_schema,
1267            *table.metadata().current_schema()
1268        );
1269
1270        let mut metadata = table.metadata().clone();
1271        Arc::make_mut(metadata.snapshots.get_mut(&entry.snapshot_id).unwrap()).schema_id =
1272            Some(1234);
1273        let table = table.with_metadata(Arc::new(metadata));
1274        let err = table
1275            .scan()
1276            .as_of_time(entry.timestamp_ms)
1277            .build()
1278            .unwrap_err();
1279        assert_eq!(err.kind(), ErrorKind::DataInvalid);
1280        assert!(err.message().contains("Schema with id 1234 not found"));
1281    }
1282
1283    #[tokio::test]
1284    async fn test_table_scan_as_of_time_reads_historical_rows() {
1285        let mut fixture = TableTestFixture::new();
1286        fixture.setup_manifest_files().await;
1287        let entry = fixture.table.metadata().history()[0].clone();
1288
1289        // The older snapshot has only x, while the current snapshot has eight columns.
1290        let mut metadata = fixture.table.metadata().clone();
1291        Arc::make_mut(metadata.snapshots.get_mut(&entry.snapshot_id).unwrap()).schema_id = Some(0);
1292        fixture.table = fixture.table.with_metadata(Arc::new(metadata));
1293        let snapshot = fixture
1294            .table
1295            .metadata()
1296            .snapshot_by_id(entry.snapshot_id)
1297            .unwrap();
1298        let schema = snapshot.schema(fixture.table.metadata()).unwrap();
1299        let arrow_schema = Arc::new(crate::arrow::schema_to_arrow_schema(&schema).unwrap());
1300
1301        // Write distinct historical data: x = 100, versus x = 1 in the current files.
1302        let path = format!("{}/historical.parquet", fixture.table_location);
1303        let batch = RecordBatch::try_new(arrow_schema.clone(), vec![Arc::new(Int64Array::from(
1304            vec![100],
1305        ))])
1306        .unwrap();
1307        let mut writer =
1308            ArrowWriter::try_new(File::create(&path).unwrap(), arrow_schema, None).unwrap();
1309        writer.write(&batch).unwrap();
1310        writer.close().unwrap();
1311
1312        let mut manifest_writer = ManifestWriterBuilder::new(
1313            fixture.next_manifest_file(),
1314            Some(snapshot.snapshot_id()),
1315            schema,
1316            fixture
1317                .table
1318                .metadata()
1319                .default_partition_spec()
1320                .as_ref()
1321                .clone(),
1322        )
1323        .build_v2_data();
1324        manifest_writer
1325            .add_entry(
1326                ManifestEntry::builder()
1327                    .status(ManifestStatus::Added)
1328                    .data_file(
1329                        DataFileBuilder::default()
1330                            .partition_spec_id(0)
1331                            .content(DataContentType::Data)
1332                            .file_format(DataFileFormat::Parquet)
1333                            .file_size_in_bytes(fs::metadata(&path).unwrap().len())
1334                            .file_path(path)
1335                            .record_count(1)
1336                            .partition(Struct::from_iter([Some(Literal::long(100))]))
1337                            .build()
1338                            .unwrap(),
1339                    )
1340                    .build(),
1341            )
1342            .unwrap();
1343        let manifest = manifest_writer.write_manifest_file().await.unwrap();
1344        let output = fixture
1345            .table
1346            .file_io()
1347            .new_output(snapshot.manifest_list())
1348            .unwrap()
1349            .writer()
1350            .await
1351            .unwrap();
1352        let mut list_writer = ManifestListWriter::v2(
1353            output,
1354            snapshot.snapshot_id(),
1355            snapshot.parent_snapshot_id(),
1356            snapshot.sequence_number(),
1357        );
1358        list_writer.add_manifests([manifest].into_iter()).unwrap();
1359        list_writer.close().await.unwrap();
1360
1361        let table = fixture.table;
1362        let [by_time, by_id] = [
1363            table.scan().as_of_time(entry.timestamp_ms),
1364            table.scan().snapshot_id(entry.snapshot_id),
1365        ]
1366        .map(|scan| {
1367            scan.with_filter(Reference::new("x").greater_than_or_equal_to(Datum::long(100)))
1368                .build()
1369                .unwrap()
1370        });
1371        let current = table.scan().build().unwrap();
1372        let mut planned_files = Vec::new();
1373        let mut rows = Vec::new();
1374        let mut schemas = Vec::new();
1375        for scan in [by_time, by_id, current] {
1376            let mut tasks: Vec<_> = scan
1377                .plan_files()
1378                .await
1379                .unwrap()
1380                .try_collect()
1381                .await
1382                .unwrap();
1383            tasks.sort_by_key(|task| task.data_file_path().to_string());
1384            planned_files.push(tasks);
1385            let batches: Vec<_> = scan
1386                .to_arrow(ArrowReaderOptions::default())
1387                .await
1388                .unwrap()
1389                .try_collect()
1390                .await
1391                .unwrap();
1392            assert!(!batches.is_empty());
1393            schemas.push(batches[0].schema());
1394            let mut values: Vec<i64> = batches
1395                .iter()
1396                .flat_map(|batch| {
1397                    batch
1398                        .column_by_name("x")
1399                        .unwrap()
1400                        .as_any()
1401                        .downcast_ref::<Int64Array>()
1402                        .unwrap()
1403                        .values()
1404                        .iter()
1405                        .copied()
1406                })
1407                .collect();
1408            values.sort_unstable();
1409            rows.push(values);
1410        }
1411        assert!(!planned_files[0].is_empty());
1412        assert_eq!(planned_files[0], planned_files[1]);
1413        assert_eq!(rows[0], vec![100]);
1414        assert_eq!(rows[0], rows[1]);
1415        // The current snapshot contains two live fixture files.
1416        assert_eq!(rows[2], vec![1; 2 * ROWS_PER_FIXTURE_FILE]);
1417        assert_eq!(schemas[0], schemas[1]);
1418        assert_eq!(schemas[0].fields().len(), 1);
1419        assert_eq!(schemas[2].fields().len(), 8);
1420    }
1421
1422    /// A scan of the current state reads the current snapshot with the table's
1423    /// current schema, not the older one that snapshot was written with.
1424    #[test]
1425    fn test_scan_of_current_state_uses_current_schema() {
1426        let table = TableTestFixture::new_with_column_added_after_last_write().table;
1427
1428        let table_scan = table.scan().build().unwrap();
1429        assert_eq!(table_scan.snapshot(), table.metadata().current_snapshot());
1430        assert_eq!(
1431            table_scan.plan_context.as_ref().unwrap().snapshot_schema,
1432            *table.metadata().current_schema()
1433        );
1434    }
1435
1436    /// A scan pinned to a snapshot uses the schema that snapshot was written
1437    /// with, even when it is the current snapshot.
1438    #[test]
1439    fn test_time_travel_scan_uses_snapshot_schema() {
1440        let table = TableTestFixture::new_with_column_added_after_last_write().table;
1441        let snapshot = table.metadata().current_snapshot().unwrap();
1442
1443        let table_scan = table
1444            .scan()
1445            .snapshot_id(snapshot.snapshot_id())
1446            .build()
1447            .unwrap();
1448        assert_eq!(table_scan.snapshot(), Some(snapshot));
1449        assert_eq!(
1450            table_scan.plan_context.as_ref().unwrap().snapshot_schema,
1451            snapshot.schema(table.metadata()).unwrap()
1452        );
1453    }
1454
1455    fn table_with_property(key: &str, value: &str) -> Table {
1456        let fixture = TableTestFixture::new();
1457        let mut metadata = fixture.table.metadata().clone();
1458        metadata
1459            .properties
1460            .insert(key.to_string(), value.to_string());
1461        Table::builder()
1462            .metadata(metadata)
1463            .identifier(fixture.table.identifier().clone())
1464            .file_io(fixture.table.file_io().clone())
1465            .metadata_location(fixture.table.metadata_location().unwrap().to_string())
1466            .runtime(test_runtime())
1467            .build()
1468            .unwrap()
1469    }
1470
1471    #[test]
1472    fn test_table_scan_without_name_mapping_property() {
1473        let table = TableTestFixture::new().table;
1474
1475        let table_scan = table.scan().build().unwrap();
1476        assert!(
1477            table_scan
1478                .plan_context
1479                .as_ref()
1480                .unwrap()
1481                .name_mapping
1482                .is_none()
1483        );
1484    }
1485
1486    #[test]
1487    fn test_table_scan_with_name_mapping_property() {
1488        let mapping_json = r#"[{"field-id":1,"names":["id","record_id"]}]"#;
1489        let table =
1490            table_with_property(TableProperties::PROPERTY_DEFAULT_NAME_MAPPING, mapping_json);
1491
1492        let table_scan = table.scan().build().unwrap();
1493        let mapping = table_scan
1494            .plan_context
1495            .as_ref()
1496            .unwrap()
1497            .name_mapping
1498            .as_ref()
1499            .expect("name_mapping should be parsed from the table property");
1500        let fields = mapping.fields();
1501        assert_eq!(fields.len(), 1);
1502        assert_eq!(fields[0].field_id(), Some(1));
1503        assert_eq!(fields[0].names(), &[
1504            "id".to_string(),
1505            "record_id".to_string()
1506        ]);
1507    }
1508
1509    #[test]
1510    fn test_table_scan_with_malformed_name_mapping_property() {
1511        let table = table_with_property(
1512            TableProperties::PROPERTY_DEFAULT_NAME_MAPPING,
1513            "{ not valid json",
1514        );
1515
1516        let err = table
1517            .scan()
1518            .build()
1519            .expect_err("malformed name mapping should fail to parse");
1520        assert_eq!(err.kind(), ErrorKind::DataInvalid);
1521    }
1522
1523    #[tokio::test]
1524    async fn test_plan_files_carries_name_mapping_into_file_scan_task() {
1525        let mut fixture = TableTestFixture::new();
1526        fixture.setup_manifest_files().await;
1527
1528        let mapping_json = r#"[{"field-id":1,"names":["id","record_id"]}]"#;
1529        let mut metadata = fixture.table.metadata().clone();
1530        metadata.properties.insert(
1531            TableProperties::PROPERTY_DEFAULT_NAME_MAPPING.to_string(),
1532            mapping_json.to_string(),
1533        );
1534        let table = Table::builder()
1535            .metadata(metadata)
1536            .identifier(fixture.table.identifier().clone())
1537            .file_io(fixture.table.file_io().clone())
1538            .metadata_location(fixture.table.metadata_location().unwrap().to_string())
1539            .runtime(test_runtime())
1540            .build()
1541            .unwrap();
1542
1543        let tasks: Vec<_> = table
1544            .scan()
1545            .build()
1546            .unwrap()
1547            .plan_files()
1548            .await
1549            .unwrap()
1550            .try_collect()
1551            .await
1552            .unwrap();
1553
1554        assert!(!tasks.is_empty(), "expected at least one FileScanTask");
1555        for task in &tasks {
1556            let mapping = task
1557                .name_mapping()
1558                .expect("name_mapping should reach the FileScanTask");
1559            assert_eq!(mapping.fields().len(), 1);
1560            assert_eq!(mapping.fields()[0].field_id(), Some(1));
1561        }
1562    }
1563
1564    #[tokio::test]
1565    async fn test_plan_files_carries_sort_order_into_file_scan_task() {
1566        let mut fixture = TableTestFixture::new();
1567
1568        // Inject the reserved unsorted order (id 0) inline rather than editing the shared
1569        // testdata fixture, so the id-0 file below exercises the `!is_unsorted()` filter
1570        // branch instead of the `and_then` short-circuit an absent entry would take.
1571        let mut metadata = fixture.table.metadata().clone();
1572        metadata
1573            .sort_orders
1574            .insert(0, Arc::new(SortOrder::unsorted_order()));
1575        fixture.table = fixture.table.with_metadata(Arc::new(metadata));
1576
1577        let expected_sort_order = fixture
1578            .table
1579            .metadata()
1580            .sort_order_by_id(3)
1581            .unwrap()
1582            .clone();
1583
1584        // sort_order_ids: resolvable (3), absent, unresolvable (99), reserved unsorted (0).
1585        fixture
1586            .setup_manifest_files_with_sort_order_ids([Some(3), None, Some(99), Some(0)])
1587            .await;
1588
1589        let tasks: Vec<_> = fixture
1590            .table
1591            .scan()
1592            .build()
1593            .unwrap()
1594            .plan_files()
1595            .await
1596            .unwrap()
1597            .try_collect()
1598            .await
1599            .unwrap();
1600
1601        assert_eq!(tasks.len(), 4, "expected all four FileScanTasks");
1602
1603        // Aggregates catch a systemic regression (every entry resolving to id 3, or
1604        // resolution dropping entirely) that the per-file checks below would each still pass.
1605        assert_eq!(
1606            tasks.iter().filter(|t| t.sort_order().is_some()).count(),
1607            1,
1608            "exactly one file resolves to a sort order"
1609        );
1610        assert_eq!(
1611            tasks.iter().filter(|t| t.sort_order_id().is_some()).count(),
1612            3,
1613            "three files carry a raw sort_order_id"
1614        );
1615
1616        let resolved = tasks
1617            .iter()
1618            .find(|t| t.data_file_path().ends_with("1.parquet"))
1619            .unwrap();
1620        assert_eq!(resolved.sort_order_id(), Some(3));
1621        assert_eq!(
1622            resolved.sort_order(),
1623            Some(&expected_sort_order),
1624            "sort_order_id 3 should resolve to the table's sort order at id 3"
1625        );
1626
1627        let missing = tasks
1628            .iter()
1629            .find(|t| t.data_file_path().ends_with("2.parquet"))
1630            .unwrap();
1631        assert_eq!(missing.sort_order_id(), None);
1632        assert!(
1633            missing.sort_order().is_none(),
1634            "a file with no sort_order_id carries no sort_order"
1635        );
1636
1637        let unresolvable = tasks
1638            .iter()
1639            .find(|t| t.data_file_path().ends_with("3.parquet"))
1640            .unwrap();
1641        assert_eq!(
1642            unresolvable.sort_order_id(),
1643            Some(99),
1644            "the raw id is preserved even when it does not resolve"
1645        );
1646        assert!(
1647            unresolvable.sort_order().is_none(),
1648            "an unresolvable sort_order_id resolves to no sort_order"
1649        );
1650
1651        let unsorted = tasks
1652            .iter()
1653            .find(|t| t.data_file_path().ends_with("4.parquet"))
1654            .unwrap();
1655        assert_eq!(unsorted.sort_order_id(), Some(0));
1656        assert!(
1657            unsorted.sort_order().is_none(),
1658            "the reserved unsorted order (id 0) resolves to no sort_order"
1659        );
1660    }
1661
1662    /// Statistics for long columns `field_ids`, with values derived from each id so that a
1663    /// column's statistics do not depend on which other columns are present.
1664    fn column_stats(field_ids: &[i32]) -> FileScanTaskColumnStats {
1665        let counts = |count: fn(u64) -> u64| -> HashMap<i32, u64> {
1666            field_ids.iter().map(|&id| (id, count(id as u64))).collect()
1667        };
1668        let bounds = |offset: i64| -> HashMap<i32, Datum> {
1669            field_ids
1670                .iter()
1671                .map(|&id| (id, Datum::long(offset + i64::from(id))))
1672                .collect()
1673        };
1674        FileScanTaskColumnStats::builder()
1675            .with_column_sizes(counts(|id| 8 * id))
1676            .with_value_counts(counts(|id| 10 + id))
1677            .with_null_value_counts(counts(|id| id))
1678            .with_nan_value_counts(counts(|id| 2 * id))
1679            .with_lower_bounds(bounds(0))
1680            .with_upper_bounds(bounds(100))
1681            .build()
1682    }
1683
1684    async fn column_stats_by_file(scan: TableScan) -> Vec<Option<FileScanTaskColumnStats>> {
1685        let mut tasks: Vec<_> = scan
1686            .plan_files()
1687            .await
1688            .unwrap()
1689            .try_collect()
1690            .await
1691            .unwrap();
1692        tasks.sort_by(|a, b| a.data_file_path().cmp(b.data_file_path()));
1693        tasks
1694            .into_iter()
1695            .map(|task| task.column_stats().cloned())
1696            .collect()
1697    }
1698
1699    #[tokio::test]
1700    async fn test_plan_files_retains_only_requested_column_stats() {
1701        let mut fixture = TableTestFixture::new();
1702        // The second file has no statistics for `x` (field id 1).
1703        fixture
1704            .setup_manifest_files_with_column_stats(&[(None, vec![
1705                column_stats(&[1, 2, 7]),
1706                column_stats(&[2, 7]),
1707            ])])
1708            .await;
1709
1710        let scan = fixture
1711            .table
1712            .scan()
1713            .with_column_stats(["x", "i64"])
1714            .build()
1715            .unwrap();
1716
1717        assert_eq!(column_stats_by_file(scan).await, vec![
1718            Some(column_stats(&[1, 7])),
1719            Some(column_stats(&[7]))
1720        ]);
1721    }
1722
1723    #[tokio::test]
1724    async fn test_plan_files_carries_column_stats_only_when_requested() {
1725        let mut fixture = TableTestFixture::new();
1726        fixture
1727            .setup_manifest_files_with_column_stats(&[(None, vec![column_stats(&[1])])])
1728            .await;
1729        let table = &fixture.table;
1730
1731        assert_eq!(
1732            column_stats_by_file(table.scan().build().unwrap()).await,
1733            vec![None]
1734        );
1735        assert_eq!(
1736            column_stats_by_file(
1737                table
1738                    .scan()
1739                    .with_column_stats(Vec::<String>::new())
1740                    .build()
1741                    .unwrap()
1742            )
1743            .await,
1744            vec![Some(FileScanTaskColumnStats::default())]
1745        );
1746    }
1747
1748    #[tokio::test]
1749    async fn test_plan_files_resolves_column_stats_names_with_scan_case_sensitivity() {
1750        let mut fixture = TableTestFixture::new();
1751        fixture
1752            .setup_manifest_files_with_column_stats(&[(None, vec![column_stats(&[1, 2])])])
1753            .await;
1754
1755        let scan = fixture
1756            .table
1757            .scan()
1758            .with_case_sensitive(false)
1759            .with_column_stats(["X"])
1760            .build()
1761            .unwrap();
1762
1763        assert_eq!(column_stats_by_file(scan).await, vec![Some(column_stats(
1764            &[1]
1765        ))]);
1766    }
1767
1768    #[test]
1769    fn test_build_rejects_unknown_column_stats_name() {
1770        let table = TableTestFixture::new().table;
1771
1772        let err = table
1773            .scan()
1774            .with_column_stats(["unknown"])
1775            .build()
1776            .unwrap_err();
1777
1778        assert_eq!(err.kind(), ErrorKind::DataInvalid);
1779    }
1780
1781    #[tokio::test]
1782    async fn test_plan_files_promotes_bounds_to_scan_schema_type() {
1783        let mut fixture = TableTestFixture::new();
1784        let y_bounds = |lower: Datum, upper: Datum| {
1785            FileScanTaskColumnStats::builder()
1786                .with_lower_bounds(HashMap::from([(2, lower)]))
1787                .with_upper_bounds(HashMap::from([(2, upper)]))
1788                .build()
1789        };
1790        // `y` is a `long` in the table schema. The first manifest was written when it was an `int`.
1791        let int_schema = Arc::new(
1792            Schema::builder()
1793                .with_fields(vec![
1794                    NestedField::required(1, "x", PrimitiveType::Long).into(),
1795                    NestedField::required(2, "y", PrimitiveType::Int).into(),
1796                ])
1797                .build()
1798                .unwrap(),
1799        );
1800        let written_as_long = y_bounds(Datum::long(6), Datum::long(9));
1801        fixture
1802            .setup_manifest_files_with_column_stats(&[
1803                (Some(int_schema), vec![y_bounds(
1804                    Datum::int(5),
1805                    Datum::int(7),
1806                )]),
1807                (None, vec![written_as_long.clone()]),
1808            ])
1809            .await;
1810
1811        let scan = fixture
1812            .table
1813            .scan()
1814            .with_column_stats(["y"])
1815            .build()
1816            .unwrap();
1817
1818        assert_eq!(column_stats_by_file(scan).await, vec![
1819            Some(y_bounds(Datum::long(5), Datum::long(7))),
1820            Some(written_as_long)
1821        ]);
1822    }
1823
1824    #[tokio::test]
1825    async fn test_plan_files_on_table_without_any_snapshots() {
1826        let table = TableTestFixture::new_empty().table;
1827        let batch_stream = table
1828            .scan()
1829            .build()
1830            .unwrap()
1831            .to_arrow(ArrowReaderOptions::default())
1832            .await
1833            .unwrap();
1834        let batches: Vec<_> = batch_stream.try_collect().await.unwrap();
1835        assert!(batches.is_empty());
1836    }
1837
1838    #[tokio::test]
1839    async fn test_plan_files_no_deletions() {
1840        let mut fixture = TableTestFixture::new();
1841        fixture.setup_manifest_files().await;
1842
1843        // Create table scan for current snapshot and plan files
1844        let table_scan = fixture.table.scan().build().unwrap();
1845
1846        let mut tasks = table_scan
1847            .plan_files()
1848            .await
1849            .unwrap()
1850            .try_fold(vec![], |mut acc, task| async move {
1851                acc.push(task);
1852                Ok(acc)
1853            })
1854            .await
1855            .unwrap();
1856
1857        assert_eq!(tasks.len(), 2);
1858
1859        tasks.sort_by_key(|t| t.data_file_path().to_string());
1860
1861        // Check first task is added data file
1862        assert_eq!(
1863            tasks[0].data_file_path(),
1864            format!("{}/1.parquet", &fixture.table_location)
1865        );
1866
1867        // Check second task is existing data file
1868        assert_eq!(
1869            tasks[1].data_file_path(),
1870            format!("{}/3.parquet", &fixture.table_location)
1871        );
1872    }
1873
1874    #[tokio::test]
1875    async fn test_scan_widens_old_partition_so_pruning_and_deletes_match() {
1876        // Guards the reader-side normalization through a real scan: a data manifest
1877        // written when the partition source `x` was `int` must have its `int(7)`
1878        // partition widened to `long(7)` during planning, so (a) a `x = 7` (long)
1879        // filter keeps the file and (b) the equality delete keyed by `long(7)`
1880        // attaches. Without the normalization the int/long mismatch drops the delete
1881        // (and prunes the file under the filter).
1882        let mut fixture = TableTestFixture::new();
1883        fixture.setup_promoted_partition_with_eq_delete(false).await;
1884
1885        let tasks: Vec<FileScanTask> = fixture
1886            .table
1887            .scan()
1888            .build()
1889            .unwrap()
1890            .plan_files()
1891            .await
1892            .unwrap()
1893            .try_collect()
1894            .await
1895            .unwrap();
1896        assert_eq!(tasks.len(), 1);
1897        assert_eq!(
1898            tasks[0].deletes().len(),
1899            1,
1900            "equality delete must attach after widening the old int partition to long"
1901        );
1902        assert_eq!(
1903            tasks[0].partition(),
1904            Some(&Struct::from_iter([Some(Literal::long(7))])),
1905            "the widened long(7) tuple must reach FileScanTask::partition()"
1906        );
1907
1908        let tasks: Vec<FileScanTask> = fixture
1909            .table
1910            .scan()
1911            .with_filter(Reference::new("x").equal_to(Datum::long(7)))
1912            .build()
1913            .unwrap()
1914            .plan_files()
1915            .await
1916            .unwrap()
1917            .try_collect()
1918            .await
1919            .unwrap();
1920        assert_eq!(
1921            tasks.len(),
1922            1,
1923            "filter on the promoted type must not prune the file"
1924        );
1925        assert_eq!(tasks[0].deletes().len(), 1);
1926
1927        let tasks: Vec<FileScanTask> = fixture
1928            .table
1929            .scan()
1930            .with_filter(Reference::new("x").equal_to(Datum::long(8)))
1931            .build()
1932            .unwrap()
1933            .plan_files()
1934            .await
1935            .unwrap()
1936            .try_collect()
1937            .await
1938            .unwrap();
1939        assert!(
1940            tasks.is_empty(),
1941            "a file in partition x=7 must be pruned by an x=8 filter"
1942        );
1943    }
1944
1945    #[tokio::test]
1946    async fn test_scan_widens_old_partition_with_pre_promotion_delete() {
1947        // Data file and equality delete both written before the promotion (int
1948        // partition). They matched on main; after both sides widen to long they must
1949        // still match, so the delete stays attached.
1950        let mut fixture = TableTestFixture::new();
1951        fixture.setup_promoted_partition_with_eq_delete(true).await;
1952
1953        let tasks: Vec<FileScanTask> = fixture
1954            .table
1955            .scan()
1956            .build()
1957            .unwrap()
1958            .plan_files()
1959            .await
1960            .unwrap()
1961            .try_collect()
1962            .await
1963            .unwrap();
1964        assert_eq!(tasks.len(), 1);
1965        assert_eq!(tasks[0].deletes().len(), 1);
1966    }
1967
1968    #[tokio::test]
1969    async fn test_scan_widens_old_partition_so_deletion_vector_matches() {
1970        // A v3 deletion vector written after the promotion (long partition) references
1971        // an older data file (int partition). The widening must make the partitions
1972        // agree so the DV attaches, instead of failing the DV partition check.
1973        let mut fixture = TableTestFixture::new();
1974        fixture.setup_promoted_partition_with_dv().await;
1975
1976        let tasks: Vec<FileScanTask> = fixture
1977            .table
1978            .scan()
1979            .build()
1980            .unwrap()
1981            .plan_files()
1982            .await
1983            .unwrap()
1984            .try_collect()
1985            .await
1986            .unwrap();
1987        assert_eq!(tasks.len(), 1);
1988        assert_eq!(tasks[0].deletes().len(), 1);
1989    }
1990
1991    #[tokio::test]
1992    async fn test_scan_keeps_eq_delete_after_dropping_promoted_source() {
1993        // Partition source `x` promoted int -> long, then dropped. The data file (int)
1994        // and its equality delete (long) must still normalize to the same long partition
1995        // so the delete stays attached; otherwise it silently detaches.
1996        let mut fixture = TableTestFixture::new();
1997        fixture.setup_promoted_partition_with_eq_delete(false).await;
1998        fixture.drop_partition_source(1);
1999        let table = fixture.table;
2000
2001        let tasks: Vec<FileScanTask> = table
2002            .scan()
2003            .build()
2004            .unwrap()
2005            .plan_files()
2006            .await
2007            .unwrap()
2008            .try_collect()
2009            .await
2010            .unwrap();
2011        assert_eq!(tasks.len(), 1);
2012        assert_eq!(tasks[0].deletes().len(), 1);
2013    }
2014
2015    #[tokio::test]
2016    async fn test_scan_keeps_deletion_vector_after_dropping_promoted_source() {
2017        // Same as above for a deletion vector: without widening the dropped numeric
2018        // source the DV partition check errors (long(7) vs int(7)).
2019        let mut fixture = TableTestFixture::new();
2020        fixture.setup_promoted_partition_with_dv().await;
2021        fixture.drop_partition_source(1);
2022        let table = fixture.table;
2023
2024        let tasks: Vec<FileScanTask> = table
2025            .scan()
2026            .build()
2027            .unwrap()
2028            .plan_files()
2029            .await
2030            .unwrap()
2031            .try_collect()
2032            .await
2033            .unwrap();
2034        assert_eq!(tasks.len(), 1);
2035        assert_eq!(tasks[0].deletes().len(), 1);
2036    }
2037
2038    #[tokio::test]
2039    async fn test_scan_keeps_eq_delete_when_dropped_source_is_date() {
2040        // A promoted numeric source `x` sits beside a dropped `date` source `d`. The gate
2041        // fires on `x`'s int -> long widening, but `d` (stored as an int) must be copied
2042        // through, not widened to long, or the data file's partition diverges from the
2043        // equality delete's and detaches it.
2044        let mut fixture = TableTestFixture::new();
2045        fixture
2046            .setup_promoted_partition_with_eq_delete_and_dropped_date()
2047            .await;
2048
2049        let tasks: Vec<FileScanTask> = fixture
2050            .table
2051            .scan()
2052            .build()
2053            .unwrap()
2054            .plan_files()
2055            .await
2056            .unwrap()
2057            .try_collect()
2058            .await
2059            .unwrap();
2060        assert_eq!(tasks.len(), 1);
2061        assert_eq!(tasks[0].deletes().len(), 1);
2062    }
2063
2064    #[tokio::test]
2065    async fn test_plan_files_carries_row_lineage_into_file_scan_task() {
2066        let mut fixture = TableTestFixture::new();
2067        fixture.setup_manifest_files().await;
2068
2069        let mut tasks: Vec<_> = fixture
2070            .table
2071            .scan()
2072            .build()
2073            .unwrap()
2074            .plan_files()
2075            .await
2076            .unwrap()
2077            .try_collect()
2078            .await
2079            .unwrap();
2080
2081        assert_eq!(tasks.len(), 2);
2082        tasks.sort_by_key(|task| task.data_file_path().to_string());
2083
2084        // The added file inherits the current snapshot's data sequence number,
2085        // the existing file keeps the one it was written with.
2086        assert_eq!(
2087            tasks[0].data_file_path(),
2088            format!("{}/1.parquet", &fixture.table_location)
2089        );
2090        assert_eq!(tasks[0].data_sequence_number(), Some(1));
2091        assert_eq!(
2092            tasks[1].data_file_path(),
2093            format!("{}/3.parquet", &fixture.table_location)
2094        );
2095        assert_eq!(tasks[1].data_sequence_number(), Some(0));
2096
2097        // first_row_id is a v3 concept; a v2 manifest carries none.
2098        assert!(tasks.iter().all(|task| task.first_row_id().is_none()));
2099    }
2100
2101    #[tokio::test]
2102    async fn test_plan_files_carries_row_lineage_from_v3_manifest() {
2103        let mut fixture = TableTestFixture::new();
2104        fixture.setup_v3_manifest_files().await;
2105
2106        let task = fixture
2107            .table
2108            .scan()
2109            .build()
2110            .unwrap()
2111            .plan_files()
2112            .await
2113            .unwrap()
2114            .try_collect::<Vec<_>>()
2115            .await
2116            .unwrap()
2117            .into_iter()
2118            .next()
2119            .expect("expected one FileScanTask");
2120
2121        // The manifest-level first_row_id (42) is inherited onto the entry on
2122        // read, then carried onto the task.
2123        assert_eq!(task.first_row_id(), Some(42));
2124        // The data sequence number is threaded through the same v3 read path.
2125        assert_eq!(task.data_sequence_number(), Some(1));
2126    }
2127
2128    #[tokio::test]
2129    async fn test_filtered_scan_with_dropped_partition_source_column() {
2130        let mut fixture = TableTestFixture::new();
2131        fixture.setup_manifest_files().await;
2132
2133        // Baseline: the same filtered scan against the table before evolution.
2134        let baseline = scan_y_gte_5(&fixture.table).await;
2135        assert!(!baseline.is_empty());
2136        assert!(baseline.iter().all(|y| *y >= 5));
2137
2138        // Evolve the table so that the manifests reference a historical spec whose source
2139        // column is no longer in the current schema: make an unpartitioned spec the
2140        // default, then drop the original spec's source column from the schema.
2141        let current_schema = fixture.table.metadata().current_schema();
2142        let evolved_schema = Schema::builder()
2143            .with_fields(
2144                current_schema
2145                    .as_struct()
2146                    .fields()
2147                    .iter()
2148                    .filter(|field| field.id != 1)
2149                    .cloned(),
2150            )
2151            .with_identifier_field_ids(vec![2])
2152            .build()
2153            .unwrap();
2154        let evolved =
2155            TableMetadataBuilder::new_from_metadata(fixture.table.metadata().clone(), None)
2156                .add_default_partition_spec(UnboundPartitionSpec::builder().build())
2157                .unwrap()
2158                .add_current_schema(evolved_schema)
2159                .unwrap()
2160                .build()
2161                .unwrap()
2162                .metadata;
2163
2164        // a commit after the evolution carries the previous manifests forward: the new
2165        // snapshot uses the evolved schema while its manifests still use historical spec 0
2166        let parent = evolved.current_snapshot().unwrap().clone();
2167        let snapshot = Snapshot::builder()
2168            .with_snapshot_id(parent.snapshot_id() + 1)
2169            .with_parent_snapshot_id(Some(parent.snapshot_id()))
2170            .with_sequence_number(evolved.last_sequence_number() + 1)
2171            .with_timestamp_ms(evolved.last_updated_ms + 1)
2172            .with_schema_id(evolved.current_schema_id())
2173            .with_manifest_list(parent.manifest_list())
2174            .with_summary(Summary {
2175                operation: Operation::Append,
2176                additional_properties: HashMap::new(),
2177            })
2178            .build();
2179        let metadata = TableMetadataBuilder::new_from_metadata(evolved, None)
2180            .set_branch_snapshot(snapshot, MAIN_BRANCH)
2181            .unwrap()
2182            .build()
2183            .unwrap()
2184            .metadata;
2185        let table = fixture.table.clone().with_metadata(Arc::new(metadata));
2186
2187        // Planning and reading must succeed, and the results must match the table before
2188        // evolution: no rows wrongly pruned and none returned unfiltered.
2189        let evolved = scan_y_gte_5(&table).await;
2190        assert_eq!(evolved, baseline);
2191    }
2192
2193    async fn scan_y_gte_5(table: &Table) -> Vec<i64> {
2194        let table_scan = table
2195            .scan()
2196            .select(["y"])
2197            .with_filter(Reference::new("y").greater_than_or_equal_to(Datum::long(5)))
2198            .build()
2199            .unwrap();
2200        let batches: Vec<_> = table_scan
2201            .to_arrow(ArrowReaderOptions::default())
2202            .await
2203            .unwrap()
2204            .try_collect()
2205            .await
2206            .unwrap();
2207
2208        let mut values: Vec<i64> = batches
2209            .iter()
2210            .flat_map(|batch| {
2211                let col = batch.column_by_name("y").unwrap();
2212                let arr = col.as_any().downcast_ref::<Int64Array>().unwrap();
2213                (0..arr.len()).map(|i| arr.value(i)).collect::<Vec<_>>()
2214            })
2215            .collect();
2216        values.sort_unstable();
2217        values
2218    }
2219
2220    #[tokio::test]
2221    async fn test_open_parquet_no_deletions() {
2222        let mut fixture = TableTestFixture::new();
2223        fixture.setup_manifest_files().await;
2224
2225        // Create table scan for current snapshot and plan files
2226        let table_scan = fixture.table.scan().build().unwrap();
2227
2228        let batch_stream = table_scan
2229            .to_arrow(
2230                ArrowReaderOptions::builder()
2231                    .with_row_selection_enabled(true)
2232                    .build(),
2233            )
2234            .await
2235            .unwrap();
2236
2237        let batches: Vec<_> = batch_stream.try_collect().await.unwrap();
2238
2239        let col = batches[0].column_by_name("x").unwrap();
2240
2241        let int64_arr = col.as_any().downcast_ref::<Int64Array>().unwrap();
2242        assert_eq!(int64_arr.value(0), 1);
2243    }
2244
2245    #[tokio::test]
2246    async fn reader_options_preserve_reads_after_dropping_scan() {
2247        let mut fixture = TableTestFixture::new();
2248        fixture.setup_multi_row_group_manifest(&[150, 299]).await;
2249        let scan = fixture
2250            .table
2251            .scan()
2252            .select(["y"])
2253            .with_filter(Reference::new("y").greater_than(Datum::long(1100)))
2254            .build()
2255            .unwrap();
2256        let options = ArrowReaderOptions::builder()
2257            .with_batch_size(17)
2258            .with_data_file_concurrency_limit(1)
2259            .with_parquet_options(
2260                ParquetReadOptions::builder()
2261                    .with_metadata_size_hint(4096)
2262                    .with_range_fetch_concurrency(2)
2263                    .build(),
2264            )
2265            .with_row_selection_enabled(true)
2266            .with_bloom_filter_enabled(true)
2267            .build();
2268        let expected: Vec<_> = scan
2269            .to_arrow(options.clone())
2270            .await
2271            .unwrap()
2272            .try_collect()
2273            .await
2274            .unwrap();
2275        let tasks: Vec<_> = scan
2276            .plan_files()
2277            .await
2278            .unwrap()
2279            .try_collect()
2280            .await
2281            .unwrap();
2282        let serialized = serde_json::to_value(&options).unwrap();
2283        assert_eq!(
2284            serialized,
2285            serde_json::json!({
2286                "batch_size": 17, "data_file_concurrency_limit": 1,
2287                "row_group_filtering_enabled": true, "row_selection_enabled": true,
2288                "bloom_filter_enabled": true,
2289                "parquet_options": {
2290                    "metadata_size_hint": 4096, "range_coalesce_bytes": 1048576, "range_fetch_concurrency": 2,
2291                    "preload_column_index": true, "preload_offset_index": true, "preload_page_index": false,
2292                },
2293            })
2294        );
2295        let deserialized: ArrowReaderOptions = serde_json::from_value(serialized).unwrap();
2296        assert_eq!(deserialized, options);
2297        let options = deserialized;
2298        drop(scan);
2299        let reader =
2300            ArrowReaderBuilder::new(FileIO::new_with_fs(), Runtime::try_current().unwrap())
2301                .with_options(options)
2302                .build();
2303        let actual: Vec<_> = reader
2304            .read(Box::pin(stream::iter(tasks.into_iter().map(Ok))))
2305            .unwrap()
2306            .stream()
2307            .try_collect()
2308            .await
2309            .unwrap();
2310        assert!(!actual.is_empty());
2311        assert!(actual.iter().all(|batch| batch.num_rows() <= 17));
2312        assert_eq!(actual, expected);
2313    }
2314
2315    #[tokio::test]
2316    async fn test_open_parquet_no_deletions_by_separate_reader() {
2317        let mut fixture = TableTestFixture::new();
2318        fixture.setup_manifest_files().await;
2319
2320        // Create table scan for current snapshot and plan files
2321        let table_scan = fixture.table.scan().build().unwrap();
2322
2323        let mut plan_task: Vec<_> = table_scan
2324            .plan_files()
2325            .await
2326            .unwrap()
2327            .try_collect()
2328            .await
2329            .unwrap();
2330        assert_eq!(plan_task.len(), 2);
2331
2332        let reader = ArrowReaderBuilder::new(
2333            fixture.table.file_io().clone(),
2334            fixture.table.runtime().clone(),
2335        )
2336        .build();
2337        let batch_stream = reader
2338            .clone()
2339            .read(Box::pin(stream::iter(vec![Ok(plan_task.remove(0))])))
2340            .unwrap()
2341            .stream();
2342        let batch_1: Vec<_> = batch_stream.try_collect().await.unwrap();
2343
2344        let reader = ArrowReaderBuilder::new(
2345            fixture.table.file_io().clone(),
2346            fixture.table.runtime().clone(),
2347        )
2348        .build();
2349        let batch_stream = reader
2350            .read(Box::pin(stream::iter(vec![Ok(plan_task.remove(0))])))
2351            .unwrap()
2352            .stream();
2353        let batch_2: Vec<_> = batch_stream.try_collect().await.unwrap();
2354
2355        assert_eq!(batch_1, batch_2);
2356    }
2357
2358    #[tokio::test]
2359    async fn test_open_parquet_with_projection() {
2360        let mut fixture = TableTestFixture::new();
2361        fixture.setup_manifest_files().await;
2362
2363        // Create table scan for current snapshot and plan files
2364        let table_scan = fixture.table.scan().select(["x", "z"]).build().unwrap();
2365
2366        let batch_stream = table_scan
2367            .to_arrow(
2368                ArrowReaderOptions::builder()
2369                    .with_row_selection_enabled(true)
2370                    .build(),
2371            )
2372            .await
2373            .unwrap();
2374
2375        let batches: Vec<_> = batch_stream.try_collect().await.unwrap();
2376
2377        assert_eq!(batches[0].num_columns(), 2);
2378
2379        let col1 = batches[0].column_by_name("x").unwrap();
2380        let int64_arr = col1.as_any().downcast_ref::<Int64Array>().unwrap();
2381        assert_eq!(int64_arr.value(0), 1);
2382
2383        let col2 = batches[0].column_by_name("z").unwrap();
2384        let int64_arr = col2.as_any().downcast_ref::<Int64Array>().unwrap();
2385        assert_eq!(int64_arr.value(0), 3);
2386
2387        // test empty scan
2388        let table_scan = fixture.table.scan().select_empty().build().unwrap();
2389        let batch_stream = table_scan
2390            .to_arrow(
2391                ArrowReaderOptions::builder()
2392                    .with_row_selection_enabled(true)
2393                    .build(),
2394            )
2395            .await
2396            .unwrap();
2397        let batches: Vec<_> = batch_stream.try_collect().await.unwrap();
2398
2399        assert_eq!(batches[0].num_columns(), 0);
2400        assert_eq!(batches[0].num_rows(), 1024);
2401    }
2402
2403    #[tokio::test]
2404    async fn test_filter_on_arrow_lt() {
2405        let mut fixture = TableTestFixture::new();
2406        fixture.setup_manifest_files().await;
2407
2408        // Filter: y < 3
2409        let mut builder = fixture.table.scan();
2410        let predicate = Reference::new("y").less_than(Datum::long(3));
2411        builder = builder.with_filter(predicate);
2412        let table_scan = builder.build().unwrap();
2413
2414        let batch_stream = table_scan
2415            .to_arrow(
2416                ArrowReaderOptions::builder()
2417                    .with_row_selection_enabled(true)
2418                    .build(),
2419            )
2420            .await
2421            .unwrap();
2422
2423        let batches: Vec<_> = batch_stream.try_collect().await.unwrap();
2424
2425        assert_eq!(batches[0].num_rows(), 512);
2426
2427        let col = batches[0].column_by_name("x").unwrap();
2428        let int64_arr = col.as_any().downcast_ref::<Int64Array>().unwrap();
2429        assert_eq!(int64_arr.value(0), 1);
2430
2431        let col = batches[0].column_by_name("y").unwrap();
2432        let int64_arr = col.as_any().downcast_ref::<Int64Array>().unwrap();
2433        assert_eq!(int64_arr.value(0), 2);
2434    }
2435
2436    #[tokio::test]
2437    async fn test_filter_on_arrow_gt_eq() {
2438        let mut fixture = TableTestFixture::new();
2439        fixture.setup_manifest_files().await;
2440
2441        // Filter: y >= 5
2442        let mut builder = fixture.table.scan();
2443        let predicate = Reference::new("y").greater_than_or_equal_to(Datum::long(5));
2444        builder = builder.with_filter(predicate);
2445        let table_scan = builder.build().unwrap();
2446
2447        let batch_stream = table_scan
2448            .to_arrow(
2449                ArrowReaderOptions::builder()
2450                    .with_row_selection_enabled(true)
2451                    .build(),
2452            )
2453            .await
2454            .unwrap();
2455
2456        let batches: Vec<_> = batch_stream.try_collect().await.unwrap();
2457
2458        assert_eq!(batches[0].num_rows(), 12);
2459
2460        let col = batches[0].column_by_name("x").unwrap();
2461        let int64_arr = col.as_any().downcast_ref::<Int64Array>().unwrap();
2462        assert_eq!(int64_arr.value(0), 1);
2463
2464        let col = batches[0].column_by_name("y").unwrap();
2465        let int64_arr = col.as_any().downcast_ref::<Int64Array>().unwrap();
2466        assert_eq!(int64_arr.value(0), 5);
2467    }
2468
2469    #[tokio::test]
2470    async fn test_filter_double_eq() {
2471        let mut fixture = TableTestFixture::new();
2472        fixture.setup_manifest_files().await;
2473
2474        // Filter: dbl == 150.0
2475        let mut builder = fixture.table.scan();
2476        let predicate = Reference::new("dbl").equal_to(Datum::double(150.0f64));
2477        builder = builder.with_filter(predicate);
2478        let table_scan = builder.build().unwrap();
2479
2480        let batch_stream = table_scan
2481            .to_arrow(
2482                ArrowReaderOptions::builder()
2483                    .with_row_selection_enabled(true)
2484                    .build(),
2485            )
2486            .await
2487            .unwrap();
2488
2489        let batches: Vec<_> = batch_stream.try_collect().await.unwrap();
2490
2491        assert_eq!(batches.len(), 2);
2492        assert_eq!(batches[0].num_rows(), 12);
2493
2494        let col = batches[0].column_by_name("dbl").unwrap();
2495        let f64_arr = col.as_any().downcast_ref::<Float64Array>().unwrap();
2496        assert_eq!(f64_arr.value(1), 150.0f64);
2497    }
2498
2499    #[tokio::test]
2500    async fn test_filter_int_eq() {
2501        let mut fixture = TableTestFixture::new();
2502        fixture.setup_manifest_files().await;
2503
2504        // Filter: i32 == 150
2505        let mut builder = fixture.table.scan();
2506        let predicate = Reference::new("i32").equal_to(Datum::int(150i32));
2507        builder = builder.with_filter(predicate);
2508        let table_scan = builder.build().unwrap();
2509
2510        let batch_stream = table_scan
2511            .to_arrow(
2512                ArrowReaderOptions::builder()
2513                    .with_row_selection_enabled(true)
2514                    .build(),
2515            )
2516            .await
2517            .unwrap();
2518
2519        let batches: Vec<_> = batch_stream.try_collect().await.unwrap();
2520
2521        assert_eq!(batches.len(), 2);
2522        assert_eq!(batches[0].num_rows(), 12);
2523
2524        let col = batches[0].column_by_name("i32").unwrap();
2525        let i32_arr = col.as_any().downcast_ref::<Int32Array>().unwrap();
2526        assert_eq!(i32_arr.value(1), 150i32);
2527    }
2528
2529    #[tokio::test]
2530    async fn test_filter_long_eq() {
2531        let mut fixture = TableTestFixture::new();
2532        fixture.setup_manifest_files().await;
2533
2534        // Filter: i64 == 150
2535        let mut builder = fixture.table.scan();
2536        let predicate = Reference::new("i64").equal_to(Datum::long(150i64));
2537        builder = builder.with_filter(predicate);
2538        let table_scan = builder.build().unwrap();
2539
2540        let batch_stream = table_scan
2541            .to_arrow(
2542                ArrowReaderOptions::builder()
2543                    .with_row_selection_enabled(true)
2544                    .build(),
2545            )
2546            .await
2547            .unwrap();
2548
2549        let batches: Vec<_> = batch_stream.try_collect().await.unwrap();
2550
2551        assert_eq!(batches.len(), 2);
2552        assert_eq!(batches[0].num_rows(), 12);
2553
2554        let col = batches[0].column_by_name("i64").unwrap();
2555        let i64_arr = col.as_any().downcast_ref::<Int64Array>().unwrap();
2556        assert_eq!(i64_arr.value(1), 150i64);
2557    }
2558
2559    #[tokio::test]
2560    async fn test_filter_bool_eq() {
2561        let mut fixture = TableTestFixture::new();
2562        fixture.setup_manifest_files().await;
2563
2564        // Filter: bool == true
2565        let mut builder = fixture.table.scan();
2566        let predicate = Reference::new("bool").equal_to(Datum::bool(true));
2567        builder = builder.with_filter(predicate);
2568        let table_scan = builder.build().unwrap();
2569
2570        let batch_stream = table_scan
2571            .to_arrow(
2572                ArrowReaderOptions::builder()
2573                    .with_row_selection_enabled(true)
2574                    .build(),
2575            )
2576            .await
2577            .unwrap();
2578
2579        let batches: Vec<_> = batch_stream.try_collect().await.unwrap();
2580
2581        assert_eq!(batches.len(), 2);
2582        assert_eq!(batches[0].num_rows(), 512);
2583
2584        let col = batches[0].column_by_name("bool").unwrap();
2585        let bool_arr = col.as_any().downcast_ref::<BooleanArray>().unwrap();
2586        assert!(bool_arr.value(1));
2587    }
2588
2589    #[tokio::test]
2590    async fn test_filter_on_arrow_is_null() {
2591        let mut fixture = TableTestFixture::new();
2592        fixture.setup_manifest_files().await;
2593
2594        // Filter: y is null
2595        let mut builder = fixture.table.scan();
2596        let predicate = Reference::new("y").is_null();
2597        builder = builder.with_filter(predicate);
2598        let table_scan = builder.build().unwrap();
2599
2600        let batch_stream = table_scan
2601            .to_arrow(
2602                ArrowReaderOptions::builder()
2603                    .with_row_selection_enabled(true)
2604                    .build(),
2605            )
2606            .await
2607            .unwrap();
2608
2609        let batches: Vec<_> = batch_stream.try_collect().await.unwrap();
2610        assert_eq!(batches.len(), 0);
2611    }
2612
2613    #[tokio::test]
2614    async fn test_filter_on_arrow_is_not_null() {
2615        let mut fixture = TableTestFixture::new();
2616        fixture.setup_manifest_files().await;
2617
2618        // Filter: y is not null
2619        let mut builder = fixture.table.scan();
2620        let predicate = Reference::new("y").is_not_null();
2621        builder = builder.with_filter(predicate);
2622        let table_scan = builder.build().unwrap();
2623
2624        let batch_stream = table_scan
2625            .to_arrow(
2626                ArrowReaderOptions::builder()
2627                    .with_row_selection_enabled(true)
2628                    .build(),
2629            )
2630            .await
2631            .unwrap();
2632
2633        let batches: Vec<_> = batch_stream.try_collect().await.unwrap();
2634        assert_eq!(batches[0].num_rows(), 1024);
2635    }
2636
2637    #[tokio::test]
2638    async fn test_filter_on_arrow_lt_and_gt() {
2639        let mut fixture = TableTestFixture::new();
2640        fixture.setup_manifest_files().await;
2641
2642        // Filter: y < 5 AND z >= 4
2643        let mut builder = fixture.table.scan();
2644        let predicate = Reference::new("y")
2645            .less_than(Datum::long(5))
2646            .and(Reference::new("z").greater_than_or_equal_to(Datum::long(4)));
2647        builder = builder.with_filter(predicate);
2648        let table_scan = builder.build().unwrap();
2649
2650        let batch_stream = table_scan
2651            .to_arrow(
2652                ArrowReaderOptions::builder()
2653                    .with_row_selection_enabled(true)
2654                    .build(),
2655            )
2656            .await
2657            .unwrap();
2658
2659        let batches: Vec<_> = batch_stream.try_collect().await.unwrap();
2660        assert_eq!(batches[0].num_rows(), 500);
2661
2662        let col = batches[0].column_by_name("x").unwrap();
2663        let expected_x = Arc::new(Int64Array::from_iter_values(vec![1; 500])) as ArrayRef;
2664        assert_eq!(col, &expected_x);
2665
2666        let col = batches[0].column_by_name("y").unwrap();
2667        let mut values = vec![];
2668        values.append(vec![3; 200].as_mut());
2669        values.append(vec![4; 300].as_mut());
2670        let expected_y = Arc::new(Int64Array::from_iter_values(values)) as ArrayRef;
2671        assert_eq!(col, &expected_y);
2672
2673        let col = batches[0].column_by_name("z").unwrap();
2674        let expected_z = Arc::new(Int64Array::from_iter_values(vec![4; 500])) as ArrayRef;
2675        assert_eq!(col, &expected_z);
2676    }
2677
2678    #[tokio::test]
2679    async fn test_filter_on_arrow_lt_or_gt() {
2680        let mut fixture = TableTestFixture::new();
2681        fixture.setup_manifest_files().await;
2682
2683        // Filter: y < 5 AND z >= 4
2684        let mut builder = fixture.table.scan();
2685        let predicate = Reference::new("y")
2686            .less_than(Datum::long(5))
2687            .or(Reference::new("z").greater_than_or_equal_to(Datum::long(4)));
2688        builder = builder.with_filter(predicate);
2689        let table_scan = builder.build().unwrap();
2690
2691        let batch_stream = table_scan
2692            .to_arrow(
2693                ArrowReaderOptions::builder()
2694                    .with_row_selection_enabled(true)
2695                    .build(),
2696            )
2697            .await
2698            .unwrap();
2699
2700        let batches: Vec<_> = batch_stream.try_collect().await.unwrap();
2701        assert_eq!(batches[0].num_rows(), 1024);
2702
2703        let col = batches[0].column_by_name("x").unwrap();
2704        let expected_x = Arc::new(Int64Array::from_iter_values(vec![1; 1024])) as ArrayRef;
2705        assert_eq!(col, &expected_x);
2706
2707        let col = batches[0].column_by_name("y").unwrap();
2708        let mut values = vec![2; 512];
2709        values.append(vec![3; 200].as_mut());
2710        values.append(vec![4; 300].as_mut());
2711        values.append(vec![5; 12].as_mut());
2712        let expected_y = Arc::new(Int64Array::from_iter_values(values)) as ArrayRef;
2713        assert_eq!(col, &expected_y);
2714
2715        let col = batches[0].column_by_name("z").unwrap();
2716        let mut values = vec![3; 512];
2717        values.append(vec![4; 512].as_mut());
2718        let expected_z = Arc::new(Int64Array::from_iter_values(values)) as ArrayRef;
2719        assert_eq!(col, &expected_z);
2720    }
2721
2722    #[tokio::test]
2723    async fn test_filter_on_arrow_startswith() {
2724        let mut fixture = TableTestFixture::new();
2725        fixture.setup_manifest_files().await;
2726
2727        // Filter: a STARTSWITH "Ice"
2728        let mut builder = fixture.table.scan();
2729        let predicate = Reference::new("a").starts_with(Datum::string("Ice"));
2730        builder = builder.with_filter(predicate);
2731        let table_scan = builder.build().unwrap();
2732
2733        let batch_stream = table_scan
2734            .to_arrow(
2735                ArrowReaderOptions::builder()
2736                    .with_row_selection_enabled(true)
2737                    .build(),
2738            )
2739            .await
2740            .unwrap();
2741
2742        let batches: Vec<_> = batch_stream.try_collect().await.unwrap();
2743
2744        assert_eq!(batches[0].num_rows(), 512);
2745
2746        let col = batches[0].column_by_name("a").unwrap();
2747        let string_arr = col.as_any().downcast_ref::<StringArray>().unwrap();
2748        assert_eq!(string_arr.value(0), "Iceberg");
2749    }
2750
2751    #[tokio::test]
2752    async fn test_filter_on_arrow_not_startswith() {
2753        let mut fixture = TableTestFixture::new();
2754        fixture.setup_manifest_files().await;
2755
2756        // Filter: a NOT STARTSWITH "Ice"
2757        let mut builder = fixture.table.scan();
2758        let predicate = Reference::new("a").not_starts_with(Datum::string("Ice"));
2759        builder = builder.with_filter(predicate);
2760        let table_scan = builder.build().unwrap();
2761
2762        let batch_stream = table_scan
2763            .to_arrow(
2764                ArrowReaderOptions::builder()
2765                    .with_row_selection_enabled(true)
2766                    .build(),
2767            )
2768            .await
2769            .unwrap();
2770
2771        let batches: Vec<_> = batch_stream.try_collect().await.unwrap();
2772
2773        assert_eq!(batches[0].num_rows(), 512);
2774
2775        let col = batches[0].column_by_name("a").unwrap();
2776        let string_arr = col.as_any().downcast_ref::<StringArray>().unwrap();
2777        assert_eq!(string_arr.value(0), "Apache");
2778    }
2779
2780    #[tokio::test]
2781    async fn test_filter_on_arrow_in() {
2782        let mut fixture = TableTestFixture::new();
2783        fixture.setup_manifest_files().await;
2784
2785        // Filter: a IN ("Sioux", "Iceberg")
2786        let mut builder = fixture.table.scan();
2787        let predicate =
2788            Reference::new("a").is_in([Datum::string("Sioux"), Datum::string("Iceberg")]);
2789        builder = builder.with_filter(predicate);
2790        let table_scan = builder.build().unwrap();
2791
2792        let batch_stream = table_scan
2793            .to_arrow(
2794                ArrowReaderOptions::builder()
2795                    .with_row_selection_enabled(true)
2796                    .build(),
2797            )
2798            .await
2799            .unwrap();
2800
2801        let batches: Vec<_> = batch_stream.try_collect().await.unwrap();
2802
2803        assert_eq!(batches[0].num_rows(), 512);
2804
2805        let col = batches[0].column_by_name("a").unwrap();
2806        let string_arr = col.as_any().downcast_ref::<StringArray>().unwrap();
2807        assert_eq!(string_arr.value(0), "Iceberg");
2808    }
2809
2810    #[tokio::test]
2811    async fn test_filter_on_arrow_not_in() {
2812        let mut fixture = TableTestFixture::new();
2813        fixture.setup_manifest_files().await;
2814
2815        // Filter: a NOT IN ("Sioux", "Iceberg")
2816        let mut builder = fixture.table.scan();
2817        let predicate =
2818            Reference::new("a").is_not_in([Datum::string("Sioux"), Datum::string("Iceberg")]);
2819        builder = builder.with_filter(predicate);
2820        let table_scan = builder.build().unwrap();
2821
2822        let batch_stream = table_scan
2823            .to_arrow(
2824                ArrowReaderOptions::builder()
2825                    .with_row_selection_enabled(true)
2826                    .build(),
2827            )
2828            .await
2829            .unwrap();
2830
2831        let batches: Vec<_> = batch_stream.try_collect().await.unwrap();
2832
2833        assert_eq!(batches[0].num_rows(), 512);
2834
2835        let col = batches[0].column_by_name("a").unwrap();
2836        let string_arr = col.as_any().downcast_ref::<StringArray>().unwrap();
2837        assert_eq!(string_arr.value(0), "Apache");
2838    }
2839
2840    fn file_scan_task_test_schema(primitive_type: PrimitiveType) -> Arc<Schema> {
2841        Arc::new(
2842            Schema::builder()
2843                .with_fields(vec![Arc::new(NestedField::required(
2844                    1,
2845                    "x",
2846                    Type::Primitive(primitive_type),
2847                ))])
2848                .build()
2849                .unwrap(),
2850        )
2851    }
2852
2853    fn assert_file_scan_task_serde_round_trip(task: FileScanTask) {
2854        // Regression test for https://github.com/apache/iceberg-rust/issues/3089.
2855        let serialized = serde_json::to_string(&task).unwrap();
2856        let deserialized: FileScanTask = serde_json::from_str(&serialized).unwrap();
2857
2858        assert_eq!(task, deserialized);
2859    }
2860
2861    fn file_scan_task_with_partition(
2862        primitive_type: PrimitiveType,
2863        transform: Transform,
2864        partition_value: Literal,
2865    ) -> FileScanTask {
2866        let schema = file_scan_task_test_schema(primitive_type);
2867        let partition_spec = Arc::new(
2868            PartitionSpec::builder(schema.clone())
2869                .add_partition_field("x", "x_partition", transform)
2870                .unwrap()
2871                .build()
2872                .unwrap(),
2873        );
2874        FileScanTask::builder()
2875            .with_data_file_path("data_file_path".to_string())
2876            .with_file_size_in_bytes(123)
2877            .with_start(10)
2878            .with_length(100)
2879            .with_project_field_ids(vec![1])
2880            .with_schema(schema)
2881            .with_data_file_format(DataFileFormat::Parquet)
2882            .with_partition(Some(Struct::from_iter([Some(partition_value)])))
2883            .with_partition_spec(Some(partition_spec))
2884            .with_case_sensitive(true)
2885            .build()
2886            .unwrap()
2887    }
2888
2889    #[test]
2890    fn test_file_scan_task_serde_without_predicate() {
2891        let task = FileScanTask::builder()
2892            .with_data_file_path("data_file_path".to_string())
2893            .with_file_size_in_bytes(0)
2894            .with_start(0)
2895            .with_length(100)
2896            .with_project_field_ids(vec![1, 2, 3])
2897            .with_schema(file_scan_task_test_schema(PrimitiveType::Binary))
2898            .with_record_count(Some(100))
2899            .with_first_row_id(Some(1000))
2900            .with_data_sequence_number(Some(5))
2901            .with_data_file_format(DataFileFormat::Parquet)
2902            .with_case_sensitive(false)
2903            .build()
2904            .unwrap();
2905        assert_file_scan_task_serde_round_trip(task);
2906    }
2907
2908    #[test]
2909    fn test_file_scan_task_serde_with_predicate() {
2910        let task = FileScanTask::builder()
2911            .with_data_file_path("data_file_path".to_string())
2912            .with_file_size_in_bytes(0)
2913            .with_start(0)
2914            .with_length(100)
2915            .with_project_field_ids(vec![1, 2, 3])
2916            .with_predicate(Some(BoundPredicate::AlwaysTrue))
2917            .with_schema(file_scan_task_test_schema(PrimitiveType::Binary))
2918            .with_data_file_format(DataFileFormat::Avro)
2919            .with_case_sensitive(false)
2920            .build()
2921            .unwrap();
2922
2923        let serialized = serde_json::to_value(&task).unwrap();
2924        assert!(serialized.get("record_count").is_none());
2925        assert_file_scan_task_serde_round_trip(task);
2926    }
2927
2928    #[test]
2929    fn test_unpartitioned_file_scan_task_serde() {
2930        let task = FileScanTask::builder()
2931            .with_data_file_path("data_file_path".to_string())
2932            .with_file_size_in_bytes(0)
2933            .with_start(0)
2934            .with_length(100)
2935            .with_project_field_ids(vec![1, 2, 3])
2936            .with_schema(file_scan_task_test_schema(PrimitiveType::Binary))
2937            .with_data_file_format(DataFileFormat::Parquet)
2938            .with_partition(Some(Struct::empty()))
2939            .with_case_sensitive(false)
2940            .build()
2941            .unwrap();
2942        assert_file_scan_task_serde_round_trip(task);
2943    }
2944
2945    #[test]
2946    fn test_file_scan_task_serde_with_all_optional_fields() {
2947        let schema = file_scan_task_test_schema(PrimitiveType::Long);
2948        let partition_spec = Arc::new(
2949            PartitionSpec::builder(schema.clone())
2950                .add_partition_field("x", "x", Transform::Identity)
2951                .unwrap()
2952                .build()
2953                .unwrap(),
2954        );
2955        let unified_partition_type = Arc::new(partition_spec.partition_type(&schema).unwrap());
2956        let sort_order = Arc::new(
2957            SortOrder::builder()
2958                .with_order_id(1)
2959                .with_sort_field(
2960                    SortField::builder()
2961                        .source_id(1)
2962                        .transform(Transform::Identity)
2963                        .direction(SortDirection::Ascending)
2964                        .null_order(NullOrder::First)
2965                        .build(),
2966                )
2967                .build(&schema)
2968                .unwrap(),
2969        );
2970        let task = FileScanTask::builder()
2971            .with_data_file_path("data_file_path".to_string())
2972            .with_file_size_in_bytes(123)
2973            .with_start(10)
2974            .with_length(100)
2975            .with_project_field_ids(vec![1])
2976            .with_schema(schema)
2977            .with_data_file_format(DataFileFormat::Parquet)
2978            .with_deletes(vec![
2979                FileScanTaskDeleteFile::builder()
2980                    .with_file_path("delete_file_path".to_string())
2981                    .with_file_size_in_bytes(23)
2982                    .with_file_type(DataContentType::EqualityDeletes)
2983                    .with_file_format(DataFileFormat::Parquet)
2984                    .with_partition_spec_id(0)
2985                    .with_equality_ids(Some(vec![1]))
2986                    .with_referenced_data_file(Some("data_file_path".to_string()))
2987                    .with_content_offset(Some(12))
2988                    .with_content_size_in_bytes(Some(34))
2989                    .with_record_count(Some(5))
2990                    .with_key_metadata(Some(vec![4, 5, 6].into_boxed_slice()))
2991                    .build()
2992                    .unwrap(),
2993            ])
2994            .with_partition(Some(Struct::from_iter([Some(Literal::long(42))])))
2995            .with_partition_spec(Some(partition_spec))
2996            .with_name_mapping(Some(Arc::new(NameMapping::new(vec![MappedField::new(
2997                Some(1),
2998                vec!["x".to_string()],
2999                vec![],
3000            )]))))
3001            .with_unified_partition_type(Some(unified_partition_type))
3002            .with_sort_order_id(Some(1))
3003            .with_sort_order(Some(sort_order))
3004            .with_column_stats(Some(
3005                FileScanTaskColumnStats::builder()
3006                    .with_column_sizes(HashMap::from([(1, 80), (2, 96)]))
3007                    .with_value_counts(HashMap::from([(1, 10), (2, 10)]))
3008                    .with_null_value_counts(HashMap::from([(1, 0), (2, 1)]))
3009                    .with_nan_value_counts(HashMap::from([(3, 2)]))
3010                    .with_lower_bounds(HashMap::from([
3011                        (1, Datum::string("apple")),
3012                        (2, Datum::decimal_from_str("-12.34").unwrap()),
3013                        (3, Datum::float(100.1_f32)),
3014                        (4, Datum::double(f64::NEG_INFINITY)),
3015                    ]))
3016                    .with_upper_bounds(HashMap::from([
3017                        (1, Datum::string("pear")),
3018                        (2, Datum::decimal_from_str("56.78").unwrap()),
3019                        (3, Datum::float(f32::INFINITY)),
3020                        (4, Datum::double(f64::INFINITY)),
3021                    ]))
3022                    .build(),
3023            ))
3024            .with_case_sensitive(true)
3025            .with_key_metadata(Some(vec![1, 2, 3].into_boxed_slice()))
3026            .build()
3027            .unwrap();
3028        assert_file_scan_task_serde_round_trip(task);
3029    }
3030
3031    #[test]
3032    fn test_file_scan_task_serde_omits_absent_column_stats() {
3033        let task = file_scan_task_with_partition(
3034            PrimitiveType::Long,
3035            Transform::Identity,
3036            Literal::long(42),
3037        );
3038
3039        let serialized = serde_json::to_value(&task).unwrap();
3040        assert!(serialized.get("column_stats").is_none());
3041
3042        let deserialized: FileScanTask = serde_json::from_value(serialized).unwrap();
3043        assert_eq!(deserialized.column_stats(), None);
3044    }
3045
3046    #[test]
3047    fn test_file_scan_task_serde_with_date_partition() {
3048        let task = file_scan_task_with_partition(
3049            PrimitiveType::Date,
3050            Transform::Identity,
3051            Literal::date(19_000),
3052        );
3053        assert_file_scan_task_serde_round_trip(task);
3054    }
3055
3056    #[test]
3057    fn test_file_scan_task_serde_with_timestamp_ns_partition() {
3058        let task = file_scan_task_with_partition(
3059            PrimitiveType::TimestampNs,
3060            Transform::Identity,
3061            Literal::timestamp_nano(1_510_871_468_123_456_789),
3062        );
3063        assert_file_scan_task_serde_round_trip(task);
3064    }
3065
3066    #[test]
3067    fn test_file_scan_task_serde_with_timestamptz_ns_partition() {
3068        let task = file_scan_task_with_partition(
3069            PrimitiveType::TimestamptzNs,
3070            Transform::Identity,
3071            Literal::timestamptz_nano(1_510_871_468_123_456_789),
3072        );
3073        assert_file_scan_task_serde_round_trip(task);
3074    }
3075
3076    #[test]
3077    fn test_file_scan_task_serde_with_decimal_partition() {
3078        let task = file_scan_task_with_partition(
3079            PrimitiveType::Decimal {
3080                precision: 9,
3081                scale: 2,
3082            },
3083            Transform::Identity,
3084            Literal::decimal(12_345),
3085        );
3086        assert_file_scan_task_serde_round_trip(task);
3087    }
3088
3089    #[test]
3090    fn test_file_scan_task_serde_with_uuid_partition() {
3091        let task = file_scan_task_with_partition(
3092            PrimitiveType::Uuid,
3093            Transform::Identity,
3094            Literal::uuid(Uuid::from_u128(0x12345678_90ab_cdef_1234_567890abcdef)),
3095        );
3096        assert_file_scan_task_serde_round_trip(task);
3097    }
3098
3099    #[test]
3100    fn test_file_scan_task_serde_with_fixed_partition() {
3101        let task = file_scan_task_with_partition(
3102            PrimitiveType::Fixed(4),
3103            Transform::Identity,
3104            Literal::fixed([1, 2, 3, 4]),
3105        );
3106        assert_file_scan_task_serde_round_trip(task);
3107    }
3108
3109    #[test]
3110    fn test_file_scan_task_serde_with_bucket_partition() {
3111        let task = file_scan_task_with_partition(
3112            PrimitiveType::String,
3113            Transform::Bucket(4),
3114            Literal::int(2),
3115        );
3116        assert_file_scan_task_serde_round_trip(task);
3117    }
3118
3119    #[tokio::test]
3120    async fn test_select_with_file_column() {
3121        let mut fixture = TableTestFixture::new();
3122        fixture.setup_manifest_files().await;
3123
3124        // Select regular columns plus the _file column
3125        let table_scan = fixture
3126            .table
3127            .scan()
3128            .select(["x", RESERVED_COL_NAME_FILE])
3129            .build()
3130            .unwrap();
3131
3132        let batch_stream = table_scan
3133            .to_arrow(
3134                ArrowReaderOptions::builder()
3135                    .with_row_selection_enabled(true)
3136                    .build(),
3137            )
3138            .await
3139            .unwrap();
3140        let batches: Vec<_> = batch_stream.try_collect().await.unwrap();
3141
3142        // Verify we have 2 columns: x and _file
3143        assert_eq!(batches[0].num_columns(), 2);
3144
3145        // Verify the x column exists and has correct data
3146        let x_col = batches[0].column_by_name("x").unwrap();
3147        let x_arr = x_col.as_primitive::<arrow_array::types::Int64Type>();
3148        assert_eq!(x_arr.value(0), 1);
3149
3150        // Verify the _file column exists
3151        let file_col = batches[0].column_by_name(RESERVED_COL_NAME_FILE);
3152        assert!(
3153            file_col.is_some(),
3154            "_file column should be present in the batch"
3155        );
3156
3157        // Verify the _file column contains a file path
3158        let file_col = file_col.unwrap();
3159        assert!(
3160            matches!(
3161                file_col.data_type(),
3162                arrow_schema::DataType::RunEndEncoded(_, _)
3163            ),
3164            "_file column should use RunEndEncoded type"
3165        );
3166
3167        // Decode the RunArray to verify it contains the file path
3168        let run_array = file_col
3169            .as_any()
3170            .downcast_ref::<RunArray<Int32Type>>()
3171            .expect("_file column should be a RunArray");
3172
3173        let values = run_array.values();
3174        let string_values = values.as_string::<i32>();
3175        assert_eq!(string_values.len(), 1, "Should have a single file path");
3176
3177        let file_path = string_values.value(0);
3178        assert!(
3179            file_path.ends_with(".parquet"),
3180            "File path should end with .parquet, got: {file_path}"
3181        );
3182    }
3183
3184    #[tokio::test]
3185    async fn test_select_file_column_position() {
3186        let mut fixture = TableTestFixture::new();
3187        fixture.setup_manifest_files().await;
3188
3189        // Select columns in specific order: x, _file, z
3190        let table_scan = fixture
3191            .table
3192            .scan()
3193            .select(["x", RESERVED_COL_NAME_FILE, "z"])
3194            .build()
3195            .unwrap();
3196
3197        let batch_stream = table_scan
3198            .to_arrow(
3199                ArrowReaderOptions::builder()
3200                    .with_row_selection_enabled(true)
3201                    .build(),
3202            )
3203            .await
3204            .unwrap();
3205        let batches: Vec<_> = batch_stream.try_collect().await.unwrap();
3206
3207        assert_eq!(batches[0].num_columns(), 3);
3208
3209        // Verify column order: x at position 0, _file at position 1, z at position 2
3210        let schema = batches[0].schema();
3211        assert_eq!(schema.field(0).name(), "x");
3212        assert_eq!(schema.field(1).name(), RESERVED_COL_NAME_FILE);
3213        assert_eq!(schema.field(2).name(), "z");
3214
3215        // Verify columns by name also works
3216        assert!(batches[0].column_by_name("x").is_some());
3217        assert!(batches[0].column_by_name(RESERVED_COL_NAME_FILE).is_some());
3218        assert!(batches[0].column_by_name("z").is_some());
3219    }
3220
3221    #[tokio::test]
3222    async fn test_select_file_column_only() {
3223        let mut fixture = TableTestFixture::new();
3224        fixture.setup_manifest_files().await;
3225
3226        // Select only the _file column
3227        let table_scan = fixture
3228            .table
3229            .scan()
3230            .select([RESERVED_COL_NAME_FILE])
3231            .build()
3232            .unwrap();
3233
3234        let batch_stream = table_scan
3235            .to_arrow(
3236                ArrowReaderOptions::builder()
3237                    .with_row_selection_enabled(true)
3238                    .build(),
3239            )
3240            .await
3241            .unwrap();
3242        let batches: Vec<_> = batch_stream.try_collect().await.unwrap();
3243
3244        // Should have exactly 1 column
3245        assert_eq!(batches[0].num_columns(), 1);
3246
3247        // Verify it's the _file column
3248        let schema = batches[0].schema();
3249        assert_eq!(schema.field(0).name(), RESERVED_COL_NAME_FILE);
3250
3251        // Verify the batch has the correct number of rows
3252        // The scan reads files 1.parquet and 3.parquet (2.parquet is deleted)
3253        // Each file has 1024 rows, so total is 2048 rows
3254        let total_rows: usize = batches.iter().map(|b| b.num_rows()).sum();
3255        assert_eq!(total_rows, 2048);
3256    }
3257
3258    #[tokio::test]
3259    async fn test_file_column_with_multiple_files() {
3260        use std::collections::HashSet;
3261
3262        let mut fixture = TableTestFixture::new();
3263        fixture.setup_manifest_files().await;
3264
3265        // Select x and _file columns
3266        let table_scan = fixture
3267            .table
3268            .scan()
3269            .select(["x", RESERVED_COL_NAME_FILE])
3270            .build()
3271            .unwrap();
3272
3273        let batch_stream = table_scan
3274            .to_arrow(
3275                ArrowReaderOptions::builder()
3276                    .with_row_selection_enabled(true)
3277                    .build(),
3278            )
3279            .await
3280            .unwrap();
3281        let batches: Vec<_> = batch_stream.try_collect().await.unwrap();
3282
3283        // Collect all unique file paths from the batches
3284        let mut file_paths = HashSet::new();
3285        for batch in &batches {
3286            let file_col = batch.column_by_name(RESERVED_COL_NAME_FILE).unwrap();
3287            let run_array = file_col
3288                .as_any()
3289                .downcast_ref::<RunArray<Int32Type>>()
3290                .expect("_file column should be a RunArray");
3291
3292            let values = run_array.values();
3293            let string_values = values.as_string::<i32>();
3294            for i in 0..string_values.len() {
3295                file_paths.insert(string_values.value(i).to_string());
3296            }
3297        }
3298
3299        // We should have multiple files (the test creates 1.parquet and 3.parquet)
3300        assert!(!file_paths.is_empty(), "Should have at least one file path");
3301
3302        // All paths should end with .parquet
3303        for path in &file_paths {
3304            assert!(
3305                path.ends_with(".parquet"),
3306                "All file paths should end with .parquet, got: {path}"
3307            );
3308        }
3309    }
3310
3311    #[tokio::test]
3312    async fn test_file_column_at_start() {
3313        let mut fixture = TableTestFixture::new();
3314        fixture.setup_manifest_files().await;
3315
3316        // Select _file at the start
3317        let table_scan = fixture
3318            .table
3319            .scan()
3320            .select([RESERVED_COL_NAME_FILE, "x", "y"])
3321            .build()
3322            .unwrap();
3323
3324        let batch_stream = table_scan
3325            .to_arrow(
3326                ArrowReaderOptions::builder()
3327                    .with_row_selection_enabled(true)
3328                    .build(),
3329            )
3330            .await
3331            .unwrap();
3332        let batches: Vec<_> = batch_stream.try_collect().await.unwrap();
3333
3334        assert_eq!(batches[0].num_columns(), 3);
3335
3336        // Verify _file is at position 0
3337        let schema = batches[0].schema();
3338        assert_eq!(schema.field(0).name(), RESERVED_COL_NAME_FILE);
3339        assert_eq!(schema.field(1).name(), "x");
3340        assert_eq!(schema.field(2).name(), "y");
3341    }
3342
3343    #[tokio::test]
3344    async fn test_file_column_at_end() {
3345        let mut fixture = TableTestFixture::new();
3346        fixture.setup_manifest_files().await;
3347
3348        // Select _file at the end
3349        let table_scan = fixture
3350            .table
3351            .scan()
3352            .select(["x", "y", RESERVED_COL_NAME_FILE])
3353            .build()
3354            .unwrap();
3355
3356        let batch_stream = table_scan
3357            .to_arrow(
3358                ArrowReaderOptions::builder()
3359                    .with_row_selection_enabled(true)
3360                    .build(),
3361            )
3362            .await
3363            .unwrap();
3364        let batches: Vec<_> = batch_stream.try_collect().await.unwrap();
3365
3366        assert_eq!(batches[0].num_columns(), 3);
3367
3368        // Verify _file is at position 2 (the end)
3369        let schema = batches[0].schema();
3370        assert_eq!(schema.field(0).name(), "x");
3371        assert_eq!(schema.field(1).name(), "y");
3372        assert_eq!(schema.field(2).name(), RESERVED_COL_NAME_FILE);
3373    }
3374
3375    #[tokio::test]
3376    async fn test_select_with_repeated_column_names() {
3377        let mut fixture = TableTestFixture::new();
3378        fixture.setup_manifest_files().await;
3379
3380        // Select with repeated column names - both regular columns and virtual columns
3381        // Repeated columns should appear multiple times in the result (duplicates are allowed)
3382        let table_scan = fixture
3383            .table
3384            .scan()
3385            .select([
3386                "x",
3387                RESERVED_COL_NAME_FILE,
3388                "x", // x repeated
3389                "y",
3390                RESERVED_COL_NAME_FILE, // _file repeated
3391                "y",                    // y repeated
3392            ])
3393            .build()
3394            .unwrap();
3395
3396        let batch_stream = table_scan
3397            .to_arrow(
3398                ArrowReaderOptions::builder()
3399                    .with_row_selection_enabled(true)
3400                    .build(),
3401            )
3402            .await
3403            .unwrap();
3404        let batches: Vec<_> = batch_stream.try_collect().await.unwrap();
3405
3406        // Verify we have exactly 6 columns (duplicates are allowed and preserved)
3407        assert_eq!(
3408            batches[0].num_columns(),
3409            6,
3410            "Should have exactly 6 columns with duplicates"
3411        );
3412
3413        let schema = batches[0].schema();
3414
3415        // Verify columns appear in the exact order requested: x, _file, x, y, _file, y
3416        assert_eq!(schema.field(0).name(), "x", "Column 0 should be x");
3417        assert_eq!(
3418            schema.field(1).name(),
3419            RESERVED_COL_NAME_FILE,
3420            "Column 1 should be _file"
3421        );
3422        assert_eq!(
3423            schema.field(2).name(),
3424            "x",
3425            "Column 2 should be x (duplicate)"
3426        );
3427        assert_eq!(schema.field(3).name(), "y", "Column 3 should be y");
3428        assert_eq!(
3429            schema.field(4).name(),
3430            RESERVED_COL_NAME_FILE,
3431            "Column 4 should be _file (duplicate)"
3432        );
3433        assert_eq!(
3434            schema.field(5).name(),
3435            "y",
3436            "Column 5 should be y (duplicate)"
3437        );
3438
3439        // Verify all columns have correct data types
3440        assert!(
3441            matches!(schema.field(0).data_type(), arrow_schema::DataType::Int64),
3442            "Column x should be Int64"
3443        );
3444        assert!(
3445            matches!(schema.field(2).data_type(), arrow_schema::DataType::Int64),
3446            "Column x (duplicate) should be Int64"
3447        );
3448        assert!(
3449            matches!(schema.field(3).data_type(), arrow_schema::DataType::Int64),
3450            "Column y should be Int64"
3451        );
3452        assert!(
3453            matches!(schema.field(5).data_type(), arrow_schema::DataType::Int64),
3454            "Column y (duplicate) should be Int64"
3455        );
3456        assert!(
3457            matches!(
3458                schema.field(1).data_type(),
3459                arrow_schema::DataType::RunEndEncoded(_, _)
3460            ),
3461            "_file column should use RunEndEncoded type"
3462        );
3463        assert!(
3464            matches!(
3465                schema.field(4).data_type(),
3466                arrow_schema::DataType::RunEndEncoded(_, _)
3467            ),
3468            "_file column (duplicate) should use RunEndEncoded type"
3469        );
3470    }
3471
3472    /// Builds a minimal single-snapshot table (no manifests on disk) whose schema
3473    /// contains a column with the given name, so scan planning resolves column names
3474    /// against a real schema. `TableScan::build()` only reads metadata, not manifest
3475    /// files, so a snapshot pointing at a dummy manifest list is enough. Used to
3476    /// reproduce issue #2837.
3477    fn table_with_data_column(column_name: &str) -> Table {
3478        use crate::spec::{
3479            FormatVersion, MAIN_BRANCH, Operation, Snapshot, SnapshotReference, SnapshotRetention,
3480            SortOrder, Summary, TableMetadataBuilder, UnboundPartitionSpec,
3481        };
3482
3483        let schema = Schema::builder()
3484            .with_schema_id(0)
3485            .with_fields(vec![
3486                NestedField::required(1, "id", Type::Primitive(PrimitiveType::Int)).into(),
3487                NestedField::required(2, column_name, Type::Primitive(PrimitiveType::Int)).into(),
3488            ])
3489            .build()
3490            .unwrap();
3491
3492        let snapshot = Snapshot::builder()
3493            .with_snapshot_id(1)
3494            .with_timestamp_ms(1)
3495            .with_sequence_number(0)
3496            .with_schema_id(0)
3497            .with_manifest_list("/snap-1.avro")
3498            .with_summary(Summary {
3499                operation: Operation::Append,
3500                additional_properties: HashMap::new(),
3501            })
3502            .build();
3503
3504        let metadata = TableMetadataBuilder::new(
3505            schema,
3506            UnboundPartitionSpec::builder().with_spec_id(0).build(),
3507            SortOrder::unsorted_order(),
3508            "s3://bucket/table".to_string(),
3509            FormatVersion::V2,
3510            HashMap::new(),
3511        )
3512        .unwrap()
3513        .add_snapshot(snapshot)
3514        .unwrap()
3515        .set_ref(MAIN_BRANCH, SnapshotReference {
3516            snapshot_id: 1,
3517            retention: SnapshotRetention::Branch {
3518                min_snapshots_to_keep: None,
3519                max_snapshot_age_ms: None,
3520                max_ref_age_ms: None,
3521            },
3522        })
3523        .unwrap()
3524        .build()
3525        .unwrap()
3526        .metadata;
3527
3528        Table::builder()
3529            .metadata(metadata)
3530            .identifier(TableIdent::from_strs(["db", "table1"]).unwrap())
3531            .file_io(FileIO::new_with_fs())
3532            .runtime(test_runtime())
3533            .build()
3534            .unwrap()
3535    }
3536
3537    /// A user data column named `pos` (a delete-file internal column name that is not a
3538    /// data-table metadata column) must be projectable rather than shadowed. Regression
3539    /// test for issue #2837.
3540    #[test]
3541    fn test_scan_projects_data_column_named_like_delete_file_column() {
3542        for column_name in ["pos", "file_path"] {
3543            let table = table_with_data_column(column_name);
3544
3545            // Projecting the data column must succeed and resolve to its real field id (2),
3546            // not the reserved delete-file field id.
3547            let table_scan = table
3548                .scan()
3549                .select([column_name])
3550                .build()
3551                .unwrap_or_else(|e| panic!("scan of data column `{column_name}` failed: {e}"));
3552
3553            assert_eq!(
3554                table_scan.plan_context.as_ref().unwrap().field_ids.as_ref(),
3555                &[2]
3556            );
3557
3558            // The default projection (all columns) must resolve to the real field ids
3559            // too, not shadow the data column with a reserved delete-file id.
3560            let default_scan = table.scan().build().unwrap();
3561            assert_eq!(
3562                default_scan
3563                    .plan_context
3564                    .as_ref()
3565                    .unwrap()
3566                    .field_ids
3567                    .as_ref(),
3568                &[1, 2]
3569            );
3570        }
3571    }
3572
3573    /// Projecting a genuinely absent column still fails with a clear "not found" error
3574    /// rather than being silently accepted as a metadata column.
3575    #[test]
3576    fn test_scan_rejects_unknown_column_named_like_delete_file_column() {
3577        // This table has no `pos` column (only `id` and `file_path`).
3578        let table = table_with_data_column("file_path");
3579
3580        let err = table
3581            .scan()
3582            .select(["pos"])
3583            .build()
3584            .expect_err("projecting an absent column should fail");
3585        assert_eq!(err.kind(), ErrorKind::DataInvalid);
3586        assert!(err.to_string().contains("not found"));
3587    }
3588
3589    #[tokio::test]
3590    async fn test_scan_deadlock() {
3591        let mut fixture = TableTestFixture::new();
3592        fixture.setup_deadlock_manifests().await;
3593
3594        // Create table scan with concurrency limit 1
3595        // This sets channel size to 1.
3596        // Data manifest has 10 entries -> will block producer.
3597        // Delete manifest is 2nd in list -> won't be processed.
3598        // Consumer 2 (Data) not started -> blocked.
3599        // Consumer 1 (Delete) waiting -> blocked.
3600        let table_scan = fixture
3601            .table
3602            .scan()
3603            .with_concurrency_limit(1)
3604            .build()
3605            .unwrap();
3606
3607        // This should timeout/hang if deadlock exists
3608        // We can use tokio::time::timeout
3609        let result = tokio::time::timeout(std::time::Duration::from_secs(5), async {
3610            table_scan
3611                .plan_files()
3612                .await
3613                .unwrap()
3614                .try_collect::<Vec<_>>()
3615                .await
3616        })
3617        .await;
3618
3619        // Assert it finished (didn't timeout)
3620        assert!(result.is_ok(), "Scan timed out - deadlock detected");
3621    }
3622
3623    /// Plans `table` under a timeout so a swallowed manifest error shows up as
3624    /// a timeout instead of a hang, and returns the collected result.
3625    async fn plan_files_with_timeout(table: &Table) -> Result<Vec<FileScanTask>> {
3626        tokio::time::timeout(std::time::Duration::from_secs(5), async {
3627            table
3628                .scan()
3629                .build()
3630                .unwrap()
3631                .plan_files()
3632                .await
3633                .unwrap()
3634                .try_collect::<Vec<_>>()
3635                .await
3636        })
3637        .await
3638        .expect("plan_files hung instead of reporting the manifest error")
3639    }
3640
3641    /// Returns the path of the first manifest in the current snapshot.
3642    async fn first_manifest_path(fixture: &TableTestFixture) -> String {
3643        let snapshot = fixture.table.metadata().current_snapshot().unwrap();
3644        let manifest_list = fixture
3645            .table
3646            .manifest_list_reader(snapshot)
3647            .load()
3648            .await
3649            .unwrap();
3650        manifest_list.entries()[0].manifest_path.clone()
3651    }
3652
3653    #[tokio::test]
3654    async fn test_plan_files_reports_corrupt_manifest_as_error() {
3655        let mut fixture = TableTestFixture::new();
3656        fixture.setup_manifest_files().await;
3657        let path = first_manifest_path(&fixture).await;
3658        fs::write(&path, b"not an avro file").unwrap();
3659
3660        let err = plan_files_with_timeout(&fixture.table)
3661            .await
3662            .expect_err("a corrupt manifest must fail the plan");
3663        assert_eq!(err.kind(), ErrorKind::Unexpected);
3664        assert!(err.to_string().contains(&path), "{err}");
3665    }
3666
3667    #[tokio::test]
3668    async fn test_plan_files_reports_missing_manifest_as_error() {
3669        let mut fixture = TableTestFixture::new();
3670        fixture.setup_manifest_files().await;
3671        let path = first_manifest_path(&fixture).await;
3672        fs::remove_file(&path).unwrap();
3673
3674        let err = plan_files_with_timeout(&fixture.table)
3675            .await
3676            .expect_err("a missing manifest must fail the plan");
3677        assert_eq!(err.kind(), ErrorKind::Unexpected);
3678        assert!(err.to_string().contains(&path), "{err}");
3679    }
3680
3681    /// A manifest parse spawned on a shut-down cpu runtime is cancelled, which
3682    /// takes the same `JoinError` path a panicking parse does. The manifest path
3683    /// in the error comes from `ObjectCache`'s "Failed to load manifest" wrapper;
3684    /// `test_read_names_manifest_when_parse_is_cancelled` covers the reader's own
3685    /// context.
3686    #[tokio::test]
3687    async fn test_plan_files_reports_cancelled_manifest_parse_as_error() {
3688        let mut fixture = TableTestFixture::new();
3689        fixture.setup_manifest_files().await;
3690        let path = first_manifest_path(&fixture).await;
3691        let io_rt = tokio::runtime::Builder::new_multi_thread()
3692            .enable_all()
3693            .build()
3694            .unwrap();
3695        let cpu_rt = tokio::runtime::Builder::new_multi_thread()
3696            .enable_all()
3697            .build()
3698            .unwrap();
3699        let rt = Runtime::new_with_split(&io_rt, &cpu_rt);
3700        cpu_rt.shutdown_background();
3701        let table = fixture.table.clone().with_runtime(rt);
3702
3703        let err = plan_files_with_timeout(&table)
3704            .await
3705            .expect_err("a cancelled manifest parse must fail the plan");
3706        assert_eq!(err.kind(), ErrorKind::Unexpected);
3707        assert!(err.to_string().contains(&path), "{err}");
3708        io_rt.shutdown_background();
3709    }
3710
3711    #[tokio::test]
3712    async fn test_select_with_spec_id_column() {
3713        let mut fixture = TableTestFixture::new();
3714        fixture.setup_manifest_files().await;
3715
3716        // Select regular columns plus the _spec_id column
3717        let table_scan = fixture
3718            .table
3719            .scan()
3720            .select(["x", RESERVED_COL_NAME_SPEC_ID, "z"])
3721            .build()
3722            .unwrap();
3723
3724        let batch_stream = table_scan
3725            .to_arrow(
3726                ArrowReaderOptions::builder()
3727                    .with_row_selection_enabled(true)
3728                    .build(),
3729            )
3730            .await
3731            .unwrap();
3732        let batches: Vec<_> = batch_stream.try_collect().await.unwrap();
3733
3734        // Verify we have 3 columns: x, _spec_id, and z
3735        assert_eq!(batches[0].num_columns(), 3);
3736
3737        // Verify the x column exists and has correct data
3738        let col1 = batches[0].column_by_name("x").unwrap();
3739        let int64_arr = col1.as_any().downcast_ref::<Int64Array>().unwrap();
3740        assert_eq!(int64_arr.value(0), 1);
3741
3742        // Verify the _spec_id column exists
3743        let spec_id_col = batches[0].column_by_name(RESERVED_COL_NAME_SPEC_ID);
3744        assert!(
3745            spec_id_col.is_some(),
3746            "_spec_id column should be present in the batch"
3747        );
3748
3749        // Verify the _spec_id data type
3750        let spec_id_col = spec_id_col.unwrap();
3751        assert!(
3752            matches!(
3753                spec_id_col.data_type(),
3754                arrow_schema::DataType::RunEndEncoded(_, _)
3755            ),
3756            "_spec_id column should use RunEndEncoded type"
3757        );
3758
3759        // Decode the RunArray to verify it contains the spec id
3760        let run_array = spec_id_col
3761            .as_any()
3762            .downcast_ref::<RunArray<Int32Type>>()
3763            .expect("_spec_id column should be a RunArray");
3764
3765        let values = run_array.values();
3766        let int_values = values.as_primitive::<Int32Type>();
3767        assert_eq!(int_values.len(), 1, "Should have a single _spec_id");
3768
3769        let spec_id = int_values.value(0);
3770        assert_eq!(spec_id, 0, "_spec_id should be 0, got: {spec_id}");
3771
3772        // Verify 'z' column exists
3773        assert!(batches[0].column_by_name("z").is_some());
3774    }
3775
3776    #[tokio::test]
3777    async fn test_select_with_last_updated_sequence_number_column() {
3778        // A v2 fixture: data files have a null first_row_id. Per the spec's Row
3779        // Lineage read rules, a file with a null first_row_id produces a null
3780        // _last_updated_sequence_number for all rows; both lineage columns are
3781        // gated on first_row_id.
3782        let mut fixture = TableTestFixture::new();
3783        fixture.setup_manifest_files().await;
3784
3785        let table_scan = fixture
3786            .table
3787            .scan()
3788            .select(["x", RESERVED_COL_NAME_LAST_UPDATED_SEQUENCE_NUMBER])
3789            .build()
3790            .unwrap();
3791
3792        let batches: Vec<_> = table_scan
3793            .to_arrow(
3794                ArrowReaderOptions::builder()
3795                    .with_row_selection_enabled(true)
3796                    .build(),
3797            )
3798            .await
3799            .unwrap()
3800            .try_collect()
3801            .await
3802            .unwrap();
3803
3804        // Every row's value is null (v2 files have no first_row_id).
3805        assert_last_updated_seq_all(&batches, None);
3806    }
3807
3808    #[tokio::test]
3809    async fn test_select_with_last_updated_sequence_number_column_v3() {
3810        // A v3 fixture: the data file inherits a first_row_id and a data sequence
3811        // number through manifest read. End to end, the projected
3812        // _last_updated_sequence_number materializes to the file's data sequence
3813        // number for every row, exercising the full inherit, populate and
3814        // materialize wiring, not just a hand-built task.
3815        let mut fixture = TableTestFixture::new();
3816        fixture.setup_v3_manifest_files().await;
3817
3818        // The added file inherits the current snapshot's sequence number; derive it
3819        // from the fixture rather than hardcoding so the assertion tracks the fixture.
3820        let expected_seq = fixture
3821            .table
3822            .metadata()
3823            .current_snapshot()
3824            .unwrap()
3825            .sequence_number();
3826
3827        let table_scan = fixture
3828            .table
3829            .scan()
3830            .select(["x", RESERVED_COL_NAME_LAST_UPDATED_SEQUENCE_NUMBER])
3831            .build()
3832            .unwrap();
3833
3834        let batches: Vec<_> = table_scan
3835            .to_arrow(
3836                ArrowReaderOptions::builder()
3837                    .with_row_selection_enabled(true)
3838                    .build(),
3839            )
3840            .await
3841            .unwrap()
3842            .try_collect()
3843            .await
3844            .unwrap();
3845
3846        assert_last_updated_seq_all(&batches, Some(expected_seq));
3847    }
3848
3849    #[tokio::test]
3850    async fn test_select_with_spec_id_column_from_unpartitioned_table() {
3851        let mut fixture = TableTestFixture::new_unpartitioned();
3852        fixture.setup_unpartitioned_manifest_files().await;
3853
3854        // Select regular columns plus the _spec_id column
3855        let table_scan = fixture
3856            .table
3857            .scan()
3858            .select(["x", RESERVED_COL_NAME_SPEC_ID])
3859            .build()
3860            .unwrap();
3861
3862        let batch_stream = table_scan
3863            .to_arrow(
3864                ArrowReaderOptions::builder()
3865                    .with_row_selection_enabled(true)
3866                    .build(),
3867            )
3868            .await
3869            .unwrap();
3870        let batches: Vec<_> = batch_stream.try_collect().await.unwrap();
3871
3872        // Verify we have 2 columns: x and _spec_id
3873        assert_eq!(batches[0].num_columns(), 2);
3874
3875        // Verify the _spec_id column exists
3876        let spec_id_col = batches[0].column_by_name(RESERVED_COL_NAME_SPEC_ID);
3877        assert!(
3878            spec_id_col.is_some(),
3879            "_spec_id column should be present in the batch"
3880        );
3881
3882        // Verify the _spec_id data type
3883        let spec_id_col = spec_id_col.unwrap();
3884        assert!(
3885            matches!(
3886                spec_id_col.data_type(),
3887                arrow_schema::DataType::RunEndEncoded(_, _)
3888            ),
3889            "_spec_id column should use RunEndEncoded type"
3890        );
3891
3892        // Decode the RunArray to verify it contains the spec id
3893        let run_array = spec_id_col
3894            .as_any()
3895            .downcast_ref::<RunArray<Int32Type>>()
3896            .expect("_spec_id column should be a RunArray");
3897
3898        let values = run_array.values();
3899        let int_values = values.as_primitive::<Int32Type>();
3900        assert_eq!(int_values.len(), 1, "Should have a single _spec_id");
3901
3902        let spec_id = int_values.value(0);
3903        assert_eq!(spec_id, 0, "_spec_id should be 0, got: {spec_id}");
3904    }
3905
3906    #[tokio::test]
3907    async fn test_select_with_spec_id_column_with_partition_evolution() {
3908        let mut fixture = TableTestFixture::new_with_partition_evolution();
3909        fixture
3910            .setup_manifest_files_with_partition_evolution()
3911            .await;
3912
3913        // Select regular columns plus the _spec_id column
3914        let table_scan = fixture
3915            .table
3916            .scan()
3917            .select(["x", RESERVED_COL_NAME_SPEC_ID, "z"])
3918            .build()
3919            .unwrap();
3920
3921        let batch_stream = table_scan
3922            .to_arrow(
3923                ArrowReaderOptions::builder()
3924                    .with_row_selection_enabled(true)
3925                    .build(),
3926            )
3927            .await
3928            .unwrap();
3929        let batches: Vec<_> = batch_stream.try_collect().await.unwrap();
3930
3931        // Verify the x column exists and has correct data
3932        let col1 = batches[0].column_by_name("x").unwrap();
3933        let int64_arr = col1.as_any().downcast_ref::<Int64Array>().unwrap();
3934        assert_eq!(int64_arr.value(0), 1);
3935
3936        // Verify the _spec_id column exists
3937        let spec_id_col = batches[0].column_by_name(RESERVED_COL_NAME_SPEC_ID);
3938        assert!(
3939            spec_id_col.is_some(),
3940            "_spec_id column should be present in the batch"
3941        );
3942
3943        // Verify the _spec_id data type
3944        let spec_id_col = spec_id_col.unwrap();
3945        assert!(
3946            matches!(
3947                spec_id_col.data_type(),
3948                arrow_schema::DataType::RunEndEncoded(_, _)
3949            ),
3950            "_spec_id column should use RunEndEncoded type"
3951        );
3952
3953        // Decode the RunArray to verify it contains the spec id
3954        let run_array = spec_id_col
3955            .as_any()
3956            .downcast_ref::<RunArray<Int32Type>>()
3957            .expect("_spec_id column should be a RunArray");
3958
3959        let values = run_array.values();
3960        let int_values = values.as_primitive::<Int32Type>();
3961        assert_eq!(int_values.len(), 1, "Should have a single _spec_id");
3962
3963        let spec_id = int_values.value(0);
3964        assert_eq!(spec_id, 2, "_spec_id should be 2, got: {spec_id}");
3965    }
3966
3967    #[tokio::test]
3968    async fn test_select_with_pos_and_file_columns() {
3969        use arrow_array::cast::AsArray;
3970
3971        let mut fixture = TableTestFixture::new();
3972        fixture.setup_manifest_files().await;
3973
3974        // Select regular columns plus the _pos column
3975        let table_scan = fixture
3976            .table
3977            .scan()
3978            .select(["x", RESERVED_COL_NAME_POS, RESERVED_COL_NAME_FILE])
3979            .build()
3980            .unwrap();
3981
3982        let batch_stream = table_scan
3983            .to_arrow(
3984                ArrowReaderOptions::builder()
3985                    .with_row_selection_enabled(true)
3986                    .build(),
3987            )
3988            .await
3989            .unwrap();
3990        let batches: Vec<_> = batch_stream.try_collect().await.unwrap();
3991        assert_eq!(batches.len(), 2);
3992
3993        // Examine batches are 1.paruqet and 3.parquet, 2.parquet is deleted.
3994        for batch in batches.iter() {
3995            // Verify we have 3 columns: x, _pos and _file
3996            assert_eq!(batch.num_columns(), 3);
3997
3998            // Verify the x column exists and has correct data
3999            let x_col = batch.column_by_name("x").unwrap();
4000            let x_arr = x_col.as_primitive::<arrow_array::types::Int64Type>();
4001            assert_eq!(x_arr.value(0), 1);
4002
4003            // The _pos column exists and verify it is Int64Array with the expected values
4004            let pos_col = batch.column(1);
4005            let pos_array: &Int64Array = pos_col
4006                .as_any()
4007                .downcast_ref::<Int64Array>()
4008                .expect("_pos column should be a Int64Array");
4009            assert_eq!(*pos_array, Int64Array::from_iter_values(0i64..1024));
4010
4011            // Verify the _file column exists
4012            let file_col = batch.column_by_name(RESERVED_COL_NAME_FILE);
4013            assert!(
4014                file_col.is_some(),
4015                "_file column should be present in the batch"
4016            );
4017        }
4018    }
4019
4020    #[tokio::test]
4021    async fn test_pos_column_at_start_with_filters() {
4022        let mut fixture = TableTestFixture::new();
4023        fixture.setup_manifest_files().await;
4024
4025        // y is in [4, 5)
4026        let predicate = Reference::new("y")
4027            .greater_than(Datum::long(4i64))
4028            .and(Reference::new("y").less_than_or_equal_to(Datum::long(5i64)));
4029        // Select _pos at the start
4030        let table_scan = fixture
4031            .table
4032            .scan()
4033            .select([RESERVED_COL_NAME_POS, "x", "y"])
4034            .with_filter(predicate)
4035            .build()
4036            .unwrap();
4037
4038        let batch_stream = table_scan
4039            .to_arrow(
4040                ArrowReaderOptions::builder()
4041                    .with_row_selection_enabled(true)
4042                    .build(),
4043            )
4044            .await
4045            .unwrap();
4046        let batches: Vec<_> = batch_stream.try_collect().await.unwrap();
4047        assert_eq!(batches.len(), 2);
4048
4049        // Examine batches are 1.paruqet and 3.parquet, 2.parquet is deleted.
4050        for batch in batches.iter() {
4051            assert_eq!(batch.num_columns(), 3);
4052            assert_eq!(batch.num_rows(), 12);
4053
4054            // Verify _pos is at position 0
4055            let schema = batch.schema();
4056            assert_eq!(schema.field(0).name(), RESERVED_COL_NAME_POS);
4057            assert_eq!(schema.field(1).name(), "x");
4058            assert_eq!(schema.field(2).name(), "y");
4059
4060            let pos_col = batch.column(0);
4061            let pos_array: &Int64Array = pos_col
4062                .as_any()
4063                .downcast_ref::<Int64Array>()
4064                .expect("_pos column should be a Int64Array");
4065            assert_eq!(*pos_array, Int64Array::from_iter_values(1012i64..1024));
4066        }
4067    }
4068
4069    #[tokio::test]
4070    async fn test_repeated_pos_column_with_filter() {
4071        let mut fixture = TableTestFixture::new();
4072        fixture.setup_manifest_files().await;
4073
4074        // a NOT STARTSWITH "Apa"
4075        let predicate = Reference::new("a").not_starts_with(Datum::string("Apa"));
4076        // Select '_pos' columns twice
4077        let table_scan = fixture
4078            .table
4079            .scan()
4080            .select([RESERVED_COL_NAME_POS, "a", RESERVED_COL_NAME_POS, "x"])
4081            .with_filter(predicate)
4082            .build()
4083            .unwrap();
4084
4085        let batch_stream = table_scan
4086            .to_arrow(
4087                ArrowReaderOptions::builder()
4088                    .with_row_selection_enabled(true)
4089                    .build(),
4090            )
4091            .await
4092            .unwrap();
4093
4094        let batches: Vec<_> = batch_stream.try_collect().await.unwrap();
4095        assert_eq!(batches.len(), 2);
4096
4097        // Examine batches are 1.paruqet and 3.parquet, 2.parquet is deleted.
4098        for batch in batches.iter() {
4099            assert_eq!(batch.num_rows(), 512);
4100
4101            // fetch the 1st _pos column by name and verify it is Int64Array with the expected values
4102            let pos_col = batch
4103                .column_by_name("_pos")
4104                .expect("_pos column should be present in the batch");
4105            let pos_array: &Int64Array = pos_col
4106                .as_any()
4107                .downcast_ref::<Int64Array>()
4108                .expect("_pos column should be a Int64Array");
4109            assert_eq!(*pos_array, Int64Array::from_iter_values(512i64..1024));
4110
4111            // fetch the 2nd _pos column by index and verify it is Int64Array with the expected values
4112            let pos_col = batch.column(2);
4113            let pos_array: &Int64Array = pos_col
4114                .as_any()
4115                .downcast_ref::<Int64Array>()
4116                .expect("_pos column should be a Int64Array");
4117            assert_eq!(*pos_array, Int64Array::from_iter_values(512i64..1024));
4118        }
4119    }
4120
4121    /// End-to-end through `TableScan`: a data file with three row groups planned
4122    /// as a single whole-file `FileScanTask` must yield contiguous, file-absolute
4123    /// `_pos` values (0..300) across the row-group boundaries.
4124    #[tokio::test]
4125    async fn test_pos_across_row_groups_via_table_scan() {
4126        let mut fixture = TableTestFixture::new();
4127        fixture.setup_multi_row_group_manifest(&[]).await;
4128
4129        // Planning must produce exactly one whole-file task with _pos projected and
4130        // no delete files, confirming TableScan does not sub-split the file.
4131        let tasks: Vec<_> = fixture
4132            .table
4133            .scan()
4134            .select(["x", RESERVED_COL_NAME_POS])
4135            .build()
4136            .unwrap()
4137            .plan_files()
4138            .await
4139            .unwrap()
4140            .try_collect()
4141            .await
4142            .unwrap();
4143        assert_eq!(tasks.len(), 1, "expected a single FileScanTask");
4144        let task = &tasks[0];
4145        assert!(
4146            task.project_field_ids().contains(&RESERVED_FIELD_ID_POS),
4147            "_pos field id must be projected into the FileScanTask"
4148        );
4149        assert_eq!(task.start(), 0, "TableScan should plan whole-file tasks");
4150        assert_eq!(task.length(), task.file_size_in_bytes());
4151        assert!(task.deletes().is_empty());
4152
4153        // Reading that task yields absolute _pos 0..300 in order.
4154        let batches: Vec<_> = fixture
4155            .table
4156            .scan()
4157            .select(["x", RESERVED_COL_NAME_POS])
4158            .build()
4159            .unwrap()
4160            .to_arrow(ArrowReaderOptions::default())
4161            .await
4162            .unwrap()
4163            .try_collect()
4164            .await
4165            .unwrap();
4166
4167        let pos: Vec<i64> = batches
4168            .iter()
4169            .flat_map(|b| {
4170                b.column_by_name(RESERVED_COL_NAME_POS)
4171                    .expect("_pos column should be present")
4172                    .as_any()
4173                    .downcast_ref::<Int64Array>()
4174                    .expect("_pos column should be a Int64Array")
4175                    .values()
4176                    .to_vec()
4177            })
4178            .collect();
4179        assert_eq!(pos, (0..300).collect::<Vec<i64>>());
4180
4181        // Sanity: x == 1000 + _pos, proving _pos aligns with the actual rows read.
4182        let x: Vec<i64> = batches
4183            .iter()
4184            .flat_map(|b| {
4185                b.column_by_name("x")
4186                    .unwrap()
4187                    .as_primitive::<arrow_array::types::Int64Type>()
4188                    .values()
4189                    .to_vec()
4190            })
4191            .collect();
4192        assert_eq!(x, (1000..1300).collect::<Vec<i64>>());
4193    }
4194
4195    /// A positional delete file registered in the manifest must be attached to the planned
4196    /// `FileScanTask` and applied on read, while surviving `_pos` values stay file-absolute.
4197    #[tokio::test]
4198    async fn test_pos_with_positional_deletes_via_table_scan() {
4199        let mut fixture = TableTestFixture::new();
4200        // Delete file-absolute positions 150 (middle row group) and 299 (last row).
4201        fixture.setup_multi_row_group_manifest(&[150, 299]).await;
4202
4203        // Planning must attach the positional delete file to the task.
4204        let tasks: Vec<_> = fixture
4205            .table
4206            .scan()
4207            .select(["x", RESERVED_COL_NAME_POS])
4208            .build()
4209            .unwrap()
4210            .plan_files()
4211            .await
4212            .unwrap()
4213            .try_collect()
4214            .await
4215            .unwrap();
4216        assert_eq!(tasks.len(), 1);
4217        assert_eq!(
4218            tasks[0].deletes().len(),
4219            1,
4220            "positional delete file should be planned into the task"
4221        );
4222        assert_eq!(
4223            tasks[0].deletes()[0].file_type(),
4224            DataContentType::PositionDeletes
4225        );
4226
4227        // Reading applies the deletes; _pos must skip 150 and 299 and stay absolute.
4228        let batches: Vec<_> = fixture
4229            .table
4230            .scan()
4231            .select(["x", RESERVED_COL_NAME_POS])
4232            .build()
4233            .unwrap()
4234            .to_arrow(ArrowReaderOptions::default())
4235            .await
4236            .unwrap()
4237            .try_collect()
4238            .await
4239            .unwrap();
4240
4241        let pos: Vec<i64> = batches
4242            .iter()
4243            .flat_map(|b| {
4244                b.column_by_name(RESERVED_COL_NAME_POS)
4245                    .expect("_pos column should be present")
4246                    .as_any()
4247                    .downcast_ref::<Int64Array>()
4248                    .expect("_pos column should be a Int64Array")
4249                    .values()
4250                    .to_vec()
4251            })
4252            .collect();
4253
4254        let total: usize = batches.iter().map(|b| b.num_rows()).sum();
4255        assert_eq!(
4256            total, 298,
4257            "two rows should be removed by positional deletes"
4258        );
4259        assert!(!pos.contains(&150) && !pos.contains(&299), "got {pos:?}");
4260        let expected: Vec<i64> = (0..150).chain(151..299).collect();
4261        assert_eq!(pos, expected);
4262    }
4263
4264    /// A filter that only matches the middle row group (`y` in [1100, 1200)) and
4265    /// prunes the other two row groups by statistics, so only the middle row group
4266    /// is read. `_pos` must report the file-absolute positions 100..200 for those
4267    /// rows, not values reset to 0..100.
4268    ///
4269    /// `y` is a non-partition column, so pruning here is driven purely by Parquet
4270    /// row-group statistics (the TableTestFixture's partition column is `x`).
4271    #[tokio::test]
4272    async fn test_pos_reads_only_middle_row_group_via_filter() {
4273        let mut fixture = TableTestFixture::new();
4274        fixture.setup_multi_row_group_manifest(&[]).await;
4275
4276        // Middle row group holds y = 1100..1200 at file positions 100..200.
4277        let predicate = Reference::new("y")
4278            .greater_than_or_equal_to(Datum::long(1100))
4279            .and(Reference::new("y").less_than(Datum::long(1200)));
4280
4281        let batches: Vec<_> = fixture
4282            .table
4283            .scan()
4284            .select(["y", RESERVED_COL_NAME_POS])
4285            .with_filter(predicate)
4286            .build()
4287            .unwrap()
4288            .to_arrow(ArrowReaderOptions::default())
4289            .await
4290            .unwrap()
4291            .try_collect()
4292            .await
4293            .unwrap();
4294
4295        let total: usize = batches.iter().map(|b| b.num_rows()).sum();
4296        assert_eq!(total, 100, "only the middle row group should be read");
4297
4298        let pos: Vec<i64> = batches
4299            .iter()
4300            .flat_map(|b| {
4301                b.column_by_name(RESERVED_COL_NAME_POS)
4302                    .expect("_pos column should be present")
4303                    .as_any()
4304                    .downcast_ref::<Int64Array>()
4305                    .expect("_pos column should be a Int64Array")
4306                    .values()
4307                    .to_vec()
4308            })
4309            .collect();
4310        assert_eq!(
4311            pos,
4312            (100..200).collect::<Vec<i64>>(),
4313            "_pos must be file-absolute for the middle row group"
4314        );
4315
4316        // Cross-check: y == 1000 + _pos for every surviving row.
4317        let y: Vec<i64> = batches
4318            .iter()
4319            .flat_map(|b| {
4320                b.column_by_name("y")
4321                    .unwrap()
4322                    .as_primitive::<arrow_array::types::Int64Type>()
4323                    .values()
4324                    .to_vec()
4325            })
4326            .collect();
4327        assert_eq!(y, (1100..1200).collect::<Vec<i64>>());
4328    }
4329
4330    #[tokio::test]
4331    async fn test_dead_only_manifests_are_skipped() {
4332        use crate::spec::ManifestContentType;
4333
4334        for content in [ManifestContentType::Data, ManifestContentType::Deletes] {
4335            let mut fixture = TableTestFixture::new();
4336            if content == ManifestContentType::Data {
4337                fixture.setup_manifest_files().await;
4338            } else {
4339                fixture.setup_deadlock_manifests().await;
4340            }
4341            let snapshot = fixture.table.metadata().current_snapshot().unwrap();
4342            let scan = fixture
4343                .table
4344                .scan()
4345                .with_filter(Reference::new("x").less_than(Datum::long(200)))
4346                .build()
4347                .unwrap();
4348            let mut expected = if content == ManifestContentType::Deletes {
4349                let tasks = scan
4350                    .plan_files()
4351                    .await
4352                    .unwrap()
4353                    .try_collect::<Vec<_>>()
4354                    .await
4355                    .unwrap();
4356                assert_eq!(tasks.len(), 10);
4357                assert!(tasks.iter().all(|task| task.deletes().len() == 1));
4358                let cow_tasks = scan
4359                    .plan_cow_rewrite_files()
4360                    .await
4361                    .unwrap()
4362                    .map_ok(|file| file.scan_task)
4363                    .try_collect::<Vec<_>>()
4364                    .await
4365                    .unwrap();
4366                [tasks, cow_tasks]
4367            } else {
4368                [vec![], vec![]]
4369            };
4370            for tasks in &mut expected {
4371                tasks.sort_by(|a, b| a.data_file_path().cmp(b.data_file_path()));
4372            }
4373            let context = scan.plan_context.as_ref().unwrap();
4374            let list = context.get_manifest_list().await.unwrap();
4375            let source = list
4376                .entries()
4377                .iter()
4378                .find(|file| file.content == content)
4379                .unwrap();
4380            let manifest = context.object_cache.get_manifest(source).await.unwrap();
4381            let builder = ManifestWriterBuilder::new(
4382                fixture.next_manifest_file(),
4383                Some(snapshot.snapshot_id()),
4384                fixture.table.metadata().current_schema().clone(),
4385                fixture
4386                    .table
4387                    .metadata()
4388                    .default_partition_spec()
4389                    .as_ref()
4390                    .clone(),
4391            );
4392            let mut writer = if content == ManifestContentType::Data {
4393                builder.build_v2_data()
4394            } else {
4395                builder.build_v2_deletes()
4396            };
4397            for entry in manifest.entries() {
4398                writer.add_delete_entry(entry.as_ref().clone()).unwrap();
4399            }
4400            let mut manifest = writer.write_manifest_file().await.unwrap();
4401            if content == ManifestContentType::Deletes {
4402                manifest.partitions.as_mut().unwrap()[0].lower_bound = Some(vec![0].into());
4403            }
4404            assert_eq!(
4405                (manifest.added_files_count, manifest.existing_files_count),
4406                (Some(0), Some(0))
4407            );
4408            assert!(manifest.deleted_files_count.unwrap() > 0);
4409            fixture
4410                .table
4411                .file_io()
4412                .delete(&manifest.manifest_path)
4413                .await
4414                .unwrap();
4415            let mut manifests = if content == ManifestContentType::Deletes {
4416                list.entries().to_vec()
4417            } else {
4418                vec![]
4419            };
4420            manifests.push(manifest);
4421            let output = fixture
4422                .table
4423                .file_io()
4424                .new_output(snapshot.manifest_list())
4425                .unwrap()
4426                .writer()
4427                .await
4428                .unwrap();
4429            let mut writer = ManifestListWriter::v2(
4430                output,
4431                snapshot.snapshot_id(),
4432                snapshot.parent_snapshot_id(),
4433                snapshot.sequence_number(),
4434            );
4435            writer.add_manifests(manifests.into_iter()).unwrap();
4436            writer.close().await.unwrap();
4437            let table = Table::builder()
4438                .metadata(fixture.table.metadata().clone())
4439                .identifier(fixture.table.identifier().clone())
4440                .file_io(fixture.table.file_io().clone())
4441                .runtime(test_runtime())
4442                .build()
4443                .unwrap();
4444            for cow in [false, true] {
4445                let scan = table
4446                    .scan()
4447                    .with_filter(Reference::new("x").less_than(Datum::long(200)))
4448                    .build()
4449                    .unwrap();
4450                let stream = if cow {
4451                    scan.plan_cow_rewrite_files()
4452                        .await
4453                        .unwrap()
4454                        .map_ok(|file| file.scan_task)
4455                        .boxed()
4456                } else {
4457                    scan.plan_files().await.unwrap()
4458                };
4459                let mut actual = tokio::time::timeout(
4460                    std::time::Duration::from_secs(10),
4461                    stream.try_collect::<Vec<_>>(),
4462                )
4463                .await
4464                .unwrap()
4465                .unwrap();
4466                actual.sort_by(|a, b| a.data_file_path().cmp(b.data_file_path()));
4467                assert_eq!(
4468                    actual,
4469                    expected[usize::from(cow)],
4470                    "content={content:?}, cow={cow}"
4471                );
4472            }
4473        }
4474    }
4475
4476    #[tokio::test]
4477    async fn test_live_and_unknown_manifest_counts_preserve_tasks() {
4478        use crate::spec::FormatVersion;
4479
4480        for (status, added, existing) in [
4481            (ManifestStatus::Added, Some(1), Some(0)),
4482            (ManifestStatus::Existing, Some(0), Some(1)),
4483            (ManifestStatus::Added, None, Some(0)),
4484            (ManifestStatus::Existing, Some(0), None),
4485            (ManifestStatus::Added, None, None),
4486        ] {
4487            let mut fixture = TableTestFixture::new();
4488            fixture.setup_manifest_files().await;
4489            let snapshot = fixture.table.metadata().current_snapshot().unwrap();
4490            let scan = fixture.table.scan().build().unwrap();
4491            let context = scan.plan_context.as_ref().unwrap();
4492            let list = context.get_manifest_list().await.unwrap();
4493            let manifest = context
4494                .object_cache
4495                .get_manifest(&list.entries()[0])
4496                .await
4497                .unwrap();
4498            let entry = manifest
4499                .entries()
4500                .iter()
4501                .find(|entry| entry.status() == status)
4502                .unwrap();
4503            let expected = vec![(entry.data_file().file_path().to_owned(), vec![])];
4504            let mut writer = ManifestWriterBuilder::new(
4505                fixture.next_manifest_file(),
4506                Some(snapshot.snapshot_id()),
4507                fixture.table.metadata().current_schema().clone(),
4508                fixture
4509                    .table
4510                    .metadata()
4511                    .default_partition_spec()
4512                    .as_ref()
4513                    .clone(),
4514            )
4515            .build_v1();
4516            if status == ManifestStatus::Added {
4517                writer.add_entry(entry.as_ref().clone()).unwrap();
4518            } else {
4519                writer.add_existing_entry(entry.as_ref().clone()).unwrap();
4520            }
4521            let mut manifest = writer.write_manifest_file().await.unwrap();
4522            assert_eq!(
4523                (manifest.added_files_count, manifest.existing_files_count),
4524                (
4525                    Some(u32::from(status == ManifestStatus::Added)),
4526                    Some(u32::from(status == ManifestStatus::Existing))
4527                )
4528            );
4529            manifest.added_files_count = added;
4530            manifest.existing_files_count = existing;
4531            let output = fixture
4532                .table
4533                .file_io()
4534                .new_output(snapshot.manifest_list())
4535                .unwrap()
4536                .writer()
4537                .await
4538                .unwrap();
4539            let mut writer = ManifestListWriter::v1(
4540                output,
4541                snapshot.snapshot_id(),
4542                snapshot.parent_snapshot_id(),
4543            );
4544            writer.add_manifests(std::iter::once(manifest)).unwrap();
4545            writer.close().await.unwrap();
4546            let mut metadata = fixture.table.metadata().clone();
4547            metadata.format_version = FormatVersion::V1;
4548            let table = Table::builder()
4549                .metadata(metadata)
4550                .identifier(fixture.table.identifier().clone())
4551                .file_io(fixture.table.file_io().clone())
4552                .runtime(test_runtime())
4553                .build()
4554                .unwrap();
4555            for cow in [false, true] {
4556                let scan = table.scan().build().unwrap();
4557                let counts = scan
4558                    .plan_context
4559                    .as_ref()
4560                    .unwrap()
4561                    .get_manifest_list()
4562                    .await
4563                    .unwrap();
4564                assert_eq!(
4565                    (
4566                        counts.entries()[0].added_files_count,
4567                        counts.entries()[0].existing_files_count
4568                    ),
4569                    (added, existing)
4570                );
4571                let stream = if cow {
4572                    scan.plan_cow_rewrite_files()
4573                        .await
4574                        .unwrap()
4575                        .map_ok(|file| file.scan_task)
4576                        .boxed()
4577                } else {
4578                    scan.plan_files().await.unwrap()
4579                };
4580                let tasks = tokio::time::timeout(
4581                    std::time::Duration::from_secs(10),
4582                    stream.try_collect::<Vec<_>>(),
4583                )
4584                .await
4585                .unwrap()
4586                .unwrap();
4587                let actual: Vec<_> = tasks
4588                    .into_iter()
4589                    .map(|task| (task.data_file_path().to_owned(), task.deletes().to_vec()))
4590                    .collect();
4591                assert_eq!(
4592                    actual, expected,
4593                    "status={status:?}, added={added:?}, existing={existing:?}, cow={cow}"
4594                );
4595            }
4596        }
4597    }
4598}