1mod cache;
21use cache::*;
22mod context;
23use context::*;
24mod task;
25
26use std::collections::HashMap;
27use std::future::Future;
28use std::sync::Arc;
29
30use arrow_array::RecordBatch;
31use futures::channel::mpsc::{Sender, channel};
32use futures::stream::BoxStream;
33use futures::{SinkExt, StreamExt, TryStreamExt};
34pub use task::*;
35
36use crate::arrow::{ArrowReaderBuilder, ArrowReaderOptions};
37pub use crate::arrow::{ScanMetrics, ScanResult};
38use crate::delete_file_index::DeleteFileIndex;
39use crate::error::invalid_data;
40use crate::expr::visitors::inclusive_metrics_evaluator::InclusiveMetricsEvaluator;
41use crate::expr::{Bind, BoundPredicate, Predicate};
42use crate::io::FileIO;
43use crate::metadata_columns::{
44 RESERVED_FIELD_ID_PARTITION, get_metadata_field_id, is_metadata_column_name,
45};
46use crate::runtime::Runtime;
47use crate::spec::{DataContentType, Schema, SchemaRef, SnapshotRef, SortOrderRef, StructType};
48use crate::table::Table;
49use crate::util::available_parallelism;
50use crate::util::snapshot::snapshot_id_as_of_time;
51use crate::{Error, ErrorKind, Result};
52
53pub type ArrowRecordBatchStream = BoxStream<'static, Result<RecordBatch>>;
55
56fn resolve_field_id(
58 schema: &Schema,
59 column_name: &str,
60 case_sensitive: bool,
61) -> Result<Option<i32>> {
62 if case_sensitive {
63 Ok(schema.field_id_by_name(column_name))
64 } else {
65 let field = schema.field_by_name_case_insensitive_checked(column_name)?;
66 Ok(field.map(|field| field.id))
67 }
68}
69
70fn require_field_id(schema: &Schema, column_name: &str, case_sensitive: bool) -> Result<i32> {
72 resolve_field_id(schema, column_name, case_sensitive)?
73 .ok_or_else(|| invalid_data!("Column {column_name} not found in table. Schema: {schema}"))
74}
75
76fn collect_scan_field_ids(
77 schema: &Schema,
78 column_names: Option<&[String]>,
79 case_sensitive: bool,
80) -> Result<Vec<i32>> {
81 let Some(column_names) = column_names else {
82 return Ok(schema.as_struct().fields().iter().map(|f| f.id).collect());
83 };
84
85 column_names
86 .iter()
87 .map(|column_name| {
88 if is_metadata_column_name(column_name) {
89 return get_metadata_field_id(column_name);
90 }
91
92 let field_id = require_field_id(schema, column_name, case_sensitive)?;
93
94 schema
95 .as_struct()
96 .field_by_id(field_id)
97 .ok_or_else(|| {
98 Error::new(
99 ErrorKind::FeatureUnsupported,
100 format!(
101 "Column {column_name} is not a direct child of schema but a nested field, which is not supported now. Schema: {schema}"
102 ),
103 )
104 })?;
105
106 Ok(field_id)
107 })
108 .collect()
109}
110
111fn resolve_column_stats_columns(
113 schema: &Schema,
114 column_names: Option<&[String]>,
115 case_sensitive: bool,
116) -> Result<Option<ColumnStatsColumns>> {
117 column_names
118 .map(|column_names| {
119 column_names
120 .iter()
121 .map(|column_name| {
122 if is_metadata_column_name(column_name) {
123 return Err(invalid_data!(
124 "Metadata column {column_name} has no column statistics"
125 ));
126 }
127 let field_id = require_field_id(schema, column_name, case_sensitive)?;
128 let column_type = schema
129 .field_by_id(field_id)
130 .and_then(|field| field.field_type.as_primitive_type())
131 .ok_or_else(|| {
132 invalid_data!(
133 "Column {column_name} is not a primitive field and has no column statistics"
134 )
135 })?;
136 Ok((field_id, column_type.clone()))
137 })
138 .collect()
139 })
140 .transpose()
141}
142
143fn bind_scan_predicate(
144 schema: &SchemaRef,
145 predicate: Option<&Predicate>,
146 case_sensitive: bool,
147) -> Result<Option<Arc<BoundPredicate>>> {
148 predicate
149 .map(|predicate| predicate.bind(schema.clone(), case_sensitive))
150 .transpose()
151 .map(|predicate| predicate.map(Arc::new))
152}
153
154fn projected_partition_type(
155 table: &Table,
156 schema: &Schema,
157 field_ids: &[i32],
158) -> Result<Option<Arc<StructType>>> {
159 if !field_ids.contains(&RESERVED_FIELD_ID_PARTITION) {
160 return Ok(None);
161 }
162
163 table
164 .metadata()
165 .unified_partition_type(schema)
166 .map(Arc::new)
167 .map(Some)
168}
169
170enum SnapshotSelection {
172 SnapshotId(i64),
173 AsOfTime(i64),
174}
175
176pub struct TableScanBuilder<'a> {
178 table: &'a Table,
179 column_names: Option<Vec<String>>,
181 column_stats_names: Option<Vec<String>>,
182 snapshot_selection: Option<SnapshotSelection>,
183 case_sensitive: bool,
184 filter: Option<Predicate>,
185 concurrency_limit_manifest_entries: usize,
186 concurrency_limit_manifest_files: usize,
187}
188
189impl<'a> TableScanBuilder<'a> {
190 pub(crate) fn new(table: &'a Table) -> Self {
191 let num_cpus = available_parallelism().get();
192
193 Self {
194 table,
195 column_names: None,
196 column_stats_names: None,
197 snapshot_selection: None,
198 case_sensitive: true,
199 filter: None,
200 concurrency_limit_manifest_entries: num_cpus,
201 concurrency_limit_manifest_files: num_cpus,
202 }
203 }
204
205 pub fn with_case_sensitive(mut self, case_sensitive: bool) -> Self {
207 self.case_sensitive = case_sensitive;
208 self
209 }
210
211 pub fn with_filter(mut self, predicate: Predicate) -> Self {
213 self.filter = Some(predicate.rewrite_not());
216 self
217 }
218
219 pub fn select_all(mut self) -> Self {
221 self.column_names = None;
222 self
223 }
224
225 pub fn select_empty(mut self) -> Self {
227 self.column_names = Some(vec![]);
228 self
229 }
230
231 pub fn select(mut self, column_names: impl IntoIterator<Item = impl ToString>) -> Self {
233 self.column_names = Some(
234 column_names
235 .into_iter()
236 .map(|item| item.to_string())
237 .collect(),
238 );
239 self
240 }
241
242 pub fn with_column_stats(
268 mut self,
269 column_names: impl IntoIterator<Item = impl ToString>,
270 ) -> Self {
271 self.column_stats_names = Some(
272 column_names
273 .into_iter()
274 .map(|item| item.to_string())
275 .collect(),
276 );
277 self
278 }
279
280 pub fn snapshot_id(mut self, snapshot_id: i64) -> Self {
287 self.snapshot_selection = Some(SnapshotSelection::SnapshotId(snapshot_id));
288 self
289 }
290
291 pub fn as_of_time(mut self, timestamp_ms: i64) -> Self {
312 self.snapshot_selection = Some(SnapshotSelection::AsOfTime(timestamp_ms));
313 self
314 }
315
316 pub fn with_concurrency_limit(mut self, limit: usize) -> Self {
318 self.concurrency_limit_manifest_files = limit;
319 self.concurrency_limit_manifest_entries = limit;
320 self
321 }
322
323 pub fn with_manifest_entry_concurrency_limit(mut self, limit: usize) -> Self {
325 self.concurrency_limit_manifest_entries = limit;
326 self
327 }
328
329 pub fn build(self) -> Result<TableScan> {
331 let selected_snapshot = match self.snapshot_selection {
332 Some(SnapshotSelection::SnapshotId(id)) => {
333 Some((id, format!("Snapshot with id {id} not found")))
334 }
335 Some(SnapshotSelection::AsOfTime(timestamp_ms)) => {
336 let id = snapshot_id_as_of_time(&self.table.metadata_ref(), timestamp_ms)?;
337 Some((
339 id,
340 format!(
341 "Snapshot with id {id} resolved from timestamp {timestamp_ms} ms not found; it may have expired"
342 ),
343 ))
344 }
345 None => None,
346 };
347 let is_time_travel = selected_snapshot.is_some();
348 let snapshot = match selected_snapshot {
349 Some((snapshot_id, not_found_message)) => self
350 .table
351 .metadata()
352 .snapshot_by_id(snapshot_id)
353 .ok_or_else(|| invalid_data!(not_found_message))?
354 .clone(),
355 None => {
356 let Some(current_snapshot_id) = self.table.metadata().current_snapshot() else {
357 return Ok(TableScan {
358 column_names: self.column_names,
359 file_io: self.table.file_io().clone(),
360 plan_context: None,
361 concurrency_limit_manifest_entries: self.concurrency_limit_manifest_entries,
362 concurrency_limit_manifest_files: self.concurrency_limit_manifest_files,
363 runtime: self.table.runtime().clone(),
364 });
365 };
366 current_snapshot_id.clone()
367 }
368 };
369
370 let schema = if is_time_travel {
373 snapshot.schema(self.table.metadata())?
374 } else {
375 self.table.metadata().current_schema().clone()
376 };
377 let field_ids =
378 collect_scan_field_ids(&schema, self.column_names.as_deref(), self.case_sensitive)?;
379 let column_stats_columns = resolve_column_stats_columns(
380 &schema,
381 self.column_stats_names.as_deref(),
382 self.case_sensitive,
383 )?;
384 let snapshot_bound_predicate =
385 bind_scan_predicate(&schema, self.filter.as_ref(), self.case_sensitive)?;
386 let name_mapping = self
387 .table
388 .metadata()
389 .table_properties()
390 .default_name_mapping()?
391 .map(Arc::new);
392 let unified_partition_type = projected_partition_type(self.table, &schema, &field_ids)?;
393
394 let sort_orders = Arc::new(
397 self.table
398 .metadata()
399 .sort_orders_iter()
400 .map(|order| (order.order_id, order.clone()))
401 .collect::<HashMap<i64, SortOrderRef>>(),
402 );
403
404 let plan_context = PlanContext {
405 snapshot,
406 table_metadata: self.table.metadata_ref(),
407 snapshot_schema: schema,
408 case_sensitive: self.case_sensitive,
409 predicate: self.filter.map(Arc::new),
410 snapshot_bound_predicate,
411 object_cache: self.table.object_cache(),
412 field_ids: Arc::new(field_ids),
413 name_mapping,
414 partition_filter_cache: Arc::new(PartitionFilterCache::new()),
415 manifest_evaluator_cache: Arc::new(ManifestEvaluatorCache::new()),
416 expression_evaluator_cache: Arc::new(ExpressionEvaluatorCache::new()),
417 unified_partition_type,
418 sort_orders,
419 column_stats_columns,
420 };
421
422 Ok(TableScan {
423 column_names: self.column_names,
424 file_io: self.table.file_io().clone(),
425 plan_context: Some(plan_context),
426 concurrency_limit_manifest_entries: self.concurrency_limit_manifest_entries,
427 concurrency_limit_manifest_files: self.concurrency_limit_manifest_files,
428 runtime: self.table.runtime().clone(),
429 })
430 }
431}
432
433#[derive(Debug)]
435pub struct TableScan {
436 plan_context: Option<PlanContext>,
440 file_io: FileIO,
441 column_names: Option<Vec<String>>,
442 concurrency_limit_manifest_files: usize,
445
446 concurrency_limit_manifest_entries: usize,
449
450 runtime: Runtime,
451}
452
453impl TableScan {
454 pub async fn plan_files(&self) -> Result<FileScanTaskStream> {
456 self.plan_data_files(|ctx| ctx.into_file_scan_task()).await
457 }
458
459 pub(crate) async fn plan_cow_rewrite_files(
460 &self,
461 ) -> Result<BoxStream<'static, Result<crate::cow_rewrite::CowRewriteFile>>> {
462 self.plan_data_files(|ctx| ctx.into_cow_rewrite_file())
463 .await
464 }
465
466 async fn plan_data_files<T, F, Fut>(
467 &self,
468 build_result: F,
469 ) -> Result<BoxStream<'static, Result<T>>>
470 where
471 T: Send + 'static,
472 F: Fn(ManifestEntryContext) -> Fut + Copy + Send + Sync + 'static,
473 Fut: Future<Output = Result<T>> + Send + 'static,
474 {
475 let Some(plan_context) = self.plan_context.as_ref() else {
476 return Ok(Box::pin(futures::stream::empty()));
477 };
478
479 let concurrency_limit_manifest_files = self.concurrency_limit_manifest_files;
480 let concurrency_limit_manifest_entries = self.concurrency_limit_manifest_entries;
481
482 let (manifest_entry_data_ctx_tx, manifest_entry_data_ctx_rx) =
484 channel(concurrency_limit_manifest_files);
485 let (manifest_entry_delete_ctx_tx, manifest_entry_delete_ctx_rx) =
486 channel(concurrency_limit_manifest_files);
487
488 let (planned_file_tx, planned_file_rx) = channel(concurrency_limit_manifest_entries);
490
491 let (delete_file_idx, delete_file_tx) = DeleteFileIndex::new(self.runtime.clone());
492
493 let manifest_list = plan_context.get_manifest_list().await?;
494
495 let manifest_file_contexts = plan_context.build_manifest_file_contexts(
499 manifest_list,
500 manifest_entry_data_ctx_tx,
501 delete_file_idx.clone(),
502 manifest_entry_delete_ctx_tx,
503 )?;
504
505 let mut channel_for_manifest_error = planned_file_tx.clone();
506 let mut channel_for_data_manifest_entry_error = planned_file_tx.clone();
507 let mut channel_for_delete_manifest_entry_error = planned_file_tx.clone();
508
509 let rt = self.runtime.clone();
510
511 rt.io().spawn(async move {
513 let result = futures::stream::iter(manifest_file_contexts)
514 .try_for_each_concurrent(concurrency_limit_manifest_files, |ctx| async move {
515 ctx.fetch_manifest_and_stream_manifest_entries().await
516 })
517 .await;
518
519 if let Err(error) = result {
520 let _ = channel_for_manifest_error.send(Err(error)).await;
521 }
522 });
523
524 {
526 let rt = rt.clone();
527 let rt_inner = rt.clone();
528 rt.cpu().spawn(async move {
529 let result = manifest_entry_delete_ctx_rx
530 .map(|me_ctx| Ok((me_ctx, delete_file_tx.clone())))
531 .try_for_each_concurrent(
532 concurrency_limit_manifest_entries,
533 |(manifest_entry_context, tx)| {
534 let rt_inner = rt_inner.clone();
535 async move {
536 rt_inner
537 .cpu()
538 .spawn(async move {
539 Self::process_delete_manifest_entry(
540 manifest_entry_context,
541 tx,
542 )
543 .await
544 })
545 .await?
546 }
547 },
548 )
549 .await;
550
551 if let Err(error) = result {
552 let _ = channel_for_delete_manifest_entry_error
553 .send(Err(error))
554 .await;
555 }
556 });
557 }
558
559 {
561 let rt_inner = rt.clone();
562 rt.cpu().spawn(async move {
563 let result = manifest_entry_data_ctx_rx
564 .map(|me_ctx| Ok((me_ctx, planned_file_tx.clone())))
565 .try_for_each_concurrent(
566 concurrency_limit_manifest_entries,
567 |(manifest_entry_context, tx)| {
568 let rt_inner = rt_inner.clone();
569 async move {
570 rt_inner
571 .cpu()
572 .spawn(async move {
573 Self::process_data_manifest_entry(
574 manifest_entry_context,
575 tx,
576 build_result,
577 )
578 .await
579 })
580 .await?
581 }
582 },
583 )
584 .await;
585
586 if let Err(error) = result {
587 let _ = channel_for_data_manifest_entry_error.send(Err(error)).await;
588 }
589 });
590 }
591
592 Ok(planned_file_rx.boxed())
593 }
594
595 pub async fn to_arrow(&self, options: ArrowReaderOptions) -> Result<ArrowRecordBatchStream> {
631 ArrowReaderBuilder::new(self.file_io.clone(), self.runtime.clone())
632 .with_options(options)
633 .build()
634 .read(self.plan_files().await?)
635 .map(|result| result.stream())
636 }
637
638 pub fn column_names(&self) -> Option<&[String]> {
640 self.column_names.as_deref()
641 }
642
643 pub fn snapshot(&self) -> Option<&SnapshotRef> {
645 self.plan_context.as_ref().map(|x| &x.snapshot)
646 }
647
648 async fn process_data_manifest_entry<T, F, Fut>(
649 manifest_entry_context: ManifestEntryContext,
650 mut planned_file_tx: Sender<Result<T>>,
651 build_result: F,
652 ) -> Result<()>
653 where
654 F: Fn(ManifestEntryContext) -> Fut + Send + Sync + 'static,
655 Fut: Future<Output = Result<T>> + Send + 'static,
656 {
657 if manifest_entry_context.manifest_entry.content_type() != DataContentType::Data {
659 return Err(Error::new(
660 ErrorKind::FeatureUnsupported,
661 "Encountered an entry for a delete file in a data file manifest",
662 ));
663 }
664
665 if let Some(ref bound_predicates) = manifest_entry_context.bound_predicates {
666 let BoundPredicates {
667 snapshot_bound_predicate,
668 partition_bound_predicate,
669 } = bound_predicates.as_ref();
670
671 let expression_evaluator_cache =
672 manifest_entry_context.expression_evaluator_cache.as_ref();
673
674 let expression_evaluator = expression_evaluator_cache.get(
675 manifest_entry_context.partition_spec_id,
676 partition_bound_predicate,
677 )?;
678
679 if !expression_evaluator.eval(manifest_entry_context.manifest_entry.data_file())? {
682 return Ok(());
683 }
684
685 if !InclusiveMetricsEvaluator::eval(
687 snapshot_bound_predicate,
688 manifest_entry_context.manifest_entry.data_file(),
689 false,
690 )? {
691 return Ok(());
692 }
693 }
694
695 planned_file_tx
699 .send(Ok(build_result(manifest_entry_context).await?))
700 .await?;
701
702 Ok(())
703 }
704
705 async fn process_delete_manifest_entry(
706 manifest_entry_context: ManifestEntryContext,
707 mut delete_file_ctx_tx: Sender<DeleteFileContext>,
708 ) -> Result<()> {
709 if manifest_entry_context.manifest_entry.content_type() == DataContentType::Data {
711 return Err(Error::new(
712 ErrorKind::FeatureUnsupported,
713 "Encountered an entry for a data file in a delete manifest",
714 ));
715 }
716
717 if let Some(ref bound_predicates) = manifest_entry_context.bound_predicates {
718 let expression_evaluator_cache =
719 manifest_entry_context.expression_evaluator_cache.as_ref();
720
721 let expression_evaluator = expression_evaluator_cache.get(
722 manifest_entry_context.partition_spec_id,
723 &bound_predicates.partition_bound_predicate,
724 )?;
725
726 if !expression_evaluator.eval(manifest_entry_context.manifest_entry.data_file())? {
729 return Ok(());
730 }
731 }
732
733 delete_file_ctx_tx
734 .send(DeleteFileContext {
735 manifest_entry: manifest_entry_context.manifest_entry.clone(),
736 partition_spec_id: manifest_entry_context.partition_spec_id,
737 })
738 .await?;
739
740 Ok(())
741 }
742}
743
744pub(crate) struct BoundPredicates {
745 partition_bound_predicate: BoundPredicate,
746 snapshot_bound_predicate: BoundPredicate,
747}
748
749#[cfg(test)]
750mod tests {
751 use std::collections::HashMap;
754 use std::fs;
755 use std::fs::File;
756 use std::sync::Arc;
757
758 use arrow_array::cast::AsArray;
759 use arrow_array::types::Int32Type;
760 use arrow_array::{
761 Array, ArrayRef, BooleanArray, Float64Array, Int32Array, Int64Array, RecordBatch, RunArray,
762 StringArray,
763 };
764 use futures::{StreamExt, TryStreamExt, stream};
765 use parquet::arrow::ArrowWriter;
766 use uuid::Uuid;
767
768 use super::{resolve_column_stats_columns, resolve_field_id};
769 use crate::arrow::{ArrowReaderBuilder, ArrowReaderOptions, ParquetReadOptions};
770 use crate::expr::{BoundPredicate, Reference};
771 use crate::io::FileIO;
772 use crate::metadata_columns::{
773 RESERVED_COL_NAME_FILE, RESERVED_COL_NAME_LAST_UPDATED_SEQUENCE_NUMBER,
774 RESERVED_COL_NAME_POS, RESERVED_COL_NAME_SPEC_ID, RESERVED_FIELD_ID_POS,
775 };
776 use crate::scan::{FileScanTask, FileScanTaskColumnStats, FileScanTaskDeleteFile, TableScan};
777 use crate::spec::{
778 DataContentType, DataFileBuilder, DataFileFormat, Datum, ListType, Literal, MAIN_BRANCH,
779 ManifestEntry, ManifestListWriter, ManifestStatus, ManifestWriterBuilder, MappedField,
780 NameMapping, NestedField, NullOrder, Operation, PartitionSpec, PrimitiveType, Schema,
781 Snapshot, SortDirection, SortField, SortOrder, Struct, StructType, Summary,
782 TableMetadataBuilder, TableProperties, Transform, Type, UnboundPartitionSpec,
783 };
784 use crate::table::Table;
785 use crate::test_utils::scan::{
786 ROWS_PER_FIXTURE_FILE, TableTestFixture, assert_last_updated_seq_all,
787 };
788 use crate::test_utils::test_runtime;
789 use crate::util::snapshot::snapshot_id_as_of_time;
790 use crate::{ErrorKind, Result, Runtime, TableIdent};
791
792 #[tokio::test]
793 async fn test_table_scan_columns() {
794 let table = TableTestFixture::new().table;
795
796 let table_scan = table.scan().select(["x", "y"]).build().unwrap();
797 assert_eq!(
798 Some(vec!["x".to_string(), "y".to_string()]),
799 table_scan.column_names
800 );
801
802 let table_scan = table
803 .scan()
804 .select(["x", "y"])
805 .select(["z"])
806 .build()
807 .unwrap();
808 assert_eq!(Some(vec!["z".to_string()]), table_scan.column_names);
809 }
810
811 #[tokio::test]
812 async fn test_select_all() {
813 let table = TableTestFixture::new().table;
814
815 let table_scan = table.scan().select_all().build().unwrap();
816 assert!(table_scan.column_names.is_none());
817 }
818
819 #[test]
820 fn test_select_no_exist_column() {
821 let table = TableTestFixture::new().table;
822
823 let table_scan = table.scan().select(["x", "y", "z", "a", "b"]).build();
824 assert!(table_scan.is_err());
825 }
826
827 #[test]
828 fn test_case_sensitive_scan_rejects_mismatched_column_case() {
829 let table = TableTestFixture::new().table;
830
831 assert!(table.scan().select(["X"]).build().is_err());
833 assert!(
834 table
835 .scan()
836 .with_filter(Reference::new("X").greater_than(Datum::long(1)))
837 .build()
838 .is_err()
839 );
840 }
841
842 fn nested_schema() -> Schema {
844 Schema::builder()
845 .with_fields(vec![
846 NestedField::required(
847 1,
848 "s",
849 Type::Struct(StructType::new(vec![
850 NestedField::optional(2, "f", PrimitiveType::Long).into(),
851 ])),
852 )
853 .into(),
854 NestedField::required(
855 3,
856 "l",
857 Type::List(ListType::new(
858 NestedField::list_element(4, PrimitiveType::Long, true).into(),
859 )),
860 )
861 .into(),
862 NestedField::required(5, "n", PrimitiveType::Long).into(),
863 ])
864 .build()
865 .unwrap()
866 }
867
868 fn column_names(names: &[&str]) -> Vec<String> {
869 names.iter().map(|name| name.to_string()).collect()
870 }
871
872 #[test]
873 fn test_resolve_column_stats_columns_resolves_primitive_names() {
874 let schema = nested_schema();
875 let long = PrimitiveType::Long;
876
877 assert_eq!(
878 resolve_column_stats_columns(&schema, Some(&column_names(&["s.f", "n"])), true)
879 .unwrap(),
880 Some(Arc::<[_]>::from([(2, long.clone()), (5, long.clone())]))
881 );
882 assert_eq!(
883 resolve_column_stats_columns(&schema, Some(&column_names(&["N"])), false).unwrap(),
884 Some(Arc::<[_]>::from([(5, long)]))
885 );
886 assert_eq!(
887 resolve_column_stats_columns(&schema, None, true).unwrap(),
888 None
889 );
890 }
891
892 #[test]
893 fn test_resolve_column_stats_columns_rejects_names_without_statistics() {
894 let schema = nested_schema();
895 let cases = [
896 (
897 "s",
898 "Column s is not a primitive field and has no column statistics",
899 ),
900 (
901 "l",
902 "Column l is not a primitive field and has no column statistics",
903 ),
904 (
905 RESERVED_COL_NAME_FILE,
906 "Metadata column _file has no column statistics",
907 ),
908 ("missing", "Column missing not found in table"),
909 ("N", "Column N not found in table"),
910 ];
911
912 for (name, expected) in cases {
913 let err = resolve_column_stats_columns(&schema, Some(&column_names(&[name])), true)
914 .unwrap_err();
915 assert_eq!(err.kind(), ErrorKind::DataInvalid, "{name}");
916 assert!(err.message().starts_with(expected), "{name}: {err}");
917 }
918 }
919
920 #[test]
921 fn test_resolve_field_id_case_insensitive_rejects_ambiguous_name() {
922 let schema = Schema::builder()
923 .with_fields(vec![
924 NestedField::optional(1, "id", PrimitiveType::Int).into(),
925 NestedField::optional(2, "ID", PrimitiveType::Int).into(),
926 NestedField::optional(3, "data", PrimitiveType::Int).into(),
927 ])
928 .build()
929 .unwrap();
930
931 assert_eq!(resolve_field_id(&schema, "id", true).unwrap(), Some(1));
932 assert_eq!(resolve_field_id(&schema, "DATA", false).unwrap(), Some(3));
933 let error = resolve_field_id(&schema, "Id", false).unwrap_err();
934 assert_eq!(error.kind(), ErrorKind::DataInvalid);
935 assert!(error.message().contains("case-insensitively"), "{error}");
936 }
937
938 #[test]
939 fn test_case_insensitive_scan_with_ambiguous_schema() {
940 let fixture = TableTestFixture::new();
941 let mut metadata = fixture.table.metadata().clone();
942 let schema = Schema::builder()
943 .with_schema_id(metadata.current_schema_id())
944 .with_fields(vec![
945 NestedField::optional(1, "id", PrimitiveType::Int).into(),
946 NestedField::optional(2, "ID", PrimitiveType::Int).into(),
947 NestedField::optional(3, "data", PrimitiveType::Int).into(),
948 ])
949 .build()
950 .unwrap();
951 metadata
952 .schemas
953 .insert(metadata.current_schema_id(), Arc::new(schema));
954 let table = fixture.table.with_metadata(Arc::new(metadata));
955
956 table
957 .scan()
958 .with_case_sensitive(false)
959 .select(["DATA"])
960 .build()
961 .unwrap();
962 let error = table
963 .scan()
964 .with_case_sensitive(false)
965 .select(["Id"])
966 .build()
967 .unwrap_err();
968 assert_eq!(error.kind(), ErrorKind::DataInvalid);
969 assert!(error.message().contains("case-insensitively"), "{error}");
970 }
971
972 #[tokio::test]
973 async fn test_case_insensitive_scan_resolves_mismatched_column_case() {
974 let mut fixture = TableTestFixture::new();
975 fixture.setup_manifest_files().await;
976
977 let table_scan = fixture
980 .table
981 .scan()
982 .with_case_sensitive(false)
983 .select(["X", "Z"])
984 .with_filter(Reference::new("Y").greater_than(Datum::long(1)))
985 .build()
986 .unwrap();
987
988 let batches: Vec<_> = table_scan
989 .to_arrow(ArrowReaderOptions::default())
990 .await
991 .unwrap()
992 .try_collect()
993 .await
994 .unwrap();
995
996 assert_eq!(batches[0].num_columns(), 2);
997 assert_eq!(
998 batches[0]
999 .column_by_name("x")
1000 .unwrap()
1001 .as_any()
1002 .downcast_ref::<Int64Array>()
1003 .unwrap()
1004 .value(0),
1005 1
1006 );
1007 assert_eq!(
1008 batches[0]
1009 .column_by_name("z")
1010 .unwrap()
1011 .as_any()
1012 .downcast_ref::<Int64Array>()
1013 .unwrap()
1014 .value(0),
1015 3
1016 );
1017 }
1018
1019 #[tokio::test]
1020 async fn test_table_scan_default_snapshot_id() {
1021 let table = TableTestFixture::new().table;
1022
1023 let table_scan = table.scan().build().unwrap();
1024 assert_eq!(
1025 table.metadata().current_snapshot().unwrap().snapshot_id(),
1026 table_scan.snapshot().unwrap().snapshot_id()
1027 );
1028 }
1029
1030 #[test]
1031 fn test_table_scan_non_exist_snapshot_id() {
1032 let table = TableTestFixture::new().table;
1033
1034 let table_scan = table.scan().snapshot_id(1024).build();
1035 assert!(table_scan.is_err());
1036 }
1037
1038 #[tokio::test]
1039 async fn test_table_scan_with_snapshot_id() {
1040 let table = TableTestFixture::new().table;
1041
1042 let table_scan = table
1043 .scan()
1044 .snapshot_id(3051729675574597004)
1045 .build()
1046 .unwrap();
1047 assert_eq!(
1048 table_scan.snapshot().unwrap().snapshot_id(),
1049 3051729675574597004
1050 );
1051 }
1052
1053 #[test]
1054 fn test_table_scan_as_of_time_after_rollback() {
1055 for elapsed_ms in [0, 1000] {
1056 let table = TableTestFixture::new().table;
1057 let mut metadata = table.metadata().clone();
1058 let original = metadata.history()[0].snapshot_id;
1059 let previous = metadata.history()[1].clone();
1060 let rollback_time = previous.timestamp_ms + elapsed_ms;
1061 metadata.snapshot_log.push(crate::spec::SnapshotLog {
1062 timestamp_ms: rollback_time,
1063 snapshot_id: original,
1064 });
1065 metadata.current_snapshot_id = Some(original);
1066 metadata.refs.get_mut(MAIN_BRANCH).unwrap().snapshot_id = original;
1067 let table = table.with_metadata(Arc::new(metadata));
1068 let scan = table.scan().as_of_time(rollback_time).build().unwrap();
1069 let expected = if elapsed_ms == 0 {
1071 previous.snapshot_id
1072 } else {
1073 original
1074 };
1075 assert_eq!(scan.snapshot().unwrap().snapshot_id(), expected);
1076 }
1077 }
1078
1079 #[test]
1080 fn test_table_scan_as_of_time_rejects_empty_history() {
1081 let table = TableTestFixture::new_empty().table;
1082 assert!(table.scan().build().unwrap().snapshot().is_none());
1083 let err = table.scan().as_of_time(0).build().unwrap_err();
1084 assert_eq!(err.kind(), ErrorKind::DataInvalid);
1085 assert!(err.message().contains("No snapshot history"));
1086 }
1087
1088 #[test]
1089 fn test_table_scan_as_of_time_rejects_before_history() {
1090 let table = TableTestFixture::new().table;
1091 let before = table.metadata().history()[0].timestamp_ms - 1;
1092 let err = table.scan().as_of_time(before).build().unwrap_err();
1093 assert_eq!(err.kind(), ErrorKind::DataInvalid);
1094 assert!(err.message().contains(&before.to_string()));
1095 }
1096
1097 #[test]
1098 fn test_table_scan_as_of_time_rejects_all_qualifying_snapshots_expired() {
1099 let table = TableTestFixture::new_with_deep_history().table;
1100 let mut metadata = table.metadata().clone();
1101 let entry = metadata.history()[3].clone();
1102 let expired_ids: Vec<_> = metadata
1103 .history()
1104 .iter()
1105 .filter(|log| log.timestamp_ms <= entry.timestamp_ms)
1106 .map(|log| log.snapshot_id)
1107 .collect();
1108 assert_eq!(expired_ids.len(), 4);
1109 for snapshot_id in expired_ids {
1110 metadata.snapshots.remove(&snapshot_id);
1111 }
1112 let table = table.with_metadata(Arc::new(metadata));
1113 assert_eq!(
1115 snapshot_id_as_of_time(&table.metadata_ref(), entry.timestamp_ms).unwrap(),
1116 entry.snapshot_id
1117 );
1118 assert!(table.scan().build().unwrap().snapshot().is_some());
1119 let err = table
1120 .scan()
1121 .as_of_time(entry.timestamp_ms)
1122 .build()
1123 .unwrap_err();
1124 assert_eq!(err.kind(), ErrorKind::DataInvalid);
1125 assert_eq!(
1126 err.message(),
1127 format!(
1128 "Snapshot with id {} resolved from timestamp {} ms not found; it may have expired",
1129 entry.snapshot_id, entry.timestamp_ms
1130 )
1131 );
1132 }
1133
1134 #[test]
1135 fn test_table_scan_last_snapshot_selection_wins() {
1136 let table = TableTestFixture::new().table;
1137 let first = &table.metadata().history()[0];
1138 let last = &table.metadata().history()[1];
1139 let cases = [
1140 (
1141 table
1142 .scan()
1143 .snapshot_id(last.snapshot_id)
1144 .as_of_time(first.timestamp_ms),
1145 first.snapshot_id,
1146 ),
1147 (
1148 table
1149 .scan()
1150 .as_of_time(first.timestamp_ms)
1151 .snapshot_id(last.snapshot_id),
1152 last.snapshot_id,
1153 ),
1154 (
1156 table.scan().snapshot_id(-1).as_of_time(first.timestamp_ms),
1157 first.snapshot_id,
1158 ),
1159 (
1160 table
1161 .scan()
1162 .as_of_time(i64::MIN)
1163 .snapshot_id(last.snapshot_id),
1164 last.snapshot_id,
1165 ),
1166 (
1167 table
1168 .scan()
1169 .snapshot_id(first.snapshot_id)
1170 .snapshot_id(last.snapshot_id),
1171 last.snapshot_id,
1172 ),
1173 (
1174 table
1175 .scan()
1176 .as_of_time(i64::MAX)
1177 .as_of_time(first.timestamp_ms),
1178 first.snapshot_id,
1179 ),
1180 ];
1181 for (builder, expected) in cases {
1182 let scan = builder.build().unwrap();
1183 assert_eq!(scan.snapshot().unwrap().snapshot_id(), expected);
1184 }
1185 }
1186
1187 #[test]
1188 fn test_table_scan_invalid_last_snapshot_selection() {
1189 let table = TableTestFixture::new().table;
1190 let first = &table.metadata().history()[0];
1191 let cases = [
1192 (
1193 table
1194 .scan()
1195 .snapshot_id(first.snapshot_id)
1196 .as_of_time(first.timestamp_ms - 1),
1197 "No snapshot history",
1198 ),
1199 (
1200 table.scan().as_of_time(first.timestamp_ms).snapshot_id(-1),
1201 "Snapshot with id -1 not found",
1202 ),
1203 ];
1204 for (builder, expected) in cases {
1205 let err = builder.build().unwrap_err();
1206 assert_eq!(err.kind(), ErrorKind::DataInvalid);
1207 assert!(err.message().contains(expected));
1208 }
1209 }
1210
1211 #[test]
1212 fn test_table_scan_as_of_time_uses_snapshot_schema() {
1213 for select_current in [false, true] {
1214 let table = TableTestFixture::new().table;
1215 let mut metadata = table.metadata().clone();
1216 let entry = metadata.history()[usize::from(select_current)].clone();
1217 Arc::make_mut(metadata.snapshots.get_mut(&entry.snapshot_id).unwrap()).schema_id =
1220 Some(0);
1221 let table = table.with_metadata(Arc::new(metadata));
1222 let scan = table
1223 .scan()
1224 .as_of_time(entry.timestamp_ms)
1225 .select(["x"])
1226 .with_filter(Reference::new("x").greater_than(Datum::long(0)))
1227 .build()
1228 .unwrap();
1229 let context = scan.plan_context.as_ref().unwrap();
1230 assert_eq!(context.snapshot_schema.schema_id(), 0);
1231 assert!(context.snapshot_bound_predicate.is_some());
1232 assert!(
1233 table
1234 .scan()
1235 .as_of_time(entry.timestamp_ms)
1236 .select(["y"])
1237 .build()
1238 .is_err()
1239 );
1240 assert!(
1241 table
1242 .scan()
1243 .as_of_time(entry.timestamp_ms)
1244 .with_filter(Reference::new("y").greater_than(Datum::long(0)))
1245 .build()
1246 .is_err()
1247 );
1248 }
1249 }
1250
1251 #[test]
1252 fn test_table_scan_as_of_time_schema_compatibility() {
1253 let table = TableTestFixture::new().table;
1254 let entry = table.metadata().history()[0].clone();
1255 assert!(
1257 table
1258 .metadata()
1259 .snapshot_by_id(entry.snapshot_id)
1260 .unwrap()
1261 .schema_id()
1262 .is_none()
1263 );
1264 let scan = table.scan().as_of_time(entry.timestamp_ms).build().unwrap();
1265 assert_eq!(
1266 scan.plan_context.as_ref().unwrap().snapshot_schema,
1267 *table.metadata().current_schema()
1268 );
1269
1270 let mut metadata = table.metadata().clone();
1271 Arc::make_mut(metadata.snapshots.get_mut(&entry.snapshot_id).unwrap()).schema_id =
1272 Some(1234);
1273 let table = table.with_metadata(Arc::new(metadata));
1274 let err = table
1275 .scan()
1276 .as_of_time(entry.timestamp_ms)
1277 .build()
1278 .unwrap_err();
1279 assert_eq!(err.kind(), ErrorKind::DataInvalid);
1280 assert!(err.message().contains("Schema with id 1234 not found"));
1281 }
1282
1283 #[tokio::test]
1284 async fn test_table_scan_as_of_time_reads_historical_rows() {
1285 let mut fixture = TableTestFixture::new();
1286 fixture.setup_manifest_files().await;
1287 let entry = fixture.table.metadata().history()[0].clone();
1288
1289 let mut metadata = fixture.table.metadata().clone();
1291 Arc::make_mut(metadata.snapshots.get_mut(&entry.snapshot_id).unwrap()).schema_id = Some(0);
1292 fixture.table = fixture.table.with_metadata(Arc::new(metadata));
1293 let snapshot = fixture
1294 .table
1295 .metadata()
1296 .snapshot_by_id(entry.snapshot_id)
1297 .unwrap();
1298 let schema = snapshot.schema(fixture.table.metadata()).unwrap();
1299 let arrow_schema = Arc::new(crate::arrow::schema_to_arrow_schema(&schema).unwrap());
1300
1301 let path = format!("{}/historical.parquet", fixture.table_location);
1303 let batch = RecordBatch::try_new(arrow_schema.clone(), vec![Arc::new(Int64Array::from(
1304 vec![100],
1305 ))])
1306 .unwrap();
1307 let mut writer =
1308 ArrowWriter::try_new(File::create(&path).unwrap(), arrow_schema, None).unwrap();
1309 writer.write(&batch).unwrap();
1310 writer.close().unwrap();
1311
1312 let mut manifest_writer = ManifestWriterBuilder::new(
1313 fixture.next_manifest_file(),
1314 Some(snapshot.snapshot_id()),
1315 schema,
1316 fixture
1317 .table
1318 .metadata()
1319 .default_partition_spec()
1320 .as_ref()
1321 .clone(),
1322 )
1323 .build_v2_data();
1324 manifest_writer
1325 .add_entry(
1326 ManifestEntry::builder()
1327 .status(ManifestStatus::Added)
1328 .data_file(
1329 DataFileBuilder::default()
1330 .partition_spec_id(0)
1331 .content(DataContentType::Data)
1332 .file_format(DataFileFormat::Parquet)
1333 .file_size_in_bytes(fs::metadata(&path).unwrap().len())
1334 .file_path(path)
1335 .record_count(1)
1336 .partition(Struct::from_iter([Some(Literal::long(100))]))
1337 .build()
1338 .unwrap(),
1339 )
1340 .build(),
1341 )
1342 .unwrap();
1343 let manifest = manifest_writer.write_manifest_file().await.unwrap();
1344 let output = fixture
1345 .table
1346 .file_io()
1347 .new_output(snapshot.manifest_list())
1348 .unwrap()
1349 .writer()
1350 .await
1351 .unwrap();
1352 let mut list_writer = ManifestListWriter::v2(
1353 output,
1354 snapshot.snapshot_id(),
1355 snapshot.parent_snapshot_id(),
1356 snapshot.sequence_number(),
1357 );
1358 list_writer.add_manifests([manifest].into_iter()).unwrap();
1359 list_writer.close().await.unwrap();
1360
1361 let table = fixture.table;
1362 let [by_time, by_id] = [
1363 table.scan().as_of_time(entry.timestamp_ms),
1364 table.scan().snapshot_id(entry.snapshot_id),
1365 ]
1366 .map(|scan| {
1367 scan.with_filter(Reference::new("x").greater_than_or_equal_to(Datum::long(100)))
1368 .build()
1369 .unwrap()
1370 });
1371 let current = table.scan().build().unwrap();
1372 let mut planned_files = Vec::new();
1373 let mut rows = Vec::new();
1374 let mut schemas = Vec::new();
1375 for scan in [by_time, by_id, current] {
1376 let mut tasks: Vec<_> = scan
1377 .plan_files()
1378 .await
1379 .unwrap()
1380 .try_collect()
1381 .await
1382 .unwrap();
1383 tasks.sort_by_key(|task| task.data_file_path().to_string());
1384 planned_files.push(tasks);
1385 let batches: Vec<_> = scan
1386 .to_arrow(ArrowReaderOptions::default())
1387 .await
1388 .unwrap()
1389 .try_collect()
1390 .await
1391 .unwrap();
1392 assert!(!batches.is_empty());
1393 schemas.push(batches[0].schema());
1394 let mut values: Vec<i64> = batches
1395 .iter()
1396 .flat_map(|batch| {
1397 batch
1398 .column_by_name("x")
1399 .unwrap()
1400 .as_any()
1401 .downcast_ref::<Int64Array>()
1402 .unwrap()
1403 .values()
1404 .iter()
1405 .copied()
1406 })
1407 .collect();
1408 values.sort_unstable();
1409 rows.push(values);
1410 }
1411 assert!(!planned_files[0].is_empty());
1412 assert_eq!(planned_files[0], planned_files[1]);
1413 assert_eq!(rows[0], vec![100]);
1414 assert_eq!(rows[0], rows[1]);
1415 assert_eq!(rows[2], vec![1; 2 * ROWS_PER_FIXTURE_FILE]);
1417 assert_eq!(schemas[0], schemas[1]);
1418 assert_eq!(schemas[0].fields().len(), 1);
1419 assert_eq!(schemas[2].fields().len(), 8);
1420 }
1421
1422 #[test]
1425 fn test_scan_of_current_state_uses_current_schema() {
1426 let table = TableTestFixture::new_with_column_added_after_last_write().table;
1427
1428 let table_scan = table.scan().build().unwrap();
1429 assert_eq!(table_scan.snapshot(), table.metadata().current_snapshot());
1430 assert_eq!(
1431 table_scan.plan_context.as_ref().unwrap().snapshot_schema,
1432 *table.metadata().current_schema()
1433 );
1434 }
1435
1436 #[test]
1439 fn test_time_travel_scan_uses_snapshot_schema() {
1440 let table = TableTestFixture::new_with_column_added_after_last_write().table;
1441 let snapshot = table.metadata().current_snapshot().unwrap();
1442
1443 let table_scan = table
1444 .scan()
1445 .snapshot_id(snapshot.snapshot_id())
1446 .build()
1447 .unwrap();
1448 assert_eq!(table_scan.snapshot(), Some(snapshot));
1449 assert_eq!(
1450 table_scan.plan_context.as_ref().unwrap().snapshot_schema,
1451 snapshot.schema(table.metadata()).unwrap()
1452 );
1453 }
1454
1455 fn table_with_property(key: &str, value: &str) -> Table {
1456 let fixture = TableTestFixture::new();
1457 let mut metadata = fixture.table.metadata().clone();
1458 metadata
1459 .properties
1460 .insert(key.to_string(), value.to_string());
1461 Table::builder()
1462 .metadata(metadata)
1463 .identifier(fixture.table.identifier().clone())
1464 .file_io(fixture.table.file_io().clone())
1465 .metadata_location(fixture.table.metadata_location().unwrap().to_string())
1466 .runtime(test_runtime())
1467 .build()
1468 .unwrap()
1469 }
1470
1471 #[test]
1472 fn test_table_scan_without_name_mapping_property() {
1473 let table = TableTestFixture::new().table;
1474
1475 let table_scan = table.scan().build().unwrap();
1476 assert!(
1477 table_scan
1478 .plan_context
1479 .as_ref()
1480 .unwrap()
1481 .name_mapping
1482 .is_none()
1483 );
1484 }
1485
1486 #[test]
1487 fn test_table_scan_with_name_mapping_property() {
1488 let mapping_json = r#"[{"field-id":1,"names":["id","record_id"]}]"#;
1489 let table =
1490 table_with_property(TableProperties::PROPERTY_DEFAULT_NAME_MAPPING, mapping_json);
1491
1492 let table_scan = table.scan().build().unwrap();
1493 let mapping = table_scan
1494 .plan_context
1495 .as_ref()
1496 .unwrap()
1497 .name_mapping
1498 .as_ref()
1499 .expect("name_mapping should be parsed from the table property");
1500 let fields = mapping.fields();
1501 assert_eq!(fields.len(), 1);
1502 assert_eq!(fields[0].field_id(), Some(1));
1503 assert_eq!(fields[0].names(), &[
1504 "id".to_string(),
1505 "record_id".to_string()
1506 ]);
1507 }
1508
1509 #[test]
1510 fn test_table_scan_with_malformed_name_mapping_property() {
1511 let table = table_with_property(
1512 TableProperties::PROPERTY_DEFAULT_NAME_MAPPING,
1513 "{ not valid json",
1514 );
1515
1516 let err = table
1517 .scan()
1518 .build()
1519 .expect_err("malformed name mapping should fail to parse");
1520 assert_eq!(err.kind(), ErrorKind::DataInvalid);
1521 }
1522
1523 #[tokio::test]
1524 async fn test_plan_files_carries_name_mapping_into_file_scan_task() {
1525 let mut fixture = TableTestFixture::new();
1526 fixture.setup_manifest_files().await;
1527
1528 let mapping_json = r#"[{"field-id":1,"names":["id","record_id"]}]"#;
1529 let mut metadata = fixture.table.metadata().clone();
1530 metadata.properties.insert(
1531 TableProperties::PROPERTY_DEFAULT_NAME_MAPPING.to_string(),
1532 mapping_json.to_string(),
1533 );
1534 let table = Table::builder()
1535 .metadata(metadata)
1536 .identifier(fixture.table.identifier().clone())
1537 .file_io(fixture.table.file_io().clone())
1538 .metadata_location(fixture.table.metadata_location().unwrap().to_string())
1539 .runtime(test_runtime())
1540 .build()
1541 .unwrap();
1542
1543 let tasks: Vec<_> = table
1544 .scan()
1545 .build()
1546 .unwrap()
1547 .plan_files()
1548 .await
1549 .unwrap()
1550 .try_collect()
1551 .await
1552 .unwrap();
1553
1554 assert!(!tasks.is_empty(), "expected at least one FileScanTask");
1555 for task in &tasks {
1556 let mapping = task
1557 .name_mapping()
1558 .expect("name_mapping should reach the FileScanTask");
1559 assert_eq!(mapping.fields().len(), 1);
1560 assert_eq!(mapping.fields()[0].field_id(), Some(1));
1561 }
1562 }
1563
1564 #[tokio::test]
1565 async fn test_plan_files_carries_sort_order_into_file_scan_task() {
1566 let mut fixture = TableTestFixture::new();
1567
1568 let mut metadata = fixture.table.metadata().clone();
1572 metadata
1573 .sort_orders
1574 .insert(0, Arc::new(SortOrder::unsorted_order()));
1575 fixture.table = fixture.table.with_metadata(Arc::new(metadata));
1576
1577 let expected_sort_order = fixture
1578 .table
1579 .metadata()
1580 .sort_order_by_id(3)
1581 .unwrap()
1582 .clone();
1583
1584 fixture
1586 .setup_manifest_files_with_sort_order_ids([Some(3), None, Some(99), Some(0)])
1587 .await;
1588
1589 let tasks: Vec<_> = fixture
1590 .table
1591 .scan()
1592 .build()
1593 .unwrap()
1594 .plan_files()
1595 .await
1596 .unwrap()
1597 .try_collect()
1598 .await
1599 .unwrap();
1600
1601 assert_eq!(tasks.len(), 4, "expected all four FileScanTasks");
1602
1603 assert_eq!(
1606 tasks.iter().filter(|t| t.sort_order().is_some()).count(),
1607 1,
1608 "exactly one file resolves to a sort order"
1609 );
1610 assert_eq!(
1611 tasks.iter().filter(|t| t.sort_order_id().is_some()).count(),
1612 3,
1613 "three files carry a raw sort_order_id"
1614 );
1615
1616 let resolved = tasks
1617 .iter()
1618 .find(|t| t.data_file_path().ends_with("1.parquet"))
1619 .unwrap();
1620 assert_eq!(resolved.sort_order_id(), Some(3));
1621 assert_eq!(
1622 resolved.sort_order(),
1623 Some(&expected_sort_order),
1624 "sort_order_id 3 should resolve to the table's sort order at id 3"
1625 );
1626
1627 let missing = tasks
1628 .iter()
1629 .find(|t| t.data_file_path().ends_with("2.parquet"))
1630 .unwrap();
1631 assert_eq!(missing.sort_order_id(), None);
1632 assert!(
1633 missing.sort_order().is_none(),
1634 "a file with no sort_order_id carries no sort_order"
1635 );
1636
1637 let unresolvable = tasks
1638 .iter()
1639 .find(|t| t.data_file_path().ends_with("3.parquet"))
1640 .unwrap();
1641 assert_eq!(
1642 unresolvable.sort_order_id(),
1643 Some(99),
1644 "the raw id is preserved even when it does not resolve"
1645 );
1646 assert!(
1647 unresolvable.sort_order().is_none(),
1648 "an unresolvable sort_order_id resolves to no sort_order"
1649 );
1650
1651 let unsorted = tasks
1652 .iter()
1653 .find(|t| t.data_file_path().ends_with("4.parquet"))
1654 .unwrap();
1655 assert_eq!(unsorted.sort_order_id(), Some(0));
1656 assert!(
1657 unsorted.sort_order().is_none(),
1658 "the reserved unsorted order (id 0) resolves to no sort_order"
1659 );
1660 }
1661
1662 fn column_stats(field_ids: &[i32]) -> FileScanTaskColumnStats {
1665 let counts = |count: fn(u64) -> u64| -> HashMap<i32, u64> {
1666 field_ids.iter().map(|&id| (id, count(id as u64))).collect()
1667 };
1668 let bounds = |offset: i64| -> HashMap<i32, Datum> {
1669 field_ids
1670 .iter()
1671 .map(|&id| (id, Datum::long(offset + i64::from(id))))
1672 .collect()
1673 };
1674 FileScanTaskColumnStats::builder()
1675 .with_column_sizes(counts(|id| 8 * id))
1676 .with_value_counts(counts(|id| 10 + id))
1677 .with_null_value_counts(counts(|id| id))
1678 .with_nan_value_counts(counts(|id| 2 * id))
1679 .with_lower_bounds(bounds(0))
1680 .with_upper_bounds(bounds(100))
1681 .build()
1682 }
1683
1684 async fn column_stats_by_file(scan: TableScan) -> Vec<Option<FileScanTaskColumnStats>> {
1685 let mut tasks: Vec<_> = scan
1686 .plan_files()
1687 .await
1688 .unwrap()
1689 .try_collect()
1690 .await
1691 .unwrap();
1692 tasks.sort_by(|a, b| a.data_file_path().cmp(b.data_file_path()));
1693 tasks
1694 .into_iter()
1695 .map(|task| task.column_stats().cloned())
1696 .collect()
1697 }
1698
1699 #[tokio::test]
1700 async fn test_plan_files_retains_only_requested_column_stats() {
1701 let mut fixture = TableTestFixture::new();
1702 fixture
1704 .setup_manifest_files_with_column_stats(&[(None, vec![
1705 column_stats(&[1, 2, 7]),
1706 column_stats(&[2, 7]),
1707 ])])
1708 .await;
1709
1710 let scan = fixture
1711 .table
1712 .scan()
1713 .with_column_stats(["x", "i64"])
1714 .build()
1715 .unwrap();
1716
1717 assert_eq!(column_stats_by_file(scan).await, vec![
1718 Some(column_stats(&[1, 7])),
1719 Some(column_stats(&[7]))
1720 ]);
1721 }
1722
1723 #[tokio::test]
1724 async fn test_plan_files_carries_column_stats_only_when_requested() {
1725 let mut fixture = TableTestFixture::new();
1726 fixture
1727 .setup_manifest_files_with_column_stats(&[(None, vec![column_stats(&[1])])])
1728 .await;
1729 let table = &fixture.table;
1730
1731 assert_eq!(
1732 column_stats_by_file(table.scan().build().unwrap()).await,
1733 vec![None]
1734 );
1735 assert_eq!(
1736 column_stats_by_file(
1737 table
1738 .scan()
1739 .with_column_stats(Vec::<String>::new())
1740 .build()
1741 .unwrap()
1742 )
1743 .await,
1744 vec![Some(FileScanTaskColumnStats::default())]
1745 );
1746 }
1747
1748 #[tokio::test]
1749 async fn test_plan_files_resolves_column_stats_names_with_scan_case_sensitivity() {
1750 let mut fixture = TableTestFixture::new();
1751 fixture
1752 .setup_manifest_files_with_column_stats(&[(None, vec![column_stats(&[1, 2])])])
1753 .await;
1754
1755 let scan = fixture
1756 .table
1757 .scan()
1758 .with_case_sensitive(false)
1759 .with_column_stats(["X"])
1760 .build()
1761 .unwrap();
1762
1763 assert_eq!(column_stats_by_file(scan).await, vec![Some(column_stats(
1764 &[1]
1765 ))]);
1766 }
1767
1768 #[test]
1769 fn test_build_rejects_unknown_column_stats_name() {
1770 let table = TableTestFixture::new().table;
1771
1772 let err = table
1773 .scan()
1774 .with_column_stats(["unknown"])
1775 .build()
1776 .unwrap_err();
1777
1778 assert_eq!(err.kind(), ErrorKind::DataInvalid);
1779 }
1780
1781 #[tokio::test]
1782 async fn test_plan_files_promotes_bounds_to_scan_schema_type() {
1783 let mut fixture = TableTestFixture::new();
1784 let y_bounds = |lower: Datum, upper: Datum| {
1785 FileScanTaskColumnStats::builder()
1786 .with_lower_bounds(HashMap::from([(2, lower)]))
1787 .with_upper_bounds(HashMap::from([(2, upper)]))
1788 .build()
1789 };
1790 let int_schema = Arc::new(
1792 Schema::builder()
1793 .with_fields(vec![
1794 NestedField::required(1, "x", PrimitiveType::Long).into(),
1795 NestedField::required(2, "y", PrimitiveType::Int).into(),
1796 ])
1797 .build()
1798 .unwrap(),
1799 );
1800 let written_as_long = y_bounds(Datum::long(6), Datum::long(9));
1801 fixture
1802 .setup_manifest_files_with_column_stats(&[
1803 (Some(int_schema), vec![y_bounds(
1804 Datum::int(5),
1805 Datum::int(7),
1806 )]),
1807 (None, vec![written_as_long.clone()]),
1808 ])
1809 .await;
1810
1811 let scan = fixture
1812 .table
1813 .scan()
1814 .with_column_stats(["y"])
1815 .build()
1816 .unwrap();
1817
1818 assert_eq!(column_stats_by_file(scan).await, vec![
1819 Some(y_bounds(Datum::long(5), Datum::long(7))),
1820 Some(written_as_long)
1821 ]);
1822 }
1823
1824 #[tokio::test]
1825 async fn test_plan_files_on_table_without_any_snapshots() {
1826 let table = TableTestFixture::new_empty().table;
1827 let batch_stream = table
1828 .scan()
1829 .build()
1830 .unwrap()
1831 .to_arrow(ArrowReaderOptions::default())
1832 .await
1833 .unwrap();
1834 let batches: Vec<_> = batch_stream.try_collect().await.unwrap();
1835 assert!(batches.is_empty());
1836 }
1837
1838 #[tokio::test]
1839 async fn test_plan_files_no_deletions() {
1840 let mut fixture = TableTestFixture::new();
1841 fixture.setup_manifest_files().await;
1842
1843 let table_scan = fixture.table.scan().build().unwrap();
1845
1846 let mut tasks = table_scan
1847 .plan_files()
1848 .await
1849 .unwrap()
1850 .try_fold(vec![], |mut acc, task| async move {
1851 acc.push(task);
1852 Ok(acc)
1853 })
1854 .await
1855 .unwrap();
1856
1857 assert_eq!(tasks.len(), 2);
1858
1859 tasks.sort_by_key(|t| t.data_file_path().to_string());
1860
1861 assert_eq!(
1863 tasks[0].data_file_path(),
1864 format!("{}/1.parquet", &fixture.table_location)
1865 );
1866
1867 assert_eq!(
1869 tasks[1].data_file_path(),
1870 format!("{}/3.parquet", &fixture.table_location)
1871 );
1872 }
1873
1874 #[tokio::test]
1875 async fn test_scan_widens_old_partition_so_pruning_and_deletes_match() {
1876 let mut fixture = TableTestFixture::new();
1883 fixture.setup_promoted_partition_with_eq_delete(false).await;
1884
1885 let tasks: Vec<FileScanTask> = fixture
1886 .table
1887 .scan()
1888 .build()
1889 .unwrap()
1890 .plan_files()
1891 .await
1892 .unwrap()
1893 .try_collect()
1894 .await
1895 .unwrap();
1896 assert_eq!(tasks.len(), 1);
1897 assert_eq!(
1898 tasks[0].deletes().len(),
1899 1,
1900 "equality delete must attach after widening the old int partition to long"
1901 );
1902 assert_eq!(
1903 tasks[0].partition(),
1904 Some(&Struct::from_iter([Some(Literal::long(7))])),
1905 "the widened long(7) tuple must reach FileScanTask::partition()"
1906 );
1907
1908 let tasks: Vec<FileScanTask> = fixture
1909 .table
1910 .scan()
1911 .with_filter(Reference::new("x").equal_to(Datum::long(7)))
1912 .build()
1913 .unwrap()
1914 .plan_files()
1915 .await
1916 .unwrap()
1917 .try_collect()
1918 .await
1919 .unwrap();
1920 assert_eq!(
1921 tasks.len(),
1922 1,
1923 "filter on the promoted type must not prune the file"
1924 );
1925 assert_eq!(tasks[0].deletes().len(), 1);
1926
1927 let tasks: Vec<FileScanTask> = fixture
1928 .table
1929 .scan()
1930 .with_filter(Reference::new("x").equal_to(Datum::long(8)))
1931 .build()
1932 .unwrap()
1933 .plan_files()
1934 .await
1935 .unwrap()
1936 .try_collect()
1937 .await
1938 .unwrap();
1939 assert!(
1940 tasks.is_empty(),
1941 "a file in partition x=7 must be pruned by an x=8 filter"
1942 );
1943 }
1944
1945 #[tokio::test]
1946 async fn test_scan_widens_old_partition_with_pre_promotion_delete() {
1947 let mut fixture = TableTestFixture::new();
1951 fixture.setup_promoted_partition_with_eq_delete(true).await;
1952
1953 let tasks: Vec<FileScanTask> = fixture
1954 .table
1955 .scan()
1956 .build()
1957 .unwrap()
1958 .plan_files()
1959 .await
1960 .unwrap()
1961 .try_collect()
1962 .await
1963 .unwrap();
1964 assert_eq!(tasks.len(), 1);
1965 assert_eq!(tasks[0].deletes().len(), 1);
1966 }
1967
1968 #[tokio::test]
1969 async fn test_scan_widens_old_partition_so_deletion_vector_matches() {
1970 let mut fixture = TableTestFixture::new();
1974 fixture.setup_promoted_partition_with_dv().await;
1975
1976 let tasks: Vec<FileScanTask> = fixture
1977 .table
1978 .scan()
1979 .build()
1980 .unwrap()
1981 .plan_files()
1982 .await
1983 .unwrap()
1984 .try_collect()
1985 .await
1986 .unwrap();
1987 assert_eq!(tasks.len(), 1);
1988 assert_eq!(tasks[0].deletes().len(), 1);
1989 }
1990
1991 #[tokio::test]
1992 async fn test_scan_keeps_eq_delete_after_dropping_promoted_source() {
1993 let mut fixture = TableTestFixture::new();
1997 fixture.setup_promoted_partition_with_eq_delete(false).await;
1998 fixture.drop_partition_source(1);
1999 let table = fixture.table;
2000
2001 let tasks: Vec<FileScanTask> = table
2002 .scan()
2003 .build()
2004 .unwrap()
2005 .plan_files()
2006 .await
2007 .unwrap()
2008 .try_collect()
2009 .await
2010 .unwrap();
2011 assert_eq!(tasks.len(), 1);
2012 assert_eq!(tasks[0].deletes().len(), 1);
2013 }
2014
2015 #[tokio::test]
2016 async fn test_scan_keeps_deletion_vector_after_dropping_promoted_source() {
2017 let mut fixture = TableTestFixture::new();
2020 fixture.setup_promoted_partition_with_dv().await;
2021 fixture.drop_partition_source(1);
2022 let table = fixture.table;
2023
2024 let tasks: Vec<FileScanTask> = table
2025 .scan()
2026 .build()
2027 .unwrap()
2028 .plan_files()
2029 .await
2030 .unwrap()
2031 .try_collect()
2032 .await
2033 .unwrap();
2034 assert_eq!(tasks.len(), 1);
2035 assert_eq!(tasks[0].deletes().len(), 1);
2036 }
2037
2038 #[tokio::test]
2039 async fn test_scan_keeps_eq_delete_when_dropped_source_is_date() {
2040 let mut fixture = TableTestFixture::new();
2045 fixture
2046 .setup_promoted_partition_with_eq_delete_and_dropped_date()
2047 .await;
2048
2049 let tasks: Vec<FileScanTask> = fixture
2050 .table
2051 .scan()
2052 .build()
2053 .unwrap()
2054 .plan_files()
2055 .await
2056 .unwrap()
2057 .try_collect()
2058 .await
2059 .unwrap();
2060 assert_eq!(tasks.len(), 1);
2061 assert_eq!(tasks[0].deletes().len(), 1);
2062 }
2063
2064 #[tokio::test]
2065 async fn test_plan_files_carries_row_lineage_into_file_scan_task() {
2066 let mut fixture = TableTestFixture::new();
2067 fixture.setup_manifest_files().await;
2068
2069 let mut tasks: Vec<_> = fixture
2070 .table
2071 .scan()
2072 .build()
2073 .unwrap()
2074 .plan_files()
2075 .await
2076 .unwrap()
2077 .try_collect()
2078 .await
2079 .unwrap();
2080
2081 assert_eq!(tasks.len(), 2);
2082 tasks.sort_by_key(|task| task.data_file_path().to_string());
2083
2084 assert_eq!(
2087 tasks[0].data_file_path(),
2088 format!("{}/1.parquet", &fixture.table_location)
2089 );
2090 assert_eq!(tasks[0].data_sequence_number(), Some(1));
2091 assert_eq!(
2092 tasks[1].data_file_path(),
2093 format!("{}/3.parquet", &fixture.table_location)
2094 );
2095 assert_eq!(tasks[1].data_sequence_number(), Some(0));
2096
2097 assert!(tasks.iter().all(|task| task.first_row_id().is_none()));
2099 }
2100
2101 #[tokio::test]
2102 async fn test_plan_files_carries_row_lineage_from_v3_manifest() {
2103 let mut fixture = TableTestFixture::new();
2104 fixture.setup_v3_manifest_files().await;
2105
2106 let task = fixture
2107 .table
2108 .scan()
2109 .build()
2110 .unwrap()
2111 .plan_files()
2112 .await
2113 .unwrap()
2114 .try_collect::<Vec<_>>()
2115 .await
2116 .unwrap()
2117 .into_iter()
2118 .next()
2119 .expect("expected one FileScanTask");
2120
2121 assert_eq!(task.first_row_id(), Some(42));
2124 assert_eq!(task.data_sequence_number(), Some(1));
2126 }
2127
2128 #[tokio::test]
2129 async fn test_filtered_scan_with_dropped_partition_source_column() {
2130 let mut fixture = TableTestFixture::new();
2131 fixture.setup_manifest_files().await;
2132
2133 let baseline = scan_y_gte_5(&fixture.table).await;
2135 assert!(!baseline.is_empty());
2136 assert!(baseline.iter().all(|y| *y >= 5));
2137
2138 let current_schema = fixture.table.metadata().current_schema();
2142 let evolved_schema = Schema::builder()
2143 .with_fields(
2144 current_schema
2145 .as_struct()
2146 .fields()
2147 .iter()
2148 .filter(|field| field.id != 1)
2149 .cloned(),
2150 )
2151 .with_identifier_field_ids(vec![2])
2152 .build()
2153 .unwrap();
2154 let evolved =
2155 TableMetadataBuilder::new_from_metadata(fixture.table.metadata().clone(), None)
2156 .add_default_partition_spec(UnboundPartitionSpec::builder().build())
2157 .unwrap()
2158 .add_current_schema(evolved_schema)
2159 .unwrap()
2160 .build()
2161 .unwrap()
2162 .metadata;
2163
2164 let parent = evolved.current_snapshot().unwrap().clone();
2167 let snapshot = Snapshot::builder()
2168 .with_snapshot_id(parent.snapshot_id() + 1)
2169 .with_parent_snapshot_id(Some(parent.snapshot_id()))
2170 .with_sequence_number(evolved.last_sequence_number() + 1)
2171 .with_timestamp_ms(evolved.last_updated_ms + 1)
2172 .with_schema_id(evolved.current_schema_id())
2173 .with_manifest_list(parent.manifest_list())
2174 .with_summary(Summary {
2175 operation: Operation::Append,
2176 additional_properties: HashMap::new(),
2177 })
2178 .build();
2179 let metadata = TableMetadataBuilder::new_from_metadata(evolved, None)
2180 .set_branch_snapshot(snapshot, MAIN_BRANCH)
2181 .unwrap()
2182 .build()
2183 .unwrap()
2184 .metadata;
2185 let table = fixture.table.clone().with_metadata(Arc::new(metadata));
2186
2187 let evolved = scan_y_gte_5(&table).await;
2190 assert_eq!(evolved, baseline);
2191 }
2192
2193 async fn scan_y_gte_5(table: &Table) -> Vec<i64> {
2194 let table_scan = table
2195 .scan()
2196 .select(["y"])
2197 .with_filter(Reference::new("y").greater_than_or_equal_to(Datum::long(5)))
2198 .build()
2199 .unwrap();
2200 let batches: Vec<_> = table_scan
2201 .to_arrow(ArrowReaderOptions::default())
2202 .await
2203 .unwrap()
2204 .try_collect()
2205 .await
2206 .unwrap();
2207
2208 let mut values: Vec<i64> = batches
2209 .iter()
2210 .flat_map(|batch| {
2211 let col = batch.column_by_name("y").unwrap();
2212 let arr = col.as_any().downcast_ref::<Int64Array>().unwrap();
2213 (0..arr.len()).map(|i| arr.value(i)).collect::<Vec<_>>()
2214 })
2215 .collect();
2216 values.sort_unstable();
2217 values
2218 }
2219
2220 #[tokio::test]
2221 async fn test_open_parquet_no_deletions() {
2222 let mut fixture = TableTestFixture::new();
2223 fixture.setup_manifest_files().await;
2224
2225 let table_scan = fixture.table.scan().build().unwrap();
2227
2228 let batch_stream = table_scan
2229 .to_arrow(
2230 ArrowReaderOptions::builder()
2231 .with_row_selection_enabled(true)
2232 .build(),
2233 )
2234 .await
2235 .unwrap();
2236
2237 let batches: Vec<_> = batch_stream.try_collect().await.unwrap();
2238
2239 let col = batches[0].column_by_name("x").unwrap();
2240
2241 let int64_arr = col.as_any().downcast_ref::<Int64Array>().unwrap();
2242 assert_eq!(int64_arr.value(0), 1);
2243 }
2244
2245 #[tokio::test]
2246 async fn reader_options_preserve_reads_after_dropping_scan() {
2247 let mut fixture = TableTestFixture::new();
2248 fixture.setup_multi_row_group_manifest(&[150, 299]).await;
2249 let scan = fixture
2250 .table
2251 .scan()
2252 .select(["y"])
2253 .with_filter(Reference::new("y").greater_than(Datum::long(1100)))
2254 .build()
2255 .unwrap();
2256 let options = ArrowReaderOptions::builder()
2257 .with_batch_size(17)
2258 .with_data_file_concurrency_limit(1)
2259 .with_parquet_options(
2260 ParquetReadOptions::builder()
2261 .with_metadata_size_hint(4096)
2262 .with_range_fetch_concurrency(2)
2263 .build(),
2264 )
2265 .with_row_selection_enabled(true)
2266 .with_bloom_filter_enabled(true)
2267 .build();
2268 let expected: Vec<_> = scan
2269 .to_arrow(options.clone())
2270 .await
2271 .unwrap()
2272 .try_collect()
2273 .await
2274 .unwrap();
2275 let tasks: Vec<_> = scan
2276 .plan_files()
2277 .await
2278 .unwrap()
2279 .try_collect()
2280 .await
2281 .unwrap();
2282 let serialized = serde_json::to_value(&options).unwrap();
2283 assert_eq!(
2284 serialized,
2285 serde_json::json!({
2286 "batch_size": 17, "data_file_concurrency_limit": 1,
2287 "row_group_filtering_enabled": true, "row_selection_enabled": true,
2288 "bloom_filter_enabled": true,
2289 "parquet_options": {
2290 "metadata_size_hint": 4096, "range_coalesce_bytes": 1048576, "range_fetch_concurrency": 2,
2291 "preload_column_index": true, "preload_offset_index": true, "preload_page_index": false,
2292 },
2293 })
2294 );
2295 let deserialized: ArrowReaderOptions = serde_json::from_value(serialized).unwrap();
2296 assert_eq!(deserialized, options);
2297 let options = deserialized;
2298 drop(scan);
2299 let reader =
2300 ArrowReaderBuilder::new(FileIO::new_with_fs(), Runtime::try_current().unwrap())
2301 .with_options(options)
2302 .build();
2303 let actual: Vec<_> = reader
2304 .read(Box::pin(stream::iter(tasks.into_iter().map(Ok))))
2305 .unwrap()
2306 .stream()
2307 .try_collect()
2308 .await
2309 .unwrap();
2310 assert!(!actual.is_empty());
2311 assert!(actual.iter().all(|batch| batch.num_rows() <= 17));
2312 assert_eq!(actual, expected);
2313 }
2314
2315 #[tokio::test]
2316 async fn test_open_parquet_no_deletions_by_separate_reader() {
2317 let mut fixture = TableTestFixture::new();
2318 fixture.setup_manifest_files().await;
2319
2320 let table_scan = fixture.table.scan().build().unwrap();
2322
2323 let mut plan_task: Vec<_> = table_scan
2324 .plan_files()
2325 .await
2326 .unwrap()
2327 .try_collect()
2328 .await
2329 .unwrap();
2330 assert_eq!(plan_task.len(), 2);
2331
2332 let reader = ArrowReaderBuilder::new(
2333 fixture.table.file_io().clone(),
2334 fixture.table.runtime().clone(),
2335 )
2336 .build();
2337 let batch_stream = reader
2338 .clone()
2339 .read(Box::pin(stream::iter(vec![Ok(plan_task.remove(0))])))
2340 .unwrap()
2341 .stream();
2342 let batch_1: Vec<_> = batch_stream.try_collect().await.unwrap();
2343
2344 let reader = ArrowReaderBuilder::new(
2345 fixture.table.file_io().clone(),
2346 fixture.table.runtime().clone(),
2347 )
2348 .build();
2349 let batch_stream = reader
2350 .read(Box::pin(stream::iter(vec![Ok(plan_task.remove(0))])))
2351 .unwrap()
2352 .stream();
2353 let batch_2: Vec<_> = batch_stream.try_collect().await.unwrap();
2354
2355 assert_eq!(batch_1, batch_2);
2356 }
2357
2358 #[tokio::test]
2359 async fn test_open_parquet_with_projection() {
2360 let mut fixture = TableTestFixture::new();
2361 fixture.setup_manifest_files().await;
2362
2363 let table_scan = fixture.table.scan().select(["x", "z"]).build().unwrap();
2365
2366 let batch_stream = table_scan
2367 .to_arrow(
2368 ArrowReaderOptions::builder()
2369 .with_row_selection_enabled(true)
2370 .build(),
2371 )
2372 .await
2373 .unwrap();
2374
2375 let batches: Vec<_> = batch_stream.try_collect().await.unwrap();
2376
2377 assert_eq!(batches[0].num_columns(), 2);
2378
2379 let col1 = batches[0].column_by_name("x").unwrap();
2380 let int64_arr = col1.as_any().downcast_ref::<Int64Array>().unwrap();
2381 assert_eq!(int64_arr.value(0), 1);
2382
2383 let col2 = batches[0].column_by_name("z").unwrap();
2384 let int64_arr = col2.as_any().downcast_ref::<Int64Array>().unwrap();
2385 assert_eq!(int64_arr.value(0), 3);
2386
2387 let table_scan = fixture.table.scan().select_empty().build().unwrap();
2389 let batch_stream = table_scan
2390 .to_arrow(
2391 ArrowReaderOptions::builder()
2392 .with_row_selection_enabled(true)
2393 .build(),
2394 )
2395 .await
2396 .unwrap();
2397 let batches: Vec<_> = batch_stream.try_collect().await.unwrap();
2398
2399 assert_eq!(batches[0].num_columns(), 0);
2400 assert_eq!(batches[0].num_rows(), 1024);
2401 }
2402
2403 #[tokio::test]
2404 async fn test_filter_on_arrow_lt() {
2405 let mut fixture = TableTestFixture::new();
2406 fixture.setup_manifest_files().await;
2407
2408 let mut builder = fixture.table.scan();
2410 let predicate = Reference::new("y").less_than(Datum::long(3));
2411 builder = builder.with_filter(predicate);
2412 let table_scan = builder.build().unwrap();
2413
2414 let batch_stream = table_scan
2415 .to_arrow(
2416 ArrowReaderOptions::builder()
2417 .with_row_selection_enabled(true)
2418 .build(),
2419 )
2420 .await
2421 .unwrap();
2422
2423 let batches: Vec<_> = batch_stream.try_collect().await.unwrap();
2424
2425 assert_eq!(batches[0].num_rows(), 512);
2426
2427 let col = batches[0].column_by_name("x").unwrap();
2428 let int64_arr = col.as_any().downcast_ref::<Int64Array>().unwrap();
2429 assert_eq!(int64_arr.value(0), 1);
2430
2431 let col = batches[0].column_by_name("y").unwrap();
2432 let int64_arr = col.as_any().downcast_ref::<Int64Array>().unwrap();
2433 assert_eq!(int64_arr.value(0), 2);
2434 }
2435
2436 #[tokio::test]
2437 async fn test_filter_on_arrow_gt_eq() {
2438 let mut fixture = TableTestFixture::new();
2439 fixture.setup_manifest_files().await;
2440
2441 let mut builder = fixture.table.scan();
2443 let predicate = Reference::new("y").greater_than_or_equal_to(Datum::long(5));
2444 builder = builder.with_filter(predicate);
2445 let table_scan = builder.build().unwrap();
2446
2447 let batch_stream = table_scan
2448 .to_arrow(
2449 ArrowReaderOptions::builder()
2450 .with_row_selection_enabled(true)
2451 .build(),
2452 )
2453 .await
2454 .unwrap();
2455
2456 let batches: Vec<_> = batch_stream.try_collect().await.unwrap();
2457
2458 assert_eq!(batches[0].num_rows(), 12);
2459
2460 let col = batches[0].column_by_name("x").unwrap();
2461 let int64_arr = col.as_any().downcast_ref::<Int64Array>().unwrap();
2462 assert_eq!(int64_arr.value(0), 1);
2463
2464 let col = batches[0].column_by_name("y").unwrap();
2465 let int64_arr = col.as_any().downcast_ref::<Int64Array>().unwrap();
2466 assert_eq!(int64_arr.value(0), 5);
2467 }
2468
2469 #[tokio::test]
2470 async fn test_filter_double_eq() {
2471 let mut fixture = TableTestFixture::new();
2472 fixture.setup_manifest_files().await;
2473
2474 let mut builder = fixture.table.scan();
2476 let predicate = Reference::new("dbl").equal_to(Datum::double(150.0f64));
2477 builder = builder.with_filter(predicate);
2478 let table_scan = builder.build().unwrap();
2479
2480 let batch_stream = table_scan
2481 .to_arrow(
2482 ArrowReaderOptions::builder()
2483 .with_row_selection_enabled(true)
2484 .build(),
2485 )
2486 .await
2487 .unwrap();
2488
2489 let batches: Vec<_> = batch_stream.try_collect().await.unwrap();
2490
2491 assert_eq!(batches.len(), 2);
2492 assert_eq!(batches[0].num_rows(), 12);
2493
2494 let col = batches[0].column_by_name("dbl").unwrap();
2495 let f64_arr = col.as_any().downcast_ref::<Float64Array>().unwrap();
2496 assert_eq!(f64_arr.value(1), 150.0f64);
2497 }
2498
2499 #[tokio::test]
2500 async fn test_filter_int_eq() {
2501 let mut fixture = TableTestFixture::new();
2502 fixture.setup_manifest_files().await;
2503
2504 let mut builder = fixture.table.scan();
2506 let predicate = Reference::new("i32").equal_to(Datum::int(150i32));
2507 builder = builder.with_filter(predicate);
2508 let table_scan = builder.build().unwrap();
2509
2510 let batch_stream = table_scan
2511 .to_arrow(
2512 ArrowReaderOptions::builder()
2513 .with_row_selection_enabled(true)
2514 .build(),
2515 )
2516 .await
2517 .unwrap();
2518
2519 let batches: Vec<_> = batch_stream.try_collect().await.unwrap();
2520
2521 assert_eq!(batches.len(), 2);
2522 assert_eq!(batches[0].num_rows(), 12);
2523
2524 let col = batches[0].column_by_name("i32").unwrap();
2525 let i32_arr = col.as_any().downcast_ref::<Int32Array>().unwrap();
2526 assert_eq!(i32_arr.value(1), 150i32);
2527 }
2528
2529 #[tokio::test]
2530 async fn test_filter_long_eq() {
2531 let mut fixture = TableTestFixture::new();
2532 fixture.setup_manifest_files().await;
2533
2534 let mut builder = fixture.table.scan();
2536 let predicate = Reference::new("i64").equal_to(Datum::long(150i64));
2537 builder = builder.with_filter(predicate);
2538 let table_scan = builder.build().unwrap();
2539
2540 let batch_stream = table_scan
2541 .to_arrow(
2542 ArrowReaderOptions::builder()
2543 .with_row_selection_enabled(true)
2544 .build(),
2545 )
2546 .await
2547 .unwrap();
2548
2549 let batches: Vec<_> = batch_stream.try_collect().await.unwrap();
2550
2551 assert_eq!(batches.len(), 2);
2552 assert_eq!(batches[0].num_rows(), 12);
2553
2554 let col = batches[0].column_by_name("i64").unwrap();
2555 let i64_arr = col.as_any().downcast_ref::<Int64Array>().unwrap();
2556 assert_eq!(i64_arr.value(1), 150i64);
2557 }
2558
2559 #[tokio::test]
2560 async fn test_filter_bool_eq() {
2561 let mut fixture = TableTestFixture::new();
2562 fixture.setup_manifest_files().await;
2563
2564 let mut builder = fixture.table.scan();
2566 let predicate = Reference::new("bool").equal_to(Datum::bool(true));
2567 builder = builder.with_filter(predicate);
2568 let table_scan = builder.build().unwrap();
2569
2570 let batch_stream = table_scan
2571 .to_arrow(
2572 ArrowReaderOptions::builder()
2573 .with_row_selection_enabled(true)
2574 .build(),
2575 )
2576 .await
2577 .unwrap();
2578
2579 let batches: Vec<_> = batch_stream.try_collect().await.unwrap();
2580
2581 assert_eq!(batches.len(), 2);
2582 assert_eq!(batches[0].num_rows(), 512);
2583
2584 let col = batches[0].column_by_name("bool").unwrap();
2585 let bool_arr = col.as_any().downcast_ref::<BooleanArray>().unwrap();
2586 assert!(bool_arr.value(1));
2587 }
2588
2589 #[tokio::test]
2590 async fn test_filter_on_arrow_is_null() {
2591 let mut fixture = TableTestFixture::new();
2592 fixture.setup_manifest_files().await;
2593
2594 let mut builder = fixture.table.scan();
2596 let predicate = Reference::new("y").is_null();
2597 builder = builder.with_filter(predicate);
2598 let table_scan = builder.build().unwrap();
2599
2600 let batch_stream = table_scan
2601 .to_arrow(
2602 ArrowReaderOptions::builder()
2603 .with_row_selection_enabled(true)
2604 .build(),
2605 )
2606 .await
2607 .unwrap();
2608
2609 let batches: Vec<_> = batch_stream.try_collect().await.unwrap();
2610 assert_eq!(batches.len(), 0);
2611 }
2612
2613 #[tokio::test]
2614 async fn test_filter_on_arrow_is_not_null() {
2615 let mut fixture = TableTestFixture::new();
2616 fixture.setup_manifest_files().await;
2617
2618 let mut builder = fixture.table.scan();
2620 let predicate = Reference::new("y").is_not_null();
2621 builder = builder.with_filter(predicate);
2622 let table_scan = builder.build().unwrap();
2623
2624 let batch_stream = table_scan
2625 .to_arrow(
2626 ArrowReaderOptions::builder()
2627 .with_row_selection_enabled(true)
2628 .build(),
2629 )
2630 .await
2631 .unwrap();
2632
2633 let batches: Vec<_> = batch_stream.try_collect().await.unwrap();
2634 assert_eq!(batches[0].num_rows(), 1024);
2635 }
2636
2637 #[tokio::test]
2638 async fn test_filter_on_arrow_lt_and_gt() {
2639 let mut fixture = TableTestFixture::new();
2640 fixture.setup_manifest_files().await;
2641
2642 let mut builder = fixture.table.scan();
2644 let predicate = Reference::new("y")
2645 .less_than(Datum::long(5))
2646 .and(Reference::new("z").greater_than_or_equal_to(Datum::long(4)));
2647 builder = builder.with_filter(predicate);
2648 let table_scan = builder.build().unwrap();
2649
2650 let batch_stream = table_scan
2651 .to_arrow(
2652 ArrowReaderOptions::builder()
2653 .with_row_selection_enabled(true)
2654 .build(),
2655 )
2656 .await
2657 .unwrap();
2658
2659 let batches: Vec<_> = batch_stream.try_collect().await.unwrap();
2660 assert_eq!(batches[0].num_rows(), 500);
2661
2662 let col = batches[0].column_by_name("x").unwrap();
2663 let expected_x = Arc::new(Int64Array::from_iter_values(vec![1; 500])) as ArrayRef;
2664 assert_eq!(col, &expected_x);
2665
2666 let col = batches[0].column_by_name("y").unwrap();
2667 let mut values = vec![];
2668 values.append(vec![3; 200].as_mut());
2669 values.append(vec![4; 300].as_mut());
2670 let expected_y = Arc::new(Int64Array::from_iter_values(values)) as ArrayRef;
2671 assert_eq!(col, &expected_y);
2672
2673 let col = batches[0].column_by_name("z").unwrap();
2674 let expected_z = Arc::new(Int64Array::from_iter_values(vec![4; 500])) as ArrayRef;
2675 assert_eq!(col, &expected_z);
2676 }
2677
2678 #[tokio::test]
2679 async fn test_filter_on_arrow_lt_or_gt() {
2680 let mut fixture = TableTestFixture::new();
2681 fixture.setup_manifest_files().await;
2682
2683 let mut builder = fixture.table.scan();
2685 let predicate = Reference::new("y")
2686 .less_than(Datum::long(5))
2687 .or(Reference::new("z").greater_than_or_equal_to(Datum::long(4)));
2688 builder = builder.with_filter(predicate);
2689 let table_scan = builder.build().unwrap();
2690
2691 let batch_stream = table_scan
2692 .to_arrow(
2693 ArrowReaderOptions::builder()
2694 .with_row_selection_enabled(true)
2695 .build(),
2696 )
2697 .await
2698 .unwrap();
2699
2700 let batches: Vec<_> = batch_stream.try_collect().await.unwrap();
2701 assert_eq!(batches[0].num_rows(), 1024);
2702
2703 let col = batches[0].column_by_name("x").unwrap();
2704 let expected_x = Arc::new(Int64Array::from_iter_values(vec![1; 1024])) as ArrayRef;
2705 assert_eq!(col, &expected_x);
2706
2707 let col = batches[0].column_by_name("y").unwrap();
2708 let mut values = vec![2; 512];
2709 values.append(vec![3; 200].as_mut());
2710 values.append(vec![4; 300].as_mut());
2711 values.append(vec![5; 12].as_mut());
2712 let expected_y = Arc::new(Int64Array::from_iter_values(values)) as ArrayRef;
2713 assert_eq!(col, &expected_y);
2714
2715 let col = batches[0].column_by_name("z").unwrap();
2716 let mut values = vec![3; 512];
2717 values.append(vec![4; 512].as_mut());
2718 let expected_z = Arc::new(Int64Array::from_iter_values(values)) as ArrayRef;
2719 assert_eq!(col, &expected_z);
2720 }
2721
2722 #[tokio::test]
2723 async fn test_filter_on_arrow_startswith() {
2724 let mut fixture = TableTestFixture::new();
2725 fixture.setup_manifest_files().await;
2726
2727 let mut builder = fixture.table.scan();
2729 let predicate = Reference::new("a").starts_with(Datum::string("Ice"));
2730 builder = builder.with_filter(predicate);
2731 let table_scan = builder.build().unwrap();
2732
2733 let batch_stream = table_scan
2734 .to_arrow(
2735 ArrowReaderOptions::builder()
2736 .with_row_selection_enabled(true)
2737 .build(),
2738 )
2739 .await
2740 .unwrap();
2741
2742 let batches: Vec<_> = batch_stream.try_collect().await.unwrap();
2743
2744 assert_eq!(batches[0].num_rows(), 512);
2745
2746 let col = batches[0].column_by_name("a").unwrap();
2747 let string_arr = col.as_any().downcast_ref::<StringArray>().unwrap();
2748 assert_eq!(string_arr.value(0), "Iceberg");
2749 }
2750
2751 #[tokio::test]
2752 async fn test_filter_on_arrow_not_startswith() {
2753 let mut fixture = TableTestFixture::new();
2754 fixture.setup_manifest_files().await;
2755
2756 let mut builder = fixture.table.scan();
2758 let predicate = Reference::new("a").not_starts_with(Datum::string("Ice"));
2759 builder = builder.with_filter(predicate);
2760 let table_scan = builder.build().unwrap();
2761
2762 let batch_stream = table_scan
2763 .to_arrow(
2764 ArrowReaderOptions::builder()
2765 .with_row_selection_enabled(true)
2766 .build(),
2767 )
2768 .await
2769 .unwrap();
2770
2771 let batches: Vec<_> = batch_stream.try_collect().await.unwrap();
2772
2773 assert_eq!(batches[0].num_rows(), 512);
2774
2775 let col = batches[0].column_by_name("a").unwrap();
2776 let string_arr = col.as_any().downcast_ref::<StringArray>().unwrap();
2777 assert_eq!(string_arr.value(0), "Apache");
2778 }
2779
2780 #[tokio::test]
2781 async fn test_filter_on_arrow_in() {
2782 let mut fixture = TableTestFixture::new();
2783 fixture.setup_manifest_files().await;
2784
2785 let mut builder = fixture.table.scan();
2787 let predicate =
2788 Reference::new("a").is_in([Datum::string("Sioux"), Datum::string("Iceberg")]);
2789 builder = builder.with_filter(predicate);
2790 let table_scan = builder.build().unwrap();
2791
2792 let batch_stream = table_scan
2793 .to_arrow(
2794 ArrowReaderOptions::builder()
2795 .with_row_selection_enabled(true)
2796 .build(),
2797 )
2798 .await
2799 .unwrap();
2800
2801 let batches: Vec<_> = batch_stream.try_collect().await.unwrap();
2802
2803 assert_eq!(batches[0].num_rows(), 512);
2804
2805 let col = batches[0].column_by_name("a").unwrap();
2806 let string_arr = col.as_any().downcast_ref::<StringArray>().unwrap();
2807 assert_eq!(string_arr.value(0), "Iceberg");
2808 }
2809
2810 #[tokio::test]
2811 async fn test_filter_on_arrow_not_in() {
2812 let mut fixture = TableTestFixture::new();
2813 fixture.setup_manifest_files().await;
2814
2815 let mut builder = fixture.table.scan();
2817 let predicate =
2818 Reference::new("a").is_not_in([Datum::string("Sioux"), Datum::string("Iceberg")]);
2819 builder = builder.with_filter(predicate);
2820 let table_scan = builder.build().unwrap();
2821
2822 let batch_stream = table_scan
2823 .to_arrow(
2824 ArrowReaderOptions::builder()
2825 .with_row_selection_enabled(true)
2826 .build(),
2827 )
2828 .await
2829 .unwrap();
2830
2831 let batches: Vec<_> = batch_stream.try_collect().await.unwrap();
2832
2833 assert_eq!(batches[0].num_rows(), 512);
2834
2835 let col = batches[0].column_by_name("a").unwrap();
2836 let string_arr = col.as_any().downcast_ref::<StringArray>().unwrap();
2837 assert_eq!(string_arr.value(0), "Apache");
2838 }
2839
2840 fn file_scan_task_test_schema(primitive_type: PrimitiveType) -> Arc<Schema> {
2841 Arc::new(
2842 Schema::builder()
2843 .with_fields(vec![Arc::new(NestedField::required(
2844 1,
2845 "x",
2846 Type::Primitive(primitive_type),
2847 ))])
2848 .build()
2849 .unwrap(),
2850 )
2851 }
2852
2853 fn assert_file_scan_task_serde_round_trip(task: FileScanTask) {
2854 let serialized = serde_json::to_string(&task).unwrap();
2856 let deserialized: FileScanTask = serde_json::from_str(&serialized).unwrap();
2857
2858 assert_eq!(task, deserialized);
2859 }
2860
2861 fn file_scan_task_with_partition(
2862 primitive_type: PrimitiveType,
2863 transform: Transform,
2864 partition_value: Literal,
2865 ) -> FileScanTask {
2866 let schema = file_scan_task_test_schema(primitive_type);
2867 let partition_spec = Arc::new(
2868 PartitionSpec::builder(schema.clone())
2869 .add_partition_field("x", "x_partition", transform)
2870 .unwrap()
2871 .build()
2872 .unwrap(),
2873 );
2874 FileScanTask::builder()
2875 .with_data_file_path("data_file_path".to_string())
2876 .with_file_size_in_bytes(123)
2877 .with_start(10)
2878 .with_length(100)
2879 .with_project_field_ids(vec![1])
2880 .with_schema(schema)
2881 .with_data_file_format(DataFileFormat::Parquet)
2882 .with_partition(Some(Struct::from_iter([Some(partition_value)])))
2883 .with_partition_spec(Some(partition_spec))
2884 .with_case_sensitive(true)
2885 .build()
2886 .unwrap()
2887 }
2888
2889 #[test]
2890 fn test_file_scan_task_serde_without_predicate() {
2891 let task = FileScanTask::builder()
2892 .with_data_file_path("data_file_path".to_string())
2893 .with_file_size_in_bytes(0)
2894 .with_start(0)
2895 .with_length(100)
2896 .with_project_field_ids(vec![1, 2, 3])
2897 .with_schema(file_scan_task_test_schema(PrimitiveType::Binary))
2898 .with_record_count(Some(100))
2899 .with_first_row_id(Some(1000))
2900 .with_data_sequence_number(Some(5))
2901 .with_data_file_format(DataFileFormat::Parquet)
2902 .with_case_sensitive(false)
2903 .build()
2904 .unwrap();
2905 assert_file_scan_task_serde_round_trip(task);
2906 }
2907
2908 #[test]
2909 fn test_file_scan_task_serde_with_predicate() {
2910 let task = FileScanTask::builder()
2911 .with_data_file_path("data_file_path".to_string())
2912 .with_file_size_in_bytes(0)
2913 .with_start(0)
2914 .with_length(100)
2915 .with_project_field_ids(vec![1, 2, 3])
2916 .with_predicate(Some(BoundPredicate::AlwaysTrue))
2917 .with_schema(file_scan_task_test_schema(PrimitiveType::Binary))
2918 .with_data_file_format(DataFileFormat::Avro)
2919 .with_case_sensitive(false)
2920 .build()
2921 .unwrap();
2922
2923 let serialized = serde_json::to_value(&task).unwrap();
2924 assert!(serialized.get("record_count").is_none());
2925 assert_file_scan_task_serde_round_trip(task);
2926 }
2927
2928 #[test]
2929 fn test_unpartitioned_file_scan_task_serde() {
2930 let task = FileScanTask::builder()
2931 .with_data_file_path("data_file_path".to_string())
2932 .with_file_size_in_bytes(0)
2933 .with_start(0)
2934 .with_length(100)
2935 .with_project_field_ids(vec![1, 2, 3])
2936 .with_schema(file_scan_task_test_schema(PrimitiveType::Binary))
2937 .with_data_file_format(DataFileFormat::Parquet)
2938 .with_partition(Some(Struct::empty()))
2939 .with_case_sensitive(false)
2940 .build()
2941 .unwrap();
2942 assert_file_scan_task_serde_round_trip(task);
2943 }
2944
2945 #[test]
2946 fn test_file_scan_task_serde_with_all_optional_fields() {
2947 let schema = file_scan_task_test_schema(PrimitiveType::Long);
2948 let partition_spec = Arc::new(
2949 PartitionSpec::builder(schema.clone())
2950 .add_partition_field("x", "x", Transform::Identity)
2951 .unwrap()
2952 .build()
2953 .unwrap(),
2954 );
2955 let unified_partition_type = Arc::new(partition_spec.partition_type(&schema).unwrap());
2956 let sort_order = Arc::new(
2957 SortOrder::builder()
2958 .with_order_id(1)
2959 .with_sort_field(
2960 SortField::builder()
2961 .source_id(1)
2962 .transform(Transform::Identity)
2963 .direction(SortDirection::Ascending)
2964 .null_order(NullOrder::First)
2965 .build(),
2966 )
2967 .build(&schema)
2968 .unwrap(),
2969 );
2970 let task = FileScanTask::builder()
2971 .with_data_file_path("data_file_path".to_string())
2972 .with_file_size_in_bytes(123)
2973 .with_start(10)
2974 .with_length(100)
2975 .with_project_field_ids(vec![1])
2976 .with_schema(schema)
2977 .with_data_file_format(DataFileFormat::Parquet)
2978 .with_deletes(vec![
2979 FileScanTaskDeleteFile::builder()
2980 .with_file_path("delete_file_path".to_string())
2981 .with_file_size_in_bytes(23)
2982 .with_file_type(DataContentType::EqualityDeletes)
2983 .with_file_format(DataFileFormat::Parquet)
2984 .with_partition_spec_id(0)
2985 .with_equality_ids(Some(vec![1]))
2986 .with_referenced_data_file(Some("data_file_path".to_string()))
2987 .with_content_offset(Some(12))
2988 .with_content_size_in_bytes(Some(34))
2989 .with_record_count(Some(5))
2990 .with_key_metadata(Some(vec![4, 5, 6].into_boxed_slice()))
2991 .build()
2992 .unwrap(),
2993 ])
2994 .with_partition(Some(Struct::from_iter([Some(Literal::long(42))])))
2995 .with_partition_spec(Some(partition_spec))
2996 .with_name_mapping(Some(Arc::new(NameMapping::new(vec![MappedField::new(
2997 Some(1),
2998 vec!["x".to_string()],
2999 vec![],
3000 )]))))
3001 .with_unified_partition_type(Some(unified_partition_type))
3002 .with_sort_order_id(Some(1))
3003 .with_sort_order(Some(sort_order))
3004 .with_column_stats(Some(
3005 FileScanTaskColumnStats::builder()
3006 .with_column_sizes(HashMap::from([(1, 80), (2, 96)]))
3007 .with_value_counts(HashMap::from([(1, 10), (2, 10)]))
3008 .with_null_value_counts(HashMap::from([(1, 0), (2, 1)]))
3009 .with_nan_value_counts(HashMap::from([(3, 2)]))
3010 .with_lower_bounds(HashMap::from([
3011 (1, Datum::string("apple")),
3012 (2, Datum::decimal_from_str("-12.34").unwrap()),
3013 (3, Datum::float(100.1_f32)),
3014 (4, Datum::double(f64::NEG_INFINITY)),
3015 ]))
3016 .with_upper_bounds(HashMap::from([
3017 (1, Datum::string("pear")),
3018 (2, Datum::decimal_from_str("56.78").unwrap()),
3019 (3, Datum::float(f32::INFINITY)),
3020 (4, Datum::double(f64::INFINITY)),
3021 ]))
3022 .build(),
3023 ))
3024 .with_case_sensitive(true)
3025 .with_key_metadata(Some(vec![1, 2, 3].into_boxed_slice()))
3026 .build()
3027 .unwrap();
3028 assert_file_scan_task_serde_round_trip(task);
3029 }
3030
3031 #[test]
3032 fn test_file_scan_task_serde_omits_absent_column_stats() {
3033 let task = file_scan_task_with_partition(
3034 PrimitiveType::Long,
3035 Transform::Identity,
3036 Literal::long(42),
3037 );
3038
3039 let serialized = serde_json::to_value(&task).unwrap();
3040 assert!(serialized.get("column_stats").is_none());
3041
3042 let deserialized: FileScanTask = serde_json::from_value(serialized).unwrap();
3043 assert_eq!(deserialized.column_stats(), None);
3044 }
3045
3046 #[test]
3047 fn test_file_scan_task_serde_with_date_partition() {
3048 let task = file_scan_task_with_partition(
3049 PrimitiveType::Date,
3050 Transform::Identity,
3051 Literal::date(19_000),
3052 );
3053 assert_file_scan_task_serde_round_trip(task);
3054 }
3055
3056 #[test]
3057 fn test_file_scan_task_serde_with_timestamp_ns_partition() {
3058 let task = file_scan_task_with_partition(
3059 PrimitiveType::TimestampNs,
3060 Transform::Identity,
3061 Literal::timestamp_nano(1_510_871_468_123_456_789),
3062 );
3063 assert_file_scan_task_serde_round_trip(task);
3064 }
3065
3066 #[test]
3067 fn test_file_scan_task_serde_with_timestamptz_ns_partition() {
3068 let task = file_scan_task_with_partition(
3069 PrimitiveType::TimestamptzNs,
3070 Transform::Identity,
3071 Literal::timestamptz_nano(1_510_871_468_123_456_789),
3072 );
3073 assert_file_scan_task_serde_round_trip(task);
3074 }
3075
3076 #[test]
3077 fn test_file_scan_task_serde_with_decimal_partition() {
3078 let task = file_scan_task_with_partition(
3079 PrimitiveType::Decimal {
3080 precision: 9,
3081 scale: 2,
3082 },
3083 Transform::Identity,
3084 Literal::decimal(12_345),
3085 );
3086 assert_file_scan_task_serde_round_trip(task);
3087 }
3088
3089 #[test]
3090 fn test_file_scan_task_serde_with_uuid_partition() {
3091 let task = file_scan_task_with_partition(
3092 PrimitiveType::Uuid,
3093 Transform::Identity,
3094 Literal::uuid(Uuid::from_u128(0x12345678_90ab_cdef_1234_567890abcdef)),
3095 );
3096 assert_file_scan_task_serde_round_trip(task);
3097 }
3098
3099 #[test]
3100 fn test_file_scan_task_serde_with_fixed_partition() {
3101 let task = file_scan_task_with_partition(
3102 PrimitiveType::Fixed(4),
3103 Transform::Identity,
3104 Literal::fixed([1, 2, 3, 4]),
3105 );
3106 assert_file_scan_task_serde_round_trip(task);
3107 }
3108
3109 #[test]
3110 fn test_file_scan_task_serde_with_bucket_partition() {
3111 let task = file_scan_task_with_partition(
3112 PrimitiveType::String,
3113 Transform::Bucket(4),
3114 Literal::int(2),
3115 );
3116 assert_file_scan_task_serde_round_trip(task);
3117 }
3118
3119 #[tokio::test]
3120 async fn test_select_with_file_column() {
3121 let mut fixture = TableTestFixture::new();
3122 fixture.setup_manifest_files().await;
3123
3124 let table_scan = fixture
3126 .table
3127 .scan()
3128 .select(["x", RESERVED_COL_NAME_FILE])
3129 .build()
3130 .unwrap();
3131
3132 let batch_stream = table_scan
3133 .to_arrow(
3134 ArrowReaderOptions::builder()
3135 .with_row_selection_enabled(true)
3136 .build(),
3137 )
3138 .await
3139 .unwrap();
3140 let batches: Vec<_> = batch_stream.try_collect().await.unwrap();
3141
3142 assert_eq!(batches[0].num_columns(), 2);
3144
3145 let x_col = batches[0].column_by_name("x").unwrap();
3147 let x_arr = x_col.as_primitive::<arrow_array::types::Int64Type>();
3148 assert_eq!(x_arr.value(0), 1);
3149
3150 let file_col = batches[0].column_by_name(RESERVED_COL_NAME_FILE);
3152 assert!(
3153 file_col.is_some(),
3154 "_file column should be present in the batch"
3155 );
3156
3157 let file_col = file_col.unwrap();
3159 assert!(
3160 matches!(
3161 file_col.data_type(),
3162 arrow_schema::DataType::RunEndEncoded(_, _)
3163 ),
3164 "_file column should use RunEndEncoded type"
3165 );
3166
3167 let run_array = file_col
3169 .as_any()
3170 .downcast_ref::<RunArray<Int32Type>>()
3171 .expect("_file column should be a RunArray");
3172
3173 let values = run_array.values();
3174 let string_values = values.as_string::<i32>();
3175 assert_eq!(string_values.len(), 1, "Should have a single file path");
3176
3177 let file_path = string_values.value(0);
3178 assert!(
3179 file_path.ends_with(".parquet"),
3180 "File path should end with .parquet, got: {file_path}"
3181 );
3182 }
3183
3184 #[tokio::test]
3185 async fn test_select_file_column_position() {
3186 let mut fixture = TableTestFixture::new();
3187 fixture.setup_manifest_files().await;
3188
3189 let table_scan = fixture
3191 .table
3192 .scan()
3193 .select(["x", RESERVED_COL_NAME_FILE, "z"])
3194 .build()
3195 .unwrap();
3196
3197 let batch_stream = table_scan
3198 .to_arrow(
3199 ArrowReaderOptions::builder()
3200 .with_row_selection_enabled(true)
3201 .build(),
3202 )
3203 .await
3204 .unwrap();
3205 let batches: Vec<_> = batch_stream.try_collect().await.unwrap();
3206
3207 assert_eq!(batches[0].num_columns(), 3);
3208
3209 let schema = batches[0].schema();
3211 assert_eq!(schema.field(0).name(), "x");
3212 assert_eq!(schema.field(1).name(), RESERVED_COL_NAME_FILE);
3213 assert_eq!(schema.field(2).name(), "z");
3214
3215 assert!(batches[0].column_by_name("x").is_some());
3217 assert!(batches[0].column_by_name(RESERVED_COL_NAME_FILE).is_some());
3218 assert!(batches[0].column_by_name("z").is_some());
3219 }
3220
3221 #[tokio::test]
3222 async fn test_select_file_column_only() {
3223 let mut fixture = TableTestFixture::new();
3224 fixture.setup_manifest_files().await;
3225
3226 let table_scan = fixture
3228 .table
3229 .scan()
3230 .select([RESERVED_COL_NAME_FILE])
3231 .build()
3232 .unwrap();
3233
3234 let batch_stream = table_scan
3235 .to_arrow(
3236 ArrowReaderOptions::builder()
3237 .with_row_selection_enabled(true)
3238 .build(),
3239 )
3240 .await
3241 .unwrap();
3242 let batches: Vec<_> = batch_stream.try_collect().await.unwrap();
3243
3244 assert_eq!(batches[0].num_columns(), 1);
3246
3247 let schema = batches[0].schema();
3249 assert_eq!(schema.field(0).name(), RESERVED_COL_NAME_FILE);
3250
3251 let total_rows: usize = batches.iter().map(|b| b.num_rows()).sum();
3255 assert_eq!(total_rows, 2048);
3256 }
3257
3258 #[tokio::test]
3259 async fn test_file_column_with_multiple_files() {
3260 use std::collections::HashSet;
3261
3262 let mut fixture = TableTestFixture::new();
3263 fixture.setup_manifest_files().await;
3264
3265 let table_scan = fixture
3267 .table
3268 .scan()
3269 .select(["x", RESERVED_COL_NAME_FILE])
3270 .build()
3271 .unwrap();
3272
3273 let batch_stream = table_scan
3274 .to_arrow(
3275 ArrowReaderOptions::builder()
3276 .with_row_selection_enabled(true)
3277 .build(),
3278 )
3279 .await
3280 .unwrap();
3281 let batches: Vec<_> = batch_stream.try_collect().await.unwrap();
3282
3283 let mut file_paths = HashSet::new();
3285 for batch in &batches {
3286 let file_col = batch.column_by_name(RESERVED_COL_NAME_FILE).unwrap();
3287 let run_array = file_col
3288 .as_any()
3289 .downcast_ref::<RunArray<Int32Type>>()
3290 .expect("_file column should be a RunArray");
3291
3292 let values = run_array.values();
3293 let string_values = values.as_string::<i32>();
3294 for i in 0..string_values.len() {
3295 file_paths.insert(string_values.value(i).to_string());
3296 }
3297 }
3298
3299 assert!(!file_paths.is_empty(), "Should have at least one file path");
3301
3302 for path in &file_paths {
3304 assert!(
3305 path.ends_with(".parquet"),
3306 "All file paths should end with .parquet, got: {path}"
3307 );
3308 }
3309 }
3310
3311 #[tokio::test]
3312 async fn test_file_column_at_start() {
3313 let mut fixture = TableTestFixture::new();
3314 fixture.setup_manifest_files().await;
3315
3316 let table_scan = fixture
3318 .table
3319 .scan()
3320 .select([RESERVED_COL_NAME_FILE, "x", "y"])
3321 .build()
3322 .unwrap();
3323
3324 let batch_stream = table_scan
3325 .to_arrow(
3326 ArrowReaderOptions::builder()
3327 .with_row_selection_enabled(true)
3328 .build(),
3329 )
3330 .await
3331 .unwrap();
3332 let batches: Vec<_> = batch_stream.try_collect().await.unwrap();
3333
3334 assert_eq!(batches[0].num_columns(), 3);
3335
3336 let schema = batches[0].schema();
3338 assert_eq!(schema.field(0).name(), RESERVED_COL_NAME_FILE);
3339 assert_eq!(schema.field(1).name(), "x");
3340 assert_eq!(schema.field(2).name(), "y");
3341 }
3342
3343 #[tokio::test]
3344 async fn test_file_column_at_end() {
3345 let mut fixture = TableTestFixture::new();
3346 fixture.setup_manifest_files().await;
3347
3348 let table_scan = fixture
3350 .table
3351 .scan()
3352 .select(["x", "y", RESERVED_COL_NAME_FILE])
3353 .build()
3354 .unwrap();
3355
3356 let batch_stream = table_scan
3357 .to_arrow(
3358 ArrowReaderOptions::builder()
3359 .with_row_selection_enabled(true)
3360 .build(),
3361 )
3362 .await
3363 .unwrap();
3364 let batches: Vec<_> = batch_stream.try_collect().await.unwrap();
3365
3366 assert_eq!(batches[0].num_columns(), 3);
3367
3368 let schema = batches[0].schema();
3370 assert_eq!(schema.field(0).name(), "x");
3371 assert_eq!(schema.field(1).name(), "y");
3372 assert_eq!(schema.field(2).name(), RESERVED_COL_NAME_FILE);
3373 }
3374
3375 #[tokio::test]
3376 async fn test_select_with_repeated_column_names() {
3377 let mut fixture = TableTestFixture::new();
3378 fixture.setup_manifest_files().await;
3379
3380 let table_scan = fixture
3383 .table
3384 .scan()
3385 .select([
3386 "x",
3387 RESERVED_COL_NAME_FILE,
3388 "x", "y",
3390 RESERVED_COL_NAME_FILE, "y", ])
3393 .build()
3394 .unwrap();
3395
3396 let batch_stream = table_scan
3397 .to_arrow(
3398 ArrowReaderOptions::builder()
3399 .with_row_selection_enabled(true)
3400 .build(),
3401 )
3402 .await
3403 .unwrap();
3404 let batches: Vec<_> = batch_stream.try_collect().await.unwrap();
3405
3406 assert_eq!(
3408 batches[0].num_columns(),
3409 6,
3410 "Should have exactly 6 columns with duplicates"
3411 );
3412
3413 let schema = batches[0].schema();
3414
3415 assert_eq!(schema.field(0).name(), "x", "Column 0 should be x");
3417 assert_eq!(
3418 schema.field(1).name(),
3419 RESERVED_COL_NAME_FILE,
3420 "Column 1 should be _file"
3421 );
3422 assert_eq!(
3423 schema.field(2).name(),
3424 "x",
3425 "Column 2 should be x (duplicate)"
3426 );
3427 assert_eq!(schema.field(3).name(), "y", "Column 3 should be y");
3428 assert_eq!(
3429 schema.field(4).name(),
3430 RESERVED_COL_NAME_FILE,
3431 "Column 4 should be _file (duplicate)"
3432 );
3433 assert_eq!(
3434 schema.field(5).name(),
3435 "y",
3436 "Column 5 should be y (duplicate)"
3437 );
3438
3439 assert!(
3441 matches!(schema.field(0).data_type(), arrow_schema::DataType::Int64),
3442 "Column x should be Int64"
3443 );
3444 assert!(
3445 matches!(schema.field(2).data_type(), arrow_schema::DataType::Int64),
3446 "Column x (duplicate) should be Int64"
3447 );
3448 assert!(
3449 matches!(schema.field(3).data_type(), arrow_schema::DataType::Int64),
3450 "Column y should be Int64"
3451 );
3452 assert!(
3453 matches!(schema.field(5).data_type(), arrow_schema::DataType::Int64),
3454 "Column y (duplicate) should be Int64"
3455 );
3456 assert!(
3457 matches!(
3458 schema.field(1).data_type(),
3459 arrow_schema::DataType::RunEndEncoded(_, _)
3460 ),
3461 "_file column should use RunEndEncoded type"
3462 );
3463 assert!(
3464 matches!(
3465 schema.field(4).data_type(),
3466 arrow_schema::DataType::RunEndEncoded(_, _)
3467 ),
3468 "_file column (duplicate) should use RunEndEncoded type"
3469 );
3470 }
3471
3472 fn table_with_data_column(column_name: &str) -> Table {
3478 use crate::spec::{
3479 FormatVersion, MAIN_BRANCH, Operation, Snapshot, SnapshotReference, SnapshotRetention,
3480 SortOrder, Summary, TableMetadataBuilder, UnboundPartitionSpec,
3481 };
3482
3483 let schema = Schema::builder()
3484 .with_schema_id(0)
3485 .with_fields(vec![
3486 NestedField::required(1, "id", Type::Primitive(PrimitiveType::Int)).into(),
3487 NestedField::required(2, column_name, Type::Primitive(PrimitiveType::Int)).into(),
3488 ])
3489 .build()
3490 .unwrap();
3491
3492 let snapshot = Snapshot::builder()
3493 .with_snapshot_id(1)
3494 .with_timestamp_ms(1)
3495 .with_sequence_number(0)
3496 .with_schema_id(0)
3497 .with_manifest_list("/snap-1.avro")
3498 .with_summary(Summary {
3499 operation: Operation::Append,
3500 additional_properties: HashMap::new(),
3501 })
3502 .build();
3503
3504 let metadata = TableMetadataBuilder::new(
3505 schema,
3506 UnboundPartitionSpec::builder().with_spec_id(0).build(),
3507 SortOrder::unsorted_order(),
3508 "s3://bucket/table".to_string(),
3509 FormatVersion::V2,
3510 HashMap::new(),
3511 )
3512 .unwrap()
3513 .add_snapshot(snapshot)
3514 .unwrap()
3515 .set_ref(MAIN_BRANCH, SnapshotReference {
3516 snapshot_id: 1,
3517 retention: SnapshotRetention::Branch {
3518 min_snapshots_to_keep: None,
3519 max_snapshot_age_ms: None,
3520 max_ref_age_ms: None,
3521 },
3522 })
3523 .unwrap()
3524 .build()
3525 .unwrap()
3526 .metadata;
3527
3528 Table::builder()
3529 .metadata(metadata)
3530 .identifier(TableIdent::from_strs(["db", "table1"]).unwrap())
3531 .file_io(FileIO::new_with_fs())
3532 .runtime(test_runtime())
3533 .build()
3534 .unwrap()
3535 }
3536
3537 #[test]
3541 fn test_scan_projects_data_column_named_like_delete_file_column() {
3542 for column_name in ["pos", "file_path"] {
3543 let table = table_with_data_column(column_name);
3544
3545 let table_scan = table
3548 .scan()
3549 .select([column_name])
3550 .build()
3551 .unwrap_or_else(|e| panic!("scan of data column `{column_name}` failed: {e}"));
3552
3553 assert_eq!(
3554 table_scan.plan_context.as_ref().unwrap().field_ids.as_ref(),
3555 &[2]
3556 );
3557
3558 let default_scan = table.scan().build().unwrap();
3561 assert_eq!(
3562 default_scan
3563 .plan_context
3564 .as_ref()
3565 .unwrap()
3566 .field_ids
3567 .as_ref(),
3568 &[1, 2]
3569 );
3570 }
3571 }
3572
3573 #[test]
3576 fn test_scan_rejects_unknown_column_named_like_delete_file_column() {
3577 let table = table_with_data_column("file_path");
3579
3580 let err = table
3581 .scan()
3582 .select(["pos"])
3583 .build()
3584 .expect_err("projecting an absent column should fail");
3585 assert_eq!(err.kind(), ErrorKind::DataInvalid);
3586 assert!(err.to_string().contains("not found"));
3587 }
3588
3589 #[tokio::test]
3590 async fn test_scan_deadlock() {
3591 let mut fixture = TableTestFixture::new();
3592 fixture.setup_deadlock_manifests().await;
3593
3594 let table_scan = fixture
3601 .table
3602 .scan()
3603 .with_concurrency_limit(1)
3604 .build()
3605 .unwrap();
3606
3607 let result = tokio::time::timeout(std::time::Duration::from_secs(5), async {
3610 table_scan
3611 .plan_files()
3612 .await
3613 .unwrap()
3614 .try_collect::<Vec<_>>()
3615 .await
3616 })
3617 .await;
3618
3619 assert!(result.is_ok(), "Scan timed out - deadlock detected");
3621 }
3622
3623 async fn plan_files_with_timeout(table: &Table) -> Result<Vec<FileScanTask>> {
3626 tokio::time::timeout(std::time::Duration::from_secs(5), async {
3627 table
3628 .scan()
3629 .build()
3630 .unwrap()
3631 .plan_files()
3632 .await
3633 .unwrap()
3634 .try_collect::<Vec<_>>()
3635 .await
3636 })
3637 .await
3638 .expect("plan_files hung instead of reporting the manifest error")
3639 }
3640
3641 async fn first_manifest_path(fixture: &TableTestFixture) -> String {
3643 let snapshot = fixture.table.metadata().current_snapshot().unwrap();
3644 let manifest_list = fixture
3645 .table
3646 .manifest_list_reader(snapshot)
3647 .load()
3648 .await
3649 .unwrap();
3650 manifest_list.entries()[0].manifest_path.clone()
3651 }
3652
3653 #[tokio::test]
3654 async fn test_plan_files_reports_corrupt_manifest_as_error() {
3655 let mut fixture = TableTestFixture::new();
3656 fixture.setup_manifest_files().await;
3657 let path = first_manifest_path(&fixture).await;
3658 fs::write(&path, b"not an avro file").unwrap();
3659
3660 let err = plan_files_with_timeout(&fixture.table)
3661 .await
3662 .expect_err("a corrupt manifest must fail the plan");
3663 assert_eq!(err.kind(), ErrorKind::Unexpected);
3664 assert!(err.to_string().contains(&path), "{err}");
3665 }
3666
3667 #[tokio::test]
3668 async fn test_plan_files_reports_missing_manifest_as_error() {
3669 let mut fixture = TableTestFixture::new();
3670 fixture.setup_manifest_files().await;
3671 let path = first_manifest_path(&fixture).await;
3672 fs::remove_file(&path).unwrap();
3673
3674 let err = plan_files_with_timeout(&fixture.table)
3675 .await
3676 .expect_err("a missing manifest must fail the plan");
3677 assert_eq!(err.kind(), ErrorKind::Unexpected);
3678 assert!(err.to_string().contains(&path), "{err}");
3679 }
3680
3681 #[tokio::test]
3687 async fn test_plan_files_reports_cancelled_manifest_parse_as_error() {
3688 let mut fixture = TableTestFixture::new();
3689 fixture.setup_manifest_files().await;
3690 let path = first_manifest_path(&fixture).await;
3691 let io_rt = tokio::runtime::Builder::new_multi_thread()
3692 .enable_all()
3693 .build()
3694 .unwrap();
3695 let cpu_rt = tokio::runtime::Builder::new_multi_thread()
3696 .enable_all()
3697 .build()
3698 .unwrap();
3699 let rt = Runtime::new_with_split(&io_rt, &cpu_rt);
3700 cpu_rt.shutdown_background();
3701 let table = fixture.table.clone().with_runtime(rt);
3702
3703 let err = plan_files_with_timeout(&table)
3704 .await
3705 .expect_err("a cancelled manifest parse must fail the plan");
3706 assert_eq!(err.kind(), ErrorKind::Unexpected);
3707 assert!(err.to_string().contains(&path), "{err}");
3708 io_rt.shutdown_background();
3709 }
3710
3711 #[tokio::test]
3712 async fn test_select_with_spec_id_column() {
3713 let mut fixture = TableTestFixture::new();
3714 fixture.setup_manifest_files().await;
3715
3716 let table_scan = fixture
3718 .table
3719 .scan()
3720 .select(["x", RESERVED_COL_NAME_SPEC_ID, "z"])
3721 .build()
3722 .unwrap();
3723
3724 let batch_stream = table_scan
3725 .to_arrow(
3726 ArrowReaderOptions::builder()
3727 .with_row_selection_enabled(true)
3728 .build(),
3729 )
3730 .await
3731 .unwrap();
3732 let batches: Vec<_> = batch_stream.try_collect().await.unwrap();
3733
3734 assert_eq!(batches[0].num_columns(), 3);
3736
3737 let col1 = batches[0].column_by_name("x").unwrap();
3739 let int64_arr = col1.as_any().downcast_ref::<Int64Array>().unwrap();
3740 assert_eq!(int64_arr.value(0), 1);
3741
3742 let spec_id_col = batches[0].column_by_name(RESERVED_COL_NAME_SPEC_ID);
3744 assert!(
3745 spec_id_col.is_some(),
3746 "_spec_id column should be present in the batch"
3747 );
3748
3749 let spec_id_col = spec_id_col.unwrap();
3751 assert!(
3752 matches!(
3753 spec_id_col.data_type(),
3754 arrow_schema::DataType::RunEndEncoded(_, _)
3755 ),
3756 "_spec_id column should use RunEndEncoded type"
3757 );
3758
3759 let run_array = spec_id_col
3761 .as_any()
3762 .downcast_ref::<RunArray<Int32Type>>()
3763 .expect("_spec_id column should be a RunArray");
3764
3765 let values = run_array.values();
3766 let int_values = values.as_primitive::<Int32Type>();
3767 assert_eq!(int_values.len(), 1, "Should have a single _spec_id");
3768
3769 let spec_id = int_values.value(0);
3770 assert_eq!(spec_id, 0, "_spec_id should be 0, got: {spec_id}");
3771
3772 assert!(batches[0].column_by_name("z").is_some());
3774 }
3775
3776 #[tokio::test]
3777 async fn test_select_with_last_updated_sequence_number_column() {
3778 let mut fixture = TableTestFixture::new();
3783 fixture.setup_manifest_files().await;
3784
3785 let table_scan = fixture
3786 .table
3787 .scan()
3788 .select(["x", RESERVED_COL_NAME_LAST_UPDATED_SEQUENCE_NUMBER])
3789 .build()
3790 .unwrap();
3791
3792 let batches: Vec<_> = table_scan
3793 .to_arrow(
3794 ArrowReaderOptions::builder()
3795 .with_row_selection_enabled(true)
3796 .build(),
3797 )
3798 .await
3799 .unwrap()
3800 .try_collect()
3801 .await
3802 .unwrap();
3803
3804 assert_last_updated_seq_all(&batches, None);
3806 }
3807
3808 #[tokio::test]
3809 async fn test_select_with_last_updated_sequence_number_column_v3() {
3810 let mut fixture = TableTestFixture::new();
3816 fixture.setup_v3_manifest_files().await;
3817
3818 let expected_seq = fixture
3821 .table
3822 .metadata()
3823 .current_snapshot()
3824 .unwrap()
3825 .sequence_number();
3826
3827 let table_scan = fixture
3828 .table
3829 .scan()
3830 .select(["x", RESERVED_COL_NAME_LAST_UPDATED_SEQUENCE_NUMBER])
3831 .build()
3832 .unwrap();
3833
3834 let batches: Vec<_> = table_scan
3835 .to_arrow(
3836 ArrowReaderOptions::builder()
3837 .with_row_selection_enabled(true)
3838 .build(),
3839 )
3840 .await
3841 .unwrap()
3842 .try_collect()
3843 .await
3844 .unwrap();
3845
3846 assert_last_updated_seq_all(&batches, Some(expected_seq));
3847 }
3848
3849 #[tokio::test]
3850 async fn test_select_with_spec_id_column_from_unpartitioned_table() {
3851 let mut fixture = TableTestFixture::new_unpartitioned();
3852 fixture.setup_unpartitioned_manifest_files().await;
3853
3854 let table_scan = fixture
3856 .table
3857 .scan()
3858 .select(["x", RESERVED_COL_NAME_SPEC_ID])
3859 .build()
3860 .unwrap();
3861
3862 let batch_stream = table_scan
3863 .to_arrow(
3864 ArrowReaderOptions::builder()
3865 .with_row_selection_enabled(true)
3866 .build(),
3867 )
3868 .await
3869 .unwrap();
3870 let batches: Vec<_> = batch_stream.try_collect().await.unwrap();
3871
3872 assert_eq!(batches[0].num_columns(), 2);
3874
3875 let spec_id_col = batches[0].column_by_name(RESERVED_COL_NAME_SPEC_ID);
3877 assert!(
3878 spec_id_col.is_some(),
3879 "_spec_id column should be present in the batch"
3880 );
3881
3882 let spec_id_col = spec_id_col.unwrap();
3884 assert!(
3885 matches!(
3886 spec_id_col.data_type(),
3887 arrow_schema::DataType::RunEndEncoded(_, _)
3888 ),
3889 "_spec_id column should use RunEndEncoded type"
3890 );
3891
3892 let run_array = spec_id_col
3894 .as_any()
3895 .downcast_ref::<RunArray<Int32Type>>()
3896 .expect("_spec_id column should be a RunArray");
3897
3898 let values = run_array.values();
3899 let int_values = values.as_primitive::<Int32Type>();
3900 assert_eq!(int_values.len(), 1, "Should have a single _spec_id");
3901
3902 let spec_id = int_values.value(0);
3903 assert_eq!(spec_id, 0, "_spec_id should be 0, got: {spec_id}");
3904 }
3905
3906 #[tokio::test]
3907 async fn test_select_with_spec_id_column_with_partition_evolution() {
3908 let mut fixture = TableTestFixture::new_with_partition_evolution();
3909 fixture
3910 .setup_manifest_files_with_partition_evolution()
3911 .await;
3912
3913 let table_scan = fixture
3915 .table
3916 .scan()
3917 .select(["x", RESERVED_COL_NAME_SPEC_ID, "z"])
3918 .build()
3919 .unwrap();
3920
3921 let batch_stream = table_scan
3922 .to_arrow(
3923 ArrowReaderOptions::builder()
3924 .with_row_selection_enabled(true)
3925 .build(),
3926 )
3927 .await
3928 .unwrap();
3929 let batches: Vec<_> = batch_stream.try_collect().await.unwrap();
3930
3931 let col1 = batches[0].column_by_name("x").unwrap();
3933 let int64_arr = col1.as_any().downcast_ref::<Int64Array>().unwrap();
3934 assert_eq!(int64_arr.value(0), 1);
3935
3936 let spec_id_col = batches[0].column_by_name(RESERVED_COL_NAME_SPEC_ID);
3938 assert!(
3939 spec_id_col.is_some(),
3940 "_spec_id column should be present in the batch"
3941 );
3942
3943 let spec_id_col = spec_id_col.unwrap();
3945 assert!(
3946 matches!(
3947 spec_id_col.data_type(),
3948 arrow_schema::DataType::RunEndEncoded(_, _)
3949 ),
3950 "_spec_id column should use RunEndEncoded type"
3951 );
3952
3953 let run_array = spec_id_col
3955 .as_any()
3956 .downcast_ref::<RunArray<Int32Type>>()
3957 .expect("_spec_id column should be a RunArray");
3958
3959 let values = run_array.values();
3960 let int_values = values.as_primitive::<Int32Type>();
3961 assert_eq!(int_values.len(), 1, "Should have a single _spec_id");
3962
3963 let spec_id = int_values.value(0);
3964 assert_eq!(spec_id, 2, "_spec_id should be 2, got: {spec_id}");
3965 }
3966
3967 #[tokio::test]
3968 async fn test_select_with_pos_and_file_columns() {
3969 use arrow_array::cast::AsArray;
3970
3971 let mut fixture = TableTestFixture::new();
3972 fixture.setup_manifest_files().await;
3973
3974 let table_scan = fixture
3976 .table
3977 .scan()
3978 .select(["x", RESERVED_COL_NAME_POS, RESERVED_COL_NAME_FILE])
3979 .build()
3980 .unwrap();
3981
3982 let batch_stream = table_scan
3983 .to_arrow(
3984 ArrowReaderOptions::builder()
3985 .with_row_selection_enabled(true)
3986 .build(),
3987 )
3988 .await
3989 .unwrap();
3990 let batches: Vec<_> = batch_stream.try_collect().await.unwrap();
3991 assert_eq!(batches.len(), 2);
3992
3993 for batch in batches.iter() {
3995 assert_eq!(batch.num_columns(), 3);
3997
3998 let x_col = batch.column_by_name("x").unwrap();
4000 let x_arr = x_col.as_primitive::<arrow_array::types::Int64Type>();
4001 assert_eq!(x_arr.value(0), 1);
4002
4003 let pos_col = batch.column(1);
4005 let pos_array: &Int64Array = pos_col
4006 .as_any()
4007 .downcast_ref::<Int64Array>()
4008 .expect("_pos column should be a Int64Array");
4009 assert_eq!(*pos_array, Int64Array::from_iter_values(0i64..1024));
4010
4011 let file_col = batch.column_by_name(RESERVED_COL_NAME_FILE);
4013 assert!(
4014 file_col.is_some(),
4015 "_file column should be present in the batch"
4016 );
4017 }
4018 }
4019
4020 #[tokio::test]
4021 async fn test_pos_column_at_start_with_filters() {
4022 let mut fixture = TableTestFixture::new();
4023 fixture.setup_manifest_files().await;
4024
4025 let predicate = Reference::new("y")
4027 .greater_than(Datum::long(4i64))
4028 .and(Reference::new("y").less_than_or_equal_to(Datum::long(5i64)));
4029 let table_scan = fixture
4031 .table
4032 .scan()
4033 .select([RESERVED_COL_NAME_POS, "x", "y"])
4034 .with_filter(predicate)
4035 .build()
4036 .unwrap();
4037
4038 let batch_stream = table_scan
4039 .to_arrow(
4040 ArrowReaderOptions::builder()
4041 .with_row_selection_enabled(true)
4042 .build(),
4043 )
4044 .await
4045 .unwrap();
4046 let batches: Vec<_> = batch_stream.try_collect().await.unwrap();
4047 assert_eq!(batches.len(), 2);
4048
4049 for batch in batches.iter() {
4051 assert_eq!(batch.num_columns(), 3);
4052 assert_eq!(batch.num_rows(), 12);
4053
4054 let schema = batch.schema();
4056 assert_eq!(schema.field(0).name(), RESERVED_COL_NAME_POS);
4057 assert_eq!(schema.field(1).name(), "x");
4058 assert_eq!(schema.field(2).name(), "y");
4059
4060 let pos_col = batch.column(0);
4061 let pos_array: &Int64Array = pos_col
4062 .as_any()
4063 .downcast_ref::<Int64Array>()
4064 .expect("_pos column should be a Int64Array");
4065 assert_eq!(*pos_array, Int64Array::from_iter_values(1012i64..1024));
4066 }
4067 }
4068
4069 #[tokio::test]
4070 async fn test_repeated_pos_column_with_filter() {
4071 let mut fixture = TableTestFixture::new();
4072 fixture.setup_manifest_files().await;
4073
4074 let predicate = Reference::new("a").not_starts_with(Datum::string("Apa"));
4076 let table_scan = fixture
4078 .table
4079 .scan()
4080 .select([RESERVED_COL_NAME_POS, "a", RESERVED_COL_NAME_POS, "x"])
4081 .with_filter(predicate)
4082 .build()
4083 .unwrap();
4084
4085 let batch_stream = table_scan
4086 .to_arrow(
4087 ArrowReaderOptions::builder()
4088 .with_row_selection_enabled(true)
4089 .build(),
4090 )
4091 .await
4092 .unwrap();
4093
4094 let batches: Vec<_> = batch_stream.try_collect().await.unwrap();
4095 assert_eq!(batches.len(), 2);
4096
4097 for batch in batches.iter() {
4099 assert_eq!(batch.num_rows(), 512);
4100
4101 let pos_col = batch
4103 .column_by_name("_pos")
4104 .expect("_pos column should be present in the batch");
4105 let pos_array: &Int64Array = pos_col
4106 .as_any()
4107 .downcast_ref::<Int64Array>()
4108 .expect("_pos column should be a Int64Array");
4109 assert_eq!(*pos_array, Int64Array::from_iter_values(512i64..1024));
4110
4111 let pos_col = batch.column(2);
4113 let pos_array: &Int64Array = pos_col
4114 .as_any()
4115 .downcast_ref::<Int64Array>()
4116 .expect("_pos column should be a Int64Array");
4117 assert_eq!(*pos_array, Int64Array::from_iter_values(512i64..1024));
4118 }
4119 }
4120
4121 #[tokio::test]
4125 async fn test_pos_across_row_groups_via_table_scan() {
4126 let mut fixture = TableTestFixture::new();
4127 fixture.setup_multi_row_group_manifest(&[]).await;
4128
4129 let tasks: Vec<_> = fixture
4132 .table
4133 .scan()
4134 .select(["x", RESERVED_COL_NAME_POS])
4135 .build()
4136 .unwrap()
4137 .plan_files()
4138 .await
4139 .unwrap()
4140 .try_collect()
4141 .await
4142 .unwrap();
4143 assert_eq!(tasks.len(), 1, "expected a single FileScanTask");
4144 let task = &tasks[0];
4145 assert!(
4146 task.project_field_ids().contains(&RESERVED_FIELD_ID_POS),
4147 "_pos field id must be projected into the FileScanTask"
4148 );
4149 assert_eq!(task.start(), 0, "TableScan should plan whole-file tasks");
4150 assert_eq!(task.length(), task.file_size_in_bytes());
4151 assert!(task.deletes().is_empty());
4152
4153 let batches: Vec<_> = fixture
4155 .table
4156 .scan()
4157 .select(["x", RESERVED_COL_NAME_POS])
4158 .build()
4159 .unwrap()
4160 .to_arrow(ArrowReaderOptions::default())
4161 .await
4162 .unwrap()
4163 .try_collect()
4164 .await
4165 .unwrap();
4166
4167 let pos: Vec<i64> = batches
4168 .iter()
4169 .flat_map(|b| {
4170 b.column_by_name(RESERVED_COL_NAME_POS)
4171 .expect("_pos column should be present")
4172 .as_any()
4173 .downcast_ref::<Int64Array>()
4174 .expect("_pos column should be a Int64Array")
4175 .values()
4176 .to_vec()
4177 })
4178 .collect();
4179 assert_eq!(pos, (0..300).collect::<Vec<i64>>());
4180
4181 let x: Vec<i64> = batches
4183 .iter()
4184 .flat_map(|b| {
4185 b.column_by_name("x")
4186 .unwrap()
4187 .as_primitive::<arrow_array::types::Int64Type>()
4188 .values()
4189 .to_vec()
4190 })
4191 .collect();
4192 assert_eq!(x, (1000..1300).collect::<Vec<i64>>());
4193 }
4194
4195 #[tokio::test]
4198 async fn test_pos_with_positional_deletes_via_table_scan() {
4199 let mut fixture = TableTestFixture::new();
4200 fixture.setup_multi_row_group_manifest(&[150, 299]).await;
4202
4203 let tasks: Vec<_> = fixture
4205 .table
4206 .scan()
4207 .select(["x", RESERVED_COL_NAME_POS])
4208 .build()
4209 .unwrap()
4210 .plan_files()
4211 .await
4212 .unwrap()
4213 .try_collect()
4214 .await
4215 .unwrap();
4216 assert_eq!(tasks.len(), 1);
4217 assert_eq!(
4218 tasks[0].deletes().len(),
4219 1,
4220 "positional delete file should be planned into the task"
4221 );
4222 assert_eq!(
4223 tasks[0].deletes()[0].file_type(),
4224 DataContentType::PositionDeletes
4225 );
4226
4227 let batches: Vec<_> = fixture
4229 .table
4230 .scan()
4231 .select(["x", RESERVED_COL_NAME_POS])
4232 .build()
4233 .unwrap()
4234 .to_arrow(ArrowReaderOptions::default())
4235 .await
4236 .unwrap()
4237 .try_collect()
4238 .await
4239 .unwrap();
4240
4241 let pos: Vec<i64> = batches
4242 .iter()
4243 .flat_map(|b| {
4244 b.column_by_name(RESERVED_COL_NAME_POS)
4245 .expect("_pos column should be present")
4246 .as_any()
4247 .downcast_ref::<Int64Array>()
4248 .expect("_pos column should be a Int64Array")
4249 .values()
4250 .to_vec()
4251 })
4252 .collect();
4253
4254 let total: usize = batches.iter().map(|b| b.num_rows()).sum();
4255 assert_eq!(
4256 total, 298,
4257 "two rows should be removed by positional deletes"
4258 );
4259 assert!(!pos.contains(&150) && !pos.contains(&299), "got {pos:?}");
4260 let expected: Vec<i64> = (0..150).chain(151..299).collect();
4261 assert_eq!(pos, expected);
4262 }
4263
4264 #[tokio::test]
4272 async fn test_pos_reads_only_middle_row_group_via_filter() {
4273 let mut fixture = TableTestFixture::new();
4274 fixture.setup_multi_row_group_manifest(&[]).await;
4275
4276 let predicate = Reference::new("y")
4278 .greater_than_or_equal_to(Datum::long(1100))
4279 .and(Reference::new("y").less_than(Datum::long(1200)));
4280
4281 let batches: Vec<_> = fixture
4282 .table
4283 .scan()
4284 .select(["y", RESERVED_COL_NAME_POS])
4285 .with_filter(predicate)
4286 .build()
4287 .unwrap()
4288 .to_arrow(ArrowReaderOptions::default())
4289 .await
4290 .unwrap()
4291 .try_collect()
4292 .await
4293 .unwrap();
4294
4295 let total: usize = batches.iter().map(|b| b.num_rows()).sum();
4296 assert_eq!(total, 100, "only the middle row group should be read");
4297
4298 let pos: Vec<i64> = batches
4299 .iter()
4300 .flat_map(|b| {
4301 b.column_by_name(RESERVED_COL_NAME_POS)
4302 .expect("_pos column should be present")
4303 .as_any()
4304 .downcast_ref::<Int64Array>()
4305 .expect("_pos column should be a Int64Array")
4306 .values()
4307 .to_vec()
4308 })
4309 .collect();
4310 assert_eq!(
4311 pos,
4312 (100..200).collect::<Vec<i64>>(),
4313 "_pos must be file-absolute for the middle row group"
4314 );
4315
4316 let y: Vec<i64> = batches
4318 .iter()
4319 .flat_map(|b| {
4320 b.column_by_name("y")
4321 .unwrap()
4322 .as_primitive::<arrow_array::types::Int64Type>()
4323 .values()
4324 .to_vec()
4325 })
4326 .collect();
4327 assert_eq!(y, (1100..1200).collect::<Vec<i64>>());
4328 }
4329
4330 #[tokio::test]
4331 async fn test_dead_only_manifests_are_skipped() {
4332 use crate::spec::ManifestContentType;
4333
4334 for content in [ManifestContentType::Data, ManifestContentType::Deletes] {
4335 let mut fixture = TableTestFixture::new();
4336 if content == ManifestContentType::Data {
4337 fixture.setup_manifest_files().await;
4338 } else {
4339 fixture.setup_deadlock_manifests().await;
4340 }
4341 let snapshot = fixture.table.metadata().current_snapshot().unwrap();
4342 let scan = fixture
4343 .table
4344 .scan()
4345 .with_filter(Reference::new("x").less_than(Datum::long(200)))
4346 .build()
4347 .unwrap();
4348 let mut expected = if content == ManifestContentType::Deletes {
4349 let tasks = scan
4350 .plan_files()
4351 .await
4352 .unwrap()
4353 .try_collect::<Vec<_>>()
4354 .await
4355 .unwrap();
4356 assert_eq!(tasks.len(), 10);
4357 assert!(tasks.iter().all(|task| task.deletes().len() == 1));
4358 let cow_tasks = scan
4359 .plan_cow_rewrite_files()
4360 .await
4361 .unwrap()
4362 .map_ok(|file| file.scan_task)
4363 .try_collect::<Vec<_>>()
4364 .await
4365 .unwrap();
4366 [tasks, cow_tasks]
4367 } else {
4368 [vec![], vec![]]
4369 };
4370 for tasks in &mut expected {
4371 tasks.sort_by(|a, b| a.data_file_path().cmp(b.data_file_path()));
4372 }
4373 let context = scan.plan_context.as_ref().unwrap();
4374 let list = context.get_manifest_list().await.unwrap();
4375 let source = list
4376 .entries()
4377 .iter()
4378 .find(|file| file.content == content)
4379 .unwrap();
4380 let manifest = context.object_cache.get_manifest(source).await.unwrap();
4381 let builder = ManifestWriterBuilder::new(
4382 fixture.next_manifest_file(),
4383 Some(snapshot.snapshot_id()),
4384 fixture.table.metadata().current_schema().clone(),
4385 fixture
4386 .table
4387 .metadata()
4388 .default_partition_spec()
4389 .as_ref()
4390 .clone(),
4391 );
4392 let mut writer = if content == ManifestContentType::Data {
4393 builder.build_v2_data()
4394 } else {
4395 builder.build_v2_deletes()
4396 };
4397 for entry in manifest.entries() {
4398 writer.add_delete_entry(entry.as_ref().clone()).unwrap();
4399 }
4400 let mut manifest = writer.write_manifest_file().await.unwrap();
4401 if content == ManifestContentType::Deletes {
4402 manifest.partitions.as_mut().unwrap()[0].lower_bound = Some(vec![0].into());
4403 }
4404 assert_eq!(
4405 (manifest.added_files_count, manifest.existing_files_count),
4406 (Some(0), Some(0))
4407 );
4408 assert!(manifest.deleted_files_count.unwrap() > 0);
4409 fixture
4410 .table
4411 .file_io()
4412 .delete(&manifest.manifest_path)
4413 .await
4414 .unwrap();
4415 let mut manifests = if content == ManifestContentType::Deletes {
4416 list.entries().to_vec()
4417 } else {
4418 vec![]
4419 };
4420 manifests.push(manifest);
4421 let output = fixture
4422 .table
4423 .file_io()
4424 .new_output(snapshot.manifest_list())
4425 .unwrap()
4426 .writer()
4427 .await
4428 .unwrap();
4429 let mut writer = ManifestListWriter::v2(
4430 output,
4431 snapshot.snapshot_id(),
4432 snapshot.parent_snapshot_id(),
4433 snapshot.sequence_number(),
4434 );
4435 writer.add_manifests(manifests.into_iter()).unwrap();
4436 writer.close().await.unwrap();
4437 let table = Table::builder()
4438 .metadata(fixture.table.metadata().clone())
4439 .identifier(fixture.table.identifier().clone())
4440 .file_io(fixture.table.file_io().clone())
4441 .runtime(test_runtime())
4442 .build()
4443 .unwrap();
4444 for cow in [false, true] {
4445 let scan = table
4446 .scan()
4447 .with_filter(Reference::new("x").less_than(Datum::long(200)))
4448 .build()
4449 .unwrap();
4450 let stream = if cow {
4451 scan.plan_cow_rewrite_files()
4452 .await
4453 .unwrap()
4454 .map_ok(|file| file.scan_task)
4455 .boxed()
4456 } else {
4457 scan.plan_files().await.unwrap()
4458 };
4459 let mut actual = tokio::time::timeout(
4460 std::time::Duration::from_secs(10),
4461 stream.try_collect::<Vec<_>>(),
4462 )
4463 .await
4464 .unwrap()
4465 .unwrap();
4466 actual.sort_by(|a, b| a.data_file_path().cmp(b.data_file_path()));
4467 assert_eq!(
4468 actual,
4469 expected[usize::from(cow)],
4470 "content={content:?}, cow={cow}"
4471 );
4472 }
4473 }
4474 }
4475
4476 #[tokio::test]
4477 async fn test_live_and_unknown_manifest_counts_preserve_tasks() {
4478 use crate::spec::FormatVersion;
4479
4480 for (status, added, existing) in [
4481 (ManifestStatus::Added, Some(1), Some(0)),
4482 (ManifestStatus::Existing, Some(0), Some(1)),
4483 (ManifestStatus::Added, None, Some(0)),
4484 (ManifestStatus::Existing, Some(0), None),
4485 (ManifestStatus::Added, None, None),
4486 ] {
4487 let mut fixture = TableTestFixture::new();
4488 fixture.setup_manifest_files().await;
4489 let snapshot = fixture.table.metadata().current_snapshot().unwrap();
4490 let scan = fixture.table.scan().build().unwrap();
4491 let context = scan.plan_context.as_ref().unwrap();
4492 let list = context.get_manifest_list().await.unwrap();
4493 let manifest = context
4494 .object_cache
4495 .get_manifest(&list.entries()[0])
4496 .await
4497 .unwrap();
4498 let entry = manifest
4499 .entries()
4500 .iter()
4501 .find(|entry| entry.status() == status)
4502 .unwrap();
4503 let expected = vec![(entry.data_file().file_path().to_owned(), vec![])];
4504 let mut writer = ManifestWriterBuilder::new(
4505 fixture.next_manifest_file(),
4506 Some(snapshot.snapshot_id()),
4507 fixture.table.metadata().current_schema().clone(),
4508 fixture
4509 .table
4510 .metadata()
4511 .default_partition_spec()
4512 .as_ref()
4513 .clone(),
4514 )
4515 .build_v1();
4516 if status == ManifestStatus::Added {
4517 writer.add_entry(entry.as_ref().clone()).unwrap();
4518 } else {
4519 writer.add_existing_entry(entry.as_ref().clone()).unwrap();
4520 }
4521 let mut manifest = writer.write_manifest_file().await.unwrap();
4522 assert_eq!(
4523 (manifest.added_files_count, manifest.existing_files_count),
4524 (
4525 Some(u32::from(status == ManifestStatus::Added)),
4526 Some(u32::from(status == ManifestStatus::Existing))
4527 )
4528 );
4529 manifest.added_files_count = added;
4530 manifest.existing_files_count = existing;
4531 let output = fixture
4532 .table
4533 .file_io()
4534 .new_output(snapshot.manifest_list())
4535 .unwrap()
4536 .writer()
4537 .await
4538 .unwrap();
4539 let mut writer = ManifestListWriter::v1(
4540 output,
4541 snapshot.snapshot_id(),
4542 snapshot.parent_snapshot_id(),
4543 );
4544 writer.add_manifests(std::iter::once(manifest)).unwrap();
4545 writer.close().await.unwrap();
4546 let mut metadata = fixture.table.metadata().clone();
4547 metadata.format_version = FormatVersion::V1;
4548 let table = Table::builder()
4549 .metadata(metadata)
4550 .identifier(fixture.table.identifier().clone())
4551 .file_io(fixture.table.file_io().clone())
4552 .runtime(test_runtime())
4553 .build()
4554 .unwrap();
4555 for cow in [false, true] {
4556 let scan = table.scan().build().unwrap();
4557 let counts = scan
4558 .plan_context
4559 .as_ref()
4560 .unwrap()
4561 .get_manifest_list()
4562 .await
4563 .unwrap();
4564 assert_eq!(
4565 (
4566 counts.entries()[0].added_files_count,
4567 counts.entries()[0].existing_files_count
4568 ),
4569 (added, existing)
4570 );
4571 let stream = if cow {
4572 scan.plan_cow_rewrite_files()
4573 .await
4574 .unwrap()
4575 .map_ok(|file| file.scan_task)
4576 .boxed()
4577 } else {
4578 scan.plan_files().await.unwrap()
4579 };
4580 let tasks = tokio::time::timeout(
4581 std::time::Duration::from_secs(10),
4582 stream.try_collect::<Vec<_>>(),
4583 )
4584 .await
4585 .unwrap()
4586 .unwrap();
4587 let actual: Vec<_> = tasks
4588 .into_iter()
4589 .map(|task| (task.data_file_path().to_owned(), task.deletes().to_vec()))
4590 .collect();
4591 assert_eq!(
4592 actual, expected,
4593 "status={status:?}, added={added:?}, existing={existing:?}, cow={cow}"
4594 );
4595 }
4596 }
4597 }
4598}