1use std::borrow::Cow;
24use std::collections::HashMap;
25use std::sync::Arc;
26use std::sync::atomic::AtomicU64;
27
28use arrow_schema::{DataType, Field};
29use futures::{StreamExt, TryStreamExt};
30use parquet::arrow::arrow_reader::{ArrowReaderMetadata, ArrowReaderOptions};
31use parquet::arrow::{
32 PARQUET_FIELD_ID_META_KEY, ParquetRecordBatchStreamBuilder, ProjectionMask, RowNumber,
33};
34use parquet::encryption::decrypt::FileDecryptionProperties;
35
36use super::row_lineage::synthesize_row_id_column;
37use super::{
38 ArrowFileReader, ArrowReader, ParquetReadOptions, add_fallback_field_ids_to_arrow_schema,
39 apply_name_mapping_to_arrow_schema, coerce_fixed_size_lists, find_leaf_by_field_id,
40};
41use crate::arrow::build_partition_constant;
42use crate::arrow::caching_delete_file_loader::CachingDeleteFileLoader;
43use crate::arrow::int96::coerce_int96_timestamps;
44use crate::arrow::record_batch_transformer::RecordBatchTransformerBuilder;
45use crate::arrow::scan_metrics::{CountingFileRead, ScanMetrics, ScanResult};
46use crate::encryption::StandardKeyMetadata;
47use crate::error::{Result, invalid_data};
48use crate::expr::BoundPredicate;
49use crate::expr::visitors::bloom_filter_evaluator::{
50 BloomFilterEvaluator, ColumnBloomFilter, collect_bloom_filter_field_ids,
51};
52use crate::io::{FileIO, FileMetadata, FileRead};
53use crate::metadata_columns::{
54 RESERVED_COL_NAME_LAST_UPDATED_SEQUENCE_NUMBER, RESERVED_COL_NAME_POS,
55 RESERVED_COL_NAME_ROW_ID, RESERVED_FIELD_ID_FILE,
56 RESERVED_FIELD_ID_LAST_UPDATED_SEQUENCE_NUMBER, RESERVED_FIELD_ID_PARTITION,
57 RESERVED_FIELD_ID_POS, RESERVED_FIELD_ID_ROW_ID, RESERVED_FIELD_ID_SPEC_ID, is_metadata_field,
58};
59use crate::runtime::Runtime;
60use crate::scan::{ArrowRecordBatchStream, FileScanTask, FileScanTaskStream};
61use crate::spec::{Datum, PartitionSpec, Struct};
62use crate::{Error, ErrorKind};
63
64impl ArrowReader {
65 pub fn read(self, tasks: FileScanTaskStream) -> Result<ScanResult> {
68 let concurrency_limit_data_files = self
69 .options
70 .data_file_concurrency_limit()
71 .expect("reader construction sets concurrency")
72 .max(1);
73 let scan_metrics = ScanMetrics::new();
74
75 let task_reader = FileScanTaskReader {
76 batch_size: self.options.batch_size(),
77 file_io: self.file_io,
78 delete_file_loader: self
79 .delete_file_loader
80 .with_scan_metrics(scan_metrics.clone()),
81 row_group_filtering_enabled: self.options.row_group_filtering_enabled(),
82 row_selection_enabled: self.options.row_selection_enabled(),
83 bloom_filter_enabled: self.options.bloom_filter_enabled(),
84 parquet_read_options: self.options.parquet_options().clone(),
85 scan_metrics: scan_metrics.clone(),
86 runtime: self.runtime,
87 };
88
89 let stream: ArrowRecordBatchStream = if concurrency_limit_data_files == 1 {
91 Box::pin(
92 tasks
93 .and_then(move |task| task_reader.clone().process(task))
94 .map_err(|err| {
95 Error::new(ErrorKind::Unexpected, "file scan task generate failed")
96 .with_source(err)
97 })
98 .try_flatten(),
99 )
100 } else {
101 Box::pin(
102 tasks
103 .map_ok(move |task| task_reader.clone().process(task))
104 .map_err(|err| {
105 Error::new(ErrorKind::Unexpected, "file scan task generate failed")
106 .with_source(err)
107 })
108 .try_buffer_unordered(concurrency_limit_data_files)
109 .try_flatten_unordered(concurrency_limit_data_files),
110 )
111 };
112
113 Ok(ScanResult::new(stream, scan_metrics))
114 }
115}
116
117const PRUNABLE_METADATA_FIELDS: &[i32] = &[
122 RESERVED_FIELD_ID_FILE,
123 RESERVED_FIELD_ID_SPEC_ID,
124 RESERVED_FIELD_ID_PARTITION,
125 RESERVED_FIELD_ID_POS,
126 RESERVED_FIELD_ID_ROW_ID,
127 RESERVED_FIELD_ID_LAST_UPDATED_SEQUENCE_NUMBER,
128];
129
130#[derive(Clone)]
133struct FileScanTaskReader {
134 batch_size: Option<usize>,
135 file_io: FileIO,
136 delete_file_loader: CachingDeleteFileLoader,
137 row_group_filtering_enabled: bool,
138 row_selection_enabled: bool,
139 bloom_filter_enabled: bool,
140 parquet_read_options: ParquetReadOptions,
141 scan_metrics: ScanMetrics,
142 runtime: Runtime,
143}
144
145fn candidate_row_groups(
147 selected_row_groups: Option<&[usize]>,
148 num_row_groups: usize,
149) -> Cow<'_, [usize]> {
150 match selected_row_groups {
151 Some(indices) => Cow::Borrowed(indices),
152 None => Cow::Owned((0..num_row_groups).collect()),
153 }
154}
155
156impl FileScanTaskReader {
157 async fn process(self, task: FileScanTask) -> Result<ArrowRecordBatchStream> {
158 if task.record_count() == Some(0) {
160 return Ok(Box::pin(futures::stream::empty()));
161 }
162
163 let should_load_page_index = (self.row_selection_enabled && task.predicate().is_some())
164 || !task.deletes().is_empty();
165
166 let delete_filter_rx = self
167 .delete_file_loader
168 .load_deletes(task.deletes(), task.schema_ref());
169
170 let path = task.data_file_path().to_string();
175 let file_io = self.file_io.clone();
176 let file_size_in_bytes = task.file_size_in_bytes();
177 let bytes_read = Arc::clone(self.scan_metrics.bytes_read_counter());
178 let key_metadata = task.key_metadata().map(|km| km.to_vec());
179 let parquet_read_options = self
180 .parquet_read_options
181 .clone()
182 .with_preload_page_index(should_load_page_index);
183 let opening = self
184 .runtime
185 .io()
186 .spawn(async move {
187 ArrowReader::open_parquet_file(
188 &path,
189 &file_io,
190 file_size_in_bytes,
191 parquet_read_options,
192 &bytes_read,
193 key_metadata.as_deref(),
194 )
195 .await
196 })
197 .abort_on_drop();
198 let (parquet_file_reader, arrow_metadata) = opening.await??;
199
200 let missing_field_ids = arrow_metadata
204 .schema()
205 .fields()
206 .iter()
207 .next()
208 .is_some_and(|f| f.metadata().get(PARQUET_FIELD_ID_META_KEY).is_none());
209
210 let use_position_fallback = missing_field_ids && task.name_mapping().is_none();
215
216 let project_pos = task.project_field_ids().contains(&RESERVED_FIELD_ID_POS);
217 let project_row_id = task.project_field_ids().contains(&RESERVED_FIELD_ID_ROW_ID);
218
219 let need_row_number = project_pos || (project_row_id && task.first_row_id().is_some());
224
225 let field_ids = task.project_field_ids();
226 let no_data_columns_projected = field_ids
228 .iter()
229 .all(|id| PRUNABLE_METADATA_FIELDS.contains(id));
230
231 let install_row_number = need_row_number || no_data_columns_projected;
232
233 let arrow_metadata = Self::configure_arrow_reader_metadata(
234 arrow_metadata,
235 &task,
236 missing_field_ids,
237 install_row_number,
238 )?;
239
240 let mut record_batch_stream_builder =
242 ParquetRecordBatchStreamBuilder::new_with_metadata(parquet_file_reader, arrow_metadata);
243
244 let project_last_updated_seq = task
248 .project_field_ids()
249 .contains(&RESERVED_FIELD_ID_LAST_UPDATED_SEQUENCE_NUMBER);
250
251 let phys_last_updated_seq_leaf = if project_last_updated_seq {
257 find_leaf_by_field_id(
258 record_batch_stream_builder.parquet_schema(),
259 RESERVED_FIELD_ID_LAST_UPDATED_SEQUENCE_NUMBER,
260 )
261 } else {
262 None
263 };
264
265 let last_updated_seq_present_by_name_only = project_last_updated_seq
268 && phys_last_updated_seq_leaf.is_none()
269 && record_batch_stream_builder
270 .schema()
271 .fields()
272 .iter()
273 .any(|f| f.name() == RESERVED_COL_NAME_LAST_UPDATED_SEQUENCE_NUMBER);
274
275 let coalesce_last_updated_seq_leaf = phys_last_updated_seq_leaf
280 .filter(|_| task.first_row_id().is_some() && task.data_sequence_number().is_some());
281
282 let phys_row_id_leaf = if project_row_id {
283 find_leaf_by_field_id(
284 record_batch_stream_builder.parquet_schema(),
285 RESERVED_FIELD_ID_ROW_ID,
286 )
287 } else {
288 None
289 };
290
291 let row_id_present_by_name_only = project_row_id
297 && !use_position_fallback
298 && phys_row_id_leaf.is_none()
299 && record_batch_stream_builder
300 .schema()
301 .fields()
302 .iter()
303 .any(|f| f.name() == RESERVED_COL_NAME_ROW_ID);
304
305 let coalesce_row_id_leaf = phys_row_id_leaf.filter(|_| task.first_row_id().is_some());
308
309 let project_field_ids_without_metadata: Vec<i32> = task
311 .project_field_ids()
312 .iter()
313 .filter(|&&id| !is_metadata_field(id))
314 .copied()
315 .collect();
316
317 let mut projection_mask = ArrowReader::get_arrow_projection_mask(
323 &project_field_ids_without_metadata,
324 task.schema(),
325 record_batch_stream_builder.parquet_schema(),
326 record_batch_stream_builder.schema(),
327 use_position_fallback, )?;
329
330 if project_field_ids_without_metadata.is_empty() && install_row_number {
339 projection_mask =
340 ProjectionMask::none(record_batch_stream_builder.parquet_schema().num_columns());
341 }
342
343 for leaf in [coalesce_last_updated_seq_leaf, coalesce_row_id_leaf]
348 .into_iter()
349 .flatten()
350 {
351 let phys_mask =
352 ProjectionMask::leaves(record_batch_stream_builder.parquet_schema(), vec![leaf]);
353 projection_mask.union(&phys_mask);
354 }
355
356 record_batch_stream_builder =
357 record_batch_stream_builder.with_projection(projection_mask.clone());
358
359 let mut record_batch_transformer_builder =
363 RecordBatchTransformerBuilder::new(task.schema_ref(), task.project_field_ids());
364
365 if task.project_field_ids().contains(&RESERVED_FIELD_ID_FILE) {
367 let file_datum = Datum::string(task.data_file_path().to_string());
368 record_batch_transformer_builder =
369 record_batch_transformer_builder.with_constant(RESERVED_FIELD_ID_FILE, file_datum);
370 }
371
372 if task
373 .project_field_ids()
374 .contains(&RESERVED_FIELD_ID_SPEC_ID)
375 {
376 let partition_spec = task
377 .partition_spec()
378 .ok_or_else(|| Error::new(ErrorKind::Unexpected, "Partition spec is missing"))?;
379
380 let spec_id_datum = Datum::int(partition_spec.spec_id());
381 record_batch_transformer_builder = record_batch_transformer_builder
382 .with_constant(RESERVED_FIELD_ID_SPEC_ID, spec_id_datum);
383 }
384
385 if project_last_updated_seq {
386 record_batch_transformer_builder =
391 match (task.first_row_id(), task.data_sequence_number()) {
392 (Some(_), Some(seq)) => {
393 let datum = Datum::long(seq);
394 if coalesce_last_updated_seq_leaf.is_some() {
395 record_batch_transformer_builder.with_coalesced_last_updated_seq_column(
398 RESERVED_FIELD_ID_LAST_UPDATED_SEQUENCE_NUMBER,
399 datum,
400 )
401 } else if last_updated_seq_present_by_name_only {
402 return Err(Error::new(
410 ErrorKind::FeatureUnsupported,
411 "Reading a physically-stored _last_updated_sequence_number column \
412 without an embedded field id is not supported",
413 ));
414 } else {
415 record_batch_transformer_builder.with_constant(
417 RESERVED_FIELD_ID_LAST_UPDATED_SEQUENCE_NUMBER,
418 datum,
419 )
420 }
421 }
422 (None, _) => record_batch_transformer_builder.with_null_metadata_column(
424 RESERVED_FIELD_ID_LAST_UPDATED_SEQUENCE_NUMBER,
425 )?,
426 (Some(_), None) => {
430 return Err(invalid_data!(
431 "Data file {} has a first_row_id but no data sequence number",
432 task.data_file_path()
433 ));
434 }
435 };
436 }
437
438 if project_row_id {
439 if task.first_row_id().is_some() && row_id_present_by_name_only {
445 return Err(Error::new(
446 ErrorKind::FeatureUnsupported,
447 "Reading a physically-stored _row_id column without an embedded field id \
448 is not supported",
449 ));
450 }
451
452 record_batch_transformer_builder =
456 record_batch_transformer_builder.with_virtual_field(RESERVED_FIELD_ID_ROW_ID);
457 }
458
459 if let (Some(partition_spec), Some(partition_data)) =
460 (task.partition_spec(), task.partition())
461 {
462 record_batch_transformer_builder = record_batch_transformer_builder
463 .with_partition(Arc::clone(partition_spec), partition_data.clone())?;
464 }
465
466 if project_pos {
467 record_batch_transformer_builder =
468 record_batch_transformer_builder.with_virtual_field(RESERVED_FIELD_ID_POS);
469 }
470
471 if task
474 .project_field_ids()
475 .contains(&RESERVED_FIELD_ID_PARTITION)
476 && let Some(unified_type) = task.unified_partition_type()
477 {
478 let (spec, partition_data) = match (task.partition_spec(), task.partition()) {
479 (Some(spec), Some(data)) => (Arc::clone(spec), data.clone()),
480 _ if unified_type.fields().is_empty() => {
485 (Arc::new(PartitionSpec::unpartition_spec()), Struct::empty())
486 }
487 _ => {
488 return Err(Error::new(
489 ErrorKind::Unexpected,
490 "cannot build _partition column: unified partition type has fields \
491 but the scan task is missing its partition spec or data",
492 ));
493 }
494 };
495 let constant = build_partition_constant(unified_type, &spec, &partition_data)?;
496 record_batch_transformer_builder =
497 record_batch_transformer_builder.with_partition_constant(constant);
498 }
499
500 let mut record_batch_transformer = record_batch_transformer_builder.build();
501
502 if let Some(batch_size) = self.batch_size {
503 record_batch_stream_builder = record_batch_stream_builder.with_batch_size(batch_size);
504 }
505
506 let delete_filter = delete_filter_rx.await.unwrap()?;
507 let delete_predicate = delete_filter.build_equality_delete_predicate(&task).await?;
508
509 let final_predicate = match (task.predicate(), delete_predicate) {
514 (None, None) => None,
515 (Some(predicate), None) => Some(predicate.clone()),
516 (None, Some(ref predicate)) => Some(predicate.clone()),
517 (Some(filter_predicate), Some(delete_predicate)) => {
518 Some(filter_predicate.clone().and(delete_predicate))
519 }
520 };
521
522 let mut selected_row_group_indices = None;
538 let mut row_selection = None;
539
540 if task.start() != 0 || task.length() != 0 {
543 let byte_range_filtered_row_groups = ArrowReader::filter_row_groups_by_byte_range(
544 record_batch_stream_builder.metadata(),
545 task.start(),
546 task.length(),
547 )?;
548 selected_row_group_indices = Some(byte_range_filtered_row_groups);
549 }
550
551 if let Some(predicate) = final_predicate {
552 let predicate = predicate.rewrite_not();
559
560 let (iceberg_field_ids, field_id_map) = ArrowReader::build_field_id_set_and_map(
561 record_batch_stream_builder.parquet_schema(),
562 record_batch_stream_builder.schema(),
563 &predicate,
564 use_position_fallback,
565 )?;
566
567 let row_filter = ArrowReader::get_row_filter(
568 &predicate,
569 record_batch_stream_builder.parquet_schema(),
570 &iceberg_field_ids,
571 &field_id_map,
572 )?;
573 record_batch_stream_builder = record_batch_stream_builder.with_row_filter(row_filter);
574
575 if self.row_group_filtering_enabled {
576 let candidates = candidate_row_groups(
577 selected_row_group_indices.as_deref(),
578 record_batch_stream_builder.metadata().num_row_groups(),
579 );
580
581 selected_row_group_indices = Some(ArrowReader::get_selected_row_group_indices(
582 &predicate,
583 record_batch_stream_builder.metadata(),
584 &field_id_map,
585 task.schema(),
586 candidates.as_ref(),
587 )?);
588 }
589
590 if self.bloom_filter_enabled {
591 let candidates = candidate_row_groups(
592 selected_row_group_indices.as_deref(),
593 record_batch_stream_builder.metadata().num_row_groups(),
594 );
595
596 let bloom_predicate = predicate.clone();
598 let bloom_candidates = candidates.as_ref().to_vec();
599 let bloom_field_ids = field_id_map.clone();
600 let mut builder = record_batch_stream_builder;
601 let (builder, bloom_filtered) = self
602 .runtime
603 .io()
604 .spawn(async move {
605 let filtered = Self::filter_row_groups_by_bloom_filter(
606 &bloom_predicate,
607 &mut builder,
608 &bloom_candidates,
609 &bloom_field_ids,
610 )
611 .await;
612 (builder, filtered)
613 })
614 .abort_on_drop()
615 .await?;
616 record_batch_stream_builder = builder;
617 let bloom_filtered = bloom_filtered?;
618
619 if bloom_filtered.len() < candidates.len() {
620 selected_row_group_indices = Some(bloom_filtered);
621 }
622 }
623
624 if self.row_selection_enabled {
625 row_selection = ArrowReader::get_row_selection_for_filter_predicate(
626 &predicate,
627 record_batch_stream_builder.metadata(),
628 &selected_row_group_indices,
629 &field_id_map,
630 task.schema(),
631 )?;
632 }
633 }
634
635 let positional_delete_indexes = delete_filter.get_delete_vector(&task);
636
637 if let Some(positional_delete_indexes) = positional_delete_indexes {
638 let delete_row_selection = {
639 let positional_delete_indexes = positional_delete_indexes.lock().unwrap();
640
641 ArrowReader::build_deletes_row_selection(
642 record_batch_stream_builder.metadata().row_groups(),
643 &selected_row_group_indices,
644 &positional_delete_indexes,
645 )
646 }?;
647
648 row_selection = match row_selection {
651 None => Some(delete_row_selection),
652 Some(filter_row_selection) => {
653 Some(filter_row_selection.intersection(&delete_row_selection))
654 }
655 };
656 }
657
658 if let Some(row_selection) = row_selection {
659 record_batch_stream_builder =
660 record_batch_stream_builder.with_row_selection(row_selection);
661 }
662
663 if let Some(selected_row_group_indices) = selected_row_group_indices {
664 record_batch_stream_builder =
665 record_batch_stream_builder.with_row_groups(selected_row_group_indices);
666 }
667
668 let first_row_id = task.first_row_id();
673 let record_batch_stream = record_batch_stream_builder.build()?.map(move |batch| {
674 let mut batch = batch.map_err(|err| -> Error { err.into() })?;
675 if project_row_id {
676 batch = synthesize_row_id_column(batch, first_row_id)?;
677 }
678 record_batch_transformer.process_record_batch(batch)
680 });
681
682 Ok(Box::pin(record_batch_stream) as ArrowRecordBatchStream)
683 }
684
685 fn configure_arrow_reader_metadata(
688 arrow_metadata: ArrowReaderMetadata,
689 task: &FileScanTask,
690 missing_field_ids: bool,
691 install_row_number: bool,
692 ) -> Result<ArrowReaderMetadata> {
693 let without_fixed_size_lists = coerce_fixed_size_lists(arrow_metadata.schema());
705 let file_schema = without_fixed_size_lists
706 .as_ref()
707 .unwrap_or(arrow_metadata.schema());
708 let arrow_schema = if missing_field_ids {
709 let schema = if let Some(name_mapping) = task.name_mapping() {
710 apply_name_mapping_to_arrow_schema(Arc::clone(file_schema), name_mapping)?
711 } else {
712 add_fallback_field_ids_to_arrow_schema(file_schema)
713 };
714 coerce_int96_timestamps(&schema, task.schema()).unwrap_or(schema)
718 } else if let Some(coerced) = coerce_int96_timestamps(file_schema, task.schema()) {
719 coerced
720 } else if without_fixed_size_lists.is_some() || install_row_number {
721 Arc::clone(file_schema)
722 } else {
723 return Ok(arrow_metadata);
724 };
725
726 let mut options = ArrowReaderOptions::new().with_schema(Arc::clone(&arrow_schema));
727 if install_row_number {
728 let row_number_field = Arc::new(
729 Field::new(RESERVED_COL_NAME_POS, DataType::Int64, false)
730 .with_metadata(HashMap::from([(
731 PARQUET_FIELD_ID_META_KEY.to_string(),
732 RESERVED_FIELD_ID_POS.to_string(),
733 )]))
734 .with_extension_type(RowNumber),
735 );
736 options = options.with_virtual_columns(vec![row_number_field])?;
737 }
738
739 ArrowReaderMetadata::try_new(Arc::clone(arrow_metadata.metadata()), options).map_err(|e| {
740 Error::new(
741 ErrorKind::Unexpected,
742 format!(
743 "Failed to create ArrowReaderMetadata with the configured reader options \
744 (missing_field_ids: {missing_field_ids}, \
745 install_row_number: {install_row_number}, schema: {arrow_schema})"
746 ),
747 )
748 .with_source(e)
749 })
750 }
751
752 async fn filter_row_groups_by_bloom_filter(
755 predicate: &BoundPredicate,
756 builder: &mut ParquetRecordBatchStreamBuilder<ArrowFileReader>,
757 candidate_row_groups: &[usize],
758 field_id_map: &HashMap<i32, usize>,
759 ) -> Result<Vec<usize>> {
760 let bloom_filter_field_ids: Vec<i32> = collect_bloom_filter_field_ids(predicate)?
763 .into_iter()
764 .filter(|id| field_id_map.contains_key(id))
765 .collect();
766
767 if bloom_filter_field_ids.is_empty() {
768 return Ok(candidate_row_groups.to_vec());
769 }
770
771 let mut result = Vec::with_capacity(candidate_row_groups.len());
772
773 for &rg_idx in candidate_row_groups {
774 let mut bloom_filters: HashMap<i32, ColumnBloomFilter> = HashMap::new();
775
776 for &field_id in &bloom_filter_field_ids {
777 let col_idx = field_id_map[&field_id];
778 let col_meta = builder.metadata().row_group(rg_idx).column(col_idx);
779
780 if col_meta.bloom_filter_offset().is_none() {
782 continue;
783 }
784
785 let physical_type = col_meta.column_type();
786 let type_length = col_meta.column_descr().type_length();
787
788 match builder
789 .get_row_group_column_bloom_filter(rg_idx, col_idx)
790 .await
791 {
792 Ok(Some(sbbf)) => {
793 bloom_filters.insert(
794 field_id,
795 ColumnBloomFilter::new(sbbf, physical_type, type_length),
796 );
797 }
798 Ok(None) => {}
799 Err(e) => {
800 tracing::debug!(
803 "Bloom filter for field {field_id} in row group {rg_idx} could not be read: {e}"
804 );
805 }
806 }
807 }
808
809 match BloomFilterEvaluator::eval(predicate, &bloom_filters) {
810 Ok(true) => result.push(rg_idx),
811 Ok(false) => { }
812 Err(e) => {
813 tracing::debug!(
814 "Bloom filter evaluation failed for row group {rg_idx}, including it: {e}"
815 );
816 result.push(rg_idx);
817 }
818 }
819 }
820
821 Ok(result)
822 }
823}
824
825impl ArrowReader {
826 pub(crate) async fn open_parquet_file(
829 data_file_path: &str,
830 file_io: &FileIO,
831 file_size_in_bytes: u64,
832 parquet_read_options: ParquetReadOptions,
833 bytes_read: &Arc<AtomicU64>,
834 key_metadata: Option<&[u8]>,
835 ) -> Result<(ArrowFileReader, ArrowReaderMetadata)> {
836 let parquet_file = file_io.new_input(data_file_path)?;
837 let counting_reader =
838 CountingFileRead::new(parquet_file.reader().await?, Arc::clone(bytes_read));
839 Self::build_parquet_reader(
840 Box::new(counting_reader),
841 file_size_in_bytes,
842 parquet_read_options,
843 key_metadata,
844 )
845 .await
846 }
847
848 async fn build_parquet_reader(
849 parquet_reader: Box<dyn FileRead>,
850 file_size_in_bytes: u64,
851 parquet_read_options: ParquetReadOptions,
852 key_metadata: Option<&[u8]>,
853 ) -> Result<(ArrowFileReader, ArrowReaderMetadata)> {
854 let mut reader = ArrowFileReader::new(
855 FileMetadata {
856 size: file_size_in_bytes,
857 },
858 parquet_reader,
859 )
860 .with_parquet_read_options(parquet_read_options);
861
862 let arrow_reader_options = Self::build_arrow_reader_options(key_metadata)?;
863
864 let arrow_metadata = ArrowReaderMetadata::load_async(&mut reader, arrow_reader_options)
865 .await
866 .map_err(|e| {
867 Error::new(ErrorKind::Unexpected, "Failed to load Parquet metadata").with_source(e)
868 })?;
869
870 Ok((reader, arrow_metadata))
871 }
872
873 fn build_arrow_reader_options(key_metadata: Option<&[u8]>) -> Result<ArrowReaderOptions> {
876 match key_metadata {
877 Some(km) => {
878 let standard_key_metadata = StandardKeyMetadata::decode(km)?;
879 let mut builder = FileDecryptionProperties::builder(
880 standard_key_metadata.encryption_key().as_bytes().to_vec(),
881 );
882 if let Some(aad) = standard_key_metadata.aad_prefix() {
883 builder = builder.with_aad_prefix(aad.to_vec());
884 }
885 let decryption_properties = builder.build().map_err(|e| {
886 Error::new(
887 ErrorKind::Unexpected,
888 "Failed to build Parquet file decryption properties",
889 )
890 .with_source(e)
891 })?;
892 Ok(
893 ArrowReaderOptions::new()
894 .with_file_decryption_properties(decryption_properties),
895 )
896 }
897 None => Ok(ArrowReaderOptions::default()),
898 }
899 }
900}
901
902#[cfg(test)]
903mod tests {
904 use std::collections::HashMap;
905 use std::fs::File;
906 use std::sync::Arc;
907
908 use arrow_array::cast::AsArray;
909 use arrow_array::{Array, ArrayRef, Int32Array, Int64Array, RecordBatch, StringArray};
910 use arrow_cast::cast;
911 use arrow_schema::{DataType, Field, Schema as ArrowSchema};
912 use futures::TryStreamExt;
913 use parquet::arrow::{ArrowWriter, PARQUET_FIELD_ID_META_KEY};
914 use parquet::basic::Compression;
915 use parquet::file::properties::WriterProperties;
916 use tempfile::TempDir;
917
918 use crate::Runtime;
919 use crate::arrow::test_utils::write_encrypted_parquet;
920 use crate::arrow::{ArrowReaderBuilder, ArrowReaderOptions as ReaderOptions};
921 use crate::io::FileIO;
922 use crate::metadata_columns::{
923 RESERVED_COL_NAME_POS, RESERVED_COL_NAME_ROW_ID, RESERVED_FIELD_ID_FILE,
924 RESERVED_FIELD_ID_POS, RESERVED_FIELD_ID_ROW_ID,
925 };
926 use crate::scan::{FileScanTask, FileScanTaskDeleteFile, FileScanTaskStream};
927 use crate::spec::{DataFileFormat, NestedField, PrimitiveType, Schema, SchemaRef, Type};
928
929 fn record_count_task(path: &str, file_size: u64, record_count: Option<u64>) -> FileScanTask {
930 let schema = Schema::builder()
931 .with_fields(vec![
932 NestedField::required(1, "id", Type::Primitive(PrimitiveType::Int)).into(),
933 ])
934 .build()
935 .unwrap();
936 FileScanTask::builder()
937 .with_data_file_path(path.to_string())
938 .with_data_file_format(DataFileFormat::Parquet)
939 .with_file_size_in_bytes(file_size)
940 .with_start(0)
941 .with_length(file_size)
942 .with_record_count(record_count)
943 .with_schema(Arc::new(schema))
944 .with_project_field_ids(vec![1])
945 .with_case_sensitive(true)
946 .build()
947 .unwrap()
948 }
949
950 async fn read_tasks(
951 tasks: Vec<FileScanTask>,
952 concurrency: usize,
953 ) -> crate::Result<Vec<RecordBatch>> {
954 let tasks = futures::stream::iter(tasks.into_iter().map(Ok));
955 ArrowReaderBuilder::new(FileIO::new_with_fs(), Runtime::current())
956 .with_options(
957 ReaderOptions::builder()
958 .with_data_file_concurrency_limit(concurrency)
959 .build(),
960 )
961 .build()
962 .read(Box::pin(tasks))?
963 .stream()
964 .try_collect()
965 .await
966 }
967
968 #[tokio::test]
969 async fn test_read_zero_record_files_without_opening_them() {
970 let temp_dir = TempDir::new().unwrap();
971 let missing_path = temp_dir
972 .path()
973 .join("missing.parquet")
974 .display()
975 .to_string();
976
977 for concurrency in [1, 2] {
978 let tasks = vec![
980 record_count_task(&missing_path, 0, Some(0)),
981 record_count_task(&missing_path, 128, Some(0)),
982 ];
983 let batches = read_tasks(tasks, concurrency).await.unwrap();
984 assert!(batches.is_empty());
985 }
986 }
987
988 #[tokio::test]
989 async fn test_read_zero_size_files_with_nonzero_or_unknown_record_count() {
990 let temp_dir = TempDir::new().unwrap();
991 let path = temp_dir.path().join("empty.parquet").display().to_string();
992 File::create(&path).unwrap();
993
994 for concurrency in [1, 2] {
995 for record_count in [Some(1), None] {
996 let task = record_count_task(&path, 0, record_count);
997 let error = read_tasks(vec![task], concurrency).await.unwrap_err();
998 assert!(
999 error
1000 .to_string()
1001 .contains("file size of 0 is less than footer"),
1002 "{error}"
1003 );
1004 }
1005 }
1006 }
1007
1008 #[tokio::test]
1009 async fn test_read_nonempty_files_alongside_zero_record_files() {
1010 use arrow_array::types::Int32Type;
1011
1012 let temp_dir = TempDir::new().unwrap();
1013 let dir = temp_dir.path().to_str().unwrap();
1014 let path = write_plain_parquet(dir, "data.parquet", vec![], vec![]);
1015 let file_size = std::fs::metadata(&path).unwrap().len();
1016 let missing_path = format!("{dir}/missing.parquet");
1017
1018 for concurrency in [1, 2] {
1019 for record_count in [Some(3), None] {
1020 let empty_task = record_count_task(&missing_path, 0, Some(0));
1021 let data_task = record_count_task(&path, file_size, record_count);
1022 let tasks = vec![empty_task.clone(), data_task, empty_task];
1023 let batches = read_tasks(tasks, concurrency).await.unwrap();
1024
1025 assert_eq!(batches.len(), 1);
1026 let ids = batches[0].column(0).as_primitive::<Int32Type>();
1027 assert_eq!(ids.values(), &[1, 2, 3]);
1028 }
1029 }
1030 }
1031
1032 const UNIX_EPOCH_JULIAN: i64 = 2_440_588;
1035 const MICROS_PER_DAY: i64 = 86_400_000_000;
1036 const INT96_TEST_NANOS_WITHIN_DAY: u64 = 43_200_000_000_000;
1038 const INT96_TEST_JULIAN_DAY: u32 = 2_953_529;
1039
1040 fn make_int96_test_value() -> (parquet::data_type::Int96, i64) {
1041 let mut val = parquet::data_type::Int96::new();
1042 val.set_data(
1043 (INT96_TEST_NANOS_WITHIN_DAY & 0xFFFFFFFF) as u32,
1044 (INT96_TEST_NANOS_WITHIN_DAY >> 32) as u32,
1045 INT96_TEST_JULIAN_DAY,
1046 );
1047 let expected_micros = (INT96_TEST_JULIAN_DAY as i64 - UNIX_EPOCH_JULIAN) * MICROS_PER_DAY
1048 + (INT96_TEST_NANOS_WITHIN_DAY / 1_000) as i64;
1049 (val, expected_micros)
1050 }
1051
1052 async fn read_int96_batches(
1053 file_path: &str,
1054 schema: SchemaRef,
1055 project_field_ids: Vec<i32>,
1056 ) -> Vec<RecordBatch> {
1057 let file_io = FileIO::new_with_fs();
1058 let reader = ArrowReaderBuilder::new(file_io, Runtime::current()).build();
1059
1060 let file_size = std::fs::metadata(file_path).unwrap().len();
1061 let task = FileScanTask::builder()
1062 .with_file_size_in_bytes(file_size)
1063 .with_start(0)
1064 .with_length(file_size)
1065 .with_data_file_path(file_path.to_string())
1066 .with_data_file_format(DataFileFormat::Parquet)
1067 .with_schema(schema)
1068 .with_project_field_ids(project_field_ids)
1069 .with_case_sensitive(false)
1070 .build()
1071 .unwrap();
1072
1073 let tasks = Box::pin(futures::stream::iter(vec![Ok(task)])) as FileScanTaskStream;
1074 reader
1075 .read(tasks)
1076 .unwrap()
1077 .stream()
1078 .try_collect()
1079 .await
1080 .unwrap()
1081 }
1082
1083 fn write_int96_parquet_file(
1085 table_location: &str,
1086 filename: &str,
1087 with_field_ids: bool,
1088 ) -> (String, Vec<i64>) {
1089 use parquet::basic::{Repetition, Type as PhysicalType};
1090 use parquet::data_type::{Int32Type, Int96, Int96Type};
1091 use parquet::file::writer::SerializedFileWriter;
1092 use parquet::schema::types::Type as SchemaType;
1093
1094 let file_path = format!("{table_location}/{filename}");
1095
1096 let mut ts_builder = SchemaType::primitive_type_builder("ts", PhysicalType::INT96)
1097 .with_repetition(Repetition::OPTIONAL);
1098 let mut id_builder = SchemaType::primitive_type_builder("id", PhysicalType::INT32)
1099 .with_repetition(Repetition::REQUIRED);
1100
1101 if with_field_ids {
1102 ts_builder = ts_builder.with_id(Some(1));
1103 id_builder = id_builder.with_id(Some(2));
1104 }
1105
1106 let schema = SchemaType::group_type_builder("schema")
1107 .with_fields(vec![
1108 Arc::new(ts_builder.build().unwrap()),
1109 Arc::new(id_builder.build().unwrap()),
1110 ])
1111 .build()
1112 .unwrap();
1113
1114 const NOON_NANOS: u64 = INT96_TEST_NANOS_WITHIN_DAY;
1116 const JULIAN_3333: u32 = INT96_TEST_JULIAN_DAY;
1117 const JULIAN_2100: u32 = 2_488_070;
1118
1119 let test_data: Vec<(u32, u32, u32, i64)> = vec![
1120 (
1122 0,
1123 0,
1124 JULIAN_3333,
1125 (JULIAN_3333 as i64 - UNIX_EPOCH_JULIAN) * MICROS_PER_DAY,
1126 ),
1127 (
1129 (NOON_NANOS & 0xFFFFFFFF) as u32,
1130 (NOON_NANOS >> 32) as u32,
1131 JULIAN_3333,
1132 (JULIAN_3333 as i64 - UNIX_EPOCH_JULIAN) * MICROS_PER_DAY
1133 + (NOON_NANOS / 1_000) as i64,
1134 ),
1135 (
1137 0,
1138 0,
1139 JULIAN_2100,
1140 (JULIAN_2100 as i64 - UNIX_EPOCH_JULIAN) * MICROS_PER_DAY,
1141 ),
1142 ];
1143
1144 let int96_values: Vec<Int96> = test_data
1145 .iter()
1146 .map(|(lo, hi, day, _)| {
1147 let mut v = Int96::new();
1148 v.set_data(*lo, *hi, *day);
1149 v
1150 })
1151 .collect();
1152
1153 let id_values: Vec<i32> = (0..test_data.len() as i32).collect();
1154 let expected_micros: Vec<i64> = test_data.iter().map(|(_, _, _, m)| *m).collect();
1155
1156 let file = File::create(&file_path).unwrap();
1157 let mut writer =
1158 SerializedFileWriter::new(file, Arc::new(schema), Default::default()).unwrap();
1159
1160 let mut row_group = writer.next_row_group().unwrap();
1161 {
1162 let mut col = row_group.next_column().unwrap().unwrap();
1164 col.typed::<Int96Type>()
1165 .write_batch(&int96_values, Some(&vec![1; test_data.len()]), None)
1166 .unwrap();
1167 col.close().unwrap();
1168 }
1169 {
1170 let mut col = row_group.next_column().unwrap().unwrap();
1171 col.typed::<Int32Type>()
1172 .write_batch(&id_values, None, None)
1173 .unwrap();
1174 col.close().unwrap();
1175 }
1176 row_group.close().unwrap();
1177 writer.close().unwrap();
1178
1179 (file_path, expected_micros)
1180 }
1181
1182 async fn assert_int96_read_matches(
1183 file_path: &str,
1184 schema: SchemaRef,
1185 project_field_ids: Vec<i32>,
1186 expected_micros: &[i64],
1187 ) {
1188 use arrow_array::TimestampMicrosecondArray;
1189
1190 let batches = read_int96_batches(file_path, schema, project_field_ids).await;
1191
1192 assert_eq!(batches.len(), 1);
1193 let ts_array = batches[0]
1194 .column(0)
1195 .as_any()
1196 .downcast_ref::<TimestampMicrosecondArray>()
1197 .expect("Expected TimestampMicrosecondArray");
1198
1199 for (i, expected) in expected_micros.iter().enumerate() {
1200 assert_eq!(
1201 ts_array.value(i),
1202 *expected,
1203 "Row {i}: got {}, expected {expected}",
1204 ts_array.value(i)
1205 );
1206 }
1207 }
1208
1209 async fn assert_encrypted_parquet_roundtrip(encryption_key: &[u8]) {
1213 let aad_prefix = b"aad_prefix";
1214
1215 let schema = Arc::new(
1216 Schema::builder()
1217 .with_schema_id(1)
1218 .with_fields(vec![
1219 NestedField::required(1, "id", Type::Primitive(PrimitiveType::Int)).into(),
1220 ])
1221 .build()
1222 .unwrap(),
1223 );
1224
1225 let arrow_schema = Arc::new(ArrowSchema::new(vec![
1226 Field::new("id", DataType::Int32, false).with_metadata(HashMap::from([(
1227 PARQUET_FIELD_ID_META_KEY.to_string(),
1228 "1".to_string(),
1229 )])),
1230 ]));
1231
1232 let tmp_dir = TempDir::new().unwrap();
1233 let table_location = tmp_dir.path().to_str().unwrap().to_string();
1234 let file_io = FileIO::new_with_fs();
1235
1236 let id_data = Arc::new(Int32Array::from(vec![10, 20, 30])) as ArrayRef;
1237 let batch = RecordBatch::try_new(arrow_schema.clone(), vec![id_data]).unwrap();
1238
1239 let file_path = format!("{table_location}/encrypted.parquet");
1240 write_encrypted_parquet(&file_path, &batch, encryption_key, Some(aad_prefix));
1241
1242 let key_metadata = crate::encryption::StandardKeyMetadata::try_new(encryption_key)
1243 .unwrap()
1244 .with_aad_prefix(aad_prefix)
1245 .encode()
1246 .unwrap();
1247
1248 let reader = ArrowReaderBuilder::new(file_io, Runtime::current()).build();
1249
1250 let task = FileScanTask::builder()
1251 .with_file_size_in_bytes(std::fs::metadata(&file_path).unwrap().len())
1252 .with_start(0)
1253 .with_length(0)
1254 .with_data_file_path(file_path)
1255 .with_data_file_format(DataFileFormat::Parquet)
1256 .with_schema(schema)
1257 .with_project_field_ids(vec![1])
1258 .with_case_sensitive(false)
1259 .with_key_metadata(Some(key_metadata))
1260 .build()
1261 .unwrap();
1262
1263 let tasks = Box::pin(futures::stream::iter(vec![Ok(task)])) as FileScanTaskStream;
1264 let batches: Vec<RecordBatch> = reader
1265 .read(tasks)
1266 .unwrap()
1267 .stream()
1268 .try_collect()
1269 .await
1270 .unwrap();
1271
1272 assert_eq!(batches.len(), 1);
1273 let ids = batches[0]
1274 .column(0)
1275 .as_any()
1276 .downcast_ref::<Int32Array>()
1277 .unwrap();
1278 assert_eq!(ids.values(), &[10, 20, 30]);
1279 }
1280
1281 #[tokio::test]
1282 async fn test_read_encrypted_parquet_aes_128() {
1283 assert_encrypted_parquet_roundtrip(b"0123456789abcdef").await;
1284 }
1285
1286 #[tokio::test]
1287 async fn test_read_encrypted_parquet_aes_256() {
1288 assert_encrypted_parquet_roundtrip(b"0123456789abcdef0123456789abcdef").await;
1289 }
1290
1291 #[tokio::test]
1292 async fn test_read_encrypted_parquet_without_key_metadata_fails() {
1293 let encryption_key = b"0123456789abcdef";
1294
1295 let schema = Arc::new(
1296 Schema::builder()
1297 .with_schema_id(1)
1298 .with_fields(vec![
1299 NestedField::required(1, "id", Type::Primitive(PrimitiveType::Int)).into(),
1300 ])
1301 .build()
1302 .unwrap(),
1303 );
1304
1305 let arrow_schema = Arc::new(ArrowSchema::new(vec![
1306 Field::new("id", DataType::Int32, false).with_metadata(HashMap::from([(
1307 PARQUET_FIELD_ID_META_KEY.to_string(),
1308 "1".to_string(),
1309 )])),
1310 ]));
1311
1312 let tmp_dir = TempDir::new().unwrap();
1313 let table_location = tmp_dir.path().to_str().unwrap().to_string();
1314 let file_io = FileIO::new_with_fs();
1315
1316 let id_data = Arc::new(Int32Array::from(vec![1, 2, 3])) as ArrayRef;
1317 let batch = RecordBatch::try_new(arrow_schema.clone(), vec![id_data]).unwrap();
1318
1319 let file_path = format!("{table_location}/encrypted_no_key.parquet");
1320 write_encrypted_parquet(&file_path, &batch, encryption_key, None);
1321
1322 let reader = ArrowReaderBuilder::new(file_io, Runtime::current()).build();
1323
1324 let task = FileScanTask::builder()
1325 .with_file_size_in_bytes(std::fs::metadata(&file_path).unwrap().len())
1326 .with_start(0)
1327 .with_length(0)
1328 .with_data_file_path(file_path)
1329 .with_data_file_format(DataFileFormat::Parquet)
1330 .with_schema(schema)
1331 .with_project_field_ids(vec![1])
1332 .with_case_sensitive(false)
1333 .build()
1334 .unwrap();
1335
1336 let tasks = Box::pin(futures::stream::iter(vec![Ok(task)])) as FileScanTaskStream;
1337 let result: Result<Vec<RecordBatch>, _> =
1338 reader.read(tasks).unwrap().stream().try_collect().await;
1339
1340 let err = result.unwrap_err();
1341 assert_eq!(err.kind(), crate::ErrorKind::Unexpected);
1342 let err_str = format!("{err}");
1343 assert!(
1344 err_str.contains("encrypted footer"),
1345 "Expected error about encrypted footer, got: {err_str}"
1346 );
1347 assert!(
1348 err_str.contains("decryption properties were not provided"),
1349 "Expected error about missing decryption properties, got: {err_str}"
1350 );
1351 }
1352
1353 fn write_plain_parquet(
1356 dir: &str,
1357 name: &str,
1358 extra_fields: Vec<Field>,
1359 extra_columns: Vec<ArrayRef>,
1360 ) -> String {
1361 let props = WriterProperties::builder()
1362 .set_compression(Compression::SNAPPY)
1363 .build();
1364 write_parquet_with(dir, name, extra_fields, extra_columns, props)
1365 }
1366
1367 fn write_parquet_with(
1368 dir: &str,
1369 name: &str,
1370 extra_fields: Vec<Field>,
1371 extra_columns: Vec<ArrayRef>,
1372 props: WriterProperties,
1373 ) -> String {
1374 let mut fields =
1375 vec![
1376 Field::new("id", DataType::Int32, false).with_metadata(HashMap::from([(
1377 PARQUET_FIELD_ID_META_KEY.to_string(),
1378 "1".to_string(),
1379 )])),
1380 ];
1381 fields.extend(extra_fields);
1382 let arrow_schema = Arc::new(ArrowSchema::new(fields));
1383
1384 let mut columns: Vec<ArrayRef> = vec![Arc::new(Int32Array::from(vec![1, 2, 3]))];
1385 columns.extend(extra_columns);
1386 let batch = RecordBatch::try_new(arrow_schema.clone(), columns).unwrap();
1387
1388 let file_path = format!("{dir}/{name}");
1389 let file = File::create(&file_path).unwrap();
1390 let mut writer = ArrowWriter::try_new(file, arrow_schema, Some(props)).unwrap();
1391 writer.write(&batch).unwrap();
1392 writer.close().unwrap();
1393 file_path
1394 }
1395
1396 fn last_updated_seq_task(
1397 file_path: String,
1398 first_row_id: Option<i64>,
1399 data_sequence_number: Option<i64>,
1400 ) -> FileScanTask {
1401 use crate::metadata_columns::RESERVED_FIELD_ID_LAST_UPDATED_SEQUENCE_NUMBER;
1402
1403 let schema = Arc::new(
1404 Schema::builder()
1405 .with_schema_id(1)
1406 .with_fields(vec![
1407 NestedField::required(1, "id", Type::Primitive(PrimitiveType::Int)).into(),
1408 ])
1409 .build()
1410 .unwrap(),
1411 );
1412
1413 FileScanTask::builder()
1414 .with_file_size_in_bytes(std::fs::metadata(&file_path).unwrap().len())
1415 .with_start(0)
1416 .with_length(0)
1417 .with_data_file_path(file_path)
1418 .with_data_file_format(DataFileFormat::Parquet)
1419 .with_schema(schema)
1420 .with_project_field_ids(vec![1, RESERVED_FIELD_ID_LAST_UPDATED_SEQUENCE_NUMBER])
1421 .with_first_row_id(first_row_id)
1422 .with_data_sequence_number(data_sequence_number)
1423 .with_case_sensitive(false)
1424 .build()
1425 .unwrap()
1426 }
1427
1428 fn assert_last_updated_seq_column(batches: &[RecordBatch], expected: &[Option<i64>]) {
1431 use arrow_array::cast::AsArray;
1432 use arrow_cast::cast;
1433 use arrow_schema::DataType;
1434
1435 use crate::metadata_columns::RESERVED_COL_NAME_LAST_UPDATED_SEQUENCE_NUMBER;
1436
1437 let mut actual = Vec::new();
1438 for batch in batches {
1439 let col = batch
1440 .column_by_name(RESERVED_COL_NAME_LAST_UPDATED_SEQUENCE_NUMBER)
1441 .expect("_last_updated_sequence_number column should be present");
1442 let logical = cast(col, &DataType::Int64).unwrap();
1443 let values = logical.as_primitive::<arrow_array::types::Int64Type>();
1444 for i in 0..values.len() {
1445 actual.push((!values.is_null(i)).then(|| values.value(i)));
1446 }
1447 }
1448 assert_eq!(actual, expected);
1449 }
1450
1451 #[tokio::test]
1452 async fn test_last_updated_sequence_number_null_when_no_first_row_id() {
1453 let tmp_dir = TempDir::new().unwrap();
1454 let dir = tmp_dir.path().to_str().unwrap();
1455 let file_path = write_plain_parquet(dir, "no_first_row_id.parquet", vec![], vec![]);
1456
1457 let task = last_updated_seq_task(file_path, None, Some(9));
1461
1462 let reader = ArrowReaderBuilder::new(FileIO::new_with_fs(), Runtime::current()).build();
1463 let tasks = Box::pin(futures::stream::iter(vec![Ok(task)])) as FileScanTaskStream;
1464 let batches: Vec<RecordBatch> = reader
1465 .read(tasks)
1466 .unwrap()
1467 .stream()
1468 .try_collect()
1469 .await
1470 .unwrap();
1471
1472 assert_last_updated_seq_column(&batches, &[None, None, None]);
1473 }
1474
1475 #[tokio::test]
1476 async fn test_last_updated_sequence_number_error_when_no_data_seq() {
1477 let tmp_dir = TempDir::new().unwrap();
1478 let dir = tmp_dir.path().to_str().unwrap();
1479 let file_path = write_plain_parquet(dir, "no_data_seq.parquet", vec![], vec![]);
1480
1481 let task = last_updated_seq_task(file_path, Some(42), None);
1485
1486 let reader = ArrowReaderBuilder::new(FileIO::new_with_fs(), Runtime::current()).build();
1487 let tasks = Box::pin(futures::stream::iter(vec![Ok(task)])) as FileScanTaskStream;
1488 let result: Result<Vec<RecordBatch>, _> =
1489 reader.read(tasks).unwrap().stream().try_collect().await;
1490
1491 let err = result.unwrap_err();
1492 assert_eq!(err.kind(), crate::ErrorKind::DataInvalid);
1493 assert!(
1494 format!("{err}").contains("no data sequence number"),
1495 "unexpected error: {err}"
1496 );
1497 }
1498
1499 #[tokio::test]
1500 async fn test_last_updated_sequence_number_derived_from_data_seq() {
1501 let tmp_dir = TempDir::new().unwrap();
1502 let dir = tmp_dir.path().to_str().unwrap();
1503 let file_path = write_plain_parquet(dir, "with_first_row_id.parquet", vec![], vec![]);
1504
1505 let task = last_updated_seq_task(file_path, Some(42), Some(7));
1508
1509 let reader = ArrowReaderBuilder::new(FileIO::new_with_fs(), Runtime::current()).build();
1510 let tasks = Box::pin(futures::stream::iter(vec![Ok(task)])) as FileScanTaskStream;
1511 let batches: Vec<RecordBatch> = reader
1512 .read(tasks)
1513 .unwrap()
1514 .stream()
1515 .try_collect()
1516 .await
1517 .unwrap();
1518
1519 assert_last_updated_seq_column(&batches, &[Some(7), Some(7), Some(7)]);
1520 }
1521
1522 #[tokio::test]
1523 async fn test_last_updated_sequence_number_mixed_files_share_schema() {
1524 use arrow_select::concat::concat_batches;
1525
1526 let tmp_dir = TempDir::new().unwrap();
1527 let dir = tmp_dir.path().to_str().unwrap();
1528
1529 let constant = last_updated_seq_task(
1535 write_plain_parquet(dir, "constant.parquet", vec![], vec![]),
1536 Some(42),
1537 Some(7),
1538 );
1539 let nulled = last_updated_seq_task(
1540 write_plain_parquet(dir, "nulled.parquet", vec![], vec![]),
1541 None,
1542 Some(7),
1543 );
1544 let coalesced = last_updated_seq_task(
1545 write_plain_parquet(
1546 dir,
1547 "coalesced.parquet",
1548 vec![physical_last_updated_seq_field()],
1549 vec![Arc::new(Int64Array::from(vec![Some(5), None, Some(8)])) as ArrayRef],
1550 ),
1551 Some(50),
1552 Some(7),
1553 );
1554
1555 let reader = ArrowReaderBuilder::new(FileIO::new_with_fs(), Runtime::current()).build();
1556 let tasks = Box::pin(futures::stream::iter(vec![
1557 Ok(constant),
1558 Ok(nulled),
1559 Ok(coalesced),
1560 ])) as FileScanTaskStream;
1561 let batches: Vec<RecordBatch> = reader
1562 .read(tasks)
1563 .unwrap()
1564 .stream()
1565 .try_collect()
1566 .await
1567 .unwrap();
1568
1569 assert_eq!(batches.len(), 3);
1570 let schema = batches[0].schema();
1572 concat_batches(&schema, &batches)
1573 .expect("constant, null and coalesce files must share one column type");
1574 }
1575
1576 fn physical_last_updated_seq_field() -> Field {
1578 use crate::metadata_columns::{
1579 RESERVED_COL_NAME_LAST_UPDATED_SEQUENCE_NUMBER,
1580 RESERVED_FIELD_ID_LAST_UPDATED_SEQUENCE_NUMBER,
1581 };
1582 Field::new(
1583 RESERVED_COL_NAME_LAST_UPDATED_SEQUENCE_NUMBER,
1584 DataType::Int64,
1585 true,
1586 )
1587 .with_metadata(HashMap::from([(
1588 PARQUET_FIELD_ID_META_KEY.to_string(),
1589 RESERVED_FIELD_ID_LAST_UPDATED_SEQUENCE_NUMBER.to_string(),
1590 )]))
1591 }
1592
1593 #[tokio::test]
1594 async fn test_last_updated_sequence_number_physical_column_coalesced() {
1595 let tmp_dir = TempDir::new().unwrap();
1596 let dir = tmp_dir.path().to_str().unwrap();
1597 let seq_col = Arc::new(Int64Array::from(vec![Some(5), None, Some(8)])) as ArrayRef;
1601 let file_path = write_plain_parquet(
1602 dir,
1603 "with_seq.parquet",
1604 vec![physical_last_updated_seq_field()],
1605 vec![seq_col],
1606 );
1607
1608 let task = last_updated_seq_task(file_path, Some(100), Some(9));
1609
1610 let reader = ArrowReaderBuilder::new(FileIO::new_with_fs(), Runtime::current()).build();
1611 let tasks = Box::pin(futures::stream::iter(vec![Ok(task)])) as FileScanTaskStream;
1612 let batches: Vec<RecordBatch> = reader
1613 .read(tasks)
1614 .unwrap()
1615 .stream()
1616 .try_collect()
1617 .await
1618 .unwrap();
1619
1620 assert_last_updated_seq_column(&batches, &[Some(5), Some(9), Some(8)]);
1622 }
1623
1624 #[tokio::test]
1625 async fn test_last_updated_sequence_number_coalesced_with_pos_column() {
1626 use crate::metadata_columns::{
1627 RESERVED_COL_NAME_POS, RESERVED_FIELD_ID_LAST_UPDATED_SEQUENCE_NUMBER,
1628 RESERVED_FIELD_ID_POS,
1629 };
1630
1631 let tmp_dir = TempDir::new().unwrap();
1632 let dir = tmp_dir.path().to_str().unwrap();
1633 let seq_col = Arc::new(Int64Array::from(vec![Some(5), None, Some(8)])) as ArrayRef;
1634 let file_path = write_plain_parquet(
1635 dir,
1636 "with_seq_and_pos.parquet",
1637 vec![physical_last_updated_seq_field()],
1638 vec![seq_col],
1639 );
1640
1641 let schema = Arc::new(
1646 Schema::builder()
1647 .with_schema_id(1)
1648 .with_fields(vec![
1649 NestedField::required(1, "id", Type::Primitive(PrimitiveType::Int)).into(),
1650 ])
1651 .build()
1652 .unwrap(),
1653 );
1654 let task = FileScanTask::builder()
1655 .with_file_size_in_bytes(std::fs::metadata(&file_path).unwrap().len())
1656 .with_start(0)
1657 .with_length(0)
1658 .with_data_file_path(file_path)
1659 .with_data_file_format(DataFileFormat::Parquet)
1660 .with_schema(schema)
1661 .with_project_field_ids(vec![
1662 1,
1663 RESERVED_FIELD_ID_POS,
1664 RESERVED_FIELD_ID_LAST_UPDATED_SEQUENCE_NUMBER,
1665 ])
1666 .with_first_row_id(Some(100))
1667 .with_data_sequence_number(Some(9))
1668 .with_case_sensitive(false)
1669 .build()
1670 .unwrap();
1671
1672 let reader = ArrowReaderBuilder::new(FileIO::new_with_fs(), Runtime::current()).build();
1673 let tasks = Box::pin(futures::stream::iter(vec![Ok(task)])) as FileScanTaskStream;
1674 let batches: Vec<RecordBatch> = reader
1675 .read(tasks)
1676 .unwrap()
1677 .stream()
1678 .try_collect()
1679 .await
1680 .unwrap();
1681
1682 assert_last_updated_seq_column(&batches, &[Some(5), Some(9), Some(8)]);
1684 let pos_col = batches[0]
1686 .column_by_name(RESERVED_COL_NAME_POS)
1687 .expect("_pos column should be present")
1688 .as_primitive::<arrow_array::types::Int64Type>();
1689 assert_eq!(pos_col.values(), &[0, 1, 2]);
1690 }
1691
1692 #[tokio::test]
1693 async fn test_last_updated_sequence_number_physical_column_nulled_without_first_row_id() {
1694 let tmp_dir = TempDir::new().unwrap();
1695 let dir = tmp_dir.path().to_str().unwrap();
1696 let seq_col = Arc::new(Int64Array::from(vec![Some(5), None, Some(8)])) as ArrayRef;
1697 let file_path = write_plain_parquet(
1698 dir,
1699 "with_seq_no_first_row_id.parquet",
1700 vec![physical_last_updated_seq_field()],
1701 vec![seq_col],
1702 );
1703
1704 let task = last_updated_seq_task(file_path, None, Some(9));
1707
1708 let reader = ArrowReaderBuilder::new(FileIO::new_with_fs(), Runtime::current()).build();
1709 let tasks = Box::pin(futures::stream::iter(vec![Ok(task)])) as FileScanTaskStream;
1710 let batches: Vec<RecordBatch> = reader
1711 .read(tasks)
1712 .unwrap()
1713 .stream()
1714 .try_collect()
1715 .await
1716 .unwrap();
1717
1718 assert_last_updated_seq_column(&batches, &[None, None, None]);
1719 }
1720
1721 #[tokio::test]
1722 async fn test_last_updated_sequence_number_present_by_name_without_id_unsupported() {
1723 let tmp_dir = TempDir::new().unwrap();
1724 let dir = tmp_dir.path().to_str().unwrap();
1725 let seq_field = Field::new(
1729 crate::metadata_columns::RESERVED_COL_NAME_LAST_UPDATED_SEQUENCE_NUMBER,
1730 DataType::Int64,
1731 true,
1732 );
1733 let seq_col = Arc::new(Int64Array::from(vec![Some(5), None, Some(8)])) as ArrayRef;
1734 let file_path =
1735 write_plain_parquet(dir, "with_seq_by_name.parquet", vec![seq_field], vec![
1736 seq_col,
1737 ]);
1738
1739 let task = last_updated_seq_task(file_path, Some(100), Some(9));
1740
1741 let reader = ArrowReaderBuilder::new(FileIO::new_with_fs(), Runtime::current()).build();
1742 let tasks = Box::pin(futures::stream::iter(vec![Ok(task)])) as FileScanTaskStream;
1743 let result: Result<Vec<RecordBatch>, _> =
1744 reader.read(tasks).unwrap().stream().try_collect().await;
1745
1746 let err = result.unwrap_err();
1747 assert_eq!(err.kind(), crate::ErrorKind::FeatureUnsupported);
1748 assert!(
1749 format!("{err}").contains("without an embedded field id"),
1750 "unexpected error: {err}"
1751 );
1752 }
1753
1754 #[tokio::test]
1755 async fn test_last_updated_sequence_number_physical_column_first_row_id_without_data_seq() {
1756 let tmp_dir = TempDir::new().unwrap();
1757 let dir = tmp_dir.path().to_str().unwrap();
1758 let seq_col = Arc::new(Int64Array::from(vec![Some(5), None, Some(8)])) as ArrayRef;
1759 let file_path = write_plain_parquet(
1760 dir,
1761 "with_seq_no_data_seq.parquet",
1762 vec![physical_last_updated_seq_field()],
1763 vec![seq_col],
1764 );
1765
1766 let task = last_updated_seq_task(file_path, Some(100), None);
1770
1771 let reader = ArrowReaderBuilder::new(FileIO::new_with_fs(), Runtime::current()).build();
1772 let tasks = Box::pin(futures::stream::iter(vec![Ok(task)])) as FileScanTaskStream;
1773 let result: Result<Vec<RecordBatch>, _> =
1774 reader.read(tasks).unwrap().stream().try_collect().await;
1775
1776 let err = result.unwrap_err();
1777 assert_eq!(err.kind(), crate::ErrorKind::DataInvalid);
1778 assert!(
1779 format!("{err}").contains("no data sequence number"),
1780 "unexpected error: {err}"
1781 );
1782 }
1783
1784 fn row_id_task(file_path: String, first_row_id: Option<i64>) -> FileScanTask {
1786 row_id_task_with_options(file_path, first_row_id, 0, 0, vec![])
1787 }
1788
1789 fn row_id_task_with_options(
1790 file_path: String,
1791 first_row_id: Option<i64>,
1792 start: u64,
1793 length: u64,
1794 deletes: Vec<FileScanTaskDeleteFile>,
1795 ) -> FileScanTask {
1796 let schema = Arc::new(
1797 Schema::builder()
1798 .with_schema_id(1)
1799 .with_fields(vec![
1800 NestedField::required(1, "id", Type::Primitive(PrimitiveType::Int)).into(),
1801 ])
1802 .build()
1803 .unwrap(),
1804 );
1805
1806 FileScanTask::builder()
1807 .with_file_size_in_bytes(std::fs::metadata(&file_path).unwrap().len())
1808 .with_start(start)
1809 .with_length(length)
1810 .with_data_file_path(file_path)
1811 .with_data_file_format(DataFileFormat::Parquet)
1812 .with_schema(schema)
1813 .with_project_field_ids(vec![1, RESERVED_FIELD_ID_ROW_ID])
1814 .with_first_row_id(first_row_id)
1815 .with_deletes(deletes)
1816 .with_case_sensitive(false)
1817 .build()
1818 .unwrap()
1819 }
1820
1821 fn assert_row_id_column(batches: &[RecordBatch], expected: &[Option<i64>]) {
1824 use arrow_array::cast::AsArray;
1825 use arrow_cast::cast;
1826 use arrow_schema::DataType;
1827
1828 let mut actual = Vec::new();
1829 for batch in batches {
1830 let col = batch
1831 .column_by_name(RESERVED_COL_NAME_ROW_ID)
1832 .expect("_row_id column should be present");
1833 let logical = cast(col, &DataType::Int64).unwrap();
1834 let values = logical.as_primitive::<arrow_array::types::Int64Type>();
1835 for i in 0..values.len() {
1836 actual.push((!values.is_null(i)).then(|| values.value(i)));
1837 }
1838 }
1839 assert_eq!(actual, expected);
1840 }
1841
1842 fn physical_row_id_field() -> Field {
1844 Field::new(RESERVED_COL_NAME_ROW_ID, DataType::Int64, true).with_metadata(HashMap::from([
1845 (
1846 PARQUET_FIELD_ID_META_KEY.to_string(),
1847 RESERVED_FIELD_ID_ROW_ID.to_string(),
1848 ),
1849 ]))
1850 }
1851
1852 #[tokio::test]
1853 async fn test_row_id_synthesized_from_first_row_id_and_pos() {
1854 let tmp_dir = TempDir::new().unwrap();
1855 let dir = tmp_dir.path().to_str().unwrap();
1856 let file_path = write_plain_parquet(dir, "row_id_synth.parquet", vec![], vec![]);
1857
1858 let task = row_id_task(file_path, Some(100));
1860
1861 let reader = ArrowReaderBuilder::new(FileIO::new_with_fs(), Runtime::current()).build();
1862 let tasks = Box::pin(futures::stream::iter(vec![Ok(task)])) as FileScanTaskStream;
1863 let batches: Vec<RecordBatch> = reader
1864 .read(tasks)
1865 .unwrap()
1866 .stream()
1867 .try_collect()
1868 .await
1869 .unwrap();
1870
1871 assert_row_id_column(&batches, &[Some(100), Some(101), Some(102)]);
1872 }
1873
1874 #[tokio::test]
1875 async fn test_row_id_physical_column_coalesced() {
1876 let tmp_dir = TempDir::new().unwrap();
1877 let dir = tmp_dir.path().to_str().unwrap();
1878 let id_col = Arc::new(Int64Array::from(vec![Some(5), None, Some(8)])) as ArrayRef;
1881 let file_path = write_plain_parquet(
1882 dir,
1883 "row_id_phys.parquet",
1884 vec![physical_row_id_field()],
1885 vec![id_col],
1886 );
1887
1888 let task = row_id_task(file_path, Some(100));
1889
1890 let reader = ArrowReaderBuilder::new(FileIO::new_with_fs(), Runtime::current()).build();
1891 let tasks = Box::pin(futures::stream::iter(vec![Ok(task)])) as FileScanTaskStream;
1892 let batches: Vec<RecordBatch> = reader
1893 .read(tasks)
1894 .unwrap()
1895 .stream()
1896 .try_collect()
1897 .await
1898 .unwrap();
1899
1900 assert_row_id_column(&batches, &[Some(5), Some(101), Some(8)]);
1902 }
1903
1904 #[tokio::test]
1905 async fn test_row_id_only_synthesis_reads_no_data_columns() {
1906 let tmp_dir = TempDir::new().unwrap();
1911 let dir = tmp_dir.path().to_str().unwrap();
1912
1913 let meta_only = metadata_projection_task_with_first_row_id(
1914 write_parquet_with_wide_column(dir, "row_id_only.parquet", vec![], vec![]),
1915 id_and_wide_schema(),
1916 vec![RESERVED_FIELD_ID_ROW_ID],
1917 Some(100),
1918 );
1919 let (batches, meta_only_bytes) = scan_task(meta_only).await;
1920
1921 assert_eq!(batches[0].num_columns(), 1);
1922 assert_row_id_column(&batches, &[Some(100), Some(101), Some(102)]);
1923
1924 let with_data = metadata_projection_task_with_first_row_id(
1926 write_parquet_with_wide_column(dir, "row_id_only_ref.parquet", vec![], vec![]),
1927 id_and_wide_schema(),
1928 vec![2, RESERVED_FIELD_ID_ROW_ID],
1929 Some(100),
1930 );
1931 let (_, with_data_bytes) = scan_task(with_data).await;
1932
1933 assert!(
1934 meta_only_bytes < with_data_bytes,
1935 "_row_id-only synthesis should read fewer bytes than a scan of the wide column: \
1936 {meta_only_bytes} vs {with_data_bytes}"
1937 );
1938 }
1939
1940 #[tokio::test]
1941 async fn test_row_id_only_null_first_row_id_reads_no_data_columns() {
1942 let tmp_dir = TempDir::new().unwrap();
1947 let dir = tmp_dir.path().to_str().unwrap();
1948
1949 let meta_only = metadata_projection_task_with_first_row_id(
1950 write_parquet_with_wide_column(dir, "row_id_null.parquet", vec![], vec![]),
1951 id_and_wide_schema(),
1952 vec![RESERVED_FIELD_ID_ROW_ID],
1953 None,
1954 );
1955 let (batches, meta_only_bytes) = scan_task(meta_only).await;
1956
1957 assert_eq!(batches[0].num_columns(), 1);
1958 assert_row_id_column(&batches, &[None, None, None]);
1959
1960 let with_data = metadata_projection_task_with_first_row_id(
1961 write_parquet_with_wide_column(dir, "row_id_null_ref.parquet", vec![], vec![]),
1962 id_and_wide_schema(),
1963 vec![2, RESERVED_FIELD_ID_ROW_ID],
1964 None,
1965 );
1966 let (_, with_data_bytes) = scan_task(with_data).await;
1967
1968 assert!(
1969 meta_only_bytes < with_data_bytes,
1970 "_row_id-only scan with a null first_row_id should read fewer bytes than a scan \
1971 of the wide column: {meta_only_bytes} vs {with_data_bytes}"
1972 );
1973 }
1974
1975 #[tokio::test]
1976 async fn test_last_updated_seq_only_reads_no_data_columns() {
1977 use crate::metadata_columns::RESERVED_FIELD_ID_LAST_UPDATED_SEQUENCE_NUMBER;
1978
1979 let tmp_dir = TempDir::new().unwrap();
1984 let dir = tmp_dir.path().to_str().unwrap();
1985
1986 let lusn_only_task = |file_path: String, project_field_ids: Vec<i32>| {
1987 FileScanTask::builder()
1988 .with_file_size_in_bytes(std::fs::metadata(&file_path).unwrap().len())
1989 .with_start(0)
1990 .with_length(0)
1991 .with_data_file_path(file_path)
1992 .with_data_file_format(DataFileFormat::Parquet)
1993 .with_schema(id_and_wide_schema())
1994 .with_project_field_ids(project_field_ids)
1995 .with_first_row_id(Some(42))
1996 .with_data_sequence_number(Some(7))
1997 .with_case_sensitive(false)
1998 .build()
1999 .unwrap()
2000 };
2001
2002 let meta_only = lusn_only_task(
2003 write_parquet_with_wide_column(dir, "lusn_only.parquet", vec![], vec![]),
2004 vec![RESERVED_FIELD_ID_LAST_UPDATED_SEQUENCE_NUMBER],
2005 );
2006 let (batches, meta_only_bytes) = scan_task(meta_only).await;
2007
2008 assert_eq!(batches[0].num_columns(), 1);
2009 assert_last_updated_seq_column(&batches, &[Some(7), Some(7), Some(7)]);
2010
2011 let with_data = lusn_only_task(
2012 write_parquet_with_wide_column(dir, "lusn_only_ref.parquet", vec![], vec![]),
2013 vec![2, RESERVED_FIELD_ID_LAST_UPDATED_SEQUENCE_NUMBER],
2014 );
2015 let (_, with_data_bytes) = scan_task(with_data).await;
2016
2017 assert!(
2018 meta_only_bytes < with_data_bytes,
2019 "_last_updated_sequence_number-only scan should read fewer bytes than a scan of \
2020 the wide column: {meta_only_bytes} vs {with_data_bytes}"
2021 );
2022 }
2023
2024 #[tokio::test]
2025 async fn test_partition_only_reads_no_data_columns() {
2026 use arrow_array::StructArray;
2027
2028 use crate::metadata_columns::{RESERVED_COL_NAME_PARTITION, RESERVED_FIELD_ID_PARTITION};
2029 use crate::spec::{Literal, PartitionSpec, Struct, Transform};
2030
2031 let tmp_dir = TempDir::new().unwrap();
2035 let dir = tmp_dir.path().to_str().unwrap();
2036 let schema = id_and_wide_schema();
2037 let spec = Arc::new(
2038 PartitionSpec::builder(schema.clone())
2039 .with_spec_id(7)
2040 .add_partition_field("id", "id", Transform::Identity)
2041 .unwrap()
2042 .build()
2043 .unwrap(),
2044 );
2045 let unified_type = Arc::new(spec.partition_type(&schema).unwrap());
2046 let partition_data = Struct::from_iter(vec![Some(Literal::int(42))]);
2047
2048 let partition_task = |file_path: String, project_field_ids: Vec<i32>| {
2049 FileScanTask::builder()
2050 .with_file_size_in_bytes(std::fs::metadata(&file_path).unwrap().len())
2051 .with_start(0)
2052 .with_length(0)
2053 .with_data_file_path(file_path)
2054 .with_data_file_format(DataFileFormat::Parquet)
2055 .with_schema(schema.clone())
2056 .with_project_field_ids(project_field_ids)
2057 .with_partition_spec(Some(spec.clone()))
2058 .with_partition(Some(partition_data.clone()))
2059 .with_unified_partition_type(Some(unified_type.clone()))
2060 .with_case_sensitive(false)
2061 .build()
2062 .unwrap()
2063 };
2064
2065 let meta_only = partition_task(
2066 write_parquet_with_wide_column(dir, "partition_only.parquet", vec![], vec![]),
2067 vec![RESERVED_FIELD_ID_PARTITION],
2068 );
2069 let (batches, meta_only_bytes) = scan_task(meta_only).await;
2070
2071 assert_eq!(batches[0].num_columns(), 1);
2072 let partition_col = batches[0]
2073 .column_by_name(RESERVED_COL_NAME_PARTITION)
2074 .expect("_partition column should be present")
2075 .as_any()
2076 .downcast_ref::<StructArray>()
2077 .unwrap();
2078 assert_eq!(partition_col.len(), 3);
2079 let inner = partition_col
2080 .column(0)
2081 .as_any()
2082 .downcast_ref::<Int32Array>()
2083 .unwrap();
2084 assert_eq!(inner.values(), &[42, 42, 42]);
2085
2086 let with_data = partition_task(
2087 write_parquet_with_wide_column(dir, "partition_only_ref.parquet", vec![], vec![]),
2088 vec![2, RESERVED_FIELD_ID_PARTITION],
2089 );
2090 let (_, with_data_bytes) = scan_task(with_data).await;
2091
2092 assert!(
2093 meta_only_bytes < with_data_bytes,
2094 "_partition-only scan should read fewer bytes than a scan of the wide column: \
2095 {meta_only_bytes} vs {with_data_bytes}"
2096 );
2097 }
2098
2099 #[tokio::test]
2100 async fn test_row_id_resolves_alongside_id_less_leaf() {
2101 let tmp_dir = TempDir::new().unwrap();
2106 let dir = tmp_dir.path().to_str().unwrap();
2107 let idless_field = Field::new("variant_internal", DataType::Utf8, true);
2108 let idless_col = Arc::new(StringArray::from(vec!["a", "b", "c"])) as ArrayRef;
2109 let row_id_col = Arc::new(Int64Array::from(vec![Some(5), None, Some(8)])) as ArrayRef;
2110 let file_path = write_plain_parquet(
2111 dir,
2112 "row_id_with_idless_leaf.parquet",
2113 vec![idless_field, physical_row_id_field()],
2114 vec![idless_col, row_id_col],
2115 );
2116
2117 let task = row_id_task(file_path, Some(100));
2118
2119 let reader = ArrowReaderBuilder::new(FileIO::new_with_fs(), Runtime::current()).build();
2120 let tasks = Box::pin(futures::stream::iter(vec![Ok(task)])) as FileScanTaskStream;
2121 let batches: Vec<RecordBatch> = reader
2122 .read(tasks)
2123 .unwrap()
2124 .stream()
2125 .try_collect()
2126 .await
2127 .unwrap();
2128
2129 assert_row_id_column(&batches, &[Some(5), Some(101), Some(8)]);
2131 }
2132
2133 #[tokio::test]
2134 async fn test_row_id_and_last_updated_seq_co_projected() {
2135 use crate::metadata_columns::RESERVED_FIELD_ID_LAST_UPDATED_SEQUENCE_NUMBER;
2136
2137 let tmp_dir = TempDir::new().unwrap();
2141 let dir = tmp_dir.path().to_str().unwrap();
2142 let row_id_col = Arc::new(Int64Array::from(vec![Some(5), None, Some(8)])) as ArrayRef;
2143 let seq_col = Arc::new(Int64Array::from(vec![Some(50), None, Some(70)])) as ArrayRef;
2144 let file_path = write_plain_parquet(
2145 dir,
2146 "row_id_and_seq.parquet",
2147 vec![physical_row_id_field(), physical_last_updated_seq_field()],
2148 vec![row_id_col, seq_col],
2149 );
2150
2151 let schema = Arc::new(
2152 Schema::builder()
2153 .with_schema_id(1)
2154 .with_fields(vec![
2155 NestedField::required(1, "id", Type::Primitive(PrimitiveType::Int)).into(),
2156 ])
2157 .build()
2158 .unwrap(),
2159 );
2160 let task = FileScanTask::builder()
2161 .with_file_size_in_bytes(std::fs::metadata(&file_path).unwrap().len())
2162 .with_start(0)
2163 .with_length(0)
2164 .with_data_file_path(file_path)
2165 .with_data_file_format(DataFileFormat::Parquet)
2166 .with_schema(schema)
2167 .with_project_field_ids(vec![
2168 1,
2169 RESERVED_FIELD_ID_ROW_ID,
2170 RESERVED_FIELD_ID_LAST_UPDATED_SEQUENCE_NUMBER,
2171 ])
2172 .with_first_row_id(Some(100))
2173 .with_data_sequence_number(Some(9))
2174 .with_case_sensitive(false)
2175 .build()
2176 .unwrap();
2177
2178 let reader = ArrowReaderBuilder::new(FileIO::new_with_fs(), Runtime::current()).build();
2179 let tasks = Box::pin(futures::stream::iter(vec![Ok(task)])) as FileScanTaskStream;
2180 let batches: Vec<RecordBatch> = reader
2181 .read(tasks)
2182 .unwrap()
2183 .stream()
2184 .try_collect()
2185 .await
2186 .unwrap();
2187
2188 assert_row_id_column(&batches, &[Some(5), Some(101), Some(8)]);
2190 assert_last_updated_seq_column(&batches, &[Some(50), Some(9), Some(70)]);
2192 }
2193
2194 #[tokio::test]
2195 async fn test_row_id_null_when_no_first_row_id() {
2196 let tmp_dir = TempDir::new().unwrap();
2197 let dir = tmp_dir.path().to_str().unwrap();
2198 let id_col = Arc::new(Int64Array::from(vec![Some(5), Some(6), Some(7)])) as ArrayRef;
2200 let file_path = write_plain_parquet(
2201 dir,
2202 "row_id_no_first.parquet",
2203 vec![physical_row_id_field()],
2204 vec![id_col],
2205 );
2206
2207 let task = row_id_task(file_path, None);
2209
2210 let reader = ArrowReaderBuilder::new(FileIO::new_with_fs(), Runtime::current()).build();
2211 let tasks = Box::pin(futures::stream::iter(vec![Ok(task)])) as FileScanTaskStream;
2212 let batches: Vec<RecordBatch> = reader
2213 .read(tasks)
2214 .unwrap()
2215 .stream()
2216 .try_collect()
2217 .await
2218 .unwrap();
2219
2220 assert_row_id_column(&batches, &[None, None, None]);
2221 }
2222
2223 #[tokio::test]
2224 async fn test_row_id_with_pos_column() {
2225 use crate::metadata_columns::RESERVED_COL_NAME_POS;
2226
2227 let tmp_dir = TempDir::new().unwrap();
2228 let dir = tmp_dir.path().to_str().unwrap();
2229 let id_col = Arc::new(Int64Array::from(vec![Some(5), None, Some(8)])) as ArrayRef;
2230 let file_path = write_plain_parquet(
2231 dir,
2232 "row_id_and_pos.parquet",
2233 vec![physical_row_id_field()],
2234 vec![id_col],
2235 );
2236
2237 let schema = Arc::new(
2241 Schema::builder()
2242 .with_schema_id(1)
2243 .with_fields(vec![
2244 NestedField::required(1, "id", Type::Primitive(PrimitiveType::Int)).into(),
2245 ])
2246 .build()
2247 .unwrap(),
2248 );
2249 let task = FileScanTask::builder()
2250 .with_file_size_in_bytes(std::fs::metadata(&file_path).unwrap().len())
2251 .with_start(0)
2252 .with_length(0)
2253 .with_data_file_path(file_path)
2254 .with_data_file_format(DataFileFormat::Parquet)
2255 .with_schema(schema)
2256 .with_project_field_ids(vec![1, RESERVED_FIELD_ID_POS, RESERVED_FIELD_ID_ROW_ID])
2257 .with_first_row_id(Some(100))
2258 .with_case_sensitive(false)
2259 .build()
2260 .unwrap();
2261
2262 let reader = ArrowReaderBuilder::new(FileIO::new_with_fs(), Runtime::current()).build();
2263 let tasks = Box::pin(futures::stream::iter(vec![Ok(task)])) as FileScanTaskStream;
2264 let batches: Vec<RecordBatch> = reader
2265 .read(tasks)
2266 .unwrap()
2267 .stream()
2268 .try_collect()
2269 .await
2270 .unwrap();
2271
2272 assert_row_id_column(&batches, &[Some(5), Some(101), Some(8)]);
2274 let pos_col = batches[0]
2276 .column_by_name(RESERVED_COL_NAME_POS)
2277 .expect("_pos column should be present")
2278 .as_primitive::<arrow_array::types::Int64Type>();
2279 assert_eq!(pos_col.values(), &[0, 1, 2]);
2280 }
2281
2282 #[tokio::test]
2283 async fn test_row_id_mixed_files_share_schema() {
2284 use arrow_select::concat::concat_batches;
2285
2286 let tmp_dir = TempDir::new().unwrap();
2287 let dir = tmp_dir.path().to_str().unwrap();
2288
2289 let synth = row_id_task(
2295 write_plain_parquet(dir, "row_id_synth2.parquet", vec![], vec![]),
2296 Some(42),
2297 );
2298 let nulled = row_id_task(
2299 write_plain_parquet(dir, "row_id_null2.parquet", vec![], vec![]),
2300 None,
2301 );
2302 let coalesced = row_id_task(
2303 write_plain_parquet(
2304 dir,
2305 "row_id_coalesced2.parquet",
2306 vec![physical_row_id_field()],
2307 vec![Arc::new(Int64Array::from(vec![Some(5), None, Some(8)])) as ArrayRef],
2308 ),
2309 Some(50),
2310 );
2311
2312 let reader = ArrowReaderBuilder::new(FileIO::new_with_fs(), Runtime::current()).build();
2313 let tasks = Box::pin(futures::stream::iter(vec![
2314 Ok(synth),
2315 Ok(nulled),
2316 Ok(coalesced),
2317 ])) as FileScanTaskStream;
2318 let batches: Vec<RecordBatch> = reader
2319 .read(tasks)
2320 .unwrap()
2321 .stream()
2322 .try_collect()
2323 .await
2324 .unwrap();
2325
2326 assert_eq!(batches.len(), 3);
2327 let schema = batches[0].schema();
2328 concat_batches(&schema, &batches)
2329 .expect("synthesis, null and coalesce files must share one column type");
2330 }
2331
2332 #[tokio::test]
2333 async fn test_row_id_present_by_name_without_id_unsupported() {
2334 let tmp_dir = TempDir::new().unwrap();
2335 let dir = tmp_dir.path().to_str().unwrap();
2336 let id_field = Field::new(RESERVED_COL_NAME_ROW_ID, DataType::Int64, true);
2339 let id_col = Arc::new(Int64Array::from(vec![Some(5), None, Some(8)])) as ArrayRef;
2340 let file_path =
2341 write_plain_parquet(dir, "row_id_by_name.parquet", vec![id_field], vec![id_col]);
2342
2343 let task = row_id_task(file_path, Some(100));
2344
2345 let reader = ArrowReaderBuilder::new(FileIO::new_with_fs(), Runtime::current()).build();
2346 let tasks = Box::pin(futures::stream::iter(vec![Ok(task)])) as FileScanTaskStream;
2347 let result: Result<Vec<RecordBatch>, _> =
2348 reader.read(tasks).unwrap().stream().try_collect().await;
2349
2350 let err = result.unwrap_err();
2351 assert_eq!(err.kind(), crate::ErrorKind::FeatureUnsupported);
2352 assert!(
2353 format!("{err}").contains("without an embedded field id"),
2354 "unexpected error: {err}"
2355 );
2356 }
2357
2358 #[tokio::test]
2359 async fn test_row_id_present_by_name_without_id_nulls_when_no_lineage() {
2360 let tmp_dir = TempDir::new().unwrap();
2361 let dir = tmp_dir.path().to_str().unwrap();
2362 let id_field = Field::new(RESERVED_COL_NAME_ROW_ID, DataType::Int64, true);
2367 let id_col = Arc::new(Int64Array::from(vec![Some(5), None, Some(8)])) as ArrayRef;
2368 let file_path = write_plain_parquet(
2369 dir,
2370 "row_id_by_name_no_lineage.parquet",
2371 vec![id_field],
2372 vec![id_col],
2373 );
2374
2375 let task = row_id_task(file_path, None);
2376
2377 let reader = ArrowReaderBuilder::new(FileIO::new_with_fs(), Runtime::current()).build();
2378 let tasks = Box::pin(futures::stream::iter(vec![Ok(task)])) as FileScanTaskStream;
2379 let batches: Vec<RecordBatch> = reader
2380 .read(tasks)
2381 .unwrap()
2382 .stream()
2383 .try_collect()
2384 .await
2385 .unwrap();
2386
2387 assert_row_id_column(&batches, &[None, None, None]);
2388 }
2389
2390 fn row_id_task_with_predicate(
2393 file_path: String,
2394 first_row_id: Option<i64>,
2395 extra_project_field_ids: Vec<i32>,
2396 predicate: crate::expr::Predicate,
2397 ) -> FileScanTask {
2398 use crate::expr::Bind;
2399
2400 let schema = Arc::new(
2401 Schema::builder()
2402 .with_schema_id(1)
2403 .with_fields(vec![
2404 NestedField::required(1, "id", Type::Primitive(PrimitiveType::Int)).into(),
2405 ])
2406 .build()
2407 .unwrap(),
2408 );
2409 let bound = predicate.bind(Arc::clone(&schema), false).unwrap();
2410
2411 let mut project_field_ids = vec![1];
2412 project_field_ids.extend(extra_project_field_ids);
2413 project_field_ids.push(RESERVED_FIELD_ID_ROW_ID);
2414
2415 FileScanTask::builder()
2416 .with_file_size_in_bytes(std::fs::metadata(&file_path).unwrap().len())
2417 .with_start(0)
2418 .with_length(0)
2419 .with_data_file_path(file_path)
2420 .with_data_file_format(DataFileFormat::Parquet)
2421 .with_schema(schema)
2422 .with_project_field_ids(project_field_ids)
2423 .with_predicate(Some(bound))
2424 .with_first_row_id(first_row_id)
2425 .with_case_sensitive(false)
2426 .build()
2427 .unwrap()
2428 }
2429
2430 #[tokio::test]
2431 async fn test_row_id_stable_under_row_selection() {
2432 use crate::expr::Reference;
2433 use crate::spec::Datum;
2434
2435 let tmp_dir = TempDir::new().unwrap();
2436 let dir = tmp_dir.path().to_str().unwrap();
2437 let file_path = write_plain_parquet(dir, "row_id_selection.parquet", vec![], vec![]);
2439
2440 let task = row_id_task_with_predicate(
2441 file_path,
2442 Some(100),
2443 vec![],
2444 Reference::new("id").not_equal_to(Datum::int(2)),
2445 );
2446
2447 let reader = ArrowReaderBuilder::new(FileIO::new_with_fs(), Runtime::current())
2449 .with_options(
2450 ReaderOptions::builder()
2451 .with_row_selection_enabled(true)
2452 .build(),
2453 )
2454 .build();
2455 let tasks = Box::pin(futures::stream::iter(vec![Ok(task)])) as FileScanTaskStream;
2456 let batches: Vec<RecordBatch> = reader
2457 .read(tasks)
2458 .unwrap()
2459 .stream()
2460 .try_collect()
2461 .await
2462 .unwrap();
2463
2464 assert_row_id_column(&batches, &[Some(100), Some(102)]);
2467 }
2468
2469 #[tokio::test]
2470 async fn test_row_id_coalesce_stable_under_row_selection() {
2471 use crate::expr::Reference;
2472 use crate::spec::Datum;
2473
2474 let tmp_dir = TempDir::new().unwrap();
2475 let dir = tmp_dir.path().to_str().unwrap();
2476 let id_col = Arc::new(Int64Array::from(vec![Some(5), None, Some(8)])) as ArrayRef;
2479 let file_path = write_plain_parquet(
2480 dir,
2481 "row_id_coalesce_selection.parquet",
2482 vec![physical_row_id_field()],
2483 vec![id_col],
2484 );
2485
2486 let task = row_id_task_with_predicate(
2487 file_path,
2488 Some(100),
2489 vec![],
2490 Reference::new("id").not_equal_to(Datum::int(2)),
2491 );
2492
2493 let reader = ArrowReaderBuilder::new(FileIO::new_with_fs(), Runtime::current())
2494 .with_options(
2495 ReaderOptions::builder()
2496 .with_row_selection_enabled(true)
2497 .build(),
2498 )
2499 .build();
2500 let tasks = Box::pin(futures::stream::iter(vec![Ok(task)])) as FileScanTaskStream;
2501 let batches: Vec<RecordBatch> = reader
2502 .read(tasks)
2503 .unwrap()
2504 .stream()
2505 .try_collect()
2506 .await
2507 .unwrap();
2508
2509 assert_row_id_column(&batches, &[Some(5), Some(8)]);
2513 }
2514
2515 #[tokio::test]
2516 async fn test_row_id_global_across_row_groups() {
2517 let tmp_dir = TempDir::new().unwrap();
2518 let dir = tmp_dir.path().to_str().unwrap();
2519
2520 let file_path = format!("{dir}/row_id_multi_rg.parquet");
2524 let field = Field::new("id", DataType::Int32, false).with_metadata(HashMap::from([(
2525 PARQUET_FIELD_ID_META_KEY.to_string(),
2526 "1".to_string(),
2527 )]));
2528 let arrow_schema = Arc::new(ArrowSchema::new(vec![field]));
2529 let batch = RecordBatch::try_new(arrow_schema.clone(), vec![Arc::new(Int32Array::from(
2530 vec![1, 2, 3, 4, 5],
2531 ))])
2532 .unwrap();
2533 let props = WriterProperties::builder()
2534 .set_compression(Compression::SNAPPY)
2535 .set_max_row_group_row_count(Some(2))
2536 .build();
2537 let file = File::create(&file_path).unwrap();
2538 let mut writer = ArrowWriter::try_new(file, arrow_schema, Some(props)).unwrap();
2539 writer.write(&batch).unwrap();
2540 writer.close().unwrap();
2541
2542 let task = row_id_task(file_path, Some(0));
2543 let reader = ArrowReaderBuilder::new(FileIO::new_with_fs(), Runtime::current()).build();
2544 let tasks = Box::pin(futures::stream::iter(vec![Ok(task)])) as FileScanTaskStream;
2545 let batches: Vec<RecordBatch> = reader
2546 .read(tasks)
2547 .unwrap()
2548 .stream()
2549 .try_collect()
2550 .await
2551 .unwrap();
2552
2553 assert_row_id_column(&batches, &[Some(0), Some(1), Some(2), Some(3), Some(4)]);
2554 }
2555
2556 #[tokio::test]
2557 async fn test_row_id_global_when_first_row_group_pruned() {
2558 use parquet::file::reader::{FileReader, SerializedFileReader};
2559
2560 let tmp_dir = TempDir::new().unwrap();
2561 let dir = tmp_dir.path().to_str().unwrap();
2562
2563 let file_path = format!("{dir}/row_id_prune_rg0.parquet");
2568 let field = Field::new("id", DataType::Int32, false).with_metadata(HashMap::from([(
2569 PARQUET_FIELD_ID_META_KEY.to_string(),
2570 "1".to_string(),
2571 )]));
2572 let arrow_schema = Arc::new(ArrowSchema::new(vec![field]));
2573 let batch = RecordBatch::try_new(arrow_schema.clone(), vec![Arc::new(Int32Array::from(
2574 vec![1, 2, 3, 4, 5, 6],
2575 ))])
2576 .unwrap();
2577 let props = WriterProperties::builder()
2578 .set_compression(Compression::SNAPPY)
2579 .set_max_row_group_row_count(Some(2))
2580 .build();
2581 let file = File::create(&file_path).unwrap();
2582 let mut writer = ArrowWriter::try_new(file, arrow_schema, Some(props)).unwrap();
2583 writer.write(&batch).unwrap();
2584 writer.close().unwrap();
2585
2586 let metadata = SerializedFileReader::new(File::open(&file_path).unwrap())
2589 .unwrap()
2590 .metadata()
2591 .clone();
2592 assert_eq!(metadata.num_row_groups(), 3);
2593 let start = 4 + metadata.row_group(0).compressed_size() as u64;
2594 let file_size = std::fs::metadata(&file_path).unwrap().len();
2595
2596 let task = row_id_task_with_options(file_path, Some(100), start, file_size - start, vec![]);
2597
2598 let reader = ArrowReaderBuilder::new(FileIO::new_with_fs(), Runtime::current()).build();
2599 let tasks = Box::pin(futures::stream::iter(vec![Ok(task)])) as FileScanTaskStream;
2600 let batches: Vec<RecordBatch> = reader
2601 .read(tasks)
2602 .unwrap()
2603 .stream()
2604 .try_collect()
2605 .await
2606 .unwrap();
2607
2608 assert_row_id_column(&batches, &[Some(102), Some(103), Some(104), Some(105)]);
2611 }
2612
2613 fn write_positional_delete(
2616 dir: &str,
2617 name: &str,
2618 data_file_path: &str,
2619 positions: &[i64],
2620 ) -> String {
2621 use arrow_array::StringArray;
2622
2623 let file_path_field =
2624 Field::new("file_path", DataType::Utf8, false).with_metadata(HashMap::from([(
2625 PARQUET_FIELD_ID_META_KEY.to_string(),
2626 "2147483546".to_string(),
2627 )]));
2628 let pos_field = Field::new("pos", DataType::Int64, false).with_metadata(HashMap::from([(
2629 PARQUET_FIELD_ID_META_KEY.to_string(),
2630 "2147483545".to_string(),
2631 )]));
2632 let arrow_schema = Arc::new(ArrowSchema::new(vec![file_path_field, pos_field]));
2633 let batch = RecordBatch::try_new(arrow_schema.clone(), vec![
2634 Arc::new(StringArray::from(vec![data_file_path; positions.len()])),
2635 Arc::new(Int64Array::from(positions.to_vec())),
2636 ])
2637 .unwrap();
2638
2639 let path = format!("{dir}/{name}");
2640 let file = File::create(&path).unwrap();
2641 let props = WriterProperties::builder()
2642 .set_compression(Compression::SNAPPY)
2643 .build();
2644 let mut writer = ArrowWriter::try_new(file, arrow_schema, Some(props)).unwrap();
2645 writer.write(&batch).unwrap();
2646 writer.close().unwrap();
2647 path
2648 }
2649
2650 #[tokio::test]
2651 async fn test_row_id_survives_positional_delete() {
2652 use crate::spec::DataContentType;
2653
2654 let tmp_dir = TempDir::new().unwrap();
2655 let dir = tmp_dir.path().to_str().unwrap();
2656 let data_path = write_plain_parquet(dir, "row_id_posdel_data.parquet", vec![], vec![]);
2658 let del_path = write_positional_delete(dir, "row_id_posdel.parquet", &data_path, &[1]);
2659
2660 let delete = FileScanTaskDeleteFile::builder()
2661 .with_file_path(del_path.clone())
2662 .with_file_size_in_bytes(std::fs::metadata(&del_path).unwrap().len())
2663 .with_file_type(DataContentType::PositionDeletes)
2664 .with_file_format(DataFileFormat::Parquet)
2665 .with_partition_spec_id(0)
2666 .build()
2667 .unwrap();
2668 let task = row_id_task_with_options(data_path, Some(100), 0, 0, vec![delete]);
2669
2670 let reader = ArrowReaderBuilder::new(FileIO::new_with_fs(), Runtime::current()).build();
2671 let tasks = Box::pin(futures::stream::iter(vec![Ok(task)])) as FileScanTaskStream;
2672 let batches: Vec<RecordBatch> = reader
2673 .read(tasks)
2674 .unwrap()
2675 .stream()
2676 .try_collect()
2677 .await
2678 .unwrap();
2679
2680 assert_row_id_column(&batches, &[Some(100), Some(102)]);
2684 }
2685
2686 #[tokio::test]
2687 async fn test_row_id_name_collision_under_positional_fallback() {
2688 let tmp_dir = TempDir::new().unwrap();
2689 let dir = tmp_dir.path().to_str().unwrap();
2690
2691 let file_path = format!("{dir}/fallback_row_id_name.parquet");
2696 let arrow_schema = Arc::new(ArrowSchema::new(vec![
2697 Field::new("id", DataType::Int32, false),
2698 Field::new(RESERVED_COL_NAME_ROW_ID, DataType::Int64, true),
2699 ]));
2700 let batch = RecordBatch::try_new(arrow_schema.clone(), vec![
2701 Arc::new(Int32Array::from(vec![1, 2, 3])),
2702 Arc::new(Int64Array::from(vec![7i64, 8, 9])),
2703 ])
2704 .unwrap();
2705 let props = WriterProperties::builder()
2706 .set_compression(Compression::SNAPPY)
2707 .build();
2708 let file = File::create(&file_path).unwrap();
2709 let mut writer = ArrowWriter::try_new(file, arrow_schema, Some(props)).unwrap();
2710 writer.write(&batch).unwrap();
2711 writer.close().unwrap();
2712
2713 let task = row_id_task(file_path, Some(100));
2714 let reader = ArrowReaderBuilder::new(FileIO::new_with_fs(), Runtime::current()).build();
2715 let tasks = Box::pin(futures::stream::iter(vec![Ok(task)])) as FileScanTaskStream;
2716 let batches: Vec<RecordBatch> = reader
2717 .read(tasks)
2718 .unwrap()
2719 .stream()
2720 .try_collect()
2721 .await
2722 .unwrap();
2723
2724 assert_row_id_column(&batches, &[Some(100), Some(101), Some(102)]);
2726 }
2727
2728 #[tokio::test]
2729 async fn test_read_encrypted_parquet_with_wrong_key_fails() {
2730 let encryption_key = b"0123456789abcdef";
2731 let wrong_key = b"fedcba9876543210";
2732
2733 let schema = Arc::new(
2734 Schema::builder()
2735 .with_schema_id(1)
2736 .with_fields(vec![
2737 NestedField::required(1, "id", Type::Primitive(PrimitiveType::Int)).into(),
2738 ])
2739 .build()
2740 .unwrap(),
2741 );
2742
2743 let arrow_schema = Arc::new(ArrowSchema::new(vec![
2744 Field::new("id", DataType::Int32, false).with_metadata(HashMap::from([(
2745 PARQUET_FIELD_ID_META_KEY.to_string(),
2746 "1".to_string(),
2747 )])),
2748 ]));
2749
2750 let tmp_dir = TempDir::new().unwrap();
2751 let table_location = tmp_dir.path().to_str().unwrap().to_string();
2752 let file_io = FileIO::new_with_fs();
2753
2754 let id_data = Arc::new(Int32Array::from(vec![1, 2, 3])) as ArrayRef;
2755 let batch = RecordBatch::try_new(arrow_schema.clone(), vec![id_data]).unwrap();
2756
2757 let file_path = format!("{table_location}/encrypted_wrong_key.parquet");
2758 write_encrypted_parquet(&file_path, &batch, encryption_key, None);
2759
2760 let wrong_key_metadata = crate::encryption::StandardKeyMetadata::try_new(wrong_key)
2761 .unwrap()
2762 .encode()
2763 .unwrap();
2764
2765 let reader = ArrowReaderBuilder::new(file_io, Runtime::current()).build();
2766
2767 let task = FileScanTask::builder()
2768 .with_file_size_in_bytes(std::fs::metadata(&file_path).unwrap().len())
2769 .with_start(0)
2770 .with_length(0)
2771 .with_data_file_path(file_path)
2772 .with_data_file_format(DataFileFormat::Parquet)
2773 .with_schema(schema)
2774 .with_project_field_ids(vec![1])
2775 .with_case_sensitive(false)
2776 .with_key_metadata(Some(wrong_key_metadata))
2777 .build()
2778 .unwrap();
2779
2780 let tasks = Box::pin(futures::stream::iter(vec![Ok(task)])) as FileScanTaskStream;
2781 let result: Result<Vec<RecordBatch>, _> =
2782 reader.read(tasks).unwrap().stream().try_collect().await;
2783
2784 let err = result.unwrap_err();
2785 assert_eq!(err.kind(), crate::ErrorKind::Unexpected);
2786 let err_str = format!("{err}");
2787 assert!(
2788 err_str.contains("unable to decrypt parquet footer"),
2789 "Expected error about decryption failure, got: {err_str}"
2790 );
2791 }
2792
2793 #[tokio::test]
2796 async fn test_read_with_concurrency_one() {
2797 use arrow_array::Int32Array;
2798
2799 let schema = Arc::new(
2800 Schema::builder()
2801 .with_schema_id(1)
2802 .with_fields(vec![
2803 NestedField::required(1, "id", Type::Primitive(PrimitiveType::Int)).into(),
2804 NestedField::required(2, "file_num", Type::Primitive(PrimitiveType::Int))
2805 .into(),
2806 ])
2807 .build()
2808 .unwrap(),
2809 );
2810
2811 let arrow_schema = Arc::new(ArrowSchema::new(vec![
2812 Field::new("id", DataType::Int32, false).with_metadata(HashMap::from([(
2813 PARQUET_FIELD_ID_META_KEY.to_string(),
2814 "1".to_string(),
2815 )])),
2816 Field::new("file_num", DataType::Int32, false).with_metadata(HashMap::from([(
2817 PARQUET_FIELD_ID_META_KEY.to_string(),
2818 "2".to_string(),
2819 )])),
2820 ]));
2821
2822 let tmp_dir = TempDir::new().unwrap();
2823 let table_location = tmp_dir.path().to_str().unwrap().to_string();
2824 let file_io = FileIO::new_with_fs();
2825
2826 let props = WriterProperties::builder()
2828 .set_compression(Compression::SNAPPY)
2829 .build();
2830
2831 for file_num in 0..3 {
2832 let id_data = Arc::new(Int32Array::from_iter_values(
2833 file_num * 10..(file_num + 1) * 10,
2834 )) as ArrayRef;
2835 let file_num_data = Arc::new(Int32Array::from(vec![file_num; 10])) as ArrayRef;
2836
2837 let to_write =
2838 RecordBatch::try_new(arrow_schema.clone(), vec![id_data, file_num_data]).unwrap();
2839
2840 let file = File::create(format!("{table_location}/file_{file_num}.parquet")).unwrap();
2841 let mut writer =
2842 ArrowWriter::try_new(file, to_write.schema(), Some(props.clone())).unwrap();
2843 writer.write(&to_write).expect("Writing batch");
2844 writer.close().unwrap();
2845 }
2846
2847 let reader = ArrowReaderBuilder::new(file_io, Runtime::current())
2849 .with_options(
2850 ReaderOptions::builder()
2851 .with_data_file_concurrency_limit(1)
2852 .build(),
2853 )
2854 .build();
2855
2856 let tasks = vec![
2858 Ok(FileScanTask::builder()
2859 .with_file_size_in_bytes(
2860 std::fs::metadata(format!("{table_location}/file_0.parquet"))
2861 .unwrap()
2862 .len(),
2863 )
2864 .with_start(0)
2865 .with_length(0)
2866 .with_data_file_path(format!("{table_location}/file_0.parquet"))
2867 .with_data_file_format(DataFileFormat::Parquet)
2868 .with_schema(schema.clone())
2869 .with_project_field_ids(vec![1, 2])
2870 .with_case_sensitive(false)
2871 .build()
2872 .unwrap()),
2873 Ok(FileScanTask::builder()
2874 .with_file_size_in_bytes(
2875 std::fs::metadata(format!("{table_location}/file_1.parquet"))
2876 .unwrap()
2877 .len(),
2878 )
2879 .with_start(0)
2880 .with_length(0)
2881 .with_data_file_path(format!("{table_location}/file_1.parquet"))
2882 .with_data_file_format(DataFileFormat::Parquet)
2883 .with_schema(schema.clone())
2884 .with_project_field_ids(vec![1, 2])
2885 .with_case_sensitive(false)
2886 .build()
2887 .unwrap()),
2888 Ok(FileScanTask::builder()
2889 .with_file_size_in_bytes(
2890 std::fs::metadata(format!("{table_location}/file_2.parquet"))
2891 .unwrap()
2892 .len(),
2893 )
2894 .with_start(0)
2895 .with_length(0)
2896 .with_data_file_path(format!("{table_location}/file_2.parquet"))
2897 .with_data_file_format(DataFileFormat::Parquet)
2898 .with_schema(schema.clone())
2899 .with_project_field_ids(vec![1, 2])
2900 .with_case_sensitive(false)
2901 .build()
2902 .unwrap()),
2903 ];
2904
2905 let tasks_stream = Box::pin(futures::stream::iter(tasks)) as FileScanTaskStream;
2906
2907 let result = reader
2908 .read(tasks_stream)
2909 .unwrap()
2910 .stream()
2911 .try_collect::<Vec<RecordBatch>>()
2912 .await
2913 .unwrap();
2914
2915 let total_rows: usize = result.iter().map(|b| b.num_rows()).sum();
2917 assert_eq!(total_rows, 30, "Should have 30 total rows");
2918
2919 let mut all_ids = Vec::new();
2921 let mut all_file_nums = Vec::new();
2922
2923 for batch in &result {
2924 let id_col = batch
2925 .column(0)
2926 .as_primitive::<arrow_array::types::Int32Type>();
2927 let file_num_col = batch
2928 .column(1)
2929 .as_primitive::<arrow_array::types::Int32Type>();
2930
2931 for i in 0..batch.num_rows() {
2932 all_ids.push(id_col.value(i));
2933 all_file_nums.push(file_num_col.value(i));
2934 }
2935 }
2936
2937 assert_eq!(all_ids.len(), 30);
2938 assert_eq!(all_file_nums.len(), 30);
2939
2940 for i in 0..10 {
2945 assert_eq!(all_file_nums[i], 0, "First 10 rows should be from file_0");
2946 assert_eq!(all_ids[i], i as i32, "IDs should be 0-9");
2947 }
2948 for i in 10..20 {
2949 assert_eq!(all_file_nums[i], 1, "Next 10 rows should be from file_1");
2950 assert_eq!(all_ids[i], i as i32, "IDs should be 10-19");
2951 }
2952 for i in 20..30 {
2953 assert_eq!(all_file_nums[i], 2, "Last 10 rows should be from file_2");
2954 assert_eq!(all_ids[i], i as i32, "IDs should be 20-29");
2955 }
2956 }
2957
2958 #[tokio::test]
2959 async fn test_read_int96_timestamps_with_field_ids() {
2960 let schema = Arc::new(
2961 Schema::builder()
2962 .with_schema_id(1)
2963 .with_fields(vec![
2964 NestedField::optional(1, "ts", Type::Primitive(PrimitiveType::Timestamp))
2965 .into(),
2966 NestedField::required(2, "id", Type::Primitive(PrimitiveType::Int)).into(),
2967 ])
2968 .build()
2969 .unwrap(),
2970 );
2971
2972 let tmp_dir = TempDir::new().unwrap();
2973 let table_location = tmp_dir.path().to_str().unwrap().to_string();
2974 let (file_path, expected_micros) =
2975 write_int96_parquet_file(&table_location, "with_ids.parquet", true);
2976
2977 assert_int96_read_matches(&file_path, schema, vec![1, 2], &expected_micros).await;
2978 }
2979
2980 #[tokio::test]
2981 async fn test_read_int96_timestamps_without_field_ids() {
2982 let schema = Arc::new(
2983 Schema::builder()
2984 .with_schema_id(1)
2985 .with_fields(vec![
2986 NestedField::optional(1, "ts", Type::Primitive(PrimitiveType::Timestamp))
2987 .into(),
2988 NestedField::required(2, "id", Type::Primitive(PrimitiveType::Int)).into(),
2989 ])
2990 .build()
2991 .unwrap(),
2992 );
2993
2994 let tmp_dir = TempDir::new().unwrap();
2995 let table_location = tmp_dir.path().to_str().unwrap().to_string();
2996 let (file_path, expected_micros) =
2997 write_int96_parquet_file(&table_location, "no_ids.parquet", false);
2998
2999 assert_int96_read_matches(&file_path, schema, vec![1, 2], &expected_micros).await;
3000 }
3001
3002 #[tokio::test]
3003 async fn test_read_int96_timestamps_with_fallback_ids_and_pos() {
3004 use arrow_array::TimestampMicrosecondArray;
3005
3006 let schema = Arc::new(
3011 Schema::builder()
3012 .with_schema_id(1)
3013 .with_fields(vec![
3014 NestedField::optional(1, "ts", Type::Primitive(PrimitiveType::Timestamp))
3015 .into(),
3016 NestedField::required(2, "id", Type::Primitive(PrimitiveType::Int)).into(),
3017 ])
3018 .build()
3019 .unwrap(),
3020 );
3021
3022 let tmp_dir = TempDir::new().unwrap();
3023 let table_location = tmp_dir.path().to_str().unwrap().to_string();
3024 let (file_path, expected_micros) =
3025 write_int96_parquet_file(&table_location, "no_ids_with_pos.parquet", false);
3026
3027 let batches =
3028 read_int96_batches(&file_path, schema, vec![1, 2, RESERVED_FIELD_ID_POS]).await;
3029
3030 assert_eq!(batches.len(), 1);
3031 let ts_col = batches[0]
3033 .column_by_name("ts")
3034 .expect("ts column should be present")
3035 .as_any()
3036 .downcast_ref::<TimestampMicrosecondArray>()
3037 .expect("Expected TimestampMicrosecondArray");
3038 for (i, expected) in expected_micros.iter().enumerate() {
3039 assert_eq!(ts_col.value(i), *expected, "Row {i}");
3040 }
3041 let pos_col = batches[0]
3043 .column_by_name(RESERVED_COL_NAME_POS)
3044 .expect("_pos column should be present")
3045 .as_primitive::<arrow_array::types::Int64Type>();
3046 assert_eq!(pos_col.values(), &[0, 1, 2]);
3047 }
3048
3049 #[tokio::test]
3050 async fn test_read_int96_timestamps_with_field_ids_and_pos() {
3051 use arrow_array::TimestampMicrosecondArray;
3052
3053 let schema = Arc::new(
3054 Schema::builder()
3055 .with_schema_id(1)
3056 .with_fields(vec![
3057 NestedField::optional(1, "ts", Type::Primitive(PrimitiveType::Timestamp))
3058 .into(),
3059 NestedField::required(2, "id", Type::Primitive(PrimitiveType::Int)).into(),
3060 ])
3061 .build()
3062 .unwrap(),
3063 );
3064
3065 let tmp_dir = TempDir::new().unwrap();
3066 let table_location = tmp_dir.path().to_str().unwrap().to_string();
3067 let (file_path, expected_micros) =
3068 write_int96_parquet_file(&table_location, "with_ids_with_pos.parquet", true);
3069
3070 let batches =
3071 read_int96_batches(&file_path, schema, vec![1, 2, RESERVED_FIELD_ID_POS]).await;
3072
3073 assert_eq!(batches.len(), 1);
3074 let ts_col = batches[0]
3075 .column_by_name("ts")
3076 .expect("ts column should be present")
3077 .as_any()
3078 .downcast_ref::<TimestampMicrosecondArray>()
3079 .expect("Expected TimestampMicrosecondArray");
3080
3081 for (i, expected) in expected_micros.iter().enumerate() {
3082 assert_eq!(ts_col.value(i), *expected, "Row {i}");
3083 }
3084
3085 let pos_col = batches[0]
3086 .column_by_name(RESERVED_COL_NAME_POS)
3087 .expect("_pos column should be present")
3088 .as_primitive::<arrow_array::types::Int64Type>();
3089
3090 assert_eq!(pos_col.values(), &[0, 1, 2]);
3091 }
3092
3093 #[tokio::test]
3094 async fn test_read_int96_timestamps_in_struct() {
3095 use arrow_array::{StructArray, TimestampMicrosecondArray};
3096 use parquet::basic::{Repetition, Type as PhysicalType};
3097 use parquet::data_type::Int96Type;
3098 use parquet::file::writer::SerializedFileWriter;
3099 use parquet::schema::types::Type as SchemaType;
3100
3101 let tmp_dir = TempDir::new().unwrap();
3102 let table_location = tmp_dir.path().to_str().unwrap().to_string();
3103 let file_path = format!("{table_location}/struct_int96.parquet");
3104
3105 let ts_type = SchemaType::primitive_type_builder("ts", PhysicalType::INT96)
3106 .with_repetition(Repetition::OPTIONAL)
3107 .with_id(Some(2))
3108 .build()
3109 .unwrap();
3110
3111 let struct_type = SchemaType::group_type_builder("data")
3112 .with_repetition(Repetition::REQUIRED)
3113 .with_id(Some(1))
3114 .with_fields(vec![Arc::new(ts_type)])
3115 .build()
3116 .unwrap();
3117
3118 let parquet_schema = SchemaType::group_type_builder("schema")
3119 .with_fields(vec![Arc::new(struct_type)])
3120 .build()
3121 .unwrap();
3122
3123 let (int96_val, expected_micros) = make_int96_test_value();
3124
3125 let file = File::create(&file_path).unwrap();
3126 let mut writer =
3127 SerializedFileWriter::new(file, Arc::new(parquet_schema), Default::default()).unwrap();
3128
3129 let mut row_group = writer.next_row_group().unwrap();
3132 {
3133 let mut col = row_group.next_column().unwrap().unwrap();
3134 col.typed::<Int96Type>()
3135 .write_batch(&[int96_val], Some(&[1]), None)
3136 .unwrap();
3137 col.close().unwrap();
3138 }
3139 row_group.close().unwrap();
3140 writer.close().unwrap();
3141
3142 let iceberg_schema = Arc::new(
3143 Schema::builder()
3144 .with_schema_id(1)
3145 .with_fields(vec![
3146 NestedField::required(
3147 1,
3148 "data",
3149 Type::Struct(crate::spec::StructType::new(vec![
3150 NestedField::optional(
3151 2,
3152 "ts",
3153 Type::Primitive(PrimitiveType::Timestamp),
3154 )
3155 .into(),
3156 ])),
3157 )
3158 .into(),
3159 ])
3160 .build()
3161 .unwrap(),
3162 );
3163
3164 let batches = read_int96_batches(&file_path, iceberg_schema, vec![1]).await;
3165
3166 assert_eq!(batches.len(), 1);
3167 let struct_array = batches[0]
3168 .column(0)
3169 .as_any()
3170 .downcast_ref::<StructArray>()
3171 .expect("Expected StructArray");
3172 let ts_array = struct_array
3173 .column(0)
3174 .as_any()
3175 .downcast_ref::<TimestampMicrosecondArray>()
3176 .expect("Expected TimestampMicrosecondArray inside struct");
3177
3178 assert_eq!(
3179 ts_array.value(0),
3180 expected_micros,
3181 "INT96 in struct: got {}, expected {expected_micros}",
3182 ts_array.value(0)
3183 );
3184 }
3185
3186 #[tokio::test]
3187 async fn test_read_int96_timestamps_in_list() {
3188 use arrow_array::{ListArray, TimestampMicrosecondArray};
3189 use parquet::basic::{Repetition, Type as PhysicalType};
3190 use parquet::data_type::Int96Type;
3191 use parquet::file::writer::SerializedFileWriter;
3192 use parquet::schema::types::Type as SchemaType;
3193
3194 let tmp_dir = TempDir::new().unwrap();
3195 let table_location = tmp_dir.path().to_str().unwrap().to_string();
3196 let file_path = format!("{table_location}/list_int96.parquet");
3197
3198 let element_type = SchemaType::primitive_type_builder("element", PhysicalType::INT96)
3205 .with_repetition(Repetition::OPTIONAL)
3206 .with_id(Some(2))
3207 .build()
3208 .unwrap();
3209
3210 let list_group = SchemaType::group_type_builder("list")
3211 .with_repetition(Repetition::REPEATED)
3212 .with_fields(vec![Arc::new(element_type)])
3213 .build()
3214 .unwrap();
3215
3216 let list_type = SchemaType::group_type_builder("timestamps")
3217 .with_repetition(Repetition::OPTIONAL)
3218 .with_id(Some(1))
3219 .with_logical_type(Some(parquet::basic::LogicalType::List))
3220 .with_fields(vec![Arc::new(list_group)])
3221 .build()
3222 .unwrap();
3223
3224 let parquet_schema = SchemaType::group_type_builder("schema")
3225 .with_fields(vec![Arc::new(list_type)])
3226 .build()
3227 .unwrap();
3228
3229 let (int96_val, expected_micros) = make_int96_test_value();
3230
3231 let file = File::create(&file_path).unwrap();
3232 let mut writer =
3233 SerializedFileWriter::new(file, Arc::new(parquet_schema), Default::default()).unwrap();
3234
3235 let mut row_group = writer.next_row_group().unwrap();
3239 {
3240 let mut col = row_group.next_column().unwrap().unwrap();
3241 col.typed::<Int96Type>()
3242 .write_batch(&[int96_val], Some(&[3]), Some(&[0]))
3243 .unwrap();
3244 col.close().unwrap();
3245 }
3246 row_group.close().unwrap();
3247 writer.close().unwrap();
3248
3249 let iceberg_schema = Arc::new(
3250 Schema::builder()
3251 .with_schema_id(1)
3252 .with_fields(vec![
3253 NestedField::optional(
3254 1,
3255 "timestamps",
3256 Type::List(crate::spec::ListType {
3257 element_field: NestedField::optional(
3258 2,
3259 "element",
3260 Type::Primitive(PrimitiveType::Timestamp),
3261 )
3262 .into(),
3263 }),
3264 )
3265 .into(),
3266 ])
3267 .build()
3268 .unwrap(),
3269 );
3270
3271 let batches = read_int96_batches(&file_path, iceberg_schema, vec![1]).await;
3272
3273 assert_eq!(batches.len(), 1);
3274 let list_array = batches[0]
3275 .column(0)
3276 .as_any()
3277 .downcast_ref::<ListArray>()
3278 .expect("Expected ListArray");
3279 let ts_array = list_array
3280 .values()
3281 .as_any()
3282 .downcast_ref::<TimestampMicrosecondArray>()
3283 .expect("Expected TimestampMicrosecondArray inside list");
3284
3285 assert_eq!(
3286 ts_array.value(0),
3287 expected_micros,
3288 "INT96 in list: got {}, expected {expected_micros}",
3289 ts_array.value(0)
3290 );
3291 }
3292
3293 #[tokio::test]
3294 async fn test_read_int96_timestamps_in_map() {
3295 use arrow_array::{MapArray, TimestampMicrosecondArray};
3296 use parquet::basic::{Repetition, Type as PhysicalType};
3297 use parquet::data_type::{ByteArrayType, Int96Type};
3298 use parquet::file::writer::SerializedFileWriter;
3299 use parquet::schema::types::Type as SchemaType;
3300
3301 let tmp_dir = TempDir::new().unwrap();
3302 let table_location = tmp_dir.path().to_str().unwrap().to_string();
3303 let file_path = format!("{table_location}/map_int96.parquet");
3304
3305 let key_type = SchemaType::primitive_type_builder("key", PhysicalType::BYTE_ARRAY)
3313 .with_repetition(Repetition::REQUIRED)
3314 .with_logical_type(Some(parquet::basic::LogicalType::String))
3315 .with_id(Some(2))
3316 .build()
3317 .unwrap();
3318
3319 let value_type = SchemaType::primitive_type_builder("value", PhysicalType::INT96)
3320 .with_repetition(Repetition::OPTIONAL)
3321 .with_id(Some(3))
3322 .build()
3323 .unwrap();
3324
3325 let key_value_group = SchemaType::group_type_builder("key_value")
3326 .with_repetition(Repetition::REPEATED)
3327 .with_fields(vec![Arc::new(key_type), Arc::new(value_type)])
3328 .build()
3329 .unwrap();
3330
3331 let map_type = SchemaType::group_type_builder("ts_map")
3332 .with_repetition(Repetition::OPTIONAL)
3333 .with_id(Some(1))
3334 .with_logical_type(Some(parquet::basic::LogicalType::Map))
3335 .with_fields(vec![Arc::new(key_value_group)])
3336 .build()
3337 .unwrap();
3338
3339 let parquet_schema = SchemaType::group_type_builder("schema")
3340 .with_fields(vec![Arc::new(map_type)])
3341 .build()
3342 .unwrap();
3343
3344 let (int96_val, expected_micros) = make_int96_test_value();
3345
3346 let file = File::create(&file_path).unwrap();
3347 let mut writer =
3348 SerializedFileWriter::new(file, Arc::new(parquet_schema), Default::default()).unwrap();
3349
3350 let mut row_group = writer.next_row_group().unwrap();
3355 {
3356 let mut col = row_group.next_column().unwrap().unwrap();
3357 col.typed::<ByteArrayType>()
3358 .write_batch(
3359 &[parquet::data_type::ByteArray::from("event_time")],
3360 Some(&[2]),
3361 Some(&[0]),
3362 )
3363 .unwrap();
3364 col.close().unwrap();
3365 }
3366 {
3367 let mut col = row_group.next_column().unwrap().unwrap();
3368 col.typed::<Int96Type>()
3369 .write_batch(&[int96_val], Some(&[3]), Some(&[0]))
3370 .unwrap();
3371 col.close().unwrap();
3372 }
3373 row_group.close().unwrap();
3374 writer.close().unwrap();
3375
3376 let iceberg_schema = Arc::new(
3377 Schema::builder()
3378 .with_schema_id(1)
3379 .with_fields(vec![
3380 NestedField::optional(
3381 1,
3382 "ts_map",
3383 Type::Map(crate::spec::MapType {
3384 key_field: NestedField::required(
3385 2,
3386 "key",
3387 Type::Primitive(PrimitiveType::String),
3388 )
3389 .into(),
3390 value_field: NestedField::optional(
3391 3,
3392 "value",
3393 Type::Primitive(PrimitiveType::Timestamp),
3394 )
3395 .into(),
3396 }),
3397 )
3398 .into(),
3399 ])
3400 .build()
3401 .unwrap(),
3402 );
3403
3404 let batches = read_int96_batches(&file_path, iceberg_schema, vec![1]).await;
3405
3406 assert_eq!(batches.len(), 1);
3407 let map_array = batches[0]
3408 .column(0)
3409 .as_any()
3410 .downcast_ref::<MapArray>()
3411 .expect("Expected MapArray");
3412 let ts_array = map_array
3413 .values()
3414 .as_any()
3415 .downcast_ref::<TimestampMicrosecondArray>()
3416 .expect("Expected TimestampMicrosecondArray as map values");
3417
3418 assert_eq!(
3419 ts_array.value(0),
3420 expected_micros,
3421 "INT96 in map: got {}, expected {expected_micros}",
3422 ts_array.value(0)
3423 );
3424 }
3425
3426 fn write_parquet_with_wide_column(
3432 dir: &str,
3433 name: &str,
3434 extra_fields: Vec<Field>,
3435 extra_columns: Vec<ArrayRef>,
3436 ) -> String {
3437 let wide_field =
3438 Field::new("wide", DataType::Utf8, false).with_metadata(HashMap::from([(
3439 PARQUET_FIELD_ID_META_KEY.to_string(),
3440 "2".to_string(),
3441 )]));
3442 let wide_values: Vec<String> = (0..3)
3445 .map(|i| {
3446 (0..2048)
3447 .map(|j| ((i * 2048 + j) % 251) as u8 as char)
3448 .collect()
3449 })
3450 .collect();
3451
3452 let mut fields = vec![wide_field];
3453 fields.extend(extra_fields);
3454 let mut columns: Vec<ArrayRef> = vec![Arc::new(StringArray::from(wide_values))];
3455 columns.extend(extra_columns);
3456 write_plain_parquet(dir, name, fields, columns)
3457 }
3458
3459 fn id_and_wide_schema() -> SchemaRef {
3462 Arc::new(
3463 Schema::builder()
3464 .with_schema_id(1)
3465 .with_fields(vec![
3466 NestedField::required(1, "id", Type::Primitive(PrimitiveType::Int)).into(),
3467 NestedField::required(2, "wide", Type::Primitive(PrimitiveType::String)).into(),
3468 ])
3469 .build()
3470 .unwrap(),
3471 )
3472 }
3473
3474 fn metadata_projection_task(
3476 file_path: String,
3477 schema: SchemaRef,
3478 project_field_ids: Vec<i32>,
3479 ) -> FileScanTask {
3480 metadata_projection_task_with_first_row_id(file_path, schema, project_field_ids, None)
3481 }
3482
3483 fn metadata_projection_task_with_first_row_id(
3484 file_path: String,
3485 schema: SchemaRef,
3486 project_field_ids: Vec<i32>,
3487 first_row_id: Option<i64>,
3488 ) -> FileScanTask {
3489 FileScanTask::builder()
3490 .with_file_size_in_bytes(std::fs::metadata(&file_path).unwrap().len())
3491 .with_start(0)
3492 .with_length(0)
3493 .with_data_file_path(file_path)
3494 .with_data_file_format(DataFileFormat::Parquet)
3495 .with_schema(schema)
3496 .with_project_field_ids(project_field_ids)
3497 .with_first_row_id(first_row_id)
3498 .with_case_sensitive(false)
3499 .build()
3500 .unwrap()
3501 }
3502
3503 async fn scan_task(task: FileScanTask) -> (Vec<RecordBatch>, u64) {
3505 let reader = ArrowReaderBuilder::new(FileIO::new_with_fs(), Runtime::current()).build();
3506 let tasks = Box::pin(futures::stream::iter(vec![Ok(task)])) as FileScanTaskStream;
3507 let scan = reader.read(tasks).unwrap();
3508 let metrics = scan.metrics().clone();
3509 let batches = scan.stream().try_collect().await.unwrap();
3510 (batches, metrics.bytes_read())
3511 }
3512
3513 #[tokio::test]
3514 async fn test_pos_only_projection_reads_no_data_columns() {
3515 let tmp_dir = TempDir::new().unwrap();
3516 let dir = tmp_dir.path().to_str().unwrap();
3517
3518 let pos_only = metadata_projection_task(
3519 write_parquet_with_wide_column(dir, "pos_only.parquet", vec![], vec![]),
3520 id_and_wide_schema(),
3521 vec![RESERVED_FIELD_ID_POS],
3522 );
3523 let (batches, pos_only_bytes) = scan_task(pos_only).await;
3524
3525 assert_eq!(batches[0].num_columns(), 1);
3527 let pos_col = batches[0]
3528 .column_by_name(RESERVED_COL_NAME_POS)
3529 .expect("_pos column should be present")
3530 .as_primitive::<arrow_array::types::Int64Type>();
3531 assert_eq!(pos_col.values(), &[0, 1, 2]);
3532
3533 let with_data = metadata_projection_task(
3536 write_parquet_with_wide_column(dir, "pos_only_ref.parquet", vec![], vec![]),
3537 id_and_wide_schema(),
3538 vec![2, RESERVED_FIELD_ID_POS],
3539 );
3540 let (_, with_data_bytes) = scan_task(with_data).await;
3541
3542 assert!(
3543 pos_only_bytes < with_data_bytes,
3544 "_pos-only scan should read fewer bytes than a scan of the wide column: \
3545 {pos_only_bytes} vs {with_data_bytes}"
3546 );
3547 }
3548
3549 #[tokio::test]
3550 async fn test_empty_projection_reads_no_data_columns() {
3551 let tmp_dir = TempDir::new().unwrap();
3552 let dir = tmp_dir.path().to_str().unwrap();
3553
3554 let empty = metadata_projection_task(
3555 write_parquet_with_wide_column(dir, "empty.parquet", vec![], vec![]),
3556 id_and_wide_schema(),
3557 vec![],
3558 );
3559 let (batches, empty_bytes) = scan_task(empty).await;
3560
3561 assert!(batches.iter().all(|b| b.num_columns() == 0));
3563 assert_eq!(batches.iter().map(|b| b.num_rows()).sum::<usize>(), 3);
3564
3565 let with_data = metadata_projection_task(
3566 write_parquet_with_wide_column(dir, "empty_ref.parquet", vec![], vec![]),
3567 id_and_wide_schema(),
3568 vec![2],
3569 );
3570 let (_, with_data_bytes) = scan_task(with_data).await;
3571
3572 assert!(
3573 empty_bytes < with_data_bytes,
3574 "empty projection should read fewer bytes than the wide column: \
3575 {empty_bytes} vs {with_data_bytes}"
3576 );
3577 }
3578
3579 #[tokio::test]
3580 async fn test_empty_projection_counts_rows_after_positional_delete() {
3581 use crate::spec::DataContentType;
3582
3583 let tmp_dir = TempDir::new().unwrap();
3584 let dir = tmp_dir.path().to_str().unwrap();
3585 let data_path = write_plain_parquet(dir, "empty_posdel_data.parquet", vec![], vec![]);
3586 let del_path = write_positional_delete(dir, "empty_posdel.parquet", &data_path, &[1]);
3587 let delete = FileScanTaskDeleteFile::builder()
3588 .with_file_path(del_path.clone())
3589 .with_file_size_in_bytes(std::fs::metadata(&del_path).unwrap().len())
3590 .with_file_type(DataContentType::PositionDeletes)
3591 .with_file_format(DataFileFormat::Parquet)
3592 .with_partition_spec_id(0)
3593 .build()
3594 .unwrap();
3595 let task = FileScanTask::builder()
3596 .with_file_size_in_bytes(std::fs::metadata(&data_path).unwrap().len())
3597 .with_start(0)
3598 .with_length(0)
3599 .with_data_file_path(data_path)
3600 .with_data_file_format(DataFileFormat::Parquet)
3601 .with_schema(id_and_wide_schema())
3602 .with_project_field_ids(vec![])
3603 .with_deletes(vec![delete])
3604 .with_case_sensitive(false)
3605 .build()
3606 .unwrap();
3607
3608 let (batches, _) = scan_task(task).await;
3609
3610 assert_eq!(batches.iter().map(|b| b.num_rows()).sum::<usize>(), 2);
3611 }
3612
3613 #[tokio::test]
3614 async fn test_empty_projection_counts_rows_after_predicate() {
3615 use crate::expr::{Bind, Reference};
3616 use crate::spec::Datum;
3617
3618 let tmp_dir = TempDir::new().unwrap();
3619 let dir = tmp_dir.path().to_str().unwrap();
3620 let file_path = write_plain_parquet(dir, "empty_predicate.parquet", vec![], vec![]);
3621 let schema = id_and_wide_schema();
3622 let bound = Reference::new("id")
3624 .not_equal_to(Datum::int(2))
3625 .bind(Arc::clone(&schema), false)
3626 .unwrap();
3627 let task = FileScanTask::builder()
3628 .with_file_size_in_bytes(std::fs::metadata(&file_path).unwrap().len())
3629 .with_start(0)
3630 .with_length(0)
3631 .with_data_file_path(file_path)
3632 .with_data_file_format(DataFileFormat::Parquet)
3633 .with_schema(schema)
3634 .with_project_field_ids(vec![])
3635 .with_predicate(Some(bound))
3636 .with_case_sensitive(false)
3637 .build()
3638 .unwrap();
3639
3640 let (batches, _) = scan_task(task).await;
3641
3642 assert_eq!(batches.iter().map(|b| b.num_rows()).sum::<usize>(), 2);
3643 }
3644
3645 #[tokio::test]
3646 async fn test_pos_only_projection_keeps_absolute_pos_under_predicate() {
3647 use crate::expr::{Bind, Reference};
3648 use crate::spec::Datum;
3649
3650 let tmp_dir = TempDir::new().unwrap();
3651 let dir = tmp_dir.path().to_str().unwrap();
3652 let file_path = write_plain_parquet(dir, "pos_only_predicate.parquet", vec![], vec![]);
3654
3655 let schema = Arc::new(
3656 Schema::builder()
3657 .with_schema_id(1)
3658 .with_fields(vec![
3659 NestedField::required(1, "id", Type::Primitive(PrimitiveType::Int)).into(),
3660 ])
3661 .build()
3662 .unwrap(),
3663 );
3664 let bound = Reference::new("id")
3665 .not_equal_to(Datum::int(2))
3666 .bind(Arc::clone(&schema), false)
3667 .unwrap();
3668 let task = FileScanTask::builder()
3669 .with_file_size_in_bytes(std::fs::metadata(&file_path).unwrap().len())
3670 .with_start(0)
3671 .with_length(0)
3672 .with_data_file_path(file_path)
3673 .with_data_file_format(DataFileFormat::Parquet)
3674 .with_schema(schema)
3675 .with_project_field_ids(vec![RESERVED_FIELD_ID_POS])
3676 .with_predicate(Some(bound))
3677 .with_case_sensitive(false)
3678 .build()
3679 .unwrap();
3680
3681 let reader = ArrowReaderBuilder::new(FileIO::new_with_fs(), Runtime::current())
3685 .with_options(
3686 ReaderOptions::builder()
3687 .with_row_selection_enabled(true)
3688 .build(),
3689 )
3690 .build();
3691 let tasks = Box::pin(futures::stream::iter(vec![Ok(task)])) as FileScanTaskStream;
3692 let batches: Vec<RecordBatch> = reader
3693 .read(tasks)
3694 .unwrap()
3695 .stream()
3696 .try_collect()
3697 .await
3698 .unwrap();
3699
3700 let pos: Vec<i64> = batches
3701 .iter()
3702 .flat_map(|b| {
3703 b.column_by_name(RESERVED_COL_NAME_POS)
3704 .expect("_pos column should be present")
3705 .as_primitive::<arrow_array::types::Int64Type>()
3706 .values()
3707 .to_vec()
3708 })
3709 .collect();
3710 assert_eq!(pos, vec![0, 2]);
3711 }
3712
3713 #[tokio::test]
3714 async fn test_pos_and_file_projection() {
3715 use crate::metadata_columns::RESERVED_COL_NAME_FILE;
3716
3717 let tmp_dir = TempDir::new().unwrap();
3718 let dir = tmp_dir.path().to_str().unwrap();
3719 let file_path = write_parquet_with_wide_column(dir, "pos_and_file.parquet", vec![], vec![]);
3722 let task = metadata_projection_task(file_path.clone(), id_and_wide_schema(), vec![
3723 RESERVED_FIELD_ID_POS,
3724 RESERVED_FIELD_ID_FILE,
3725 ]);
3726 let (batches, _) = scan_task(task).await;
3727
3728 assert_eq!(batches[0].num_columns(), 2);
3730 let pos_col = batches[0]
3731 .column_by_name(RESERVED_COL_NAME_POS)
3732 .expect("_pos column should be present")
3733 .as_primitive::<arrow_array::types::Int64Type>();
3734 assert_eq!(pos_col.values(), &[0, 1, 2]);
3735 let file_col = batches[0]
3736 .column_by_name(RESERVED_COL_NAME_FILE)
3737 .expect("_file column should be present");
3738 let file_col = cast(file_col, &DataType::Utf8).unwrap();
3739 let file_col = file_col.as_any().downcast_ref::<StringArray>().unwrap();
3740 assert_eq!(file_col.value(0), file_path);
3741 }
3742
3743 #[tokio::test]
3744 async fn test_pos_and_physical_seq_projection_reads_only_the_leaf() {
3745 use crate::metadata_columns::{
3746 RESERVED_COL_NAME_LAST_UPDATED_SEQUENCE_NUMBER,
3747 RESERVED_FIELD_ID_LAST_UPDATED_SEQUENCE_NUMBER,
3748 };
3749
3750 let tmp_dir = TempDir::new().unwrap();
3754 let dir = tmp_dir.path().to_str().unwrap();
3755
3756 let write = |name: &str| {
3758 write_parquet_with_wide_column(
3759 dir,
3760 name,
3761 vec![physical_last_updated_seq_field()],
3762 vec![Arc::new(Int64Array::from(vec![Some(5), None, Some(8)])) as ArrayRef],
3763 )
3764 };
3765
3766 let seq_task = |path: String, ids: Vec<i32>| {
3767 FileScanTask::builder()
3768 .with_file_size_in_bytes(std::fs::metadata(&path).unwrap().len())
3769 .with_start(0)
3770 .with_length(0)
3771 .with_data_file_path(path)
3772 .with_data_file_format(DataFileFormat::Parquet)
3773 .with_schema(id_and_wide_schema())
3774 .with_project_field_ids(ids)
3775 .with_first_row_id(Some(100))
3776 .with_data_sequence_number(Some(9))
3777 .with_case_sensitive(false)
3778 .build()
3779 .unwrap()
3780 };
3781
3782 let meta_only = seq_task(write("pos_seq.parquet"), vec![
3783 RESERVED_FIELD_ID_POS,
3784 RESERVED_FIELD_ID_LAST_UPDATED_SEQUENCE_NUMBER,
3785 ]);
3786 let (batches, meta_only_bytes) = scan_task(meta_only).await;
3787
3788 let pos_col = batches[0]
3790 .column_by_name(RESERVED_COL_NAME_POS)
3791 .expect("_pos column should be present")
3792 .as_primitive::<arrow_array::types::Int64Type>();
3793 assert_eq!(pos_col.values(), &[0, 1, 2]);
3794 let seq_col = batches[0]
3795 .column_by_name(RESERVED_COL_NAME_LAST_UPDATED_SEQUENCE_NUMBER)
3796 .expect("_last_updated_sequence_number column should be present");
3797 let seq_col = cast(seq_col, &DataType::Int64).unwrap();
3798 let seq_col = seq_col.as_any().downcast_ref::<Int64Array>().unwrap();
3799 assert_eq!(seq_col.value(0), 5);
3801 assert_eq!(seq_col.value(1), 9);
3802 assert_eq!(seq_col.value(2), 8);
3803 assert!(batches[0].column_by_name("wide").is_none());
3804
3805 let with_data = seq_task(write("pos_seq_ref.parquet"), vec![
3808 2,
3809 RESERVED_FIELD_ID_POS,
3810 RESERVED_FIELD_ID_LAST_UPDATED_SEQUENCE_NUMBER,
3811 ]);
3812 let (_, with_data_bytes) = scan_task(with_data).await;
3813
3814 assert!(
3815 meta_only_bytes < with_data_bytes,
3816 "_pos + physical sequence scan should read fewer bytes than one that also \
3817 reads the wide column: {meta_only_bytes} vs {with_data_bytes}"
3818 );
3819 }
3820
3821 #[tokio::test]
3822 async fn test_seq_only_projection_reads_only_the_leaf() {
3823 use crate::metadata_columns::{
3824 RESERVED_COL_NAME_LAST_UPDATED_SEQUENCE_NUMBER,
3825 RESERVED_FIELD_ID_LAST_UPDATED_SEQUENCE_NUMBER,
3826 };
3827
3828 let tmp_dir = TempDir::new().unwrap();
3832 let dir = tmp_dir.path().to_str().unwrap();
3833 let write = |name: &str| {
3834 write_parquet_with_wide_column(
3835 dir,
3836 name,
3837 vec![physical_last_updated_seq_field()],
3838 vec![Arc::new(Int64Array::from(vec![Some(5), None, Some(8)])) as ArrayRef],
3839 )
3840 };
3841 let seq_task = |path: String, ids: Vec<i32>| {
3842 FileScanTask::builder()
3843 .with_file_size_in_bytes(std::fs::metadata(&path).unwrap().len())
3844 .with_start(0)
3845 .with_length(0)
3846 .with_data_file_path(path)
3847 .with_data_file_format(DataFileFormat::Parquet)
3848 .with_schema(id_and_wide_schema())
3849 .with_project_field_ids(ids)
3850 .with_first_row_id(Some(100))
3851 .with_data_sequence_number(Some(9))
3852 .with_case_sensitive(false)
3853 .build()
3854 .unwrap()
3855 };
3856
3857 let meta_only = seq_task(write("seq_only.parquet"), vec![
3858 RESERVED_FIELD_ID_LAST_UPDATED_SEQUENCE_NUMBER,
3859 ]);
3860 let (batches, meta_only_bytes) = scan_task(meta_only).await;
3861
3862 let seq_col = batches[0]
3863 .column_by_name(RESERVED_COL_NAME_LAST_UPDATED_SEQUENCE_NUMBER)
3864 .expect("_last_updated_sequence_number column should be present");
3865 let seq_col = cast(seq_col, &DataType::Int64).unwrap();
3866 let seq_col = seq_col.as_any().downcast_ref::<Int64Array>().unwrap();
3867 assert_eq!(seq_col.value(0), 5);
3868 assert_eq!(seq_col.value(1), 9);
3869 assert_eq!(seq_col.value(2), 8);
3870 let total_rows: usize = batches.iter().map(|b| b.num_rows()).sum();
3871 assert_eq!(total_rows, 3);
3872 assert!(batches[0].column_by_name("wide").is_none());
3873
3874 let with_data = seq_task(write("seq_only_ref.parquet"), vec![
3875 2,
3876 RESERVED_FIELD_ID_LAST_UPDATED_SEQUENCE_NUMBER,
3877 ]);
3878 let (_, with_data_bytes) = scan_task(with_data).await;
3879
3880 assert!(
3881 meta_only_bytes < with_data_bytes,
3882 "seq-only scan should read fewer bytes than one that also reads the wide \
3883 column: {meta_only_bytes} vs {with_data_bytes}"
3884 );
3885 }
3886
3887 #[tokio::test]
3888 async fn test_seq_only_null_first_row_id_reads_no_data_columns() {
3889 use crate::metadata_columns::{
3890 RESERVED_COL_NAME_LAST_UPDATED_SEQUENCE_NUMBER,
3891 RESERVED_FIELD_ID_LAST_UPDATED_SEQUENCE_NUMBER,
3892 };
3893
3894 let tmp_dir = TempDir::new().unwrap();
3899 let dir = tmp_dir.path().to_str().unwrap();
3900 let seq_task = |name: &str, ids: Vec<i32>| {
3901 let file_path = write_parquet_with_wide_column(
3902 dir,
3903 name,
3904 vec![physical_last_updated_seq_field()],
3905 vec![Arc::new(Int64Array::from(vec![Some(5), Some(6), Some(7)])) as ArrayRef],
3906 );
3907 FileScanTask::builder()
3908 .with_file_size_in_bytes(std::fs::metadata(&file_path).unwrap().len())
3909 .with_start(0)
3910 .with_length(0)
3911 .with_data_file_path(file_path)
3912 .with_data_file_format(DataFileFormat::Parquet)
3913 .with_schema(id_and_wide_schema())
3914 .with_project_field_ids(ids)
3915 .with_first_row_id(None)
3916 .with_data_sequence_number(Some(9))
3917 .with_case_sensitive(false)
3918 .build()
3919 .unwrap()
3920 };
3921
3922 let meta_only = seq_task("seq_only_null_first.parquet", vec![
3923 RESERVED_FIELD_ID_LAST_UPDATED_SEQUENCE_NUMBER,
3924 ]);
3925 let (batches, meta_only_bytes) = scan_task(meta_only).await;
3926
3927 assert_eq!(batches[0].num_columns(), 1);
3928 let total_rows: usize = batches.iter().map(|b| b.num_rows()).sum();
3929 assert_eq!(total_rows, 3);
3930 let seq_col = batches[0]
3931 .column_by_name(RESERVED_COL_NAME_LAST_UPDATED_SEQUENCE_NUMBER)
3932 .expect("_last_updated_sequence_number column should be present");
3933 let seq_col = cast(seq_col, &DataType::Int64).unwrap();
3934 let seq_col = seq_col.as_any().downcast_ref::<Int64Array>().unwrap();
3935 assert!((0..3).all(|i| seq_col.is_null(i)));
3936
3937 let with_data = seq_task("seq_only_null_first_ref.parquet", vec![
3938 2,
3939 RESERVED_FIELD_ID_LAST_UPDATED_SEQUENCE_NUMBER,
3940 ]);
3941 let (_, with_data_bytes) = scan_task(with_data).await;
3942
3943 assert!(
3944 meta_only_bytes < with_data_bytes,
3945 "seq-only scan with a null first_row_id should read fewer bytes than a scan of \
3946 the wide column: {meta_only_bytes} vs {with_data_bytes}"
3947 );
3948 }
3949
3950 #[tokio::test]
3951 async fn test_file_only_reads_no_data_columns() {
3952 use crate::metadata_columns::RESERVED_COL_NAME_FILE;
3953
3954 let tmp_dir = TempDir::new().unwrap();
3958 let dir = tmp_dir.path().to_str().unwrap();
3959
3960 let file_path = write_parquet_with_wide_column(dir, "file_only.parquet", vec![], vec![]);
3961 let meta_only = metadata_projection_task(file_path.clone(), id_and_wide_schema(), vec![
3962 RESERVED_FIELD_ID_FILE,
3963 ]);
3964 let (batches, meta_only_bytes) = scan_task(meta_only).await;
3965
3966 assert_eq!(batches[0].num_columns(), 1);
3967 let total_rows: usize = batches.iter().map(|b| b.num_rows()).sum();
3968 assert_eq!(total_rows, 3);
3969 let file_col = batches[0]
3970 .column_by_name(RESERVED_COL_NAME_FILE)
3971 .expect("_file column should be present");
3972 let file_col = cast(file_col, &DataType::Utf8).unwrap();
3973 let file_col = file_col.as_any().downcast_ref::<StringArray>().unwrap();
3974 assert_eq!(file_col.value(0), file_path);
3975
3976 let with_data = metadata_projection_task(
3977 write_parquet_with_wide_column(dir, "file_only_ref.parquet", vec![], vec![]),
3978 id_and_wide_schema(),
3979 vec![2, RESERVED_FIELD_ID_FILE],
3980 );
3981 let (_, with_data_bytes) = scan_task(with_data).await;
3982
3983 assert!(
3984 meta_only_bytes < with_data_bytes,
3985 "_file-only scan should read fewer bytes than a scan of the wide column: \
3986 {meta_only_bytes} vs {with_data_bytes}"
3987 );
3988 }
3989
3990 #[tokio::test]
3991 async fn test_spec_id_only_reads_no_data_columns() {
3992 use crate::metadata_columns::{RESERVED_COL_NAME_SPEC_ID, RESERVED_FIELD_ID_SPEC_ID};
3993 use crate::spec::{Literal, PartitionSpec, Struct, Transform};
3994
3995 let tmp_dir = TempDir::new().unwrap();
3999 let dir = tmp_dir.path().to_str().unwrap();
4000 let schema = id_and_wide_schema();
4001 let spec = Arc::new(
4002 PartitionSpec::builder(schema.clone())
4003 .with_spec_id(7)
4004 .add_partition_field("id", "id", Transform::Identity)
4005 .unwrap()
4006 .build()
4007 .unwrap(),
4008 );
4009
4010 let spec_id_task = |file_path: String, project_field_ids: Vec<i32>| {
4011 FileScanTask::builder()
4012 .with_file_size_in_bytes(std::fs::metadata(&file_path).unwrap().len())
4013 .with_start(0)
4014 .with_length(0)
4015 .with_data_file_path(file_path)
4016 .with_data_file_format(DataFileFormat::Parquet)
4017 .with_schema(schema.clone())
4018 .with_project_field_ids(project_field_ids)
4019 .with_partition(Some(Struct::from_iter([Some(Literal::int(42))])))
4020 .with_partition_spec(Some(spec.clone()))
4021 .with_case_sensitive(false)
4022 .build()
4023 .unwrap()
4024 };
4025
4026 let meta_only = spec_id_task(
4027 write_parquet_with_wide_column(dir, "spec_id_only.parquet", vec![], vec![]),
4028 vec![RESERVED_FIELD_ID_SPEC_ID],
4029 );
4030 let (batches, meta_only_bytes) = scan_task(meta_only).await;
4031
4032 assert_eq!(batches[0].num_columns(), 1);
4033 let total_rows: usize = batches.iter().map(|b| b.num_rows()).sum();
4034 assert_eq!(total_rows, 3);
4035 let spec_id_col = batches[0]
4036 .column_by_name(RESERVED_COL_NAME_SPEC_ID)
4037 .expect("_spec_id column should be present");
4038 let spec_id_col = cast(spec_id_col, &DataType::Int32).unwrap();
4039 let spec_id_col = spec_id_col.as_any().downcast_ref::<Int32Array>().unwrap();
4040 assert_eq!(spec_id_col.values(), &[7, 7, 7]);
4041
4042 let with_data = spec_id_task(
4043 write_parquet_with_wide_column(dir, "spec_id_only_ref.parquet", vec![], vec![]),
4044 vec![2, RESERVED_FIELD_ID_SPEC_ID],
4045 );
4046 let (_, with_data_bytes) = scan_task(with_data).await;
4047
4048 assert!(
4049 meta_only_bytes < with_data_bytes,
4050 "_spec_id-only scan should read fewer bytes than a scan of the wide column: \
4051 {meta_only_bytes} vs {with_data_bytes}"
4052 );
4053 }
4054
4055 #[tokio::test]
4056 async fn test_empty_projection_preserves_row_count() {
4057 let tmp_dir = TempDir::new().unwrap();
4058 let dir = tmp_dir.path().to_str().unwrap();
4059 let file_path = write_plain_parquet(dir, "empty_projection.parquet", vec![], vec![]);
4060 let schema = Arc::new(
4061 Schema::builder()
4062 .with_schema_id(1)
4063 .with_fields(vec![
4064 NestedField::required(1, "id", Type::Primitive(PrimitiveType::Int)).into(),
4065 ])
4066 .build()
4067 .unwrap(),
4068 );
4069 let task = metadata_projection_task(file_path, schema, vec![]);
4070 let (batches, _) = scan_task(task).await;
4071
4072 let total_rows: usize = batches.iter().map(|b| b.num_rows()).sum();
4074 assert_eq!(total_rows, 3);
4075 }
4076
4077 mod open_progress {
4080 use std::ops::Range;
4081 use std::sync::Mutex;
4082
4083 use async_trait::async_trait;
4084 use bytes::Bytes;
4085 use futures::StreamExt;
4086 use futures::stream::BoxStream;
4087 use parquet::arrow::arrow_reader::ParquetRecordBatchReaderBuilder;
4088 use serde::{Deserialize, Serialize};
4089
4090 use super::*;
4091 use crate::Result;
4092 use crate::arrow::ParquetReadOptions;
4093 use crate::arrow::scan_metrics::ScanMetrics;
4094 use crate::expr::{Bind, Predicate, Reference};
4095 use crate::io::{
4096 FileIOBuilder, FileMetadata, FileRead, FileWrite, InputFile, MemoryStorage, OutputFile,
4097 Storage, StorageConfig, StorageFactory,
4098 };
4099 use crate::scan::ArrowRecordBatchStream;
4100 use crate::spec::Datum;
4101
4102 const CONCURRENCY: usize = 2;
4103 const METADATA_SIZE_HINT: usize = 64;
4105 const OPEN_BYTES: u64 = 547;
4109 const YIELDS: usize = 100;
4112
4113 #[derive(Debug, Default)]
4114 struct Reads {
4115 last_start: Option<u64>,
4116 started: usize,
4117 finished: usize,
4118 }
4119
4120 #[derive(Debug, Clone, Default)]
4124 struct ReadLog(Arc<Mutex<Reads>>);
4125
4126 impl ReadLog {
4127 fn last_start(&self) -> Option<u64> {
4129 self.0.lock().unwrap().last_start
4130 }
4131
4132 fn in_flight(&self) -> usize {
4133 let reads = self.0.lock().unwrap();
4134 reads.started - reads.finished
4135 }
4136 }
4137
4138 #[derive(Debug, Clone, Default, Serialize, Deserialize)]
4140 struct DeferredStorage {
4141 #[serde(skip)]
4142 files: MemoryStorage,
4143 #[serde(skip)]
4144 log: ReadLog,
4145 }
4146
4147 struct DeferredRead {
4148 contents: Bytes,
4149 log: ReadLog,
4150 }
4151
4152 #[async_trait]
4153 impl FileRead for DeferredRead {
4154 async fn read(&self, range: Range<u64>) -> Result<Bytes> {
4155 {
4156 let mut reads = self.log.0.lock().unwrap();
4157 reads.last_start = Some(range.start);
4158 reads.started += 1;
4159 }
4160 tokio::task::yield_now().await;
4163 self.log.0.lock().unwrap().finished += 1;
4164 Ok(self
4165 .contents
4166 .slice(range.start as usize..range.end as usize))
4167 }
4168 }
4169
4170 #[async_trait]
4171 #[typetag::serde]
4172 impl Storage for DeferredStorage {
4173 async fn reader(&self, path: &str) -> Result<Box<dyn FileRead>> {
4174 Ok(Box::new(DeferredRead {
4175 contents: self.files.read(path).await?,
4176 log: self.log.clone(),
4177 }))
4178 }
4179
4180 fn new_input(&self, path: &str) -> Result<InputFile> {
4181 Ok(InputFile::new(Arc::new(self.clone()), path.to_string()))
4182 }
4183
4184 async fn exists(&self, path: &str) -> Result<bool> {
4185 self.files.exists(path).await
4186 }
4187
4188 async fn metadata(&self, path: &str) -> Result<FileMetadata> {
4189 self.files.metadata(path).await
4190 }
4191
4192 async fn read(&self, path: &str) -> Result<Bytes> {
4193 self.files.read(path).await
4194 }
4195
4196 async fn write(&self, path: &str, bs: Bytes) -> Result<()> {
4197 self.files.write(path, bs).await
4198 }
4199
4200 async fn writer(&self, path: &str) -> Result<Box<dyn FileWrite>> {
4201 self.files.writer(path).await
4202 }
4203
4204 async fn delete(&self, path: &str) -> Result<()> {
4205 self.files.delete(path).await
4206 }
4207
4208 async fn delete_prefix(&self, path: &str) -> Result<()> {
4209 self.files.delete_prefix(path).await
4210 }
4211
4212 async fn delete_stream(&self, paths: BoxStream<'static, String>) -> Result<()> {
4213 self.files.delete_stream(paths).await
4214 }
4215
4216 fn new_output(&self, path: &str) -> Result<OutputFile> {
4217 Ok(OutputFile::new(Arc::new(self.clone()), path.to_string()))
4218 }
4219 }
4220
4221 #[derive(Debug, Default, Serialize, Deserialize)]
4222 struct DeferredStorageFactory {
4223 #[serde(skip)]
4224 storage: DeferredStorage,
4225 }
4226
4227 #[typetag::serde]
4228 impl StorageFactory for DeferredStorageFactory {
4229 fn build(&self, _config: &StorageConfig) -> Result<Arc<dyn Storage>> {
4230 Ok(Arc::new(self.storage.clone()))
4231 }
4232 }
4233
4234 async fn deferred_scan_of(
4237 contents: Bytes,
4238 files: usize,
4239 predicate: Option<Predicate>,
4240 ) -> (ScanMetrics, ReadLog, ArrowRecordBatchStream) {
4241 let schema = Arc::new(
4242 Schema::builder()
4243 .with_schema_id(1)
4244 .with_fields(vec![
4245 NestedField::required(1, "id", Type::Primitive(PrimitiveType::Int)).into(),
4246 ])
4247 .build()
4248 .unwrap(),
4249 );
4250 let bound = predicate.map(|p| p.bind(Arc::clone(&schema), false).unwrap());
4251
4252 let storage = DeferredStorage::default();
4253 let log = storage.log.clone();
4254 let mut tasks = Vec::with_capacity(files);
4255 for file_num in 0..files {
4256 let path = format!("memory:/data/file_{file_num}.parquet");
4257 storage.write(&path, contents.clone()).await.unwrap();
4258 tasks.push(
4259 FileScanTask::builder()
4260 .with_data_file_path(path)
4261 .with_file_size_in_bytes(contents.len() as u64)
4262 .with_start(0)
4263 .with_length(0)
4264 .with_data_file_format(DataFileFormat::Parquet)
4265 .with_schema(Arc::clone(&schema))
4266 .with_project_field_ids(vec![1])
4267 .with_predicate(bound.clone())
4268 .with_case_sensitive(false)
4269 .build()
4270 .unwrap(),
4271 );
4272 }
4273
4274 let file_io = FileIOBuilder::new(Arc::new(DeferredStorageFactory { storage })).build();
4275
4276 let reader = ArrowReaderBuilder::new(file_io, Runtime::current())
4277 .with_options(
4278 ReaderOptions::builder()
4279 .with_data_file_concurrency_limit(CONCURRENCY)
4280 .with_bloom_filter_enabled(bound.is_some())
4281 .with_parquet_options(
4282 ParquetReadOptions::builder()
4283 .with_metadata_size_hint(METADATA_SIZE_HINT)
4284 .build(),
4285 )
4286 .build(),
4287 )
4288 .build();
4289 let tasks =
4290 Box::pin(futures::stream::iter(tasks.into_iter().map(Ok))) as FileScanTaskStream;
4291 let result = reader.read(tasks).unwrap();
4292 (result.metrics().clone(), log, result.stream())
4293 }
4294
4295 async fn deferred_scan() -> (ScanMetrics, ArrowRecordBatchStream) {
4297 let tmp_dir = TempDir::new().unwrap();
4298 let on_disk = write_plain_parquet(
4299 tmp_dir.path().to_str().unwrap(),
4300 "data.parquet",
4301 vec![],
4302 vec![],
4303 );
4304 let contents = Bytes::from(std::fs::read(on_disk).unwrap());
4305 let (metrics, _, scan) = deferred_scan_of(contents, CONCURRENCY, None).await;
4306 (metrics, scan)
4307 }
4308
4309 async fn spin(done: impl Fn() -> bool) {
4311 for _ in 0..YIELDS {
4312 if done() {
4313 return;
4314 }
4315 tokio::task::yield_now().await;
4316 }
4317 }
4318
4319 #[tokio::test]
4321 async fn test_open_proceeds_without_the_scan_being_polled() {
4322 let (metrics, scan) = deferred_scan().await;
4323 let mut scan = Box::pin(scan);
4324 let every_open = OPEN_BYTES * CONCURRENCY as u64;
4325
4326 let _ = futures::poll!(scan.next());
4329 spin(|| metrics.bytes_read() >= every_open).await;
4330 assert_eq!(
4331 metrics.bytes_read(),
4332 every_open,
4333 "the opens stopped where the scan stopped polling"
4334 );
4335 }
4336
4337 #[tokio::test]
4339 async fn test_dropped_scan_stops_opening_files() {
4340 let (metrics, scan) = deferred_scan().await;
4341 let mut scan = Box::pin(scan);
4342 let every_open = OPEN_BYTES * CONCURRENCY as u64;
4343
4344 let _ = futures::poll!(scan.next());
4346 spin(|| metrics.bytes_read() > 0).await;
4347 let in_flight = metrics.bytes_read();
4348 assert!(in_flight > 0, "no open reached the storage");
4349 assert!(in_flight < every_open, "the opens finished before the drop");
4350
4351 drop(scan);
4352 spin(|| false).await;
4353 assert_eq!(
4354 metrics.bytes_read(),
4355 in_flight,
4356 "a dropped scan read {} more bytes",
4357 metrics.bytes_read() - in_flight
4358 );
4359 }
4360
4361 #[tokio::test]
4364 async fn test_bloom_filter_fetch_proceeds_without_the_scan_being_polled() {
4365 let tmp_dir = TempDir::new().unwrap();
4366 let props = WriterProperties::builder()
4367 .set_compression(Compression::SNAPPY)
4368 .set_bloom_filter_enabled(true)
4369 .build();
4370 let on_disk = write_parquet_with(
4371 tmp_dir.path().to_str().unwrap(),
4372 "bloom.parquet",
4373 vec![],
4374 vec![],
4375 props,
4376 );
4377 let bloom_offset =
4378 ParquetRecordBatchReaderBuilder::try_new(File::open(&on_disk).unwrap())
4379 .unwrap()
4380 .metadata()
4381 .row_group(0)
4382 .column(0)
4383 .bloom_filter_offset()
4384 .expect("the writer put a bloom filter on `id`") as u64;
4385
4386 let contents = Bytes::from(std::fs::read(&on_disk).unwrap());
4387 let predicate = Reference::new("id").equal_to(Datum::int(2));
4388 let (_, log, scan) = deferred_scan_of(contents, 1, Some(predicate)).await;
4389 let mut scan = Box::pin(scan);
4390
4391 let at_the_bloom_filter = || log.last_start() == Some(bloom_offset);
4394 for _ in 0..YIELDS {
4395 if at_the_bloom_filter() {
4396 break;
4397 }
4398 let _ = futures::poll!(scan.next());
4399 tokio::task::yield_now().await;
4400 }
4401 assert!(at_the_bloom_filter(), "the bloom filter was never read");
4402
4403 spin(|| log.in_flight() == 0).await;
4405 assert_eq!(
4406 log.in_flight(),
4407 0,
4408 "a read was left in flight where the scan stopped polling"
4409 );
4410 }
4411 }
4412}