1use std::collections::HashMap;
19use std::sync::Arc;
20
21use futures::stream::BoxStream;
22use serde::{Deserialize, Serialize};
23use typed_builder::TypedBuilder;
24
25use crate::error::invalid_data;
26use crate::expr::BoundPredicate;
27use crate::spec::{
28 DataContentType, DataFile, DataFileFormat, Datum, ManifestEntryRef, NameMapping, PartitionSpec,
29 PrimitiveType, Schema, SchemaRef, SortOrderRef, Struct, StructType,
30};
31use crate::{Error, Result};
32
33pub type FileScanTaskStream = BoxStream<'static, Result<FileScanTask>>;
35
36#[derive(Debug, Clone, Deserialize, PartialEq, TypedBuilder)]
38#[serde(try_from = "crate::scan::task::_serde::FileScanTaskSerde")]
39#[builder(
40 field_defaults(setter(prefix = "with_")),
41 build_method(into = Result<FileScanTask>)
42)]
43pub struct FileScanTask {
44 file_size_in_bytes: u64,
47 start: u64,
49 length: u64,
51 #[builder(default)]
56 record_count: Option<u64>,
57
58 #[builder(default)]
63 first_row_id: Option<i64>,
64
65 #[builder(default)]
72 data_sequence_number: Option<i64>,
73
74 data_file_path: String,
76
77 data_file_format: DataFileFormat,
79
80 schema: SchemaRef,
82 project_field_ids: Vec<i32>,
84 #[builder(default)]
86 predicate: Option<BoundPredicate>,
87
88 #[builder(default)]
90 deletes: Vec<FileScanTaskDeleteFile>,
91
92 #[builder(default)]
96 partition: Option<Struct>,
97
98 #[builder(default)]
102 partition_spec: Option<Arc<PartitionSpec>>,
103
104 #[builder(default)]
108 name_mapping: Option<Arc<NameMapping>>,
109
110 #[builder(default)]
119 unified_partition_type: Option<Arc<StructType>>,
120
121 #[builder(default)]
130 sort_order_id: Option<i32>,
131
132 #[builder(default)]
139 sort_order: Option<SortOrderRef>,
140
141 #[builder(default, setter(transform = |stats: Option<FileScanTaskColumnStats>| stats.map(Box::new)))]
142 column_stats: Option<Box<FileScanTaskColumnStats>>,
143
144 case_sensitive: bool,
146
147 #[builder(default)]
156 key_metadata: Option<Box<[u8]>>,
157}
158
159impl FileScanTask {
160 pub fn file_size_in_bytes(&self) -> u64 {
162 self.file_size_in_bytes
163 }
164
165 pub fn start(&self) -> u64 {
167 self.start
168 }
169
170 pub fn length(&self) -> u64 {
172 self.length
173 }
174
175 pub fn record_count(&self) -> Option<u64> {
177 self.record_count
178 }
179
180 pub fn first_row_id(&self) -> Option<i64> {
182 self.first_row_id
183 }
184
185 pub fn data_sequence_number(&self) -> Option<i64> {
187 self.data_sequence_number
188 }
189
190 pub fn data_file_path(&self) -> &str {
192 &self.data_file_path
193 }
194
195 pub fn data_file_format(&self) -> DataFileFormat {
197 self.data_file_format
198 }
199
200 pub fn schema(&self) -> &Schema {
202 &self.schema
203 }
204
205 pub fn schema_ref(&self) -> SchemaRef {
207 self.schema.clone()
208 }
209
210 pub fn project_field_ids(&self) -> &[i32] {
212 &self.project_field_ids
213 }
214
215 pub fn predicate(&self) -> Option<&BoundPredicate> {
217 self.predicate.as_ref()
218 }
219
220 pub(crate) fn clear_predicate(&mut self) {
227 self.predicate = None;
228 }
229
230 pub fn deletes(&self) -> &[FileScanTaskDeleteFile] {
232 &self.deletes
233 }
234
235 pub fn partition(&self) -> Option<&Struct> {
237 self.partition.as_ref()
238 }
239
240 pub fn partition_spec(&self) -> Option<&Arc<PartitionSpec>> {
242 self.partition_spec.as_ref()
243 }
244
245 pub fn name_mapping(&self) -> Option<&Arc<NameMapping>> {
247 self.name_mapping.as_ref()
248 }
249
250 pub fn unified_partition_type(&self) -> Option<&Arc<StructType>> {
252 self.unified_partition_type.as_ref()
253 }
254
255 pub fn sort_order_id(&self) -> Option<i32> {
257 self.sort_order_id
258 }
259
260 pub fn sort_order(&self) -> Option<&SortOrderRef> {
262 self.sort_order.as_ref()
263 }
264
265 pub fn column_stats(&self) -> Option<&FileScanTaskColumnStats> {
269 self.column_stats.as_deref()
270 }
271
272 pub fn case_sensitive(&self) -> bool {
274 self.case_sensitive
275 }
276
277 pub fn key_metadata(&self) -> Option<&[u8]> {
279 self.key_metadata.as_deref()
280 }
281
282 fn validate(&self) -> Result<()> {
283 match (self.partition.as_ref(), self.partition_spec.as_deref()) {
284 (None, None) => Ok(()),
285 (None, Some(partition_spec)) if partition_spec.is_unpartitioned() => Ok(()),
286 (None, Some(_)) => Err(invalid_data!(
287 "FileScanTask with a partitioned spec requires partition values"
288 )),
289 (Some(partition), None) if partition.fields().is_empty() => Ok(()),
290 (Some(_), None) => Err(invalid_data!(
291 "Non-empty FileScanTask partition requires a partition spec"
292 )),
293 (Some(partition), Some(partition_spec))
294 if partition.fields().len() != partition_spec.fields().len() =>
295 {
296 Err(invalid_data!(
297 "FileScanTask partition has {} fields but partition spec has {} fields",
298 partition.fields().len(),
299 partition_spec.fields().len()
300 ))
301 }
302 (Some(_), Some(partition_spec)) => {
303 partition_spec.partition_type(&self.schema)?;
304 Ok(())
305 }
306 }
307 }
308}
309
310impl From<FileScanTask> for Result<FileScanTask> {
311 fn from(task: FileScanTask) -> Self {
312 task.validate()?;
313 Ok(task)
314 }
315}
316
317#[derive(Debug)]
318pub(crate) struct DeleteFileContext {
319 pub(crate) manifest_entry: ManifestEntryRef,
320 pub(crate) partition_spec_id: i32,
321}
322
323impl TryFrom<&DeleteFileContext> for FileScanTaskDeleteFile {
324 type Error = Error;
325
326 fn try_from(ctx: &DeleteFileContext) -> Result<Self> {
327 let file_path = ctx.manifest_entry.file_path();
331 let to_offset = |value: Option<i64>, field: &str| -> Result<Option<u64>> {
332 value
333 .map(|value| {
334 u64::try_from(value).map_err(|_| {
335 invalid_data!("delete file {file_path} has negative {field} {value}")
336 })
337 })
338 .transpose()
339 };
340
341 FileScanTaskDeleteFile::builder()
342 .with_file_path(ctx.manifest_entry.file_path().to_string())
343 .with_file_size_in_bytes(ctx.manifest_entry.file_size_in_bytes())
344 .with_file_type(ctx.manifest_entry.content_type())
345 .with_file_format(ctx.manifest_entry.data_file().file_format())
346 .with_partition_spec_id(ctx.partition_spec_id)
347 .with_equality_ids(ctx.manifest_entry.data_file.equality_ids.clone())
348 .with_referenced_data_file(ctx.manifest_entry.data_file.referenced_data_file.clone())
349 .with_content_offset(to_offset(
350 ctx.manifest_entry.data_file.content_offset,
351 "content_offset",
352 )?)
353 .with_content_size_in_bytes(to_offset(
354 ctx.manifest_entry.data_file.content_size_in_bytes,
355 "content_size_in_bytes",
356 )?)
357 .with_record_count(Some(ctx.manifest_entry.record_count()))
358 .with_key_metadata(
359 ctx.manifest_entry
360 .data_file
361 .key_metadata
362 .as_deref()
363 .map(Box::from),
364 )
365 .build()
366 }
367}
368
369#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, TypedBuilder)]
371#[serde(try_from = "crate::scan::task::_serde::FileScanTaskDeleteFileSerde")]
372#[builder(
373 field_defaults(setter(prefix = "with_")),
374 build_method(into = Result<FileScanTaskDeleteFile>)
375)]
376pub struct FileScanTaskDeleteFile {
377 file_path: String,
379
380 #[serde(default)]
388 #[serde(skip_serializing_if = "Option::is_none")]
389 #[builder(setter(strip_option(fallback = with_file_size_in_bytes_opt)))]
390 file_size_in_bytes: Option<u64>,
391
392 file_type: DataContentType,
394
395 file_format: DataFileFormat,
398
399 partition_spec_id: i32,
401
402 #[builder(default)]
404 equality_ids: Option<Vec<i32>>,
405
406 #[serde(default)]
409 #[serde(skip_serializing_if = "Option::is_none")]
410 #[builder(default)]
411 referenced_data_file: Option<String>,
412
413 #[serde(default)]
418 #[serde(skip_serializing_if = "Option::is_none")]
419 #[builder(default)]
420 content_offset: Option<u64>,
421
422 #[serde(default)]
426 #[serde(skip_serializing_if = "Option::is_none")]
427 #[builder(default)]
428 content_size_in_bytes: Option<u64>,
429
430 #[serde(default)]
433 #[serde(skip_serializing_if = "Option::is_none")]
434 #[builder(default)]
435 record_count: Option<u64>,
436
437 #[serde(default)]
447 #[serde(skip_serializing_if = "Option::is_none")]
448 #[builder(default)]
449 key_metadata: Option<Box<[u8]>>,
450}
451
452mod _serde {
453 use std::sync::Arc;
454
455 use serde::{Deserialize, Serialize};
456
457 use super::{FileScanTask, FileScanTaskColumnStats, FileScanTaskDeleteFile};
458 use crate::error::invalid_data;
459 use crate::expr::BoundPredicate;
460 use crate::spec::{
461 DataContentType, DataFileFormat, Literal, NameMapping, PartitionSpec, RawLiteral,
462 SchemaRef, SortOrderRef, StructType, Type,
463 };
464 use crate::{Error, Result};
465
466 #[derive(Deserialize)]
467 pub(super) struct FileScanTaskSerde {
468 file_size_in_bytes: u64,
469 start: u64,
470 length: u64,
471 record_count: Option<u64>,
472 first_row_id: Option<i64>,
473 data_sequence_number: Option<i64>,
474 data_file_path: String,
475 data_file_format: DataFileFormat,
476 schema: SchemaRef,
477 project_field_ids: Vec<i32>,
478 predicate: Option<BoundPredicate>,
479 deletes: Vec<FileScanTaskDeleteFile>,
480 #[serde(default)]
481 partition: Option<RawLiteral>,
482 #[serde(default)]
483 partition_spec: Option<Arc<PartitionSpec>>,
484 #[serde(default)]
485 name_mapping: Option<Arc<NameMapping>>,
486 #[serde(default)]
487 unified_partition_type: Option<Arc<StructType>>,
488 #[serde(default)]
489 sort_order_id: Option<i32>,
490 #[serde(default)]
491 sort_order: Option<SortOrderRef>,
492 #[serde(default)]
493 column_stats: Option<FileScanTaskColumnStats>,
494 case_sensitive: bool,
495 #[serde(default)]
496 key_metadata: Option<Box<[u8]>>,
497 }
498
499 #[derive(Serialize)]
500 struct FileScanTaskRefSerde<'a> {
501 file_size_in_bytes: u64,
502 start: u64,
503 length: u64,
504 #[serde(skip_serializing_if = "Option::is_none")]
505 record_count: Option<u64>,
506 #[serde(skip_serializing_if = "Option::is_none")]
507 first_row_id: Option<i64>,
508 #[serde(skip_serializing_if = "Option::is_none")]
509 data_sequence_number: Option<i64>,
510 data_file_path: &'a str,
511 data_file_format: DataFileFormat,
512 schema: &'a SchemaRef,
513 project_field_ids: &'a [i32],
514 #[serde(skip_serializing_if = "Option::is_none")]
515 predicate: Option<&'a BoundPredicate>,
516 deletes: &'a [FileScanTaskDeleteFile],
517 #[serde(skip_serializing_if = "Option::is_none")]
518 partition: Option<RawLiteral>,
519 #[serde(skip_serializing_if = "Option::is_none")]
520 partition_spec: Option<&'a Arc<PartitionSpec>>,
521 #[serde(skip_serializing_if = "Option::is_none")]
522 name_mapping: Option<&'a Arc<NameMapping>>,
523 #[serde(skip_serializing_if = "Option::is_none")]
524 unified_partition_type: Option<&'a Arc<StructType>>,
525 #[serde(skip_serializing_if = "Option::is_none")]
526 sort_order_id: Option<i32>,
527 #[serde(skip_serializing_if = "Option::is_none")]
528 sort_order: Option<&'a SortOrderRef>,
529 #[serde(skip_serializing_if = "Option::is_none")]
530 column_stats: Option<&'a FileScanTaskColumnStats>,
531 case_sensitive: bool,
532 #[serde(skip_serializing_if = "Option::is_none")]
533 key_metadata: Option<&'a [u8]>,
534 }
535
536 fn partition_type(
537 partition_spec: Option<&PartitionSpec>,
538 schema: &crate::spec::Schema,
539 ) -> Result<Type> {
540 let partition_type = match partition_spec {
541 Some(partition_spec) => partition_spec.partition_type(schema)?,
542 None => PartitionSpec::unpartition_spec().partition_type(schema)?,
543 };
544 Ok(Type::Struct(partition_type))
545 }
546
547 impl<'a> TryFrom<&'a FileScanTask> for FileScanTaskRefSerde<'a> {
548 type Error = Error;
549
550 fn try_from(value: &'a FileScanTask) -> Result<Self> {
551 let partition = value
552 .partition
553 .as_ref()
554 .map(|partition| {
555 let partition_type =
556 partition_type(value.partition_spec.as_deref(), &value.schema)?;
557 RawLiteral::try_from(Literal::Struct(partition.clone()), &partition_type)
558 })
559 .transpose()?;
560
561 Ok(Self {
562 file_size_in_bytes: value.file_size_in_bytes,
563 start: value.start,
564 length: value.length,
565 record_count: value.record_count,
566 first_row_id: value.first_row_id,
567 data_sequence_number: value.data_sequence_number,
568 data_file_path: &value.data_file_path,
569 data_file_format: value.data_file_format,
570 schema: &value.schema,
571 project_field_ids: &value.project_field_ids,
572 predicate: value.predicate.as_ref(),
573 deletes: &value.deletes,
574 partition,
575 partition_spec: value.partition_spec.as_ref(),
576 name_mapping: value.name_mapping.as_ref(),
577 unified_partition_type: value.unified_partition_type.as_ref(),
578 sort_order_id: value.sort_order_id,
579 sort_order: value.sort_order.as_ref(),
580 column_stats: value.column_stats.as_deref(),
581 case_sensitive: value.case_sensitive,
582 key_metadata: value.key_metadata.as_deref(),
583 })
584 }
585 }
586
587 impl Serialize for FileScanTask {
588 fn serialize<S>(&self, serializer: S) -> std::result::Result<S::Ok, S::Error>
589 where S: serde::Serializer {
590 FileScanTaskRefSerde::try_from(self)
591 .map_err(serde::ser::Error::custom)?
592 .serialize(serializer)
593 }
594 }
595
596 impl TryFrom<FileScanTaskSerde> for FileScanTask {
597 type Error = Error;
598
599 fn try_from(value: FileScanTaskSerde) -> Result<Self> {
600 let partition = value
601 .partition
602 .map(|partition| {
603 let partition_type =
604 partition_type(value.partition_spec.as_deref(), &value.schema)?;
605 match partition.try_into(&partition_type)? {
606 Some(Literal::Struct(partition)) => Ok(partition),
607 _ => Err(invalid_data!("FileScanTask partition must be a struct")),
608 }
609 })
610 .transpose()?;
611
612 Self::builder()
613 .with_file_size_in_bytes(value.file_size_in_bytes)
614 .with_start(value.start)
615 .with_length(value.length)
616 .with_record_count(value.record_count)
617 .with_first_row_id(value.first_row_id)
618 .with_data_sequence_number(value.data_sequence_number)
619 .with_data_file_path(value.data_file_path)
620 .with_data_file_format(value.data_file_format)
621 .with_schema(value.schema)
622 .with_project_field_ids(value.project_field_ids)
623 .with_predicate(value.predicate)
624 .with_deletes(value.deletes)
625 .with_partition(partition)
626 .with_partition_spec(value.partition_spec)
627 .with_name_mapping(value.name_mapping)
628 .with_unified_partition_type(value.unified_partition_type)
629 .with_sort_order_id(value.sort_order_id)
630 .with_sort_order(value.sort_order)
631 .with_column_stats(value.column_stats)
632 .with_case_sensitive(value.case_sensitive)
633 .with_key_metadata(value.key_metadata)
634 .build()
635 }
636 }
637
638 #[derive(Deserialize)]
639 pub(super) struct FileScanTaskDeleteFileSerde {
640 file_path: String,
641 #[serde(default)]
642 file_size_in_bytes: Option<u64>,
643 file_type: DataContentType,
644 file_format: DataFileFormat,
645 partition_spec_id: i32,
646 equality_ids: Option<Vec<i32>>,
647 #[serde(default)]
648 referenced_data_file: Option<String>,
649 #[serde(default)]
650 content_offset: Option<u64>,
651 #[serde(default)]
652 content_size_in_bytes: Option<u64>,
653 #[serde(default)]
654 record_count: Option<u64>,
655 #[serde(default)]
656 key_metadata: Option<Box<[u8]>>,
657 }
658
659 impl TryFrom<FileScanTaskDeleteFileSerde> for FileScanTaskDeleteFile {
660 type Error = Error;
661
662 fn try_from(value: FileScanTaskDeleteFileSerde) -> Result<Self> {
663 Self::builder()
664 .with_file_path(value.file_path)
665 .with_file_size_in_bytes_opt(value.file_size_in_bytes)
666 .with_file_type(value.file_type)
667 .with_file_format(value.file_format)
668 .with_partition_spec_id(value.partition_spec_id)
669 .with_equality_ids(value.equality_ids)
670 .with_referenced_data_file(value.referenced_data_file)
671 .with_content_offset(value.content_offset)
672 .with_content_size_in_bytes(value.content_size_in_bytes)
673 .with_record_count(value.record_count)
674 .with_key_metadata(value.key_metadata)
675 .build()
676 }
677 }
678
679 pub(super) mod bounds {
682 use std::collections::HashMap;
683
684 use serde::ser::SerializeMap;
685 use serde::{Deserialize, Deserializer, Serialize, Serializer};
686 use serde_bytes::ByteBuf;
687
688 use crate::spec::{Datum, PrimitiveType};
689
690 #[derive(Serialize, Deserialize)]
691 struct BoundSerde {
692 r#type: PrimitiveType,
693 bytes: ByteBuf,
694 }
695
696 pub(in crate::scan::task) fn serialize<S: Serializer>(
697 bounds: &HashMap<i32, Datum>,
698 serializer: S,
699 ) -> Result<S::Ok, S::Error> {
700 let mut map = serializer.serialize_map(Some(bounds.len()))?;
701 for (field_id, datum) in bounds {
702 let bound = BoundSerde {
703 r#type: datum.data_type().clone(),
704 bytes: datum.to_bytes().map_err(serde::ser::Error::custom)?,
705 };
706 map.serialize_entry(field_id, &bound)?;
707 }
708 map.end()
709 }
710
711 pub(in crate::scan::task) fn deserialize<'de, D: Deserializer<'de>>(
712 deserializer: D,
713 ) -> Result<HashMap<i32, Datum>, D::Error> {
714 HashMap::<i32, BoundSerde>::deserialize(deserializer)?
715 .into_iter()
716 .map(|(field_id, bound)| {
717 Datum::try_from_bytes(&bound.bytes, bound.r#type)
718 .map(|datum| (field_id, datum))
719 .map_err(serde::de::Error::custom)
720 })
721 .collect()
722 }
723 }
724}
725
726impl FileScanTaskDeleteFile {
727 pub fn file_path(&self) -> &str {
729 &self.file_path
730 }
731
732 pub fn file_size_in_bytes(&self) -> Option<u64> {
735 self.file_size_in_bytes
736 }
737
738 pub fn file_type(&self) -> DataContentType {
740 self.file_type
741 }
742
743 pub fn file_format(&self) -> DataFileFormat {
745 self.file_format
746 }
747
748 pub fn partition_spec_id(&self) -> i32 {
750 self.partition_spec_id
751 }
752
753 pub fn equality_ids(&self) -> Option<&[i32]> {
755 self.equality_ids.as_deref()
756 }
757
758 pub fn referenced_data_file(&self) -> Option<&str> {
760 self.referenced_data_file.as_deref()
761 }
762
763 pub fn content_offset(&self) -> Option<u64> {
765 self.content_offset
766 }
767
768 pub fn content_size_in_bytes(&self) -> Option<u64> {
770 self.content_size_in_bytes
771 }
772
773 pub fn record_count(&self) -> Option<u64> {
775 self.record_count
776 }
777
778 pub fn key_metadata(&self) -> Option<&[u8]> {
780 self.key_metadata.as_deref()
781 }
782
783 fn is_deletion_vector(&self) -> bool {
784 self.file_type == DataContentType::PositionDeletes
785 && self.file_format == DataFileFormat::Puffin
786 }
787
788 fn validate(&self) -> Result<()> {
789 if self.content_offset.is_some() && self.content_size_in_bytes.is_none() {
795 return Err(invalid_data!(
796 "delete file {} is missing content_size_in_bytes while content_offset is present",
797 self.file_path
798 ));
799 }
800
801 if !self.is_deletion_vector() {
804 return Ok(());
805 }
806
807 let missing = if self.referenced_data_file.is_none() {
808 Some("referenced_data_file")
809 } else if self.content_offset.is_none() {
810 Some("content_offset")
811 } else if self.content_size_in_bytes.is_none() {
812 Some("content_size_in_bytes")
813 } else if self.record_count.is_none() {
814 Some("record_count")
815 } else {
816 None
817 };
818
819 if let Some(field) = missing {
820 return Err(invalid_data!(
821 "deletion vector {} is missing {field}",
822 self.file_path
823 ));
824 }
825
826 Ok(())
827 }
828}
829
830impl From<FileScanTaskDeleteFile> for Result<FileScanTaskDeleteFile> {
831 fn from(task: FileScanTaskDeleteFile) -> Self {
832 task.validate()?;
833 Ok(task)
834 }
835}
836
837#[derive(Debug, Clone, PartialEq, Default, Serialize, Deserialize, TypedBuilder)]
848#[builder(field_defaults(default, setter(prefix = "with_")))]
849pub struct FileScanTaskColumnStats {
850 #[serde(default, skip_serializing_if = "HashMap::is_empty")]
851 column_sizes: HashMap<i32, u64>,
852
853 #[serde(default, skip_serializing_if = "HashMap::is_empty")]
854 value_counts: HashMap<i32, u64>,
855
856 #[serde(default, skip_serializing_if = "HashMap::is_empty")]
857 null_value_counts: HashMap<i32, u64>,
858
859 #[serde(default, skip_serializing_if = "HashMap::is_empty")]
860 nan_value_counts: HashMap<i32, u64>,
861
862 #[serde(
863 default,
864 skip_serializing_if = "HashMap::is_empty",
865 with = "crate::scan::task::_serde::bounds"
866 )]
867 lower_bounds: HashMap<i32, Datum>,
868
869 #[serde(
870 default,
871 skip_serializing_if = "HashMap::is_empty",
872 with = "crate::scan::task::_serde::bounds"
873 )]
874 upper_bounds: HashMap<i32, Datum>,
875}
876
877pub(crate) type ColumnStatsColumns = Arc<[(i32, PrimitiveType)]>;
879
880fn bound_as_column_type(bound: &Datum, column_type: &PrimitiveType) -> Option<Datum> {
882 if bound.data_type() == column_type {
883 return Some(bound.clone());
884 }
885 if let Some(promoted) = bound.promote_to(column_type) {
886 return Some(promoted);
887 }
888 match (bound.data_type(), column_type) {
889 (
890 PrimitiveType::Decimal {
891 precision: from_precision,
892 scale: from_scale,
893 },
894 PrimitiveType::Decimal { precision, scale },
895 ) if from_scale == scale && from_precision <= precision => {
896 Some(Datum::new(column_type.clone(), bound.literal().clone()))
897 }
898 _ => None,
899 }
900}
901
902impl FileScanTaskColumnStats {
903 pub(crate) fn from_data_file(data_file: &DataFile, columns: &[(i32, PrimitiveType)]) -> Self {
904 fn retain<V: Clone>(
905 map: &HashMap<i32, V>,
906 columns: &[(i32, PrimitiveType)],
907 ) -> HashMap<i32, V> {
908 if map.is_empty() {
909 return HashMap::new();
910 }
911 let mut retained = HashMap::with_capacity(columns.len().min(map.len()));
912 retained.extend(
913 columns
914 .iter()
915 .filter_map(|(id, _)| map.get(id).map(|value| (*id, value.clone()))),
916 );
917 retained
918 }
919
920 fn retain_bounds(
921 bounds: &HashMap<i32, Datum>,
922 columns: &[(i32, PrimitiveType)],
923 ) -> HashMap<i32, Datum> {
924 if bounds.is_empty() {
925 return HashMap::new();
926 }
927 let mut retained = HashMap::with_capacity(columns.len().min(bounds.len()));
928 retained.extend(columns.iter().filter_map(|(id, column_type)| {
929 let bound = bound_as_column_type(bounds.get(id)?, column_type)?;
930 Some((*id, bound))
931 }));
932 retained
933 }
934
935 Self {
936 column_sizes: retain(data_file.column_sizes(), columns),
937 value_counts: retain(data_file.value_counts(), columns),
938 null_value_counts: retain(data_file.null_value_counts(), columns),
939 nan_value_counts: retain(data_file.nan_value_counts(), columns),
940 lower_bounds: retain_bounds(data_file.lower_bounds(), columns),
941 upper_bounds: retain_bounds(data_file.upper_bounds(), columns),
942 }
943 }
944
945 pub fn column_sizes(&self) -> &HashMap<i32, u64> {
947 &self.column_sizes
948 }
949
950 pub fn value_counts(&self) -> &HashMap<i32, u64> {
952 &self.value_counts
953 }
954
955 pub fn null_value_counts(&self) -> &HashMap<i32, u64> {
957 &self.null_value_counts
958 }
959
960 pub fn nan_value_counts(&self) -> &HashMap<i32, u64> {
962 &self.nan_value_counts
963 }
964
965 pub fn lower_bounds(&self) -> &HashMap<i32, Datum> {
967 &self.lower_bounds
968 }
969
970 pub fn upper_bounds(&self) -> &HashMap<i32, Datum> {
972 &self.upper_bounds
973 }
974}
975
976#[cfg(test)]
977mod tests {
978 use std::collections::HashMap;
979
980 use super::*;
981 use crate::ErrorKind;
982 use crate::spec::{
983 DataFileBuilder, Literal, ManifestEntry, ManifestStatus, NestedField, PrimitiveLiteral,
984 PrimitiveType, Transform, Type,
985 };
986
987 fn build_file_scan_task(
988 schema: SchemaRef,
989 partition: Option<Struct>,
990 partition_spec: Option<Arc<PartitionSpec>>,
991 ) -> Result<FileScanTask> {
992 FileScanTask::builder()
993 .with_file_size_in_bytes(100)
994 .with_start(0)
995 .with_length(100)
996 .with_data_file_path("data_file_path".to_string())
997 .with_data_file_format(DataFileFormat::Parquet)
998 .with_schema(schema)
999 .with_project_field_ids(vec![])
1000 .with_partition(partition)
1001 .with_partition_spec(partition_spec)
1002 .with_case_sensitive(false)
1003 .build()
1004 }
1005
1006 fn schema_and_spec(
1007 primitive_type: PrimitiveType,
1008 transform: Transform,
1009 ) -> (SchemaRef, Arc<PartitionSpec>) {
1010 let schema = Arc::new(
1011 Schema::builder()
1012 .with_fields(vec![Arc::new(NestedField::required(
1013 1,
1014 "x",
1015 Type::Primitive(primitive_type),
1016 ))])
1017 .build()
1018 .unwrap(),
1019 );
1020 let partition_spec = Arc::new(
1021 PartitionSpec::builder(schema.clone())
1022 .add_partition_field("x", "x_partition", transform)
1023 .unwrap()
1024 .build()
1025 .unwrap(),
1026 );
1027 (schema, partition_spec)
1028 }
1029
1030 fn build_delete_file_task(
1031 file_type: DataContentType,
1032 file_format: DataFileFormat,
1033 ) -> Result<FileScanTaskDeleteFile> {
1034 FileScanTaskDeleteFile::builder()
1035 .with_file_path("delete-file".to_string())
1036 .with_file_size_in_bytes(100)
1037 .with_file_type(file_type)
1038 .with_file_format(file_format)
1039 .with_partition_spec_id(0)
1040 .build()
1041 }
1042
1043 fn assert_delete_file_builder_error(
1044 result: Result<FileScanTaskDeleteFile>,
1045 expected_message: &str,
1046 ) {
1047 match result {
1048 Ok(task) => panic!(
1049 "expected delete file builder to fail with `{expected_message}`, but got Ok({task:?})"
1050 ),
1051 Err(err) => {
1052 assert_eq!(err.kind(), ErrorKind::DataInvalid);
1053 assert_eq!(err.message(), expected_message);
1054 }
1055 }
1056 }
1057
1058 #[test]
1059 fn test_file_scan_task_builder_rejects_non_empty_partition_without_spec() {
1060 let err = build_file_scan_task(
1062 Arc::new(Schema::builder().build().unwrap()),
1063 Some(Struct::from_iter([Some(Literal::long(42))])),
1064 None,
1065 )
1066 .unwrap_err();
1067
1068 assert_eq!(err.kind(), ErrorKind::DataInvalid);
1069 assert_eq!(
1070 err.message(),
1071 "Non-empty FileScanTask partition requires a partition spec"
1072 );
1073 }
1074
1075 #[test]
1076 fn test_file_scan_task_builder_accepts_empty_partition_without_spec() {
1077 build_file_scan_task(
1078 Arc::new(Schema::builder().build().unwrap()),
1079 Some(Struct::empty()),
1080 None,
1081 )
1082 .unwrap();
1083 }
1084
1085 #[test]
1086 fn test_file_scan_task_builder_rejects_partitioned_spec_without_partition() {
1087 let (schema, partition_spec) = schema_and_spec(PrimitiveType::Long, Transform::Identity);
1088
1089 let err = build_file_scan_task(schema, None, Some(partition_spec)).unwrap_err();
1090
1091 assert_eq!(err.kind(), ErrorKind::DataInvalid);
1092 assert_eq!(
1093 err.message(),
1094 "FileScanTask with a partitioned spec requires partition values"
1095 );
1096 }
1097
1098 #[test]
1099 fn test_file_scan_task_builder_accepts_unpartitioned_spec_without_partition() {
1100 build_file_scan_task(
1101 Arc::new(Schema::builder().build().unwrap()),
1102 None,
1103 Some(Arc::new(PartitionSpec::unpartition_spec())),
1104 )
1105 .unwrap();
1106 }
1107
1108 #[test]
1109 fn test_file_scan_task_builder_rejects_partition_arity_mismatch() {
1110 let (schema, partition_spec) = schema_and_spec(PrimitiveType::Long, Transform::Identity);
1111
1112 let err =
1113 build_file_scan_task(schema, Some(Struct::empty()), Some(partition_spec)).unwrap_err();
1114
1115 assert_eq!(err.kind(), ErrorKind::DataInvalid);
1116 assert!(err.message().contains("partition has 0 fields"));
1117 assert!(err.message().contains("partition spec has 1 fields"));
1118 }
1119
1120 #[test]
1121 fn test_file_scan_task_builder_accepts_dropped_partition_source_column() {
1122 let (_historical_schema, partition_spec) =
1123 schema_and_spec(PrimitiveType::Long, Transform::Identity);
1124 let current_schema = Arc::new(
1125 Schema::builder()
1126 .with_fields(vec![Arc::new(NestedField::required(
1127 2,
1128 "y",
1129 Type::Primitive(PrimitiveType::String),
1130 ))])
1131 .build()
1132 .unwrap(),
1133 );
1134
1135 build_file_scan_task(
1136 current_schema,
1137 Some(Struct::from_iter([Some(Literal::long(42))])),
1138 Some(partition_spec),
1139 )
1140 .unwrap();
1141 }
1142
1143 #[test]
1144 fn test_file_scan_task_builder_rejects_partition_spec_incompatible_with_schema() {
1145 let (_historical_schema, partition_spec) =
1146 schema_and_spec(PrimitiveType::Timestamp, Transform::Day);
1147 let current_schema = Arc::new(
1148 Schema::builder()
1149 .with_fields(vec![Arc::new(NestedField::required(
1150 1,
1151 "x",
1152 Type::Primitive(PrimitiveType::String),
1153 ))])
1154 .build()
1155 .unwrap(),
1156 );
1157
1158 let err = build_file_scan_task(
1159 current_schema,
1160 Some(Struct::from_iter([Some(Literal::date(20_000))])),
1161 Some(partition_spec),
1162 )
1163 .unwrap_err();
1164
1165 assert_eq!(err.kind(), ErrorKind::DataInvalid);
1166 }
1167
1168 #[test]
1169 fn test_delete_file_builder_accepts_valid_deletion_vector() {
1170 let task = FileScanTaskDeleteFile::builder()
1171 .with_file_path("dv.puffin".to_string())
1172 .with_file_size_in_bytes(100)
1173 .with_file_type(DataContentType::PositionDeletes)
1174 .with_file_format(DataFileFormat::Puffin)
1175 .with_partition_spec_id(7)
1176 .with_referenced_data_file(Some("data.parquet".to_string()))
1177 .with_content_offset(Some(11))
1178 .with_content_size_in_bytes(Some(13))
1179 .with_record_count(Some(3))
1180 .with_key_metadata(Some(vec![17, 19].into_boxed_slice()))
1181 .build()
1182 .unwrap();
1183
1184 assert_eq!(task.file_path(), "dv.puffin");
1185 assert_eq!(task.file_size_in_bytes(), Some(100));
1186 assert_eq!(task.file_type(), DataContentType::PositionDeletes);
1187 assert_eq!(task.file_format(), DataFileFormat::Puffin);
1188 assert_eq!(task.partition_spec_id(), 7);
1189 assert_eq!(task.equality_ids(), None);
1190 assert_eq!(task.referenced_data_file(), Some("data.parquet"));
1191 assert_eq!(task.content_offset(), Some(11));
1192 assert_eq!(task.content_size_in_bytes(), Some(13));
1193 assert_eq!(task.record_count(), Some(3));
1194 assert_eq!(task.key_metadata(), Some([17, 19].as_slice()));
1195 }
1196
1197 #[test]
1198 fn test_delete_file_builder_rejects_dv_missing_referenced_data_file() {
1199 assert_delete_file_builder_error(
1200 FileScanTaskDeleteFile::builder()
1201 .with_file_path("dv.puffin".to_string())
1202 .with_file_size_in_bytes(100)
1203 .with_file_type(DataContentType::PositionDeletes)
1204 .with_file_format(DataFileFormat::Puffin)
1205 .with_partition_spec_id(0)
1206 .with_content_offset(Some(7))
1207 .with_content_size_in_bytes(Some(11))
1208 .with_record_count(Some(3))
1209 .build(),
1210 "deletion vector dv.puffin is missing referenced_data_file",
1211 );
1212 }
1213
1214 #[test]
1215 fn test_delete_file_builder_rejects_dv_missing_content_offset() {
1216 assert_delete_file_builder_error(
1217 FileScanTaskDeleteFile::builder()
1218 .with_file_path("dv.puffin".to_string())
1219 .with_file_size_in_bytes(100)
1220 .with_file_type(DataContentType::PositionDeletes)
1221 .with_file_format(DataFileFormat::Puffin)
1222 .with_partition_spec_id(0)
1223 .with_referenced_data_file(Some("data.parquet".to_string()))
1224 .with_content_size_in_bytes(Some(11))
1225 .with_record_count(Some(3))
1226 .build(),
1227 "deletion vector dv.puffin is missing content_offset",
1228 );
1229 }
1230
1231 #[test]
1232 fn test_delete_file_builder_rejects_dv_missing_content_size() {
1233 assert_delete_file_builder_error(
1234 FileScanTaskDeleteFile::builder()
1235 .with_file_path("dv.puffin".to_string())
1236 .with_file_size_in_bytes(100)
1237 .with_file_type(DataContentType::PositionDeletes)
1238 .with_file_format(DataFileFormat::Puffin)
1239 .with_partition_spec_id(0)
1240 .with_referenced_data_file(Some("data.parquet".to_string()))
1241 .with_content_offset(Some(7))
1242 .with_record_count(Some(3))
1243 .build(),
1244 "delete file dv.puffin is missing content_size_in_bytes while content_offset is present",
1245 );
1246 }
1247
1248 #[test]
1249 fn test_delete_file_builder_rejects_dv_missing_record_count() {
1250 assert_delete_file_builder_error(
1251 FileScanTaskDeleteFile::builder()
1252 .with_file_path("dv.puffin".to_string())
1253 .with_file_size_in_bytes(100)
1254 .with_file_type(DataContentType::PositionDeletes)
1255 .with_file_format(DataFileFormat::Puffin)
1256 .with_partition_spec_id(0)
1257 .with_referenced_data_file(Some("data.parquet".to_string()))
1258 .with_content_offset(Some(7))
1259 .with_content_size_in_bytes(Some(11))
1260 .build(),
1261 "deletion vector dv.puffin is missing record_count",
1262 );
1263 }
1264
1265 #[test]
1266 fn test_delete_file_builder_rejects_non_dv_offset_without_size() {
1267 assert_delete_file_builder_error(
1271 FileScanTaskDeleteFile::builder()
1272 .with_file_path("position-deletes.parquet".to_string())
1273 .with_file_size_in_bytes(100)
1274 .with_file_type(DataContentType::PositionDeletes)
1275 .with_file_format(DataFileFormat::Parquet)
1276 .with_partition_spec_id(0)
1277 .with_content_offset(Some(7))
1278 .build(),
1279 "delete file position-deletes.parquet is missing content_size_in_bytes while content_offset is present",
1280 );
1281 }
1282
1283 fn delete_manifest_entry(
1284 file_path: &str,
1285 file_format: DataFileFormat,
1286 content_offset: Option<i64>,
1287 content_size_in_bytes: Option<i64>,
1288 ) -> DeleteFileContext {
1289 let mut data_file = DataFileBuilder::default()
1290 .content(DataContentType::PositionDeletes)
1291 .file_path(file_path.to_string())
1292 .file_format(file_format)
1293 .partition(Struct::empty())
1294 .record_count(3)
1295 .file_size_in_bytes(100)
1296 .column_sizes(HashMap::new())
1297 .value_counts(HashMap::new())
1298 .null_value_counts(HashMap::new())
1299 .partition_spec_id(0)
1300 .referenced_data_file(Some("data.parquet".to_string()))
1301 .build()
1302 .unwrap();
1303 data_file.content_offset = content_offset;
1304 data_file.content_size_in_bytes = content_size_in_bytes;
1305
1306 DeleteFileContext {
1307 manifest_entry: Arc::new(
1308 ManifestEntry::builder()
1309 .status(ManifestStatus::Added)
1310 .data_file(data_file)
1311 .build(),
1312 ),
1313 partition_spec_id: 0,
1314 }
1315 }
1316
1317 fn assert_delete_context_error(ctx: &DeleteFileContext, expected_message: &str) {
1318 match FileScanTaskDeleteFile::try_from(ctx) {
1319 Ok(task) => panic!(
1320 "expected conversion to fail with `{expected_message}`, but got Ok({task:?})"
1321 ),
1322 Err(error) => {
1323 assert_eq!(error.kind(), ErrorKind::DataInvalid);
1324 assert!(
1325 error.message().contains(expected_message),
1326 "expected `{expected_message}`, got `{}`",
1327 error.message()
1328 );
1329 }
1330 }
1331 }
1332
1333 #[test]
1337 fn test_delete_context_rejects_negative_dv_offset() {
1338 assert_delete_context_error(
1339 &delete_manifest_entry("dv.puffin", DataFileFormat::Puffin, Some(-1), Some(11)),
1340 "delete file dv.puffin has negative content_offset -1",
1341 );
1342 }
1343
1344 #[test]
1345 fn test_delete_context_rejects_negative_dv_size() {
1346 assert_delete_context_error(
1347 &delete_manifest_entry("dv.puffin", DataFileFormat::Puffin, Some(7), Some(-1)),
1348 "delete file dv.puffin has negative content_size_in_bytes -1",
1349 );
1350 }
1351
1352 #[test]
1355 fn test_delete_context_rejects_negative_coordinates_for_non_dv() {
1356 assert_delete_context_error(
1357 &delete_manifest_entry(
1358 "position-deletes.parquet",
1359 DataFileFormat::Parquet,
1360 Some(-1),
1361 None,
1362 ),
1363 "delete file position-deletes.parquet has negative content_offset -1",
1364 );
1365 assert_delete_context_error(
1366 &delete_manifest_entry(
1367 "position-deletes.parquet",
1368 DataFileFormat::Parquet,
1369 None,
1370 Some(-1),
1371 ),
1372 "delete file position-deletes.parquet has negative content_size_in_bytes -1",
1373 );
1374 }
1375
1376 #[test]
1377 fn test_bound_as_column_type() {
1378 let decimal = |precision, scale| PrimitiveType::Decimal { precision, scale };
1379 let decimal_datum =
1380 |precision, scale| Datum::new(decimal(precision, scale), PrimitiveLiteral::Int128(150));
1381 let cases = [
1382 (Datum::int(-5), PrimitiveType::Long, Some(Datum::long(-5))),
1383 (
1384 Datum::float(f32::NEG_INFINITY),
1385 PrimitiveType::Double,
1386 Some(Datum::double(f64::NEG_INFINITY)),
1387 ),
1388 (
1389 Datum::float(1.5_f32),
1390 PrimitiveType::Double,
1391 Some(Datum::double(1.5)),
1392 ),
1393 (
1394 Datum::string("a"),
1395 PrimitiveType::String,
1396 Some(Datum::string("a")),
1397 ),
1398 (
1399 decimal_datum(9, 2),
1400 decimal(18, 2),
1401 Some(decimal_datum(18, 2)),
1402 ),
1403 (decimal_datum(9, 2), decimal(18, 3), None),
1404 (decimal_datum(38, 2), decimal(18, 2), None),
1405 (Datum::long(6), PrimitiveType::Int, None),
1406 (Datum::string("a"), PrimitiveType::Long, None),
1407 ];
1408
1409 for (bound, column_type, expected) in cases {
1410 assert_eq!(
1411 bound_as_column_type(&bound, &column_type),
1412 expected,
1413 "{bound:?} as {column_type:?}"
1414 );
1415 }
1416 }
1417
1418 #[test]
1419 fn test_delete_file_builder_accepts_non_dv_delete_without_dv_fields() {
1420 build_delete_file_task(DataContentType::PositionDeletes, DataFileFormat::Parquet).unwrap();
1421 build_delete_file_task(DataContentType::EqualityDeletes, DataFileFormat::Parquet).unwrap();
1422 }
1423}