Skip to main content

iceberg/scan/
task.rs

1// Licensed to the Apache Software Foundation (ASF) under one
2// or more contributor license agreements.  See the NOTICE file
3// distributed with this work for additional information
4// regarding copyright ownership.  The ASF licenses this file
5// to you under the Apache License, Version 2.0 (the
6// "License"); you may not use this file except in compliance
7// with the License.  You may obtain a copy of the License at
8//
9//   http://www.apache.org/licenses/LICENSE-2.0
10//
11// Unless required by applicable law or agreed to in writing,
12// software distributed under the License is distributed on an
13// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14// KIND, either express or implied.  See the License for the
15// specific language governing permissions and limitations
16// under the License.
17
18use std::collections::HashMap;
19use std::sync::Arc;
20
21use futures::stream::BoxStream;
22use serde::{Deserialize, Serialize};
23use typed_builder::TypedBuilder;
24
25use crate::error::invalid_data;
26use crate::expr::BoundPredicate;
27use crate::spec::{
28    DataContentType, DataFile, DataFileFormat, Datum, ManifestEntryRef, NameMapping, PartitionSpec,
29    PrimitiveType, Schema, SchemaRef, SortOrderRef, Struct, StructType,
30};
31use crate::{Error, Result};
32
33/// A stream of [`FileScanTask`].
34pub type FileScanTaskStream = BoxStream<'static, Result<FileScanTask>>;
35
36/// A task to scan part of file.
37#[derive(Debug, Clone, Deserialize, PartialEq, TypedBuilder)]
38#[serde(try_from = "crate::scan::task::_serde::FileScanTaskSerde")]
39#[builder(
40    field_defaults(setter(prefix = "with_")),
41    build_method(into = Result<FileScanTask>)
42)]
43pub struct FileScanTask {
44    /// The total size of the data file in bytes, from the manifest entry.
45    /// Used to skip a stat/HEAD request when reading Parquet footers.
46    file_size_in_bytes: u64,
47    /// The start offset of the file to scan.
48    start: u64,
49    /// The length of the file to scan.
50    length: u64,
51    /// The number of records in the file to scan.
52    ///
53    /// This is an optional field, and only available if we are
54    /// reading the entire data file.
55    #[builder(default)]
56    record_count: Option<u64>,
57
58    /// The first row id assigned to the data file.
59    ///
60    /// Used to derive the `_row_id` metadata column: for a row without an
61    /// explicit `_row_id`, it is this value plus the row's ordinal position.
62    #[builder(default)]
63    first_row_id: Option<i64>,
64
65    /// The data sequence number of the file, as opposed to its file sequence
66    /// number: the sequence number preserved when a file is carried forward
67    /// across a rewrite. May be null for an existing entry in a malformed
68    /// manifest that lacks one.
69    ///
70    /// Used to derive the `_last_updated_sequence_number` metadata column.
71    #[builder(default)]
72    data_sequence_number: Option<i64>,
73
74    /// The data file path corresponding to the task.
75    data_file_path: String,
76
77    /// The format of the file to scan.
78    data_file_format: DataFileFormat,
79
80    /// The schema of the file to scan.
81    schema: SchemaRef,
82    /// The field ids to project.
83    project_field_ids: Vec<i32>,
84    /// The predicate to filter.
85    #[builder(default)]
86    predicate: Option<BoundPredicate>,
87
88    /// The list of delete files that may need to be applied to this data file
89    #[builder(default)]
90    deletes: Vec<FileScanTaskDeleteFile>,
91
92    /// Partition data from the manifest entry, used to identify which columns can use
93    /// constant values from partition metadata vs. reading from the data file.
94    /// Per the Iceberg spec, only identity-transformed partition fields should use constants.
95    #[builder(default)]
96    partition: Option<Struct>,
97
98    /// The partition spec for this file, used to distinguish identity transforms
99    /// (which use partition metadata constants) from non-identity transforms like
100    /// bucket/truncate (which must read source columns from the data file).
101    #[builder(default)]
102    partition_spec: Option<Arc<PartitionSpec>>,
103
104    /// Name mapping from table metadata (property: schema.name-mapping.default),
105    /// used to resolve field IDs from column names when Parquet files lack field IDs
106    /// or have field ID conflicts.
107    #[builder(default)]
108    name_mapping: Option<Arc<NameMapping>>,
109
110    /// The unified partition type across all specs in the table.
111    /// When `RESERVED_FIELD_ID_PARTITION` is in the projected field IDs, the reader
112    /// uses this type along with the task's partition_spec and partition data to
113    /// materialize the `_partition` struct column at read time.
114    ///
115    /// This is a table-level value (same for all tasks in a scan), stored per-task
116    /// so that readers are self-contained without needing back-pointers to table
117    /// metadata. The cost is one Arc clone per task.
118    #[builder(default)]
119    unified_partition_type: Option<Arc<StructType>>,
120
121    /// The raw `sort_order_id` recorded on the data file (spec field 140), carried through
122    /// unresolved. See [`DataFile::sort_order_id()`](crate::spec::DataFile::sort_order_id).
123    /// `None` if the file has no sort order id.
124    ///
125    /// This preserves the distinction that [`sort_order`](Self::sort_order) collapses: a
126    /// non-zero id that does not resolve against the table's sort orders still marks the file
127    /// as physically sorted, which a consumer such as the DataFusion sorted-scan optimizer
128    /// needs even when the order definition itself is unavailable.
129    #[builder(default)]
130    sort_order_id: Option<i32>,
131
132    /// The sort order that this file's rows are sorted by, resolved from the data file's
133    /// [`sort_order_id`](Self::sort_order_id) against the table's known sort orders. `Some`
134    /// only when the id resolves to an order that has sort fields. `None` if the file has no
135    /// sort order id, the id does not resolve, or it resolves to an order with no sort fields
136    /// (the reserved unsorted order, id 0 per the spec). Use [`sort_order_id`](Self::sort_order_id)
137    /// to tell those cases apart.
138    #[builder(default)]
139    sort_order: Option<SortOrderRef>,
140
141    #[builder(default, setter(transform = |stats: Option<FileScanTaskColumnStats>| stats.map(Box::new)))]
142    column_stats: Option<Box<FileScanTaskColumnStats>>,
143
144    /// Whether this scan task should treat column names as case-sensitive when binding predicates.
145    case_sensitive: bool,
146
147    /// Key metadata for encrypted data files (Parquet Modular Encryption).
148    /// When present, the reader uses this to build `FileDecryptionProperties`.
149    ///
150    /// Note on the trust boundary: for the standard encryption scheme this
151    /// carries `StandardKeyMetadata`, whose payload is the *plaintext* DEK.
152    /// Because `FileScanTask` implements [`Serialize`], that plaintext DEK is part
153    /// of the serialized scan plan should these tasks ever be serialized and sent
154    /// over the network.
155    #[builder(default)]
156    key_metadata: Option<Box<[u8]>>,
157}
158
159impl FileScanTask {
160    /// Returns the total size of the data file in bytes.
161    pub fn file_size_in_bytes(&self) -> u64 {
162        self.file_size_in_bytes
163    }
164
165    /// Returns the start offset of the file to scan.
166    pub fn start(&self) -> u64 {
167        self.start
168    }
169
170    /// Returns the length of the file to scan.
171    pub fn length(&self) -> u64 {
172        self.length
173    }
174
175    /// Returns the number of records in the file when the whole file is scanned.
176    pub fn record_count(&self) -> Option<u64> {
177        self.record_count
178    }
179
180    /// Returns the first row id assigned to the data file.
181    pub fn first_row_id(&self) -> Option<i64> {
182        self.first_row_id
183    }
184
185    /// Returns the data sequence number of the file.
186    pub fn data_sequence_number(&self) -> Option<i64> {
187        self.data_sequence_number
188    }
189
190    /// Returns the data file path of this file scan task.
191    pub fn data_file_path(&self) -> &str {
192        &self.data_file_path
193    }
194
195    /// Returns the format of the data file.
196    pub fn data_file_format(&self) -> DataFileFormat {
197        self.data_file_format
198    }
199
200    /// Returns the schema of this file scan task as a reference.
201    pub fn schema(&self) -> &Schema {
202        &self.schema
203    }
204
205    /// Returns the schema of this file scan task as a [`SchemaRef`].
206    pub fn schema_ref(&self) -> SchemaRef {
207        self.schema.clone()
208    }
209
210    /// Returns the project field id of this file scan task.
211    pub fn project_field_ids(&self) -> &[i32] {
212        &self.project_field_ids
213    }
214
215    /// Returns the predicate of this file scan task.
216    pub fn predicate(&self) -> Option<&BoundPredicate> {
217        self.predicate.as_ref()
218    }
219
220    /// Clears the row predicate of this file scan task.
221    ///
222    /// The COW rewrite path uses this after candidate selection: candidates are
223    /// chosen with the predicate during planning, but each chosen file must then
224    /// be read in full so the rewrite sees every surviving row. Delete-file
225    /// application is unaffected.
226    pub(crate) fn clear_predicate(&mut self) {
227        self.predicate = None;
228    }
229
230    /// Returns the delete files that may need to be applied to the data file.
231    pub fn deletes(&self) -> &[FileScanTaskDeleteFile] {
232        &self.deletes
233    }
234
235    /// Returns the partition data from the manifest entry.
236    pub fn partition(&self) -> Option<&Struct> {
237        self.partition.as_ref()
238    }
239
240    /// Returns the partition spec for the data file.
241    pub fn partition_spec(&self) -> Option<&Arc<PartitionSpec>> {
242        self.partition_spec.as_ref()
243    }
244
245    /// Returns the name mapping used to resolve field ids.
246    pub fn name_mapping(&self) -> Option<&Arc<NameMapping>> {
247        self.name_mapping.as_ref()
248    }
249
250    /// Returns the unified partition type across all table partition specs.
251    pub fn unified_partition_type(&self) -> Option<&Arc<StructType>> {
252        self.unified_partition_type.as_ref()
253    }
254
255    /// Returns the raw sort order id recorded on the data file, unresolved.
256    pub fn sort_order_id(&self) -> Option<i32> {
257        self.sort_order_id
258    }
259
260    /// Returns the sort order this file's rows are sorted by, if resolved.
261    pub fn sort_order(&self) -> Option<&SortOrderRef> {
262        self.sort_order.as_ref()
263    }
264
265    /// Returns the data file's column statistics for the columns requested with
266    /// [`TableScanBuilder::with_column_stats`](crate::scan::TableScanBuilder::with_column_stats),
267    /// or `None` if none were requested.
268    pub fn column_stats(&self) -> Option<&FileScanTaskColumnStats> {
269        self.column_stats.as_deref()
270    }
271
272    /// Returns whether names are treated as case-sensitive.
273    pub fn case_sensitive(&self) -> bool {
274        self.case_sensitive
275    }
276
277    /// Returns the key metadata for the encrypted data file.
278    pub fn key_metadata(&self) -> Option<&[u8]> {
279        self.key_metadata.as_deref()
280    }
281
282    fn validate(&self) -> Result<()> {
283        match (self.partition.as_ref(), self.partition_spec.as_deref()) {
284            (None, None) => Ok(()),
285            (None, Some(partition_spec)) if partition_spec.is_unpartitioned() => Ok(()),
286            (None, Some(_)) => Err(invalid_data!(
287                "FileScanTask with a partitioned spec requires partition values"
288            )),
289            (Some(partition), None) if partition.fields().is_empty() => Ok(()),
290            (Some(_), None) => Err(invalid_data!(
291                "Non-empty FileScanTask partition requires a partition spec"
292            )),
293            (Some(partition), Some(partition_spec))
294                if partition.fields().len() != partition_spec.fields().len() =>
295            {
296                Err(invalid_data!(
297                    "FileScanTask partition has {} fields but partition spec has {} fields",
298                    partition.fields().len(),
299                    partition_spec.fields().len()
300                ))
301            }
302            (Some(_), Some(partition_spec)) => {
303                partition_spec.partition_type(&self.schema)?;
304                Ok(())
305            }
306        }
307    }
308}
309
310impl From<FileScanTask> for Result<FileScanTask> {
311    fn from(task: FileScanTask) -> Self {
312        task.validate()?;
313        Ok(task)
314    }
315}
316
317#[derive(Debug)]
318pub(crate) struct DeleteFileContext {
319    pub(crate) manifest_entry: ManifestEntryRef,
320    pub(crate) partition_spec_id: i32,
321}
322
323impl TryFrom<&DeleteFileContext> for FileScanTaskDeleteFile {
324    type Error = Error;
325
326    fn try_from(ctx: &DeleteFileContext) -> Result<Self> {
327        // The manifest stores these as i64. Convert here so a negative byte offset is
328        // rejected once at the boundary, for every delete-file kind rather than only
329        // deletion vectors, instead of being carried inward and re-checked per consumer.
330        let file_path = ctx.manifest_entry.file_path();
331        let to_offset = |value: Option<i64>, field: &str| -> Result<Option<u64>> {
332            value
333                .map(|value| {
334                    u64::try_from(value).map_err(|_| {
335                        invalid_data!("delete file {file_path} has negative {field} {value}")
336                    })
337                })
338                .transpose()
339        };
340
341        FileScanTaskDeleteFile::builder()
342            .with_file_path(ctx.manifest_entry.file_path().to_string())
343            .with_file_size_in_bytes(ctx.manifest_entry.file_size_in_bytes())
344            .with_file_type(ctx.manifest_entry.content_type())
345            .with_file_format(ctx.manifest_entry.data_file().file_format())
346            .with_partition_spec_id(ctx.partition_spec_id)
347            .with_equality_ids(ctx.manifest_entry.data_file.equality_ids.clone())
348            .with_referenced_data_file(ctx.manifest_entry.data_file.referenced_data_file.clone())
349            .with_content_offset(to_offset(
350                ctx.manifest_entry.data_file.content_offset,
351                "content_offset",
352            )?)
353            .with_content_size_in_bytes(to_offset(
354                ctx.manifest_entry.data_file.content_size_in_bytes,
355                "content_size_in_bytes",
356            )?)
357            .with_record_count(Some(ctx.manifest_entry.record_count()))
358            .with_key_metadata(
359                ctx.manifest_entry
360                    .data_file
361                    .key_metadata
362                    .as_deref()
363                    .map(Box::from),
364            )
365            .build()
366    }
367}
368
369/// A task to scan part of file.
370#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, TypedBuilder)]
371#[serde(try_from = "crate::scan::task::_serde::FileScanTaskDeleteFileSerde")]
372#[builder(
373    field_defaults(setter(prefix = "with_")),
374    build_method(into = Result<FileScanTaskDeleteFile>)
375)]
376pub struct FileScanTaskDeleteFile {
377    /// The delete file path
378    file_path: String,
379
380    /// The total size of the delete file in bytes. Tasks planned from manifest entries carry
381    /// the recorded size. An externally constructed task may pass `None` when the size is
382    /// unavailable or the recorded size can't be trusted, such as position delete files
383    /// rewritten by `rewrite_table_path` before Iceberg Java 1.12.0
384    /// (<https://github.com/apache/iceberg/issues/12554>). The Parquet delete-file loader then
385    /// resolves it with one metadata request when the file is first loaded. Deletion vectors
386    /// do not use it.
387    #[serde(default)]
388    #[serde(skip_serializing_if = "Option::is_none")]
389    #[builder(setter(strip_option(fallback = with_file_size_in_bytes_opt)))]
390    file_size_in_bytes: Option<u64>,
391
392    /// delete file type
393    file_type: DataContentType,
394
395    /// The delete file's format, from the manifest entry. A `PositionDeletes` entry written as
396    /// `Puffin` is a V3 deletion vector; one written as `Parquet` is a position delete file.
397    file_format: DataFileFormat,
398
399    /// partition id
400    partition_spec_id: i32,
401
402    /// equality ids for equality deletes (null for anything other than equality-deletes)
403    #[builder(default)]
404    equality_ids: Option<Vec<i32>>,
405
406    /// For a deletion vector, the location of the data file whose rows it deletes. Required for
407    /// deletion vectors, and may also be set on a position delete file scoped to one data file.
408    #[serde(default)]
409    #[serde(skip_serializing_if = "Option::is_none")]
410    #[builder(default)]
411    referenced_data_file: Option<String>,
412
413    /// For a deletion vector, the offset of the blob within its Puffin file. Set only for
414    /// deletion vectors, where it locates the blob for direct access. Unsigned because it is a
415    /// byte offset: a negative value is rejected once, when the task is built from a manifest
416    /// entry, rather than re-checked by every consumer.
417    #[serde(default)]
418    #[serde(skip_serializing_if = "Option::is_none")]
419    #[builder(default)]
420    content_offset: Option<u64>,
421
422    /// For a deletion vector, the length in bytes of the blob within its Puffin file.
423    /// Required together with `content_offset`; both are absent for non-DV delete files.
424    /// Unsigned for the same reason as `content_offset`.
425    #[serde(default)]
426    #[serde(skip_serializing_if = "Option::is_none")]
427    #[builder(default)]
428    content_size_in_bytes: Option<u64>,
429
430    /// The number of records in the delete file, from the manifest entry; for a deletion vector,
431    /// the cardinality of its bitmap. `None` only for a task not built from a manifest entry.
432    #[serde(default)]
433    #[serde(skip_serializing_if = "Option::is_none")]
434    #[builder(default)]
435    record_count: Option<u64>,
436
437    /// Key metadata for an encrypted delete file. When present, the reader uses this to
438    /// decrypt the file: for a Parquet equality or position delete file, this builds
439    /// `FileDecryptionProperties` (Parquet Modular Encryption); for a deletion vector, whose
440    /// Puffin file has no native encryption, this wraps the range read in an
441    /// `EncryptedInputFile` (AGS1 stream encryption).
442    ///
443    /// Same plaintext-DEK trust boundary as [`FileScanTask::key_metadata`]:
444    /// this is serialized into the scan plan and crosses the planner -> worker
445    /// channel in the clear for the standard encryption scheme.
446    #[serde(default)]
447    #[serde(skip_serializing_if = "Option::is_none")]
448    #[builder(default)]
449    key_metadata: Option<Box<[u8]>>,
450}
451
452mod _serde {
453    use std::sync::Arc;
454
455    use serde::{Deserialize, Serialize};
456
457    use super::{FileScanTask, FileScanTaskColumnStats, FileScanTaskDeleteFile};
458    use crate::error::invalid_data;
459    use crate::expr::BoundPredicate;
460    use crate::spec::{
461        DataContentType, DataFileFormat, Literal, NameMapping, PartitionSpec, RawLiteral,
462        SchemaRef, SortOrderRef, StructType, Type,
463    };
464    use crate::{Error, Result};
465
466    #[derive(Deserialize)]
467    pub(super) struct FileScanTaskSerde {
468        file_size_in_bytes: u64,
469        start: u64,
470        length: u64,
471        record_count: Option<u64>,
472        first_row_id: Option<i64>,
473        data_sequence_number: Option<i64>,
474        data_file_path: String,
475        data_file_format: DataFileFormat,
476        schema: SchemaRef,
477        project_field_ids: Vec<i32>,
478        predicate: Option<BoundPredicate>,
479        deletes: Vec<FileScanTaskDeleteFile>,
480        #[serde(default)]
481        partition: Option<RawLiteral>,
482        #[serde(default)]
483        partition_spec: Option<Arc<PartitionSpec>>,
484        #[serde(default)]
485        name_mapping: Option<Arc<NameMapping>>,
486        #[serde(default)]
487        unified_partition_type: Option<Arc<StructType>>,
488        #[serde(default)]
489        sort_order_id: Option<i32>,
490        #[serde(default)]
491        sort_order: Option<SortOrderRef>,
492        #[serde(default)]
493        column_stats: Option<FileScanTaskColumnStats>,
494        case_sensitive: bool,
495        #[serde(default)]
496        key_metadata: Option<Box<[u8]>>,
497    }
498
499    #[derive(Serialize)]
500    struct FileScanTaskRefSerde<'a> {
501        file_size_in_bytes: u64,
502        start: u64,
503        length: u64,
504        #[serde(skip_serializing_if = "Option::is_none")]
505        record_count: Option<u64>,
506        #[serde(skip_serializing_if = "Option::is_none")]
507        first_row_id: Option<i64>,
508        #[serde(skip_serializing_if = "Option::is_none")]
509        data_sequence_number: Option<i64>,
510        data_file_path: &'a str,
511        data_file_format: DataFileFormat,
512        schema: &'a SchemaRef,
513        project_field_ids: &'a [i32],
514        #[serde(skip_serializing_if = "Option::is_none")]
515        predicate: Option<&'a BoundPredicate>,
516        deletes: &'a [FileScanTaskDeleteFile],
517        #[serde(skip_serializing_if = "Option::is_none")]
518        partition: Option<RawLiteral>,
519        #[serde(skip_serializing_if = "Option::is_none")]
520        partition_spec: Option<&'a Arc<PartitionSpec>>,
521        #[serde(skip_serializing_if = "Option::is_none")]
522        name_mapping: Option<&'a Arc<NameMapping>>,
523        #[serde(skip_serializing_if = "Option::is_none")]
524        unified_partition_type: Option<&'a Arc<StructType>>,
525        #[serde(skip_serializing_if = "Option::is_none")]
526        sort_order_id: Option<i32>,
527        #[serde(skip_serializing_if = "Option::is_none")]
528        sort_order: Option<&'a SortOrderRef>,
529        #[serde(skip_serializing_if = "Option::is_none")]
530        column_stats: Option<&'a FileScanTaskColumnStats>,
531        case_sensitive: bool,
532        #[serde(skip_serializing_if = "Option::is_none")]
533        key_metadata: Option<&'a [u8]>,
534    }
535
536    fn partition_type(
537        partition_spec: Option<&PartitionSpec>,
538        schema: &crate::spec::Schema,
539    ) -> Result<Type> {
540        let partition_type = match partition_spec {
541            Some(partition_spec) => partition_spec.partition_type(schema)?,
542            None => PartitionSpec::unpartition_spec().partition_type(schema)?,
543        };
544        Ok(Type::Struct(partition_type))
545    }
546
547    impl<'a> TryFrom<&'a FileScanTask> for FileScanTaskRefSerde<'a> {
548        type Error = Error;
549
550        fn try_from(value: &'a FileScanTask) -> Result<Self> {
551            let partition = value
552                .partition
553                .as_ref()
554                .map(|partition| {
555                    let partition_type =
556                        partition_type(value.partition_spec.as_deref(), &value.schema)?;
557                    RawLiteral::try_from(Literal::Struct(partition.clone()), &partition_type)
558                })
559                .transpose()?;
560
561            Ok(Self {
562                file_size_in_bytes: value.file_size_in_bytes,
563                start: value.start,
564                length: value.length,
565                record_count: value.record_count,
566                first_row_id: value.first_row_id,
567                data_sequence_number: value.data_sequence_number,
568                data_file_path: &value.data_file_path,
569                data_file_format: value.data_file_format,
570                schema: &value.schema,
571                project_field_ids: &value.project_field_ids,
572                predicate: value.predicate.as_ref(),
573                deletes: &value.deletes,
574                partition,
575                partition_spec: value.partition_spec.as_ref(),
576                name_mapping: value.name_mapping.as_ref(),
577                unified_partition_type: value.unified_partition_type.as_ref(),
578                sort_order_id: value.sort_order_id,
579                sort_order: value.sort_order.as_ref(),
580                column_stats: value.column_stats.as_deref(),
581                case_sensitive: value.case_sensitive,
582                key_metadata: value.key_metadata.as_deref(),
583            })
584        }
585    }
586
587    impl Serialize for FileScanTask {
588        fn serialize<S>(&self, serializer: S) -> std::result::Result<S::Ok, S::Error>
589        where S: serde::Serializer {
590            FileScanTaskRefSerde::try_from(self)
591                .map_err(serde::ser::Error::custom)?
592                .serialize(serializer)
593        }
594    }
595
596    impl TryFrom<FileScanTaskSerde> for FileScanTask {
597        type Error = Error;
598
599        fn try_from(value: FileScanTaskSerde) -> Result<Self> {
600            let partition = value
601                .partition
602                .map(|partition| {
603                    let partition_type =
604                        partition_type(value.partition_spec.as_deref(), &value.schema)?;
605                    match partition.try_into(&partition_type)? {
606                        Some(Literal::Struct(partition)) => Ok(partition),
607                        _ => Err(invalid_data!("FileScanTask partition must be a struct")),
608                    }
609                })
610                .transpose()?;
611
612            Self::builder()
613                .with_file_size_in_bytes(value.file_size_in_bytes)
614                .with_start(value.start)
615                .with_length(value.length)
616                .with_record_count(value.record_count)
617                .with_first_row_id(value.first_row_id)
618                .with_data_sequence_number(value.data_sequence_number)
619                .with_data_file_path(value.data_file_path)
620                .with_data_file_format(value.data_file_format)
621                .with_schema(value.schema)
622                .with_project_field_ids(value.project_field_ids)
623                .with_predicate(value.predicate)
624                .with_deletes(value.deletes)
625                .with_partition(partition)
626                .with_partition_spec(value.partition_spec)
627                .with_name_mapping(value.name_mapping)
628                .with_unified_partition_type(value.unified_partition_type)
629                .with_sort_order_id(value.sort_order_id)
630                .with_sort_order(value.sort_order)
631                .with_column_stats(value.column_stats)
632                .with_case_sensitive(value.case_sensitive)
633                .with_key_metadata(value.key_metadata)
634                .build()
635        }
636    }
637
638    #[derive(Deserialize)]
639    pub(super) struct FileScanTaskDeleteFileSerde {
640        file_path: String,
641        #[serde(default)]
642        file_size_in_bytes: Option<u64>,
643        file_type: DataContentType,
644        file_format: DataFileFormat,
645        partition_spec_id: i32,
646        equality_ids: Option<Vec<i32>>,
647        #[serde(default)]
648        referenced_data_file: Option<String>,
649        #[serde(default)]
650        content_offset: Option<u64>,
651        #[serde(default)]
652        content_size_in_bytes: Option<u64>,
653        #[serde(default)]
654        record_count: Option<u64>,
655        #[serde(default)]
656        key_metadata: Option<Box<[u8]>>,
657    }
658
659    impl TryFrom<FileScanTaskDeleteFileSerde> for FileScanTaskDeleteFile {
660        type Error = Error;
661
662        fn try_from(value: FileScanTaskDeleteFileSerde) -> Result<Self> {
663            Self::builder()
664                .with_file_path(value.file_path)
665                .with_file_size_in_bytes_opt(value.file_size_in_bytes)
666                .with_file_type(value.file_type)
667                .with_file_format(value.file_format)
668                .with_partition_spec_id(value.partition_spec_id)
669                .with_equality_ids(value.equality_ids)
670                .with_referenced_data_file(value.referenced_data_file)
671                .with_content_offset(value.content_offset)
672                .with_content_size_in_bytes(value.content_size_in_bytes)
673                .with_record_count(value.record_count)
674                .with_key_metadata(value.key_metadata)
675                .build()
676        }
677    }
678
679    /// Serializes column bounds as their type and the spec's single-value binary encoding,
680    /// because the JSON encoding of [`RawLiteral`] does not round-trip every float value.
681    pub(super) mod bounds {
682        use std::collections::HashMap;
683
684        use serde::ser::SerializeMap;
685        use serde::{Deserialize, Deserializer, Serialize, Serializer};
686        use serde_bytes::ByteBuf;
687
688        use crate::spec::{Datum, PrimitiveType};
689
690        #[derive(Serialize, Deserialize)]
691        struct BoundSerde {
692            r#type: PrimitiveType,
693            bytes: ByteBuf,
694        }
695
696        pub(in crate::scan::task) fn serialize<S: Serializer>(
697            bounds: &HashMap<i32, Datum>,
698            serializer: S,
699        ) -> Result<S::Ok, S::Error> {
700            let mut map = serializer.serialize_map(Some(bounds.len()))?;
701            for (field_id, datum) in bounds {
702                let bound = BoundSerde {
703                    r#type: datum.data_type().clone(),
704                    bytes: datum.to_bytes().map_err(serde::ser::Error::custom)?,
705                };
706                map.serialize_entry(field_id, &bound)?;
707            }
708            map.end()
709        }
710
711        pub(in crate::scan::task) fn deserialize<'de, D: Deserializer<'de>>(
712            deserializer: D,
713        ) -> Result<HashMap<i32, Datum>, D::Error> {
714            HashMap::<i32, BoundSerde>::deserialize(deserializer)?
715                .into_iter()
716                .map(|(field_id, bound)| {
717                    Datum::try_from_bytes(&bound.bytes, bound.r#type)
718                        .map(|datum| (field_id, datum))
719                        .map_err(serde::de::Error::custom)
720                })
721                .collect()
722        }
723    }
724}
725
726impl FileScanTaskDeleteFile {
727    /// Returns the delete file path.
728    pub fn file_path(&self) -> &str {
729        &self.file_path
730    }
731
732    /// Returns the total size of the delete file in bytes, or `None` if the task does not
733    /// carry it.
734    pub fn file_size_in_bytes(&self) -> Option<u64> {
735        self.file_size_in_bytes
736    }
737
738    /// Returns the delete file content type.
739    pub fn file_type(&self) -> DataContentType {
740        self.file_type
741    }
742
743    /// Returns the delete file format.
744    pub fn file_format(&self) -> DataFileFormat {
745        self.file_format
746    }
747
748    /// Returns the partition spec id.
749    pub fn partition_spec_id(&self) -> i32 {
750        self.partition_spec_id
751    }
752
753    /// Returns the equality field ids for an equality delete file.
754    pub fn equality_ids(&self) -> Option<&[i32]> {
755        self.equality_ids.as_deref()
756    }
757
758    /// Returns the referenced data file path.
759    pub fn referenced_data_file(&self) -> Option<&str> {
760        self.referenced_data_file.as_deref()
761    }
762
763    /// Returns the deletion vector blob offset.
764    pub fn content_offset(&self) -> Option<u64> {
765        self.content_offset
766    }
767
768    /// Returns the deletion vector blob size in bytes.
769    pub fn content_size_in_bytes(&self) -> Option<u64> {
770        self.content_size_in_bytes
771    }
772
773    /// Returns the number of records in the delete file.
774    pub fn record_count(&self) -> Option<u64> {
775        self.record_count
776    }
777
778    /// Returns the key metadata for the encrypted delete file.
779    pub fn key_metadata(&self) -> Option<&[u8]> {
780        self.key_metadata.as_deref()
781    }
782
783    fn is_deletion_vector(&self) -> bool {
784        self.file_type == DataContentType::PositionDeletes
785            && self.file_format == DataFileFormat::Puffin
786    }
787
788    fn validate(&self) -> Result<()> {
789        // Negative offsets/sizes are unrepresentable: both fields are u64, and the
790        // conversion from the manifest entry's i64 rejects a negative there.
791
792        // Spec: `content_size_in_bytes` is required whenever `content_offset` is
793        // present, for every delete-file kind (not only deletion vectors).
794        if self.content_offset.is_some() && self.content_size_in_bytes.is_none() {
795            return Err(invalid_data!(
796                "delete file {} is missing content_size_in_bytes while content_offset is present",
797                self.file_path
798            ));
799        }
800
801        // Presence of the deletion-vector coordinates is required only for
802        // deletion vectors; other delete files legitimately carry none of these.
803        if !self.is_deletion_vector() {
804            return Ok(());
805        }
806
807        let missing = if self.referenced_data_file.is_none() {
808            Some("referenced_data_file")
809        } else if self.content_offset.is_none() {
810            Some("content_offset")
811        } else if self.content_size_in_bytes.is_none() {
812            Some("content_size_in_bytes")
813        } else if self.record_count.is_none() {
814            Some("record_count")
815        } else {
816            None
817        };
818
819        if let Some(field) = missing {
820            return Err(invalid_data!(
821                "deletion vector {} is missing {field}",
822                self.file_path
823            ));
824        }
825
826        Ok(())
827    }
828}
829
830impl From<FileScanTaskDeleteFile> for Result<FileScanTaskDeleteFile> {
831    fn from(task: FileScanTaskDeleteFile) -> Self {
832        task.validate()?;
833        Ok(task)
834    }
835}
836
837/// Column statistics of a [`FileScanTask`]'s data file, keyed by field id and limited to the
838/// columns requested for the scan.
839///
840/// A requested column missing from a map has unknown statistics of that kind. Bounds are
841/// inclusive, and all statistics cover the whole file, not the scan's filter, delete files or
842/// task splits.
843///
844/// A manifest written before a type promotion stores the narrower type, so bounds are promoted to
845/// the column's type in the scan schema: `int` to `long`, `float` to `double`, and a decimal to a
846/// wider precision with the same scale. A bound of any other type is omitted.
847#[derive(Debug, Clone, PartialEq, Default, Serialize, Deserialize, TypedBuilder)]
848#[builder(field_defaults(default, setter(prefix = "with_")))]
849pub struct FileScanTaskColumnStats {
850    #[serde(default, skip_serializing_if = "HashMap::is_empty")]
851    column_sizes: HashMap<i32, u64>,
852
853    #[serde(default, skip_serializing_if = "HashMap::is_empty")]
854    value_counts: HashMap<i32, u64>,
855
856    #[serde(default, skip_serializing_if = "HashMap::is_empty")]
857    null_value_counts: HashMap<i32, u64>,
858
859    #[serde(default, skip_serializing_if = "HashMap::is_empty")]
860    nan_value_counts: HashMap<i32, u64>,
861
862    #[serde(
863        default,
864        skip_serializing_if = "HashMap::is_empty",
865        with = "crate::scan::task::_serde::bounds"
866    )]
867    lower_bounds: HashMap<i32, Datum>,
868
869    #[serde(
870        default,
871        skip_serializing_if = "HashMap::is_empty",
872        with = "crate::scan::task::_serde::bounds"
873    )]
874    upper_bounds: HashMap<i32, Datum>,
875}
876
877/// The field id and type of each column whose statistics a scan keeps.
878pub(crate) type ColumnStatsColumns = Arc<[(i32, PrimitiveType)]>;
879
880/// Returns `bound` typed as `column_type`, or `None` if it cannot be promoted to it.
881fn bound_as_column_type(bound: &Datum, column_type: &PrimitiveType) -> Option<Datum> {
882    if bound.data_type() == column_type {
883        return Some(bound.clone());
884    }
885    if let Some(promoted) = bound.promote_to(column_type) {
886        return Some(promoted);
887    }
888    match (bound.data_type(), column_type) {
889        (
890            PrimitiveType::Decimal {
891                precision: from_precision,
892                scale: from_scale,
893            },
894            PrimitiveType::Decimal { precision, scale },
895        ) if from_scale == scale && from_precision <= precision => {
896            Some(Datum::new(column_type.clone(), bound.literal().clone()))
897        }
898        _ => None,
899    }
900}
901
902impl FileScanTaskColumnStats {
903    pub(crate) fn from_data_file(data_file: &DataFile, columns: &[(i32, PrimitiveType)]) -> Self {
904        fn retain<V: Clone>(
905            map: &HashMap<i32, V>,
906            columns: &[(i32, PrimitiveType)],
907        ) -> HashMap<i32, V> {
908            if map.is_empty() {
909                return HashMap::new();
910            }
911            let mut retained = HashMap::with_capacity(columns.len().min(map.len()));
912            retained.extend(
913                columns
914                    .iter()
915                    .filter_map(|(id, _)| map.get(id).map(|value| (*id, value.clone()))),
916            );
917            retained
918        }
919
920        fn retain_bounds(
921            bounds: &HashMap<i32, Datum>,
922            columns: &[(i32, PrimitiveType)],
923        ) -> HashMap<i32, Datum> {
924            if bounds.is_empty() {
925                return HashMap::new();
926            }
927            let mut retained = HashMap::with_capacity(columns.len().min(bounds.len()));
928            retained.extend(columns.iter().filter_map(|(id, column_type)| {
929                let bound = bound_as_column_type(bounds.get(id)?, column_type)?;
930                Some((*id, bound))
931            }));
932            retained
933        }
934
935        Self {
936            column_sizes: retain(data_file.column_sizes(), columns),
937            value_counts: retain(data_file.value_counts(), columns),
938            null_value_counts: retain(data_file.null_value_counts(), columns),
939            nan_value_counts: retain(data_file.nan_value_counts(), columns),
940            lower_bounds: retain_bounds(data_file.lower_bounds(), columns),
941            upper_bounds: retain_bounds(data_file.upper_bounds(), columns),
942        }
943    }
944
945    /// Returns the size in bytes of each column's data in the file, per field id.
946    pub fn column_sizes(&self) -> &HashMap<i32, u64> {
947        &self.column_sizes
948    }
949
950    /// Returns the number of values per field id, including nulls and NaNs.
951    pub fn value_counts(&self) -> &HashMap<i32, u64> {
952        &self.value_counts
953    }
954
955    /// Returns the number of null values per field id.
956    pub fn null_value_counts(&self) -> &HashMap<i32, u64> {
957        &self.null_value_counts
958    }
959
960    /// Returns the number of NaN values per field id.
961    pub fn nan_value_counts(&self) -> &HashMap<i32, u64> {
962        &self.nan_value_counts
963    }
964
965    /// Returns the inclusive lower bound per field id.
966    pub fn lower_bounds(&self) -> &HashMap<i32, Datum> {
967        &self.lower_bounds
968    }
969
970    /// Returns the inclusive upper bound per field id.
971    pub fn upper_bounds(&self) -> &HashMap<i32, Datum> {
972        &self.upper_bounds
973    }
974}
975
976#[cfg(test)]
977mod tests {
978    use std::collections::HashMap;
979
980    use super::*;
981    use crate::ErrorKind;
982    use crate::spec::{
983        DataFileBuilder, Literal, ManifestEntry, ManifestStatus, NestedField, PrimitiveLiteral,
984        PrimitiveType, Transform, Type,
985    };
986
987    fn build_file_scan_task(
988        schema: SchemaRef,
989        partition: Option<Struct>,
990        partition_spec: Option<Arc<PartitionSpec>>,
991    ) -> Result<FileScanTask> {
992        FileScanTask::builder()
993            .with_file_size_in_bytes(100)
994            .with_start(0)
995            .with_length(100)
996            .with_data_file_path("data_file_path".to_string())
997            .with_data_file_format(DataFileFormat::Parquet)
998            .with_schema(schema)
999            .with_project_field_ids(vec![])
1000            .with_partition(partition)
1001            .with_partition_spec(partition_spec)
1002            .with_case_sensitive(false)
1003            .build()
1004    }
1005
1006    fn schema_and_spec(
1007        primitive_type: PrimitiveType,
1008        transform: Transform,
1009    ) -> (SchemaRef, Arc<PartitionSpec>) {
1010        let schema = Arc::new(
1011            Schema::builder()
1012                .with_fields(vec![Arc::new(NestedField::required(
1013                    1,
1014                    "x",
1015                    Type::Primitive(primitive_type),
1016                ))])
1017                .build()
1018                .unwrap(),
1019        );
1020        let partition_spec = Arc::new(
1021            PartitionSpec::builder(schema.clone())
1022                .add_partition_field("x", "x_partition", transform)
1023                .unwrap()
1024                .build()
1025                .unwrap(),
1026        );
1027        (schema, partition_spec)
1028    }
1029
1030    fn build_delete_file_task(
1031        file_type: DataContentType,
1032        file_format: DataFileFormat,
1033    ) -> Result<FileScanTaskDeleteFile> {
1034        FileScanTaskDeleteFile::builder()
1035            .with_file_path("delete-file".to_string())
1036            .with_file_size_in_bytes(100)
1037            .with_file_type(file_type)
1038            .with_file_format(file_format)
1039            .with_partition_spec_id(0)
1040            .build()
1041    }
1042
1043    fn assert_delete_file_builder_error(
1044        result: Result<FileScanTaskDeleteFile>,
1045        expected_message: &str,
1046    ) {
1047        match result {
1048            Ok(task) => panic!(
1049                "expected delete file builder to fail with `{expected_message}`, but got Ok({task:?})"
1050            ),
1051            Err(err) => {
1052                assert_eq!(err.kind(), ErrorKind::DataInvalid);
1053                assert_eq!(err.message(), expected_message);
1054            }
1055        }
1056    }
1057
1058    #[test]
1059    fn test_file_scan_task_builder_rejects_non_empty_partition_without_spec() {
1060        // Regression test for https://github.com/apache/iceberg-rust/issues/3130.
1061        let err = build_file_scan_task(
1062            Arc::new(Schema::builder().build().unwrap()),
1063            Some(Struct::from_iter([Some(Literal::long(42))])),
1064            None,
1065        )
1066        .unwrap_err();
1067
1068        assert_eq!(err.kind(), ErrorKind::DataInvalid);
1069        assert_eq!(
1070            err.message(),
1071            "Non-empty FileScanTask partition requires a partition spec"
1072        );
1073    }
1074
1075    #[test]
1076    fn test_file_scan_task_builder_accepts_empty_partition_without_spec() {
1077        build_file_scan_task(
1078            Arc::new(Schema::builder().build().unwrap()),
1079            Some(Struct::empty()),
1080            None,
1081        )
1082        .unwrap();
1083    }
1084
1085    #[test]
1086    fn test_file_scan_task_builder_rejects_partitioned_spec_without_partition() {
1087        let (schema, partition_spec) = schema_and_spec(PrimitiveType::Long, Transform::Identity);
1088
1089        let err = build_file_scan_task(schema, None, Some(partition_spec)).unwrap_err();
1090
1091        assert_eq!(err.kind(), ErrorKind::DataInvalid);
1092        assert_eq!(
1093            err.message(),
1094            "FileScanTask with a partitioned spec requires partition values"
1095        );
1096    }
1097
1098    #[test]
1099    fn test_file_scan_task_builder_accepts_unpartitioned_spec_without_partition() {
1100        build_file_scan_task(
1101            Arc::new(Schema::builder().build().unwrap()),
1102            None,
1103            Some(Arc::new(PartitionSpec::unpartition_spec())),
1104        )
1105        .unwrap();
1106    }
1107
1108    #[test]
1109    fn test_file_scan_task_builder_rejects_partition_arity_mismatch() {
1110        let (schema, partition_spec) = schema_and_spec(PrimitiveType::Long, Transform::Identity);
1111
1112        let err =
1113            build_file_scan_task(schema, Some(Struct::empty()), Some(partition_spec)).unwrap_err();
1114
1115        assert_eq!(err.kind(), ErrorKind::DataInvalid);
1116        assert!(err.message().contains("partition has 0 fields"));
1117        assert!(err.message().contains("partition spec has 1 fields"));
1118    }
1119
1120    #[test]
1121    fn test_file_scan_task_builder_accepts_dropped_partition_source_column() {
1122        let (_historical_schema, partition_spec) =
1123            schema_and_spec(PrimitiveType::Long, Transform::Identity);
1124        let current_schema = Arc::new(
1125            Schema::builder()
1126                .with_fields(vec![Arc::new(NestedField::required(
1127                    2,
1128                    "y",
1129                    Type::Primitive(PrimitiveType::String),
1130                ))])
1131                .build()
1132                .unwrap(),
1133        );
1134
1135        build_file_scan_task(
1136            current_schema,
1137            Some(Struct::from_iter([Some(Literal::long(42))])),
1138            Some(partition_spec),
1139        )
1140        .unwrap();
1141    }
1142
1143    #[test]
1144    fn test_file_scan_task_builder_rejects_partition_spec_incompatible_with_schema() {
1145        let (_historical_schema, partition_spec) =
1146            schema_and_spec(PrimitiveType::Timestamp, Transform::Day);
1147        let current_schema = Arc::new(
1148            Schema::builder()
1149                .with_fields(vec![Arc::new(NestedField::required(
1150                    1,
1151                    "x",
1152                    Type::Primitive(PrimitiveType::String),
1153                ))])
1154                .build()
1155                .unwrap(),
1156        );
1157
1158        let err = build_file_scan_task(
1159            current_schema,
1160            Some(Struct::from_iter([Some(Literal::date(20_000))])),
1161            Some(partition_spec),
1162        )
1163        .unwrap_err();
1164
1165        assert_eq!(err.kind(), ErrorKind::DataInvalid);
1166    }
1167
1168    #[test]
1169    fn test_delete_file_builder_accepts_valid_deletion_vector() {
1170        let task = FileScanTaskDeleteFile::builder()
1171            .with_file_path("dv.puffin".to_string())
1172            .with_file_size_in_bytes(100)
1173            .with_file_type(DataContentType::PositionDeletes)
1174            .with_file_format(DataFileFormat::Puffin)
1175            .with_partition_spec_id(7)
1176            .with_referenced_data_file(Some("data.parquet".to_string()))
1177            .with_content_offset(Some(11))
1178            .with_content_size_in_bytes(Some(13))
1179            .with_record_count(Some(3))
1180            .with_key_metadata(Some(vec![17, 19].into_boxed_slice()))
1181            .build()
1182            .unwrap();
1183
1184        assert_eq!(task.file_path(), "dv.puffin");
1185        assert_eq!(task.file_size_in_bytes(), Some(100));
1186        assert_eq!(task.file_type(), DataContentType::PositionDeletes);
1187        assert_eq!(task.file_format(), DataFileFormat::Puffin);
1188        assert_eq!(task.partition_spec_id(), 7);
1189        assert_eq!(task.equality_ids(), None);
1190        assert_eq!(task.referenced_data_file(), Some("data.parquet"));
1191        assert_eq!(task.content_offset(), Some(11));
1192        assert_eq!(task.content_size_in_bytes(), Some(13));
1193        assert_eq!(task.record_count(), Some(3));
1194        assert_eq!(task.key_metadata(), Some([17, 19].as_slice()));
1195    }
1196
1197    #[test]
1198    fn test_delete_file_builder_rejects_dv_missing_referenced_data_file() {
1199        assert_delete_file_builder_error(
1200            FileScanTaskDeleteFile::builder()
1201                .with_file_path("dv.puffin".to_string())
1202                .with_file_size_in_bytes(100)
1203                .with_file_type(DataContentType::PositionDeletes)
1204                .with_file_format(DataFileFormat::Puffin)
1205                .with_partition_spec_id(0)
1206                .with_content_offset(Some(7))
1207                .with_content_size_in_bytes(Some(11))
1208                .with_record_count(Some(3))
1209                .build(),
1210            "deletion vector dv.puffin is missing referenced_data_file",
1211        );
1212    }
1213
1214    #[test]
1215    fn test_delete_file_builder_rejects_dv_missing_content_offset() {
1216        assert_delete_file_builder_error(
1217            FileScanTaskDeleteFile::builder()
1218                .with_file_path("dv.puffin".to_string())
1219                .with_file_size_in_bytes(100)
1220                .with_file_type(DataContentType::PositionDeletes)
1221                .with_file_format(DataFileFormat::Puffin)
1222                .with_partition_spec_id(0)
1223                .with_referenced_data_file(Some("data.parquet".to_string()))
1224                .with_content_size_in_bytes(Some(11))
1225                .with_record_count(Some(3))
1226                .build(),
1227            "deletion vector dv.puffin is missing content_offset",
1228        );
1229    }
1230
1231    #[test]
1232    fn test_delete_file_builder_rejects_dv_missing_content_size() {
1233        assert_delete_file_builder_error(
1234            FileScanTaskDeleteFile::builder()
1235                .with_file_path("dv.puffin".to_string())
1236                .with_file_size_in_bytes(100)
1237                .with_file_type(DataContentType::PositionDeletes)
1238                .with_file_format(DataFileFormat::Puffin)
1239                .with_partition_spec_id(0)
1240                .with_referenced_data_file(Some("data.parquet".to_string()))
1241                .with_content_offset(Some(7))
1242                .with_record_count(Some(3))
1243                .build(),
1244            "delete file dv.puffin is missing content_size_in_bytes while content_offset is present",
1245        );
1246    }
1247
1248    #[test]
1249    fn test_delete_file_builder_rejects_dv_missing_record_count() {
1250        assert_delete_file_builder_error(
1251            FileScanTaskDeleteFile::builder()
1252                .with_file_path("dv.puffin".to_string())
1253                .with_file_size_in_bytes(100)
1254                .with_file_type(DataContentType::PositionDeletes)
1255                .with_file_format(DataFileFormat::Puffin)
1256                .with_partition_spec_id(0)
1257                .with_referenced_data_file(Some("data.parquet".to_string()))
1258                .with_content_offset(Some(7))
1259                .with_content_size_in_bytes(Some(11))
1260                .build(),
1261            "deletion vector dv.puffin is missing record_count",
1262        );
1263    }
1264
1265    #[test]
1266    fn test_delete_file_builder_rejects_non_dv_offset_without_size() {
1267        // Spec pairing rule applies to every delete-file kind: a Parquet
1268        // position-delete file that carries a content_offset must also carry a
1269        // content_size_in_bytes, even though it is not a deletion vector.
1270        assert_delete_file_builder_error(
1271            FileScanTaskDeleteFile::builder()
1272                .with_file_path("position-deletes.parquet".to_string())
1273                .with_file_size_in_bytes(100)
1274                .with_file_type(DataContentType::PositionDeletes)
1275                .with_file_format(DataFileFormat::Parquet)
1276                .with_partition_spec_id(0)
1277                .with_content_offset(Some(7))
1278                .build(),
1279            "delete file position-deletes.parquet is missing content_size_in_bytes while content_offset is present",
1280        );
1281    }
1282
1283    fn delete_manifest_entry(
1284        file_path: &str,
1285        file_format: DataFileFormat,
1286        content_offset: Option<i64>,
1287        content_size_in_bytes: Option<i64>,
1288    ) -> DeleteFileContext {
1289        let mut data_file = DataFileBuilder::default()
1290            .content(DataContentType::PositionDeletes)
1291            .file_path(file_path.to_string())
1292            .file_format(file_format)
1293            .partition(Struct::empty())
1294            .record_count(3)
1295            .file_size_in_bytes(100)
1296            .column_sizes(HashMap::new())
1297            .value_counts(HashMap::new())
1298            .null_value_counts(HashMap::new())
1299            .partition_spec_id(0)
1300            .referenced_data_file(Some("data.parquet".to_string()))
1301            .build()
1302            .unwrap();
1303        data_file.content_offset = content_offset;
1304        data_file.content_size_in_bytes = content_size_in_bytes;
1305
1306        DeleteFileContext {
1307            manifest_entry: Arc::new(
1308                ManifestEntry::builder()
1309                    .status(ManifestStatus::Added)
1310                    .data_file(data_file)
1311                    .build(),
1312            ),
1313            partition_spec_id: 0,
1314        }
1315    }
1316
1317    fn assert_delete_context_error(ctx: &DeleteFileContext, expected_message: &str) {
1318        match FileScanTaskDeleteFile::try_from(ctx) {
1319            Ok(task) => panic!(
1320                "expected conversion to fail with `{expected_message}`, but got Ok({task:?})"
1321            ),
1322            Err(error) => {
1323                assert_eq!(error.kind(), ErrorKind::DataInvalid);
1324                assert!(
1325                    error.message().contains(expected_message),
1326                    "expected `{expected_message}`, got `{}`",
1327                    error.message()
1328                );
1329            }
1330        }
1331    }
1332
1333    /// The manifest entry supplies `content_offset` / `content_size_in_bytes` as
1334    /// `i64`; the conversion into a task rejects a negative value as it enters,
1335    /// since the task fields are `u64`.
1336    #[test]
1337    fn test_delete_context_rejects_negative_dv_offset() {
1338        assert_delete_context_error(
1339            &delete_manifest_entry("dv.puffin", DataFileFormat::Puffin, Some(-1), Some(11)),
1340            "delete file dv.puffin has negative content_offset -1",
1341        );
1342    }
1343
1344    #[test]
1345    fn test_delete_context_rejects_negative_dv_size() {
1346        assert_delete_context_error(
1347            &delete_manifest_entry("dv.puffin", DataFileFormat::Puffin, Some(7), Some(-1)),
1348            "delete file dv.puffin has negative content_size_in_bytes -1",
1349        );
1350    }
1351
1352    /// The negative-coordinate check applies to every delete-file kind, including
1353    /// a Parquet position-delete file rather than only deletion vectors.
1354    #[test]
1355    fn test_delete_context_rejects_negative_coordinates_for_non_dv() {
1356        assert_delete_context_error(
1357            &delete_manifest_entry(
1358                "position-deletes.parquet",
1359                DataFileFormat::Parquet,
1360                Some(-1),
1361                None,
1362            ),
1363            "delete file position-deletes.parquet has negative content_offset -1",
1364        );
1365        assert_delete_context_error(
1366            &delete_manifest_entry(
1367                "position-deletes.parquet",
1368                DataFileFormat::Parquet,
1369                None,
1370                Some(-1),
1371            ),
1372            "delete file position-deletes.parquet has negative content_size_in_bytes -1",
1373        );
1374    }
1375
1376    #[test]
1377    fn test_bound_as_column_type() {
1378        let decimal = |precision, scale| PrimitiveType::Decimal { precision, scale };
1379        let decimal_datum =
1380            |precision, scale| Datum::new(decimal(precision, scale), PrimitiveLiteral::Int128(150));
1381        let cases = [
1382            (Datum::int(-5), PrimitiveType::Long, Some(Datum::long(-5))),
1383            (
1384                Datum::float(f32::NEG_INFINITY),
1385                PrimitiveType::Double,
1386                Some(Datum::double(f64::NEG_INFINITY)),
1387            ),
1388            (
1389                Datum::float(1.5_f32),
1390                PrimitiveType::Double,
1391                Some(Datum::double(1.5)),
1392            ),
1393            (
1394                Datum::string("a"),
1395                PrimitiveType::String,
1396                Some(Datum::string("a")),
1397            ),
1398            (
1399                decimal_datum(9, 2),
1400                decimal(18, 2),
1401                Some(decimal_datum(18, 2)),
1402            ),
1403            (decimal_datum(9, 2), decimal(18, 3), None),
1404            (decimal_datum(38, 2), decimal(18, 2), None),
1405            (Datum::long(6), PrimitiveType::Int, None),
1406            (Datum::string("a"), PrimitiveType::Long, None),
1407        ];
1408
1409        for (bound, column_type, expected) in cases {
1410            assert_eq!(
1411                bound_as_column_type(&bound, &column_type),
1412                expected,
1413                "{bound:?} as {column_type:?}"
1414            );
1415        }
1416    }
1417
1418    #[test]
1419    fn test_delete_file_builder_accepts_non_dv_delete_without_dv_fields() {
1420        build_delete_file_task(DataContentType::PositionDeletes, DataFileFormat::Parquet).unwrap();
1421        build_delete_file_task(DataContentType::EqualityDeletes, DataFileFormat::Parquet).unwrap();
1422    }
1423}