Skip to main content

iceberg/spec/manifest/
data_file.rs

1// Licensed to the Apache Software Foundation (ASF) under one
2// or more contributor license agreements.  See the NOTICE file
3// distributed with this work for additional information
4// regarding copyright ownership.  The ASF licenses this file
5// to you under the Apache License, Version 2.0 (the
6// "License"); you may not use this file except in compliance
7// with the License.  You may obtain a copy of the License at
8//
9//   http://www.apache.org/licenses/LICENSE-2.0
10//
11// Unless required by applicable law or agreed to in writing,
12// software distributed under the License is distributed on an
13// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14// KIND, either express or implied.  See the License for the
15// specific language governing permissions and limitations
16// under the License.
17
18use std::collections::HashMap;
19use std::io::{Read, Write};
20use std::str::FromStr;
21
22use apache_avro::{Reader as AvroReader, from_value, to_value};
23use serde_derive::{Deserialize, Serialize};
24use serde_with::{DeserializeFromStr, SerializeDisplay};
25
26use super::_serde::DataFileSerde;
27use super::{
28    Datum, FormatVersion, Schema, data_file_schema_v1, data_file_schema_v2, data_file_schema_v3,
29};
30use crate::Error;
31use crate::avro::new_writer;
32use crate::error::{Result, invalid_data};
33use crate::spec::{
34    DEFAULT_PARTITION_SPEC_ID, Literal, NestedFieldRef, PrimitiveType, Struct, StructType, Type,
35};
36
37/// Data file carries data file path, partition tuple, metrics, …
38#[derive(Debug, PartialEq, Clone, Eq, Builder)]
39pub struct DataFile {
40    /// field id: 134
41    ///
42    /// Type of content stored by the data file: data, equality deletes,
43    /// or position deletes (all v1 files are data files)
44    pub(crate) content: DataContentType,
45    /// field id: 100
46    ///
47    /// Full URI for the file with FS scheme
48    pub(crate) file_path: String,
49    /// field id: 101
50    ///
51    /// String file format name, `avro`, `orc`, `parquet`, or `puffin`
52    pub(crate) file_format: DataFileFormat,
53    /// field id: 102
54    ///
55    /// Partition data tuple, schema based on the partition spec output using
56    /// partition field ids for the struct field ids
57    #[builder(default = "Struct::empty()")]
58    pub(crate) partition: Struct,
59    /// field id: 103
60    ///
61    /// Number of records in this file, or the cardinality of a deletion vector
62    pub(crate) record_count: u64,
63    /// field id: 104
64    ///
65    /// Total file size in bytes
66    pub(crate) file_size_in_bytes: u64,
67    /// field id: 108
68    /// key field id: 117
69    /// value field id: 118
70    ///
71    /// Map from column id to the total size on disk of all regions that
72    /// store the column. Does not include bytes necessary to read other
73    /// columns, like footers. Leave null for row-oriented formats (Avro)
74    #[builder(default)]
75    pub(crate) column_sizes: HashMap<i32, u64>,
76    /// field id: 109
77    /// key field id: 119
78    /// value field id: 120
79    ///
80    /// Map from column id to number of values in the column (including null
81    /// and NaN values)
82    #[builder(default)]
83    pub(crate) value_counts: HashMap<i32, u64>,
84    /// field id: 110
85    /// key field id: 121
86    /// value field id: 122
87    ///
88    /// Map from column id to number of null values in the column
89    #[builder(default)]
90    pub(crate) null_value_counts: HashMap<i32, u64>,
91    /// field id: 137
92    /// key field id: 138
93    /// value field id: 139
94    ///
95    /// Map from column id to number of NaN values in the column
96    #[builder(default)]
97    pub(crate) nan_value_counts: HashMap<i32, u64>,
98    /// field id: 125
99    /// key field id: 126
100    /// value field id: 127
101    ///
102    /// Map from column id to lower bound in the column serialized as binary.
103    /// Each value must be less than or equal to all non-null, non-NaN values
104    /// in the column for the file.
105    ///
106    /// Reference:
107    ///
108    /// - [Binary single-value serialization](https://iceberg.apache.org/spec/#binary-single-value-serialization)
109    #[builder(default)]
110    pub(crate) lower_bounds: HashMap<i32, Datum>,
111    /// field id: 128
112    /// key field id: 129
113    /// value field id: 130
114    ///
115    /// Map from column id to upper bound in the column serialized as binary.
116    /// Each value must be greater than or equal to all non-null, non-Nan
117    /// values in the column for the file.
118    ///
119    /// Reference:
120    ///
121    /// - [Binary single-value serialization](https://iceberg.apache.org/spec/#binary-single-value-serialization)
122    #[builder(default)]
123    pub(crate) upper_bounds: HashMap<i32, Datum>,
124    /// field id: 131
125    ///
126    /// Implementation-specific key metadata for encryption
127    #[builder(default)]
128    pub(crate) key_metadata: Option<Vec<u8>>,
129    /// field id: 132
130    /// element field id: 133
131    ///
132    /// Split offsets for the data file. For example, all row group offsets
133    /// in a Parquet file. Must be sorted ascending. Optional field that
134    /// should be serialized as null when not present.
135    #[builder(default)]
136    pub(crate) split_offsets: Option<Vec<i64>>,
137    /// field id: 135
138    /// element field id: 136
139    ///
140    /// Field ids used to determine row equality in equality delete files.
141    /// Required when content is EqualityDeletes and should be null
142    /// otherwise. Fields with ids listed in this column must be present
143    /// in the delete file
144    #[builder(default)]
145    pub(crate) equality_ids: Option<Vec<i32>>,
146    /// field id: 140
147    ///
148    /// ID representing sort order for this file.
149    ///
150    /// If sort order ID is missing or unknown, then the order is assumed to
151    /// be unsorted. Only data files and equality delete files should be
152    /// written with a non-null order id. Position deletes are required to be
153    /// sorted by file and position, not a table order, and should set sort
154    /// order id to null. Readers must ignore sort order id for position
155    /// delete files.
156    #[builder(default, setter(strip_option))]
157    pub(crate) sort_order_id: Option<i32>,
158    /// field id: 142
159    ///
160    /// The _row_id for the first row in the data file.
161    /// For more details, refer to <https://github.com/apache/iceberg/blob/main/format/spec.md#first-row-id-inheritance>
162    #[builder(default)]
163    pub(crate) first_row_id: Option<i64>,
164    /// This field is not included in spec. It is just store in memory representation used
165    /// in process.
166    #[builder(default = "DEFAULT_PARTITION_SPEC_ID")]
167    pub(crate) partition_spec_id: i32,
168    /// field id: 143
169    ///
170    /// Fully qualified location (URI with FS scheme) of a data file that all deletes reference.
171    /// Position delete metadata can use `referenced_data_file` when all deletes tracked by the
172    /// entry are in a single data file. Setting the referenced file is required for deletion vectors.
173    #[builder(default)]
174    pub(crate) referenced_data_file: Option<String>,
175    /// field: 144
176    ///
177    /// The offset in the file where the content starts.
178    /// The `content_offset` and `content_size_in_bytes` fields are used to reference a specific blob
179    /// for direct access to a deletion vector. For deletion vectors, these values are required and must
180    /// exactly match the `offset` and `length` stored in the Puffin footer for the deletion vector blob.
181    #[builder(default)]
182    pub(crate) content_offset: Option<i64>,
183    /// field: 145
184    ///
185    /// The length of a referenced content stored in the file; required if `content_offset` is present
186    #[builder(default)]
187    pub(crate) content_size_in_bytes: Option<i64>,
188}
189
190fn promote_partition_value(
191    value: Option<&Literal>,
192    current_field: &NestedFieldRef,
193    written_field: &NestedFieldRef,
194) -> Option<Literal> {
195    match value {
196        Some(Literal::Primitive(primitive)) => {
197            let promoted = match current_field.field_type.as_primitive_type() {
198                // The source column was dropped, so there is no current type to widen to.
199                // Widen a numeric to its promotion ceiling so this value and its deletes
200                // normalize the same way; keep every other type (e.g. a `date`, stored as
201                // an `int`) at its written type so deletes that skip normalization match.
202                Some(PrimitiveType::Unknown) => {
203                    match written_field.field_type.as_primitive_type() {
204                        Some(PrimitiveType::Int | PrimitiveType::Float) => primitive.widen(),
205                        _ => None,
206                    }
207                }
208                Some(target) => primitive.promote_to(target),
209                None => None,
210            };
211            promoted.map(Literal::Primitive).or_else(|| value.cloned())
212        }
213        other => other.cloned(),
214    }
215}
216
217impl DataFile {
218    /// Returns this file's partition tuple normalized to the current schema: a field
219    /// promoted (`int` -> `long`, `float` -> `double`) since the manifest was written is
220    /// widened to its `current` type; a dropped numeric source, which has no `current`
221    /// type, is widened to its promotion ceiling by its `written` type; all other values
222    /// are copied through. Column bounds are widened separately, lazily, where they are
223    /// compared.
224    ///
225    /// `current` and `written` are the file's spec resolved against the current schema and
226    /// against the manifest's schema, so they match the stored tuple in length.
227    ///
228    /// # Errors
229    /// Returns `DataInvalid` if the stored tuple and `current` have different lengths; the
230    /// lengths should always match, since both come from the same spec.
231    pub(crate) fn promoted_partition(
232        &self,
233        current: &StructType,
234        written: &StructType,
235    ) -> Result<Struct> {
236        if self.partition.fields().len() != current.fields().len() {
237            return Err(invalid_data!(
238                "Partition tuple has {} values but the spec partition type has {} fields",
239                self.partition.fields().len(),
240                current.fields().len()
241            ));
242        }
243
244        Ok(self
245            .partition
246            .iter()
247            .zip(current.fields())
248            .zip(written.fields())
249            .map(|((value, current_field), written_field)| {
250                promote_partition_value(value, current_field, written_field)
251            })
252            .collect())
253    }
254
255    /// Get the content type of the data file (data, equality deletes, or position deletes)
256    pub fn content_type(&self) -> DataContentType {
257        self.content
258    }
259    /// Get the file path as full URI with FS scheme
260    pub fn file_path(&self) -> &str {
261        &self.file_path
262    }
263    /// Get the file format of the file (avro, orc or parquet).
264    pub fn file_format(&self) -> DataFileFormat {
265        self.file_format
266    }
267    /// Get the partition values of the file.
268    pub fn partition(&self) -> &Struct {
269        &self.partition
270    }
271    /// Get the record count in the data file.
272    pub fn record_count(&self) -> u64 {
273        self.record_count
274    }
275    /// Get the file size in bytes.
276    pub fn file_size_in_bytes(&self) -> u64 {
277        self.file_size_in_bytes
278    }
279    /// Get the column sizes.
280    /// Map from column id to the total size on disk of all regions that
281    /// store the column. Does not include bytes necessary to read other
282    /// columns, like footers. Null for row-oriented formats (Avro)
283    pub fn column_sizes(&self) -> &HashMap<i32, u64> {
284        &self.column_sizes
285    }
286    /// Get the columns value counts for the data file.
287    /// Map from column id to number of values in the column (including null
288    /// and NaN values)
289    pub fn value_counts(&self) -> &HashMap<i32, u64> {
290        &self.value_counts
291    }
292    /// Get the null value counts of the data file.
293    /// Map from column id to number of null values in the column
294    pub fn null_value_counts(&self) -> &HashMap<i32, u64> {
295        &self.null_value_counts
296    }
297    /// Get the nan value counts of the data file.
298    /// Map from column id to number of NaN values in the column
299    pub fn nan_value_counts(&self) -> &HashMap<i32, u64> {
300        &self.nan_value_counts
301    }
302    /// Get the lower bounds of the data file values per column.
303    /// Map from column id to lower bound in the column serialized as binary.
304    pub fn lower_bounds(&self) -> &HashMap<i32, Datum> {
305        &self.lower_bounds
306    }
307    /// Get the upper bounds of the data file values per column.
308    /// Map from column id to upper bound in the column serialized as binary.
309    pub fn upper_bounds(&self) -> &HashMap<i32, Datum> {
310        &self.upper_bounds
311    }
312    /// Get the Implementation-specific key metadata for the data file.
313    pub fn key_metadata(&self) -> Option<&[u8]> {
314        self.key_metadata.as_deref()
315    }
316    /// Get the split offsets of the data file.
317    /// For example, all row group offsets in a Parquet file.
318    /// Returns `None` if no split offsets are present.
319    pub fn split_offsets(&self) -> Option<&[i64]> {
320        self.split_offsets.as_deref()
321    }
322    /// Get the equality ids of the data file.
323    /// Field ids used to determine row equality in equality delete files.
324    /// null when content is not EqualityDeletes.
325    pub fn equality_ids(&self) -> Option<Vec<i32>> {
326        self.equality_ids.clone()
327    }
328    /// Get the first row id in the data file.
329    pub fn first_row_id(&self) -> Option<i64> {
330        self.first_row_id
331    }
332    /// Get the sort order id of the data file.
333    /// Only data files and equality delete files should be
334    /// written with a non-null order id. Position deletes are required to be
335    /// sorted by file and position, not a table order, and should set sort
336    /// order id to null. Readers must ignore sort order id for position
337    /// delete files.
338    pub fn sort_order_id(&self) -> Option<i32> {
339        self.sort_order_id
340    }
341    /// Get the fully qualified referenced location for the corresponding data file.
342    /// Positional delete files could have the field set, and deletion vectors must the field set.
343    pub fn referenced_data_file(&self) -> Option<String> {
344        self.referenced_data_file.clone()
345    }
346    /// Get the offset in the file where the blob content starts.
347    /// Only meaningful for puffin blobs, and required for deletion vectors.
348    pub fn content_offset(&self) -> Option<i64> {
349        self.content_offset
350    }
351    /// Get the length of a puffin blob.
352    /// Only meaningful for puffin blobs, and required for deletion vectors.
353    pub fn content_size_in_bytes(&self) -> Option<i64> {
354        self.content_size_in_bytes
355    }
356}
357
358/// Convert data files to avro bytes and write to writer.
359/// Return the bytes written.
360pub fn write_data_files_to_avro<W: Write>(
361    writer: &mut W,
362    data_files: impl IntoIterator<Item = DataFile>,
363    partition_type: &StructType,
364    version: FormatVersion,
365) -> Result<usize> {
366    let avro_schema = match version {
367        FormatVersion::V1 => data_file_schema_v1(partition_type).unwrap(),
368        FormatVersion::V2 => data_file_schema_v2(partition_type).unwrap(),
369        FormatVersion::V3 => data_file_schema_v3(partition_type).unwrap(),
370    };
371    let mut writer = new_writer(&avro_schema, writer)?;
372
373    // Wrap once and reuse across files so the field-name lookup is not rebuilt
374    // for each data file.
375    let partition_struct_type = Type::Struct(partition_type.clone());
376    for data_file in data_files {
377        let value = to_value(DataFileSerde::try_from(
378            data_file,
379            &partition_struct_type,
380            version,
381        )?)?
382        .resolve(&avro_schema)?;
383        writer.append_value(value)?;
384    }
385
386    Ok(writer.flush()?)
387}
388
389/// Parse data files from avro bytes.
390pub fn read_data_files_from_avro<R: Read>(
391    reader: &mut R,
392    schema: &Schema,
393    partition_spec_id: i32,
394    partition_type: &StructType,
395    version: FormatVersion,
396) -> Result<Vec<DataFile>> {
397    let avro_schema = match version {
398        FormatVersion::V1 => data_file_schema_v1(partition_type).unwrap(),
399        FormatVersion::V2 => data_file_schema_v2(partition_type).unwrap(),
400        FormatVersion::V3 => data_file_schema_v3(partition_type).unwrap(),
401    };
402
403    let reader = AvroReader::builder(reader)
404        .reader_schema(&avro_schema)
405        .build()?;
406    // Wrap once and reuse across files so the field-name lookup is not rebuilt
407    // for each data file.
408    let partition_struct_type = Type::Struct(partition_type.clone());
409    reader
410        .into_iter()
411        .map(|value| {
412            from_value::<DataFileSerde>(&value?)?.try_into(
413                partition_spec_id,
414                &partition_struct_type,
415                schema,
416            )
417        })
418        .collect::<Result<Vec<_>>>()
419}
420
421/// Type of content stored by the data file: data, equality deletes, or
422/// position deletes (all v1 files are data files)
423#[derive(Debug, PartialEq, Eq, Clone, Copy, Serialize, Deserialize, Default)]
424pub enum DataContentType {
425    /// value: 0
426    #[default]
427    Data = 0,
428    /// value: 1
429    PositionDeletes = 1,
430    /// value: 2
431    EqualityDeletes = 2,
432}
433
434impl TryFrom<i32> for DataContentType {
435    type Error = Error;
436
437    fn try_from(v: i32) -> Result<DataContentType> {
438        match v {
439            0 => Ok(DataContentType::Data),
440            1 => Ok(DataContentType::PositionDeletes),
441            2 => Ok(DataContentType::EqualityDeletes),
442            _ => Err(invalid_data!("data content type {v} is invalid")),
443        }
444    }
445}
446
447/// Format of this data.
448#[derive(Debug, PartialEq, Eq, Clone, Copy, SerializeDisplay, DeserializeFromStr)]
449pub enum DataFileFormat {
450    /// Avro file format: <https://avro.apache.org/>
451    Avro,
452    /// Orc file format: <https://orc.apache.org/>
453    Orc,
454    /// Parquet file format: <https://parquet.apache.org/>
455    Parquet,
456    /// Puffin file format: <https://iceberg.apache.org/puffin-spec/>
457    Puffin,
458}
459
460impl FromStr for DataFileFormat {
461    type Err = Error;
462
463    fn from_str(s: &str) -> Result<Self> {
464        match s.to_lowercase().as_str() {
465            "avro" => Ok(Self::Avro),
466            "orc" => Ok(Self::Orc),
467            "parquet" => Ok(Self::Parquet),
468            "puffin" => Ok(Self::Puffin),
469            _ => Err(invalid_data!("Unsupported data file format: {s}")),
470        }
471    }
472}
473
474impl std::fmt::Display for DataFileFormat {
475    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
476        match self {
477            DataFileFormat::Avro => write!(f, "avro"),
478            DataFileFormat::Orc => write!(f, "orc"),
479            DataFileFormat::Parquet => write!(f, "parquet"),
480            DataFileFormat::Puffin => write!(f, "puffin"),
481        }
482    }
483}
484
485#[cfg(test)]
486mod test {
487    use crate::ErrorKind;
488    use crate::spec::{
489        DataContentType, DataFileBuilder, DataFileFormat, Literal, NestedField, PrimitiveType,
490        Struct, StructType, Type,
491    };
492
493    #[test]
494    fn test_data_content_type_default() {
495        assert_eq!(DataContentType::default(), DataContentType::Data);
496    }
497
498    #[test]
499    fn test_data_content_type_default_value() {
500        assert_eq!(DataContentType::default() as i32, 0);
501    }
502
503    #[test]
504    fn promoted_partition_widens_int_and_float() {
505        // Partition columns promoted int -> long and float -> double; a file
506        // written before the promotion still carries the narrower types.
507        let current = StructType::new(vec![
508            NestedField::required(1000, "b", Type::Primitive(PrimitiveType::Long)).into(),
509            NestedField::required(1001, "c", Type::Primitive(PrimitiveType::Double)).into(),
510        ]);
511        let written = StructType::new(vec![
512            NestedField::required(1000, "b", Type::Primitive(PrimitiveType::Int)).into(),
513            NestedField::required(1001, "c", Type::Primitive(PrimitiveType::Float)).into(),
514        ]);
515
516        let data_file = DataFileBuilder::default()
517            .content(DataContentType::Data)
518            .file_path("x.parquet".to_string())
519            .file_format(DataFileFormat::Parquet)
520            .partition(Struct::from_iter([
521                Some(Literal::int(7)),
522                Some(Literal::float(1.5_f32)),
523            ]))
524            .record_count(1)
525            .file_size_in_bytes(1)
526            .partition_spec_id(0)
527            .build()
528            .unwrap();
529
530        assert_eq!(
531            data_file.promoted_partition(&current, &written).unwrap(),
532            Struct::from_iter([Some(Literal::long(7)), Some(Literal::double(1.5))])
533        );
534    }
535
536    #[test]
537    fn promoted_partition_widens_dropped_numeric_source() {
538        // A dropped partition source resolves to `Unknown`, so there is no current
539        // type; a numeric still widens to its ceiling by its written type, keeping it
540        // aligned with deletes written while the source still existed.
541        let current = StructType::new(vec![
542            NestedField::required(1000, "b", Type::Primitive(PrimitiveType::Unknown)).into(),
543        ]);
544        let written = StructType::new(vec![
545            NestedField::required(1000, "b", Type::Primitive(PrimitiveType::Int)).into(),
546        ]);
547
548        let data_file = DataFileBuilder::default()
549            .content(DataContentType::Data)
550            .file_path("x.parquet".to_string())
551            .file_format(DataFileFormat::Parquet)
552            .partition(Struct::from_iter([Some(Literal::int(7))]))
553            .record_count(1)
554            .file_size_in_bytes(1)
555            .partition_spec_id(0)
556            .build()
557            .unwrap();
558
559        assert_eq!(
560            data_file.promoted_partition(&current, &written).unwrap(),
561            Struct::from_iter([Some(Literal::long(7))])
562        );
563    }
564
565    #[test]
566    fn promoted_partition_preserves_dropped_date_source() {
567        // A `date` is stored as an `int`, so widening a dropped date the way a dropped
568        // int widens would turn it into a `long` and diverge from delete manifests that
569        // skip normalization. A dropped non-numeric source is copied through unchanged.
570        let current = StructType::new(vec![
571            NestedField::required(1000, "d", Type::Primitive(PrimitiveType::Unknown)).into(),
572        ]);
573        let written = StructType::new(vec![
574            NestedField::required(1000, "d", Type::Primitive(PrimitiveType::Date)).into(),
575        ]);
576
577        let data_file = DataFileBuilder::default()
578            .content(DataContentType::Data)
579            .file_path("x.parquet".to_string())
580            .file_format(DataFileFormat::Parquet)
581            .partition(Struct::from_iter([Some(Literal::date(20_000))]))
582            .record_count(1)
583            .file_size_in_bytes(1)
584            .partition_spec_id(0)
585            .build()
586            .unwrap();
587
588        assert_eq!(
589            data_file.promoted_partition(&current, &written).unwrap(),
590            data_file.partition
591        );
592    }
593
594    #[test]
595    fn promoted_partition_leaves_matching_types_unchanged() {
596        let partition_type = StructType::new(vec![
597            NestedField::required(1000, "b", Type::Primitive(PrimitiveType::Long)).into(),
598        ]);
599
600        let data_file = DataFileBuilder::default()
601            .content(DataContentType::Data)
602            .file_path("x.parquet".to_string())
603            .file_format(DataFileFormat::Parquet)
604            .partition(Struct::from_iter([Some(Literal::long(7))]))
605            .record_count(1)
606            .file_size_in_bytes(1)
607            .partition_spec_id(0)
608            .build()
609            .unwrap();
610
611        assert_eq!(
612            data_file
613                .promoted_partition(&partition_type, &partition_type)
614                .unwrap(),
615            data_file.partition
616        );
617    }
618
619    #[test]
620    fn promoted_partition_rejects_tuple_length_mismatch() {
621        let data_file = |values: Vec<Option<Literal>>| {
622            DataFileBuilder::default()
623                .content(DataContentType::Data)
624                .file_path("x.parquet".to_string())
625                .file_format(DataFileFormat::Parquet)
626                .partition(Struct::from_iter(values))
627                .record_count(1)
628                .file_size_in_bytes(1)
629                .partition_spec_id(0)
630                .build()
631                .unwrap()
632        };
633        let one_field = StructType::new(vec![
634            NestedField::required(1000, "b", Type::Primitive(PrimitiveType::Long)).into(),
635        ]);
636        let two_fields = StructType::new(vec![
637            NestedField::required(1000, "b", Type::Primitive(PrimitiveType::Long)).into(),
638            NestedField::required(1001, "c", Type::Primitive(PrimitiveType::Long)).into(),
639        ]);
640
641        let longer = data_file(vec![Some(Literal::int(7)), Some(Literal::int(8))]);
642        assert_eq!(
643            longer
644                .promoted_partition(&one_field, &one_field)
645                .unwrap_err()
646                .kind(),
647            ErrorKind::DataInvalid
648        );
649
650        let shorter = data_file(vec![Some(Literal::int(7))]);
651        assert_eq!(
652            shorter
653                .promoted_partition(&two_fields, &two_fields)
654                .unwrap_err()
655                .kind(),
656            ErrorKind::DataInvalid
657        );
658    }
659}