Skip to main content

iceberg/spec/manifest/
entry.rs

1// Licensed to the Apache Software Foundation (ASF) under one
2// or more contributor license agreements.  See the NOTICE file
3// distributed with this work for additional information
4// regarding copyright ownership.  The ASF licenses this file
5// to you under the Apache License, Version 2.0 (the
6// "License"); you may not use this file except in compliance
7// with the License.  You may obtain a copy of the License at
8//
9//   http://www.apache.org/licenses/LICENSE-2.0
10//
11// Unless required by applicable law or agreed to in writing,
12// software distributed under the License is distributed on an
13// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14// KIND, either express or implied.  See the License for the
15// specific language governing permissions and limitations
16// under the License.
17
18use std::sync::Arc;
19
20use apache_avro::Schema as AvroSchema;
21use once_cell::sync::Lazy;
22use typed_builder::TypedBuilder;
23
24use crate::Error;
25use crate::avro::schema_to_avro_schema;
26use crate::error::{Result, invalid_data};
27use crate::spec::{
28    DataContentType, DataFile, INITIAL_SEQUENCE_NUMBER, ListType, Literal, ManifestFile, MapType,
29    NestedField, NestedFieldRef, PrimitiveLiteral, PrimitiveType, Schema, StructType, Type,
30};
31
32/// Reference to [`ManifestEntry`].
33pub type ManifestEntryRef = Arc<ManifestEntry>;
34
35/// A manifest is an immutable Avro file that lists data files or delete
36/// files, along with each file’s partition data tuple, metrics, and tracking
37/// information.
38#[derive(Debug, PartialEq, Eq, Clone, TypedBuilder)]
39pub struct ManifestEntry {
40    /// field: 0
41    ///
42    /// Used to track additions and deletions.
43    pub status: ManifestStatus,
44    /// field id: 1
45    ///
46    /// Snapshot id where the file was added, or deleted if status is 2.
47    /// Inherited when null.
48    #[builder(default, setter(strip_option(fallback = snapshot_id_opt)))]
49    pub snapshot_id: Option<i64>,
50    /// field id: 3
51    ///
52    /// Data sequence number of the file.
53    /// Inherited when null and status is 1 (added).
54    #[builder(default, setter(strip_option(fallback = sequence_number_opt)))]
55    pub sequence_number: Option<i64>,
56    /// field id: 4
57    ///
58    /// File sequence number indicating when the file was added.
59    /// Inherited when null and status is 1 (added).
60    #[builder(default, setter(strip_option(fallback = file_sequence_number_opt)))]
61    pub file_sequence_number: Option<i64>,
62    /// field id: 2
63    ///
64    /// File path, partition tuple, metrics, …
65    pub data_file: DataFile,
66}
67
68impl ManifestEntry {
69    /// Check if this manifest entry is deleted.
70    pub fn is_alive(&self) -> bool {
71        matches!(
72            self.status,
73            ManifestStatus::Added | ManifestStatus::Existing
74        )
75    }
76
77    /// Status of this manifest entry
78    pub fn status(&self) -> ManifestStatus {
79        self.status
80    }
81
82    /// Content type of this manifest entry.
83    #[inline]
84    pub fn content_type(&self) -> DataContentType {
85        self.data_file.content
86    }
87
88    /// File format of this manifest entry.
89    #[inline]
90    pub fn file_format(&self) -> DataFileFormat {
91        self.data_file.file_format
92    }
93
94    /// Data file path of this manifest entry.
95    #[inline]
96    pub fn file_path(&self) -> &str {
97        &self.data_file.file_path
98    }
99
100    /// Data file record count of the manifest entry.
101    #[inline]
102    pub fn record_count(&self) -> u64 {
103        self.data_file.record_count
104    }
105
106    /// Inherit data from manifest list, such as snapshot id, sequence number.
107    pub(crate) fn inherit_data(&mut self, snapshot_entry: &ManifestFile) {
108        if self.snapshot_id.is_none() {
109            self.snapshot_id = Some(snapshot_entry.added_snapshot_id);
110        }
111
112        if self.sequence_number.is_none()
113            && (self.status == ManifestStatus::Added
114                || snapshot_entry.sequence_number == INITIAL_SEQUENCE_NUMBER)
115        {
116            self.sequence_number = Some(snapshot_entry.sequence_number);
117        }
118
119        if self.file_sequence_number.is_none()
120            && (self.status == ManifestStatus::Added
121                || snapshot_entry.sequence_number == INITIAL_SEQUENCE_NUMBER)
122        {
123            self.file_sequence_number = Some(snapshot_entry.sequence_number);
124        }
125    }
126
127    /// Snapshot id
128    #[inline]
129    pub fn snapshot_id(&self) -> Option<i64> {
130        self.snapshot_id
131    }
132
133    /// Data sequence number.
134    #[inline]
135    pub fn sequence_number(&self) -> Option<i64> {
136        self.sequence_number
137    }
138
139    /// File size in bytes.
140    #[inline]
141    pub fn file_size_in_bytes(&self) -> u64 {
142        self.data_file.file_size_in_bytes
143    }
144
145    /// get a reference to the actual data file
146    #[inline]
147    pub fn data_file(&self) -> &DataFile {
148        &self.data_file
149    }
150}
151
152/// Used to track additions and deletions in ManifestEntry.
153#[derive(Debug, PartialEq, Eq, Clone, Copy)]
154pub enum ManifestStatus {
155    /// Value: 0
156    Existing = 0,
157    /// Value: 1
158    Added = 1,
159    /// Value: 2
160    ///
161    /// Deletes are informational only and not used in scans.
162    Deleted = 2,
163}
164
165impl TryFrom<i32> for ManifestStatus {
166    type Error = Error;
167
168    fn try_from(v: i32) -> Result<ManifestStatus> {
169        match v {
170            0 => Ok(ManifestStatus::Existing),
171            1 => Ok(ManifestStatus::Added),
172            2 => Ok(ManifestStatus::Deleted),
173            _ => Err(invalid_data!("manifest status {v} is invalid")),
174        }
175    }
176}
177
178use super::DataFileFormat;
179
180/// Field ID of `manifest_entry.data_file`.
181pub(crate) const DATA_FILE_FIELD_ID: i32 = 2;
182/// Field ID of `manifest_entry.sequence_number`.
183pub(crate) const SEQUENCE_NUMBER_FIELD_ID: i32 = 3;
184/// Field ID of `manifest_entry.file_sequence_number`.
185pub(crate) const FILE_SEQUENCE_NUMBER_FIELD_ID: i32 = 4;
186/// Field ID of `data_file.partition`.
187pub(crate) const DATA_FILE_PARTITION_FIELD_ID: i32 = 102;
188/// Field ID of `data_file.content`.
189pub(crate) const DATA_FILE_CONTENT_FIELD_ID: i32 = 134;
190
191static STATUS: Lazy<NestedFieldRef> = {
192    Lazy::new(|| {
193        Arc::new(NestedField::required(
194            0,
195            "status",
196            Type::Primitive(PrimitiveType::Int),
197        ))
198    })
199};
200
201static SNAPSHOT_ID_V1: Lazy<NestedFieldRef> = {
202    Lazy::new(|| {
203        Arc::new(NestedField::required(
204            1,
205            "snapshot_id",
206            Type::Primitive(PrimitiveType::Long),
207        ))
208    })
209};
210
211static SNAPSHOT_ID_V2: Lazy<NestedFieldRef> = {
212    Lazy::new(|| {
213        Arc::new(NestedField::optional(
214            1,
215            "snapshot_id",
216            Type::Primitive(PrimitiveType::Long),
217        ))
218    })
219};
220
221static SEQUENCE_NUMBER: Lazy<NestedFieldRef> = {
222    Lazy::new(|| {
223        Arc::new(NestedField::optional(
224            SEQUENCE_NUMBER_FIELD_ID,
225            "sequence_number",
226            Type::Primitive(PrimitiveType::Long),
227        ))
228    })
229};
230
231static FILE_SEQUENCE_NUMBER: Lazy<NestedFieldRef> = {
232    Lazy::new(|| {
233        Arc::new(NestedField::optional(
234            FILE_SEQUENCE_NUMBER_FIELD_ID,
235            "file_sequence_number",
236            Type::Primitive(PrimitiveType::Long),
237        ))
238    })
239};
240
241static CONTENT: Lazy<NestedFieldRef> = {
242    Lazy::new(|| {
243        Arc::new(
244            NestedField::required(
245                DATA_FILE_CONTENT_FIELD_ID,
246                "content",
247                Type::Primitive(PrimitiveType::Int),
248            )
249            // 0 refers to DataContentType::DATA
250            .with_initial_default(Literal::Primitive(PrimitiveLiteral::Int(0))),
251        )
252    })
253};
254
255static FILE_PATH: Lazy<NestedFieldRef> = {
256    Lazy::new(|| {
257        Arc::new(NestedField::required(
258            100,
259            "file_path",
260            Type::Primitive(PrimitiveType::String),
261        ))
262    })
263};
264
265static FILE_FORMAT: Lazy<NestedFieldRef> = {
266    Lazy::new(|| {
267        Arc::new(NestedField::required(
268            101,
269            "file_format",
270            Type::Primitive(PrimitiveType::String),
271        ))
272    })
273};
274
275static RECORD_COUNT: Lazy<NestedFieldRef> = {
276    Lazy::new(|| {
277        Arc::new(NestedField::required(
278            103,
279            "record_count",
280            Type::Primitive(PrimitiveType::Long),
281        ))
282    })
283};
284
285static FILE_SIZE_IN_BYTES: Lazy<NestedFieldRef> = {
286    Lazy::new(|| {
287        Arc::new(NestedField::required(
288            104,
289            "file_size_in_bytes",
290            Type::Primitive(PrimitiveType::Long),
291        ))
292    })
293};
294
295// Deprecated. Always write a default in v1. Do not write in v2.
296static BLOCK_SIZE_IN_BYTES: Lazy<NestedFieldRef> = {
297    Lazy::new(|| {
298        Arc::new(NestedField::required(
299            105,
300            "block_size_in_bytes",
301            Type::Primitive(PrimitiveType::Long),
302        ))
303    })
304};
305
306static COLUMN_SIZES: Lazy<NestedFieldRef> = {
307    Lazy::new(|| {
308        Arc::new(NestedField::optional(
309            108,
310            "column_sizes",
311            Type::Map(MapType {
312                key_field: Arc::new(NestedField::required(
313                    117,
314                    "key",
315                    Type::Primitive(PrimitiveType::Int),
316                )),
317                value_field: Arc::new(NestedField::required(
318                    118,
319                    "value",
320                    Type::Primitive(PrimitiveType::Long),
321                )),
322            }),
323        ))
324    })
325};
326
327static VALUE_COUNTS: Lazy<NestedFieldRef> = {
328    Lazy::new(|| {
329        Arc::new(NestedField::optional(
330            109,
331            "value_counts",
332            Type::Map(MapType {
333                key_field: Arc::new(NestedField::required(
334                    119,
335                    "key",
336                    Type::Primitive(PrimitiveType::Int),
337                )),
338                value_field: Arc::new(NestedField::required(
339                    120,
340                    "value",
341                    Type::Primitive(PrimitiveType::Long),
342                )),
343            }),
344        ))
345    })
346};
347
348static NULL_VALUE_COUNTS: Lazy<NestedFieldRef> = {
349    Lazy::new(|| {
350        Arc::new(NestedField::optional(
351            110,
352            "null_value_counts",
353            Type::Map(MapType {
354                key_field: Arc::new(NestedField::required(
355                    121,
356                    "key",
357                    Type::Primitive(PrimitiveType::Int),
358                )),
359                value_field: Arc::new(NestedField::required(
360                    122,
361                    "value",
362                    Type::Primitive(PrimitiveType::Long),
363                )),
364            }),
365        ))
366    })
367};
368
369static NAN_VALUE_COUNTS: Lazy<NestedFieldRef> = {
370    Lazy::new(|| {
371        Arc::new(NestedField::optional(
372            137,
373            "nan_value_counts",
374            Type::Map(MapType {
375                key_field: Arc::new(NestedField::required(
376                    138,
377                    "key",
378                    Type::Primitive(PrimitiveType::Int),
379                )),
380                value_field: Arc::new(NestedField::required(
381                    139,
382                    "value",
383                    Type::Primitive(PrimitiveType::Long),
384                )),
385            }),
386        ))
387    })
388};
389
390static LOWER_BOUNDS: Lazy<NestedFieldRef> = {
391    Lazy::new(|| {
392        Arc::new(NestedField::optional(
393            125,
394            "lower_bounds",
395            Type::Map(MapType {
396                key_field: Arc::new(NestedField::required(
397                    126,
398                    "key",
399                    Type::Primitive(PrimitiveType::Int),
400                )),
401                value_field: Arc::new(NestedField::required(
402                    127,
403                    "value",
404                    Type::Primitive(PrimitiveType::Binary),
405                )),
406            }),
407        ))
408    })
409};
410
411static UPPER_BOUNDS: Lazy<NestedFieldRef> = {
412    Lazy::new(|| {
413        Arc::new(NestedField::optional(
414            128,
415            "upper_bounds",
416            Type::Map(MapType {
417                key_field: Arc::new(NestedField::required(
418                    129,
419                    "key",
420                    Type::Primitive(PrimitiveType::Int),
421                )),
422                value_field: Arc::new(NestedField::required(
423                    130,
424                    "value",
425                    Type::Primitive(PrimitiveType::Binary),
426                )),
427            }),
428        ))
429    })
430};
431
432static KEY_METADATA: Lazy<NestedFieldRef> = {
433    Lazy::new(|| {
434        Arc::new(NestedField::optional(
435            131,
436            "key_metadata",
437            Type::Primitive(PrimitiveType::Binary),
438        ))
439    })
440};
441
442static SPLIT_OFFSETS: Lazy<NestedFieldRef> = {
443    Lazy::new(|| {
444        Arc::new(NestedField::optional(
445            132,
446            "split_offsets",
447            Type::List(ListType {
448                element_field: Arc::new(NestedField::required(
449                    133,
450                    "element",
451                    Type::Primitive(PrimitiveType::Long),
452                )),
453            }),
454        ))
455    })
456};
457
458static EQUALITY_IDS: Lazy<NestedFieldRef> = {
459    Lazy::new(|| {
460        Arc::new(NestedField::optional(
461            135,
462            "equality_ids",
463            Type::List(ListType {
464                element_field: Arc::new(NestedField::required(
465                    136,
466                    "element",
467                    Type::Primitive(PrimitiveType::Int),
468                )),
469            }),
470        ))
471    })
472};
473
474static SORT_ORDER_ID: Lazy<NestedFieldRef> = {
475    Lazy::new(|| {
476        Arc::new(NestedField::optional(
477            140,
478            "sort_order_id",
479            Type::Primitive(PrimitiveType::Int),
480        ))
481    })
482};
483
484static FIRST_ROW_ID: Lazy<NestedFieldRef> = {
485    Lazy::new(|| {
486        Arc::new(NestedField::optional(
487            142,
488            "first_row_id",
489            Type::Primitive(PrimitiveType::Long),
490        ))
491    })
492};
493
494static REFERENCE_DATA_FILE: Lazy<NestedFieldRef> = {
495    Lazy::new(|| {
496        Arc::new(NestedField::optional(
497            143,
498            "referenced_data_file",
499            Type::Primitive(PrimitiveType::String),
500        ))
501    })
502};
503
504static CONTENT_OFFSET: Lazy<NestedFieldRef> = {
505    Lazy::new(|| {
506        Arc::new(NestedField::optional(
507            144,
508            "content_offset",
509            Type::Primitive(PrimitiveType::Long),
510        ))
511    })
512};
513
514static CONTENT_SIZE_IN_BYTES: Lazy<NestedFieldRef> = {
515    Lazy::new(|| {
516        Arc::new(NestedField::optional(
517            145,
518            "content_size_in_bytes",
519            Type::Primitive(PrimitiveType::Long),
520        ))
521    })
522};
523
524fn data_file_fields_v3(partition_type: &StructType) -> Vec<NestedFieldRef> {
525    vec![
526        CONTENT.clone(),
527        FILE_PATH.clone(),
528        FILE_FORMAT.clone(),
529        Arc::new(NestedField::required(
530            DATA_FILE_PARTITION_FIELD_ID,
531            "partition",
532            Type::Struct(partition_type.clone()),
533        )),
534        RECORD_COUNT.clone(),
535        FILE_SIZE_IN_BYTES.clone(),
536        COLUMN_SIZES.clone(),
537        VALUE_COUNTS.clone(),
538        NULL_VALUE_COUNTS.clone(),
539        NAN_VALUE_COUNTS.clone(),
540        LOWER_BOUNDS.clone(),
541        UPPER_BOUNDS.clone(),
542        KEY_METADATA.clone(),
543        SPLIT_OFFSETS.clone(),
544        EQUALITY_IDS.clone(),
545        SORT_ORDER_ID.clone(),
546        FIRST_ROW_ID.clone(),
547        REFERENCE_DATA_FILE.clone(),
548        CONTENT_OFFSET.clone(),
549        CONTENT_SIZE_IN_BYTES.clone(),
550    ]
551}
552
553/// Fields of the `manifest_entry` struct in the latest format version. A
554/// [`ManifestEntry`] of any version fits them: earlier versions lack only
555/// optional fields, apart from the deprecated v1 `block_size_in_bytes`, which
556/// [`DataFile`] doesn't keep.
557pub(crate) fn manifest_entry_fields(partition_type: &StructType) -> Vec<NestedFieldRef> {
558    vec![
559        STATUS.clone(),
560        SNAPSHOT_ID_V2.clone(),
561        SEQUENCE_NUMBER.clone(),
562        FILE_SEQUENCE_NUMBER.clone(),
563        Arc::new(NestedField::required(
564            DATA_FILE_FIELD_ID,
565            "data_file",
566            Type::Struct(StructType::new(data_file_fields_v3(partition_type))),
567        )),
568    ]
569}
570
571pub(super) fn data_file_schema_v3(partition_type: &StructType) -> Result<AvroSchema> {
572    let schema = Schema::builder()
573        .with_fields(data_file_fields_v3(partition_type))
574        .build()?;
575    schema_to_avro_schema("data_file", &schema)
576}
577
578fn data_file_fields_v2(partition_type: &StructType) -> Vec<NestedFieldRef> {
579    vec![
580        CONTENT.clone(),
581        FILE_PATH.clone(),
582        FILE_FORMAT.clone(),
583        Arc::new(NestedField::required(
584            DATA_FILE_PARTITION_FIELD_ID,
585            "partition",
586            Type::Struct(partition_type.clone()),
587        )),
588        RECORD_COUNT.clone(),
589        FILE_SIZE_IN_BYTES.clone(),
590        COLUMN_SIZES.clone(),
591        VALUE_COUNTS.clone(),
592        NULL_VALUE_COUNTS.clone(),
593        NAN_VALUE_COUNTS.clone(),
594        LOWER_BOUNDS.clone(),
595        UPPER_BOUNDS.clone(),
596        KEY_METADATA.clone(),
597        SPLIT_OFFSETS.clone(),
598        EQUALITY_IDS.clone(),
599        SORT_ORDER_ID.clone(),
600        REFERENCE_DATA_FILE.clone(),
601    ]
602}
603
604pub(super) fn data_file_schema_v2(partition_type: &StructType) -> Result<AvroSchema> {
605    let schema = Schema::builder()
606        .with_fields(data_file_fields_v2(partition_type))
607        .build()?;
608    schema_to_avro_schema("data_file", &schema)
609}
610
611pub(super) fn manifest_schema_v3(partition_type: &StructType) -> Result<AvroSchema> {
612    manifest_schema_v2_v3(data_file_fields_v3(partition_type))
613}
614
615pub(super) fn manifest_schema_v2(partition_type: &StructType) -> Result<AvroSchema> {
616    manifest_schema_v2_v3(data_file_fields_v2(partition_type))
617}
618
619/// Root manifest schema is unchanged between format version 2 and 3.
620/// Manifest entry schema is different between versions.
621fn manifest_schema_v2_v3(data_file_fields: Vec<NestedFieldRef>) -> Result<AvroSchema> {
622    let fields = vec![
623        STATUS.clone(),
624        SNAPSHOT_ID_V2.clone(),
625        SEQUENCE_NUMBER.clone(),
626        FILE_SEQUENCE_NUMBER.clone(),
627        Arc::new(NestedField::required(
628            DATA_FILE_FIELD_ID,
629            "data_file",
630            Type::Struct(StructType::new(data_file_fields)),
631        )),
632    ];
633    let schema = Schema::builder().with_fields(fields).build()?;
634    schema_to_avro_schema("manifest_entry", &schema)
635}
636
637fn data_file_fields_v1(partition_type: &StructType) -> Vec<NestedFieldRef> {
638    vec![
639        FILE_PATH.clone(),
640        FILE_FORMAT.clone(),
641        Arc::new(NestedField::required(
642            DATA_FILE_PARTITION_FIELD_ID,
643            "partition",
644            Type::Struct(partition_type.clone()),
645        )),
646        RECORD_COUNT.clone(),
647        FILE_SIZE_IN_BYTES.clone(),
648        BLOCK_SIZE_IN_BYTES.clone(),
649        COLUMN_SIZES.clone(),
650        VALUE_COUNTS.clone(),
651        NULL_VALUE_COUNTS.clone(),
652        NAN_VALUE_COUNTS.clone(),
653        LOWER_BOUNDS.clone(),
654        UPPER_BOUNDS.clone(),
655        KEY_METADATA.clone(),
656        SPLIT_OFFSETS.clone(),
657        SORT_ORDER_ID.clone(),
658    ]
659}
660
661pub(super) fn data_file_schema_v1(partition_type: &StructType) -> Result<AvroSchema> {
662    let schema = Schema::builder()
663        .with_fields(data_file_fields_v1(partition_type))
664        .build()?;
665    schema_to_avro_schema("data_file", &schema)
666}
667
668pub(super) fn manifest_schema_v1(partition_type: &StructType) -> Result<AvroSchema> {
669    let fields = vec![
670        STATUS.clone(),
671        SNAPSHOT_ID_V1.clone(),
672        Arc::new(NestedField::required(
673            DATA_FILE_FIELD_ID,
674            "data_file",
675            Type::Struct(StructType::new(data_file_fields_v1(partition_type))),
676        )),
677    ];
678    let schema = Schema::builder().with_fields(fields).build()?;
679    schema_to_avro_schema("manifest_entry", &schema)
680}