Skip to main content

iceberg/spec/manifest/
writer.rs

1// Licensed to the Apache Software Foundation (ASF) under one
2// or more contributor license agreements.  See the NOTICE file
3// distributed with this work for additional information
4// regarding copyright ownership.  The ASF licenses this file
5// to you under the Apache License, Version 2.0 (the
6// "License"); you may not use this file except in compliance
7// with the License.  You may obtain a copy of the License at
8//
9//   http://www.apache.org/licenses/LICENSE-2.0
10//
11// Unless required by applicable law or agreed to in writing,
12// software distributed under the License is distributed on an
13// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14// KIND, either express or implied.  See the License for the
15// specific language governing permissions and limitations
16// under the License.
17
18use std::cmp::min;
19
20use apache_avro::to_value;
21use bytes::Bytes;
22use itertools::Itertools;
23use serde_json::to_vec;
24
25use super::{
26    Datum, FormatVersion, ManifestContentType, PartitionSpec, PrimitiveType,
27    UNASSIGNED_SEQUENCE_NUMBER,
28};
29use crate::avro::new_writer;
30use crate::encryption::EncryptedOutputFile;
31use crate::error::{Result, invalid_data};
32use crate::io::{FileMetadata, FileWrite, OutputFile};
33use crate::spec::manifest::_serde::{ManifestEntryV1, ManifestEntryV2, ManifestEntryV3};
34use crate::spec::manifest::{manifest_schema_v1, manifest_schema_v2, manifest_schema_v3};
35use crate::spec::{
36    DataContentType, DataFile, FieldSummary, ManifestEntry, ManifestFile, ManifestMetadata,
37    ManifestStatus, PrimitiveLiteral, SchemaRef, StructType, Type,
38};
39
40/// Placeholder for snapshot ID. The field with this value must be replaced
41/// with the actual snapshot ID before it is committed.
42const UNASSIGNED_SNAPSHOT_ID: i64 = -1;
43
44/// Retains the output until close provides the size required by encrypted key metadata.
45pub(crate) enum ManifestOutput {
46    Plain(OutputFile),
47    Encrypted(EncryptedOutputFile),
48}
49
50impl ManifestOutput {
51    async fn writer(&self) -> Result<Box<dyn FileWrite>> {
52        match self {
53            Self::Plain(output) => output.writer().await,
54            Self::Encrypted(output) => output.writer().await,
55        }
56    }
57
58    fn encoded_key_metadata(&self, file_metadata: &FileMetadata) -> Result<Option<Vec<u8>>> {
59        match self {
60            Self::Plain(_) => Ok(None),
61            Self::Encrypted(output) => Ok(Some(
62                output
63                    .key_metadata_with_saved_file_metadata(file_metadata)
64                    .encode()?
65                    .into_vec(),
66            )),
67        }
68    }
69}
70
71/// The builder used to create a [`ManifestWriter`].
72pub struct ManifestWriterBuilder {
73    output: ManifestOutput,
74    location: String,
75    snapshot_id: Option<i64>,
76    schema: SchemaRef,
77    partition_spec: PartitionSpec,
78}
79
80impl ManifestWriterBuilder {
81    /// Create a new builder for unencrypted manifests.
82    pub fn new(
83        output: OutputFile,
84        snapshot_id: Option<i64>,
85        schema: SchemaRef,
86        partition_spec: PartitionSpec,
87    ) -> Self {
88        let location = output.location().to_owned();
89        Self {
90            output: ManifestOutput::Plain(output),
91            location,
92            snapshot_id,
93            schema,
94            partition_spec,
95        }
96    }
97
98    /// Create a new builder from an [`EncryptedOutputFile`].
99    ///
100    /// Use this when writing manifests with transparent encryption.
101    pub fn new_from_encrypted(
102        encrypted_output: EncryptedOutputFile,
103        snapshot_id: Option<i64>,
104        schema: SchemaRef,
105        partition_spec: PartitionSpec,
106    ) -> Result<Self> {
107        let location = encrypted_output.location().to_owned();
108        Ok(Self {
109            output: ManifestOutput::Encrypted(encrypted_output),
110            location,
111            snapshot_id,
112            schema,
113            partition_spec,
114        })
115    }
116
117    /// Build a [`ManifestWriter`] for format version 1.
118    pub fn build_v1(self) -> ManifestWriter {
119        let metadata = ManifestMetadata::builder()
120            .schema_id(self.schema.schema_id())
121            .schema(self.schema)
122            .partition_spec(self.partition_spec)
123            .format_version(FormatVersion::V1)
124            .content(ManifestContentType::Data)
125            .build();
126        ManifestWriter::new(self.output, self.location, self.snapshot_id, metadata, None)
127    }
128
129    /// Build a [`ManifestWriter`] for format version 2, data content.
130    pub fn build_v2_data(self) -> ManifestWriter {
131        let metadata = ManifestMetadata::builder()
132            .schema_id(self.schema.schema_id())
133            .schema(self.schema)
134            .partition_spec(self.partition_spec)
135            .format_version(FormatVersion::V2)
136            .content(ManifestContentType::Data)
137            .build();
138
139        ManifestWriter::new(self.output, self.location, self.snapshot_id, metadata, None)
140    }
141
142    /// Build a [`ManifestWriter`] for format version 2, deletes content.
143    pub fn build_v2_deletes(self) -> ManifestWriter {
144        let metadata = ManifestMetadata::builder()
145            .schema_id(self.schema.schema_id())
146            .schema(self.schema)
147            .partition_spec(self.partition_spec)
148            .format_version(FormatVersion::V2)
149            .content(ManifestContentType::Deletes)
150            .build();
151        ManifestWriter::new(self.output, self.location, self.snapshot_id, metadata, None)
152    }
153
154    /// Build a [`ManifestWriter`] for format version 2, data content.
155    pub fn build_v3_data(self) -> ManifestWriter {
156        let metadata = ManifestMetadata::builder()
157            .schema_id(self.schema.schema_id())
158            .schema(self.schema)
159            .partition_spec(self.partition_spec)
160            .format_version(FormatVersion::V3)
161            .content(ManifestContentType::Data)
162            .build();
163        ManifestWriter::new(
164            self.output,
165            self.location,
166            self.snapshot_id,
167            metadata,
168            // First row id is assigned by the [`ManifestListWriter`] when the manifest
169            // is added to the list.
170            None,
171        )
172    }
173
174    /// Build a [`ManifestWriter`] for format version 3, deletes content.
175    pub fn build_v3_deletes(self) -> ManifestWriter {
176        let metadata = ManifestMetadata::builder()
177            .schema_id(self.schema.schema_id())
178            .schema(self.schema)
179            .partition_spec(self.partition_spec)
180            .format_version(FormatVersion::V3)
181            .content(ManifestContentType::Deletes)
182            .build();
183        ManifestWriter::new(self.output, self.location, self.snapshot_id, metadata, None)
184    }
185}
186
187/// A manifest writer.
188pub struct ManifestWriter {
189    output: ManifestOutput,
190    location: String,
191
192    snapshot_id: Option<i64>,
193
194    added_files: u32,
195    added_rows: u64,
196    existing_files: u32,
197    existing_rows: u64,
198    deleted_files: u32,
199    deleted_rows: u64,
200    first_row_id: Option<u64>,
201
202    min_seq_num: Option<i64>,
203
204    manifest_entries: Vec<ManifestEntry>,
205
206    metadata: ManifestMetadata,
207}
208
209impl ManifestWriter {
210    /// Create a new manifest writer.
211    pub(crate) fn new(
212        output: ManifestOutput,
213        location: String,
214        snapshot_id: Option<i64>,
215        metadata: ManifestMetadata,
216        first_row_id: Option<u64>,
217    ) -> Self {
218        Self {
219            output,
220            location,
221            snapshot_id,
222            added_files: 0,
223            added_rows: 0,
224            existing_files: 0,
225            existing_rows: 0,
226            deleted_files: 0,
227            deleted_rows: 0,
228            first_row_id,
229            min_seq_num: None,
230            manifest_entries: Vec::new(),
231            metadata,
232        }
233    }
234
235    fn construct_partition_summaries(
236        &mut self,
237        partition_type: &StructType,
238    ) -> Result<Vec<FieldSummary>> {
239        let mut field_stats: Vec<_> = partition_type
240            .fields()
241            .iter()
242            .map(|f| PartitionFieldStats::new(f.field_type.as_primitive_type().unwrap().clone()))
243            .collect();
244        for partition in self.manifest_entries.iter().map(|e| &e.data_file.partition) {
245            for (literal, stat) in partition.iter().zip_eq(field_stats.iter_mut()) {
246                let primitive_literal = literal.map(|v| v.as_primitive_literal().unwrap());
247                stat.update(primitive_literal)?;
248            }
249        }
250        Ok(field_stats.into_iter().map(|stat| stat.finish()).collect())
251    }
252
253    fn check_data_file(&self, data_file: &DataFile) -> Result<()> {
254        match self.metadata.content {
255            ManifestContentType::Data => {
256                if data_file.content != DataContentType::Data {
257                    return Err(invalid_data!(
258                        "Date file at path {} with manifest content type `data`, should have DataContentType `Data`, but has `{:?}`",
259                        data_file.file_path(),
260                        data_file.content
261                    ));
262                }
263            }
264            ManifestContentType::Deletes => {
265                if data_file.content != DataContentType::EqualityDeletes
266                    && data_file.content != DataContentType::PositionDeletes
267                {
268                    return Err(invalid_data!(
269                        "Date file at path {} with manifest content type `deletes`, should have DataContentType `Data`, but has `{:?}`",
270                        data_file.file_path(),
271                        data_file.content
272                    ));
273                }
274            }
275        }
276        Ok(())
277    }
278
279    /// Add a new manifest entry. This method will update following status of the entry:
280    /// - Update the entry status to `Added`
281    /// - Set the snapshot id to the current snapshot id
282    /// - Set the sequence number to `None` if it is invalid(smaller than 0)
283    /// - Set the file sequence number to `None`
284    pub(crate) fn add_entry(&mut self, mut entry: ManifestEntry) -> Result<()> {
285        self.check_data_file(&entry.data_file)?;
286        if entry.sequence_number().is_some_and(|n| n >= 0) {
287            entry.status = ManifestStatus::Added;
288            entry.snapshot_id = self.snapshot_id;
289            entry.file_sequence_number = None;
290        } else {
291            entry.status = ManifestStatus::Added;
292            entry.snapshot_id = self.snapshot_id;
293            entry.sequence_number = None;
294            entry.file_sequence_number = None;
295        };
296        self.add_entry_inner(entry)?;
297        Ok(())
298    }
299
300    /// Add file as an added entry with a specific sequence number. The entry's snapshot ID will be this manifest's snapshot ID. The entry's data sequence
301    /// number will be the provided data sequence number. The entry's file sequence number will be
302    /// assigned at commit.
303    pub fn add_file(&mut self, data_file: DataFile, sequence_number: i64) -> Result<()> {
304        self.check_data_file(&data_file)?;
305        let entry = ManifestEntry {
306            status: ManifestStatus::Added,
307            snapshot_id: self.snapshot_id,
308            sequence_number: (sequence_number >= 0).then_some(sequence_number),
309            file_sequence_number: None,
310            data_file,
311        };
312        self.add_entry_inner(entry)?;
313        Ok(())
314    }
315
316    /// Add a delete manifest entry. This method will update following status of the entry:
317    /// - Update the entry status to `Deleted`
318    /// - Set the snapshot id to the current snapshot id
319    ///
320    /// # TODO
321    /// Remove this allow later
322    #[allow(dead_code)]
323    pub(crate) fn add_delete_entry(&mut self, mut entry: ManifestEntry) -> Result<()> {
324        self.check_data_file(&entry.data_file)?;
325        entry.status = ManifestStatus::Deleted;
326        entry.snapshot_id = self.snapshot_id;
327        self.add_entry_inner(entry)?;
328        Ok(())
329    }
330
331    /// Add a file as delete manifest entry. The entry's snapshot ID will be this manifest's snapshot ID.
332    /// However, the original data and file sequence numbers of the file must be preserved when
333    /// the file is marked as deleted.
334    pub fn add_delete_file(
335        &mut self,
336        data_file: DataFile,
337        sequence_number: i64,
338        file_sequence_number: Option<i64>,
339    ) -> Result<()> {
340        self.check_data_file(&data_file)?;
341        let entry = ManifestEntry {
342            status: ManifestStatus::Deleted,
343            snapshot_id: self.snapshot_id,
344            sequence_number: Some(sequence_number),
345            file_sequence_number,
346            data_file,
347        };
348        self.add_entry_inner(entry)?;
349        Ok(())
350    }
351
352    /// Add an existing manifest entry. This method will update following status of the entry:
353    /// - Update the entry status to `Existing`
354    ///
355    /// # TODO
356    /// Remove this allow later
357    #[allow(dead_code)]
358    pub(crate) fn add_existing_entry(&mut self, mut entry: ManifestEntry) -> Result<()> {
359        self.check_data_file(&entry.data_file)?;
360        entry.status = ManifestStatus::Existing;
361        self.add_entry_inner(entry)?;
362        Ok(())
363    }
364
365    /// Add an file as existing manifest entry. The original data and file sequence numbers, snapshot ID,
366    /// which were assigned at commit, must be preserved when adding an existing entry.
367    pub fn add_existing_file(
368        &mut self,
369        data_file: DataFile,
370        snapshot_id: i64,
371        sequence_number: i64,
372        file_sequence_number: Option<i64>,
373    ) -> Result<()> {
374        self.check_data_file(&data_file)?;
375        let entry = ManifestEntry {
376            status: ManifestStatus::Existing,
377            snapshot_id: Some(snapshot_id),
378            sequence_number: Some(sequence_number),
379            file_sequence_number,
380            data_file,
381        };
382        self.add_entry_inner(entry)?;
383        Ok(())
384    }
385
386    fn add_entry_inner(&mut self, entry: ManifestEntry) -> Result<()> {
387        // Check if the entry has sequence number
388        if (entry.status == ManifestStatus::Deleted || entry.status == ManifestStatus::Existing)
389            && (entry.sequence_number.is_none() || entry.file_sequence_number.is_none())
390        {
391            return Err(invalid_data!(
392                "Manifest entry with status Existing or Deleted should have sequence number"
393            ));
394        }
395
396        // Update the statistics
397        match entry.status {
398            ManifestStatus::Added => {
399                self.added_files += 1;
400                self.added_rows += entry.data_file.record_count;
401            }
402            ManifestStatus::Deleted => {
403                self.deleted_files += 1;
404                self.deleted_rows += entry.data_file.record_count;
405            }
406            ManifestStatus::Existing => {
407                self.existing_files += 1;
408                self.existing_rows += entry.data_file.record_count;
409            }
410        }
411        if entry.is_alive()
412            && let Some(seq_num) = entry.sequence_number
413        {
414            self.min_seq_num = Some(self.min_seq_num.map_or(seq_num, |v| min(v, seq_num)));
415        }
416        self.manifest_entries.push(entry);
417        Ok(())
418    }
419
420    /// Write manifest file and return it.
421    pub async fn write_manifest_file(mut self) -> Result<ManifestFile> {
422        // Create the avro writer
423        let partition_type = self
424            .metadata
425            .partition_spec
426            .partition_type(&self.metadata.schema)?;
427        // Wrap once and reuse for every entry so the per-entry conversion does
428        // not rebuild the partition type's field-name lookup on each call.
429        let partition_struct_type = Type::Struct(partition_type.clone());
430        let table_schema = &self.metadata.schema;
431        let avro_schema = match self.metadata.format_version {
432            FormatVersion::V1 => manifest_schema_v1(&partition_type)?,
433            FormatVersion::V2 => manifest_schema_v2(&partition_type)?,
434            FormatVersion::V3 => manifest_schema_v3(&partition_type)?,
435        };
436        let mut avro_writer = new_writer(&avro_schema, Vec::new())?;
437        avro_writer.add_user_metadata(
438            "schema".to_string(),
439            to_vec(table_schema)
440                .map_err(|err| invalid_data!("Fail to serialize table schema").with_source(err))?,
441        )?;
442        avro_writer.add_user_metadata(
443            "schema-id".to_string(),
444            table_schema.schema_id().to_string(),
445        )?;
446        avro_writer.add_user_metadata(
447            "partition-spec".to_string(),
448            to_vec(&self.metadata.partition_spec.fields()).map_err(|err| {
449                invalid_data!("Fail to serialize partition spec").with_source(err)
450            })?,
451        )?;
452        avro_writer.add_user_metadata(
453            "partition-spec-id".to_string(),
454            self.metadata.partition_spec.spec_id().to_string(),
455        )?;
456        avro_writer.add_user_metadata(
457            "format-version".to_string(),
458            (self.metadata.format_version as u8).to_string(),
459        )?;
460        match self.metadata.format_version {
461            FormatVersion::V1 => {}
462            FormatVersion::V2 | FormatVersion::V3 => {
463                avro_writer
464                    .add_user_metadata("content".to_string(), self.metadata.content.to_string())?;
465            }
466        }
467
468        let partition_summary = self.construct_partition_summaries(&partition_type)?;
469        // Write manifest entries
470        for entry in std::mem::take(&mut self.manifest_entries) {
471            let value = match self.metadata.format_version {
472                FormatVersion::V1 => {
473                    to_value(ManifestEntryV1::try_from(entry, &partition_struct_type)?)?
474                        .resolve(&avro_schema)?
475                }
476                FormatVersion::V2 => {
477                    to_value(ManifestEntryV2::try_from(entry, &partition_struct_type)?)?
478                        .resolve(&avro_schema)?
479                }
480                FormatVersion::V3 => {
481                    to_value(ManifestEntryV3::try_from(entry, &partition_struct_type)?)?
482                        .resolve(&avro_schema)?
483                }
484            };
485
486            avro_writer.append_value(value)?;
487        }
488
489        let content = avro_writer.into_inner()?;
490        let mut writer = self.output.writer().await?;
491        writer.write(Bytes::from(content)).await?;
492        let file_metadata = writer.close().await?;
493        let key_metadata = self.output.encoded_key_metadata(&file_metadata)?;
494
495        Ok(ManifestFile {
496            manifest_path: self.location,
497            // Manifest lengths are on-disk sizes, including encryption overhead.
498            manifest_length: file_metadata.size.try_into()?,
499            partition_spec_id: self.metadata.partition_spec.spec_id(),
500            content: self.metadata.content,
501            // sequence_number and min_sequence_number with UNASSIGNED_SEQUENCE_NUMBER will be replace with
502            // real sequence number in `ManifestListWriter`.
503            sequence_number: UNASSIGNED_SEQUENCE_NUMBER,
504            min_sequence_number: self.min_seq_num.unwrap_or(UNASSIGNED_SEQUENCE_NUMBER),
505            added_snapshot_id: self.snapshot_id.unwrap_or(UNASSIGNED_SNAPSHOT_ID),
506            added_files_count: Some(self.added_files),
507            existing_files_count: Some(self.existing_files),
508            deleted_files_count: Some(self.deleted_files),
509            added_rows_count: Some(self.added_rows),
510            existing_rows_count: Some(self.existing_rows),
511            deleted_rows_count: Some(self.deleted_rows),
512            partitions: Some(partition_summary),
513            key_metadata,
514            first_row_id: self.first_row_id,
515        })
516    }
517}
518
519struct PartitionFieldStats {
520    partition_type: PrimitiveType,
521
522    contains_null: bool,
523    contains_nan: Option<bool>,
524    lower_bound: Option<Datum>,
525    upper_bound: Option<Datum>,
526}
527
528impl PartitionFieldStats {
529    pub(crate) fn new(partition_type: PrimitiveType) -> Self {
530        Self {
531            partition_type,
532            contains_null: false,
533            contains_nan: Some(false),
534            upper_bound: None,
535            lower_bound: None,
536        }
537    }
538
539    pub(crate) fn update(&mut self, value: Option<PrimitiveLiteral>) -> Result<()> {
540        let Some(value) = value else {
541            self.contains_null = true;
542            return Ok(());
543        };
544        if !self.partition_type.compatible(&value) {
545            return Err(invalid_data!("value is not compatible with type"));
546        }
547        let value = Datum::new(self.partition_type.clone(), value);
548
549        if value.is_nan() {
550            self.contains_nan = Some(true);
551            return Ok(());
552        }
553
554        self.lower_bound = Some(self.lower_bound.take().map_or(value.clone(), |original| {
555            if value < original {
556                value.clone()
557            } else {
558                original
559            }
560        }));
561        self.upper_bound = Some(self.upper_bound.take().map_or(value.clone(), |original| {
562            if value > original { value } else { original }
563        }));
564
565        Ok(())
566    }
567
568    pub(crate) fn finish(self) -> FieldSummary {
569        FieldSummary {
570            contains_null: self.contains_null,
571            contains_nan: self.contains_nan,
572            upper_bound: self.upper_bound.map(|v| v.to_bytes().unwrap()),
573            lower_bound: self.lower_bound.map(|v| v.to_bytes().unwrap()),
574        }
575    }
576}
577
578#[cfg(test)]
579mod tests {
580    use std::collections::HashMap;
581    use std::fs;
582    use std::sync::Arc;
583
584    use apache_avro::Reader as AvroReader;
585    use apache_avro::schema::Schema as AvroSchema;
586    use tempfile::TempDir;
587
588    use super::*;
589    use crate::avro::{assert_deflate_codec, define_named_types_repeatedly};
590    use crate::io::FileIO;
591    use crate::spec::{
592        DataContentType, DataFileBuilder, DataFileFormat, Literal, Manifest, NestedField,
593        PrimitiveType, Schema, Struct, Transform, Type,
594    };
595
596    #[tokio::test]
597    async fn test_add_delete_existing() {
598        let schema = Arc::new(
599            Schema::builder()
600                .with_fields(vec![
601                    Arc::new(NestedField::optional(
602                        1,
603                        "id",
604                        Type::Primitive(PrimitiveType::Int),
605                    )),
606                    Arc::new(NestedField::optional(
607                        2,
608                        "name",
609                        Type::Primitive(PrimitiveType::String),
610                    )),
611                ])
612                .build()
613                .unwrap(),
614        );
615        let metadata = ManifestMetadata {
616            schema_id: 0,
617            schema: schema.clone(),
618            partition_spec: PartitionSpec::builder(schema)
619                .with_spec_id(0)
620                .build()
621                .unwrap(),
622            content: ManifestContentType::Data,
623            format_version: FormatVersion::V2,
624        };
625        let mut entries = vec![
626                ManifestEntry {
627                    status: ManifestStatus::Added,
628                    snapshot_id: None,
629                    sequence_number: Some(1),
630                    file_sequence_number: Some(1),
631                    data_file: DataFile {
632                        content: DataContentType::Data,
633                        file_path: "s3a://icebergdata/demo/s1/t1/data/00000-0-ba56fbfa-f2ff-40c9-bb27-565ad6dc2be8-00000.parquet".to_string(),
634                        file_format: DataFileFormat::Parquet,
635                        partition: Struct::empty(),
636                        record_count: 1,
637                        file_size_in_bytes: 5442,
638                        column_sizes: HashMap::from([(1, 61), (2, 73)]),
639                        value_counts: HashMap::from([(1, 1), (2, 1)]),
640                        null_value_counts: HashMap::from([(1, 0), (2, 0)]),
641                        nan_value_counts: HashMap::new(),
642                        lower_bounds: HashMap::new(),
643                        upper_bounds: HashMap::new(),
644                        key_metadata: Some(Vec::new()),
645                        split_offsets: Some(vec![4]),
646                        equality_ids: None,
647                        sort_order_id: None,
648                        partition_spec_id: 0,
649                        first_row_id: None,
650                        referenced_data_file: None,
651                        content_offset: None,
652                        content_size_in_bytes: None,
653                    },
654                },
655                ManifestEntry {
656                    status: ManifestStatus::Deleted,
657                    snapshot_id: Some(1),
658                    sequence_number: Some(1),
659                    file_sequence_number: Some(1),
660                    data_file: DataFile {
661                        content: DataContentType::Data,
662                        file_path: "s3a://icebergdata/demo/s1/t1/data/00000-0-ba56fbfa-f2ff-40c9-bb27-565ad6dc2be8-00000.parquet".to_string(),
663                        file_format: DataFileFormat::Parquet,
664                        partition: Struct::empty(),
665                        record_count: 1,
666                        file_size_in_bytes: 5442,
667                        column_sizes: HashMap::from([(1, 61), (2, 73)]),
668                        value_counts: HashMap::from([(1, 1), (2, 1)]),
669                        null_value_counts: HashMap::from([(1, 0), (2, 0)]),
670                        nan_value_counts: HashMap::new(),
671                        lower_bounds: HashMap::new(),
672                        upper_bounds: HashMap::new(),
673                        key_metadata: Some(Vec::new()),
674                        split_offsets: Some(vec![4]),
675                        equality_ids: None,
676                        sort_order_id: None,
677                        partition_spec_id: 0,
678                        first_row_id: None,
679                        referenced_data_file: None,
680                        content_offset: None,
681                        content_size_in_bytes: None,
682                    },
683                },
684                ManifestEntry {
685                    status: ManifestStatus::Existing,
686                    snapshot_id: Some(1),
687                    sequence_number: Some(1),
688                    file_sequence_number: Some(1),
689                    data_file: DataFile {
690                        content: DataContentType::Data,
691                        file_path: "s3a://icebergdata/demo/s1/t1/data/00000-0-ba56fbfa-f2ff-40c9-bb27-565ad6dc2be8-00000.parquet".to_string(),
692                        file_format: DataFileFormat::Parquet,
693                        partition: Struct::empty(),
694                        record_count: 1,
695                        file_size_in_bytes: 5442,
696                        column_sizes: HashMap::from([(1, 61), (2, 73)]),
697                        value_counts: HashMap::from([(1, 1), (2, 1)]),
698                        null_value_counts: HashMap::from([(1, 0), (2, 0)]),
699                        nan_value_counts: HashMap::new(),
700                        lower_bounds: HashMap::new(),
701                        upper_bounds: HashMap::new(),
702                        key_metadata: Some(Vec::new()),
703                        split_offsets: Some(vec![4]),
704                        equality_ids: None,
705                        sort_order_id: None,
706                        partition_spec_id: 0,
707                        first_row_id: None,
708                        referenced_data_file: None,
709                        content_offset: None,
710                        content_size_in_bytes: None,
711                    },
712                },
713            ];
714
715        // write manifest to file
716        let tmp_dir = TempDir::new().unwrap();
717        let path = tmp_dir.path().join("test_manifest.avro");
718        let io = FileIO::new_with_fs();
719        let output_file = io.new_output(path.to_str().unwrap()).unwrap();
720        let mut writer = ManifestWriterBuilder::new(
721            output_file,
722            Some(3),
723            metadata.schema.clone(),
724            metadata.partition_spec.clone(),
725        )
726        .build_v2_data();
727        writer.add_entry(entries[0].clone()).unwrap();
728        writer.add_delete_entry(entries[1].clone()).unwrap();
729        writer.add_existing_entry(entries[2].clone()).unwrap();
730        writer.write_manifest_file().await.unwrap();
731
732        // read back the manifest file and check the content
733        let bytes = fs::read(path).expect("read_file must succeed");
734        assert_deflate_codec(&bytes);
735        let actual_manifest = Manifest::parse_avro(&bytes).unwrap();
736
737        // The snapshot id is assigned when the entry is added and delete to the manifest. Existing entries are keep original.
738        entries[0].snapshot_id = Some(3);
739        entries[1].snapshot_id = Some(3);
740        // file sequence number is assigned to None when the entry is added and delete to the manifest.
741        entries[0].file_sequence_number = None;
742        assert_eq!(actual_manifest, Manifest::new(metadata, entries));
743    }
744
745    #[tokio::test]
746    async fn test_v3_delete_manifest_delete_file_roundtrip() {
747        let schema = Arc::new(
748            Schema::builder()
749                .with_fields(vec![
750                    Arc::new(NestedField::optional(
751                        1,
752                        "id",
753                        Type::Primitive(PrimitiveType::Long),
754                    )),
755                    Arc::new(NestedField::optional(
756                        2,
757                        "data",
758                        Type::Primitive(PrimitiveType::String),
759                    )),
760                ])
761                .build()
762                .unwrap(),
763        );
764
765        let partition_spec = PartitionSpec::builder(schema.clone())
766            .with_spec_id(0)
767            .build()
768            .unwrap();
769
770        // Create a position delete file entry
771        let delete_entry = ManifestEntry {
772            status: ManifestStatus::Added,
773            snapshot_id: None,
774            sequence_number: None,
775            file_sequence_number: None,
776            data_file: DataFile {
777                content: DataContentType::PositionDeletes,
778                file_path: "s3://bucket/table/data/delete-00000.parquet".to_string(),
779                file_format: DataFileFormat::Parquet,
780                partition: Struct::empty(),
781                record_count: 10,
782                file_size_in_bytes: 1024,
783                column_sizes: HashMap::new(),
784                value_counts: HashMap::new(),
785                null_value_counts: HashMap::new(),
786                nan_value_counts: HashMap::new(),
787                lower_bounds: HashMap::new(),
788                upper_bounds: HashMap::new(),
789                key_metadata: None,
790                split_offsets: None,
791                equality_ids: None,
792                sort_order_id: None,
793                partition_spec_id: 0,
794                first_row_id: None,
795                referenced_data_file: None,
796                content_offset: None,
797                content_size_in_bytes: None,
798            },
799        };
800
801        // Write a V3 delete manifest
802        let tmp_dir = TempDir::new().unwrap();
803        let path = tmp_dir.path().join("v3_delete_manifest.avro");
804        let io = FileIO::new_with_fs();
805        let output_file = io.new_output(path.to_str().unwrap()).unwrap();
806
807        let mut writer = ManifestWriterBuilder::new(
808            output_file,
809            Some(1),
810            schema.clone(),
811            partition_spec.clone(),
812        )
813        .build_v3_deletes();
814
815        writer.add_entry(delete_entry).unwrap();
816        let manifest_file = writer.write_manifest_file().await.unwrap();
817
818        // The returned ManifestFile correctly reports Deletes content
819        assert_eq!(manifest_file.content, ManifestContentType::Deletes);
820
821        // Read back the manifest file
822        let bytes = fs::read(&path).expect("read_file must succeed");
823        assert_deflate_codec(&bytes);
824        assert_eq!(manifest_file.manifest_length, bytes.len() as i64);
825        let actual_manifest = Manifest::parse_avro(&bytes).unwrap();
826
827        // Verify the content type is correctly preserved as Deletes
828        assert_eq!(
829            actual_manifest.metadata().content,
830            ManifestContentType::Deletes,
831        );
832    }
833
834    /// Writes a manifest partitioned by identity on two columns of `field_type`,
835    /// reads it back, and returns the manifest file.
836    async fn roundtrip_two_partition_fields_of_type(
837        field_type: Type,
838        values: [Literal; 2],
839    ) -> Vec<u8> {
840        let schema = Arc::new(
841            Schema::builder()
842                .with_fields(vec![
843                    NestedField::optional(1, "a", field_type.clone()).into(),
844                    NestedField::optional(2, "b", field_type).into(),
845                ])
846                .build()
847                .unwrap(),
848        );
849        let partition_spec = PartitionSpec::builder(schema.clone())
850            .with_spec_id(0)
851            .add_partition_field("a", "a", Transform::Identity)
852            .unwrap()
853            .add_partition_field("b", "b", Transform::Identity)
854            .unwrap()
855            .build()
856            .unwrap();
857        let partition = Struct::from_iter(values.map(Some));
858        let tmp_dir = TempDir::new().unwrap();
859        let path = tmp_dir.path().join("manifest.avro");
860        let output_file = FileIO::new_with_fs()
861            .new_output(path.to_str().unwrap())
862            .unwrap();
863        let mut writer = ManifestWriterBuilder::new(output_file, Some(1), schema, partition_spec)
864            .build_v2_data();
865        writer
866            .add_file(
867                DataFileBuilder::default()
868                    .content(DataContentType::Data)
869                    .file_path("s3://bucket/table/data/a.parquet".to_string())
870                    .file_format(DataFileFormat::Parquet)
871                    .partition(partition.clone())
872                    .record_count(1)
873                    .file_size_in_bytes(1)
874                    .partition_spec_id(0)
875                    .build()
876                    .unwrap(),
877                1,
878            )
879            .unwrap();
880        writer.write_manifest_file().await.unwrap();
881        let bs = fs::read(&path).unwrap();
882
883        // Also read the manifest as iceberg-rust wrote it before defining each
884        // named type once.
885        for bs in [bs.clone(), define_named_types_repeatedly(&bs)] {
886            let manifest = Manifest::parse_avro(&bs).unwrap();
887
888            assert_eq!(*manifest.entries()[0].data_file().partition(), partition);
889        }
890        bs
891    }
892
893    #[tokio::test]
894    async fn test_write_manifest_header_marks_int_keyed_maps() {
895        // Java and PyIceberg read an array of key-value records as a map only if
896        // it has `logicalType: map`. apache-avro 0.22 drops the attribute when it
897        // parses a schema (https://github.com/apache/avro-rs/issues/654), so this
898        // checks the schema JSON in the written header.
899        let bs = roundtrip_two_partition_fields_of_type(Type::Primitive(PrimitiveType::Int), [
900            Literal::int(1),
901            Literal::int(2),
902        ])
903        .await;
904
905        let file = String::from_utf8_lossy(&bs);
906        // `column_sizes`, `value_counts`, `null_value_counts`,
907        // `nan_value_counts`, `lower_bounds`, and `upper_bounds`.
908        assert_eq!(file.matches(r#""logicalType":"map""#).count(), 6);
909    }
910
911    #[tokio::test]
912    async fn test_write_manifest_with_repeated_decimal_partition_type() {
913        roundtrip_two_partition_fields_of_type(
914            Type::Primitive(PrimitiveType::Decimal {
915                precision: 10,
916                scale: 2,
917            }),
918            [Literal::decimal(12345), Literal::decimal(-678)],
919        )
920        .await;
921    }
922
923    #[tokio::test]
924    async fn test_write_manifest_with_repeated_fixed_partition_type() {
925        roundtrip_two_partition_fields_of_type(Type::Primitive(PrimitiveType::Fixed(4)), [
926            Literal::fixed(vec![1, 2, 3, 4]),
927            Literal::fixed(vec![5, 6, 7, 8]),
928        ])
929        .await;
930    }
931
932    /// Writes a manifest with one added data file and returns the field names
933    /// of its Avro `data_file` record.
934    async fn written_data_file_field_names(
935        build: fn(ManifestWriterBuilder) -> ManifestWriter,
936    ) -> Vec<String> {
937        let schema = Arc::new(
938            Schema::builder()
939                .with_fields(vec![
940                    NestedField::optional(1, "id", Type::Primitive(PrimitiveType::Int)).into(),
941                ])
942                .build()
943                .unwrap(),
944        );
945        let partition_spec = PartitionSpec::builder(schema.clone())
946            .with_spec_id(0)
947            .build()
948            .unwrap();
949        let tmp_dir = TempDir::new().unwrap();
950        let path = tmp_dir.path().join("manifest.avro");
951        let output_file = FileIO::new_with_fs()
952            .new_output(path.to_str().unwrap())
953            .unwrap();
954        let mut writer = build(ManifestWriterBuilder::new(
955            output_file,
956            Some(1),
957            schema,
958            partition_spec,
959        ));
960        writer
961            .add_file(
962                DataFileBuilder::default()
963                    .content(DataContentType::Data)
964                    .file_path("s3://bucket/table/data/a.parquet".to_string())
965                    .file_format(DataFileFormat::Parquet)
966                    .partition(Struct::empty())
967                    .record_count(1)
968                    .file_size_in_bytes(1)
969                    .partition_spec_id(0)
970                    .build()
971                    .unwrap(),
972                1,
973            )
974            .unwrap();
975        writer.write_manifest_file().await.unwrap();
976        let bs = fs::read(&path).unwrap();
977
978        let reader = AvroReader::new(&bs[..]).unwrap();
979        let AvroSchema::Record(entry) = reader.writer_schema() else {
980            panic!("manifest_entry is not a record");
981        };
982        let data_file_schema = &entry.fields[entry.lookup["data_file"]].schema;
983        let AvroSchema::Record(data_file) = data_file_schema else {
984            panic!("data_file is not a record");
985        };
986        data_file.fields.iter().map(|f| f.name.clone()).collect()
987    }
988
989    #[tokio::test]
990    async fn test_v1_manifest_data_file_schema() {
991        let field_names = written_data_file_field_names(ManifestWriterBuilder::build_v1).await;
992        assert_eq!(field_names, vec![
993            "file_path",
994            "file_format",
995            "partition",
996            "record_count",
997            "file_size_in_bytes",
998            "block_size_in_bytes",
999            "column_sizes",
1000            "value_counts",
1001            "null_value_counts",
1002            "nan_value_counts",
1003            "lower_bounds",
1004            "upper_bounds",
1005            "key_metadata",
1006            "split_offsets",
1007            "sort_order_id",
1008        ]);
1009    }
1010
1011    #[tokio::test]
1012    async fn test_v2_manifest_data_file_schema() {
1013        let field_names = written_data_file_field_names(ManifestWriterBuilder::build_v2_data).await;
1014        assert_eq!(field_names, vec![
1015            "content",
1016            "file_path",
1017            "file_format",
1018            "partition",
1019            "record_count",
1020            "file_size_in_bytes",
1021            "column_sizes",
1022            "value_counts",
1023            "null_value_counts",
1024            "nan_value_counts",
1025            "lower_bounds",
1026            "upper_bounds",
1027            "key_metadata",
1028            "split_offsets",
1029            "equality_ids",
1030            "sort_order_id",
1031            "referenced_data_file",
1032        ]);
1033    }
1034
1035    #[tokio::test]
1036    async fn test_v3_manifest_data_file_schema() {
1037        let field_names = written_data_file_field_names(ManifestWriterBuilder::build_v3_data).await;
1038        assert_eq!(field_names, vec![
1039            "content",
1040            "file_path",
1041            "file_format",
1042            "partition",
1043            "record_count",
1044            "file_size_in_bytes",
1045            "column_sizes",
1046            "value_counts",
1047            "null_value_counts",
1048            "nan_value_counts",
1049            "lower_bounds",
1050            "upper_bounds",
1051            "key_metadata",
1052            "split_offsets",
1053            "equality_ids",
1054            "sort_order_id",
1055            "first_row_id",
1056            "referenced_data_file",
1057            "content_offset",
1058            "content_size_in_bytes",
1059        ]);
1060    }
1061}