Skip to main content

iceberg/spec/manifest_list/
writer.rs

1// Licensed to the Apache Software Foundation (ASF) under one
2// or more contributor license agreements.  See the NOTICE file
3// distributed with this work for additional information
4// regarding copyright ownership.  The ASF licenses this file
5// to you under the Apache License, Version 2.0 (the
6// "License"); you may not use this file except in compliance
7// with the License.  You may obtain a copy of the License at
8//
9//   http://www.apache.org/licenses/LICENSE-2.0
10//
11// Unless required by applicable law or agreed to in writing,
12// software distributed under the License is distributed on an
13// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14// KIND, either express or implied.  See the License for the
15// specific language governing permissions and limitations
16// under the License.
17
18use std::collections::HashMap;
19
20use apache_avro::Writer;
21use bytes::Bytes;
22
23use super::_const_schema::{
24    MANIFEST_LIST_AVRO_SCHEMA_V1, MANIFEST_LIST_AVRO_SCHEMA_V2, MANIFEST_LIST_AVRO_SCHEMA_V3,
25};
26use super::_serde::{ManifestFileV1, ManifestFileV2, ManifestFileV3};
27use super::{FormatVersion, ManifestContentType, ManifestFile, UNASSIGNED_SEQUENCE_NUMBER};
28use crate::avro::new_writer;
29use crate::error::{Result, invalid_data};
30use crate::io::{FileMetadata, FileWrite};
31use crate::{Error, ErrorKind};
32
33/// A manifest list writer.
34pub struct ManifestListWriter {
35    format_version: FormatVersion,
36    writer: Box<dyn FileWrite>,
37    avro_writer: Writer<'static, Vec<u8>>,
38    sequence_number: i64,
39    snapshot_id: i64,
40    next_row_id: Option<u64>,
41}
42
43impl std::fmt::Debug for ManifestListWriter {
44    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
45        f.debug_struct("ManifestListWriter")
46            .field("format_version", &self.format_version)
47            .field("avro_writer", &self.avro_writer.schema())
48            .finish_non_exhaustive()
49    }
50}
51
52impl ManifestListWriter {
53    /// Get the next row ID that will be assigned to the next data manifest added.
54    pub fn next_row_id(&self) -> Option<u64> {
55        self.next_row_id
56    }
57
58    /// Construct a v1 [`ManifestListWriter`] that writes to a provided [`FileWrite`].
59    pub fn v1(
60        writer: Box<dyn FileWrite>,
61        snapshot_id: i64,
62        parent_snapshot_id: Option<i64>,
63    ) -> Self {
64        let mut metadata = HashMap::from_iter([
65            ("snapshot-id".to_string(), snapshot_id.to_string()),
66            ("format-version".to_string(), "1".to_string()),
67        ]);
68        if let Some(parent_snapshot_id) = parent_snapshot_id {
69            metadata.insert(
70                "parent-snapshot-id".to_string(),
71                parent_snapshot_id.to_string(),
72            );
73        }
74        Self::new(FormatVersion::V1, writer, metadata, 0, snapshot_id, None)
75    }
76
77    /// Construct a v2 [`ManifestListWriter`] that writes to a provided [`FileWrite`].
78    pub fn v2(
79        writer: Box<dyn FileWrite>,
80        snapshot_id: i64,
81        parent_snapshot_id: Option<i64>,
82        sequence_number: i64,
83    ) -> Self {
84        let mut metadata = HashMap::from_iter([
85            ("snapshot-id".to_string(), snapshot_id.to_string()),
86            ("sequence-number".to_string(), sequence_number.to_string()),
87            ("format-version".to_string(), "2".to_string()),
88        ]);
89        metadata.insert(
90            "parent-snapshot-id".to_string(),
91            parent_snapshot_id
92                .map(|v| v.to_string())
93                .unwrap_or("null".to_string()),
94        );
95        Self::new(
96            FormatVersion::V2,
97            writer,
98            metadata,
99            sequence_number,
100            snapshot_id,
101            None,
102        )
103    }
104
105    /// Construct a v3 [`ManifestListWriter`] that writes to a provided [`FileWrite`].
106    pub fn v3(
107        writer: Box<dyn FileWrite>,
108        snapshot_id: i64,
109        parent_snapshot_id: Option<i64>,
110        sequence_number: i64,
111        first_row_id: Option<u64>, // Always None for delete manifests
112    ) -> Self {
113        let mut metadata = HashMap::from_iter([
114            ("snapshot-id".to_string(), snapshot_id.to_string()),
115            ("sequence-number".to_string(), sequence_number.to_string()),
116            ("format-version".to_string(), "3".to_string()),
117        ]);
118        metadata.insert(
119            "parent-snapshot-id".to_string(),
120            parent_snapshot_id
121                .map(|v| v.to_string())
122                .unwrap_or("null".to_string()),
123        );
124        metadata.insert(
125            "first-row-id".to_string(),
126            first_row_id
127                .map(|v| v.to_string())
128                .unwrap_or("null".to_string()),
129        );
130        Self::new(
131            FormatVersion::V3,
132            writer,
133            metadata,
134            sequence_number,
135            snapshot_id,
136            first_row_id,
137        )
138    }
139
140    fn new(
141        format_version: FormatVersion,
142        writer: Box<dyn FileWrite>,
143        metadata: HashMap<String, String>,
144        sequence_number: i64,
145        snapshot_id: i64,
146        first_row_id: Option<u64>,
147    ) -> Self {
148        let avro_schema = match format_version {
149            FormatVersion::V1 => &MANIFEST_LIST_AVRO_SCHEMA_V1,
150            FormatVersion::V2 => &MANIFEST_LIST_AVRO_SCHEMA_V2,
151            FormatVersion::V3 => &MANIFEST_LIST_AVRO_SCHEMA_V3,
152        };
153        let mut avro_writer = new_writer(avro_schema, Vec::new()).expect(
154            "Manifest list Avro schemas should resolve because they refer to no named types.",
155        );
156        for (key, value) in metadata {
157            avro_writer
158                .add_user_metadata(key, value)
159                .expect("Avro metadata should be added to the writer before the first record.");
160        }
161        Self {
162            format_version,
163            writer,
164            avro_writer,
165            sequence_number,
166            snapshot_id,
167            next_row_id: first_row_id,
168        }
169    }
170
171    /// Append manifests to be written.
172    ///
173    /// If V3 Manifests are added and the `first_row_id` of any data manifest is unassigned,
174    /// it will be assigned based on the `next_row_id` of the writer, and the `next_row_id` of the writer will be updated accordingly.
175    /// If `first_row_id` is already assigned, it will be validated against the `next_row_id` of the writer.
176    pub fn add_manifests(&mut self, manifests: impl Iterator<Item = ManifestFile>) -> Result<()> {
177        match self.format_version {
178            FormatVersion::V1 => {
179                for manifest in manifests {
180                    let manifests: ManifestFileV1 = manifest.try_into()?;
181                    self.avro_writer.append_ser(manifests)?;
182                }
183            }
184            FormatVersion::V2 | FormatVersion::V3 => {
185                for mut manifest in manifests {
186                    self.assign_sequence_numbers(&mut manifest)?;
187
188                    if self.format_version == FormatVersion::V2 {
189                        let manifest_entry: ManifestFileV2 = manifest.try_into()?;
190                        self.avro_writer.append_ser(manifest_entry)?;
191                    } else if self.format_version == FormatVersion::V3 {
192                        self.assign_first_row_id(&mut manifest)?;
193                        let manifest_entry: ManifestFileV3 = manifest.try_into()?;
194                        self.avro_writer.append_ser(manifest_entry)?;
195                    }
196                }
197            }
198        }
199        Ok(())
200    }
201
202    /// Write the manifest list and return its stored size.
203    pub async fn close(mut self) -> Result<FileMetadata> {
204        let data = self.avro_writer.into_inner()?;
205        self.writer.write(Bytes::from(data)).await?;
206        self.writer.close().await
207    }
208
209    /// Assign sequence numbers to manifest if they are unassigned
210    fn assign_sequence_numbers(&self, manifest: &mut ManifestFile) -> Result<()> {
211        if manifest.sequence_number == UNASSIGNED_SEQUENCE_NUMBER {
212            if manifest.added_snapshot_id != self.snapshot_id {
213                return Err(invalid_data!(
214                    "Found unassigned sequence number for a manifest from snapshot {}.",
215                    manifest.added_snapshot_id
216                ));
217            }
218            manifest.sequence_number = self.sequence_number;
219        }
220
221        if manifest.min_sequence_number == UNASSIGNED_SEQUENCE_NUMBER {
222            if manifest.added_snapshot_id != self.snapshot_id {
223                return Err(invalid_data!(
224                    "Found unassigned sequence number for a manifest from snapshot {}.",
225                    manifest.added_snapshot_id
226                ));
227            }
228            manifest.min_sequence_number = self.sequence_number;
229        }
230
231        Ok(())
232    }
233
234    /// Returns number of newly assigned first-row-ids, if any.
235    fn assign_first_row_id(&mut self, manifest: &mut ManifestFile) -> Result<()> {
236        match manifest.content {
237            ManifestContentType::Data => {
238                match (self.next_row_id, manifest.first_row_id) {
239                    (Some(_), Some(_)) => {
240                        // Case: Manifest with already assigned first row ID.
241                        // No need to increase next_row_id, as this manifest is already assigned.
242                    }
243                    (None, Some(manifest_first_row_id)) => {
244                        // Case: Assigned first row ID for data manifest, but the writer does not have a next-row-id assigned.
245                        return Err(Error::new(
246                            ErrorKind::Unexpected,
247                            format!(
248                                "Found invalid first-row-id assignment for Manifest {}. Writer does not have a next-row-id assigned, but the manifest has first-row-id assigned to {}.",
249                                manifest.manifest_path, manifest_first_row_id,
250                            ),
251                        ));
252                    }
253                    (Some(writer_next_row_id), None) => {
254                        // Case: Unassigned first row ID for data manifest. This is either a new
255                        // manifest, or a manifest from a pre-v3 snapshot. We need to assign one.
256                        let (existing_rows_count, added_rows_count) =
257                            require_row_counts_in_manifest(manifest)?;
258                        manifest.first_row_id = Some(writer_next_row_id);
259
260                        self.next_row_id = writer_next_row_id
261                        .checked_add(existing_rows_count)
262                        .and_then(|sum| sum.checked_add(added_rows_count))
263                        .ok_or_else(|| {
264                            invalid_data!(
265                                    "Row ID overflow when computing next row ID for Manifest {}. Next Row ID: {writer_next_row_id}, Existing Rows Count: {existing_rows_count}, Added Rows Count: {added_rows_count}",
266                                    manifest.manifest_path
267                                )
268                        }).map(Some)?;
269                    }
270                    (None, None) => {
271                        // Case: Table without row lineage. No action needed.
272                    }
273                }
274            }
275            ManifestContentType::Deletes => {
276                // Deletes never have a first-row-id assigned.
277                manifest.first_row_id = None;
278            }
279        };
280
281        Ok(())
282    }
283}
284
285fn require_row_counts_in_manifest(manifest: &ManifestFile) -> Result<(u64, u64)> {
286    let existing_rows_count = manifest.existing_rows_count.ok_or_else(|| {
287        invalid_data!(
288                "Cannot include a Manifest without existing-rows-count to a table with row lineage enabled. Manifest path: {}",
289                manifest.manifest_path,
290            )
291    })?;
292    let added_rows_count = manifest.added_rows_count.ok_or_else(|| {
293        invalid_data!(
294                "Cannot include a Manifest without added-rows-count to a table with row lineage enabled. Manifest path: {}",
295                manifest.manifest_path,
296            )
297    })?;
298    Ok((existing_rows_count, added_rows_count))
299}
300
301#[cfg(test)]
302mod test {
303    use std::fs;
304    use std::path::Path;
305    use std::sync::Arc;
306
307    use tempfile::TempDir;
308
309    use super::ManifestListWriter;
310    use crate::avro::assert_deflate_codec;
311    use crate::encryption::kms::{KeyManagementClient, MemoryKeyManagementClient};
312    use crate::encryption::{EncryptedInputFile, EncryptionManager};
313    use crate::io::{FileIO, FileWrite};
314    use crate::spec::{
315        Datum, FieldSummary, ManifestContentType, ManifestFile, ManifestList,
316        UNASSIGNED_SEQUENCE_NUMBER,
317    };
318
319    #[tokio::test]
320    async fn test_manifest_list_writer_v1() {
321        let expected_manifest_list = ManifestList {
322            entries: vec![ManifestFile {
323                manifest_path: "/opt/bitnami/spark/warehouse/db/table/metadata/10d28031-9739-484c-92db-cdf2975cead4-m0.avro".to_string(),
324                manifest_length: 5806,
325                partition_spec_id: 1,
326                content: ManifestContentType::Data,
327                sequence_number: 0,
328                min_sequence_number: 0,
329                added_snapshot_id: 1646658105718557341,
330                added_files_count: Some(3),
331                existing_files_count: Some(0),
332                deleted_files_count: Some(0),
333                added_rows_count: Some(3),
334                existing_rows_count: Some(0),
335                deleted_rows_count: Some(0),
336                partitions: Some(
337                    vec![FieldSummary { contains_null: false, contains_nan: Some(false), lower_bound: Some(Datum::long(1).to_bytes().unwrap()), upper_bound: Some(Datum::long(1).to_bytes().unwrap())}],
338                ),
339                key_metadata: None,
340                first_row_id: None,
341            }]
342        };
343
344        let temp_dir = TempDir::new().unwrap();
345        let path = temp_dir.path().join("manifest_list_v1.avro");
346        let io = FileIO::new_with_fs();
347        let file_writer = file_writer(&path, io).await;
348
349        let mut writer = ManifestListWriter::v1(file_writer, 1646658105718557341, Some(0));
350        writer
351            .add_manifests(expected_manifest_list.entries.clone().into_iter())
352            .unwrap();
353        writer.close().await.unwrap();
354
355        let bs = fs::read(path).unwrap();
356        assert_deflate_codec(&bs);
357
358        let manifest_list =
359            ManifestList::parse_with_version(&bs, crate::spec::FormatVersion::V1).unwrap();
360        assert_eq!(manifest_list, expected_manifest_list);
361
362        temp_dir.close().unwrap();
363    }
364
365    #[tokio::test]
366    async fn test_manifest_list_writer_v2() {
367        let snapshot_id = 377075049360453639;
368        let seq_num = 1;
369        let mut expected_manifest_list = ManifestList {
370            entries: vec![ManifestFile {
371                manifest_path: "s3a://icebergdata/demo/s1/t1/metadata/05ffe08b-810f-49b3-a8f4-e88fc99b254a-m0.avro".to_string(),
372                manifest_length: 6926,
373                partition_spec_id: 1,
374                content: ManifestContentType::Data,
375                sequence_number: UNASSIGNED_SEQUENCE_NUMBER,
376                min_sequence_number: UNASSIGNED_SEQUENCE_NUMBER,
377                added_snapshot_id: snapshot_id,
378                added_files_count: Some(1),
379                existing_files_count: Some(0),
380                deleted_files_count: Some(0),
381                added_rows_count: Some(3),
382                existing_rows_count: Some(0),
383                deleted_rows_count: Some(0),
384                partitions: Some(
385                    vec![FieldSummary { contains_null: false, contains_nan: Some(false), lower_bound: Some(Datum::long(1).to_bytes().unwrap()), upper_bound: Some(Datum::long(1).to_bytes().unwrap())}]
386                ),
387                key_metadata: None,
388                first_row_id: None,
389            }]
390        };
391
392        let temp_dir = TempDir::new().unwrap();
393        let path = temp_dir.path().join("manifest_list_v2.avro");
394        let io = FileIO::new_with_fs();
395        let file_writer = file_writer(&path, io).await;
396
397        let mut writer = ManifestListWriter::v2(file_writer, snapshot_id, Some(0), seq_num);
398        writer
399            .add_manifests(expected_manifest_list.entries.clone().into_iter())
400            .unwrap();
401        writer.close().await.unwrap();
402
403        let bs = fs::read(path).unwrap();
404        assert_deflate_codec(&bs);
405        let manifest_list =
406            ManifestList::parse_with_version(&bs, crate::spec::FormatVersion::V2).unwrap();
407        expected_manifest_list.entries[0].sequence_number = seq_num;
408        expected_manifest_list.entries[0].min_sequence_number = seq_num;
409        assert_eq!(manifest_list, expected_manifest_list);
410
411        temp_dir.close().unwrap();
412    }
413
414    #[tokio::test]
415    async fn test_manifest_list_writer_v3() {
416        let snapshot_id = 377075049360453639;
417        let seq_num = 1;
418        let mut expected_manifest_list = ManifestList {
419            entries: vec![ManifestFile {
420                manifest_path: "s3a://icebergdata/demo/s1/t1/metadata/05ffe08b-810f-49b3-a8f4-e88fc99b254a-m0.avro".to_string(),
421                manifest_length: 6926,
422                partition_spec_id: 1,
423                content: ManifestContentType::Data,
424                sequence_number: UNASSIGNED_SEQUENCE_NUMBER,
425                min_sequence_number: UNASSIGNED_SEQUENCE_NUMBER,
426                added_snapshot_id: snapshot_id,
427                added_files_count: Some(1),
428                existing_files_count: Some(0),
429                deleted_files_count: Some(0),
430                added_rows_count: Some(3),
431                existing_rows_count: Some(0),
432                deleted_rows_count: Some(0),
433                partitions: Some(
434                    vec![FieldSummary { contains_null: false, contains_nan: Some(false), lower_bound: Some(Datum::long(1).to_bytes().unwrap()), upper_bound: Some(Datum::long(1).to_bytes().unwrap())}]
435                ),
436                key_metadata: None,
437                first_row_id: Some(10),
438            }]
439        };
440
441        let temp_dir = TempDir::new().unwrap();
442        let path = temp_dir.path().join("manifest_list_v2.avro");
443        let io = FileIO::new_with_fs();
444        let file_writer = file_writer(&path, io).await;
445
446        let mut writer =
447            ManifestListWriter::v3(file_writer, snapshot_id, Some(0), seq_num, Some(10));
448        writer
449            .add_manifests(expected_manifest_list.entries.clone().into_iter())
450            .unwrap();
451        writer.close().await.unwrap();
452
453        let bs = fs::read(path).unwrap();
454        assert_deflate_codec(&bs);
455        let manifest_list =
456            ManifestList::parse_with_version(&bs, crate::spec::FormatVersion::V3).unwrap();
457        expected_manifest_list.entries[0].sequence_number = seq_num;
458        expected_manifest_list.entries[0].min_sequence_number = seq_num;
459        expected_manifest_list.entries[0].first_row_id = Some(10);
460        assert_eq!(manifest_list, expected_manifest_list);
461
462        temp_dir.close().unwrap();
463    }
464
465    #[tokio::test]
466    async fn test_manifest_list_writer_v1_as_v2() {
467        let expected_manifest_list = ManifestList {
468            entries: vec![ManifestFile {
469                manifest_path: "/opt/bitnami/spark/warehouse/db/table/metadata/10d28031-9739-484c-92db-cdf2975cead4-m0.avro".to_string(),
470                manifest_length: 5806,
471                partition_spec_id: 1,
472                content: ManifestContentType::Data,
473                sequence_number: 0,
474                min_sequence_number: 0,
475                added_snapshot_id: 1646658105718557341,
476                added_files_count: Some(3),
477                existing_files_count: Some(0),
478                deleted_files_count: Some(0),
479                added_rows_count: Some(3),
480                existing_rows_count: Some(0),
481                deleted_rows_count: Some(0),
482                partitions: Some(
483                    vec![FieldSummary { contains_null: false, contains_nan: Some(false), lower_bound: Some(Datum::long(1).to_bytes().unwrap()), upper_bound: Some(Datum::long(1).to_bytes().unwrap())}]
484                ),
485                key_metadata: None,
486                first_row_id: None,
487            }]
488        };
489
490        let temp_dir = TempDir::new().unwrap();
491        let path = temp_dir.path().join("manifest_list_v1.avro");
492        let io = FileIO::new_with_fs();
493        let file_writer = file_writer(&path, io).await;
494
495        let mut writer = ManifestListWriter::v1(file_writer, 1646658105718557341, Some(0));
496        writer
497            .add_manifests(expected_manifest_list.entries.clone().into_iter())
498            .unwrap();
499        writer.close().await.unwrap();
500
501        let bs = fs::read(path).unwrap();
502
503        let manifest_list =
504            ManifestList::parse_with_version(&bs, crate::spec::FormatVersion::V2).unwrap();
505        assert_eq!(manifest_list, expected_manifest_list);
506
507        temp_dir.close().unwrap();
508    }
509
510    #[tokio::test]
511    async fn test_manifest_list_writer_v1_as_v3() {
512        let expected_manifest_list = ManifestList {
513            entries: vec![ManifestFile {
514                manifest_path: "/opt/bitnami/spark/warehouse/db/table/metadata/10d28031-9739-484c-92db-cdf2975cead4-m0.avro".to_string(),
515                manifest_length: 5806,
516                partition_spec_id: 1,
517                content: ManifestContentType::Data,
518                sequence_number: 0,
519                min_sequence_number: 0,
520                added_snapshot_id: 1646658105718557341,
521                added_files_count: Some(3),
522                existing_files_count: Some(0),
523                deleted_files_count: Some(0),
524                added_rows_count: Some(3),
525                existing_rows_count: Some(0),
526                deleted_rows_count: Some(0),
527                partitions: Some(
528                    vec![FieldSummary { contains_null: false, contains_nan: Some(false), lower_bound: Some(Datum::long(1).to_bytes().unwrap()), upper_bound: Some(Datum::long(1).to_bytes().unwrap())}]
529                ),
530                key_metadata: None,
531                first_row_id: None,
532            }]
533        };
534
535        let temp_dir = TempDir::new().unwrap();
536        let path = temp_dir.path().join("manifest_list_v1.avro");
537        let io = FileIO::new_with_fs();
538        let file_writer = file_writer(&path, io).await;
539
540        let mut writer = ManifestListWriter::v1(file_writer, 1646658105718557341, Some(0));
541        writer
542            .add_manifests(expected_manifest_list.entries.clone().into_iter())
543            .unwrap();
544        writer.close().await.unwrap();
545
546        let bs = fs::read(path).unwrap();
547
548        let manifest_list =
549            ManifestList::parse_with_version(&bs, crate::spec::FormatVersion::V3).unwrap();
550        assert_eq!(manifest_list, expected_manifest_list);
551
552        temp_dir.close().unwrap();
553    }
554
555    #[tokio::test]
556    async fn test_manifest_list_writer_v2_as_v3() {
557        let snapshot_id = 377075049360453639;
558        let seq_num = 1;
559        let mut expected_manifest_list = ManifestList {
560            entries: vec![ManifestFile {
561                manifest_path: "s3a://icebergdata/demo/s1/t1/metadata/05ffe08b-810f-49b3-a8f4-e88fc99b254a-m0.avro".to_string(),
562                manifest_length: 6926,
563                partition_spec_id: 1,
564                content: ManifestContentType::Data,
565                sequence_number: UNASSIGNED_SEQUENCE_NUMBER,
566                min_sequence_number: UNASSIGNED_SEQUENCE_NUMBER,
567                added_snapshot_id: snapshot_id,
568                added_files_count: Some(1),
569                existing_files_count: Some(0),
570                deleted_files_count: Some(0),
571                added_rows_count: Some(3),
572                existing_rows_count: Some(0),
573                deleted_rows_count: Some(0),
574                partitions: Some(
575                    vec![FieldSummary { contains_null: false, contains_nan: Some(false), lower_bound: Some(Datum::long(1).to_bytes().unwrap()), upper_bound: Some(Datum::long(1).to_bytes().unwrap())}]
576                ),
577                key_metadata: None,
578                first_row_id: None,
579            }]
580        };
581
582        let temp_dir = TempDir::new().unwrap();
583        let path = temp_dir.path().join("manifest_list_v2.avro");
584        let io = FileIO::new_with_fs();
585        let file_writer = file_writer(&path, io).await;
586
587        let mut writer = ManifestListWriter::v2(file_writer, snapshot_id, Some(0), seq_num);
588        writer
589            .add_manifests(expected_manifest_list.entries.clone().into_iter())
590            .unwrap();
591        writer.close().await.unwrap();
592
593        let bs = fs::read(path).unwrap();
594
595        let manifest_list =
596            ManifestList::parse_with_version(&bs, crate::spec::FormatVersion::V3).unwrap();
597        expected_manifest_list.entries[0].sequence_number = seq_num;
598        expected_manifest_list.entries[0].min_sequence_number = seq_num;
599        assert_eq!(manifest_list, expected_manifest_list);
600
601        temp_dir.close().unwrap();
602    }
603
604    #[tokio::test]
605    async fn test_manifest_list_writer_v3_encrypted_round_trip() {
606        let (mgr, file_io) = fresh_encryption_manager_and_io();
607        let path = "memory:///manifest_list_v3_encrypted.avro";
608
609        let encrypted_output = mgr.encrypt(file_io.new_output(path).unwrap());
610
611        let snapshot_id = 9_000_000_000_000_001i64;
612        let seq_num = 7i64;
613        let mut expected = ManifestList {
614            entries: vec![ManifestFile {
615                manifest_path: "memory:///encrypted/v3_m0.avro".to_string(),
616                manifest_length: 1234,
617                partition_spec_id: 0,
618                content: ManifestContentType::Data,
619                sequence_number: UNASSIGNED_SEQUENCE_NUMBER,
620                min_sequence_number: UNASSIGNED_SEQUENCE_NUMBER,
621                added_snapshot_id: snapshot_id,
622                added_files_count: Some(2),
623                existing_files_count: Some(0),
624                deleted_files_count: Some(0),
625                added_rows_count: Some(10),
626                existing_rows_count: Some(0),
627                deleted_rows_count: Some(0),
628                partitions: Some(vec![FieldSummary {
629                    contains_null: false,
630                    contains_nan: Some(false),
631                    lower_bound: Some(Datum::long(1).to_bytes().unwrap()),
632                    upper_bound: Some(Datum::long(1).to_bytes().unwrap()),
633                }]),
634                key_metadata: None,
635                first_row_id: None,
636            }],
637        };
638
639        let file_writer = encrypted_output.writer().await.unwrap();
640        let mut writer = ManifestListWriter::v3(file_writer, snapshot_id, Some(0), seq_num, None);
641        writer
642            .add_manifests(expected.entries.clone().into_iter())
643            .unwrap();
644        let file_metadata = writer.close().await.unwrap();
645
646        let raw_bytes = file_io.new_input(path).unwrap().read().await.unwrap();
647        assert!(
648            ManifestList::parse_with_version(&raw_bytes, crate::spec::FormatVersion::V3).is_err(),
649            "raw bytes should be ciphertext, not parseable as Avro"
650        );
651
652        let key_metadata = encrypted_output.key_metadata_with_saved_file_metadata(&file_metadata);
653        assert_eq!(key_metadata.file_length(), Some(raw_bytes.len() as u64));
654        let plaintext = EncryptedInputFile::new(file_io.new_input(path).unwrap(), key_metadata)
655            .read()
656            .await
657            .unwrap();
658        let manifest_list =
659            ManifestList::parse_with_version(&plaintext, crate::spec::FormatVersion::V3).unwrap();
660
661        expected.entries[0].sequence_number = seq_num;
662        expected.entries[0].min_sequence_number = seq_num;
663        assert_eq!(manifest_list, expected);
664    }
665
666    fn fresh_encryption_manager_and_io() -> (EncryptionManager, FileIO) {
667        let kms = MemoryKeyManagementClient::new();
668        kms.add_master_key("master-1").unwrap();
669        let mgr = EncryptionManager::builder()
670            .kms_client(Arc::new(kms) as Arc<dyn KeyManagementClient>)
671            .table_key_id("master-1")
672            .build();
673        (mgr, FileIO::new_with_memory())
674    }
675
676    async fn file_writer(path: &Path, io: FileIO) -> Box<dyn FileWrite> {
677        io.new_output(path.to_str().unwrap())
678            .unwrap()
679            .writer()
680            .await
681            .unwrap()
682    }
683}