Skip to main content

iceberg/transaction/
append.rs

1// Licensed to the Apache Software Foundation (ASF) under one
2// or more contributor license agreements.  See the NOTICE file
3// distributed with this work for additional information
4// regarding copyright ownership.  The ASF licenses this file
5// to you under the Apache License, Version 2.0 (the
6// "License"); you may not use this file except in compliance
7// with the License.  You may obtain a copy of the License at
8//
9//   http://www.apache.org/licenses/LICENSE-2.0
10//
11// Unless required by applicable law or agreed to in writing,
12// software distributed under the License is distributed on an
13// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14// KIND, either express or implied.  See the License for the
15// specific language governing permissions and limitations
16// under the License.
17
18use std::collections::{HashMap, HashSet};
19
20use async_trait::async_trait;
21use uuid::Uuid;
22
23use crate::error::{Result, invalid_data};
24use crate::spec::{DataContentType, DataFile};
25use crate::table::Table;
26use crate::transaction::action::CommitStatus;
27use crate::transaction::snapshot::simple::{AppendIntent, SimpleSnapshotProducer};
28use crate::transaction::snapshot::validate_files_not_live;
29use crate::transaction::{ActionCommit, TransactionAction};
30use crate::{Error, ErrorKind};
31
32/// FastAppendAction is a transaction action for fast append data files to the table.
33#[derive(Clone)]
34pub struct FastAppendAction {
35    check_duplicate: bool,
36    /// Overrides the commit UUID of the producer created in `new_state`.
37    commit_uuid: Option<Uuid>,
38    snapshot_properties: HashMap<String, String>,
39    /// Free of duplicate paths; see [`Self::add_data_files`].
40    added_data_files: Vec<DataFile>,
41    /// Paths already in `added_data_files`.
42    added_paths: HashSet<String>,
43}
44
45impl FastAppendAction {
46    pub(crate) fn new() -> Self {
47        Self {
48            check_duplicate: true,
49            commit_uuid: None,
50            snapshot_properties: HashMap::default(),
51            added_data_files: vec![],
52            added_paths: HashSet::new(),
53        }
54    }
55
56    /// Set whether to check duplicate files
57    pub fn with_check_duplicate(mut self, v: bool) -> Self {
58        self.check_duplicate = v;
59        self
60    }
61
62    /// Add data files to the snapshot.
63    ///
64    /// A path added more than once keeps its first occurrence, so a manifest
65    /// never references the same file twice. This is independent of the
66    /// `check_duplicate`-gated check against files already in the table.
67    pub fn add_data_files(mut self, data_files: impl IntoIterator<Item = DataFile>) -> Self {
68        for data_file in data_files {
69            if self.added_paths.insert(data_file.file_path.clone()) {
70                self.added_data_files.push(data_file);
71            }
72        }
73        self
74    }
75
76    /// Set commit UUID for the snapshot.
77    pub fn set_commit_uuid(mut self, commit_uuid: Uuid) -> Self {
78        self.commit_uuid = Some(commit_uuid);
79        self
80    }
81
82    /// Set snapshot summary properties.
83    pub fn set_snapshot_properties(mut self, snapshot_properties: HashMap<String, String>) -> Self {
84        self.snapshot_properties = snapshot_properties;
85        self
86    }
87
88    /// Checks the parts of this action that do not depend on the table.
89    fn validate(&self) -> Result<()> {
90        // TODO: Allowing snapshot property setup with no added data files is a workaround.
91        // We should clean it up after all necessary actions are supported.
92        // For details, please refer to https://github.com/apache/iceberg-rust/issues/1548
93        if self.added_data_files.is_empty() && self.snapshot_properties.is_empty() {
94            return Err(Error::new(
95                ErrorKind::PreconditionFailed,
96                "No added data files or added snapshot properties found when write a manifest file",
97            ));
98        }
99        if let Some(data_file) = self
100            .added_data_files
101            .iter()
102            .find(|f| f.content_type() != DataContentType::Data)
103        {
104            return Err(invalid_data!(
105                "Only data content type is allowed for fast append, got {:?} for {}",
106                data_file.content_type(),
107                data_file.file_path()
108            ));
109        }
110        Ok(())
111    }
112}
113
114#[async_trait]
115impl TransactionAction for FastAppendAction {
116    type State = SimpleSnapshotProducer;
117
118    fn new_state(&self) -> SimpleSnapshotProducer {
119        SimpleSnapshotProducer::new(AppendIntent {
120            commit_uuid: self.commit_uuid,
121            snapshot_properties: self.snapshot_properties.clone(),
122            data_files: self.added_data_files.clone(),
123        })
124    }
125
126    async fn commit(
127        &self,
128        state: &mut SimpleSnapshotProducer,
129        table: &Table,
130    ) -> Result<ActionCommit> {
131        self.validate()?;
132        if self.check_duplicate {
133            validate_files_not_live(table, &self.added_data_files).await?;
134        }
135        state.apply(table).await
136    }
137
138    async fn cleanup(self, _state: SimpleSnapshotProducer, _table: &Table, _status: CommitStatus) {
139        // TODO(#3417): remove the producer's uncommitted artifacts once terminal
140        // cleanup is wired up.
141    }
142}
143
144#[cfg(test)]
145mod tests {
146    use std::collections::HashMap;
147    use std::fs;
148    use std::sync::Arc;
149    use std::sync::atomic::{AtomicU32, Ordering};
150
151    use minijinja::{AutoEscape, Environment, Value, context};
152    use tempfile::TempDir;
153    use uuid::Uuid;
154
155    use crate::catalog::MockCatalog;
156    use crate::encryption::kms::MemoryKeyManagementClient;
157    use crate::encryption::{SensitiveBytes, StandardKeyMetadata};
158    use crate::io::FileIO;
159    use crate::spec::{
160        DataContentType, DataFile, DataFileBuilder, DataFileFormat, Literal, MAIN_BRANCH,
161        ManifestEntry, ManifestListWriter, ManifestStatus, ManifestWriterBuilder, SnapshotRef,
162        Struct, TableMetadata, TableProperties,
163    };
164    use crate::table::Table;
165    use crate::test_utils::{commit_with_fresh_state, make_encrypted_table, test_runtime};
166    use crate::transaction::tests::{
167        make_v1_table, make_v2_minimal_table, make_v3_minimal_table, with_bucket_partition_spec,
168        with_retry_properties,
169    };
170    use crate::transaction::{ApplyTransactionAction, FastAppendAction, Transaction};
171    use crate::{ErrorKind, TableCommit, TableIdent, TableRequirement, TableUpdate};
172
173    fn render_template(template: &str, ctx: Value) -> String {
174        let mut env = Environment::new();
175        env.set_auto_escape_callback(|_| AutoEscape::None);
176        env.render_str(template, ctx).unwrap()
177    }
178
179    /// Builds a table whose current snapshot's manifest list contains a data manifest
180    /// followed by a delete-only manifest (one entry with `ManifestStatus::Deleted`,
181    /// so `deleted_files_count > 0` while `added_files_count == existing_files_count == 0`).
182    ///
183    /// Returns the table plus the `manifest_path` of the delete-only manifest so callers
184    /// can assert whether a subsequent append carries it forward.
185    async fn make_table_with_delete_only_manifest() -> (Table, TempDir, String) {
186        let tmp_dir = TempDir::new().unwrap();
187        let table_location = tmp_dir.path().join("table1");
188        let manifest_list_location = table_location.join("metadata/manifests_list_1.avro");
189        let table_metadata_location = table_location.join("metadata/v1.json");
190
191        let file_io = FileIO::new_with_fs();
192
193        let template = fs::read_to_string(format!(
194            "{}/testdata/example_table_metadata_v2.json",
195            env!("CARGO_MANIFEST_DIR")
196        ))
197        .unwrap();
198        // The template has two snapshots; point the current one at our manifest list.
199        let metadata_json = render_template(&template, context! {
200            table_location => &table_location,
201            manifest_list_1_location => &manifest_list_location,
202            manifest_list_2_location => &manifest_list_location,
203            table_metadata_1_location => &table_metadata_location,
204        });
205        let table_metadata = serde_json::from_str::<TableMetadata>(&metadata_json).unwrap();
206
207        let table = Table::builder()
208            .metadata(table_metadata)
209            .identifier(TableIdent::from_strs(["db", "table1"]).unwrap())
210            .file_io(file_io)
211            .metadata_location(table_metadata_location.to_str().unwrap())
212            .runtime(test_runtime())
213            .build()
214            .unwrap();
215
216        let current_snapshot = table.metadata().current_snapshot().unwrap();
217        let schema = current_snapshot.schema(table.metadata()).unwrap();
218        let partition_spec = table.metadata().default_partition_spec();
219
220        let next_manifest_file = |location: &str| {
221            table
222                .file_io()
223                .new_output(format!(
224                    "{}/metadata/manifest_{}.avro",
225                    location,
226                    Uuid::new_v4()
227                ))
228                .unwrap()
229        };
230        let table_location_str = table_location.to_str().unwrap().to_string();
231
232        // Data manifest: one Added data file.
233        let mut data_writer = ManifestWriterBuilder::new(
234            next_manifest_file(&table_location_str),
235            Some(current_snapshot.snapshot_id()),
236            schema.clone(),
237            partition_spec.as_ref().clone(),
238        )
239        .build_v2_data();
240        data_writer
241            .add_entry(
242                ManifestEntry::builder()
243                    .status(ManifestStatus::Added)
244                    .data_file(
245                        DataFileBuilder::default()
246                            .partition_spec_id(0)
247                            .content(DataContentType::Data)
248                            .file_path(format!("{table_location_str}/data.parquet"))
249                            .file_format(DataFileFormat::Parquet)
250                            .file_size_in_bytes(100)
251                            .record_count(1)
252                            .partition(Struct::from_iter([Some(Literal::long(100))]))
253                            .build()
254                            .unwrap(),
255                    )
256                    .build(),
257            )
258            .unwrap();
259        let data_manifest = data_writer.write_manifest_file().await.unwrap();
260
261        // Delete-only manifest: a single Deleted entry, nothing added or existing.
262        let mut delete_writer = ManifestWriterBuilder::new(
263            next_manifest_file(&table_location_str),
264            Some(current_snapshot.snapshot_id()),
265            schema.clone(),
266            partition_spec.as_ref().clone(),
267        )
268        .build_v2_data();
269        delete_writer
270            .add_delete_entry(
271                ManifestEntry::builder()
272                    .status(ManifestStatus::Deleted)
273                    .sequence_number(0)
274                    .file_sequence_number(0)
275                    .data_file(
276                        DataFileBuilder::default()
277                            .partition_spec_id(0)
278                            .content(DataContentType::Data)
279                            .file_path(format!("{table_location_str}/removed.parquet"))
280                            .file_format(DataFileFormat::Parquet)
281                            .file_size_in_bytes(100)
282                            .record_count(1)
283                            .partition(Struct::from_iter([Some(Literal::long(100))]))
284                            .build()
285                            .unwrap(),
286                    )
287                    .build(),
288            )
289            .unwrap();
290        let delete_manifest = delete_writer.write_manifest_file().await.unwrap();
291        let delete_manifest_path = delete_manifest.manifest_path.clone();
292
293        // Sanity: the delete manifest really is delete-only.
294        assert!(delete_manifest.has_deleted_files());
295        assert!(!delete_manifest.has_added_files());
296        assert!(!delete_manifest.has_existing_files());
297
298        let mut manifest_list_writer = ManifestListWriter::v2(
299            table
300                .file_io()
301                .new_output(current_snapshot.manifest_list())
302                .unwrap()
303                .writer()
304                .await
305                .unwrap(),
306            current_snapshot.snapshot_id(),
307            current_snapshot.parent_snapshot_id(),
308            current_snapshot.sequence_number(),
309        );
310        manifest_list_writer
311            .add_manifests(vec![data_manifest, delete_manifest].into_iter())
312            .unwrap();
313        manifest_list_writer.close().await.unwrap();
314
315        (table, tmp_dir, delete_manifest_path)
316    }
317
318    /// Regression test for #2148: a `fast_append` must carry delete-only manifests
319    /// forward into the new snapshot. Dropping them lets the files they mark as
320    /// removed reappear as live data on the next append.
321    #[tokio::test]
322    async fn test_fast_append_preserves_delete_only_manifest() {
323        let (table, _tmp_dir, delete_manifest_path) = make_table_with_delete_only_manifest().await;
324
325        // Append a new data file via the public transaction API.
326        let new_file = DataFileBuilder::default()
327            .content(DataContentType::Data)
328            .file_path(format!("{}/appended.parquet", table.metadata().location()))
329            .file_format(DataFileFormat::Parquet)
330            .file_size_in_bytes(100)
331            .record_count(1)
332            .partition_spec_id(table.metadata().default_partition_spec_id())
333            .partition(Struct::from_iter([Some(Literal::long(100))]))
334            .build()
335            .unwrap();
336
337        let tx = Transaction::new(&table);
338        let action = tx.fast_append().add_data_files(vec![new_file]);
339        let mut action_commit = commit_with_fresh_state(action, &table).await.unwrap();
340        let updates = action_commit.take_updates();
341
342        let new_snapshot: SnapshotRef = if let TableUpdate::AddSnapshot { snapshot } = &updates[0] {
343            SnapshotRef::new(snapshot.clone())
344        } else {
345            unreachable!("first update of a fast append should be AddSnapshot")
346        };
347
348        let manifest_list = table
349            .manifest_list_reader(&new_snapshot)
350            .load()
351            .await
352            .unwrap();
353
354        assert!(
355            manifest_list
356                .entries()
357                .iter()
358                .any(|m| m.manifest_path == delete_manifest_path),
359            "delete-only manifest {delete_manifest_path} was dropped from the new snapshot's \
360             manifest list; the files it removed would reappear as live data"
361        );
362    }
363
364    /// Load the data files written by a single-manifest fast-append commit.
365    async fn committed_data_files(table: &Table, updates: &[TableUpdate]) -> Vec<DataFile> {
366        let TableUpdate::AddSnapshot { snapshot } = &updates[0] else {
367            unreachable!("first update is always AddSnapshot")
368        };
369        let manifest_list = table
370            .manifest_list_reader(&SnapshotRef::new(snapshot.clone()))
371            .load()
372            .await
373            .unwrap();
374        assert_eq!(1, manifest_list.entries().len());
375        table
376            .manifest_reader()
377            .read(&manifest_list.entries()[0])
378            .await
379            .unwrap()
380            .entries()
381            .iter()
382            .map(|entry| entry.data_file().clone())
383            .collect()
384    }
385
386    #[tokio::test]
387    async fn test_fast_append_writes_encrypted_manifest() {
388        let table = make_encrypted_table().await;
389        assert!(
390            table.encryption_manager().is_some(),
391            "fixture table should have an EncryptionManager"
392        );
393
394        let new_file = DataFileBuilder::default()
395            .content(DataContentType::Data)
396            .file_path("memory:///table/data/00000.parquet".to_string())
397            .file_format(DataFileFormat::Parquet)
398            .partition(Struct::empty())
399            .record_count(100)
400            .file_size_in_bytes(4096)
401            .partition_spec_id(table.metadata().default_partition_spec_id())
402            .build()
403            .unwrap();
404
405        let tx = Transaction::new(&table);
406        let action = tx.fast_append().add_data_files(vec![new_file]);
407        let mut action_commit = commit_with_fresh_state(action, &table).await.unwrap();
408        let updates = action_commit.take_updates();
409
410        let new_snapshot: SnapshotRef = updates
411            .iter()
412            .find_map(|u| match u {
413                TableUpdate::AddSnapshot { snapshot } => Some(SnapshotRef::new(snapshot.clone())),
414                _ => None,
415            })
416            .expect("a fast append should emit an AddSnapshot update");
417
418        let manifest_list_key_metadata = table
419            .encryption_manager()
420            .unwrap()
421            .decrypt_manifest_list_key_metadata(new_snapshot.encryption_key_id().unwrap())
422            .await
423            .unwrap();
424        let manifest_list_size = table
425            .file_io()
426            .new_input(new_snapshot.manifest_list())
427            .unwrap()
428            .metadata()
429            .await
430            .unwrap()
431            .size;
432        assert_eq!(
433            manifest_list_key_metadata.file_length(),
434            Some(manifest_list_size)
435        );
436
437        let manifest_list = table
438            .manifest_list_reader(&new_snapshot)
439            .load()
440            .await
441            .unwrap();
442        let manifest_file = manifest_list
443            .entries()
444            .iter()
445            .find(|m| m.added_files_count.unwrap_or(0) > 0)
446            .expect("new snapshot should carry the appended data manifest");
447
448        // The manifest list entry must carry decodable key metadata.
449        let key_metadata_bytes = manifest_file
450            .key_metadata
451            .as_ref()
452            .expect("encrypted manifest must record key metadata");
453        let key_metadata = StandardKeyMetadata::decode(key_metadata_bytes)
454            .expect("recorded key metadata must decode as StandardKeyMetadata");
455        let manifest_size = table
456            .file_io()
457            .new_input(&manifest_file.manifest_path)
458            .unwrap()
459            .metadata()
460            .await
461            .unwrap()
462            .size;
463        assert_eq!(key_metadata.file_length(), Some(manifest_size));
464        assert_eq!(manifest_file.manifest_length, manifest_size as i64);
465
466        // The reader self-decrypts using the recorded key metadata and must
467        // recover the entry we appended. Because the read goes through the
468        // decryption path, this succeeding also proves the bytes
469        // on disk were genuinely encrypted (not silently written as plaintext).
470        let manifest = table.manifest_reader().read(manifest_file).await.unwrap();
471        assert_eq!(manifest.entries().len(), 1);
472        assert_eq!(
473            manifest.entries()[0].data_file().file_path(),
474            "memory:///table/data/00000.parquet"
475        );
476    }
477
478    #[tokio::test]
479    async fn test_fast_append_rejects_delete_files() {
480        let table = make_v2_minimal_table();
481        let delete_file = DataFileBuilder::default()
482            .content(DataContentType::EqualityDeletes)
483            .file_path("test/1-deletes.parquet".to_string())
484            .file_format(DataFileFormat::Parquet)
485            .file_size_in_bytes(100)
486            .record_count(1)
487            .partition_spec_id(table.metadata().default_partition_spec_id())
488            .partition(Struct::from_iter([Some(Literal::long(300))]))
489            .equality_ids(Some(vec![1]))
490            .build()
491            .unwrap();
492
493        let tx = Transaction::new(&table);
494        let action = tx.fast_append().add_data_files(vec![delete_file]);
495        let err = commit_with_fresh_state(action, &table)
496            .await
497            .err()
498            .expect("delete files must be rejected");
499        assert_eq!(err.kind(), ErrorKind::DataInvalid);
500    }
501
502    #[tokio::test]
503    async fn test_empty_data_append_action() {
504        let table = make_v2_minimal_table();
505        let tx = Transaction::new(&table);
506        let action = tx.fast_append().add_data_files(vec![]);
507        assert!(commit_with_fresh_state(action, &table).await.is_err());
508    }
509
510    /// A `fast_append` must write the manifest list and the manifest
511    /// files under the `write.metadata.path` prefix when configured,
512    /// rather than the default `<location>/metadata` directory.
513    #[tokio::test]
514    async fn test_fast_append_honors_write_metadata_path() {
515        let base = make_v2_minimal_table();
516        let metadata_root = format!("{}/custom-meta", base.metadata().location());
517        let metadata = base
518            .metadata()
519            .clone()
520            .into_builder(None)
521            .set_properties(HashMap::from([(
522                "write.metadata.path".to_string(),
523                metadata_root.clone(),
524            )]))
525            .unwrap()
526            .build()
527            .unwrap()
528            .metadata;
529        let table = base.with_metadata(Arc::new(metadata));
530
531        let data_file = DataFileBuilder::default()
532            .content(DataContentType::Data)
533            .file_path(format!("{}/data/1.parquet", table.metadata().location()))
534            .file_format(DataFileFormat::Parquet)
535            .file_size_in_bytes(100)
536            .record_count(1)
537            .partition_spec_id(table.metadata().default_partition_spec_id())
538            .partition(Struct::from_iter([Some(Literal::long(300))]))
539            .build()
540            .unwrap();
541
542        let tx = Transaction::new(&table);
543        let action = tx.fast_append().add_data_files(vec![data_file]);
544        let mut action_commit = commit_with_fresh_state(action, &table).await.unwrap();
545        let updates = action_commit.take_updates();
546
547        let new_snapshot: SnapshotRef = if let TableUpdate::AddSnapshot { snapshot } = &updates[0] {
548            SnapshotRef::new(snapshot.clone())
549        } else {
550            unreachable!("first update of a fast append should be AddSnapshot")
551        };
552
553        let prefix = format!("{metadata_root}/");
554
555        // Manifest list
556        assert!(
557            new_snapshot.manifest_list().starts_with(prefix.as_str()),
558            "manifest list {} not under configured write.metadata.path {metadata_root}",
559            new_snapshot.manifest_list()
560        );
561
562        // Manifest files
563        let manifest_list = table
564            .manifest_list_reader(&new_snapshot)
565            .load()
566            .await
567            .unwrap();
568        assert!(
569            !manifest_list.entries().is_empty(),
570            "expected at least one manifest entry"
571        );
572        for entry in manifest_list.entries() {
573            assert!(
574                entry.manifest_path.starts_with(prefix.as_str()),
575                "manifest {} not under configured write.metadata.path {metadata_root}",
576                entry.manifest_path
577            );
578        }
579    }
580
581    #[tokio::test]
582    async fn test_set_snapshot_properties() {
583        let table = make_v2_minimal_table();
584        let tx = Transaction::new(&table);
585
586        let mut snapshot_properties = HashMap::new();
587        snapshot_properties.insert("key".to_string(), "val".to_string());
588
589        let data_file = DataFileBuilder::default()
590            .content(DataContentType::Data)
591            .file_path("test/1.parquet".to_string())
592            .file_format(DataFileFormat::Parquet)
593            .file_size_in_bytes(100)
594            .record_count(1)
595            .partition_spec_id(table.metadata().default_partition_spec_id())
596            .partition(Struct::from_iter([Some(Literal::long(300))]))
597            .build()
598            .unwrap();
599
600        let action = tx
601            .fast_append()
602            .set_snapshot_properties(snapshot_properties)
603            .add_data_files(vec![data_file]);
604        let mut action_commit = commit_with_fresh_state(action, &table).await.unwrap();
605        let updates = action_commit.take_updates();
606
607        // Check customized properties is contained in snapshot summary properties.
608        let new_snapshot = if let TableUpdate::AddSnapshot { snapshot } = &updates[0] {
609            snapshot
610        } else {
611            unreachable!()
612        };
613        assert_eq!(
614            new_snapshot
615                .summary()
616                .additional_properties
617                .get("key")
618                .unwrap(),
619            "val"
620        );
621    }
622
623    /// See `testdata/manifests_lists/README.md`.
624    const FIXTURE_MASTER_KEY_ID: &str = "master-1";
625    const FIXTURE_MASTER_KEY_BYTES: [u8; 16] = [
626        0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e,
627        0x0f,
628    ];
629
630    async fn make_v3_encrypted_table() -> Table {
631        let json = fs::read_to_string(format!(
632            "{}/testdata/table_metadata/TableMetadataV3ValidEncryption.json",
633            env!("CARGO_MANIFEST_DIR")
634        ))
635        .unwrap();
636        let metadata = serde_json::from_str::<TableMetadata>(&json).unwrap();
637
638        let kms = MemoryKeyManagementClient::new();
639        kms.add_master_key_bytes(
640            FIXTURE_MASTER_KEY_ID,
641            SensitiveBytes::new(FIXTURE_MASTER_KEY_BYTES),
642        )
643        .unwrap();
644
645        let file_io = FileIO::new_with_memory();
646
647        let manifest_list_bytes = fs::read(format!(
648            "{}/testdata/manifests_lists/manifest-list-v3-encrypted.avro",
649            env!("CARGO_MANIFEST_DIR")
650        ))
651        .unwrap();
652        let parent_manifest_list = metadata.current_snapshot().unwrap().manifest_list();
653        file_io
654            .new_output(parent_manifest_list)
655            .unwrap()
656            .write(manifest_list_bytes.into())
657            .await
658            .unwrap();
659
660        Table::builder()
661            .metadata(metadata)
662            .metadata_location("memory:///table/metadata/v1.json")
663            .identifier(TableIdent::from_strs(["ns1", "enc"]).unwrap())
664            .file_io(file_io)
665            .kms_client(Arc::new(kms))
666            .runtime(test_runtime())
667            .build()
668            .unwrap()
669    }
670
671    #[tokio::test]
672    async fn test_commit_with_encryption_adds_keys_and_records_snapshot_key_id() {
673        let table = make_v3_encrypted_table().await;
674
675        let data_file = DataFileBuilder::default()
676            .content(DataContentType::Data)
677            .file_path("test/1.parquet".to_string())
678            .file_format(DataFileFormat::Parquet)
679            .file_size_in_bytes(100)
680            .record_count(1)
681            .partition_spec_id(table.metadata().default_partition_spec_id())
682            .partition(Struct::empty())
683            .build()
684            .unwrap();
685
686        let tx = Transaction::new(&table);
687        let action = tx.fast_append().add_data_files(vec![data_file]);
688        let mut action_commit = commit_with_fresh_state(action, &table).await.unwrap();
689        let updates = action_commit.take_updates();
690
691        let added_key_ids: Vec<String> = updates
692            .iter()
693            .filter_map(|u| match u {
694                TableUpdate::AddEncryptionKey { encryption_key } => {
695                    Some(encryption_key.key_id().to_string())
696                }
697                _ => None,
698            })
699            .collect();
700        assert!(!added_key_ids.is_empty(), "got {updates:?}");
701
702        // Encryption keys are added before the snapshot, so it isn't updates[0] here.
703        let new_snapshot = updates
704            .iter()
705            .find_map(|u| match u {
706                TableUpdate::AddSnapshot { snapshot } => Some(snapshot),
707                _ => None,
708            })
709            .expect("commit should add a snapshot");
710
711        let snapshot_key_id = new_snapshot
712            .encryption_key_id()
713            .expect("encrypted snapshot should record its manifest-list key id");
714        assert!(
715            added_key_ids.iter().any(|id| id == snapshot_key_id),
716            "snapshot key id {snapshot_key_id} not in added keys {added_key_ids:?}"
717        );
718
719        let new_snapshot_ref: SnapshotRef = Arc::new(new_snapshot.clone());
720        let manifest_list = table
721            .manifest_list_reader(&new_snapshot_ref)
722            .load()
723            .await
724            .expect("newly written encrypted manifest list should decrypt and parse");
725        assert_eq!(
726            manifest_list.entries().len(),
727            1,
728            "append should record exactly the one new data manifest"
729        );
730    }
731
732    #[tokio::test]
733    async fn test_snapshot_properties_cannot_override_computed_metrics() {
734        // A user-supplied snapshot property must not shadow a computed metric key
735        // such as `added-data-files`. Matching iceberg-java, the computed value
736        // wins, so the summary reflects the real count and a bad value can neither
737        // corrupt the summary nor panic total computation (see #2184-adjacent fix).
738        let table = make_v2_minimal_table();
739        let tx = Transaction::new(&table);
740
741        let mut snapshot_properties = HashMap::new();
742        // Both a benign-but-wrong value and a non-integer value collide with
743        // computed metric keys; neither should reach the final summary.
744        snapshot_properties.insert("added-data-files".to_string(), "9999".to_string());
745        snapshot_properties.insert("added-records".to_string(), "not-a-number".to_string());
746
747        let data_file = DataFileBuilder::default()
748            .content(DataContentType::Data)
749            .file_path("test/1.parquet".to_string())
750            .file_format(DataFileFormat::Parquet)
751            .file_size_in_bytes(100)
752            .record_count(1)
753            .partition_spec_id(table.metadata().default_partition_spec_id())
754            .partition(Struct::from_iter([Some(Literal::long(300))]))
755            .build()
756            .unwrap();
757
758        let action = tx
759            .fast_append()
760            .set_snapshot_properties(snapshot_properties)
761            .add_data_files(vec![data_file]);
762        // Must not panic during total computation.
763        let mut action_commit = commit_with_fresh_state(action, &table).await.unwrap();
764        let updates = action_commit.take_updates();
765
766        let new_snapshot = if let TableUpdate::AddSnapshot { snapshot } = &updates[0] {
767            snapshot
768        } else {
769            unreachable!()
770        };
771        let props = &new_snapshot.summary().additional_properties;
772
773        // Computed metric wins over the user's colliding values.
774        assert_eq!(
775            props.get("added-data-files").unwrap(),
776            "1",
777            "computed added-data-files must override the user-supplied value"
778        );
779        assert_eq!(
780            props.get("added-records").unwrap(),
781            "1",
782            "computed added-records must override the user-supplied non-integer value"
783        );
784    }
785
786    #[tokio::test]
787    async fn test_snapshot_summary_records_writer_identity() {
788        let table = make_v2_minimal_table();
789        let tx = Transaction::new(&table);
790
791        let snapshot_properties = HashMap::from([
792            ("key".to_string(), "val".to_string()),
793            ("iceberg-version".to_string(), "caller".to_string()),
794        ]);
795
796        let action = tx
797            .fast_append()
798            .set_snapshot_properties(snapshot_properties);
799        let mut action_commit = commit_with_fresh_state(action, &table).await.unwrap();
800        let updates = action_commit.take_updates();
801
802        let new_snapshot = if let TableUpdate::AddSnapshot { snapshot } = &updates[0] {
803            snapshot
804        } else {
805            unreachable!()
806        };
807        let props = &new_snapshot.summary().additional_properties;
808        let version = env!("CARGO_PKG_VERSION");
809
810        assert_eq!(
811            props.get("iceberg-version").unwrap(),
812            &format!("Apache Iceberg Rust {version}")
813        );
814        assert_eq!(props.get("key").unwrap(), "val");
815    }
816
817    #[tokio::test]
818    async fn test_append_snapshot_properties() {
819        let table = make_v2_minimal_table();
820        let tx = Transaction::new(&table);
821
822        let mut snapshot_properties = HashMap::new();
823        snapshot_properties.insert("key".to_string(), "val".to_string());
824
825        let action = tx
826            .fast_append()
827            .set_snapshot_properties(snapshot_properties);
828        let mut action_commit = commit_with_fresh_state(action, &table).await.unwrap();
829        let updates = action_commit.take_updates();
830
831        // Check customized properties is contained in snapshot summary properties.
832        let new_snapshot = if let TableUpdate::AddSnapshot { snapshot } = &updates[0] {
833            snapshot
834        } else {
835            unreachable!()
836        };
837        assert_eq!(
838            new_snapshot
839                .summary()
840                .additional_properties
841                .get("key")
842                .unwrap(),
843            "val"
844        );
845    }
846
847    #[tokio::test]
848    async fn test_fast_append_file_with_incompatible_partition_value() {
849        let table = make_v2_minimal_table();
850        let tx = Transaction::new(&table);
851        let action = tx.fast_append();
852
853        // check add data file with incompatible partition value
854        let data_file = DataFileBuilder::default()
855            .content(DataContentType::Data)
856            .file_path("test/3.parquet".to_string())
857            .file_format(DataFileFormat::Parquet)
858            .file_size_in_bytes(100)
859            .record_count(1)
860            .partition_spec_id(table.metadata().default_partition_spec_id())
861            .partition(Struct::from_iter([Some(Literal::string("test"))]))
862            .build()
863            .unwrap();
864
865        let action = action.add_data_files(vec![data_file.clone()]);
866
867        assert!(commit_with_fresh_state(action, &table).await.is_err());
868    }
869
870    #[tokio::test]
871    async fn test_fast_append_dedupes_intra_batch_duplicate_paths() {
872        let table = make_v2_minimal_table();
873        let tx = Transaction::new(&table);
874
875        let make_file = |size: u64, records: u64| {
876            DataFileBuilder::default()
877                .content(DataContentType::Data)
878                .file_path("test/dup.parquet".to_string())
879                .file_format(DataFileFormat::Parquet)
880                .file_size_in_bytes(size)
881                .record_count(records)
882                .partition_spec_id(table.metadata().default_partition_spec_id())
883                .partition(Struct::from_iter([Some(Literal::long(1))]))
884                .build()
885                .unwrap()
886        };
887
888        // Same path three times, across two `add_data_files` calls: the manifest
889        // keeps a single entry, the first one.
890        let action = tx
891            .fast_append()
892            .add_data_files(vec![make_file(100, 10), make_file(200, 20)])
893            .add_data_files(vec![make_file(300, 30)]);
894        let mut action_commit = commit_with_fresh_state(action, &table).await.unwrap();
895        let files = committed_data_files(&table, &action_commit.take_updates()).await;
896        assert_eq!(1, files.len());
897        assert_eq!(100, files[0].file_size_in_bytes());
898    }
899
900    #[tokio::test]
901    async fn test_fast_append_dedupes_regardless_of_check_duplicate_flag() {
902        let table = make_v2_minimal_table();
903        let tx = Transaction::new(&table);
904
905        let make_file = || {
906            DataFileBuilder::default()
907                .content(DataContentType::Data)
908                .file_path("test/dup.parquet".to_string())
909                .file_format(DataFileFormat::Parquet)
910                .file_size_in_bytes(100)
911                .record_count(10)
912                .partition_spec_id(table.metadata().default_partition_spec_id())
913                .partition(Struct::from_iter([Some(Literal::long(1))]))
914                .build()
915                .unwrap()
916        };
917
918        // `check_duplicate` only gates the cross-snapshot check; intra-batch dedupe runs regardless.
919        let action = tx
920            .fast_append()
921            .with_check_duplicate(false)
922            .add_data_files(vec![make_file(), make_file()]);
923        let mut action_commit = commit_with_fresh_state(action, &table).await.unwrap();
924        let files = committed_data_files(&table, &action_commit.take_updates()).await;
925        assert_eq!(1, files.len());
926    }
927
928    #[tokio::test]
929    async fn test_fast_append() {
930        let table = make_v2_minimal_table();
931        let tx = Transaction::new(&table);
932        let action = tx.fast_append();
933
934        let data_file = DataFileBuilder::default()
935            .content(DataContentType::Data)
936            .file_path("test/3.parquet".to_string())
937            .file_format(DataFileFormat::Parquet)
938            .file_size_in_bytes(100)
939            .record_count(1)
940            .partition_spec_id(table.metadata().default_partition_spec_id())
941            .partition(Struct::from_iter([Some(Literal::long(300))]))
942            .build()
943            .unwrap();
944
945        let action = action.add_data_files(vec![data_file.clone()]);
946        let mut action_commit = commit_with_fresh_state(action, &table).await.unwrap();
947        let updates = action_commit.take_updates();
948        let requirements = action_commit.take_requirements();
949
950        // check updates and requirements
951        assert!(
952            matches!((&updates[0],&updates[1]), (TableUpdate::AddSnapshot { snapshot },TableUpdate::SetSnapshotRef { reference,ref_name }) if snapshot.snapshot_id() == reference.snapshot_id && ref_name == MAIN_BRANCH)
953        );
954        assert_eq!(
955            vec![
956                TableRequirement::UuidMatch {
957                    uuid: table.metadata().uuid()
958                },
959                TableRequirement::RefSnapshotIdMatch {
960                    r#ref: MAIN_BRANCH.to_string(),
961                    snapshot_id: table.metadata().current_snapshot_id
962                }
963            ],
964            requirements
965        );
966
967        // check manifest list
968        let new_snapshot: SnapshotRef = if let TableUpdate::AddSnapshot { snapshot } = &updates[0] {
969            SnapshotRef::new(snapshot.clone())
970        } else {
971            unreachable!()
972        };
973        let manifest_list = table
974            .manifest_list_reader(&new_snapshot)
975            .load()
976            .await
977            .unwrap();
978        assert_eq!(1, manifest_list.entries().len());
979        assert_eq!(
980            manifest_list.entries()[0].sequence_number,
981            new_snapshot.sequence_number()
982        );
983
984        // check manifest
985        let manifest = table
986            .manifest_reader()
987            .read(&manifest_list.entries()[0])
988            .await
989            .unwrap();
990        assert_eq!(1, manifest.entries().len());
991        assert_eq!(
992            new_snapshot.sequence_number(),
993            manifest.entries()[0]
994                .sequence_number()
995                .expect("Inherit sequence number by load manifest")
996        );
997
998        assert_eq!(
999            new_snapshot.snapshot_id(),
1000            manifest.entries()[0].snapshot_id().unwrap()
1001        );
1002        assert_eq!(data_file, *manifest.entries()[0].data_file());
1003    }
1004
1005    #[tokio::test]
1006    async fn test_fast_append_groups_files_by_partition_spec() {
1007        for table in [
1008            make_v1_table(),
1009            make_v2_minimal_table(),
1010            make_v3_minimal_table(),
1011        ] {
1012            let metadata = table
1013                .metadata()
1014                .clone()
1015                .into_builder(None)
1016                .set_properties(HashMap::from([(
1017                    TableProperties::PROPERTY_WRITE_PARTITION_SUMMARY_LIMIT.to_string(),
1018                    "2".to_string(),
1019                )]))
1020                .unwrap()
1021                .build()
1022                .unwrap()
1023                .metadata;
1024            let table = table.with_metadata(Arc::new(metadata));
1025            let table = with_bucket_partition_spec(table).with_metadata_location(
1026                "s3://bucket/test/location/metadata/00000-9c12d441-03fe-4693-9a96-a0705ddf69c1.metadata.json".to_string(),
1027            );
1028            let old_file = DataFileBuilder::default()
1029                .content(DataContentType::Data)
1030                .file_path("test/old.parquet".to_string())
1031                .file_format(DataFileFormat::Parquet)
1032                .file_size_in_bytes(100)
1033                .record_count(1)
1034                .partition_spec_id(0)
1035                .partition(Struct::from_iter([Some(Literal::long(300))]))
1036                .build()
1037                .unwrap();
1038            let mut other_old_file = old_file.clone();
1039            other_old_file.file_path = "test/other-old.parquet".to_string();
1040            let mut new_file = old_file.clone();
1041            new_file.file_path = "test/new.parquet".to_string();
1042            new_file.partition_spec_id = table.metadata().default_partition_spec_id();
1043            new_file.partition =
1044                Struct::from_iter([Some(Literal::long(300)), Some(Literal::int(3))]);
1045
1046            let action = Transaction::new(&table).fast_append().add_data_files(vec![
1047                new_file.clone(),
1048                old_file.clone(),
1049                other_old_file.clone(),
1050            ]);
1051            let mut commit = commit_with_fresh_state(action, &table).await.unwrap();
1052            let table = TableCommit::builder()
1053                .ident(table.identifier().clone())
1054                .updates(commit.take_updates())
1055                .requirements(commit.take_requirements())
1056                .build()
1057                .apply(table)
1058                .unwrap();
1059            let snapshot = table.metadata().current_snapshot().unwrap();
1060            let list = table.manifest_list_reader(snapshot).load().await.unwrap();
1061            let mut files_by_spec = Vec::new();
1062            for manifest_file in list.entries() {
1063                let manifest = table.manifest_reader().read(manifest_file).await.unwrap();
1064                assert_eq!(
1065                    manifest.metadata().partition_spec(),
1066                    table
1067                        .metadata()
1068                        .partition_spec_by_id(manifest_file.partition_spec_id)
1069                        .unwrap()
1070                        .as_ref()
1071                );
1072                let files = manifest
1073                    .entries()
1074                    .iter()
1075                    .map(|entry| entry.data_file().clone())
1076                    .collect::<Vec<_>>();
1077                files_by_spec.push((manifest_file.partition_spec_id, files));
1078            }
1079            let mut expected = vec![
1080                (0, vec![old_file.clone(), other_old_file]),
1081                (new_file.partition_spec_id, vec![new_file.clone()]),
1082            ];
1083            if let Some(mut row_id) = snapshot.first_row_id() {
1084                for (_, files) in &mut expected {
1085                    for file in files {
1086                        file.first_row_id = Some(row_id.try_into().unwrap());
1087                        row_id += file.record_count;
1088                    }
1089                }
1090            }
1091            assert_eq!(files_by_spec, expected);
1092
1093            let expected_summary = HashMap::from([
1094                ("added-data-files".to_string(), "3".to_string()),
1095                ("added-records".to_string(), "3".to_string()),
1096                ("added-files-size".to_string(), "300".to_string()),
1097                ("changed-partition-count".to_string(), "2".to_string()),
1098                ("total-data-files".to_string(), "3".to_string()),
1099                ("total-delete-files".to_string(), "0".to_string()),
1100                ("total-records".to_string(), "3".to_string()),
1101                ("total-files-size".to_string(), "300".to_string()),
1102                ("total-position-deletes".to_string(), "0".to_string()),
1103                ("total-equality-deletes".to_string(), "0".to_string()),
1104                (
1105                    "partitions.x=300".to_string(),
1106                    "added-data-files=2,added-files-size=200,added-records=2".to_string(),
1107                ),
1108                (
1109                    "partitions.x=300/y_bucket=3".to_string(),
1110                    "added-data-files=1,added-files-size=100,added-records=1".to_string(),
1111                ),
1112                (
1113                    "iceberg-version".to_string(),
1114                    format!("Apache Iceberg Rust {}", env!("CARGO_PKG_VERSION")),
1115                ),
1116            ]);
1117            let mut actual_summary = snapshot.summary().additional_properties.clone();
1118            for (key, value) in &mut actual_summary {
1119                if key.starts_with("partitions.") {
1120                    let mut metrics = value.split(',').collect::<Vec<_>>();
1121                    metrics.sort_unstable();
1122                    *value = metrics.join(",");
1123                }
1124            }
1125            assert_eq!(actual_summary, expected_summary);
1126        }
1127    }
1128
1129    #[tokio::test]
1130    async fn test_fast_append_validates_non_default_partition_type() {
1131        let table = with_bucket_partition_spec(make_v2_minimal_table());
1132        let data_file = DataFileBuilder::default()
1133            .content(DataContentType::Data)
1134            .file_path("test/invalid.parquet".to_string())
1135            .file_format(DataFileFormat::Parquet)
1136            .file_size_in_bytes(100)
1137            .record_count(1)
1138            .partition_spec_id(0)
1139            .partition(Struct::from_iter([
1140                Some(Literal::long(300)),
1141                Some(Literal::int(3)),
1142            ]))
1143            .build()
1144            .unwrap();
1145        let action = Transaction::new(&table)
1146            .fast_append()
1147            .add_data_files(vec![data_file]);
1148        let err = commit_with_fresh_state(action, &table)
1149            .await
1150            .err()
1151            .expect("partition must match the file's spec");
1152        assert_eq!(
1153            (err.kind(), err.to_string()),
1154            (
1155                ErrorKind::DataInvalid,
1156                "DataInvalid, context: { file_path: test/invalid.parquet, partition_spec_id: 0 } => Partition value is not compatible with partition type".to_string()
1157            )
1158        );
1159    }
1160
1161    /// Two writers append concurrently. Writer A starts from S0, but writer B
1162    /// commits S1 first, so A's first attempt fails the `RefSnapshotIdMatch`
1163    /// requirement and A rebases onto S1 to commit S2.
1164    ///
1165    /// The producer state survives the retry: A's manifest is written once and
1166    /// reused, while parent, sequence number, manifest list and summary are
1167    /// recomputed against S1 (see #3418).
1168    #[tokio::test]
1169    async fn test_fast_append_retry_rebases_onto_concurrent_commit() {
1170        let (s1, s2) = rebase_onto_concurrent_commit(make_v2_minimal_table()).await;
1171        assert_eq!(s1.first_row_id(), None);
1172        assert_eq!(s2.first_row_id(), None);
1173    }
1174
1175    /// On a v3 table the retry must also move the row range: S2's first row id
1176    /// is S1's `next_row_id`, not S0's.
1177    #[tokio::test]
1178    async fn test_fast_append_retry_rebases_row_range_onto_concurrent_commit() {
1179        let (s1, s2) = rebase_onto_concurrent_commit(make_v3_minimal_table()).await;
1180
1181        // B's single-row file takes row 0 in S1, so A's file starts at row 1.
1182        assert_eq!(s1.row_range(), Some((0, 1)));
1183        assert_eq!(s2.row_range(), Some((1, 1)));
1184    }
1185
1186    /// Runs the concurrent-append rebase scenario on `s0` and returns the
1187    /// snapshots committed by writer B (S1) and writer A (S2).
1188    async fn rebase_onto_concurrent_commit(s0: Table) -> (SnapshotRef, SnapshotRef) {
1189        // `TableCommit::apply` derives the next metadata location from the
1190        // current one, so it needs a well-formed name.
1191        let s0 = with_retry_properties(s0, "3").with_metadata_location(
1192            "memory:///table/metadata/00000-9c12d441-03fe-4693-9a96-a0705ddf69c1.metadata.json"
1193                .to_string(),
1194        );
1195        let data_file = |path: &str| {
1196            DataFileBuilder::default()
1197                .content(DataContentType::Data)
1198                .file_path(path.to_string())
1199                .file_format(DataFileFormat::Parquet)
1200                .file_size_in_bytes(100)
1201                .record_count(1)
1202                .partition_spec_id(s0.metadata().default_partition_spec_id())
1203                .partition(Struct::from_iter([Some(Literal::long(300))]))
1204                .build()
1205                .unwrap()
1206        };
1207
1208        // Writer B commits S1 on top of S0.
1209        let mut b_commit = commit_with_fresh_state(
1210            FastAppendAction::new().add_data_files(vec![data_file("test/b.parquet")]),
1211            &s0,
1212        )
1213        .await
1214        .unwrap();
1215        let s1 = TableCommit::builder()
1216            .ident(s0.identifier().clone())
1217            .updates(b_commit.take_updates())
1218            .requirements(b_commit.take_requirements())
1219            .build()
1220            .apply(s0.clone())
1221            .unwrap();
1222        let s1_snapshot = s1.metadata().current_snapshot().unwrap().clone();
1223
1224        // The catalog serves S0 to A's first attempt and S1 afterwards; every
1225        // update is validated against S1, so the first attempt's
1226        // `RefSnapshotIdMatch { snapshot_id: None }` fails with a real conflict.
1227        let mut mock_catalog = MockCatalog::new();
1228        let loads = AtomicU32::new(0);
1229        let (load_s0, load_s1) = (s0.clone(), s1.clone());
1230        mock_catalog
1231            .expect_load_table()
1232            .times(2)
1233            .returning_st(move |_| {
1234                let table = if loads.fetch_add(1, Ordering::SeqCst) == 0 {
1235                    load_s0.clone()
1236                } else {
1237                    load_s1.clone()
1238                };
1239                Box::pin(async move { Ok(table) })
1240            });
1241        let apply_s1 = s1.clone();
1242        mock_catalog
1243            .expect_update_table()
1244            .times(2)
1245            .returning_st(move |commit| {
1246                let s1 = apply_s1.clone();
1247                Box::pin(async move { commit.apply(s1) })
1248            });
1249
1250        // Writer A starts from S0.
1251        let a_uuid = Uuid::now_v7();
1252        let tx = Transaction::new(&s0);
1253        let tx = tx
1254            .fast_append()
1255            .set_commit_uuid(a_uuid)
1256            .add_data_files(vec![data_file("test/a.parquet")])
1257            .apply(tx)
1258            .unwrap();
1259        let s2 = tx.commit(&mock_catalog).await.unwrap();
1260        let s2_snapshot = s2.metadata().current_snapshot().unwrap().clone();
1261
1262        // S2 is rebased onto S1, not S0.
1263        assert_eq!(
1264            s2_snapshot.parent_snapshot_id(),
1265            Some(s1_snapshot.snapshot_id())
1266        );
1267        assert_eq!(
1268            s2_snapshot.sequence_number(),
1269            s1_snapshot.sequence_number() + 1
1270        );
1271        // The committed manifest list is A's second attempt.
1272        assert!(
1273            s2_snapshot
1274                .manifest_list()
1275                .ends_with(&format!("-1-{a_uuid}.avro")),
1276            "unexpected manifest list path {}",
1277            s2_snapshot.manifest_list()
1278        );
1279
1280        // S2 carries B's manifest forward and adds A's. A's manifest is the one
1281        // written during the first attempt (counter 0), reused after rebase.
1282        let manifest_list = s2.manifest_list_reader(&s2_snapshot).load().await.unwrap();
1283        assert_eq!(manifest_list.entries().len(), 2);
1284        let b_manifest = manifest_list
1285            .entries()
1286            .iter()
1287            .find(|m| m.added_snapshot_id == s1_snapshot.snapshot_id())
1288            .expect("S2 should carry B's manifest forward");
1289        let a_manifest = manifest_list
1290            .entries()
1291            .iter()
1292            .find(|m| m.added_snapshot_id == s2_snapshot.snapshot_id())
1293            .expect("S2 should add A's manifest");
1294        assert!(
1295            a_manifest
1296                .manifest_path
1297                .ends_with(&format!("{a_uuid}-m0.avro")),
1298            "unexpected manifest path {}",
1299            a_manifest.manifest_path
1300        );
1301        assert_eq!(a_manifest.sequence_number, s2_snapshot.sequence_number());
1302        assert_eq!(b_manifest.sequence_number, s1_snapshot.sequence_number());
1303
1304        // Summary totals are computed against S1, not S0.
1305        let summary = &s2_snapshot.summary().additional_properties;
1306        assert_eq!(summary.get("added-data-files").unwrap(), "1");
1307        assert_eq!(summary.get("total-data-files").unwrap(), "2");
1308        assert_eq!(summary.get("total-records").unwrap(), "2");
1309
1310        (s1_snapshot, s2_snapshot)
1311    }
1312}