1use std::collections::{HashMap, HashSet};
19
20use async_trait::async_trait;
21use uuid::Uuid;
22
23use crate::error::{Result, invalid_data};
24use crate::spec::{DataContentType, DataFile};
25use crate::table::Table;
26use crate::transaction::action::CommitStatus;
27use crate::transaction::snapshot::simple::{AppendIntent, SimpleSnapshotProducer};
28use crate::transaction::snapshot::validate_files_not_live;
29use crate::transaction::{ActionCommit, TransactionAction};
30use crate::{Error, ErrorKind};
31
32#[derive(Clone)]
34pub struct FastAppendAction {
35 check_duplicate: bool,
36 commit_uuid: Option<Uuid>,
38 snapshot_properties: HashMap<String, String>,
39 added_data_files: Vec<DataFile>,
41 added_paths: HashSet<String>,
43}
44
45impl FastAppendAction {
46 pub(crate) fn new() -> Self {
47 Self {
48 check_duplicate: true,
49 commit_uuid: None,
50 snapshot_properties: HashMap::default(),
51 added_data_files: vec![],
52 added_paths: HashSet::new(),
53 }
54 }
55
56 pub fn with_check_duplicate(mut self, v: bool) -> Self {
58 self.check_duplicate = v;
59 self
60 }
61
62 pub fn add_data_files(mut self, data_files: impl IntoIterator<Item = DataFile>) -> Self {
68 for data_file in data_files {
69 if self.added_paths.insert(data_file.file_path.clone()) {
70 self.added_data_files.push(data_file);
71 }
72 }
73 self
74 }
75
76 pub fn set_commit_uuid(mut self, commit_uuid: Uuid) -> Self {
78 self.commit_uuid = Some(commit_uuid);
79 self
80 }
81
82 pub fn set_snapshot_properties(mut self, snapshot_properties: HashMap<String, String>) -> Self {
84 self.snapshot_properties = snapshot_properties;
85 self
86 }
87
88 fn validate(&self) -> Result<()> {
90 if self.added_data_files.is_empty() && self.snapshot_properties.is_empty() {
94 return Err(Error::new(
95 ErrorKind::PreconditionFailed,
96 "No added data files or added snapshot properties found when write a manifest file",
97 ));
98 }
99 if let Some(data_file) = self
100 .added_data_files
101 .iter()
102 .find(|f| f.content_type() != DataContentType::Data)
103 {
104 return Err(invalid_data!(
105 "Only data content type is allowed for fast append, got {:?} for {}",
106 data_file.content_type(),
107 data_file.file_path()
108 ));
109 }
110 Ok(())
111 }
112}
113
114#[async_trait]
115impl TransactionAction for FastAppendAction {
116 type State = SimpleSnapshotProducer;
117
118 fn new_state(&self) -> SimpleSnapshotProducer {
119 SimpleSnapshotProducer::new(AppendIntent {
120 commit_uuid: self.commit_uuid,
121 snapshot_properties: self.snapshot_properties.clone(),
122 data_files: self.added_data_files.clone(),
123 })
124 }
125
126 async fn commit(
127 &self,
128 state: &mut SimpleSnapshotProducer,
129 table: &Table,
130 ) -> Result<ActionCommit> {
131 self.validate()?;
132 if self.check_duplicate {
133 validate_files_not_live(table, &self.added_data_files).await?;
134 }
135 state.apply(table).await
136 }
137
138 async fn cleanup(self, _state: SimpleSnapshotProducer, _table: &Table, _status: CommitStatus) {
139 }
142}
143
144#[cfg(test)]
145mod tests {
146 use std::collections::HashMap;
147 use std::fs;
148 use std::sync::Arc;
149 use std::sync::atomic::{AtomicU32, Ordering};
150
151 use minijinja::{AutoEscape, Environment, Value, context};
152 use tempfile::TempDir;
153 use uuid::Uuid;
154
155 use crate::catalog::MockCatalog;
156 use crate::encryption::kms::MemoryKeyManagementClient;
157 use crate::encryption::{SensitiveBytes, StandardKeyMetadata};
158 use crate::io::FileIO;
159 use crate::spec::{
160 DataContentType, DataFile, DataFileBuilder, DataFileFormat, Literal, MAIN_BRANCH,
161 ManifestEntry, ManifestListWriter, ManifestStatus, ManifestWriterBuilder, SnapshotRef,
162 Struct, TableMetadata, TableProperties,
163 };
164 use crate::table::Table;
165 use crate::test_utils::{commit_with_fresh_state, make_encrypted_table, test_runtime};
166 use crate::transaction::tests::{
167 make_v1_table, make_v2_minimal_table, make_v3_minimal_table, with_bucket_partition_spec,
168 with_retry_properties,
169 };
170 use crate::transaction::{ApplyTransactionAction, FastAppendAction, Transaction};
171 use crate::{ErrorKind, TableCommit, TableIdent, TableRequirement, TableUpdate};
172
173 fn render_template(template: &str, ctx: Value) -> String {
174 let mut env = Environment::new();
175 env.set_auto_escape_callback(|_| AutoEscape::None);
176 env.render_str(template, ctx).unwrap()
177 }
178
179 async fn make_table_with_delete_only_manifest() -> (Table, TempDir, String) {
186 let tmp_dir = TempDir::new().unwrap();
187 let table_location = tmp_dir.path().join("table1");
188 let manifest_list_location = table_location.join("metadata/manifests_list_1.avro");
189 let table_metadata_location = table_location.join("metadata/v1.json");
190
191 let file_io = FileIO::new_with_fs();
192
193 let template = fs::read_to_string(format!(
194 "{}/testdata/example_table_metadata_v2.json",
195 env!("CARGO_MANIFEST_DIR")
196 ))
197 .unwrap();
198 let metadata_json = render_template(&template, context! {
200 table_location => &table_location,
201 manifest_list_1_location => &manifest_list_location,
202 manifest_list_2_location => &manifest_list_location,
203 table_metadata_1_location => &table_metadata_location,
204 });
205 let table_metadata = serde_json::from_str::<TableMetadata>(&metadata_json).unwrap();
206
207 let table = Table::builder()
208 .metadata(table_metadata)
209 .identifier(TableIdent::from_strs(["db", "table1"]).unwrap())
210 .file_io(file_io)
211 .metadata_location(table_metadata_location.to_str().unwrap())
212 .runtime(test_runtime())
213 .build()
214 .unwrap();
215
216 let current_snapshot = table.metadata().current_snapshot().unwrap();
217 let schema = current_snapshot.schema(table.metadata()).unwrap();
218 let partition_spec = table.metadata().default_partition_spec();
219
220 let next_manifest_file = |location: &str| {
221 table
222 .file_io()
223 .new_output(format!(
224 "{}/metadata/manifest_{}.avro",
225 location,
226 Uuid::new_v4()
227 ))
228 .unwrap()
229 };
230 let table_location_str = table_location.to_str().unwrap().to_string();
231
232 let mut data_writer = ManifestWriterBuilder::new(
234 next_manifest_file(&table_location_str),
235 Some(current_snapshot.snapshot_id()),
236 schema.clone(),
237 partition_spec.as_ref().clone(),
238 )
239 .build_v2_data();
240 data_writer
241 .add_entry(
242 ManifestEntry::builder()
243 .status(ManifestStatus::Added)
244 .data_file(
245 DataFileBuilder::default()
246 .partition_spec_id(0)
247 .content(DataContentType::Data)
248 .file_path(format!("{table_location_str}/data.parquet"))
249 .file_format(DataFileFormat::Parquet)
250 .file_size_in_bytes(100)
251 .record_count(1)
252 .partition(Struct::from_iter([Some(Literal::long(100))]))
253 .build()
254 .unwrap(),
255 )
256 .build(),
257 )
258 .unwrap();
259 let data_manifest = data_writer.write_manifest_file().await.unwrap();
260
261 let mut delete_writer = ManifestWriterBuilder::new(
263 next_manifest_file(&table_location_str),
264 Some(current_snapshot.snapshot_id()),
265 schema.clone(),
266 partition_spec.as_ref().clone(),
267 )
268 .build_v2_data();
269 delete_writer
270 .add_delete_entry(
271 ManifestEntry::builder()
272 .status(ManifestStatus::Deleted)
273 .sequence_number(0)
274 .file_sequence_number(0)
275 .data_file(
276 DataFileBuilder::default()
277 .partition_spec_id(0)
278 .content(DataContentType::Data)
279 .file_path(format!("{table_location_str}/removed.parquet"))
280 .file_format(DataFileFormat::Parquet)
281 .file_size_in_bytes(100)
282 .record_count(1)
283 .partition(Struct::from_iter([Some(Literal::long(100))]))
284 .build()
285 .unwrap(),
286 )
287 .build(),
288 )
289 .unwrap();
290 let delete_manifest = delete_writer.write_manifest_file().await.unwrap();
291 let delete_manifest_path = delete_manifest.manifest_path.clone();
292
293 assert!(delete_manifest.has_deleted_files());
295 assert!(!delete_manifest.has_added_files());
296 assert!(!delete_manifest.has_existing_files());
297
298 let mut manifest_list_writer = ManifestListWriter::v2(
299 table
300 .file_io()
301 .new_output(current_snapshot.manifest_list())
302 .unwrap()
303 .writer()
304 .await
305 .unwrap(),
306 current_snapshot.snapshot_id(),
307 current_snapshot.parent_snapshot_id(),
308 current_snapshot.sequence_number(),
309 );
310 manifest_list_writer
311 .add_manifests(vec![data_manifest, delete_manifest].into_iter())
312 .unwrap();
313 manifest_list_writer.close().await.unwrap();
314
315 (table, tmp_dir, delete_manifest_path)
316 }
317
318 #[tokio::test]
322 async fn test_fast_append_preserves_delete_only_manifest() {
323 let (table, _tmp_dir, delete_manifest_path) = make_table_with_delete_only_manifest().await;
324
325 let new_file = DataFileBuilder::default()
327 .content(DataContentType::Data)
328 .file_path(format!("{}/appended.parquet", table.metadata().location()))
329 .file_format(DataFileFormat::Parquet)
330 .file_size_in_bytes(100)
331 .record_count(1)
332 .partition_spec_id(table.metadata().default_partition_spec_id())
333 .partition(Struct::from_iter([Some(Literal::long(100))]))
334 .build()
335 .unwrap();
336
337 let tx = Transaction::new(&table);
338 let action = tx.fast_append().add_data_files(vec![new_file]);
339 let mut action_commit = commit_with_fresh_state(action, &table).await.unwrap();
340 let updates = action_commit.take_updates();
341
342 let new_snapshot: SnapshotRef = if let TableUpdate::AddSnapshot { snapshot } = &updates[0] {
343 SnapshotRef::new(snapshot.clone())
344 } else {
345 unreachable!("first update of a fast append should be AddSnapshot")
346 };
347
348 let manifest_list = table
349 .manifest_list_reader(&new_snapshot)
350 .load()
351 .await
352 .unwrap();
353
354 assert!(
355 manifest_list
356 .entries()
357 .iter()
358 .any(|m| m.manifest_path == delete_manifest_path),
359 "delete-only manifest {delete_manifest_path} was dropped from the new snapshot's \
360 manifest list; the files it removed would reappear as live data"
361 );
362 }
363
364 async fn committed_data_files(table: &Table, updates: &[TableUpdate]) -> Vec<DataFile> {
366 let TableUpdate::AddSnapshot { snapshot } = &updates[0] else {
367 unreachable!("first update is always AddSnapshot")
368 };
369 let manifest_list = table
370 .manifest_list_reader(&SnapshotRef::new(snapshot.clone()))
371 .load()
372 .await
373 .unwrap();
374 assert_eq!(1, manifest_list.entries().len());
375 table
376 .manifest_reader()
377 .read(&manifest_list.entries()[0])
378 .await
379 .unwrap()
380 .entries()
381 .iter()
382 .map(|entry| entry.data_file().clone())
383 .collect()
384 }
385
386 #[tokio::test]
387 async fn test_fast_append_writes_encrypted_manifest() {
388 let table = make_encrypted_table().await;
389 assert!(
390 table.encryption_manager().is_some(),
391 "fixture table should have an EncryptionManager"
392 );
393
394 let new_file = DataFileBuilder::default()
395 .content(DataContentType::Data)
396 .file_path("memory:///table/data/00000.parquet".to_string())
397 .file_format(DataFileFormat::Parquet)
398 .partition(Struct::empty())
399 .record_count(100)
400 .file_size_in_bytes(4096)
401 .partition_spec_id(table.metadata().default_partition_spec_id())
402 .build()
403 .unwrap();
404
405 let tx = Transaction::new(&table);
406 let action = tx.fast_append().add_data_files(vec![new_file]);
407 let mut action_commit = commit_with_fresh_state(action, &table).await.unwrap();
408 let updates = action_commit.take_updates();
409
410 let new_snapshot: SnapshotRef = updates
411 .iter()
412 .find_map(|u| match u {
413 TableUpdate::AddSnapshot { snapshot } => Some(SnapshotRef::new(snapshot.clone())),
414 _ => None,
415 })
416 .expect("a fast append should emit an AddSnapshot update");
417
418 let manifest_list_key_metadata = table
419 .encryption_manager()
420 .unwrap()
421 .decrypt_manifest_list_key_metadata(new_snapshot.encryption_key_id().unwrap())
422 .await
423 .unwrap();
424 let manifest_list_size = table
425 .file_io()
426 .new_input(new_snapshot.manifest_list())
427 .unwrap()
428 .metadata()
429 .await
430 .unwrap()
431 .size;
432 assert_eq!(
433 manifest_list_key_metadata.file_length(),
434 Some(manifest_list_size)
435 );
436
437 let manifest_list = table
438 .manifest_list_reader(&new_snapshot)
439 .load()
440 .await
441 .unwrap();
442 let manifest_file = manifest_list
443 .entries()
444 .iter()
445 .find(|m| m.added_files_count.unwrap_or(0) > 0)
446 .expect("new snapshot should carry the appended data manifest");
447
448 let key_metadata_bytes = manifest_file
450 .key_metadata
451 .as_ref()
452 .expect("encrypted manifest must record key metadata");
453 let key_metadata = StandardKeyMetadata::decode(key_metadata_bytes)
454 .expect("recorded key metadata must decode as StandardKeyMetadata");
455 let manifest_size = table
456 .file_io()
457 .new_input(&manifest_file.manifest_path)
458 .unwrap()
459 .metadata()
460 .await
461 .unwrap()
462 .size;
463 assert_eq!(key_metadata.file_length(), Some(manifest_size));
464 assert_eq!(manifest_file.manifest_length, manifest_size as i64);
465
466 let manifest = table.manifest_reader().read(manifest_file).await.unwrap();
471 assert_eq!(manifest.entries().len(), 1);
472 assert_eq!(
473 manifest.entries()[0].data_file().file_path(),
474 "memory:///table/data/00000.parquet"
475 );
476 }
477
478 #[tokio::test]
479 async fn test_fast_append_rejects_delete_files() {
480 let table = make_v2_minimal_table();
481 let delete_file = DataFileBuilder::default()
482 .content(DataContentType::EqualityDeletes)
483 .file_path("test/1-deletes.parquet".to_string())
484 .file_format(DataFileFormat::Parquet)
485 .file_size_in_bytes(100)
486 .record_count(1)
487 .partition_spec_id(table.metadata().default_partition_spec_id())
488 .partition(Struct::from_iter([Some(Literal::long(300))]))
489 .equality_ids(Some(vec![1]))
490 .build()
491 .unwrap();
492
493 let tx = Transaction::new(&table);
494 let action = tx.fast_append().add_data_files(vec![delete_file]);
495 let err = commit_with_fresh_state(action, &table)
496 .await
497 .err()
498 .expect("delete files must be rejected");
499 assert_eq!(err.kind(), ErrorKind::DataInvalid);
500 }
501
502 #[tokio::test]
503 async fn test_empty_data_append_action() {
504 let table = make_v2_minimal_table();
505 let tx = Transaction::new(&table);
506 let action = tx.fast_append().add_data_files(vec![]);
507 assert!(commit_with_fresh_state(action, &table).await.is_err());
508 }
509
510 #[tokio::test]
514 async fn test_fast_append_honors_write_metadata_path() {
515 let base = make_v2_minimal_table();
516 let metadata_root = format!("{}/custom-meta", base.metadata().location());
517 let metadata = base
518 .metadata()
519 .clone()
520 .into_builder(None)
521 .set_properties(HashMap::from([(
522 "write.metadata.path".to_string(),
523 metadata_root.clone(),
524 )]))
525 .unwrap()
526 .build()
527 .unwrap()
528 .metadata;
529 let table = base.with_metadata(Arc::new(metadata));
530
531 let data_file = DataFileBuilder::default()
532 .content(DataContentType::Data)
533 .file_path(format!("{}/data/1.parquet", table.metadata().location()))
534 .file_format(DataFileFormat::Parquet)
535 .file_size_in_bytes(100)
536 .record_count(1)
537 .partition_spec_id(table.metadata().default_partition_spec_id())
538 .partition(Struct::from_iter([Some(Literal::long(300))]))
539 .build()
540 .unwrap();
541
542 let tx = Transaction::new(&table);
543 let action = tx.fast_append().add_data_files(vec![data_file]);
544 let mut action_commit = commit_with_fresh_state(action, &table).await.unwrap();
545 let updates = action_commit.take_updates();
546
547 let new_snapshot: SnapshotRef = if let TableUpdate::AddSnapshot { snapshot } = &updates[0] {
548 SnapshotRef::new(snapshot.clone())
549 } else {
550 unreachable!("first update of a fast append should be AddSnapshot")
551 };
552
553 let prefix = format!("{metadata_root}/");
554
555 assert!(
557 new_snapshot.manifest_list().starts_with(prefix.as_str()),
558 "manifest list {} not under configured write.metadata.path {metadata_root}",
559 new_snapshot.manifest_list()
560 );
561
562 let manifest_list = table
564 .manifest_list_reader(&new_snapshot)
565 .load()
566 .await
567 .unwrap();
568 assert!(
569 !manifest_list.entries().is_empty(),
570 "expected at least one manifest entry"
571 );
572 for entry in manifest_list.entries() {
573 assert!(
574 entry.manifest_path.starts_with(prefix.as_str()),
575 "manifest {} not under configured write.metadata.path {metadata_root}",
576 entry.manifest_path
577 );
578 }
579 }
580
581 #[tokio::test]
582 async fn test_set_snapshot_properties() {
583 let table = make_v2_minimal_table();
584 let tx = Transaction::new(&table);
585
586 let mut snapshot_properties = HashMap::new();
587 snapshot_properties.insert("key".to_string(), "val".to_string());
588
589 let data_file = DataFileBuilder::default()
590 .content(DataContentType::Data)
591 .file_path("test/1.parquet".to_string())
592 .file_format(DataFileFormat::Parquet)
593 .file_size_in_bytes(100)
594 .record_count(1)
595 .partition_spec_id(table.metadata().default_partition_spec_id())
596 .partition(Struct::from_iter([Some(Literal::long(300))]))
597 .build()
598 .unwrap();
599
600 let action = tx
601 .fast_append()
602 .set_snapshot_properties(snapshot_properties)
603 .add_data_files(vec![data_file]);
604 let mut action_commit = commit_with_fresh_state(action, &table).await.unwrap();
605 let updates = action_commit.take_updates();
606
607 let new_snapshot = if let TableUpdate::AddSnapshot { snapshot } = &updates[0] {
609 snapshot
610 } else {
611 unreachable!()
612 };
613 assert_eq!(
614 new_snapshot
615 .summary()
616 .additional_properties
617 .get("key")
618 .unwrap(),
619 "val"
620 );
621 }
622
623 const FIXTURE_MASTER_KEY_ID: &str = "master-1";
625 const FIXTURE_MASTER_KEY_BYTES: [u8; 16] = [
626 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e,
627 0x0f,
628 ];
629
630 async fn make_v3_encrypted_table() -> Table {
631 let json = fs::read_to_string(format!(
632 "{}/testdata/table_metadata/TableMetadataV3ValidEncryption.json",
633 env!("CARGO_MANIFEST_DIR")
634 ))
635 .unwrap();
636 let metadata = serde_json::from_str::<TableMetadata>(&json).unwrap();
637
638 let kms = MemoryKeyManagementClient::new();
639 kms.add_master_key_bytes(
640 FIXTURE_MASTER_KEY_ID,
641 SensitiveBytes::new(FIXTURE_MASTER_KEY_BYTES),
642 )
643 .unwrap();
644
645 let file_io = FileIO::new_with_memory();
646
647 let manifest_list_bytes = fs::read(format!(
648 "{}/testdata/manifests_lists/manifest-list-v3-encrypted.avro",
649 env!("CARGO_MANIFEST_DIR")
650 ))
651 .unwrap();
652 let parent_manifest_list = metadata.current_snapshot().unwrap().manifest_list();
653 file_io
654 .new_output(parent_manifest_list)
655 .unwrap()
656 .write(manifest_list_bytes.into())
657 .await
658 .unwrap();
659
660 Table::builder()
661 .metadata(metadata)
662 .metadata_location("memory:///table/metadata/v1.json")
663 .identifier(TableIdent::from_strs(["ns1", "enc"]).unwrap())
664 .file_io(file_io)
665 .kms_client(Arc::new(kms))
666 .runtime(test_runtime())
667 .build()
668 .unwrap()
669 }
670
671 #[tokio::test]
672 async fn test_commit_with_encryption_adds_keys_and_records_snapshot_key_id() {
673 let table = make_v3_encrypted_table().await;
674
675 let data_file = DataFileBuilder::default()
676 .content(DataContentType::Data)
677 .file_path("test/1.parquet".to_string())
678 .file_format(DataFileFormat::Parquet)
679 .file_size_in_bytes(100)
680 .record_count(1)
681 .partition_spec_id(table.metadata().default_partition_spec_id())
682 .partition(Struct::empty())
683 .build()
684 .unwrap();
685
686 let tx = Transaction::new(&table);
687 let action = tx.fast_append().add_data_files(vec![data_file]);
688 let mut action_commit = commit_with_fresh_state(action, &table).await.unwrap();
689 let updates = action_commit.take_updates();
690
691 let added_key_ids: Vec<String> = updates
692 .iter()
693 .filter_map(|u| match u {
694 TableUpdate::AddEncryptionKey { encryption_key } => {
695 Some(encryption_key.key_id().to_string())
696 }
697 _ => None,
698 })
699 .collect();
700 assert!(!added_key_ids.is_empty(), "got {updates:?}");
701
702 let new_snapshot = updates
704 .iter()
705 .find_map(|u| match u {
706 TableUpdate::AddSnapshot { snapshot } => Some(snapshot),
707 _ => None,
708 })
709 .expect("commit should add a snapshot");
710
711 let snapshot_key_id = new_snapshot
712 .encryption_key_id()
713 .expect("encrypted snapshot should record its manifest-list key id");
714 assert!(
715 added_key_ids.iter().any(|id| id == snapshot_key_id),
716 "snapshot key id {snapshot_key_id} not in added keys {added_key_ids:?}"
717 );
718
719 let new_snapshot_ref: SnapshotRef = Arc::new(new_snapshot.clone());
720 let manifest_list = table
721 .manifest_list_reader(&new_snapshot_ref)
722 .load()
723 .await
724 .expect("newly written encrypted manifest list should decrypt and parse");
725 assert_eq!(
726 manifest_list.entries().len(),
727 1,
728 "append should record exactly the one new data manifest"
729 );
730 }
731
732 #[tokio::test]
733 async fn test_snapshot_properties_cannot_override_computed_metrics() {
734 let table = make_v2_minimal_table();
739 let tx = Transaction::new(&table);
740
741 let mut snapshot_properties = HashMap::new();
742 snapshot_properties.insert("added-data-files".to_string(), "9999".to_string());
745 snapshot_properties.insert("added-records".to_string(), "not-a-number".to_string());
746
747 let data_file = DataFileBuilder::default()
748 .content(DataContentType::Data)
749 .file_path("test/1.parquet".to_string())
750 .file_format(DataFileFormat::Parquet)
751 .file_size_in_bytes(100)
752 .record_count(1)
753 .partition_spec_id(table.metadata().default_partition_spec_id())
754 .partition(Struct::from_iter([Some(Literal::long(300))]))
755 .build()
756 .unwrap();
757
758 let action = tx
759 .fast_append()
760 .set_snapshot_properties(snapshot_properties)
761 .add_data_files(vec![data_file]);
762 let mut action_commit = commit_with_fresh_state(action, &table).await.unwrap();
764 let updates = action_commit.take_updates();
765
766 let new_snapshot = if let TableUpdate::AddSnapshot { snapshot } = &updates[0] {
767 snapshot
768 } else {
769 unreachable!()
770 };
771 let props = &new_snapshot.summary().additional_properties;
772
773 assert_eq!(
775 props.get("added-data-files").unwrap(),
776 "1",
777 "computed added-data-files must override the user-supplied value"
778 );
779 assert_eq!(
780 props.get("added-records").unwrap(),
781 "1",
782 "computed added-records must override the user-supplied non-integer value"
783 );
784 }
785
786 #[tokio::test]
787 async fn test_snapshot_summary_records_writer_identity() {
788 let table = make_v2_minimal_table();
789 let tx = Transaction::new(&table);
790
791 let snapshot_properties = HashMap::from([
792 ("key".to_string(), "val".to_string()),
793 ("iceberg-version".to_string(), "caller".to_string()),
794 ]);
795
796 let action = tx
797 .fast_append()
798 .set_snapshot_properties(snapshot_properties);
799 let mut action_commit = commit_with_fresh_state(action, &table).await.unwrap();
800 let updates = action_commit.take_updates();
801
802 let new_snapshot = if let TableUpdate::AddSnapshot { snapshot } = &updates[0] {
803 snapshot
804 } else {
805 unreachable!()
806 };
807 let props = &new_snapshot.summary().additional_properties;
808 let version = env!("CARGO_PKG_VERSION");
809
810 assert_eq!(
811 props.get("iceberg-version").unwrap(),
812 &format!("Apache Iceberg Rust {version}")
813 );
814 assert_eq!(props.get("key").unwrap(), "val");
815 }
816
817 #[tokio::test]
818 async fn test_append_snapshot_properties() {
819 let table = make_v2_minimal_table();
820 let tx = Transaction::new(&table);
821
822 let mut snapshot_properties = HashMap::new();
823 snapshot_properties.insert("key".to_string(), "val".to_string());
824
825 let action = tx
826 .fast_append()
827 .set_snapshot_properties(snapshot_properties);
828 let mut action_commit = commit_with_fresh_state(action, &table).await.unwrap();
829 let updates = action_commit.take_updates();
830
831 let new_snapshot = if let TableUpdate::AddSnapshot { snapshot } = &updates[0] {
833 snapshot
834 } else {
835 unreachable!()
836 };
837 assert_eq!(
838 new_snapshot
839 .summary()
840 .additional_properties
841 .get("key")
842 .unwrap(),
843 "val"
844 );
845 }
846
847 #[tokio::test]
848 async fn test_fast_append_file_with_incompatible_partition_value() {
849 let table = make_v2_minimal_table();
850 let tx = Transaction::new(&table);
851 let action = tx.fast_append();
852
853 let data_file = DataFileBuilder::default()
855 .content(DataContentType::Data)
856 .file_path("test/3.parquet".to_string())
857 .file_format(DataFileFormat::Parquet)
858 .file_size_in_bytes(100)
859 .record_count(1)
860 .partition_spec_id(table.metadata().default_partition_spec_id())
861 .partition(Struct::from_iter([Some(Literal::string("test"))]))
862 .build()
863 .unwrap();
864
865 let action = action.add_data_files(vec![data_file.clone()]);
866
867 assert!(commit_with_fresh_state(action, &table).await.is_err());
868 }
869
870 #[tokio::test]
871 async fn test_fast_append_dedupes_intra_batch_duplicate_paths() {
872 let table = make_v2_minimal_table();
873 let tx = Transaction::new(&table);
874
875 let make_file = |size: u64, records: u64| {
876 DataFileBuilder::default()
877 .content(DataContentType::Data)
878 .file_path("test/dup.parquet".to_string())
879 .file_format(DataFileFormat::Parquet)
880 .file_size_in_bytes(size)
881 .record_count(records)
882 .partition_spec_id(table.metadata().default_partition_spec_id())
883 .partition(Struct::from_iter([Some(Literal::long(1))]))
884 .build()
885 .unwrap()
886 };
887
888 let action = tx
891 .fast_append()
892 .add_data_files(vec![make_file(100, 10), make_file(200, 20)])
893 .add_data_files(vec![make_file(300, 30)]);
894 let mut action_commit = commit_with_fresh_state(action, &table).await.unwrap();
895 let files = committed_data_files(&table, &action_commit.take_updates()).await;
896 assert_eq!(1, files.len());
897 assert_eq!(100, files[0].file_size_in_bytes());
898 }
899
900 #[tokio::test]
901 async fn test_fast_append_dedupes_regardless_of_check_duplicate_flag() {
902 let table = make_v2_minimal_table();
903 let tx = Transaction::new(&table);
904
905 let make_file = || {
906 DataFileBuilder::default()
907 .content(DataContentType::Data)
908 .file_path("test/dup.parquet".to_string())
909 .file_format(DataFileFormat::Parquet)
910 .file_size_in_bytes(100)
911 .record_count(10)
912 .partition_spec_id(table.metadata().default_partition_spec_id())
913 .partition(Struct::from_iter([Some(Literal::long(1))]))
914 .build()
915 .unwrap()
916 };
917
918 let action = tx
920 .fast_append()
921 .with_check_duplicate(false)
922 .add_data_files(vec![make_file(), make_file()]);
923 let mut action_commit = commit_with_fresh_state(action, &table).await.unwrap();
924 let files = committed_data_files(&table, &action_commit.take_updates()).await;
925 assert_eq!(1, files.len());
926 }
927
928 #[tokio::test]
929 async fn test_fast_append() {
930 let table = make_v2_minimal_table();
931 let tx = Transaction::new(&table);
932 let action = tx.fast_append();
933
934 let data_file = DataFileBuilder::default()
935 .content(DataContentType::Data)
936 .file_path("test/3.parquet".to_string())
937 .file_format(DataFileFormat::Parquet)
938 .file_size_in_bytes(100)
939 .record_count(1)
940 .partition_spec_id(table.metadata().default_partition_spec_id())
941 .partition(Struct::from_iter([Some(Literal::long(300))]))
942 .build()
943 .unwrap();
944
945 let action = action.add_data_files(vec![data_file.clone()]);
946 let mut action_commit = commit_with_fresh_state(action, &table).await.unwrap();
947 let updates = action_commit.take_updates();
948 let requirements = action_commit.take_requirements();
949
950 assert!(
952 matches!((&updates[0],&updates[1]), (TableUpdate::AddSnapshot { snapshot },TableUpdate::SetSnapshotRef { reference,ref_name }) if snapshot.snapshot_id() == reference.snapshot_id && ref_name == MAIN_BRANCH)
953 );
954 assert_eq!(
955 vec![
956 TableRequirement::UuidMatch {
957 uuid: table.metadata().uuid()
958 },
959 TableRequirement::RefSnapshotIdMatch {
960 r#ref: MAIN_BRANCH.to_string(),
961 snapshot_id: table.metadata().current_snapshot_id
962 }
963 ],
964 requirements
965 );
966
967 let new_snapshot: SnapshotRef = if let TableUpdate::AddSnapshot { snapshot } = &updates[0] {
969 SnapshotRef::new(snapshot.clone())
970 } else {
971 unreachable!()
972 };
973 let manifest_list = table
974 .manifest_list_reader(&new_snapshot)
975 .load()
976 .await
977 .unwrap();
978 assert_eq!(1, manifest_list.entries().len());
979 assert_eq!(
980 manifest_list.entries()[0].sequence_number,
981 new_snapshot.sequence_number()
982 );
983
984 let manifest = table
986 .manifest_reader()
987 .read(&manifest_list.entries()[0])
988 .await
989 .unwrap();
990 assert_eq!(1, manifest.entries().len());
991 assert_eq!(
992 new_snapshot.sequence_number(),
993 manifest.entries()[0]
994 .sequence_number()
995 .expect("Inherit sequence number by load manifest")
996 );
997
998 assert_eq!(
999 new_snapshot.snapshot_id(),
1000 manifest.entries()[0].snapshot_id().unwrap()
1001 );
1002 assert_eq!(data_file, *manifest.entries()[0].data_file());
1003 }
1004
1005 #[tokio::test]
1006 async fn test_fast_append_groups_files_by_partition_spec() {
1007 for table in [
1008 make_v1_table(),
1009 make_v2_minimal_table(),
1010 make_v3_minimal_table(),
1011 ] {
1012 let metadata = table
1013 .metadata()
1014 .clone()
1015 .into_builder(None)
1016 .set_properties(HashMap::from([(
1017 TableProperties::PROPERTY_WRITE_PARTITION_SUMMARY_LIMIT.to_string(),
1018 "2".to_string(),
1019 )]))
1020 .unwrap()
1021 .build()
1022 .unwrap()
1023 .metadata;
1024 let table = table.with_metadata(Arc::new(metadata));
1025 let table = with_bucket_partition_spec(table).with_metadata_location(
1026 "s3://bucket/test/location/metadata/00000-9c12d441-03fe-4693-9a96-a0705ddf69c1.metadata.json".to_string(),
1027 );
1028 let old_file = DataFileBuilder::default()
1029 .content(DataContentType::Data)
1030 .file_path("test/old.parquet".to_string())
1031 .file_format(DataFileFormat::Parquet)
1032 .file_size_in_bytes(100)
1033 .record_count(1)
1034 .partition_spec_id(0)
1035 .partition(Struct::from_iter([Some(Literal::long(300))]))
1036 .build()
1037 .unwrap();
1038 let mut other_old_file = old_file.clone();
1039 other_old_file.file_path = "test/other-old.parquet".to_string();
1040 let mut new_file = old_file.clone();
1041 new_file.file_path = "test/new.parquet".to_string();
1042 new_file.partition_spec_id = table.metadata().default_partition_spec_id();
1043 new_file.partition =
1044 Struct::from_iter([Some(Literal::long(300)), Some(Literal::int(3))]);
1045
1046 let action = Transaction::new(&table).fast_append().add_data_files(vec![
1047 new_file.clone(),
1048 old_file.clone(),
1049 other_old_file.clone(),
1050 ]);
1051 let mut commit = commit_with_fresh_state(action, &table).await.unwrap();
1052 let table = TableCommit::builder()
1053 .ident(table.identifier().clone())
1054 .updates(commit.take_updates())
1055 .requirements(commit.take_requirements())
1056 .build()
1057 .apply(table)
1058 .unwrap();
1059 let snapshot = table.metadata().current_snapshot().unwrap();
1060 let list = table.manifest_list_reader(snapshot).load().await.unwrap();
1061 let mut files_by_spec = Vec::new();
1062 for manifest_file in list.entries() {
1063 let manifest = table.manifest_reader().read(manifest_file).await.unwrap();
1064 assert_eq!(
1065 manifest.metadata().partition_spec(),
1066 table
1067 .metadata()
1068 .partition_spec_by_id(manifest_file.partition_spec_id)
1069 .unwrap()
1070 .as_ref()
1071 );
1072 let files = manifest
1073 .entries()
1074 .iter()
1075 .map(|entry| entry.data_file().clone())
1076 .collect::<Vec<_>>();
1077 files_by_spec.push((manifest_file.partition_spec_id, files));
1078 }
1079 let mut expected = vec![
1080 (0, vec![old_file.clone(), other_old_file]),
1081 (new_file.partition_spec_id, vec![new_file.clone()]),
1082 ];
1083 if let Some(mut row_id) = snapshot.first_row_id() {
1084 for (_, files) in &mut expected {
1085 for file in files {
1086 file.first_row_id = Some(row_id.try_into().unwrap());
1087 row_id += file.record_count;
1088 }
1089 }
1090 }
1091 assert_eq!(files_by_spec, expected);
1092
1093 let expected_summary = HashMap::from([
1094 ("added-data-files".to_string(), "3".to_string()),
1095 ("added-records".to_string(), "3".to_string()),
1096 ("added-files-size".to_string(), "300".to_string()),
1097 ("changed-partition-count".to_string(), "2".to_string()),
1098 ("total-data-files".to_string(), "3".to_string()),
1099 ("total-delete-files".to_string(), "0".to_string()),
1100 ("total-records".to_string(), "3".to_string()),
1101 ("total-files-size".to_string(), "300".to_string()),
1102 ("total-position-deletes".to_string(), "0".to_string()),
1103 ("total-equality-deletes".to_string(), "0".to_string()),
1104 (
1105 "partitions.x=300".to_string(),
1106 "added-data-files=2,added-files-size=200,added-records=2".to_string(),
1107 ),
1108 (
1109 "partitions.x=300/y_bucket=3".to_string(),
1110 "added-data-files=1,added-files-size=100,added-records=1".to_string(),
1111 ),
1112 (
1113 "iceberg-version".to_string(),
1114 format!("Apache Iceberg Rust {}", env!("CARGO_PKG_VERSION")),
1115 ),
1116 ]);
1117 let mut actual_summary = snapshot.summary().additional_properties.clone();
1118 for (key, value) in &mut actual_summary {
1119 if key.starts_with("partitions.") {
1120 let mut metrics = value.split(',').collect::<Vec<_>>();
1121 metrics.sort_unstable();
1122 *value = metrics.join(",");
1123 }
1124 }
1125 assert_eq!(actual_summary, expected_summary);
1126 }
1127 }
1128
1129 #[tokio::test]
1130 async fn test_fast_append_validates_non_default_partition_type() {
1131 let table = with_bucket_partition_spec(make_v2_minimal_table());
1132 let data_file = DataFileBuilder::default()
1133 .content(DataContentType::Data)
1134 .file_path("test/invalid.parquet".to_string())
1135 .file_format(DataFileFormat::Parquet)
1136 .file_size_in_bytes(100)
1137 .record_count(1)
1138 .partition_spec_id(0)
1139 .partition(Struct::from_iter([
1140 Some(Literal::long(300)),
1141 Some(Literal::int(3)),
1142 ]))
1143 .build()
1144 .unwrap();
1145 let action = Transaction::new(&table)
1146 .fast_append()
1147 .add_data_files(vec![data_file]);
1148 let err = commit_with_fresh_state(action, &table)
1149 .await
1150 .err()
1151 .expect("partition must match the file's spec");
1152 assert_eq!(
1153 (err.kind(), err.to_string()),
1154 (
1155 ErrorKind::DataInvalid,
1156 "DataInvalid, context: { file_path: test/invalid.parquet, partition_spec_id: 0 } => Partition value is not compatible with partition type".to_string()
1157 )
1158 );
1159 }
1160
1161 #[tokio::test]
1169 async fn test_fast_append_retry_rebases_onto_concurrent_commit() {
1170 let (s1, s2) = rebase_onto_concurrent_commit(make_v2_minimal_table()).await;
1171 assert_eq!(s1.first_row_id(), None);
1172 assert_eq!(s2.first_row_id(), None);
1173 }
1174
1175 #[tokio::test]
1178 async fn test_fast_append_retry_rebases_row_range_onto_concurrent_commit() {
1179 let (s1, s2) = rebase_onto_concurrent_commit(make_v3_minimal_table()).await;
1180
1181 assert_eq!(s1.row_range(), Some((0, 1)));
1183 assert_eq!(s2.row_range(), Some((1, 1)));
1184 }
1185
1186 async fn rebase_onto_concurrent_commit(s0: Table) -> (SnapshotRef, SnapshotRef) {
1189 let s0 = with_retry_properties(s0, "3").with_metadata_location(
1192 "memory:///table/metadata/00000-9c12d441-03fe-4693-9a96-a0705ddf69c1.metadata.json"
1193 .to_string(),
1194 );
1195 let data_file = |path: &str| {
1196 DataFileBuilder::default()
1197 .content(DataContentType::Data)
1198 .file_path(path.to_string())
1199 .file_format(DataFileFormat::Parquet)
1200 .file_size_in_bytes(100)
1201 .record_count(1)
1202 .partition_spec_id(s0.metadata().default_partition_spec_id())
1203 .partition(Struct::from_iter([Some(Literal::long(300))]))
1204 .build()
1205 .unwrap()
1206 };
1207
1208 let mut b_commit = commit_with_fresh_state(
1210 FastAppendAction::new().add_data_files(vec![data_file("test/b.parquet")]),
1211 &s0,
1212 )
1213 .await
1214 .unwrap();
1215 let s1 = TableCommit::builder()
1216 .ident(s0.identifier().clone())
1217 .updates(b_commit.take_updates())
1218 .requirements(b_commit.take_requirements())
1219 .build()
1220 .apply(s0.clone())
1221 .unwrap();
1222 let s1_snapshot = s1.metadata().current_snapshot().unwrap().clone();
1223
1224 let mut mock_catalog = MockCatalog::new();
1228 let loads = AtomicU32::new(0);
1229 let (load_s0, load_s1) = (s0.clone(), s1.clone());
1230 mock_catalog
1231 .expect_load_table()
1232 .times(2)
1233 .returning_st(move |_| {
1234 let table = if loads.fetch_add(1, Ordering::SeqCst) == 0 {
1235 load_s0.clone()
1236 } else {
1237 load_s1.clone()
1238 };
1239 Box::pin(async move { Ok(table) })
1240 });
1241 let apply_s1 = s1.clone();
1242 mock_catalog
1243 .expect_update_table()
1244 .times(2)
1245 .returning_st(move |commit| {
1246 let s1 = apply_s1.clone();
1247 Box::pin(async move { commit.apply(s1) })
1248 });
1249
1250 let a_uuid = Uuid::now_v7();
1252 let tx = Transaction::new(&s0);
1253 let tx = tx
1254 .fast_append()
1255 .set_commit_uuid(a_uuid)
1256 .add_data_files(vec![data_file("test/a.parquet")])
1257 .apply(tx)
1258 .unwrap();
1259 let s2 = tx.commit(&mock_catalog).await.unwrap();
1260 let s2_snapshot = s2.metadata().current_snapshot().unwrap().clone();
1261
1262 assert_eq!(
1264 s2_snapshot.parent_snapshot_id(),
1265 Some(s1_snapshot.snapshot_id())
1266 );
1267 assert_eq!(
1268 s2_snapshot.sequence_number(),
1269 s1_snapshot.sequence_number() + 1
1270 );
1271 assert!(
1273 s2_snapshot
1274 .manifest_list()
1275 .ends_with(&format!("-1-{a_uuid}.avro")),
1276 "unexpected manifest list path {}",
1277 s2_snapshot.manifest_list()
1278 );
1279
1280 let manifest_list = s2.manifest_list_reader(&s2_snapshot).load().await.unwrap();
1283 assert_eq!(manifest_list.entries().len(), 2);
1284 let b_manifest = manifest_list
1285 .entries()
1286 .iter()
1287 .find(|m| m.added_snapshot_id == s1_snapshot.snapshot_id())
1288 .expect("S2 should carry B's manifest forward");
1289 let a_manifest = manifest_list
1290 .entries()
1291 .iter()
1292 .find(|m| m.added_snapshot_id == s2_snapshot.snapshot_id())
1293 .expect("S2 should add A's manifest");
1294 assert!(
1295 a_manifest
1296 .manifest_path
1297 .ends_with(&format!("{a_uuid}-m0.avro")),
1298 "unexpected manifest path {}",
1299 a_manifest.manifest_path
1300 );
1301 assert_eq!(a_manifest.sequence_number, s2_snapshot.sequence_number());
1302 assert_eq!(b_manifest.sequence_number, s1_snapshot.sequence_number());
1303
1304 let summary = &s2_snapshot.summary().additional_properties;
1306 assert_eq!(summary.get("added-data-files").unwrap(), "1");
1307 assert_eq!(summary.get("total-data-files").unwrap(), "2");
1308 assert_eq!(summary.get("total-records").unwrap(), "2");
1309
1310 (s1_snapshot, s2_snapshot)
1311 }
1312}