1mod action;
54
55pub use action::*;
56mod append;
57mod expire_snapshots;
58mod snapshot;
59mod sort_order;
60mod update_location;
61mod update_properties;
62mod update_schema;
63mod update_statistics;
64mod upgrade_format_version;
65
66use std::sync::Arc;
67use std::time::Duration;
68
69use backon::{BackoffBuilder, ExponentialBackoff, ExponentialBuilder, RetryableWithContext};
70pub use update_schema::AddColumn;
71
72use crate::error::Result;
73use crate::spec::TableProperties;
74use crate::table::Table;
75use crate::transaction::action::TransactionActionEntry;
76pub use crate::transaction::append::FastAppendAction;
77pub use crate::transaction::expire_snapshots::ExpireSnapshotsAction;
78pub use crate::transaction::sort_order::ReplaceSortOrderAction;
79pub use crate::transaction::update_location::UpdateLocationAction;
80pub use crate::transaction::update_properties::UpdatePropertiesAction;
81pub use crate::transaction::update_schema::UpdateSchemaAction;
82pub use crate::transaction::update_statistics::UpdateStatisticsAction;
83pub use crate::transaction::upgrade_format_version::UpgradeFormatVersionAction;
84use crate::{Catalog, Error, ErrorKind, TableCommit, TableRequirement, TableUpdate};
85
86pub struct Transaction {
88 table: Table,
89 actions: Vec<TransactionActionEntry>,
90}
91
92impl Transaction {
93 pub fn new(table: &Table) -> Self {
95 Self {
96 table: table.clone(),
97 actions: vec![],
98 }
99 }
100
101 fn update_table_metadata(table: Table, updates: &[TableUpdate]) -> Result<Table> {
102 let mut metadata_builder = table.metadata().clone().into_builder(None);
103 for update in updates {
104 metadata_builder = update.clone().apply(metadata_builder)?;
105 }
106
107 Ok(table.with_metadata(Arc::new(metadata_builder.build()?.metadata)))
108 }
109
110 fn apply(
113 table: Table,
114 mut action_commit: ActionCommit,
115 existing_updates: &mut Vec<TableUpdate>,
116 existing_requirements: &mut Vec<TableRequirement>,
117 ) -> Result<Table> {
118 let updates = action_commit.take_updates();
119 let requirements = action_commit.take_requirements();
120
121 for requirement in &requirements {
122 requirement.check(Some(table.metadata()))?;
123 }
124
125 let updated_table = Self::update_table_metadata(table, &updates)?;
126
127 existing_updates.extend(updates);
128 existing_requirements.extend(requirements);
129
130 Ok(updated_table)
131 }
132
133 pub fn upgrade_table_version(&self) -> UpgradeFormatVersionAction {
135 UpgradeFormatVersionAction::new()
136 }
137
138 pub fn update_table_properties(&self) -> UpdatePropertiesAction {
140 UpdatePropertiesAction::new()
141 }
142
143 pub fn update_schema(&self) -> UpdateSchemaAction {
145 UpdateSchemaAction::new()
146 }
147
148 pub fn fast_append(&self) -> FastAppendAction {
150 FastAppendAction::new()
151 }
152
153 pub fn replace_sort_order(&self) -> ReplaceSortOrderAction {
155 ReplaceSortOrderAction::new()
156 }
157
158 pub fn update_location(&self) -> UpdateLocationAction {
160 UpdateLocationAction::new()
161 }
162
163 pub fn update_statistics(&self) -> UpdateStatisticsAction {
165 UpdateStatisticsAction::new()
166 }
167
168 pub fn expire_snapshots(&self) -> ExpireSnapshotsAction {
170 ExpireSnapshotsAction::new()
171 }
172
173 pub async fn commit(self, catalog: &dyn Catalog) -> Result<Table> {
178 if self.table.readonly() {
179 return Err(Error::new(
180 ErrorKind::PreconditionFailed,
181 "Cannot modify a read-only table",
182 )
183 .with_context("table", self.table.identifier().to_string()));
184 }
185
186 if self.actions.is_empty() {
187 return Ok(self.table);
189 }
190
191 let table_props = self.table.metadata().table_properties();
192
193 let backoff = Self::build_backoff(table_props)?;
194 let tx = self;
195
196 (|mut tx: Transaction| async {
197 let result = tx.do_commit(catalog).await;
198 (tx, result)
199 })
200 .retry(backoff)
201 .sleep(tokio::time::sleep)
202 .context(tx)
203 .when(|e| e.retryable())
204 .await
205 .1
206 }
207
208 fn build_backoff(props: TableProperties<'_>) -> Result<ExponentialBackoff> {
209 Ok(ExponentialBuilder::new()
210 .with_min_delay(Duration::from_millis(props.commit_min_retry_wait_ms()?))
211 .with_max_delay(Duration::from_millis(props.commit_max_retry_wait_ms()?))
212 .with_total_delay(Some(Duration::from_millis(
213 props.commit_total_retry_timeout_ms()?,
214 )))
215 .with_max_times(props.commit_num_retries()?)
216 .with_factor(2.0)
217 .with_jitter()
218 .build())
219 }
220
221 async fn do_commit(&mut self, catalog: &dyn Catalog) -> Result<Table> {
222 let refreshed = catalog.load_table(self.table.identifier()).await?;
223
224 if self.table.metadata() != refreshed.metadata()
225 || self.table.metadata_location() != refreshed.metadata_location()
226 {
227 self.table = refreshed.clone();
229 }
230
231 let mut current_table = self.table.clone();
232 let mut existing_updates: Vec<TableUpdate> = vec![];
233 let mut existing_requirements: Vec<TableRequirement> = vec![];
234
235 for entry in self.actions.iter_mut() {
236 let action_commit = entry.commit(¤t_table).await?;
237 current_table = Self::apply(
239 current_table,
240 action_commit,
241 &mut existing_updates,
242 &mut existing_requirements,
243 )?;
244 }
245
246 let table_commit = TableCommit::builder()
247 .ident(self.table.identifier().to_owned())
248 .updates(existing_updates)
249 .requirements(existing_requirements)
250 .build();
251
252 catalog.update_table(table_commit).await
253 }
254}
255
256#[cfg(test)]
257mod tests {
258 use std::collections::HashMap;
259 use std::fs::File;
260 use std::io::BufReader;
261 use std::sync::Arc;
262 use std::sync::atomic::{AtomicU32, Ordering};
263
264 use crate::catalog::MockCatalog;
265 use crate::io::FileIO;
266 use crate::memory::tests::new_memory_catalog;
267 use crate::spec::{
268 DataContentType, DataFileBuilder, DataFileFormat, Literal, PartitionSpec, Struct,
269 TableMetadata, TableProperties, Transform,
270 };
271 use crate::table::{StaticTable, Table};
272 use crate::test_utils::{make_encrypted_table, test_runtime};
273 use crate::transaction::{ApplyTransactionAction, Transaction};
274 use crate::{Catalog, Error, ErrorKind, TableCreation, TableIdent};
275
276 pub fn make_v1_table() -> Table {
277 let file = File::open(format!(
278 "{}/testdata/table_metadata/{}",
279 env!("CARGO_MANIFEST_DIR"),
280 "TableMetadataV1Valid.json"
281 ))
282 .unwrap();
283 let reader = BufReader::new(file);
284 let resp = serde_json::from_reader::<_, TableMetadata>(reader).unwrap();
285
286 Table::builder()
287 .metadata(resp)
288 .metadata_location("s3://bucket/test/location/metadata/v1.json")
289 .identifier(TableIdent::from_strs(["ns1", "test1"]).unwrap())
290 .file_io(FileIO::new_with_memory())
291 .runtime(test_runtime())
292 .build()
293 .unwrap()
294 }
295
296 pub fn make_v2_table() -> Table {
297 let file = File::open(format!(
298 "{}/testdata/table_metadata/{}",
299 env!("CARGO_MANIFEST_DIR"),
300 "TableMetadataV2Valid.json"
301 ))
302 .unwrap();
303 let reader = BufReader::new(file);
304 let resp = serde_json::from_reader::<_, TableMetadata>(reader).unwrap();
305
306 Table::builder()
307 .metadata(resp)
308 .metadata_location("s3://bucket/test/location/metadata/v1.json")
309 .identifier(TableIdent::from_strs(["ns1", "test1"]).unwrap())
310 .file_io(FileIO::new_with_memory())
311 .runtime(test_runtime())
312 .build()
313 .unwrap()
314 }
315
316 pub fn make_v2_minimal_table() -> Table {
317 let file = File::open(format!(
318 "{}/testdata/table_metadata/{}",
319 env!("CARGO_MANIFEST_DIR"),
320 "TableMetadataV2ValidMinimal.json"
321 ))
322 .unwrap();
323 let reader = BufReader::new(file);
324 let resp = serde_json::from_reader::<_, TableMetadata>(reader).unwrap();
325
326 Table::builder()
327 .metadata(resp)
328 .metadata_location("s3://bucket/test/location/metadata/v1.json")
329 .identifier(TableIdent::from_strs(["ns1", "test1"]).unwrap())
330 .file_io(FileIO::new_with_memory())
331 .runtime(test_runtime())
332 .build()
333 .unwrap()
334 }
335
336 pub(crate) fn make_v3_minimal_table() -> Table {
337 let file = File::open(format!(
338 "{}/testdata/table_metadata/{}",
339 env!("CARGO_MANIFEST_DIR"),
340 "TableMetadataV3ValidMinimal.json"
341 ))
342 .unwrap();
343 let reader = BufReader::new(file);
344 let resp = serde_json::from_reader::<_, TableMetadata>(reader).unwrap();
345
346 Table::builder()
347 .metadata(resp)
348 .metadata_location("s3://bucket/test/location/metadata/v1.json")
349 .identifier(TableIdent::from_strs(["ns1", "test1"]).unwrap())
350 .file_io(FileIO::new_with_memory())
351 .runtime(test_runtime())
352 .build()
353 .unwrap()
354 }
355
356 pub(super) fn with_bucket_partition_spec(table: Table) -> Table {
357 let spec = PartitionSpec::builder(table.metadata().current_schema().clone())
358 .add_partition_field("x", "x", Transform::Identity)
359 .unwrap()
360 .add_partition_field("y", "y_bucket", Transform::Bucket(16))
361 .unwrap()
362 .build()
363 .unwrap()
364 .into_unbound();
365 let metadata = table
366 .metadata()
367 .clone()
368 .into_builder(None)
369 .add_default_partition_spec(spec)
370 .unwrap()
371 .build()
372 .unwrap()
373 .metadata;
374 table.with_metadata(Arc::new(metadata))
375 }
376
377 async fn make_v2_minimal_readonly_table() -> Table {
378 let table = make_v2_minimal_table();
379 StaticTable::from_metadata(
380 table.metadata().clone(),
381 table.identifier().clone(),
382 FileIO::new_with_memory(),
383 )
384 .await
385 .unwrap()
386 .into_table()
387 }
388
389 pub(crate) async fn make_v3_minimal_table_in_catalog(catalog: &impl Catalog) -> Table {
390 let table_ident =
391 TableIdent::from_strs([format!("ns1-{}", uuid::Uuid::new_v4()), "test1".to_string()])
392 .unwrap();
393
394 catalog
395 .create_namespace(table_ident.namespace(), HashMap::new())
396 .await
397 .unwrap();
398
399 let file = File::open(format!(
400 "{}/testdata/table_metadata/{}",
401 env!("CARGO_MANIFEST_DIR"),
402 "TableMetadataV3ValidMinimal.json"
403 ))
404 .unwrap();
405 let reader = BufReader::new(file);
406 let base_metadata = serde_json::from_reader::<_, TableMetadata>(reader).unwrap();
407
408 let table_creation = TableCreation::builder()
409 .schema((**base_metadata.current_schema()).clone())
410 .partition_spec((**base_metadata.default_partition_spec()).clone())
411 .sort_order((**base_metadata.default_sort_order()).clone())
412 .name(table_ident.name().to_string())
413 .format_version(crate::spec::FormatVersion::V3)
414 .build();
415
416 catalog
417 .create_table(table_ident.namespace(), table_creation)
418 .await
419 .unwrap()
420 }
421
422 pub(super) fn setup_test_table(num_retries: &str) -> Table {
424 with_retry_properties(make_v2_table(), num_retries)
425 }
426
427 pub(super) fn with_retry_properties(table: Table, num_retries: &str) -> Table {
429 let mut props = HashMap::new();
431 props.insert("commit.retry.min-wait-ms".to_string(), "10".to_string());
432 props.insert("commit.retry.max-wait-ms".to_string(), "100".to_string());
433 props.insert(
434 "commit.retry.total-timeout-ms".to_string(),
435 "1000".to_string(),
436 );
437 props.insert(
438 "commit.retry.num-retries".to_string(),
439 num_retries.to_string(),
440 );
441
442 let metadata = table
444 .metadata()
445 .clone()
446 .into_builder(None)
447 .set_properties(props)
448 .unwrap()
449 .build()
450 .unwrap()
451 .metadata;
452
453 table.with_metadata(Arc::new(metadata))
454 }
455
456 fn create_test_transaction(table: &Table) -> Transaction {
458 let tx = Transaction::new(table);
459 tx.update_table_properties()
460 .set("test.key".to_string(), "test.value".to_string())
461 .apply(tx)
462 .unwrap()
463 }
464
465 fn setup_mock_catalog_with_retryable_errors(
467 success_after_attempts: Option<u32>,
468 expected_calls: usize,
469 ) -> MockCatalog {
470 let mut mock_catalog = MockCatalog::new();
471
472 mock_catalog
473 .expect_load_table()
474 .returning_st(|_| Box::pin(async move { Ok(make_v2_table()) }));
475
476 let attempts = AtomicU32::new(0);
477 mock_catalog
478 .expect_update_table()
479 .times(expected_calls)
480 .returning_st(move |_| {
481 if let Some(success_after_attempts) = success_after_attempts {
482 attempts.fetch_add(1, Ordering::SeqCst);
483 if attempts.load(Ordering::SeqCst) <= success_after_attempts {
484 Box::pin(async move {
485 Err(
486 Error::new(ErrorKind::CatalogCommitConflicts, "Commit conflict")
487 .with_retryable(true),
488 )
489 })
490 } else {
491 Box::pin(async move { Ok(make_v2_table()) })
492 }
493 } else {
494 Box::pin(async move {
496 Err(
497 Error::new(ErrorKind::CatalogCommitConflicts, "Commit conflict")
498 .with_retryable(true),
499 )
500 })
501 }
502 });
503
504 mock_catalog
505 }
506
507 fn setup_mock_catalog_with_non_retryable_error() -> MockCatalog {
509 let mut mock_catalog = MockCatalog::new();
510
511 mock_catalog
512 .expect_load_table()
513 .returning_st(|_| Box::pin(async move { Ok(make_v2_table()) }));
514
515 mock_catalog
516 .expect_update_table()
517 .times(1) .returning_st(move |_| {
519 Box::pin(async move {
520 Err(Error::new(ErrorKind::Unexpected, "Non-retryable error")
521 .with_retryable(false))
522 })
523 });
524
525 mock_catalog
526 }
527
528 #[tokio::test]
529 async fn test_commit_retryable_error() {
530 let table = setup_test_table("3");
532
533 let tx = create_test_transaction(&table);
535
536 let mock_catalog = setup_mock_catalog_with_retryable_errors(Some(2), 3);
538
539 let result = tx.commit(&mock_catalog).await;
541
542 assert!(result.is_ok(), "Transaction should eventually succeed");
544 }
545
546 #[tokio::test]
547 async fn test_commit_non_retryable_error() {
548 let table = setup_test_table("3");
550
551 let tx = create_test_transaction(&table);
553
554 let mock_catalog = setup_mock_catalog_with_non_retryable_error();
556
557 let result = tx.commit(&mock_catalog).await;
559
560 assert!(result.is_err(), "Transaction should fail immediately");
562 if let Err(err) = result {
563 assert_eq!(err.kind(), ErrorKind::Unexpected);
564 assert_eq!(err.message(), "Non-retryable error");
565 assert!(!err.retryable(), "Error should not be retryable");
566 }
567 }
568
569 #[tokio::test]
570 async fn test_commit_max_retries_exceeded() {
571 let table = setup_test_table("2");
573
574 let tx = create_test_transaction(&table);
576
577 let mock_catalog = setup_mock_catalog_with_retryable_errors(None, 3); let result = tx.commit(&mock_catalog).await;
582
583 assert!(result.is_err(), "Transaction should fail after max retries");
585 if let Err(err) = result {
586 assert_eq!(err.kind(), ErrorKind::CatalogCommitConflicts);
587 assert_eq!(err.message(), "Commit conflict");
588 assert!(err.retryable(), "Error should be retryable");
589 }
590 }
591
592 #[test]
593 fn test_commit_backoff_is_jittered() {
594 let table = setup_test_table("4");
595 let delays = || {
596 Transaction::build_backoff(table.metadata().table_properties())
597 .unwrap()
598 .collect::<Vec<_>>()
599 };
600
601 assert_ne!(delays(), delays());
602 }
603
604 #[tokio::test]
605 async fn test_transaction_snapshot_summary() {
606 let catalog = new_memory_catalog().await;
607 let table = make_v3_minimal_table_in_catalog(&catalog).await;
608
609 let mut file_seq = 0u32;
610 let mut append_file = |table: &Table, record_count: u64, file_size: u64| {
611 file_seq += 1;
612 let file = DataFileBuilder::default()
613 .content(DataContentType::Data)
614 .file_path(format!("test/{file_seq}.parquet"))
615 .file_format(DataFileFormat::Parquet)
616 .file_size_in_bytes(file_size)
617 .record_count(record_count)
618 .partition(Struct::from_iter([Some(Literal::long(1))]))
619 .partition_spec_id(0)
620 .build()
621 .unwrap();
622 let tx = Transaction::new(table);
623 tx.fast_append()
624 .add_data_files(vec![file])
625 .apply(tx)
626 .unwrap()
627 };
628
629 let table = append_file(&table, 10, 100)
630 .commit(&catalog)
631 .await
632 .unwrap();
633 let table = append_file(&table, 20, 200)
634 .commit(&catalog)
635 .await
636 .unwrap();
637
638 let summary = &table
639 .metadata()
640 .current_snapshot()
641 .unwrap()
642 .summary()
643 .additional_properties;
644
645 assert_eq!(summary.get("total-records").unwrap(), "30");
646 assert_eq!(summary.get("total-data-files").unwrap(), "2");
647 assert_eq!(summary.get("total-files-size").unwrap(), "300");
648 }
649
650 #[tokio::test]
651 async fn test_commit_to_encrypted_table() {
652 let table = make_encrypted_table().await.with_metadata_location(
653 "memory:///table/metadata/00000-9c12d441-03fe-4693-9a96-a0705ddf69c1.metadata.json"
654 .to_string(),
655 );
656 let refreshed_table = table.clone();
657 let update_table = table.clone();
658 let mut mock_catalog = MockCatalog::new();
659 mock_catalog
660 .expect_load_table()
661 .times(1)
662 .returning_st(move |_| {
663 let refreshed_table = refreshed_table.clone();
664 Box::pin(async move { Ok(refreshed_table) })
665 });
666 mock_catalog
667 .expect_update_table()
668 .times(1)
669 .returning_st(move |commit| {
670 let update_table = update_table.clone();
671 Box::pin(async move { commit.apply(update_table) })
672 });
673
674 let tx = Transaction::new(&table);
675 let tx = tx
676 .update_table_properties()
677 .set("test.key".to_string(), "test.value".to_string())
678 .apply(tx)
679 .unwrap();
680
681 let updated_table = tx.commit(&mock_catalog).await.unwrap();
682
683 assert_eq!(
684 updated_table
685 .metadata()
686 .properties()
687 .get(TableProperties::PROPERTY_ENCRYPTION_KEY_ID)
688 .map(String::as_str),
689 Some("master-1")
690 );
691 assert_eq!(
692 updated_table
693 .metadata()
694 .properties()
695 .get("test.key")
696 .map(String::as_str),
697 Some("test.value")
698 );
699 assert!(updated_table.encryption_manager().is_some());
700 }
701
702 #[tokio::test]
703 async fn test_commit_readonly_table_fails() {
704 let table = make_v2_minimal_readonly_table().await;
705 let catalog = MockCatalog::new();
706
707 let err = Transaction::new(&table).commit(&catalog).await.unwrap_err();
709
710 assert_eq!(err.kind(), ErrorKind::PreconditionFailed);
711 assert_eq!(err.message(), "Cannot modify a read-only table");
712 assert!(!err.retryable(), "Error should not be retryable");
713 }
714
715 #[tokio::test]
716 async fn test_commit_readonly_table_with_actions_fails() {
717 let table = make_v2_minimal_readonly_table().await;
718 let catalog = MockCatalog::new();
719
720 let err = create_test_transaction(&table)
721 .commit(&catalog)
722 .await
723 .unwrap_err();
724
725 assert_eq!(err.kind(), ErrorKind::PreconditionFailed);
726 }
727
728 #[tokio::test]
729 async fn test_commit_empty_transaction_is_noop() {
730 let table = make_v2_minimal_table();
731 let catalog = MockCatalog::new();
732
733 let result = Transaction::new(&table).commit(&catalog).await.unwrap();
734
735 assert_eq!(result.metadata(), table.metadata());
736 }
737}
738
739#[cfg(test)]
740mod test_row_lineage {
741 use crate::memory::tests::new_memory_catalog;
742 use crate::spec::{
743 DataContentType, DataFile, DataFileBuilder, DataFileFormat, Literal, Struct,
744 };
745 use crate::transaction::tests::make_v3_minimal_table_in_catalog;
746 use crate::transaction::{ApplyTransactionAction, Transaction};
747
748 #[tokio::test]
749 async fn test_fast_append_with_row_lineage() {
750 fn file_with_rows(record_count: u64) -> DataFile {
752 DataFileBuilder::default()
753 .content(DataContentType::Data)
754 .file_path(format!("test/{record_count}.parquet"))
755 .file_format(DataFileFormat::Parquet)
756 .file_size_in_bytes(100)
757 .record_count(record_count)
758 .partition(Struct::from_iter([Some(Literal::long(0))]))
759 .partition_spec_id(0)
760 .build()
761 .unwrap()
762 }
763 let catalog = new_memory_catalog().await;
764
765 let table = make_v3_minimal_table_in_catalog(&catalog).await;
766
767 assert_eq!(table.metadata().next_row_id(), 0);
769
770 let tx = Transaction::new(&table);
772 let data_file_30 = file_with_rows(30);
773 let action = tx.fast_append().add_data_files(vec![data_file_30]);
774 let tx = action.apply(tx).unwrap();
775 let table = tx.commit(&catalog).await.unwrap();
776
777 let snapshot = table.metadata().current_snapshot().unwrap();
779 assert_eq!(snapshot.first_row_id(), Some(0));
780 assert_eq!(table.metadata().next_row_id(), 30);
781
782 let snapshot = table.metadata().current_snapshot().unwrap();
784 let manifest_list = table.manifest_list_reader(snapshot).load().await.unwrap();
785
786 assert_eq!(manifest_list.entries().len(), 1);
787 let manifest_file = &manifest_list.entries()[0];
788 assert_eq!(manifest_file.first_row_id, Some(0));
789
790 let tx = Transaction::new(&table);
792 let data_file_17 = file_with_rows(17);
793 let data_file_11 = file_with_rows(11);
794 let action = tx
795 .fast_append()
796 .add_data_files(vec![data_file_17, data_file_11]);
797 let tx = action.apply(tx).unwrap();
798 let table = tx.commit(&catalog).await.unwrap();
799
800 let snapshot = table.metadata().current_snapshot().unwrap();
802 assert_eq!(snapshot.first_row_id(), Some(30));
803 assert_eq!(table.metadata().next_row_id(), 30 + 17 + 11);
804
805 let manifest_list = table.manifest_list_reader(snapshot).load().await.unwrap();
807 assert_eq!(manifest_list.entries().len(), 2);
808 let manifest_file = &manifest_list.entries()[1];
809 assert_eq!(manifest_file.first_row_id, Some(30));
810 }
811}