Skip to main content

iceberg/transaction/
mod.rs

1// Licensed to the Apache Software Foundation (ASF) under one
2// or more contributor license agreements.  See the NOTICE file
3// distributed with this work for additional information
4// regarding copyright ownership.  The ASF licenses this file
5// to you under the Apache License, Version 2.0 (the
6// "License"); you may not use this file except in compliance
7// with the License.  You may obtain a copy of the License at
8//
9//   http://www.apache.org/licenses/LICENSE-2.0
10//
11// Unless required by applicable law or agreed to in writing,
12// software distributed under the License is distributed on an
13// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14// KIND, either express or implied.  See the License for the
15// specific language governing permissions and limitations
16// under the License.
17
18//! This module contains transaction api.
19//!
20//! The transaction API enables changes to be made to an existing table.
21//!
22//! Note that this may also have side effects, such as producing new manifest
23//! files.
24//!
25//! Below is a basic example using the "fast-append" action:
26//!
27//! ```ignore
28//! use iceberg::transaction::{ApplyTransactionAction, Transaction};
29//! use iceberg::Catalog;
30//!
31//! // Create a transaction.
32//! let tx = Transaction::new(my_table);
33//!
34//! // Create a `FastAppendAction` which will not rewrite or append
35//! // to existing metadata. This will create a new manifest.
36//! let action = tx.fast_append().add_data_files(my_data_files);
37//!
38//! // Apply the fast-append action to the given transaction, returning
39//! // the newly updated `Transaction`.
40//! let tx = action.apply(tx).unwrap();
41//!
42//!
43//! // End the transaction by committing to an `iceberg::Catalog`
44//! // implementation. This will cause a table update to occur.
45//! let table = tx
46//!     .commit(&some_catalog_impl)
47//!     .await
48//!     .unwrap();
49//! ```
50
51/// The `ApplyTransactionAction` trait provides an `apply` method
52/// that allows users to apply a transaction action to a `Transaction`.
53mod action;
54
55pub use action::*;
56mod append;
57mod expire_snapshots;
58mod snapshot;
59mod sort_order;
60mod update_location;
61mod update_properties;
62mod update_schema;
63mod update_statistics;
64mod upgrade_format_version;
65
66use std::sync::Arc;
67use std::time::Duration;
68
69use backon::{BackoffBuilder, ExponentialBackoff, ExponentialBuilder, RetryableWithContext};
70pub use update_schema::AddColumn;
71
72use crate::error::Result;
73use crate::spec::TableProperties;
74use crate::table::Table;
75use crate::transaction::action::TransactionActionEntry;
76pub use crate::transaction::append::FastAppendAction;
77pub use crate::transaction::expire_snapshots::ExpireSnapshotsAction;
78pub use crate::transaction::sort_order::ReplaceSortOrderAction;
79pub use crate::transaction::update_location::UpdateLocationAction;
80pub use crate::transaction::update_properties::UpdatePropertiesAction;
81pub use crate::transaction::update_schema::UpdateSchemaAction;
82pub use crate::transaction::update_statistics::UpdateStatisticsAction;
83pub use crate::transaction::upgrade_format_version::UpgradeFormatVersionAction;
84use crate::{Catalog, Error, ErrorKind, TableCommit, TableRequirement, TableUpdate};
85
86/// Table transaction.
87pub struct Transaction {
88    table: Table,
89    actions: Vec<TransactionActionEntry>,
90}
91
92impl Transaction {
93    /// Creates a new transaction.
94    pub fn new(table: &Table) -> Self {
95        Self {
96            table: table.clone(),
97            actions: vec![],
98        }
99    }
100
101    fn update_table_metadata(table: Table, updates: &[TableUpdate]) -> Result<Table> {
102        let mut metadata_builder = table.metadata().clone().into_builder(None);
103        for update in updates {
104            metadata_builder = update.clone().apply(metadata_builder)?;
105        }
106
107        Ok(table.with_metadata(Arc::new(metadata_builder.build()?.metadata)))
108    }
109
110    /// Applies an [`ActionCommit`] to the given [`Table`], returning a new [`Table`] with updated metadata.
111    /// Also appends any derived [`TableUpdate`]s and [`TableRequirement`]s to the provided vectors.
112    fn apply(
113        table: Table,
114        mut action_commit: ActionCommit,
115        existing_updates: &mut Vec<TableUpdate>,
116        existing_requirements: &mut Vec<TableRequirement>,
117    ) -> Result<Table> {
118        let updates = action_commit.take_updates();
119        let requirements = action_commit.take_requirements();
120
121        for requirement in &requirements {
122            requirement.check(Some(table.metadata()))?;
123        }
124
125        let updated_table = Self::update_table_metadata(table, &updates)?;
126
127        existing_updates.extend(updates);
128        existing_requirements.extend(requirements);
129
130        Ok(updated_table)
131    }
132
133    /// Sets table to a new version.
134    pub fn upgrade_table_version(&self) -> UpgradeFormatVersionAction {
135        UpgradeFormatVersionAction::new()
136    }
137
138    /// Update table's property.
139    pub fn update_table_properties(&self) -> UpdatePropertiesAction {
140        UpdatePropertiesAction::new()
141    }
142
143    /// Creates an update schema action.
144    pub fn update_schema(&self) -> UpdateSchemaAction {
145        UpdateSchemaAction::new()
146    }
147
148    /// Creates a fast append action.
149    pub fn fast_append(&self) -> FastAppendAction {
150        FastAppendAction::new()
151    }
152
153    /// Creates replace sort order action.
154    pub fn replace_sort_order(&self) -> ReplaceSortOrderAction {
155        ReplaceSortOrderAction::new()
156    }
157
158    /// Set the location of table
159    pub fn update_location(&self) -> UpdateLocationAction {
160        UpdateLocationAction::new()
161    }
162
163    /// Update the statistics of table
164    pub fn update_statistics(&self) -> UpdateStatisticsAction {
165        UpdateStatisticsAction::new()
166    }
167
168    /// Expire snapshots from the table metadata.
169    pub fn expire_snapshots(&self) -> ExpireSnapshotsAction {
170        ExpireSnapshotsAction::new()
171    }
172
173    /// Commit transaction.
174    ///
175    /// Returns `ErrorKind::PreconditionFailed` if the table is read-only
176    /// (`Table::readonly()`), even when the transaction has no actions.
177    pub async fn commit(self, catalog: &dyn Catalog) -> Result<Table> {
178        if self.table.readonly() {
179            return Err(Error::new(
180                ErrorKind::PreconditionFailed,
181                "Cannot modify a read-only table",
182            )
183            .with_context("table", self.table.identifier().to_string()));
184        }
185
186        if self.actions.is_empty() {
187            // nothing to commit
188            return Ok(self.table);
189        }
190
191        let table_props = self.table.metadata().table_properties();
192
193        let backoff = Self::build_backoff(table_props)?;
194        let tx = self;
195
196        (|mut tx: Transaction| async {
197            let result = tx.do_commit(catalog).await;
198            (tx, result)
199        })
200        .retry(backoff)
201        .sleep(tokio::time::sleep)
202        .context(tx)
203        .when(|e| e.retryable())
204        .await
205        .1
206    }
207
208    fn build_backoff(props: TableProperties<'_>) -> Result<ExponentialBackoff> {
209        Ok(ExponentialBuilder::new()
210            .with_min_delay(Duration::from_millis(props.commit_min_retry_wait_ms()?))
211            .with_max_delay(Duration::from_millis(props.commit_max_retry_wait_ms()?))
212            .with_total_delay(Some(Duration::from_millis(
213                props.commit_total_retry_timeout_ms()?,
214            )))
215            .with_max_times(props.commit_num_retries()?)
216            .with_factor(2.0)
217            .with_jitter()
218            .build())
219    }
220
221    async fn do_commit(&mut self, catalog: &dyn Catalog) -> Result<Table> {
222        let refreshed = catalog.load_table(self.table.identifier()).await?;
223
224        if self.table.metadata() != refreshed.metadata()
225            || self.table.metadata_location() != refreshed.metadata_location()
226        {
227            // current base is stale, use refreshed as base and re-apply transaction actions
228            self.table = refreshed.clone();
229        }
230
231        let mut current_table = self.table.clone();
232        let mut existing_updates: Vec<TableUpdate> = vec![];
233        let mut existing_requirements: Vec<TableRequirement> = vec![];
234
235        for entry in self.actions.iter_mut() {
236            let action_commit = entry.commit(&current_table).await?;
237            // apply action commit to current_table
238            current_table = Self::apply(
239                current_table,
240                action_commit,
241                &mut existing_updates,
242                &mut existing_requirements,
243            )?;
244        }
245
246        let table_commit = TableCommit::builder()
247            .ident(self.table.identifier().to_owned())
248            .updates(existing_updates)
249            .requirements(existing_requirements)
250            .build();
251
252        catalog.update_table(table_commit).await
253    }
254}
255
256#[cfg(test)]
257mod tests {
258    use std::collections::HashMap;
259    use std::fs::File;
260    use std::io::BufReader;
261    use std::sync::Arc;
262    use std::sync::atomic::{AtomicU32, Ordering};
263
264    use crate::catalog::MockCatalog;
265    use crate::io::FileIO;
266    use crate::memory::tests::new_memory_catalog;
267    use crate::spec::{
268        DataContentType, DataFileBuilder, DataFileFormat, Literal, PartitionSpec, Struct,
269        TableMetadata, TableProperties, Transform,
270    };
271    use crate::table::{StaticTable, Table};
272    use crate::test_utils::{make_encrypted_table, test_runtime};
273    use crate::transaction::{ApplyTransactionAction, Transaction};
274    use crate::{Catalog, Error, ErrorKind, TableCreation, TableIdent};
275
276    pub fn make_v1_table() -> Table {
277        let file = File::open(format!(
278            "{}/testdata/table_metadata/{}",
279            env!("CARGO_MANIFEST_DIR"),
280            "TableMetadataV1Valid.json"
281        ))
282        .unwrap();
283        let reader = BufReader::new(file);
284        let resp = serde_json::from_reader::<_, TableMetadata>(reader).unwrap();
285
286        Table::builder()
287            .metadata(resp)
288            .metadata_location("s3://bucket/test/location/metadata/v1.json")
289            .identifier(TableIdent::from_strs(["ns1", "test1"]).unwrap())
290            .file_io(FileIO::new_with_memory())
291            .runtime(test_runtime())
292            .build()
293            .unwrap()
294    }
295
296    pub fn make_v2_table() -> Table {
297        let file = File::open(format!(
298            "{}/testdata/table_metadata/{}",
299            env!("CARGO_MANIFEST_DIR"),
300            "TableMetadataV2Valid.json"
301        ))
302        .unwrap();
303        let reader = BufReader::new(file);
304        let resp = serde_json::from_reader::<_, TableMetadata>(reader).unwrap();
305
306        Table::builder()
307            .metadata(resp)
308            .metadata_location("s3://bucket/test/location/metadata/v1.json")
309            .identifier(TableIdent::from_strs(["ns1", "test1"]).unwrap())
310            .file_io(FileIO::new_with_memory())
311            .runtime(test_runtime())
312            .build()
313            .unwrap()
314    }
315
316    pub fn make_v2_minimal_table() -> Table {
317        let file = File::open(format!(
318            "{}/testdata/table_metadata/{}",
319            env!("CARGO_MANIFEST_DIR"),
320            "TableMetadataV2ValidMinimal.json"
321        ))
322        .unwrap();
323        let reader = BufReader::new(file);
324        let resp = serde_json::from_reader::<_, TableMetadata>(reader).unwrap();
325
326        Table::builder()
327            .metadata(resp)
328            .metadata_location("s3://bucket/test/location/metadata/v1.json")
329            .identifier(TableIdent::from_strs(["ns1", "test1"]).unwrap())
330            .file_io(FileIO::new_with_memory())
331            .runtime(test_runtime())
332            .build()
333            .unwrap()
334    }
335
336    pub(crate) fn make_v3_minimal_table() -> Table {
337        let file = File::open(format!(
338            "{}/testdata/table_metadata/{}",
339            env!("CARGO_MANIFEST_DIR"),
340            "TableMetadataV3ValidMinimal.json"
341        ))
342        .unwrap();
343        let reader = BufReader::new(file);
344        let resp = serde_json::from_reader::<_, TableMetadata>(reader).unwrap();
345
346        Table::builder()
347            .metadata(resp)
348            .metadata_location("s3://bucket/test/location/metadata/v1.json")
349            .identifier(TableIdent::from_strs(["ns1", "test1"]).unwrap())
350            .file_io(FileIO::new_with_memory())
351            .runtime(test_runtime())
352            .build()
353            .unwrap()
354    }
355
356    pub(super) fn with_bucket_partition_spec(table: Table) -> Table {
357        let spec = PartitionSpec::builder(table.metadata().current_schema().clone())
358            .add_partition_field("x", "x", Transform::Identity)
359            .unwrap()
360            .add_partition_field("y", "y_bucket", Transform::Bucket(16))
361            .unwrap()
362            .build()
363            .unwrap()
364            .into_unbound();
365        let metadata = table
366            .metadata()
367            .clone()
368            .into_builder(None)
369            .add_default_partition_spec(spec)
370            .unwrap()
371            .build()
372            .unwrap()
373            .metadata;
374        table.with_metadata(Arc::new(metadata))
375    }
376
377    async fn make_v2_minimal_readonly_table() -> Table {
378        let table = make_v2_minimal_table();
379        StaticTable::from_metadata(
380            table.metadata().clone(),
381            table.identifier().clone(),
382            FileIO::new_with_memory(),
383        )
384        .await
385        .unwrap()
386        .into_table()
387    }
388
389    pub(crate) async fn make_v3_minimal_table_in_catalog(catalog: &impl Catalog) -> Table {
390        let table_ident =
391            TableIdent::from_strs([format!("ns1-{}", uuid::Uuid::new_v4()), "test1".to_string()])
392                .unwrap();
393
394        catalog
395            .create_namespace(table_ident.namespace(), HashMap::new())
396            .await
397            .unwrap();
398
399        let file = File::open(format!(
400            "{}/testdata/table_metadata/{}",
401            env!("CARGO_MANIFEST_DIR"),
402            "TableMetadataV3ValidMinimal.json"
403        ))
404        .unwrap();
405        let reader = BufReader::new(file);
406        let base_metadata = serde_json::from_reader::<_, TableMetadata>(reader).unwrap();
407
408        let table_creation = TableCreation::builder()
409            .schema((**base_metadata.current_schema()).clone())
410            .partition_spec((**base_metadata.default_partition_spec()).clone())
411            .sort_order((**base_metadata.default_sort_order()).clone())
412            .name(table_ident.name().to_string())
413            .format_version(crate::spec::FormatVersion::V3)
414            .build();
415
416        catalog
417            .create_table(table_ident.namespace(), table_creation)
418            .await
419            .unwrap()
420    }
421
422    /// Helper function to create a test table with retry properties
423    pub(super) fn setup_test_table(num_retries: &str) -> Table {
424        with_retry_properties(make_v2_table(), num_retries)
425    }
426
427    /// Sets fast commit retry properties on `table`.
428    pub(super) fn with_retry_properties(table: Table, num_retries: &str) -> Table {
429        // Set retry properties
430        let mut props = HashMap::new();
431        props.insert("commit.retry.min-wait-ms".to_string(), "10".to_string());
432        props.insert("commit.retry.max-wait-ms".to_string(), "100".to_string());
433        props.insert(
434            "commit.retry.total-timeout-ms".to_string(),
435            "1000".to_string(),
436        );
437        props.insert(
438            "commit.retry.num-retries".to_string(),
439            num_retries.to_string(),
440        );
441
442        // Update table properties
443        let metadata = table
444            .metadata()
445            .clone()
446            .into_builder(None)
447            .set_properties(props)
448            .unwrap()
449            .build()
450            .unwrap()
451            .metadata;
452
453        table.with_metadata(Arc::new(metadata))
454    }
455
456    /// Helper function to create a transaction with a simple update action
457    fn create_test_transaction(table: &Table) -> Transaction {
458        let tx = Transaction::new(table);
459        tx.update_table_properties()
460            .set("test.key".to_string(), "test.value".to_string())
461            .apply(tx)
462            .unwrap()
463    }
464
465    /// Helper function to set up a mock catalog with retryable errors
466    fn setup_mock_catalog_with_retryable_errors(
467        success_after_attempts: Option<u32>,
468        expected_calls: usize,
469    ) -> MockCatalog {
470        let mut mock_catalog = MockCatalog::new();
471
472        mock_catalog
473            .expect_load_table()
474            .returning_st(|_| Box::pin(async move { Ok(make_v2_table()) }));
475
476        let attempts = AtomicU32::new(0);
477        mock_catalog
478            .expect_update_table()
479            .times(expected_calls)
480            .returning_st(move |_| {
481                if let Some(success_after_attempts) = success_after_attempts {
482                    attempts.fetch_add(1, Ordering::SeqCst);
483                    if attempts.load(Ordering::SeqCst) <= success_after_attempts {
484                        Box::pin(async move {
485                            Err(
486                                Error::new(ErrorKind::CatalogCommitConflicts, "Commit conflict")
487                                    .with_retryable(true),
488                            )
489                        })
490                    } else {
491                        Box::pin(async move { Ok(make_v2_table()) })
492                    }
493                } else {
494                    // Always fail with retryable error
495                    Box::pin(async move {
496                        Err(
497                            Error::new(ErrorKind::CatalogCommitConflicts, "Commit conflict")
498                                .with_retryable(true),
499                        )
500                    })
501                }
502            });
503
504        mock_catalog
505    }
506
507    /// Helper function to set up a mock catalog with non-retryable error
508    fn setup_mock_catalog_with_non_retryable_error() -> MockCatalog {
509        let mut mock_catalog = MockCatalog::new();
510
511        mock_catalog
512            .expect_load_table()
513            .returning_st(|_| Box::pin(async move { Ok(make_v2_table()) }));
514
515        mock_catalog
516            .expect_update_table()
517            .times(1) // Should only be called once since error is not retryable
518            .returning_st(move |_| {
519                Box::pin(async move {
520                    Err(Error::new(ErrorKind::Unexpected, "Non-retryable error")
521                        .with_retryable(false))
522                })
523            });
524
525        mock_catalog
526    }
527
528    #[tokio::test]
529    async fn test_commit_retryable_error() {
530        // Create a test table with retry properties
531        let table = setup_test_table("3");
532
533        // Create a transaction with a simple update action
534        let tx = create_test_transaction(&table);
535
536        // Create a mock catalog that fails twice then succeeds
537        let mock_catalog = setup_mock_catalog_with_retryable_errors(Some(2), 3);
538
539        // Commit the transaction
540        let result = tx.commit(&mock_catalog).await;
541
542        // Verify the result
543        assert!(result.is_ok(), "Transaction should eventually succeed");
544    }
545
546    #[tokio::test]
547    async fn test_commit_non_retryable_error() {
548        // Create a test table with retry properties
549        let table = setup_test_table("3");
550
551        // Create a transaction with a simple update action
552        let tx = create_test_transaction(&table);
553
554        // Create a mock catalog that fails with non-retryable error
555        let mock_catalog = setup_mock_catalog_with_non_retryable_error();
556
557        // Commit the transaction
558        let result = tx.commit(&mock_catalog).await;
559
560        // Verify the result
561        assert!(result.is_err(), "Transaction should fail immediately");
562        if let Err(err) = result {
563            assert_eq!(err.kind(), ErrorKind::Unexpected);
564            assert_eq!(err.message(), "Non-retryable error");
565            assert!(!err.retryable(), "Error should not be retryable");
566        }
567    }
568
569    #[tokio::test]
570    async fn test_commit_max_retries_exceeded() {
571        // Create a test table with retry properties (only allow 2 retries)
572        let table = setup_test_table("2");
573
574        // Create a transaction with a simple update action
575        let tx = create_test_transaction(&table);
576
577        // Create a mock catalog that always fails with retryable error
578        let mock_catalog = setup_mock_catalog_with_retryable_errors(None, 3); // Initial attempt + 2 retries = 3 total attempts
579
580        // Commit the transaction
581        let result = tx.commit(&mock_catalog).await;
582
583        // Verify the result
584        assert!(result.is_err(), "Transaction should fail after max retries");
585        if let Err(err) = result {
586            assert_eq!(err.kind(), ErrorKind::CatalogCommitConflicts);
587            assert_eq!(err.message(), "Commit conflict");
588            assert!(err.retryable(), "Error should be retryable");
589        }
590    }
591
592    #[test]
593    fn test_commit_backoff_is_jittered() {
594        let table = setup_test_table("4");
595        let delays = || {
596            Transaction::build_backoff(table.metadata().table_properties())
597                .unwrap()
598                .collect::<Vec<_>>()
599        };
600
601        assert_ne!(delays(), delays());
602    }
603
604    #[tokio::test]
605    async fn test_transaction_snapshot_summary() {
606        let catalog = new_memory_catalog().await;
607        let table = make_v3_minimal_table_in_catalog(&catalog).await;
608
609        let mut file_seq = 0u32;
610        let mut append_file = |table: &Table, record_count: u64, file_size: u64| {
611            file_seq += 1;
612            let file = DataFileBuilder::default()
613                .content(DataContentType::Data)
614                .file_path(format!("test/{file_seq}.parquet"))
615                .file_format(DataFileFormat::Parquet)
616                .file_size_in_bytes(file_size)
617                .record_count(record_count)
618                .partition(Struct::from_iter([Some(Literal::long(1))]))
619                .partition_spec_id(0)
620                .build()
621                .unwrap();
622            let tx = Transaction::new(table);
623            tx.fast_append()
624                .add_data_files(vec![file])
625                .apply(tx)
626                .unwrap()
627        };
628
629        let table = append_file(&table, /*record_count=*/ 10, /*file_size=*/ 100)
630            .commit(&catalog)
631            .await
632            .unwrap();
633        let table = append_file(&table, /*record_count=*/ 20, /*file_size=*/ 200)
634            .commit(&catalog)
635            .await
636            .unwrap();
637
638        let summary = &table
639            .metadata()
640            .current_snapshot()
641            .unwrap()
642            .summary()
643            .additional_properties;
644
645        assert_eq!(summary.get("total-records").unwrap(), "30");
646        assert_eq!(summary.get("total-data-files").unwrap(), "2");
647        assert_eq!(summary.get("total-files-size").unwrap(), "300");
648    }
649
650    #[tokio::test]
651    async fn test_commit_to_encrypted_table() {
652        let table = make_encrypted_table().await.with_metadata_location(
653            "memory:///table/metadata/00000-9c12d441-03fe-4693-9a96-a0705ddf69c1.metadata.json"
654                .to_string(),
655        );
656        let refreshed_table = table.clone();
657        let update_table = table.clone();
658        let mut mock_catalog = MockCatalog::new();
659        mock_catalog
660            .expect_load_table()
661            .times(1)
662            .returning_st(move |_| {
663                let refreshed_table = refreshed_table.clone();
664                Box::pin(async move { Ok(refreshed_table) })
665            });
666        mock_catalog
667            .expect_update_table()
668            .times(1)
669            .returning_st(move |commit| {
670                let update_table = update_table.clone();
671                Box::pin(async move { commit.apply(update_table) })
672            });
673
674        let tx = Transaction::new(&table);
675        let tx = tx
676            .update_table_properties()
677            .set("test.key".to_string(), "test.value".to_string())
678            .apply(tx)
679            .unwrap();
680
681        let updated_table = tx.commit(&mock_catalog).await.unwrap();
682
683        assert_eq!(
684            updated_table
685                .metadata()
686                .properties()
687                .get(TableProperties::PROPERTY_ENCRYPTION_KEY_ID)
688                .map(String::as_str),
689            Some("master-1")
690        );
691        assert_eq!(
692            updated_table
693                .metadata()
694                .properties()
695                .get("test.key")
696                .map(String::as_str),
697            Some("test.value")
698        );
699        assert!(updated_table.encryption_manager().is_some());
700    }
701
702    #[tokio::test]
703    async fn test_commit_readonly_table_fails() {
704        let table = make_v2_minimal_readonly_table().await;
705        let catalog = MockCatalog::new();
706
707        // make sure even empty transactions fail
708        let err = Transaction::new(&table).commit(&catalog).await.unwrap_err();
709
710        assert_eq!(err.kind(), ErrorKind::PreconditionFailed);
711        assert_eq!(err.message(), "Cannot modify a read-only table");
712        assert!(!err.retryable(), "Error should not be retryable");
713    }
714
715    #[tokio::test]
716    async fn test_commit_readonly_table_with_actions_fails() {
717        let table = make_v2_minimal_readonly_table().await;
718        let catalog = MockCatalog::new();
719
720        let err = create_test_transaction(&table)
721            .commit(&catalog)
722            .await
723            .unwrap_err();
724
725        assert_eq!(err.kind(), ErrorKind::PreconditionFailed);
726    }
727
728    #[tokio::test]
729    async fn test_commit_empty_transaction_is_noop() {
730        let table = make_v2_minimal_table();
731        let catalog = MockCatalog::new();
732
733        let result = Transaction::new(&table).commit(&catalog).await.unwrap();
734
735        assert_eq!(result.metadata(), table.metadata());
736    }
737}
738
739#[cfg(test)]
740mod test_row_lineage {
741    use crate::memory::tests::new_memory_catalog;
742    use crate::spec::{
743        DataContentType, DataFile, DataFileBuilder, DataFileFormat, Literal, Struct,
744    };
745    use crate::transaction::tests::make_v3_minimal_table_in_catalog;
746    use crate::transaction::{ApplyTransactionAction, Transaction};
747
748    #[tokio::test]
749    async fn test_fast_append_with_row_lineage() {
750        // Helper function to create a data file with specified number of rows
751        fn file_with_rows(record_count: u64) -> DataFile {
752            DataFileBuilder::default()
753                .content(DataContentType::Data)
754                .file_path(format!("test/{record_count}.parquet"))
755                .file_format(DataFileFormat::Parquet)
756                .file_size_in_bytes(100)
757                .record_count(record_count)
758                .partition(Struct::from_iter([Some(Literal::long(0))]))
759                .partition_spec_id(0)
760                .build()
761                .unwrap()
762        }
763        let catalog = new_memory_catalog().await;
764
765        let table = make_v3_minimal_table_in_catalog(&catalog).await;
766
767        // Check initial state - next_row_id should be 0
768        assert_eq!(table.metadata().next_row_id(), 0);
769
770        // First fast append with 30 rows
771        let tx = Transaction::new(&table);
772        let data_file_30 = file_with_rows(30);
773        let action = tx.fast_append().add_data_files(vec![data_file_30]);
774        let tx = action.apply(tx).unwrap();
775        let table = tx.commit(&catalog).await.unwrap();
776
777        // Check snapshot and table state after first append
778        let snapshot = table.metadata().current_snapshot().unwrap();
779        assert_eq!(snapshot.first_row_id(), Some(0));
780        assert_eq!(table.metadata().next_row_id(), 30);
781
782        // Check written manifest for first_row_id
783        let snapshot = table.metadata().current_snapshot().unwrap();
784        let manifest_list = table.manifest_list_reader(snapshot).load().await.unwrap();
785
786        assert_eq!(manifest_list.entries().len(), 1);
787        let manifest_file = &manifest_list.entries()[0];
788        assert_eq!(manifest_file.first_row_id, Some(0));
789
790        // Second fast append with 17 and 11 rows
791        let tx = Transaction::new(&table);
792        let data_file_17 = file_with_rows(17);
793        let data_file_11 = file_with_rows(11);
794        let action = tx
795            .fast_append()
796            .add_data_files(vec![data_file_17, data_file_11]);
797        let tx = action.apply(tx).unwrap();
798        let table = tx.commit(&catalog).await.unwrap();
799
800        // Check snapshot and table state after second append
801        let snapshot = table.metadata().current_snapshot().unwrap();
802        assert_eq!(snapshot.first_row_id(), Some(30));
803        assert_eq!(table.metadata().next_row_id(), 30 + 17 + 11);
804
805        // Check written manifest for first_row_id
806        let manifest_list = table.manifest_list_reader(snapshot).load().await.unwrap();
807        assert_eq!(manifest_list.entries().len(), 2);
808        let manifest_file = &manifest_list.entries()[1];
809        assert_eq!(manifest_file.first_row_id, Some(30));
810    }
811}