Skip to main content

iceberg/spec/
table_metadata_builder.rs

1// Licensed to the Apache Software Foundation (ASF) under one
2// or more contributor license agreements.  See the NOTICE file
3// distributed with this work for additional information
4// regarding copyright ownership.  The ASF licenses this file
5// to you under the Apache License, Version 2.0 (the
6// "License"); you may not use this file except in compliance
7// with the License.  You may obtain a copy of the License at
8//
9//   http://www.apache.org/licenses/LICENSE-2.0
10//
11// Unless required by applicable law or agreed to in writing,
12// software distributed under the License is distributed on an
13// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14// KIND, either express or implied.  See the License for the
15// specific language governing permissions and limitations
16// under the License.
17
18use std::collections::{HashMap, HashSet};
19use std::sync::Arc;
20
21use uuid::Uuid;
22
23use super::schema::assign_fresh_ids;
24use super::{
25    DEFAULT_PARTITION_SPEC_ID, DEFAULT_SCHEMA_ID, FormatVersion, MAIN_BRANCH, MetadataLog,
26    ONE_MINUTE_MS, PartitionField, PartitionSpec, PartitionSpecBuilder, PartitionStatisticsFile,
27    Schema, SchemaRef, Snapshot, SnapshotLog, SnapshotReference, SnapshotRetention, SortOrder,
28    SortOrderRef, StatisticsFile, StructType, TableMetadata, TableProperties, Transform,
29    UNPARTITIONED_LAST_ASSIGNED_ID, UnboundPartitionField, UnboundPartitionSpec,
30};
31use crate::error::{Error, ErrorKind, Result, invalid_data};
32use crate::spec::{EncryptedKey, INITIAL_ROW_ID, MIN_FORMAT_VERSION_ROW_LINEAGE};
33use crate::{TableCreation, TableUpdate};
34
35pub(crate) const FIRST_FIELD_ID: i32 = 1;
36
37/// Manipulating table metadata.
38///
39/// For this builder the order of called functions matters. Functions are applied in-order.
40/// All operations applied to the `TableMetadata` are tracked in `changes` as  a chronologically
41/// ordered vec of `TableUpdate`.
42/// If an operation does not lead to a change of the `TableMetadata`, the corresponding update
43/// is omitted from `changes`.
44///
45/// Unlike a typical builder pattern, the order of function calls matters.
46/// Some basic rules:
47/// - `add_schema` must be called before `set_current_schema`.
48/// - If a new partition spec and schema are added, the schema should be added first.
49#[derive(Debug, Clone)]
50pub struct TableMetadataBuilder {
51    metadata: TableMetadata,
52    changes: Vec<TableUpdate>,
53    last_added_schema_id: Option<i32>,
54    last_added_spec_id: Option<i32>,
55    last_added_order_id: Option<i64>,
56    // None if this is a new table (from_metadata) method not used
57    previous_history_entry: Option<MetadataLog>,
58    last_updated_ms: Option<i64>,
59}
60
61#[derive(Debug, Clone, PartialEq)]
62/// Result of modifying or creating a `TableMetadata`.
63pub struct TableMetadataBuildResult {
64    /// The new `TableMetadata`.
65    pub metadata: TableMetadata,
66    /// The changes that were applied to the metadata.
67    pub changes: Vec<TableUpdate>,
68    /// Expired metadata logs
69    pub expired_metadata_logs: Vec<MetadataLog>,
70}
71
72impl TableMetadataBuilder {
73    /// Proxy id for "last added" items, including schema, partition spec, sort order.
74    pub const LAST_ADDED: i32 = -1;
75
76    /// Create a `TableMetadata` object from scratch.
77    ///
78    /// This method re-assign ids of fields in the schema, schema.id, sort_order.id and
79    /// spec.id. It should only be used to create new table metadata from scratch.
80    pub fn new(
81        schema: Schema,
82        spec: impl Into<UnboundPartitionSpec>,
83        sort_order: SortOrder,
84        location: String,
85        format_version: FormatVersion,
86        properties: HashMap<String, String>,
87    ) -> Result<Self> {
88        // Re-assign field_ids, schema.id, sort_order.id and spec.id for a new table.
89        let (fresh_schema, fresh_spec, fresh_sort_order) =
90            Self::reassign_ids(schema, spec.into(), sort_order)?;
91        let schema_id = fresh_schema.schema_id();
92
93        let builder = Self {
94            metadata: TableMetadata {
95                format_version,
96                table_uuid: Uuid::now_v7(),
97                location: "".to_string(), // Overwritten immediately by set_location
98                last_sequence_number: 0,
99                last_updated_ms: 0,    // Overwritten by build() if not set before
100                last_column_id: -1,    // Overwritten immediately by add_current_schema
101                current_schema_id: -1, // Overwritten immediately by add_current_schema
102                schemas: HashMap::new(),
103                partition_specs: HashMap::new(),
104                default_spec: Arc::new(
105                    // The spec id (-1) is just a proxy value and can be any negative number.
106                    // 0 would lead to wrong changes in the builder if the provided spec by the user is
107                    // also unpartitioned.
108                    // The `default_spec` value is always replaced at the end of this method by he `add_default_partition_spec`
109                    // method.
110                    PartitionSpec::unpartition_spec().with_spec_id(-1),
111                ), // Overwritten immediately by add_default_partition_spec
112                default_partition_type: StructType::new(vec![]),
113                last_partition_id: UNPARTITIONED_LAST_ASSIGNED_ID,
114                properties: HashMap::new(),
115                current_snapshot_id: None,
116                snapshots: HashMap::new(),
117                snapshot_log: vec![],
118                sort_orders: HashMap::new(),
119                metadata_log: vec![],
120                default_sort_order_id: -1, // Overwritten immediately by add_default_sort_order
121                refs: HashMap::default(),
122                statistics: HashMap::new(),
123                partition_statistics: HashMap::new(),
124                encryption_keys: HashMap::new(),
125                next_row_id: INITIAL_ROW_ID,
126            },
127            last_updated_ms: None,
128            changes: vec![],
129            last_added_schema_id: Some(schema_id),
130            last_added_spec_id: None,
131            last_added_order_id: None,
132            previous_history_entry: None,
133        };
134
135        builder
136            .set_location(location)
137            .add_current_schema(fresh_schema)?
138            .add_default_partition_spec(fresh_spec.into_unbound())?
139            .add_default_sort_order(fresh_sort_order)?
140            .set_properties(properties)
141    }
142
143    /// Creates a new table metadata builder from the given metadata to modify it.
144    /// `current_file_location` is the location where the current version
145    /// of the metadata file is stored. This is used to update the metadata log.
146    /// If `current_file_location` is `None`, the metadata log will not be updated.
147    /// This should only be used to stage-create tables.
148    #[must_use]
149    pub fn new_from_metadata(
150        previous: TableMetadata,
151        current_file_location: Option<String>,
152    ) -> Self {
153        Self {
154            previous_history_entry: current_file_location.map(|l| MetadataLog {
155                metadata_file: l,
156                timestamp_ms: previous.last_updated_ms,
157            }),
158            metadata: previous,
159            changes: Vec::default(),
160            last_added_schema_id: None,
161            last_added_spec_id: None,
162            last_added_order_id: None,
163            last_updated_ms: None,
164        }
165    }
166
167    /// Creates a new table metadata builder from the given table creation.
168    pub fn from_table_creation(table_creation: TableCreation) -> Result<Self> {
169        let TableCreation {
170            name: _,
171            location,
172            schema,
173            partition_spec,
174            sort_order,
175            properties,
176            format_version,
177        } = table_creation;
178
179        let location =
180            location.ok_or_else(|| invalid_data!("Can't create table without location"))?;
181        let partition_spec = partition_spec.unwrap_or(UnboundPartitionSpec {
182            spec_id: None,
183            fields: vec![],
184        });
185
186        Self::new(
187            schema,
188            partition_spec,
189            sort_order.unwrap_or(SortOrder::unsorted_order()),
190            location,
191            format_version,
192            properties,
193        )
194    }
195
196    /// Replaces the table's schema, partition spec and sort order, leaving it with no
197    /// current snapshot. Backs `REPLACE TABLE`.
198    ///
199    /// The table keeps its uuid, properties and snapshots, so earlier snapshots stay readable.
200    /// Only the `main` branch is removed; other branches and tags are kept.
201    ///
202    /// Columns are matched to the current schema by full name and keep their field ids, even
203    /// when their type changes; new columns get ids above `last_column_id`, never a dropped
204    /// column's id.
205    ///
206    /// Partition fields reuse the highest id an equivalent field has in any of the table's specs,
207    /// and other fields get ids above `last_partition_id`.
208    ///
209    /// `location` moves the table, `None` leaves it in place. `format_version` upgrades the table,
210    /// `None` keeps its version. `properties` are applied on top of the existing ones, so an
211    /// existing `write.parquet.compression-codec` is kept.
212    ///
213    /// The caller adds the snapshot holding the new contents, if any, before building.
214    ///
215    /// # Errors
216    /// - The table is format version 1, which is not supported.
217    /// - `format_version` would downgrade the table.
218    /// - The partition spec or sort order names a column missing from `schema`.
219    /// - `schema` takes the name of a partition field in one of the table's specs without being
220    ///   that field's source column.
221    /// - `properties` contains a reserved property.
222    // TODO: Use for table replacement in https://github.com/apache/iceberg-rust/issues/3056.
223    #[allow(dead_code)]
224    pub(crate) fn build_replacement(
225        self,
226        schema: Schema,
227        spec: impl Into<UnboundPartitionSpec>,
228        sort_order: SortOrder,
229        location: Option<String>,
230        format_version: Option<FormatVersion>,
231        properties: HashMap<String, String>,
232    ) -> Result<Self> {
233        if self.metadata.format_version == FormatVersion::V1 {
234            return Err(Error::new(
235                ErrorKind::FeatureUnsupported,
236                "Table replacement is not supported for format version 1",
237            ));
238        }
239
240        let format_version = format_version.unwrap_or(self.metadata.format_version);
241        let spec = spec.into();
242
243        let previous_id_to_name = schema.field_id_to_name_map().clone();
244        let fresh_schema = assign_fresh_ids(
245            schema,
246            self.get_current_schema()?,
247            self.metadata.last_column_id,
248        )?;
249
250        let (fresh_spec, fresh_sort_order) =
251            Self::rebind_to_schema(&previous_id_to_name, &fresh_schema, spec, sort_order)?;
252        let fresh_spec = self.reassign_partition_field_ids(fresh_spec, &fresh_schema)?;
253
254        let mut builder = self.upgrade_format_version(format_version)?;
255        if builder.metadata.refs.contains_key(MAIN_BRANCH) {
256            builder = builder.remove_ref(MAIN_BRANCH);
257        }
258
259        let builder = builder
260            .add_current_schema(fresh_schema)?
261            .add_default_partition_spec(fresh_spec)?
262            .add_default_sort_order(fresh_sort_order)?;
263
264        let builder = match location {
265            Some(location) => builder.set_location(location),
266            None => builder,
267        };
268
269        builder.set_properties(properties)
270    }
271
272    /// Changes uuid of table metadata.
273    pub fn assign_uuid(mut self, uuid: Uuid) -> Self {
274        if self.metadata.table_uuid != uuid {
275            self.metadata.table_uuid = uuid;
276            self.changes.push(TableUpdate::AssignUuid { uuid });
277        }
278
279        self
280    }
281
282    /// Upgrade `FormatVersion`. Downgrades are not allowed.
283    ///
284    /// # Errors
285    /// - Cannot downgrade to older format versions.
286    pub fn upgrade_format_version(mut self, format_version: FormatVersion) -> Result<Self> {
287        if format_version < self.metadata.format_version {
288            return Err(invalid_data!(
289                "Cannot downgrade FormatVersion from {} to {}",
290                self.metadata.format_version,
291                format_version
292            ));
293        }
294
295        if format_version != self.metadata.format_version {
296            match format_version {
297                FormatVersion::V1 => {
298                    // No changes needed for V1
299                }
300                FormatVersion::V2 => {
301                    self.metadata.format_version = format_version;
302                    self.changes
303                        .push(TableUpdate::UpgradeFormatVersion { format_version });
304                }
305                FormatVersion::V3 => {
306                    self.metadata.format_version = format_version;
307                    // Set next-row-id to 0 when upgrading to v3 as per Iceberg spec
308                    self.metadata.next_row_id = INITIAL_ROW_ID;
309                    self.changes
310                        .push(TableUpdate::UpgradeFormatVersion { format_version });
311                }
312            }
313        }
314
315        Ok(self)
316    }
317
318    /// Set properties. If a property already exists, it will be overwritten.
319    ///
320    /// If a reserved property is set, the corresponding action is performed and the property is not persisted.
321    /// Currently the following reserved properties are supported:
322    /// * format-version: Set the format version of the table.
323    ///
324    /// # Errors
325    /// - If properties contains a reserved property
326    pub fn set_properties(mut self, properties: HashMap<String, String>) -> Result<Self> {
327        // List of specified properties that are RESERVED and should not be persisted.
328        let reserved_properties = properties
329            .keys()
330            .filter(|key| TableProperties::RESERVED_PROPERTIES.contains(&key.as_str()))
331            .map(ToString::to_string)
332            .collect::<Vec<_>>();
333
334        if !reserved_properties.is_empty() {
335            return Err(invalid_data!(
336                "Table properties should not contain reserved properties, but got: [{}]",
337                reserved_properties.join(", ")
338            ));
339        }
340
341        if properties.is_empty() {
342            return Ok(self);
343        }
344
345        self.metadata.properties.extend(properties.clone());
346        self.changes.push(TableUpdate::SetProperties {
347            updates: properties,
348        });
349
350        Ok(self)
351    }
352
353    /// Remove properties from the table metadata.
354    /// Does nothing if the key is not present.
355    ///
356    /// # Errors
357    /// - If properties to remove contains a reserved property
358    pub fn remove_properties(mut self, properties: &[String]) -> Result<Self> {
359        // remove duplicates
360        let properties = properties.iter().cloned().collect::<HashSet<_>>();
361
362        // disallow removal of reserved properties
363        let reserved_properties = properties
364            .iter()
365            .filter(|key| TableProperties::RESERVED_PROPERTIES.contains(&key.as_str()))
366            .map(ToString::to_string)
367            .collect::<Vec<_>>();
368
369        if !reserved_properties.is_empty() {
370            return Err(invalid_data!(
371                "Table properties to remove contain reserved properties: [{}]",
372                reserved_properties.join(", ")
373            ));
374        }
375
376        for property in &properties {
377            self.metadata.properties.remove(property);
378        }
379
380        if !properties.is_empty() {
381            self.changes.push(TableUpdate::RemoveProperties {
382                removals: properties.into_iter().collect(),
383            });
384        }
385
386        Ok(self)
387    }
388
389    /// Set the location of the table, stripping any trailing slashes.
390    pub fn set_location(mut self, location: String) -> Self {
391        let location = location.trim_end_matches('/').to_string();
392        if self.metadata.location != location {
393            self.changes.push(TableUpdate::SetLocation {
394                location: location.clone(),
395            });
396            self.metadata.location = location;
397        }
398
399        self
400    }
401
402    /// Add a snapshot to the table metadata.
403    ///
404    /// # Errors
405    /// - Snapshot id already exists.
406    /// - For format version > 1: the sequence number of the snapshot is lower than the highest sequence number specified so far.
407    /// - For format version >= 3: the first-row-id of the snapshot is lower than the next-row-id of the table.
408    /// - For format version >= 3: added-rows is null or first-row-id is null.
409    /// - For format version >= 3: next-row-id would overflow when adding added-rows.
410    pub fn add_snapshot(mut self, snapshot: Snapshot) -> Result<Self> {
411        if self
412            .metadata
413            .snapshots
414            .contains_key(&snapshot.snapshot_id())
415        {
416            return Err(invalid_data!(
417                "Snapshot already exists for: '{}'",
418                snapshot.snapshot_id()
419            ));
420        }
421
422        if self.metadata.format_version != FormatVersion::V1
423            && snapshot.sequence_number() <= self.metadata.last_sequence_number
424            && snapshot.parent_snapshot_id().is_some()
425        {
426            return Err(invalid_data!(
427                "Cannot add snapshot with sequence number {} older than last sequence number {}",
428                snapshot.sequence_number(),
429                self.metadata.last_sequence_number
430            ));
431        }
432
433        if let Some(last) = self.metadata.snapshot_log.last() {
434            // commits can happen concurrently from different machines.
435            // A tolerance helps us avoid failure for small clock skew
436            if snapshot.timestamp_ms() - last.timestamp_ms < -ONE_MINUTE_MS {
437                return Err(invalid_data!(
438                    "Invalid snapshot timestamp {}: before last snapshot timestamp {}",
439                    snapshot.timestamp_ms(),
440                    last.timestamp_ms
441                ));
442            }
443        }
444
445        let max_last_updated = self
446            .last_updated_ms
447            .unwrap_or_default()
448            .max(self.metadata.last_updated_ms);
449        if snapshot.timestamp_ms() - max_last_updated < -ONE_MINUTE_MS {
450            return Err(invalid_data!(
451                "Invalid snapshot timestamp {}: before last updated timestamp {}",
452                snapshot.timestamp_ms(),
453                max_last_updated
454            ));
455        }
456
457        let mut added_rows = None;
458        if self.metadata.format_version >= MIN_FORMAT_VERSION_ROW_LINEAGE {
459            if let Some((first_row_id, added_rows_count)) = snapshot.row_range() {
460                if first_row_id < self.metadata.next_row_id {
461                    return Err(invalid_data!(
462                        "Cannot add a snapshot, first-row-id is behind table next-row-id: {first_row_id} < {}",
463                        self.metadata.next_row_id
464                    ));
465                }
466
467                added_rows = Some(added_rows_count);
468            } else {
469                return Err(invalid_data!(
470                    "Cannot add a snapshot: first-row-id is null. first-row-id must be set for format version >= {MIN_FORMAT_VERSION_ROW_LINEAGE}",
471                ));
472            }
473        }
474
475        if let Some(added_rows) = added_rows {
476            self.metadata.next_row_id = self
477                .metadata
478                .next_row_id
479                .checked_add(added_rows)
480                .ok_or_else(|| {
481                    invalid_data!(
482                        "Cannot add snapshot: next-row-id overflowed when adding added-rows"
483                    )
484                })?;
485        }
486
487        // Mutation happens in next line - must be infallible from here
488        self.changes.push(TableUpdate::AddSnapshot {
489            snapshot: snapshot.clone(),
490        });
491
492        self.last_updated_ms = Some(snapshot.timestamp_ms());
493        self.metadata.last_sequence_number = snapshot.sequence_number();
494        self.metadata
495            .snapshots
496            .insert(snapshot.snapshot_id(), snapshot.into());
497
498        Ok(self)
499    }
500
501    /// Append a snapshot to the specified branch.
502    /// Retention settings from the `branch` are re-used.
503    ///
504    /// # Errors
505    /// - Any of the preconditions of `self.add_snapshot` are not met.
506    pub fn set_branch_snapshot(self, snapshot: Snapshot, branch: &str) -> Result<Self> {
507        let reference = self.metadata.refs.get(branch).cloned();
508
509        let reference = if let Some(mut reference) = reference {
510            if !reference.is_branch() {
511                return Err(invalid_data!(
512                    "Cannot append snapshot to non-branch reference '{branch}'",
513                ));
514            }
515
516            reference.snapshot_id = snapshot.snapshot_id();
517            reference
518        } else {
519            SnapshotReference {
520                snapshot_id: snapshot.snapshot_id(),
521                retention: SnapshotRetention::Branch {
522                    min_snapshots_to_keep: None,
523                    max_snapshot_age_ms: None,
524                    max_ref_age_ms: None,
525                },
526            }
527        };
528
529        self.add_snapshot(snapshot)?.set_ref(branch, reference)
530    }
531
532    /// Remove snapshots by its ids from the table metadata.
533    /// Does nothing if a snapshot id is not present.
534    /// Records removed snapshot IDs once each, in the order they first appear
535    /// in `snapshot_ids`.
536    /// Refs pointing to a removed snapshot are removed as well, see [`Self::remove_ref`].
537    pub fn remove_snapshots(mut self, snapshot_ids: &[i64]) -> Self {
538        let mut removed_snapshots = Vec::with_capacity(snapshot_ids.len());
539
540        for &snapshot_id in snapshot_ids {
541            if self.metadata.snapshots.remove(&snapshot_id).is_some() {
542                removed_snapshots.push(snapshot_id);
543            }
544        }
545
546        if !removed_snapshots.is_empty() {
547            self.changes.push(TableUpdate::RemoveSnapshots {
548                snapshot_ids: removed_snapshots,
549            });
550        }
551
552        // Remove refs that are no longer valid
553        let mut dangling_refs: Vec<String> = self
554            .metadata
555            .refs
556            .iter()
557            .filter(|(_, v)| !self.metadata.snapshots.contains_key(&v.snapshot_id))
558            .map(|(name, _)| name.clone())
559            .collect();
560        dangling_refs.sort();
561
562        for ref_name in dangling_refs {
563            self = self.remove_ref(&ref_name);
564        }
565
566        self
567    }
568
569    /// Set a reference to a snapshot.
570    ///
571    /// # Errors
572    /// - The snapshot id is unknown.
573    pub fn set_ref(mut self, ref_name: &str, reference: SnapshotReference) -> Result<Self> {
574        if self
575            .metadata
576            .refs
577            .get(ref_name)
578            .is_some_and(|snap_ref| snap_ref.eq(&reference))
579        {
580            return Ok(self);
581        }
582
583        let Some(snapshot) = self.metadata.snapshots.get(&reference.snapshot_id) else {
584            return Err(invalid_data!(
585                "Cannot set '{ref_name}' to unknown snapshot: '{}'",
586                reference.snapshot_id
587            ));
588        };
589
590        // Update last_updated_ms to the exact timestamp of the snapshot if it was added in this commit
591        let is_added_snapshot = self.changes.iter().any(|update| {
592            matches!(update, TableUpdate::AddSnapshot { snapshot: snap } if snap.snapshot_id() == snapshot.snapshot_id())
593        });
594        if is_added_snapshot {
595            self.last_updated_ms = Some(snapshot.timestamp_ms());
596        }
597
598        // Current snapshot id is set only for the main branch
599        if ref_name == MAIN_BRANCH {
600            self.metadata.current_snapshot_id = Some(snapshot.snapshot_id());
601            let timestamp_ms = if let Some(last_updated_ms) = self.last_updated_ms {
602                last_updated_ms
603            } else {
604                let last_updated_ms = chrono::Utc::now().timestamp_millis();
605                self.last_updated_ms = Some(last_updated_ms);
606                last_updated_ms
607            };
608
609            self.metadata.snapshot_log.push(SnapshotLog {
610                snapshot_id: snapshot.snapshot_id(),
611                timestamp_ms,
612            });
613        }
614
615        self.changes.push(TableUpdate::SetSnapshotRef {
616            ref_name: ref_name.to_string(),
617            reference: reference.clone(),
618        });
619        self.metadata.refs.insert(ref_name.to_string(), reference);
620
621        Ok(self)
622    }
623
624    /// Remove a reference
625    ///
626    /// If `ref_name='main'` the current snapshot id is set to `None`.
627    pub fn remove_ref(mut self, ref_name: &str) -> Self {
628        if ref_name == MAIN_BRANCH {
629            self.metadata.current_snapshot_id = None;
630        }
631
632        if self.metadata.refs.remove(ref_name).is_some() || ref_name == MAIN_BRANCH {
633            self.changes.push(TableUpdate::RemoveSnapshotRef {
634                ref_name: ref_name.to_string(),
635            });
636        }
637
638        self
639    }
640
641    /// Set statistics for a snapshot
642    pub fn set_statistics(mut self, statistics: StatisticsFile) -> Self {
643        self.metadata
644            .statistics
645            .insert(statistics.snapshot_id, statistics.clone());
646        self.changes.push(TableUpdate::SetStatistics {
647            statistics: statistics.clone(),
648        });
649        self
650    }
651
652    /// Remove statistics for a snapshot
653    pub fn remove_statistics(mut self, snapshot_id: i64) -> Self {
654        let previous = self.metadata.statistics.remove(&snapshot_id);
655        if previous.is_some() {
656            self.changes
657                .push(TableUpdate::RemoveStatistics { snapshot_id });
658        }
659        self
660    }
661
662    /// Set partition statistics
663    pub fn set_partition_statistics(
664        mut self,
665        partition_statistics_file: PartitionStatisticsFile,
666    ) -> Self {
667        self.metadata.partition_statistics.insert(
668            partition_statistics_file.snapshot_id,
669            partition_statistics_file.clone(),
670        );
671        self.changes.push(TableUpdate::SetPartitionStatistics {
672            partition_statistics: partition_statistics_file,
673        });
674        self
675    }
676
677    /// Remove partition statistics
678    pub fn remove_partition_statistics(mut self, snapshot_id: i64) -> Self {
679        let previous = self.metadata.partition_statistics.remove(&snapshot_id);
680        if previous.is_some() {
681            self.changes
682                .push(TableUpdate::RemovePartitionStatistics { snapshot_id });
683        }
684        self
685    }
686
687    /// Add a schema to the table metadata.
688    ///
689    /// The provided `schema.schema_id` may not be used.
690    ///
691    /// Important: Use this method with caution. The builder does not check
692    /// if the added schema is compatible with the current schema.
693    pub fn add_schema(mut self, schema: Schema) -> Result<Self> {
694        let new_schema_id = self.reuse_or_create_new_schema_id(&schema);
695        let schema_found = self.metadata.schemas.contains_key(&new_schema_id);
696
697        if schema_found {
698            if self.last_added_schema_id != Some(new_schema_id) {
699                self.changes.push(TableUpdate::AddSchema {
700                    schema: schema.clone(),
701                });
702                self.last_added_schema_id = Some(new_schema_id);
703            }
704
705            return Ok(self);
706        }
707
708        // New schemas might contain only old columns. In this case last_column_id should not be
709        // reduced.
710        self.metadata.last_column_id =
711            std::cmp::max(self.metadata.last_column_id, schema.highest_field_id());
712
713        // Set schema-id
714        let schema = match new_schema_id == schema.schema_id() {
715            true => schema,
716            false => schema.with_schema_id(new_schema_id),
717        };
718
719        self.metadata
720            .schemas
721            .insert(new_schema_id, schema.clone().into());
722
723        self.changes.push(TableUpdate::AddSchema { schema });
724
725        self.last_added_schema_id = Some(new_schema_id);
726
727        Ok(self)
728    }
729
730    /// Set the current schema id.
731    ///
732    /// If `schema_id` is -1, the last added schema is set as the current schema.
733    ///
734    /// Errors:
735    /// - provided `schema_id` is -1 but no schema has been added via `add_schema`.
736    /// - No schema with the provided `schema_id` exists.
737    /// - A field of the schema takes the name of a partition field it is not the source of.
738    pub fn set_current_schema(mut self, mut schema_id: i32) -> Result<Self> {
739        if schema_id == Self::LAST_ADDED {
740            schema_id = self.last_added_schema_id.ok_or_else(|| {
741                invalid_data!(
742                    "Cannot set current schema to last added schema: no schema has been added."
743                )
744            })?;
745        };
746        let schema_id = schema_id; // Make immutable
747
748        if schema_id == self.metadata.current_schema_id {
749            return Ok(self);
750        }
751
752        let schema = self.metadata.schemas.get(&schema_id).ok_or_else(|| {
753            invalid_data!("Cannot set current schema to unknown schema with id: '{schema_id}'")
754        })?;
755        self.validate_schema_field_names(schema)?;
756
757        // Old partition specs and sort-orders should be preserved even if they are not compatible with the new schema,
758        // so that older metadata can still be interpreted.
759        // Default partition spec and sort order are checked in the build() method
760        // which allows other default partition specs and sort orders to be set before the build.
761
762        self.metadata.current_schema_id = schema_id;
763
764        if self.last_added_schema_id == Some(schema_id) {
765            self.changes.push(TableUpdate::SetCurrentSchema {
766                schema_id: Self::LAST_ADDED,
767            });
768        } else {
769            self.changes
770                .push(TableUpdate::SetCurrentSchema { schema_id });
771        }
772
773        Ok(self)
774    }
775
776    /// Add a schema and set it as the current schema.
777    pub fn add_current_schema(self, schema: Schema) -> Result<Self> {
778        self.add_schema(schema)?
779            .set_current_schema(Self::LAST_ADDED)
780    }
781
782    /// Validate schema field names against the partition field names of every spec.
783    ///
784    /// A partition field whose source column is absent from `schema` is skipped: its spec is
785    /// historical, kept only so old metadata can be interpreted, so the name is free to reuse.
786    /// Otherwise the name may only be taken by the source column itself, which is the identity
787    /// partition case that [`Self::validate_partition_field_names`] allows in the other direction.
788    ///
789    /// # Errors
790    /// - A schema field takes the name of a partition field it is not the source of.
791    fn validate_schema_field_names(&self, schema: &Schema) -> Result<()> {
792        for (spec_id, spec) in &self.metadata.partition_specs {
793            for partition_field in spec.fields() {
794                if schema.field_by_id(partition_field.source_id).is_none() {
795                    continue;
796                }
797
798                let Some(schema_field) = schema.field_by_name(&partition_field.name) else {
799                    continue;
800                };
801
802                if schema_field.id != partition_field.source_id {
803                    return Err(invalid_data!(
804                        "Cannot set current schema {}: field '{}' with id {} conflicts with a \
805                         partition field of that name in spec {} sourced from field id {}",
806                        schema.schema_id(),
807                        partition_field.name,
808                        schema_field.id,
809                        spec_id,
810                        partition_field.source_id
811                    ));
812                }
813            }
814        }
815
816        Ok(())
817    }
818
819    /// Validate partition field names against schema field names across all historical schemas.
820    ///
821    /// Due to Iceberg's multi-version property, partition fields can share names with schema fields
822    /// if they meet specific requirements (identity or void transform + matching source field ID).
823    /// This validation enforces those rules across all historical schema versions.
824    ///
825    /// # Errors
826    /// - Partition field name conflicts with schema field name but doesn't use identity or void
827    ///   transform.
828    /// - Partition field uses identity or void transform but references wrong source field ID.
829    fn validate_partition_field_names(&self, unbound_spec: &UnboundPartitionSpec) -> Result<()> {
830        if self.metadata.schemas.is_empty() {
831            return Ok(());
832        }
833
834        let current_schema = self.get_current_schema()?;
835        for partition_field in unbound_spec.fields() {
836            let exists_in_any_schema = self
837                .metadata
838                .name_exists_in_any_schema(partition_field.name());
839
840            // Skip if partition field name doesn't conflict with any schema field
841            if !exists_in_any_schema {
842                continue;
843            }
844
845            // If name exists in schemas, validate against current schema rules
846            if let Some(schema_field) = current_schema.field_by_name(partition_field.name()) {
847                let has_matching_source_id = schema_field.id == partition_field.source_id()?;
848
849                if !partition_field.transform().may_share_name_with_source() {
850                    return Err(invalid_data!(
851                        "Cannot create partition with name '{}' that conflicts with schema field and is not an identity or void transform.",
852                        partition_field.name()
853                    ));
854                }
855
856                if !has_matching_source_id {
857                    return Err(invalid_data!(
858                        "Cannot create identity or void partition sourced from different field in schema. \
859                             Field name '{}' has id `{}` in schema but partition source id is `{}`",
860                        partition_field.name(),
861                        schema_field.id,
862                        partition_field.source_id()?
863                    ));
864                }
865            }
866        }
867
868        Ok(())
869    }
870
871    /// Add a partition spec to the table metadata.
872    ///
873    /// The spec is bound eagerly to the current schema.
874    /// If a schema is added in the same set of changes, the schema should be added first.
875    ///
876    /// Even if `unbound_spec.spec_id` is provided as `Some`, it may not be used.
877    ///
878    /// # Errors
879    /// - The partition spec cannot be bound to the current schema.
880    /// - The partition spec has non-sequential field ids and the table format version is 1.
881    pub fn add_partition_spec(mut self, unbound_spec: UnboundPartitionSpec) -> Result<Self> {
882        let schema = self.get_current_schema()?.clone();
883
884        // Check if partition field names conflict with schema field names across all schemas
885        self.validate_partition_field_names(&unbound_spec)?;
886
887        // Reuse field IDs for equivalent fields from existing partition specs
888        let unbound_spec = self.reuse_partition_field_ids(unbound_spec)?;
889
890        let spec = PartitionSpecBuilder::new_from_unbound(unbound_spec.clone(), schema)?
891            .with_last_assigned_field_id(self.metadata.last_partition_id)
892            .build()?;
893
894        let new_spec_id = self.reuse_or_create_new_spec_id(&spec);
895        let spec_found = self.metadata.partition_specs.contains_key(&new_spec_id);
896        let spec = spec.with_spec_id(new_spec_id);
897        let unbound_spec = unbound_spec.with_spec_id(new_spec_id);
898
899        if spec_found {
900            if self.last_added_spec_id != Some(new_spec_id) {
901                self.changes
902                    .push(TableUpdate::AddSpec { spec: unbound_spec });
903                self.last_added_spec_id = Some(new_spec_id);
904            }
905
906            return Ok(self);
907        }
908
909        if self.metadata.format_version <= FormatVersion::V1 && !spec.has_sequential_ids() {
910            return Err(invalid_data!(
911                "Cannot add partition spec with non-sequential field ids to format version 1 table"
912            ));
913        }
914
915        let highest_field_id = spec
916            .highest_field_id()
917            .unwrap_or(UNPARTITIONED_LAST_ASSIGNED_ID);
918        self.metadata
919            .partition_specs
920            .insert(new_spec_id, Arc::new(spec));
921        self.changes
922            .push(TableUpdate::AddSpec { spec: unbound_spec });
923
924        self.last_added_spec_id = Some(new_spec_id);
925        self.metadata.last_partition_id =
926            std::cmp::max(self.metadata.last_partition_id, highest_field_id);
927
928        Ok(self)
929    }
930
931    /// Reuse partition field IDs for equivalent fields from existing partition specs.
932    ///
933    /// According to the Iceberg spec, partition field IDs must be reused if an existing
934    /// partition spec contains an equivalent field (same source_id and transform).
935    fn reuse_partition_field_ids(
936        &self,
937        unbound_spec: UnboundPartitionSpec,
938    ) -> Result<UnboundPartitionSpec> {
939        let equivalent_field_ids = Self::highest_equivalent_field_ids(
940            self.metadata
941                .partition_specs
942                .values()
943                .flat_map(|spec| spec.fields()),
944        );
945
946        // Create new fields with reused field IDs where possible
947        let fields = unbound_spec
948            .fields
949            .into_iter()
950            .map(|field| {
951                if field.field_id().is_none()
952                    && let Some(&existing_field_id) =
953                        equivalent_field_ids.get(&(field.source_ids().to_vec(), field.transform()))
954                {
955                    field.with_field_id(existing_field_id)
956                } else {
957                    field
958                }
959            })
960            .collect();
961
962        Ok(UnboundPartitionSpec {
963            spec_id: unbound_spec.spec_id,
964            fields,
965        })
966    }
967
968    /// Map each (source_ids, transform) pair to the highest field id assigned to it.
969    ///
970    /// Taking the highest keeps the result independent of the order `fields` arrive in, which for
971    /// the specs of a `TableMetadata` is an arbitrary `HashMap` iteration order.
972    fn highest_equivalent_field_ids<'a>(
973        fields: impl Iterator<Item = &'a PartitionField>,
974    ) -> HashMap<(Vec<i32>, Transform), i32> {
975        let mut equivalent_field_ids = HashMap::new();
976        for field in fields {
977            equivalent_field_ids
978                .entry((vec![field.source_id], field.transform))
979                .and_modify(|id: &mut i32| *id = (*id).max(field.field_id))
980                .or_insert(field.field_id);
981        }
982        equivalent_field_ids
983    }
984
985    /// Set the default partition spec.
986    ///
987    /// # Errors
988    /// - spec_id is -1 but no spec has been added via `add_partition_spec`.
989    /// - No partition spec with the provided `spec_id` exists.
990    pub fn set_default_partition_spec(mut self, mut spec_id: i32) -> Result<Self> {
991        if spec_id == Self::LAST_ADDED {
992            spec_id = self.last_added_spec_id.ok_or_else(|| {
993                invalid_data!(
994                    "Cannot set default partition spec to last added spec: no spec has been added."
995                )
996            })?;
997        }
998
999        if self.metadata.default_spec.spec_id() == spec_id {
1000            return Ok(self);
1001        }
1002
1003        if !self.metadata.partition_specs.contains_key(&spec_id) {
1004            return Err(invalid_data!(
1005                "Cannot set default partition spec to unknown spec with id: '{spec_id}'",
1006            ));
1007        }
1008
1009        let schemaless_spec = self
1010            .metadata
1011            .partition_specs
1012            .get(&spec_id)
1013            .ok_or_else(|| {
1014                invalid_data!(
1015                    "Cannot set default partition spec to unknown spec with id: '{spec_id}'",
1016                )
1017            })?
1018            .clone();
1019        let spec = Arc::unwrap_or_clone(schemaless_spec);
1020        let spec_type = spec.partition_type(self.get_current_schema()?)?;
1021        self.metadata.default_spec = Arc::new(spec);
1022        self.metadata.default_partition_type = spec_type;
1023
1024        if self.last_added_spec_id == Some(spec_id) {
1025            self.changes.push(TableUpdate::SetDefaultSpec {
1026                spec_id: Self::LAST_ADDED,
1027            });
1028        } else {
1029            self.changes.push(TableUpdate::SetDefaultSpec { spec_id });
1030        }
1031
1032        Ok(self)
1033    }
1034
1035    /// Add a partition spec and set it as the default
1036    pub fn add_default_partition_spec(self, unbound_spec: UnboundPartitionSpec) -> Result<Self> {
1037        self.add_partition_spec(unbound_spec)?
1038            .set_default_partition_spec(Self::LAST_ADDED)
1039    }
1040
1041    /// Remove partition specs by their ids from the table metadata.
1042    /// Does nothing if a spec id is not present. Active partition specs
1043    /// should not be removed.
1044    ///
1045    /// # Errors
1046    /// - Cannot remove the default partition spec.
1047    pub fn remove_partition_specs(mut self, spec_ids: &[i32]) -> Result<Self> {
1048        if spec_ids.contains(&self.metadata.default_spec.spec_id()) {
1049            return Err(invalid_data!("Cannot remove default partition spec"));
1050        }
1051
1052        let mut removed_specs = Vec::with_capacity(spec_ids.len());
1053        spec_ids.iter().for_each(|id| {
1054            if self.metadata.partition_specs.remove(id).is_some() {
1055                removed_specs.push(*id);
1056            }
1057        });
1058
1059        if !removed_specs.is_empty() {
1060            self.changes.push(TableUpdate::RemovePartitionSpecs {
1061                spec_ids: removed_specs,
1062            });
1063        }
1064
1065        Ok(self)
1066    }
1067
1068    /// Add a sort order to the table metadata.
1069    ///
1070    /// The spec is bound eagerly to the current schema and must be valid for it.
1071    /// If a schema is added in the same set of changes, the schema should be added first.
1072    ///
1073    /// Even if `sort_order.order_id` is provided, it may not be used.
1074    ///
1075    /// # Errors
1076    /// - Sort order id to add already exists.
1077    /// - Sort order is incompatible with the current schema.
1078    pub fn add_sort_order(mut self, sort_order: SortOrder) -> Result<Self> {
1079        let new_order_id = self.reuse_or_create_new_sort_id(&sort_order);
1080        let sort_order_found = self.metadata.sort_orders.contains_key(&new_order_id);
1081
1082        if sort_order_found {
1083            if self.last_added_order_id != Some(new_order_id) {
1084                self.changes.push(TableUpdate::AddSortOrder {
1085                    sort_order: sort_order.clone().with_order_id(new_order_id),
1086                });
1087                self.last_added_order_id = Some(new_order_id);
1088            }
1089
1090            return Ok(self);
1091        }
1092
1093        let schema = self.get_current_schema()?.clone().as_ref().clone();
1094        let sort_order = SortOrder::builder()
1095            .with_order_id(new_order_id)
1096            .with_fields(sort_order.fields)
1097            .build(&schema)
1098            .map_err(|e| {
1099                invalid_data!("Sort order to add is incompatible with current schema: {e}")
1100                    .with_source(e)
1101            })?;
1102
1103        self.last_added_order_id = Some(new_order_id);
1104        self.metadata
1105            .sort_orders
1106            .insert(new_order_id, sort_order.clone().into());
1107        self.changes.push(TableUpdate::AddSortOrder { sort_order });
1108
1109        Ok(self)
1110    }
1111
1112    /// Set the default sort order. If `sort_order_id` is -1, the last added sort order is set as default.
1113    ///
1114    /// # Errors
1115    /// - sort_order_id is -1 but no sort order has been added via `add_sort_order`.
1116    /// - No sort order with the provided `sort_order_id` exists.
1117    pub fn set_default_sort_order(mut self, mut sort_order_id: i64) -> Result<Self> {
1118        if sort_order_id == Self::LAST_ADDED as i64 {
1119            sort_order_id = self.last_added_order_id.ok_or_else(|| {
1120                invalid_data!(
1121                    "Cannot set default sort order to last added order: no order has been added."
1122                )
1123            })?;
1124        }
1125
1126        if self.metadata.default_sort_order_id == sort_order_id {
1127            return Ok(self);
1128        }
1129
1130        if !self.metadata.sort_orders.contains_key(&sort_order_id) {
1131            return Err(invalid_data!(
1132                "Cannot set default sort order to unknown order with id: '{sort_order_id}'"
1133            ));
1134        }
1135
1136        self.metadata.default_sort_order_id = sort_order_id;
1137
1138        if self.last_added_order_id == Some(sort_order_id) {
1139            self.changes.push(TableUpdate::SetDefaultSortOrder {
1140                sort_order_id: Self::LAST_ADDED as i64,
1141            });
1142        } else {
1143            self.changes
1144                .push(TableUpdate::SetDefaultSortOrder { sort_order_id });
1145        }
1146
1147        Ok(self)
1148    }
1149
1150    /// Add a sort order and set it as the default
1151    fn add_default_sort_order(self, sort_order: SortOrder) -> Result<Self> {
1152        self.add_sort_order(sort_order)?
1153            .set_default_sort_order(Self::LAST_ADDED as i64)
1154    }
1155
1156    /// Add an encryption key to the table metadata.
1157    pub fn add_encryption_key(mut self, key: EncryptedKey) -> Self {
1158        let key_id = key.key_id().to_string();
1159        if self.metadata.encryption_keys.contains_key(&key_id) {
1160            // already exists
1161            return self;
1162        }
1163
1164        self.metadata.encryption_keys.insert(key_id, key.clone());
1165        self.changes.push(TableUpdate::AddEncryptionKey {
1166            encryption_key: key,
1167        });
1168        self
1169    }
1170
1171    /// Remove an encryption key from the table metadata.
1172    pub fn remove_encryption_key(mut self, key_id: &str) -> Self {
1173        if self.metadata.encryption_keys.remove(key_id).is_some() {
1174            self.changes.push(TableUpdate::RemoveEncryptionKey {
1175                key_id: key_id.to_string(),
1176            });
1177        }
1178        self
1179    }
1180
1181    /// Build the table metadata.
1182    pub fn build(mut self) -> Result<TableMetadataBuildResult> {
1183        self.metadata.last_updated_ms = self
1184            .last_updated_ms
1185            .unwrap_or_else(|| chrono::Utc::now().timestamp_millis());
1186
1187        self.check_format_compatibility()?;
1188
1189        // Check compatibility of the current schema to the default partition spec and sort order.
1190        // We use the `get_xxx` methods from the builder to avoid using the panicking
1191        // `TableMetadata.default_partition_spec` etc. methods.
1192        let schema = self.get_current_schema()?.clone();
1193        let sort_order = Arc::unwrap_or_clone(self.get_default_sort_order()?);
1194
1195        self.metadata.default_spec = Arc::new(
1196            Arc::unwrap_or_clone(self.metadata.default_spec)
1197                .into_unbound()
1198                .bind(schema.clone())?,
1199        );
1200        self.metadata.default_partition_type =
1201            self.metadata.default_spec.partition_type(&schema)?;
1202        SortOrder::builder()
1203            .with_fields(sort_order.fields)
1204            .build(&schema)?;
1205
1206        self.update_snapshot_log()?;
1207
1208        // Validate the metadata log with the entry for the previous file, as every reader does.
1209        if let Some(hist_entry) = self.previous_history_entry.take() {
1210            self.metadata.metadata_log.push(hist_entry);
1211        }
1212        let expired_metadata_logs = self.expire_metadata_log();
1213        self.metadata.try_normalize()?;
1214
1215        Ok(TableMetadataBuildResult {
1216            metadata: self.metadata,
1217            changes: self.changes,
1218            expired_metadata_logs,
1219        })
1220    }
1221
1222    fn expire_metadata_log(&mut self) -> Vec<MetadataLog> {
1223        let max_size = self
1224            .metadata
1225            .properties
1226            .get(TableProperties::PROPERTY_METADATA_PREVIOUS_VERSIONS_MAX)
1227            .and_then(|v| v.parse::<usize>().ok())
1228            .unwrap_or(TableProperties::PROPERTY_METADATA_PREVIOUS_VERSIONS_MAX_DEFAULT)
1229            .max(1);
1230
1231        if self.metadata.metadata_log.len() > max_size {
1232            self.metadata
1233                .metadata_log
1234                .drain(0..self.metadata.metadata_log.len() - max_size)
1235                .collect()
1236        } else {
1237            Vec::new()
1238        }
1239    }
1240
1241    fn update_snapshot_log(&mut self) -> Result<()> {
1242        let intermediate_snapshots = self.get_intermediate_snapshots();
1243        let has_removed_snapshots = self
1244            .changes
1245            .iter()
1246            .any(|update| matches!(update, TableUpdate::RemoveSnapshots { .. }));
1247
1248        if intermediate_snapshots.is_empty() && !has_removed_snapshots {
1249            return Ok(());
1250        }
1251
1252        let mut new_snapshot_log = Vec::new();
1253        for log_entry in &self.metadata.snapshot_log {
1254            let snapshot_id = log_entry.snapshot_id;
1255            if self.metadata.snapshots.contains_key(&snapshot_id) {
1256                if !intermediate_snapshots.contains(&snapshot_id) {
1257                    new_snapshot_log.push(log_entry.clone());
1258                }
1259            } else if has_removed_snapshots {
1260                // any invalid entry causes the history before it to be removed. otherwise, there could be
1261                // history gaps that cause time-travel queries to produce incorrect results. for example,
1262                // if history is [(t1, s1), (t2, s2), (t3, s3)] and s2 is removed, the history cannot be
1263                // [(t1, s1), (t3, s3)] because it appears that s3 was current during the time between t2
1264                // and t3 when in fact s2 was the current snapshot.
1265                new_snapshot_log.clear();
1266            }
1267        }
1268
1269        if let Some(current_snapshot_id) = self.metadata.current_snapshot_id {
1270            let last_id = new_snapshot_log.last().map(|entry| entry.snapshot_id);
1271            if last_id != Some(current_snapshot_id) {
1272                return Err(invalid_data!(
1273                    "Cannot set invalid snapshot log: latest entry is not the current snapshot"
1274                ));
1275            }
1276        };
1277
1278        self.metadata.snapshot_log = new_snapshot_log;
1279        Ok(())
1280    }
1281
1282    /// Finds intermediate snapshots that have not been committed as the current snapshot.
1283    ///
1284    /// Transactions can create snapshots that are never the current snapshot because several
1285    /// changes are combined by the transaction into one table metadata update. when each
1286    /// intermediate snapshot is added to table metadata, it is added to the snapshot log, assuming
1287    /// that it will be the current snapshot. when there are multiple snapshot updates, the log must
1288    /// be corrected by suppressing the intermediate snapshot entries.
1289    ///     
1290    /// A snapshot is an intermediate snapshot if it was added but is not the current snapshot.
1291    fn get_intermediate_snapshots(&self) -> HashSet<i64> {
1292        let added_snapshot_ids = self
1293            .changes
1294            .iter()
1295            .filter_map(|update| match update {
1296                TableUpdate::AddSnapshot { snapshot } => Some(snapshot.snapshot_id()),
1297                _ => None,
1298            })
1299            .collect::<HashSet<_>>();
1300
1301        self.changes
1302            .iter()
1303            .filter_map(|update| match update {
1304                TableUpdate::SetSnapshotRef {
1305                    ref_name,
1306                    reference,
1307                } => {
1308                    if added_snapshot_ids.contains(&reference.snapshot_id)
1309                        && ref_name == MAIN_BRANCH
1310                        && reference.snapshot_id
1311                            != self
1312                                .metadata
1313                                .current_snapshot_id
1314                                .unwrap_or(i64::from(Self::LAST_ADDED))
1315                    {
1316                        Some(reference.snapshot_id)
1317                    } else {
1318                        None
1319                    }
1320                }
1321                _ => None,
1322            })
1323            .collect()
1324    }
1325
1326    /// Gives each field of `spec` the highest id an equivalent field has in the table's specs, or
1327    /// the next id above `last_partition_id`.
1328    fn reassign_partition_field_ids(
1329        &self,
1330        spec: PartitionSpec,
1331        schema: &Schema,
1332    ) -> Result<UnboundPartitionSpec> {
1333        let fields = spec
1334            .fields()
1335            .iter()
1336            .map(|field| {
1337                UnboundPartitionField::builder()
1338                    .source_ids(vec![field.source_id])
1339                    .name(field.name.clone())
1340                    .transform(field.transform)
1341                    .build()
1342            })
1343            .collect::<Result<Vec<_>>>()?;
1344        let spec = self.reuse_partition_field_ids(
1345            UnboundPartitionSpec::builder()
1346                .add_partition_fields(fields)?
1347                .build(),
1348        )?;
1349
1350        Ok(
1351            PartitionSpecBuilder::new_from_unbound(spec, schema.clone())?
1352                .with_last_assigned_field_id(self.metadata.last_partition_id)
1353                .build()?
1354                .into_unbound(),
1355        )
1356    }
1357
1358    fn reassign_ids(
1359        schema: Schema,
1360        spec: UnboundPartitionSpec,
1361        sort_order: SortOrder,
1362    ) -> Result<(Schema, PartitionSpec, SortOrder)> {
1363        // Re-assign field ids and schema ids for a new table.
1364        let previous_id_to_name = schema.field_id_to_name_map().clone();
1365        let fresh_schema = schema
1366            .into_builder()
1367            .with_schema_id(DEFAULT_SCHEMA_ID)
1368            .with_reassigned_field_ids(FIRST_FIELD_ID)
1369            .build()?;
1370
1371        let (fresh_spec, fresh_sort_order) =
1372            Self::rebind_to_schema(&previous_id_to_name, &fresh_schema, spec, sort_order)?;
1373
1374        Ok((fresh_schema, fresh_spec, fresh_sort_order))
1375    }
1376
1377    /// Rebuilds `spec` and `sort_order` against `fresh_schema`, looking up each source column by
1378    /// the name `previous_id_to_name` gives for the field id they were built against.
1379    fn rebind_to_schema(
1380        previous_id_to_name: &HashMap<i32, String>,
1381        fresh_schema: &Schema,
1382        spec: UnboundPartitionSpec,
1383        sort_order: SortOrder,
1384    ) -> Result<(PartitionSpec, SortOrder)> {
1385        // Re-build partition spec with new ids
1386        let mut fresh_spec = PartitionSpecBuilder::new(fresh_schema.clone());
1387        for field in spec.fields() {
1388            let source_id = field.source_id()?;
1389            let source_field_name = previous_id_to_name.get(&source_id).ok_or_else(|| {
1390                invalid_data!(
1391                    "Cannot find source column with id {} for partition column {} in schema.",
1392                    source_id,
1393                    field.name()
1394                )
1395            })?;
1396            fresh_spec = fresh_spec.add_partition_field(
1397                source_field_name,
1398                field.name(),
1399                field.transform(),
1400            )?;
1401        }
1402        let fresh_spec = fresh_spec.build()?;
1403
1404        // Re-build sort order with new ids
1405        let mut fresh_order = SortOrder::builder();
1406        for mut field in sort_order.fields {
1407            let source_field_name = previous_id_to_name.get(&field.source_id).ok_or_else(|| {
1408                invalid_data!(
1409                    "Cannot find source column with id {} for sort column in schema.",
1410                    field.source_id
1411                )
1412            })?;
1413            let new_field_id = fresh_schema
1414                       .field_by_name(source_field_name)
1415                       .ok_or_else(|| {
1416                           Error::new(
1417                               ErrorKind::Unexpected,
1418                               format!(
1419                                   "Cannot find source column with name {source_field_name} for sort column in re-assigned schema."
1420                               ),
1421                           )
1422                       })?.id;
1423            field.source_id = new_field_id;
1424            fresh_order.with_sort_field(field);
1425        }
1426        let fresh_sort_order = fresh_order.build(fresh_schema)?;
1427
1428        Ok((fresh_spec, fresh_sort_order))
1429    }
1430
1431    fn check_format_compatibility(&self) -> Result<()> {
1432        let format_version = self.metadata.format_version;
1433        if format_version < FormatVersion::V3 {
1434            let encryption_properties = [
1435                TableProperties::PROPERTY_ENCRYPTION_KEY_ID,
1436                TableProperties::PROPERTY_ENCRYPTION_DATA_KEY_LENGTH,
1437            ]
1438            .into_iter()
1439            .filter(|key| self.metadata.properties.contains_key(*key))
1440            .collect::<Vec<_>>();
1441            if !encryption_properties.is_empty() {
1442                return Err(invalid_data!(
1443                    "Invalid properties for {format_version}: [{}]",
1444                    encryption_properties.join(", ")
1445                ));
1446            }
1447
1448            if !self.metadata.encryption_keys.is_empty() {
1449                return Err(invalid_data!(
1450                    "Encryption keys are not supported for {format_version} tables"
1451                ));
1452            }
1453        }
1454
1455        Ok(())
1456    }
1457
1458    fn reuse_or_create_new_schema_id(&self, new_schema: &Schema) -> i32 {
1459        self.metadata
1460            .schemas
1461            .iter()
1462            .find_map(|(id, schema)| new_schema.is_same_schema(schema).then_some(*id))
1463            .unwrap_or_else(|| self.get_highest_schema_id() + 1)
1464    }
1465
1466    fn get_highest_schema_id(&self) -> i32 {
1467        *self
1468            .metadata
1469            .schemas
1470            .keys()
1471            .max()
1472            .unwrap_or(&self.metadata.current_schema_id)
1473    }
1474
1475    fn get_current_schema(&self) -> Result<&SchemaRef> {
1476        self.metadata
1477            .schemas
1478            .get(&self.metadata.current_schema_id)
1479            .ok_or_else(|| {
1480                invalid_data!(
1481                    "Current schema with id '{}' not found in table metadata.",
1482                    self.metadata.current_schema_id
1483                )
1484            })
1485    }
1486
1487    fn get_default_sort_order(&self) -> Result<SortOrderRef> {
1488        self.metadata
1489            .sort_orders
1490            .get(&self.metadata.default_sort_order_id)
1491            .cloned()
1492            .ok_or_else(|| {
1493                invalid_data!(
1494                    "Default sort order with id '{}' not found in table metadata.",
1495                    self.metadata.default_sort_order_id
1496                )
1497            })
1498    }
1499
1500    /// If a compatible spec already exists, use the same ID. Otherwise, use 1 more than the highest ID.
1501    fn reuse_or_create_new_spec_id(&self, new_spec: &PartitionSpec) -> i32 {
1502        self.metadata
1503            .partition_specs
1504            .iter()
1505            .find_map(|(id, old_spec)| new_spec.is_compatible_with(old_spec).then_some(*id))
1506            .unwrap_or_else(|| {
1507                self.get_highest_spec_id()
1508                    .map(|id| id + 1)
1509                    .unwrap_or(DEFAULT_PARTITION_SPEC_ID)
1510            })
1511    }
1512
1513    fn get_highest_spec_id(&self) -> Option<i32> {
1514        self.metadata.partition_specs.keys().max().copied()
1515    }
1516
1517    /// If a compatible sort-order already exists, use the same ID. Otherwise, use 1 more than the highest ID.
1518    fn reuse_or_create_new_sort_id(&self, new_sort_order: &SortOrder) -> i64 {
1519        if new_sort_order.is_unsorted() {
1520            return SortOrder::unsorted_order().order_id;
1521        }
1522
1523        self.metadata
1524            .sort_orders
1525            .iter()
1526            .find_map(|(id, sort_order)| {
1527                sort_order.fields.eq(&new_sort_order.fields).then_some(*id)
1528            })
1529            .unwrap_or_else(|| {
1530                self.highest_sort_order_id()
1531                    .unwrap_or(SortOrder::unsorted_order().order_id)
1532                    + 1
1533            })
1534    }
1535
1536    fn highest_sort_order_id(&self) -> Option<i64> {
1537        self.metadata.sort_orders.keys().max().copied()
1538    }
1539
1540    /// Remove schemas by their ids from the table metadata.
1541    /// Does nothing if a schema id is not present. Active schemas should not be removed.
1542    pub fn remove_schemas(mut self, schema_id_to_remove: &[i32]) -> Result<Self> {
1543        if schema_id_to_remove.contains(&self.metadata.current_schema_id) {
1544            return Err(invalid_data!("Cannot remove current schema"));
1545        }
1546
1547        if schema_id_to_remove.is_empty() {
1548            return Ok(self);
1549        }
1550
1551        let mut removed_schemas = Vec::with_capacity(schema_id_to_remove.len());
1552        self.metadata.schemas.retain(|id, _schema| {
1553            if schema_id_to_remove.contains(id) {
1554                removed_schemas.push(*id);
1555                false
1556            } else {
1557                true
1558            }
1559        });
1560
1561        self.changes.push(TableUpdate::RemoveSchemas {
1562            schema_ids: removed_schemas,
1563        });
1564
1565        Ok(self)
1566    }
1567}
1568
1569impl From<TableMetadataBuildResult> for TableMetadata {
1570    fn from(result: TableMetadataBuildResult) -> Self {
1571        result.metadata
1572    }
1573}
1574
1575#[cfg(test)]
1576mod tests {
1577    use std::fs::File;
1578    use std::io::BufReader;
1579    use std::thread::sleep;
1580
1581    use super::*;
1582    use crate::TableIdent;
1583    use crate::io::FileIO;
1584    use crate::spec::{
1585        BlobMetadata, NestedField, NullOrder, Operation, PartitionSpec, PrimitiveType, Schema,
1586        SnapshotRetention, SortDirection, SortField, StructType, Summary, TableProperties,
1587        Transform, Type, UnboundPartitionField,
1588    };
1589    use crate::table::Table;
1590    use crate::test_utils::scan::TableTestFixture;
1591    use crate::test_utils::test_runtime;
1592
1593    const TEST_LOCATION: &str = "s3://bucket/test/location";
1594    const LAST_ASSIGNED_COLUMN_ID: i32 = 3;
1595
1596    fn schema() -> Schema {
1597        Schema::builder()
1598            .with_fields(vec![
1599                NestedField::required(1, "x", Type::Primitive(PrimitiveType::Long)).into(),
1600                NestedField::required(2, "y", Type::Primitive(PrimitiveType::Long)).into(),
1601                NestedField::required(3, "z", Type::Primitive(PrimitiveType::Long)).into(),
1602            ])
1603            .build()
1604            .unwrap()
1605    }
1606
1607    fn sort_order() -> SortOrder {
1608        let schema = schema();
1609        SortOrder::builder()
1610            .with_order_id(1)
1611            .with_sort_field(SortField {
1612                source_id: 3,
1613                transform: Transform::Bucket(4),
1614                direction: SortDirection::Descending,
1615                null_order: NullOrder::First,
1616            })
1617            .build(&schema)
1618            .unwrap()
1619    }
1620
1621    fn partition_spec() -> UnboundPartitionSpec {
1622        UnboundPartitionSpec::builder()
1623            .with_spec_id(0)
1624            .add_partition_field(
1625                UnboundPartitionField::builder()
1626                    .source_ids(vec![2])
1627                    .name("y")
1628                    .transform(Transform::Identity)
1629                    .build()
1630                    .unwrap(),
1631            )
1632            .unwrap()
1633            .build()
1634    }
1635
1636    fn builder_without_changes(format_version: FormatVersion) -> TableMetadataBuilder {
1637        TableMetadataBuilder::new(
1638            schema(),
1639            partition_spec(),
1640            sort_order(),
1641            TEST_LOCATION.to_string(),
1642            format_version,
1643            HashMap::new(),
1644        )
1645        .unwrap()
1646        .build()
1647        .unwrap()
1648        .metadata
1649        .into_builder(Some(
1650            "s3://bucket/test/location/metadata/metadata1.json".to_string(),
1651        ))
1652    }
1653
1654    #[test]
1655    fn test_minimal_build() {
1656        let metadata = TableMetadataBuilder::new(
1657            schema(),
1658            partition_spec(),
1659            sort_order(),
1660            TEST_LOCATION.to_string(),
1661            FormatVersion::V1,
1662            HashMap::new(),
1663        )
1664        .unwrap()
1665        .build()
1666        .unwrap()
1667        .metadata;
1668
1669        assert_eq!(metadata.format_version, FormatVersion::V1);
1670        assert_eq!(metadata.location, TEST_LOCATION);
1671        assert_eq!(metadata.current_schema_id, 0);
1672        assert_eq!(metadata.default_spec.spec_id(), 0);
1673        assert_eq!(metadata.default_sort_order_id, 1);
1674        assert_eq!(metadata.last_partition_id, 1000);
1675        assert_eq!(metadata.last_column_id, 3);
1676        assert_eq!(metadata.snapshots.len(), 0);
1677        assert_eq!(metadata.current_snapshot_id, None);
1678        assert_eq!(metadata.refs.len(), 0);
1679        assert_eq!(metadata.properties.len(), 0);
1680        assert_eq!(metadata.metadata_log.len(), 0);
1681        assert_eq!(metadata.last_sequence_number, 0);
1682        assert_eq!(metadata.last_column_id, LAST_ASSIGNED_COLUMN_ID);
1683
1684        // Test can serialize v1
1685        let _ = serde_json::to_string(&metadata).unwrap();
1686
1687        // Test can serialize v2
1688        let metadata = metadata
1689            .into_builder(Some(
1690                "s3://bucket/test/location/metadata/metadata1.json".to_string(),
1691            ))
1692            .upgrade_format_version(FormatVersion::V2)
1693            .unwrap()
1694            .build()
1695            .unwrap()
1696            .metadata;
1697
1698        assert_eq!(metadata.format_version, FormatVersion::V2);
1699        let _ = serde_json::to_string(&metadata).unwrap();
1700    }
1701
1702    #[test]
1703    fn test_build_unpartitioned_unsorted() {
1704        let schema = Schema::builder().build().unwrap();
1705        let metadata = TableMetadataBuilder::new(
1706            schema.clone(),
1707            PartitionSpec::unpartition_spec(),
1708            SortOrder::unsorted_order(),
1709            TEST_LOCATION.to_string(),
1710            FormatVersion::V2,
1711            HashMap::new(),
1712        )
1713        .unwrap()
1714        .build()
1715        .unwrap()
1716        .metadata;
1717
1718        assert_eq!(metadata.format_version, FormatVersion::V2);
1719        assert_eq!(metadata.location, TEST_LOCATION);
1720        assert_eq!(metadata.current_schema_id, 0);
1721        assert_eq!(metadata.default_spec.spec_id(), 0);
1722        assert_eq!(metadata.default_sort_order_id, 0);
1723        assert_eq!(metadata.last_partition_id, UNPARTITIONED_LAST_ASSIGNED_ID);
1724        assert_eq!(metadata.last_column_id, 0);
1725        assert_eq!(metadata.snapshots.len(), 0);
1726        assert_eq!(metadata.current_snapshot_id, None);
1727        assert_eq!(metadata.refs.len(), 0);
1728        assert_eq!(metadata.properties.len(), 0);
1729        assert_eq!(metadata.metadata_log.len(), 0);
1730        assert_eq!(metadata.last_sequence_number, 0);
1731    }
1732
1733    #[test]
1734    fn test_reassigns_ids() {
1735        let schema = Schema::builder()
1736            .with_schema_id(10)
1737            .with_fields(vec![
1738                NestedField::required(11, "a", Type::Primitive(PrimitiveType::Long)).into(),
1739                NestedField::required(12, "b", Type::Primitive(PrimitiveType::Long)).into(),
1740                NestedField::required(
1741                    13,
1742                    "struct",
1743                    Type::Struct(StructType::new(vec![
1744                        NestedField::required(14, "nested", Type::Primitive(PrimitiveType::Long))
1745                            .into(),
1746                    ])),
1747                )
1748                .into(),
1749                NestedField::required(15, "c", Type::Primitive(PrimitiveType::Long)).into(),
1750            ])
1751            .build()
1752            .unwrap();
1753        let spec = PartitionSpec::builder(schema.clone())
1754            .with_spec_id(20)
1755            .add_partition_field("a", "a", Transform::Identity)
1756            .unwrap()
1757            .add_partition_field("struct.nested", "nested_partition", Transform::Identity)
1758            .unwrap()
1759            .build()
1760            .unwrap();
1761        let sort_order = SortOrder::builder()
1762            .with_fields(vec![SortField {
1763                source_id: 11,
1764                transform: Transform::Identity,
1765                direction: SortDirection::Ascending,
1766                null_order: NullOrder::First,
1767            }])
1768            .with_order_id(10)
1769            .build(&schema)
1770            .unwrap();
1771
1772        let (fresh_schema, fresh_spec, fresh_sort_order) =
1773            TableMetadataBuilder::reassign_ids(schema, spec.into_unbound(), sort_order).unwrap();
1774
1775        let expected_schema = Schema::builder()
1776            .with_fields(vec![
1777                NestedField::required(1, "a", Type::Primitive(PrimitiveType::Long)).into(),
1778                NestedField::required(2, "b", Type::Primitive(PrimitiveType::Long)).into(),
1779                NestedField::required(
1780                    3,
1781                    "struct",
1782                    Type::Struct(StructType::new(vec![
1783                        NestedField::required(5, "nested", Type::Primitive(PrimitiveType::Long))
1784                            .into(),
1785                    ])),
1786                )
1787                .into(),
1788                NestedField::required(4, "c", Type::Primitive(PrimitiveType::Long)).into(),
1789            ])
1790            .build()
1791            .unwrap();
1792
1793        let expected_spec = PartitionSpec::builder(expected_schema.clone())
1794            .with_spec_id(0)
1795            .add_partition_field("a", "a", Transform::Identity)
1796            .unwrap()
1797            .add_partition_field("struct.nested", "nested_partition", Transform::Identity)
1798            .unwrap()
1799            .build()
1800            .unwrap();
1801
1802        let expected_sort_order = SortOrder::builder()
1803            .with_fields(vec![SortField {
1804                source_id: 1,
1805                transform: Transform::Identity,
1806                direction: SortDirection::Ascending,
1807                null_order: NullOrder::First,
1808            }])
1809            .with_order_id(1)
1810            .build(&expected_schema)
1811            .unwrap();
1812
1813        assert_eq!(fresh_schema, expected_schema);
1814        assert_eq!(fresh_spec, expected_spec);
1815        assert_eq!(fresh_sort_order, expected_sort_order);
1816    }
1817
1818    #[test]
1819    fn test_ids_are_reassigned_for_new_metadata() {
1820        let schema = schema().into_builder().with_schema_id(10).build().unwrap();
1821
1822        let metadata = TableMetadataBuilder::new(
1823            schema,
1824            partition_spec(),
1825            sort_order(),
1826            TEST_LOCATION.to_string(),
1827            FormatVersion::V1,
1828            HashMap::new(),
1829        )
1830        .unwrap()
1831        .build()
1832        .unwrap()
1833        .metadata;
1834
1835        assert_eq!(metadata.current_schema_id, 0);
1836        assert_eq!(metadata.current_schema().schema_id(), 0);
1837    }
1838
1839    #[test]
1840    fn test_new_metadata_changes() {
1841        let changes = TableMetadataBuilder::new(
1842            schema(),
1843            partition_spec(),
1844            sort_order(),
1845            TEST_LOCATION.to_string(),
1846            FormatVersion::V1,
1847            HashMap::from_iter(vec![("property 1".to_string(), "value 1".to_string())]),
1848        )
1849        .unwrap()
1850        .build()
1851        .unwrap()
1852        .changes;
1853
1854        pretty_assertions::assert_eq!(changes, vec![
1855            TableUpdate::SetLocation {
1856                location: TEST_LOCATION.to_string()
1857            },
1858            TableUpdate::AddSchema { schema: schema() },
1859            TableUpdate::SetCurrentSchema { schema_id: -1 },
1860            TableUpdate::AddSpec {
1861                // Because this is a new tables, field-ids are assigned
1862                // partition_spec() has None set for field-id
1863                spec: PartitionSpec::builder(schema())
1864                    .with_spec_id(0)
1865                    .add_unbound_field(
1866                        UnboundPartitionField::builder()
1867                            .source_ids(vec![2])
1868                            .field_id(1000)
1869                            .name("y".to_string())
1870                            .transform(Transform::Identity)
1871                            .build()
1872                            .unwrap()
1873                    )
1874                    .unwrap()
1875                    .build()
1876                    .unwrap()
1877                    .into_unbound(),
1878            },
1879            TableUpdate::SetDefaultSpec { spec_id: -1 },
1880            TableUpdate::AddSortOrder {
1881                sort_order: sort_order(),
1882            },
1883            TableUpdate::SetDefaultSortOrder { sort_order_id: -1 },
1884            TableUpdate::SetProperties {
1885                updates: HashMap::from_iter(vec![(
1886                    "property 1".to_string(),
1887                    "value 1".to_string()
1888                )]),
1889            }
1890        ]);
1891    }
1892
1893    #[test]
1894    fn test_new_metadata_changes_unpartitioned_unsorted() {
1895        let schema = Schema::builder().build().unwrap();
1896        let changes = TableMetadataBuilder::new(
1897            schema.clone(),
1898            PartitionSpec::unpartition_spec().into_unbound(),
1899            SortOrder::unsorted_order(),
1900            TEST_LOCATION.to_string(),
1901            FormatVersion::V1,
1902            HashMap::new(),
1903        )
1904        .unwrap()
1905        .build()
1906        .unwrap()
1907        .changes;
1908
1909        pretty_assertions::assert_eq!(changes, vec![
1910            TableUpdate::SetLocation {
1911                location: TEST_LOCATION.to_string()
1912            },
1913            TableUpdate::AddSchema {
1914                schema: Schema::builder().build().unwrap(),
1915            },
1916            TableUpdate::SetCurrentSchema { schema_id: -1 },
1917            TableUpdate::AddSpec {
1918                // Because this is a new tables, field-ids are assigned
1919                // partition_spec() has None set for field-id
1920                spec: PartitionSpec::builder(schema)
1921                    .with_spec_id(0)
1922                    .build()
1923                    .unwrap()
1924                    .into_unbound(),
1925            },
1926            TableUpdate::SetDefaultSpec { spec_id: -1 },
1927            TableUpdate::AddSortOrder {
1928                sort_order: SortOrder::unsorted_order(),
1929            },
1930            TableUpdate::SetDefaultSortOrder { sort_order_id: -1 },
1931        ]);
1932    }
1933
1934    #[test]
1935    fn test_add_partition_spec() {
1936        let builder = builder_without_changes(FormatVersion::V2);
1937
1938        let added_spec = UnboundPartitionSpec::builder()
1939            .with_spec_id(10)
1940            .add_partition_fields(vec![
1941                // The previous field - has field_id set
1942                UnboundPartitionField::builder()
1943                    .source_ids(vec![2])
1944                    .field_id(1000)
1945                    .name("y".to_string())
1946                    .transform(Transform::Identity)
1947                    .build()
1948                    .unwrap(),
1949                // A new field without field id - should still be without field id in changes
1950                UnboundPartitionField::builder()
1951                    .source_ids(vec![3])
1952                    .name("z".to_string())
1953                    .transform(Transform::Identity)
1954                    .build()
1955                    .unwrap(),
1956            ])
1957            .unwrap()
1958            .build();
1959
1960        let build_result = builder
1961            .add_partition_spec(added_spec.clone())
1962            .unwrap()
1963            .build()
1964            .unwrap();
1965
1966        // Spec id should be re-assigned
1967        let expected_change = added_spec.with_spec_id(1);
1968        let expected_spec = PartitionSpec::builder(schema())
1969            .with_spec_id(1)
1970            .add_unbound_field(
1971                UnboundPartitionField::builder()
1972                    .source_ids(vec![2])
1973                    .field_id(1000)
1974                    .name("y".to_string())
1975                    .transform(Transform::Identity)
1976                    .build()
1977                    .unwrap(),
1978            )
1979            .unwrap()
1980            .add_unbound_field(
1981                UnboundPartitionField::builder()
1982                    .source_ids(vec![3])
1983                    .field_id(1001)
1984                    .name("z".to_string())
1985                    .transform(Transform::Identity)
1986                    .build()
1987                    .unwrap(),
1988            )
1989            .unwrap()
1990            .build()
1991            .unwrap();
1992
1993        assert_eq!(build_result.changes.len(), 1);
1994        assert_eq!(
1995            build_result.metadata.partition_spec_by_id(1),
1996            Some(&Arc::new(expected_spec))
1997        );
1998        assert_eq!(build_result.metadata.default_spec.spec_id(), 0);
1999        assert_eq!(build_result.metadata.last_partition_id, 1001);
2000        pretty_assertions::assert_eq!(build_result.changes[0], TableUpdate::AddSpec {
2001            spec: expected_change
2002        });
2003
2004        // Remove the spec
2005        let build_result = build_result
2006            .metadata
2007            .into_builder(Some(
2008                "s3://bucket/test/location/metadata/metadata1.json".to_string(),
2009            ))
2010            .remove_partition_specs(&[1])
2011            .unwrap()
2012            .build()
2013            .unwrap();
2014
2015        assert_eq!(build_result.changes.len(), 1);
2016        assert_eq!(build_result.metadata.partition_specs.len(), 1);
2017        assert!(build_result.metadata.partition_spec_by_id(1).is_none());
2018    }
2019
2020    #[test]
2021    fn test_set_default_partition_spec() {
2022        let builder = builder_without_changes(FormatVersion::V2);
2023        let schema = builder.get_current_schema().unwrap().clone();
2024        let added_spec = UnboundPartitionSpec::builder()
2025            .with_spec_id(10)
2026            .add_partition_field(
2027                UnboundPartitionField::builder()
2028                    .source_ids(vec![1])
2029                    .name("y_bucket[2]")
2030                    .transform(Transform::Bucket(2))
2031                    .build()
2032                    .unwrap(),
2033            )
2034            .unwrap()
2035            .build();
2036
2037        let build_result = builder
2038            .add_partition_spec(added_spec.clone())
2039            .unwrap()
2040            .set_default_partition_spec(-1)
2041            .unwrap()
2042            .build()
2043            .unwrap();
2044
2045        let expected_spec = PartitionSpec::builder(schema)
2046            .with_spec_id(1)
2047            .add_unbound_field(
2048                UnboundPartitionField::builder()
2049                    .source_ids(vec![1])
2050                    .field_id(1001)
2051                    .name("y_bucket[2]".to_string())
2052                    .transform(Transform::Bucket(2))
2053                    .build()
2054                    .unwrap(),
2055            )
2056            .unwrap()
2057            .build()
2058            .unwrap();
2059
2060        assert_eq!(build_result.changes.len(), 2);
2061        assert_eq!(build_result.metadata.default_spec, Arc::new(expected_spec));
2062        assert_eq!(build_result.changes, vec![
2063            TableUpdate::AddSpec {
2064                // Should contain the actual ID that was used
2065                spec: added_spec.with_spec_id(1)
2066            },
2067            TableUpdate::SetDefaultSpec { spec_id: -1 }
2068        ]);
2069    }
2070
2071    #[test]
2072    fn test_set_existing_default_partition_spec() {
2073        let builder = builder_without_changes(FormatVersion::V2);
2074        // Add and set an unbound spec as current
2075        let unbound_spec = UnboundPartitionSpec::builder().with_spec_id(1).build();
2076        let build_result = builder
2077            .add_partition_spec(unbound_spec.clone())
2078            .unwrap()
2079            .set_default_partition_spec(-1)
2080            .unwrap()
2081            .build()
2082            .unwrap();
2083
2084        assert_eq!(build_result.changes.len(), 2);
2085        assert_eq!(build_result.changes[0], TableUpdate::AddSpec {
2086            spec: unbound_spec.clone()
2087        });
2088        assert_eq!(build_result.changes[1], TableUpdate::SetDefaultSpec {
2089            spec_id: -1
2090        });
2091        assert_eq!(
2092            build_result.metadata.default_spec,
2093            Arc::new(
2094                unbound_spec
2095                    .bind(build_result.metadata.current_schema().clone())
2096                    .unwrap()
2097            )
2098        );
2099
2100        // Set old spec again
2101        let build_result = build_result
2102            .metadata
2103            .into_builder(Some(
2104                "s3://bucket/test/location/metadata/metadata1.json".to_string(),
2105            ))
2106            .set_default_partition_spec(0)
2107            .unwrap()
2108            .build()
2109            .unwrap();
2110
2111        assert_eq!(build_result.changes.len(), 1);
2112        assert_eq!(build_result.changes[0], TableUpdate::SetDefaultSpec {
2113            spec_id: 0
2114        });
2115        assert_eq!(
2116            build_result.metadata.default_spec,
2117            Arc::new(
2118                partition_spec()
2119                    .bind(build_result.metadata.current_schema().clone())
2120                    .unwrap()
2121            )
2122        );
2123    }
2124
2125    #[test]
2126    fn test_add_sort_order() {
2127        let builder = builder_without_changes(FormatVersion::V2);
2128
2129        let added_sort_order = SortOrder::builder()
2130            .with_order_id(10)
2131            .with_fields(vec![SortField {
2132                source_id: 1,
2133                transform: Transform::Identity,
2134                direction: SortDirection::Ascending,
2135                null_order: NullOrder::First,
2136            }])
2137            .build(&schema())
2138            .unwrap();
2139
2140        let build_result = builder
2141            .add_sort_order(added_sort_order.clone())
2142            .unwrap()
2143            .build()
2144            .unwrap();
2145
2146        let expected_sort_order = added_sort_order.with_order_id(2);
2147
2148        assert_eq!(build_result.changes.len(), 1);
2149        assert_eq!(build_result.metadata.sort_orders.keys().max(), Some(&2));
2150        pretty_assertions::assert_eq!(
2151            build_result.metadata.sort_order_by_id(2),
2152            Some(&Arc::new(expected_sort_order.clone()))
2153        );
2154        pretty_assertions::assert_eq!(build_result.changes[0], TableUpdate::AddSortOrder {
2155            sort_order: expected_sort_order
2156        });
2157    }
2158
2159    #[test]
2160    fn test_add_compatible_schema() {
2161        let builder = builder_without_changes(FormatVersion::V2);
2162
2163        let added_schema = Schema::builder()
2164            .with_schema_id(1)
2165            .with_fields(vec![
2166                NestedField::required(1, "x", Type::Primitive(PrimitiveType::Long)).into(),
2167                NestedField::required(2, "y", Type::Primitive(PrimitiveType::Long)).into(),
2168                NestedField::required(3, "z", Type::Primitive(PrimitiveType::Long)).into(),
2169                NestedField::required(4, "a", Type::Primitive(PrimitiveType::Long)).into(),
2170            ])
2171            .build()
2172            .unwrap();
2173
2174        let build_result = builder
2175            .add_current_schema(added_schema.clone())
2176            .unwrap()
2177            .build()
2178            .unwrap();
2179
2180        assert_eq!(build_result.changes.len(), 2);
2181        assert_eq!(build_result.metadata.schemas.keys().max(), Some(&1));
2182        pretty_assertions::assert_eq!(
2183            build_result.metadata.schema_by_id(1),
2184            Some(&Arc::new(added_schema.clone()))
2185        );
2186        pretty_assertions::assert_eq!(build_result.changes[0], TableUpdate::AddSchema {
2187            schema: added_schema
2188        });
2189        assert_eq!(build_result.changes[1], TableUpdate::SetCurrentSchema {
2190            schema_id: -1
2191        });
2192    }
2193
2194    #[test]
2195    fn test_set_current_schema_change_is_minus_one_if_schema_was_added_in_this_change() {
2196        let builder = builder_without_changes(FormatVersion::V2);
2197
2198        let added_schema = Schema::builder()
2199            .with_schema_id(1)
2200            .with_fields(vec![
2201                NestedField::required(1, "x", Type::Primitive(PrimitiveType::Long)).into(),
2202                NestedField::required(2, "y", Type::Primitive(PrimitiveType::Long)).into(),
2203                NestedField::required(3, "z", Type::Primitive(PrimitiveType::Long)).into(),
2204                NestedField::required(4, "a", Type::Primitive(PrimitiveType::Long)).into(),
2205            ])
2206            .build()
2207            .unwrap();
2208
2209        let build_result = builder
2210            .add_schema(added_schema.clone())
2211            .unwrap()
2212            .set_current_schema(1)
2213            .unwrap()
2214            .build()
2215            .unwrap();
2216
2217        assert_eq!(build_result.changes.len(), 2);
2218        assert_eq!(build_result.changes[1], TableUpdate::SetCurrentSchema {
2219            schema_id: -1
2220        });
2221    }
2222
2223    #[test]
2224    fn test_no_metadata_log_for_create_table() {
2225        let build_result = TableMetadataBuilder::new(
2226            schema(),
2227            partition_spec(),
2228            sort_order(),
2229            TEST_LOCATION.to_string(),
2230            FormatVersion::V2,
2231            HashMap::new(),
2232        )
2233        .unwrap()
2234        .build()
2235        .unwrap();
2236
2237        assert_eq!(build_result.metadata.metadata_log.len(), 0);
2238    }
2239
2240    const KEY_ID: (&str, &str) = (TableProperties::PROPERTY_ENCRYPTION_KEY_ID, "master-1");
2241    const KEY_LENGTH: (&str, &str) = (TableProperties::PROPERTY_ENCRYPTION_DATA_KEY_LENGTH, "16");
2242
2243    fn string_map(entries: &[(&str, &str)]) -> HashMap<String, String> {
2244        entries
2245            .iter()
2246            .map(|(key, value)| (key.to_string(), value.to_string()))
2247            .collect()
2248    }
2249
2250    fn build_error(builder: TableMetadataBuilder) -> (ErrorKind, String) {
2251        let err = builder.build().unwrap_err();
2252        (err.kind(), err.message().to_string())
2253    }
2254
2255    #[test]
2256    fn test_build_rejects_encryption_properties_before_v3() {
2257        for version in [FormatVersion::V1, FormatVersion::V2] {
2258            for (properties, names) in [
2259                (vec![KEY_ID], "encryption.key-id"),
2260                (vec![KEY_LENGTH], "encryption.data-key-length"),
2261                (
2262                    vec![KEY_ID, KEY_LENGTH],
2263                    "encryption.key-id, encryption.data-key-length",
2264                ),
2265            ] {
2266                let builder = builder_without_changes(version)
2267                    .set_properties(string_map(&properties))
2268                    .unwrap();
2269                assert_eq!(
2270                    build_error(builder),
2271                    (
2272                        ErrorKind::DataInvalid,
2273                        format!("Invalid properties for {version}: [{names}]")
2274                    )
2275                );
2276            }
2277        }
2278    }
2279
2280    #[test]
2281    fn test_build_rejects_encryption_keys_before_v3() {
2282        let key = EncryptedKey::builder()
2283            .key_id("key-1")
2284            .encrypted_key_metadata(vec![1, 2, 3, 4])
2285            .build();
2286
2287        for version in [FormatVersion::V1, FormatVersion::V2] {
2288            let builder = builder_without_changes(version).add_encryption_key(key.clone());
2289            assert_eq!(
2290                build_error(builder),
2291                (
2292                    ErrorKind::DataInvalid,
2293                    format!("Encryption keys are not supported for {version} tables")
2294                )
2295            );
2296        }
2297    }
2298
2299    #[test]
2300    fn test_build_accepts_encryption_properties_on_v3() {
2301        let properties = string_map(&[KEY_ID, KEY_LENGTH]);
2302
2303        let metadata = builder_without_changes(FormatVersion::V3)
2304            .set_properties(properties.clone())
2305            .unwrap()
2306            .build()
2307            .unwrap()
2308            .metadata;
2309
2310        assert_eq!(metadata.properties, properties);
2311    }
2312
2313    #[test]
2314    fn test_pre_v3_encryption_property_blocks_updates_until_removed() {
2315        let legacy = || {
2316            let mut builder = builder_without_changes(FormatVersion::V2);
2317            builder.metadata.properties = string_map(&[KEY_ID]);
2318            builder
2319        };
2320
2321        let other_update = legacy()
2322            .set_properties(string_map(&[("key", "value")]))
2323            .unwrap();
2324        assert_eq!(
2325            build_error(other_update),
2326            (
2327                ErrorKind::DataInvalid,
2328                "Invalid properties for v2: [encryption.key-id]".to_string()
2329            )
2330        );
2331
2332        let repaired = legacy()
2333            .remove_properties(&[KEY_ID.0.to_string()])
2334            .unwrap()
2335            .build()
2336            .unwrap();
2337        assert_eq!(repaired.metadata.properties, HashMap::new());
2338    }
2339
2340    #[test]
2341    fn test_table_properties_view_reflects_metadata_updates() {
2342        let property = TableProperties::PROPERTY_COMMIT_NUM_RETRIES.to_string();
2343        let metadata = builder_without_changes(FormatVersion::V2)
2344            .set_properties(HashMap::from([(property.clone(), "7".to_string())]))
2345            .unwrap()
2346            .build()
2347            .unwrap()
2348            .metadata;
2349
2350        assert_eq!(metadata.table_properties().commit_num_retries().unwrap(), 7);
2351
2352        let metadata = metadata
2353            .into_builder(None)
2354            .remove_properties(&[property])
2355            .unwrap()
2356            .build()
2357            .unwrap()
2358            .metadata;
2359
2360        assert_eq!(
2361            metadata.table_properties().commit_num_retries().unwrap(),
2362            TableProperties::PROPERTY_COMMIT_NUM_RETRIES_DEFAULT
2363        );
2364    }
2365
2366    #[test]
2367    fn test_no_metadata_log_entry_for_no_previous_location() {
2368        // Used for first commit after stage-creation of tables
2369        let metadata = builder_without_changes(FormatVersion::V2)
2370            .build()
2371            .unwrap()
2372            .metadata;
2373        assert_eq!(metadata.metadata_log.len(), 1);
2374
2375        let build_result = metadata
2376            .into_builder(None)
2377            .set_properties(HashMap::from_iter(vec![(
2378                "foo".to_string(),
2379                "bar".to_string(),
2380            )]))
2381            .unwrap()
2382            .build()
2383            .unwrap();
2384
2385        assert_eq!(build_result.metadata.metadata_log.len(), 1);
2386    }
2387
2388    #[test]
2389    fn test_from_metadata_generates_metadata_log() {
2390        let metadata_path = "s3://bucket/test/location/metadata/metadata1.json";
2391        let builder = TableMetadataBuilder::new(
2392            schema(),
2393            partition_spec(),
2394            sort_order(),
2395            TEST_LOCATION.to_string(),
2396            FormatVersion::V2,
2397            HashMap::new(),
2398        )
2399        .unwrap()
2400        .build()
2401        .unwrap()
2402        .metadata
2403        .into_builder(Some(metadata_path.to_string()));
2404
2405        let builder = builder
2406            .add_default_sort_order(SortOrder::unsorted_order())
2407            .unwrap();
2408
2409        let build_result = builder.build().unwrap();
2410
2411        assert_eq!(build_result.metadata.metadata_log.len(), 1);
2412        assert_eq!(
2413            build_result.metadata.metadata_log[0].metadata_file,
2414            metadata_path
2415        );
2416    }
2417
2418    #[test]
2419    fn test_set_ref() {
2420        let builder = builder_without_changes(FormatVersion::V2);
2421
2422        let snapshot = Snapshot::builder()
2423            .with_snapshot_id(1)
2424            .with_timestamp_ms(builder.metadata.last_updated_ms + 1)
2425            .with_sequence_number(0)
2426            .with_schema_id(0)
2427            .with_manifest_list("/snap-1.avro")
2428            .with_summary(Summary {
2429                operation: Operation::Append,
2430                additional_properties: HashMap::from_iter(vec![
2431                    (
2432                        "spark.app.id".to_string(),
2433                        "local-1662532784305".to_string(),
2434                    ),
2435                    ("added-data-files".to_string(), "4".to_string()),
2436                    ("added-records".to_string(), "4".to_string()),
2437                    ("added-files-size".to_string(), "6001".to_string()),
2438                ]),
2439            })
2440            .build();
2441
2442        let builder = builder.add_snapshot(snapshot.clone()).unwrap();
2443
2444        assert!(
2445            builder
2446                .clone()
2447                .set_ref(MAIN_BRANCH, SnapshotReference {
2448                    snapshot_id: 10,
2449                    retention: SnapshotRetention::Branch {
2450                        min_snapshots_to_keep: Some(10),
2451                        max_snapshot_age_ms: None,
2452                        max_ref_age_ms: None,
2453                    },
2454                })
2455                .unwrap_err()
2456                .to_string()
2457                .contains("Cannot set 'main' to unknown snapshot: '10'")
2458        );
2459
2460        let build_result = builder
2461            .set_ref(MAIN_BRANCH, SnapshotReference {
2462                snapshot_id: 1,
2463                retention: SnapshotRetention::Branch {
2464                    min_snapshots_to_keep: Some(10),
2465                    max_snapshot_age_ms: None,
2466                    max_ref_age_ms: None,
2467                },
2468            })
2469            .unwrap()
2470            .build()
2471            .unwrap();
2472        assert_eq!(build_result.metadata.snapshots.len(), 1);
2473        assert_eq!(
2474            build_result.metadata.snapshot_by_id(1),
2475            Some(&Arc::new(snapshot.clone()))
2476        );
2477        assert_eq!(build_result.metadata.snapshot_log, vec![SnapshotLog {
2478            snapshot_id: 1,
2479            timestamp_ms: snapshot.timestamp_ms()
2480        }])
2481    }
2482
2483    #[test]
2484    fn test_snapshot_log_skips_intermediates() {
2485        let builder = builder_without_changes(FormatVersion::V2);
2486
2487        let snapshot_1 = Snapshot::builder()
2488            .with_snapshot_id(1)
2489            .with_timestamp_ms(builder.metadata.last_updated_ms + 1)
2490            .with_sequence_number(0)
2491            .with_schema_id(0)
2492            .with_manifest_list("/snap-1.avro")
2493            .with_summary(Summary {
2494                operation: Operation::Append,
2495                additional_properties: HashMap::from_iter(vec![
2496                    (
2497                        "spark.app.id".to_string(),
2498                        "local-1662532784305".to_string(),
2499                    ),
2500                    ("added-data-files".to_string(), "4".to_string()),
2501                    ("added-records".to_string(), "4".to_string()),
2502                    ("added-files-size".to_string(), "6001".to_string()),
2503                ]),
2504            })
2505            .build();
2506
2507        let snapshot_2 = Snapshot::builder()
2508            .with_snapshot_id(2)
2509            .with_timestamp_ms(builder.metadata.last_updated_ms + 1)
2510            .with_sequence_number(0)
2511            .with_schema_id(0)
2512            .with_manifest_list("/snap-1.avro")
2513            .with_summary(Summary {
2514                operation: Operation::Append,
2515                additional_properties: HashMap::from_iter(vec![
2516                    (
2517                        "spark.app.id".to_string(),
2518                        "local-1662532784305".to_string(),
2519                    ),
2520                    ("added-data-files".to_string(), "4".to_string()),
2521                    ("added-records".to_string(), "4".to_string()),
2522                    ("added-files-size".to_string(), "6001".to_string()),
2523                ]),
2524            })
2525            .build();
2526
2527        let result = builder
2528            .add_snapshot(snapshot_1)
2529            .unwrap()
2530            .set_ref(MAIN_BRANCH, SnapshotReference {
2531                snapshot_id: 1,
2532                retention: SnapshotRetention::Branch {
2533                    min_snapshots_to_keep: Some(10),
2534                    max_snapshot_age_ms: None,
2535                    max_ref_age_ms: None,
2536                },
2537            })
2538            .unwrap()
2539            .set_branch_snapshot(snapshot_2.clone(), MAIN_BRANCH)
2540            .unwrap()
2541            .build()
2542            .unwrap();
2543
2544        assert_eq!(result.metadata.snapshot_log, vec![SnapshotLog {
2545            snapshot_id: 2,
2546            timestamp_ms: snapshot_2.timestamp_ms()
2547        }]);
2548        assert_eq!(result.metadata.current_snapshot().unwrap().snapshot_id(), 2);
2549    }
2550
2551    #[test]
2552    fn test_remove_main_ref_keeps_snapshot_log() {
2553        let builder = builder_without_changes(FormatVersion::V2);
2554
2555        let snapshot = Snapshot::builder()
2556            .with_snapshot_id(1)
2557            .with_timestamp_ms(builder.metadata.last_updated_ms + 1)
2558            .with_sequence_number(0)
2559            .with_schema_id(0)
2560            .with_manifest_list("/snap-1.avro")
2561            .with_summary(Summary {
2562                operation: Operation::Append,
2563                additional_properties: HashMap::from_iter(vec![
2564                    (
2565                        "spark.app.id".to_string(),
2566                        "local-1662532784305".to_string(),
2567                    ),
2568                    ("added-data-files".to_string(), "4".to_string()),
2569                    ("added-records".to_string(), "4".to_string()),
2570                    ("added-files-size".to_string(), "6001".to_string()),
2571                ]),
2572            })
2573            .build();
2574
2575        let result = builder
2576            .add_snapshot(snapshot.clone())
2577            .unwrap()
2578            .set_ref(MAIN_BRANCH, SnapshotReference {
2579                snapshot_id: 1,
2580                retention: SnapshotRetention::Branch {
2581                    min_snapshots_to_keep: Some(10),
2582                    max_snapshot_age_ms: None,
2583                    max_ref_age_ms: None,
2584                },
2585            })
2586            .unwrap()
2587            .build()
2588            .unwrap();
2589
2590        // Verify snapshot log was created
2591        assert_eq!(result.metadata.snapshot_log.len(), 1);
2592        assert_eq!(result.metadata.snapshot_log[0].snapshot_id, 1);
2593        assert_eq!(result.metadata.current_snapshot_id, Some(1));
2594
2595        // Remove the main ref
2596        let result_after_remove = result
2597            .metadata
2598            .into_builder(Some(
2599                "s3://bucket/test/location/metadata/metadata2.json".to_string(),
2600            ))
2601            .remove_ref(MAIN_BRANCH)
2602            .build()
2603            .unwrap();
2604
2605        // Verify snapshot log is kept even after removing main ref
2606        assert_eq!(result_after_remove.metadata.snapshot_log.len(), 1);
2607        assert_eq!(result_after_remove.metadata.snapshot_log[0].snapshot_id, 1);
2608        assert_eq!(result_after_remove.metadata.current_snapshot_id, None);
2609        assert_eq!(result_after_remove.changes.len(), 1);
2610        assert_eq!(
2611            result_after_remove.changes[0],
2612            TableUpdate::RemoveSnapshotRef {
2613                ref_name: MAIN_BRANCH.to_string()
2614            }
2615        );
2616    }
2617
2618    #[test]
2619    fn test_set_branch_snapshot_creates_branch_if_not_exists() {
2620        let builder = builder_without_changes(FormatVersion::V2);
2621
2622        let snapshot = Snapshot::builder()
2623            .with_snapshot_id(2)
2624            .with_timestamp_ms(builder.metadata.last_updated_ms + 1)
2625            .with_sequence_number(0)
2626            .with_schema_id(0)
2627            .with_manifest_list("/snap-1.avro")
2628            .with_summary(Summary {
2629                operation: Operation::Append,
2630                additional_properties: HashMap::new(),
2631            })
2632            .build();
2633
2634        let build_result = builder
2635            .set_branch_snapshot(snapshot.clone(), "new_branch")
2636            .unwrap()
2637            .build()
2638            .unwrap();
2639
2640        let reference = SnapshotReference {
2641            snapshot_id: 2,
2642            retention: SnapshotRetention::Branch {
2643                min_snapshots_to_keep: None,
2644                max_snapshot_age_ms: None,
2645                max_ref_age_ms: None,
2646            },
2647        };
2648
2649        assert_eq!(build_result.metadata.refs.len(), 1);
2650        assert_eq!(
2651            build_result.metadata.refs.get("new_branch"),
2652            Some(&reference)
2653        );
2654        assert_eq!(build_result.changes, vec![
2655            TableUpdate::AddSnapshot { snapshot },
2656            TableUpdate::SetSnapshotRef {
2657                ref_name: "new_branch".to_string(),
2658                reference
2659            }
2660        ]);
2661    }
2662
2663    /// Builds metadata with one snapshot per id and the given refs, as committed metadata
2664    /// without pending changes.
2665    fn metadata_with_snapshots_and_refs(
2666        snapshot_ids: &[i64],
2667        refs: &[(&str, SnapshotReference)],
2668    ) -> TableMetadata {
2669        let mut builder = builder_without_changes(FormatVersion::V2);
2670        let timestamp_ms = builder.metadata.last_updated_ms + 1;
2671        for (i, snapshot_id) in snapshot_ids.iter().enumerate() {
2672            let snapshot = Snapshot::builder()
2673                .with_snapshot_id(*snapshot_id)
2674                .with_timestamp_ms(timestamp_ms + i as i64)
2675                .with_sequence_number(i as i64 + 1)
2676                .with_schema_id(0)
2677                .with_manifest_list(format!("/snap-{snapshot_id}.avro"))
2678                .with_summary(Summary {
2679                    operation: Operation::Append,
2680                    additional_properties: HashMap::new(),
2681                })
2682                .build();
2683            builder = builder.add_snapshot(snapshot).unwrap();
2684        }
2685        for (ref_name, reference) in refs {
2686            builder = builder.set_ref(ref_name, reference.clone()).unwrap();
2687        }
2688        builder.build().unwrap().metadata
2689    }
2690
2691    fn branch_ref(snapshot_id: i64) -> SnapshotReference {
2692        SnapshotReference {
2693            snapshot_id,
2694            retention: SnapshotRetention::Branch {
2695                min_snapshots_to_keep: None,
2696                max_snapshot_age_ms: None,
2697                max_ref_age_ms: None,
2698            },
2699        }
2700    }
2701
2702    fn tag_ref(snapshot_id: i64) -> SnapshotReference {
2703        SnapshotReference {
2704            snapshot_id,
2705            retention: SnapshotRetention::Tag {
2706                max_ref_age_ms: None,
2707            },
2708        }
2709    }
2710
2711    #[test]
2712    fn test_remove_snapshots_records_removal_of_refs_to_removed_snapshots() {
2713        let metadata = metadata_with_snapshots_and_refs(&[1, 2], &[
2714            (MAIN_BRANCH, branch_ref(2)),
2715            ("branch-on-1", branch_ref(1)),
2716            ("tag-on-1", tag_ref(1)),
2717            ("branch-on-2", branch_ref(2)),
2718        ]);
2719
2720        let result = metadata
2721            .into_builder(Some(
2722                "s3://bucket/test/location/metadata/metadata2.json".to_string(),
2723            ))
2724            .remove_snapshots(&[1])
2725            .build()
2726            .unwrap();
2727
2728        assert_eq!(result.changes, vec![
2729            TableUpdate::RemoveSnapshots {
2730                snapshot_ids: vec![1]
2731            },
2732            TableUpdate::RemoveSnapshotRef {
2733                ref_name: "branch-on-1".to_string()
2734            },
2735            TableUpdate::RemoveSnapshotRef {
2736                ref_name: "tag-on-1".to_string()
2737            },
2738        ]);
2739        assert_eq!(
2740            result.metadata.refs,
2741            HashMap::from([
2742                (MAIN_BRANCH.to_string(), branch_ref(2)),
2743                ("branch-on-2".to_string(), branch_ref(2)),
2744            ])
2745        );
2746        assert_eq!(result.metadata.current_snapshot_id, Some(2));
2747    }
2748
2749    #[test]
2750    fn test_remove_snapshots_clears_current_snapshot_when_main_snapshot_removed() {
2751        let metadata = metadata_with_snapshots_and_refs(&[1, 2], &[
2752            (MAIN_BRANCH, branch_ref(2)),
2753            ("branch-on-1", branch_ref(1)),
2754        ]);
2755
2756        let result = metadata
2757            .into_builder(Some(
2758                "s3://bucket/test/location/metadata/metadata2.json".to_string(),
2759            ))
2760            .remove_snapshots(&[2])
2761            .build()
2762            .unwrap();
2763
2764        assert_eq!(result.changes, vec![
2765            TableUpdate::RemoveSnapshots {
2766                snapshot_ids: vec![2]
2767            },
2768            TableUpdate::RemoveSnapshotRef {
2769                ref_name: MAIN_BRANCH.to_string()
2770            },
2771        ]);
2772        assert_eq!(result.metadata.current_snapshot_id, None);
2773        assert_eq!(
2774            result.metadata.refs,
2775            HashMap::from([("branch-on-1".to_string(), branch_ref(1))])
2776        );
2777        assert!(result.metadata.snapshot_log.is_empty());
2778    }
2779
2780    #[test]
2781    fn test_remove_snapshots_keeps_refs_to_retained_snapshots() {
2782        let metadata = metadata_with_snapshots_and_refs(&[1, 2, 3], &[
2783            (MAIN_BRANCH, branch_ref(3)),
2784            ("branch-on-2", branch_ref(2)),
2785            ("tag-on-3", tag_ref(3)),
2786        ]);
2787        let refs_before = metadata.refs.clone();
2788
2789        let result = metadata
2790            .into_builder(Some(
2791                "s3://bucket/test/location/metadata/metadata2.json".to_string(),
2792            ))
2793            .remove_snapshots(&[1])
2794            .build()
2795            .unwrap();
2796
2797        assert_eq!(result.changes, vec![TableUpdate::RemoveSnapshots {
2798            snapshot_ids: vec![1]
2799        }]);
2800        assert_eq!(result.metadata.refs, refs_before);
2801        assert_eq!(result.metadata.current_snapshot_id, Some(3));
2802    }
2803
2804    #[test]
2805    fn test_cannot_add_duplicate_snapshot_id() {
2806        let builder = builder_without_changes(FormatVersion::V2);
2807
2808        let snapshot = Snapshot::builder()
2809            .with_snapshot_id(2)
2810            .with_timestamp_ms(builder.metadata.last_updated_ms + 1)
2811            .with_sequence_number(0)
2812            .with_schema_id(0)
2813            .with_manifest_list("/snap-1.avro")
2814            .with_summary(Summary {
2815                operation: Operation::Append,
2816                additional_properties: HashMap::from_iter(vec![
2817                    (
2818                        "spark.app.id".to_string(),
2819                        "local-1662532784305".to_string(),
2820                    ),
2821                    ("added-data-files".to_string(), "4".to_string()),
2822                    ("added-records".to_string(), "4".to_string()),
2823                    ("added-files-size".to_string(), "6001".to_string()),
2824                ]),
2825            })
2826            .build();
2827
2828        let builder = builder.add_snapshot(snapshot.clone()).unwrap();
2829        builder.add_snapshot(snapshot).unwrap_err();
2830    }
2831
2832    #[test]
2833    fn test_remove_snapshots_records_existing_ids_once() {
2834        let fixture = TableTestFixture::new_with_deep_history();
2835        let metadata = fixture.table.metadata().clone();
2836        let first_id = metadata.snapshot_log[0].snapshot_id;
2837        let third_id = metadata.snapshot_log[2].snapshot_id;
2838        let mut expected_metadata = metadata.clone();
2839        expected_metadata.snapshots = [
2840            3055729675574597004,
2841            3057729675574597004,
2842            3059729675574597004,
2843        ]
2844        .into_iter()
2845        .map(|id| (id, metadata.snapshot_by_id(id).unwrap().clone()))
2846        .collect();
2847        expected_metadata.snapshot_log = metadata.snapshot_log[3..].to_vec();
2848        let expected = TableMetadataBuildResult {
2849            metadata: expected_metadata,
2850            changes: vec![TableUpdate::RemoveSnapshots {
2851                snapshot_ids: vec![third_id, first_id],
2852            }],
2853            expired_metadata_logs: vec![],
2854        };
2855        let mut builder = metadata.into_builder(None);
2856        builder.last_updated_ms = Some(expected.metadata.last_updated_ms);
2857
2858        let result = builder
2859            .remove_snapshots(&[third_id, 0, first_id, third_id, 0, first_id])
2860            .build()
2861            .unwrap();
2862
2863        assert_eq!(result, expected);
2864    }
2865
2866    #[test]
2867    fn test_add_incompatible_current_schema_fails() {
2868        let builder = builder_without_changes(FormatVersion::V2);
2869
2870        let added_schema = Schema::builder()
2871            .with_schema_id(1)
2872            .with_fields(vec![])
2873            .build()
2874            .unwrap();
2875
2876        let err = builder
2877            .add_current_schema(added_schema)
2878            .unwrap()
2879            .build()
2880            .unwrap_err();
2881
2882        assert!(
2883            err.to_string()
2884                .contains("Cannot find partition source field")
2885        );
2886    }
2887
2888    #[test]
2889    fn test_add_partition_spec_for_v1_requires_sequential_ids() {
2890        let builder = builder_without_changes(FormatVersion::V1);
2891
2892        let added_spec = UnboundPartitionSpec::builder()
2893            .with_spec_id(10)
2894            .add_partition_fields(vec![
2895                UnboundPartitionField::builder()
2896                    .source_ids(vec![2])
2897                    .field_id(1000)
2898                    .name("y".to_string())
2899                    .transform(Transform::Identity)
2900                    .build()
2901                    .unwrap(),
2902                UnboundPartitionField::builder()
2903                    .source_ids(vec![3])
2904                    .field_id(1002)
2905                    .name("z".to_string())
2906                    .transform(Transform::Identity)
2907                    .build()
2908                    .unwrap(),
2909            ])
2910            .unwrap()
2911            .build();
2912
2913        let err = builder.add_partition_spec(added_spec).unwrap_err();
2914        assert!(err.to_string().contains(
2915            "Cannot add partition spec with non-sequential field ids to format version 1 table"
2916        ));
2917    }
2918
2919    #[test]
2920    fn test_add_partition_spec_accepts_void_fields_of_removed_fields() {
2921        let builder = builder_without_changes(FormatVersion::V1);
2922        let void_field = |field_id: i32, name: &str| {
2923            UnboundPartitionField::builder()
2924                .source_ids(vec![2])
2925                .field_id(field_id)
2926                .name(name)
2927                .transform(Transform::Void)
2928                .build()
2929                .unwrap()
2930        };
2931        let added_spec = UnboundPartitionSpec {
2932            spec_id: None,
2933            fields: vec![void_field(1000, "y"), void_field(1001, "y_null")],
2934        };
2935
2936        let metadata = builder
2937            .add_default_partition_spec(added_spec.clone())
2938            .unwrap()
2939            .build()
2940            .unwrap()
2941            .metadata;
2942
2943        let expected = added_spec.with_spec_id(1).bind(schema()).unwrap();
2944        assert_eq!(metadata.default_partition_spec().as_ref(), &expected);
2945        assert_eq!(metadata.last_partition_id, 1001);
2946    }
2947
2948    #[test]
2949    fn test_add_partition_spec_accepts_two_time_granularities_of_one_column() {
2950        let schema = Schema::builder()
2951            .with_fields(vec![
2952                NestedField::required(1, "ts", Type::Primitive(PrimitiveType::Timestamptz)).into(),
2953            ])
2954            .build()
2955            .unwrap();
2956        let time_field = |field_id: i32, name: &str, transform: Transform| {
2957            UnboundPartitionField::builder()
2958                .source_ids(vec![1])
2959                .field_id(field_id)
2960                .name(name)
2961                .transform(transform)
2962                .build()
2963                .unwrap()
2964        };
2965        let added_spec = UnboundPartitionSpec {
2966            spec_id: None,
2967            fields: vec![
2968                time_field(1000, "ts_day", Transform::Day),
2969                time_field(1001, "ts_hour", Transform::Hour),
2970            ],
2971        };
2972        let metadata = TableMetadataBuilder::new(
2973            schema.clone(),
2974            PartitionSpec::unpartition_spec(),
2975            SortOrder::unsorted_order(),
2976            TEST_LOCATION.to_string(),
2977            FormatVersion::V2,
2978            HashMap::new(),
2979        )
2980        .unwrap()
2981        .build()
2982        .unwrap()
2983        .metadata;
2984
2985        let metadata = metadata
2986            .into_builder(None)
2987            .add_default_partition_spec(added_spec.clone())
2988            .unwrap()
2989            .build()
2990            .unwrap()
2991            .metadata;
2992        // Every later commit re-validates the default spec.
2993        let metadata = metadata
2994            .into_builder(None)
2995            .set_properties(HashMap::from([("key".to_string(), "value".to_string())]))
2996            .unwrap()
2997            .build()
2998            .unwrap()
2999            .metadata;
3000
3001        let expected = added_spec.with_spec_id(1).bind(schema).unwrap();
3002        assert_eq!(metadata.default_partition_spec().as_ref(), &expected);
3003    }
3004
3005    /// Adds a branch snapshot from a writer whose clock is `skew_ms` ahead, then changes a
3006    /// property, which takes the current time. Returns the snapshot time and the second build.
3007    fn build_after_fast_branch_snapshot(skew_ms: i64) -> (i64, Result<TableMetadataBuildResult>) {
3008        let builder = builder_without_changes(FormatVersion::V2);
3009        let fast = builder.metadata.last_updated_ms + skew_ms;
3010        let snapshot = Snapshot::builder()
3011            .with_snapshot_id(2)
3012            .with_timestamp_ms(fast)
3013            .with_sequence_number(1)
3014            .with_schema_id(0)
3015            .with_manifest_list("/snap-2.avro")
3016            .with_summary(Summary {
3017                operation: Operation::Append,
3018                additional_properties: HashMap::new(),
3019            })
3020            .build();
3021        let on_branch = builder
3022            .set_branch_snapshot(snapshot, "b")
3023            .unwrap()
3024            .build()
3025            .unwrap()
3026            .metadata;
3027        assert_eq!(on_branch.last_updated_ms, fast);
3028
3029        let result = on_branch
3030            .into_builder(Some(
3031                "s3://bucket/test/location/metadata/metadata2.json".to_string(),
3032            ))
3033            .set_properties(HashMap::from([("k".to_string(), "v".to_string())]))
3034            .unwrap()
3035            .build();
3036        (fast, result)
3037    }
3038
3039    #[test]
3040    fn test_metadata_change_after_a_fast_branch_snapshot_is_refused_in_build() {
3041        // The change is two hours before the entry for the previous file. No reader could load
3042        // the result, so the build fails.
3043        let (fast, result) = build_after_fast_branch_snapshot(2 * 60 * 60_000);
3044        let err = result.unwrap_err();
3045        assert_eq!(err.kind(), ErrorKind::DataInvalid);
3046        assert!(
3047            err.message()
3048                .ends_with(&format!(": before last metadata log entry at {fast}")),
3049            "{}",
3050            err.message()
3051        );
3052    }
3053
3054    #[test]
3055    fn test_metadata_change_after_a_slightly_fast_branch_snapshot_builds() {
3056        // 30 seconds of clock skew is within the one-minute tolerance.
3057        let (fast, result) = build_after_fast_branch_snapshot(30_000);
3058        assert_eq!(
3059            result.unwrap().metadata.metadata_log.last(),
3060            Some(&MetadataLog {
3061                metadata_file: "s3://bucket/test/location/metadata/metadata2.json".to_string(),
3062                timestamp_ms: fast,
3063            })
3064        );
3065    }
3066
3067    #[test]
3068    fn test_expire_metadata_log() {
3069        let builder = builder_without_changes(FormatVersion::V2);
3070        let metadata = builder
3071            .set_properties(HashMap::from_iter(vec![(
3072                TableProperties::PROPERTY_METADATA_PREVIOUS_VERSIONS_MAX.to_string(),
3073                "2".to_string(),
3074            )]))
3075            .unwrap()
3076            .build()
3077            .unwrap();
3078        assert_eq!(metadata.metadata.metadata_log.len(), 1);
3079        assert_eq!(metadata.expired_metadata_logs.len(), 0);
3080
3081        let metadata = metadata
3082            .metadata
3083            .into_builder(Some("path2".to_string()))
3084            .set_properties(HashMap::from_iter(vec![(
3085                "change_nr".to_string(),
3086                "1".to_string(),
3087            )]))
3088            .unwrap()
3089            .build()
3090            .unwrap();
3091
3092        assert_eq!(metadata.metadata.metadata_log.len(), 2);
3093        assert_eq!(metadata.expired_metadata_logs.len(), 0);
3094
3095        let metadata = metadata
3096            .metadata
3097            .into_builder(Some("path2".to_string()))
3098            .set_properties(HashMap::from_iter(vec![(
3099                "change_nr".to_string(),
3100                "2".to_string(),
3101            )]))
3102            .unwrap()
3103            .build()
3104            .unwrap();
3105        assert_eq!(metadata.metadata.metadata_log.len(), 2);
3106        assert_eq!(metadata.expired_metadata_logs.len(), 1);
3107    }
3108
3109    #[test]
3110    fn test_v2_sequence_number_cannot_decrease() {
3111        let builder = builder_without_changes(FormatVersion::V2);
3112
3113        let snapshot = Snapshot::builder()
3114            .with_snapshot_id(1)
3115            .with_timestamp_ms(builder.metadata.last_updated_ms + 1)
3116            .with_sequence_number(1)
3117            .with_schema_id(0)
3118            .with_manifest_list("/snap-1")
3119            .with_summary(Summary {
3120                operation: Operation::Append,
3121                additional_properties: HashMap::new(),
3122            })
3123            .build();
3124
3125        let builder = builder
3126            .add_snapshot(snapshot.clone())
3127            .unwrap()
3128            .set_ref(MAIN_BRANCH, SnapshotReference {
3129                snapshot_id: 1,
3130                retention: SnapshotRetention::Branch {
3131                    min_snapshots_to_keep: Some(10),
3132                    max_snapshot_age_ms: None,
3133                    max_ref_age_ms: None,
3134                },
3135            })
3136            .unwrap();
3137
3138        let snapshot = Snapshot::builder()
3139            .with_snapshot_id(2)
3140            .with_timestamp_ms(builder.metadata.last_updated_ms + 1)
3141            .with_sequence_number(0)
3142            .with_schema_id(0)
3143            .with_manifest_list("/snap-0")
3144            .with_parent_snapshot_id(Some(1))
3145            .with_summary(Summary {
3146                operation: Operation::Append,
3147                additional_properties: HashMap::new(),
3148            })
3149            .build();
3150
3151        let err = builder
3152            .set_branch_snapshot(snapshot, MAIN_BRANCH)
3153            .unwrap_err();
3154        assert!(
3155            err.to_string()
3156                .contains("Cannot add snapshot with sequence number")
3157        );
3158    }
3159
3160    #[test]
3161    fn test_default_spec_cannot_be_removed() {
3162        let builder = builder_without_changes(FormatVersion::V2);
3163
3164        builder.remove_partition_specs(&[0]).unwrap_err();
3165    }
3166
3167    #[test]
3168    fn test_statistics() {
3169        let builder = builder_without_changes(FormatVersion::V2);
3170
3171        let statistics = StatisticsFile {
3172            snapshot_id: 3055729675574597004,
3173            statistics_path: "s3://a/b/stats.puffin".to_string(),
3174            file_size_in_bytes: 413,
3175            file_footer_size_in_bytes: 42,
3176            key_metadata: None,
3177            blob_metadata: vec![BlobMetadata {
3178                snapshot_id: 3055729675574597004,
3179                sequence_number: 1,
3180                fields: vec![1],
3181                r#type: "ndv".to_string(),
3182                properties: HashMap::new(),
3183            }],
3184        };
3185        let build_result = builder.set_statistics(statistics.clone()).build().unwrap();
3186
3187        assert_eq!(
3188            build_result.metadata.statistics,
3189            HashMap::from_iter(vec![(3055729675574597004, statistics.clone())])
3190        );
3191        assert_eq!(build_result.changes, vec![TableUpdate::SetStatistics {
3192            statistics: statistics.clone()
3193        }]);
3194
3195        // Remove
3196        let builder = build_result.metadata.into_builder(None);
3197        let build_result = builder
3198            .remove_statistics(statistics.snapshot_id)
3199            .build()
3200            .unwrap();
3201
3202        assert_eq!(build_result.metadata.statistics.len(), 0);
3203        assert_eq!(build_result.changes, vec![TableUpdate::RemoveStatistics {
3204            snapshot_id: statistics.snapshot_id
3205        }]);
3206
3207        // Remove again yields no changes
3208        let builder = build_result.metadata.into_builder(None);
3209        let build_result = builder
3210            .remove_statistics(statistics.snapshot_id)
3211            .build()
3212            .unwrap();
3213        assert_eq!(build_result.metadata.statistics.len(), 0);
3214        assert_eq!(build_result.changes.len(), 0);
3215    }
3216
3217    #[test]
3218    fn test_add_partition_statistics() {
3219        let builder = builder_without_changes(FormatVersion::V2);
3220
3221        let statistics = PartitionStatisticsFile {
3222            snapshot_id: 3055729675574597004,
3223            statistics_path: "s3://a/b/partition-stats.parquet".to_string(),
3224            file_size_in_bytes: 43,
3225        };
3226
3227        let build_result = builder
3228            .set_partition_statistics(statistics.clone())
3229            .build()
3230            .unwrap();
3231        assert_eq!(
3232            build_result.metadata.partition_statistics,
3233            HashMap::from_iter(vec![(3055729675574597004, statistics.clone())])
3234        );
3235        assert_eq!(build_result.changes, vec![
3236            TableUpdate::SetPartitionStatistics {
3237                partition_statistics: statistics.clone()
3238            }
3239        ]);
3240
3241        // Remove
3242        let builder = build_result.metadata.into_builder(None);
3243        let build_result = builder
3244            .remove_partition_statistics(statistics.snapshot_id)
3245            .build()
3246            .unwrap();
3247        assert_eq!(build_result.metadata.partition_statistics.len(), 0);
3248        assert_eq!(build_result.changes, vec![
3249            TableUpdate::RemovePartitionStatistics {
3250                snapshot_id: statistics.snapshot_id
3251            }
3252        ]);
3253
3254        // Remove again yields no changes
3255        let builder = build_result.metadata.into_builder(None);
3256        let build_result = builder
3257            .remove_partition_statistics(statistics.snapshot_id)
3258            .build()
3259            .unwrap();
3260        assert_eq!(build_result.metadata.partition_statistics.len(), 0);
3261        assert_eq!(build_result.changes.len(), 0);
3262    }
3263
3264    #[test]
3265    fn last_update_increased_for_property_only_update() {
3266        let builder = builder_without_changes(FormatVersion::V2);
3267
3268        let metadata = builder.build().unwrap().metadata;
3269        let last_updated_ms = metadata.last_updated_ms;
3270        sleep(std::time::Duration::from_millis(2));
3271
3272        let build_result = metadata
3273            .into_builder(Some(
3274                "s3://bucket/test/location/metadata/metadata1.json".to_string(),
3275            ))
3276            .set_properties(HashMap::from_iter(vec![(
3277                "foo".to_string(),
3278                "bar".to_string(),
3279            )]))
3280            .unwrap()
3281            .build()
3282            .unwrap();
3283
3284        assert!(
3285            build_result.metadata.last_updated_ms > last_updated_ms,
3286            "{} > {}",
3287            build_result.metadata.last_updated_ms,
3288            last_updated_ms
3289        );
3290    }
3291
3292    #[test]
3293    fn test_construct_default_main_branch() {
3294        // Load the table without ref
3295        let file = File::open(format!(
3296            "{}/testdata/table_metadata/{}",
3297            env!("CARGO_MANIFEST_DIR"),
3298            "TableMetadataV2Valid.json"
3299        ))
3300        .unwrap();
3301        let reader = BufReader::new(file);
3302        let resp = serde_json::from_reader::<_, TableMetadata>(reader).unwrap();
3303
3304        let table = Table::builder()
3305            .metadata(resp)
3306            .metadata_location("s3://bucket/test/location/metadata/v1.json")
3307            .identifier(TableIdent::from_strs(["ns1", "test1"]).unwrap())
3308            .file_io(FileIO::new_with_memory())
3309            .runtime(test_runtime())
3310            .build()
3311            .unwrap();
3312
3313        assert_eq!(
3314            table.metadata().refs.get(MAIN_BRANCH).unwrap().snapshot_id,
3315            table.metadata().current_snapshot_id().unwrap()
3316        );
3317    }
3318
3319    #[test]
3320    fn test_active_schema_cannot_be_removed() {
3321        let builder = builder_without_changes(FormatVersion::V2);
3322        builder.remove_schemas(&[0]).unwrap_err();
3323    }
3324
3325    #[test]
3326    fn test_remove_schemas() {
3327        let file = File::open(format!(
3328            "{}/testdata/table_metadata/{}",
3329            env!("CARGO_MANIFEST_DIR"),
3330            "TableMetadataV2Valid.json"
3331        ))
3332        .unwrap();
3333        let reader = BufReader::new(file);
3334        let resp = serde_json::from_reader::<_, TableMetadata>(reader).unwrap();
3335
3336        let table = Table::builder()
3337            .metadata(resp)
3338            .metadata_location("s3://bucket/test/location/metadata/v1.json")
3339            .identifier(TableIdent::from_strs(["ns1", "test1"]).unwrap())
3340            .file_io(FileIO::new_with_memory())
3341            .runtime(test_runtime())
3342            .build()
3343            .unwrap();
3344
3345        assert_eq!(2, table.metadata().schemas.len());
3346
3347        {
3348            // can not remove active schema
3349            let meta_data_builder = table.metadata().clone().into_builder(None);
3350            meta_data_builder.remove_schemas(&[1]).unwrap_err();
3351        }
3352
3353        let mut meta_data_builder = table.metadata().clone().into_builder(None);
3354        meta_data_builder = meta_data_builder.remove_schemas(&[0]).unwrap();
3355        let build_result = meta_data_builder.build().unwrap();
3356        assert_eq!(1, build_result.metadata.schemas.len());
3357        assert_eq!(1, build_result.metadata.current_schema_id);
3358        assert_eq!(1, build_result.metadata.current_schema().schema_id());
3359        assert_eq!(1, build_result.changes.len());
3360
3361        let remove_schema_ids =
3362            if let TableUpdate::RemoveSchemas { schema_ids } = &build_result.changes[0] {
3363                schema_ids
3364            } else {
3365                unreachable!("Expected RemoveSchema change")
3366            };
3367        assert_eq!(remove_schema_ids, &[0]);
3368    }
3369
3370    #[test]
3371    fn test_schema_evolution_now_correctly_validates_partition_field_name_conflicts() {
3372        let initial_schema = Schema::builder()
3373            .with_fields(vec![
3374                NestedField::required(1, "data", Type::Primitive(PrimitiveType::String)).into(),
3375            ])
3376            .build()
3377            .unwrap();
3378
3379        let partition_spec_with_bucket = UnboundPartitionSpec::builder()
3380            .with_spec_id(0)
3381            .add_partition_field(
3382                UnboundPartitionField::builder()
3383                    .source_ids(vec![1])
3384                    .name("bucket_data")
3385                    .transform(Transform::Bucket(16))
3386                    .build()
3387                    .unwrap(),
3388            )
3389            .unwrap()
3390            .build();
3391
3392        let metadata = TableMetadataBuilder::new(
3393            initial_schema,
3394            partition_spec_with_bucket,
3395            SortOrder::unsorted_order(),
3396            TEST_LOCATION.to_string(),
3397            FormatVersion::V2,
3398            HashMap::new(),
3399        )
3400        .unwrap()
3401        .build()
3402        .unwrap()
3403        .metadata;
3404
3405        let partition_field_names: Vec<String> = metadata
3406            .default_partition_spec()
3407            .fields()
3408            .iter()
3409            .map(|f| f.name.clone())
3410            .collect();
3411        assert!(partition_field_names.contains(&"bucket_data".to_string()));
3412
3413        let evolved_schema = Schema::builder()
3414            .with_fields(vec![
3415                NestedField::required(1, "data", Type::Primitive(PrimitiveType::String)).into(),
3416                // Adding a schema field with the same name as an existing partition field
3417                NestedField::required(2, "bucket_data", Type::Primitive(PrimitiveType::Int)).into(),
3418            ])
3419            .build()
3420            .unwrap();
3421
3422        let builder = metadata.into_builder(Some(
3423            "s3://bucket/test/location/metadata/metadata1.json".to_string(),
3424        ));
3425
3426        // Try to add the evolved schema - this should now fail immediately with a clear error
3427        let result = builder.add_current_schema(evolved_schema);
3428
3429        assert!(result.is_err());
3430        let error = result.unwrap_err();
3431        let error_message = error.message();
3432        assert!(error_message.contains("Cannot set current schema"));
3433        assert!(error_message.contains("sourced from field id 1"));
3434    }
3435
3436    #[test]
3437    fn test_schema_evolution_should_validate_on_schema_add_not_metadata_build() {
3438        let initial_schema = Schema::builder()
3439            .with_fields(vec![
3440                NestedField::required(1, "data", Type::Primitive(PrimitiveType::String)).into(),
3441            ])
3442            .build()
3443            .unwrap();
3444
3445        let partition_spec = UnboundPartitionSpec::builder()
3446            .with_spec_id(0)
3447            .add_partition_field(
3448                UnboundPartitionField::builder()
3449                    .source_ids(vec![1])
3450                    .name("partition_col")
3451                    .transform(Transform::Bucket(16))
3452                    .build()
3453                    .unwrap(),
3454            )
3455            .unwrap()
3456            .build();
3457
3458        let metadata = TableMetadataBuilder::new(
3459            initial_schema,
3460            partition_spec,
3461            SortOrder::unsorted_order(),
3462            TEST_LOCATION.to_string(),
3463            FormatVersion::V2,
3464            HashMap::new(),
3465        )
3466        .unwrap()
3467        .build()
3468        .unwrap()
3469        .metadata;
3470
3471        let non_conflicting_schema = Schema::builder()
3472            .with_fields(vec![
3473                NestedField::required(1, "data", Type::Primitive(PrimitiveType::String)).into(),
3474                NestedField::required(2, "new_field", Type::Primitive(PrimitiveType::Int)).into(),
3475            ])
3476            .build()
3477            .unwrap();
3478
3479        // This should succeed since there's no name conflict
3480        let result = metadata
3481            .clone()
3482            .into_builder(Some("test_location".to_string()))
3483            .add_current_schema(non_conflicting_schema)
3484            .unwrap()
3485            .build();
3486
3487        assert!(result.is_ok());
3488    }
3489
3490    #[test]
3491    fn test_partition_spec_evolution_validates_schema_field_name_conflicts() {
3492        let initial_schema = Schema::builder()
3493            .with_fields(vec![
3494                NestedField::required(1, "data", Type::Primitive(PrimitiveType::String)).into(),
3495                NestedField::required(2, "existing_field", Type::Primitive(PrimitiveType::Int))
3496                    .into(),
3497            ])
3498            .build()
3499            .unwrap();
3500
3501        let partition_spec = UnboundPartitionSpec::builder()
3502            .with_spec_id(0)
3503            .add_partition_field(
3504                UnboundPartitionField::builder()
3505                    .source_ids(vec![1])
3506                    .name("data_bucket")
3507                    .transform(Transform::Bucket(16))
3508                    .build()
3509                    .unwrap(),
3510            )
3511            .unwrap()
3512            .build();
3513
3514        let metadata = TableMetadataBuilder::new(
3515            initial_schema,
3516            partition_spec,
3517            SortOrder::unsorted_order(),
3518            TEST_LOCATION.to_string(),
3519            FormatVersion::V2,
3520            HashMap::new(),
3521        )
3522        .unwrap()
3523        .build()
3524        .unwrap()
3525        .metadata;
3526
3527        let builder = metadata.into_builder(Some(
3528            "s3://bucket/test/location/metadata/metadata1.json".to_string(),
3529        ));
3530
3531        let conflicting_partition_spec = UnboundPartitionSpec::builder()
3532            .with_spec_id(1)
3533            .add_partition_field(
3534                UnboundPartitionField::builder()
3535                    .source_ids(vec![1])
3536                    .name("existing_field")
3537                    .transform(Transform::Bucket(8))
3538                    .build()
3539                    .unwrap(),
3540            )
3541            .unwrap()
3542            .build();
3543
3544        let result = builder.add_partition_spec(conflicting_partition_spec);
3545
3546        assert!(result.is_err());
3547        let error = result.unwrap_err();
3548        let error_message = error.message();
3549        // The error comes from our multi-version validation
3550        assert!(error_message.contains(
3551            "Cannot create partition with name 'existing_field' that conflicts with schema field"
3552        ));
3553        assert!(error_message.contains("and is not an identity or void transform"));
3554    }
3555
3556    #[test]
3557    fn test_schema_evolution_validates_against_all_historical_schemas() {
3558        // Create a table with an initial schema that has a field "existing_field"
3559        let initial_schema = Schema::builder()
3560            .with_fields(vec![
3561                NestedField::required(1, "data", Type::Primitive(PrimitiveType::String)).into(),
3562                NestedField::required(2, "existing_field", Type::Primitive(PrimitiveType::Int))
3563                    .into(),
3564            ])
3565            .build()
3566            .unwrap();
3567
3568        let partition_spec = UnboundPartitionSpec::builder()
3569            .with_spec_id(0)
3570            .add_partition_field(
3571                UnboundPartitionField::builder()
3572                    .source_ids(vec![1])
3573                    .name("bucket_data")
3574                    .transform(Transform::Bucket(16))
3575                    .build()
3576                    .unwrap(),
3577            )
3578            .unwrap()
3579            .build();
3580
3581        let metadata = TableMetadataBuilder::new(
3582            initial_schema,
3583            partition_spec,
3584            SortOrder::unsorted_order(),
3585            TEST_LOCATION.to_string(),
3586            FormatVersion::V2,
3587            HashMap::new(),
3588        )
3589        .unwrap()
3590        .build()
3591        .unwrap()
3592        .metadata;
3593
3594        // Add a second schema that removes the existing_field but keeps the data field
3595        let second_schema = Schema::builder()
3596            .with_fields(vec![
3597                NestedField::required(1, "data", Type::Primitive(PrimitiveType::String)).into(),
3598                NestedField::required(3, "new_field", Type::Primitive(PrimitiveType::String))
3599                    .into(),
3600            ])
3601            .build()
3602            .unwrap();
3603
3604        let metadata = metadata
3605            .into_builder(Some("test_location".to_string()))
3606            .add_current_schema(second_schema)
3607            .unwrap()
3608            .build()
3609            .unwrap()
3610            .metadata;
3611
3612        // Now try to add a third schema that reintroduces "existing_field"
3613        // This should succeed because "existing_field" exists in a historical schema,
3614        // even though there's a partition field named "bucket_data"
3615        let third_schema = Schema::builder()
3616            .with_fields(vec![
3617                NestedField::required(1, "data", Type::Primitive(PrimitiveType::String)).into(),
3618                NestedField::required(3, "new_field", Type::Primitive(PrimitiveType::String))
3619                    .into(),
3620                NestedField::required(4, "existing_field", Type::Primitive(PrimitiveType::Int))
3621                    .into(),
3622            ])
3623            .build()
3624            .unwrap();
3625
3626        let builder = metadata
3627            .clone()
3628            .into_builder(Some("test_location".to_string()));
3629
3630        // This should succeed because "existing_field" exists in a historical schema
3631        let result = builder.add_current_schema(third_schema);
3632        assert!(result.is_ok());
3633
3634        // However, trying to add a schema field that conflicts with the partition field
3635        // and doesn't exist in any historical schema should fail
3636        let conflicting_schema = Schema::builder()
3637            .with_fields(vec![
3638                NestedField::required(1, "data", Type::Primitive(PrimitiveType::String)).into(),
3639                NestedField::required(3, "new_field", Type::Primitive(PrimitiveType::String))
3640                    .into(),
3641                NestedField::required(4, "existing_field", Type::Primitive(PrimitiveType::Int))
3642                    .into(),
3643                NestedField::required(5, "bucket_data", Type::Primitive(PrimitiveType::String))
3644                    .into(), // conflicts with partition field
3645            ])
3646            .build()
3647            .unwrap();
3648
3649        let builder2 = metadata.into_builder(Some("test_location".to_string()));
3650        let result2 = builder2.add_current_schema(conflicting_schema);
3651
3652        // This should fail because "bucket_data" takes the name of a partition field whose
3653        // source column `data` is still in the schema
3654        assert!(result2.is_err());
3655        let error = result2.unwrap_err();
3656        assert!(error.message().contains("Cannot set current schema"));
3657        assert!(error.message().contains("sourced from field id 1"));
3658    }
3659
3660    #[test]
3661    fn test_schema_evolution_allows_existing_partition_field_if_exists_in_historical_schema() {
3662        // Create initial schema with a field
3663        let initial_schema = Schema::builder()
3664            .with_fields(vec![
3665                NestedField::required(1, "data", Type::Primitive(PrimitiveType::String)).into(),
3666                NestedField::required(2, "partition_data", Type::Primitive(PrimitiveType::Int))
3667                    .into(),
3668            ])
3669            .build()
3670            .unwrap();
3671
3672        let partition_spec = UnboundPartitionSpec::builder()
3673            .with_spec_id(0)
3674            .add_partition_field(
3675                UnboundPartitionField::builder()
3676                    .source_ids(vec![2])
3677                    .name("partition_data")
3678                    .transform(Transform::Identity)
3679                    .build()
3680                    .unwrap(),
3681            )
3682            .unwrap()
3683            .build();
3684
3685        let metadata = TableMetadataBuilder::new(
3686            initial_schema,
3687            partition_spec,
3688            SortOrder::unsorted_order(),
3689            TEST_LOCATION.to_string(),
3690            FormatVersion::V2,
3691            HashMap::new(),
3692        )
3693        .unwrap()
3694        .build()
3695        .unwrap()
3696        .metadata;
3697
3698        // Add a new schema that still contains the partition_data field
3699        let evolved_schema = Schema::builder()
3700            .with_fields(vec![
3701                NestedField::required(1, "data", Type::Primitive(PrimitiveType::String)).into(),
3702                NestedField::required(2, "partition_data", Type::Primitive(PrimitiveType::Int))
3703                    .into(),
3704                NestedField::required(3, "new_field", Type::Primitive(PrimitiveType::String))
3705                    .into(),
3706            ])
3707            .build()
3708            .unwrap();
3709
3710        // This should succeed because partition_data exists in historical schemas
3711        let result = metadata
3712            .into_builder(Some("test_location".to_string()))
3713            .add_current_schema(evolved_schema);
3714
3715        assert!(result.is_ok());
3716    }
3717
3718    /// A v2 table of `data: string` partitioned by `bucket(data, 16)` as `bucket_data`.
3719    fn bucket_partitioned_metadata() -> TableMetadata {
3720        let schema = Schema::builder()
3721            .with_fields(vec![
3722                NestedField::required(1, "data", Type::Primitive(PrimitiveType::String)).into(),
3723            ])
3724            .build()
3725            .unwrap();
3726        let spec = UnboundPartitionSpec::builder()
3727            .with_spec_id(0)
3728            .add_partition_field(
3729                UnboundPartitionField::builder()
3730                    .source_ids(vec![1])
3731                    .name("bucket_data")
3732                    .transform(Transform::Bucket(16))
3733                    .build()
3734                    .unwrap(),
3735            )
3736            .unwrap()
3737            .build();
3738
3739        TableMetadataBuilder::new(
3740            schema,
3741            spec,
3742            SortOrder::unsorted_order(),
3743            TEST_LOCATION.to_string(),
3744            FormatVersion::V2,
3745            HashMap::new(),
3746        )
3747        .unwrap()
3748        .build()
3749        .unwrap()
3750        .metadata
3751    }
3752
3753    /// Replaces the default spec with an unpartitioned one. Spec 0 is retained but superseded.
3754    fn unpartition(metadata: TableMetadata, file: &str) -> TableMetadata {
3755        let metadata = metadata
3756            .into_builder(Some(file.to_string()))
3757            .add_default_partition_spec(UnboundPartitionSpec::builder().build())
3758            .unwrap()
3759            .build()
3760            .unwrap()
3761            .metadata;
3762        assert_ne!(metadata.default_partition_spec().spec_id(), 0);
3763        assert!(metadata.partition_specs.contains_key(&0));
3764        metadata
3765    }
3766
3767    #[test]
3768    fn test_schema_evolution_allows_a_superseded_partition_field_name_once_its_source_is_dropped() {
3769        // `data`, the source of spec 0's `bucket_data`, is gone, so spec 0 can only be historical
3770        // and its names are free. Iceberg Java skips its name check for exactly this case.
3771        let schema = Schema::builder()
3772            .with_fields(vec![
3773                NestedField::required(2, "bucket_data", Type::Primitive(PrimitiveType::Int)).into(),
3774            ])
3775            .build()
3776            .unwrap();
3777
3778        let metadata = unpartition(
3779            bucket_partitioned_metadata(),
3780            "s3://bucket/t/metadata/m1.json",
3781        )
3782        .into_builder(Some("s3://bucket/t/metadata/m2.json".to_string()))
3783        .add_current_schema(schema.clone())
3784        .unwrap()
3785        .build()
3786        .unwrap()
3787        .metadata;
3788
3789        assert_eq!(metadata.current_schema().as_struct(), schema.as_struct());
3790        assert!(metadata.current_schema().field_by_name("data").is_none());
3791        // Spec 0 is kept so the snapshots written under it stay interpretable.
3792        assert_eq!(
3793            metadata.partition_spec_by_id(0).unwrap().fields()[0].name,
3794            "bucket_data"
3795        );
3796    }
3797
3798    #[test]
3799    fn test_schema_evolution_rejects_a_partition_field_name_while_its_source_remains() {
3800        // `data` is still here, so `bucket_data` would be ambiguous. Iceberg Java rejects this
3801        // whether or not spec 0 is still the default, because it rebuilds every spec.
3802        let schema = Schema::builder()
3803            .with_fields(vec![
3804                NestedField::required(1, "data", Type::Primitive(PrimitiveType::String)).into(),
3805                NestedField::required(2, "bucket_data", Type::Primitive(PrimitiveType::Int)).into(),
3806            ])
3807            .build()
3808            .unwrap();
3809
3810        let error = bucket_partitioned_metadata()
3811            .into_builder(Some("s3://bucket/t/metadata/m1.json".to_string()))
3812            .add_current_schema(schema.clone())
3813            .unwrap_err();
3814        assert_eq!(error.kind(), ErrorKind::DataInvalid);
3815        assert!(
3816            error.message().contains("sourced from field id 1"),
3817            "{error}"
3818        );
3819
3820        let error = unpartition(
3821            bucket_partitioned_metadata(),
3822            "s3://bucket/t/metadata/m1.json",
3823        )
3824        .into_builder(Some("s3://bucket/t/metadata/m2.json".to_string()))
3825        .add_current_schema(schema)
3826        .unwrap_err();
3827        assert_eq!(error.kind(), ErrorKind::DataInvalid);
3828        assert!(
3829            error.message().contains("sourced from field id 1"),
3830            "{error}"
3831        );
3832    }
3833
3834    #[test]
3835    fn test_schema_evolution_partition_name_check_is_order_independent() {
3836        // Dropping the source column and dropping the partitioning in one set of changes must
3837        // behave the same either way round.
3838        let schema = Schema::builder()
3839            .with_fields(vec![
3840                NestedField::required(2, "bucket_data", Type::Primitive(PrimitiveType::Int)).into(),
3841            ])
3842            .build()
3843            .unwrap();
3844        let unpartitioned = UnboundPartitionSpec::builder().build();
3845
3846        let schema_first = bucket_partitioned_metadata()
3847            .into_builder(Some("s3://bucket/t/metadata/m1.json".to_string()))
3848            .add_current_schema(schema.clone())
3849            .and_then(|b| b.add_default_partition_spec(unpartitioned.clone()))
3850            .and_then(|b| b.build());
3851        let spec_first = bucket_partitioned_metadata()
3852            .into_builder(Some("s3://bucket/t/metadata/m1.json".to_string()))
3853            .add_default_partition_spec(unpartitioned)
3854            .and_then(|b| b.add_current_schema(schema))
3855            .and_then(|b| b.build());
3856
3857        assert!(schema_first.is_ok(), "{:?}", schema_first.err());
3858        assert!(spec_first.is_ok(), "{:?}", spec_first.err());
3859    }
3860
3861    #[test]
3862    fn test_set_current_schema_rejects_a_partition_field_name_from_a_spec_added_after_the_schema() {
3863        let schema = Schema::builder()
3864            .with_fields(vec![
3865                NestedField::required(1, "data", Type::Primitive(PrimitiveType::String)).into(),
3866            ])
3867            .build()
3868            .unwrap();
3869        let conflicting_schema = Schema::builder()
3870            .with_fields(vec![
3871                NestedField::required(1, "data", Type::Primitive(PrimitiveType::String)).into(),
3872                NestedField::required(2, "bucket_data", Type::Primitive(PrimitiveType::Int)).into(),
3873            ])
3874            .build()
3875            .unwrap();
3876        let spec = UnboundPartitionSpec::builder()
3877            .add_partition_field(
3878                UnboundPartitionField::builder()
3879                    .source_ids(vec![1])
3880                    .name("bucket_data")
3881                    .transform(Transform::Bucket(16))
3882                    .build()
3883                    .unwrap(),
3884            )
3885            .unwrap()
3886            .build();
3887
3888        let error = TableMetadataBuilder::new(
3889            schema,
3890            UnboundPartitionSpec::builder().build(),
3891            SortOrder::unsorted_order(),
3892            TEST_LOCATION.to_string(),
3893            FormatVersion::V2,
3894            HashMap::new(),
3895        )
3896        .unwrap()
3897        .add_schema(conflicting_schema)
3898        .and_then(|b| b.add_partition_spec(spec))
3899        .and_then(|b| b.set_current_schema(TableMetadataBuilder::LAST_ADDED))
3900        .unwrap_err();
3901        assert_eq!(error.kind(), ErrorKind::DataInvalid);
3902        assert!(
3903            error.message().contains("sourced from field id 1"),
3904            "{error}"
3905        );
3906    }
3907
3908    #[test]
3909    fn test_schema_with_a_conflicting_partition_field_name_can_be_added_but_not_made_current() {
3910        let conflicting_schema = Schema::builder()
3911            .with_fields(vec![
3912                NestedField::required(1, "data", Type::Primitive(PrimitiveType::String)).into(),
3913                NestedField::required(2, "bucket_data", Type::Primitive(PrimitiveType::Int)).into(),
3914            ])
3915            .build()
3916            .unwrap();
3917
3918        let metadata = bucket_partitioned_metadata()
3919            .into_builder(Some("s3://bucket/t/metadata/m1.json".to_string()))
3920            .add_schema(conflicting_schema.clone())
3921            .unwrap()
3922            .build()
3923            .unwrap()
3924            .metadata;
3925        assert_eq!(metadata.current_schema_id(), 0);
3926        assert_eq!(
3927            metadata.schema_by_id(1).unwrap().as_struct(),
3928            conflicting_schema.as_struct()
3929        );
3930
3931        let error = metadata
3932            .into_builder(Some("s3://bucket/t/metadata/m2.json".to_string()))
3933            .set_current_schema(1)
3934            .unwrap_err();
3935        assert_eq!(error.kind(), ErrorKind::DataInvalid);
3936        assert!(
3937            error.message().contains("sourced from field id 1"),
3938            "{error}"
3939        );
3940    }
3941
3942    #[test]
3943    fn test_schema_evolution_prevents_new_field_conflicting_with_partition_field() {
3944        // Create initial schema WITHOUT the conflicting field
3945        let initial_schema = Schema::builder()
3946            .with_fields(vec![
3947                NestedField::required(1, "data", Type::Primitive(PrimitiveType::String)).into(),
3948            ])
3949            .build()
3950            .unwrap();
3951
3952        let partition_spec = UnboundPartitionSpec::builder()
3953            .with_spec_id(0)
3954            .add_partition_field(
3955                UnboundPartitionField::builder()
3956                    .source_ids(vec![1])
3957                    .name("bucket_data")
3958                    .transform(Transform::Bucket(16))
3959                    .build()
3960                    .unwrap(),
3961            )
3962            .unwrap()
3963            .build();
3964
3965        let metadata = TableMetadataBuilder::new(
3966            initial_schema,
3967            partition_spec,
3968            SortOrder::unsorted_order(),
3969            TEST_LOCATION.to_string(),
3970            FormatVersion::V2,
3971            HashMap::new(),
3972        )
3973        .unwrap()
3974        .build()
3975        .unwrap()
3976        .metadata;
3977
3978        // Try to add a schema with a field that conflicts with partition field name
3979        let conflicting_schema = Schema::builder()
3980            .with_fields(vec![
3981                NestedField::required(1, "data", Type::Primitive(PrimitiveType::String)).into(),
3982                // This field name conflicts with the partition field "bucket_data"
3983                NestedField::required(2, "bucket_data", Type::Primitive(PrimitiveType::Int)).into(),
3984            ])
3985            .build()
3986            .unwrap();
3987
3988        let builder = metadata.into_builder(Some("test_location".to_string()));
3989        let result = builder.add_current_schema(conflicting_schema);
3990
3991        // This should fail because "bucket_data" takes the name of a partition field whose
3992        // source column `data` is still in the schema
3993        assert!(result.is_err());
3994        let error = result.unwrap_err();
3995        assert!(error.message().contains("Cannot set current schema"));
3996        assert!(error.message().contains("sourced from field id 1"));
3997    }
3998
3999    #[test]
4000    fn test_partition_spec_evolution_allows_non_conflicting_names() {
4001        let initial_schema = Schema::builder()
4002            .with_fields(vec![
4003                NestedField::required(1, "data", Type::Primitive(PrimitiveType::String)).into(),
4004                NestedField::required(2, "existing_field", Type::Primitive(PrimitiveType::Int))
4005                    .into(),
4006            ])
4007            .build()
4008            .unwrap();
4009
4010        let partition_spec = UnboundPartitionSpec::builder()
4011            .with_spec_id(0)
4012            .add_partition_field(
4013                UnboundPartitionField::builder()
4014                    .source_ids(vec![1])
4015                    .name("data_bucket")
4016                    .transform(Transform::Bucket(16))
4017                    .build()
4018                    .unwrap(),
4019            )
4020            .unwrap()
4021            .build();
4022
4023        let metadata = TableMetadataBuilder::new(
4024            initial_schema,
4025            partition_spec,
4026            SortOrder::unsorted_order(),
4027            TEST_LOCATION.to_string(),
4028            FormatVersion::V2,
4029            HashMap::new(),
4030        )
4031        .unwrap()
4032        .build()
4033        .unwrap()
4034        .metadata;
4035
4036        let builder = metadata.into_builder(Some(
4037            "s3://bucket/test/location/metadata/metadata1.json".to_string(),
4038        ));
4039
4040        // Try to add a partition spec with a field name that does NOT conflict with existing schema fields
4041        let non_conflicting_partition_spec = UnboundPartitionSpec::builder()
4042            .with_spec_id(1)
4043            .add_partition_field(
4044                UnboundPartitionField::builder()
4045                    .source_ids(vec![2])
4046                    .name("new_partition_field")
4047                    .transform(Transform::Bucket(8))
4048                    .build()
4049                    .unwrap(),
4050            )
4051            .unwrap()
4052            .build();
4053
4054        let result = builder.add_partition_spec(non_conflicting_partition_spec);
4055
4056        assert!(result.is_ok());
4057    }
4058
4059    #[test]
4060    fn test_row_lineage_addition() {
4061        let new_rows = 30;
4062        let base = builder_without_changes(FormatVersion::V3)
4063            .build()
4064            .unwrap()
4065            .metadata;
4066        let add_rows = Snapshot::builder()
4067            .with_snapshot_id(0)
4068            .with_timestamp_ms(base.last_updated_ms + 1)
4069            .with_sequence_number(0)
4070            .with_schema_id(0)
4071            .with_manifest_list("foo")
4072            .with_parent_snapshot_id(None)
4073            .with_summary(Summary {
4074                operation: Operation::Append,
4075                additional_properties: HashMap::new(),
4076            })
4077            .with_row_range(base.next_row_id(), new_rows)
4078            .build();
4079
4080        let first_addition = base
4081            .into_builder(None)
4082            .add_snapshot(add_rows.clone())
4083            .unwrap()
4084            .build()
4085            .unwrap()
4086            .metadata;
4087
4088        assert_eq!(first_addition.next_row_id(), new_rows);
4089
4090        let add_more_rows = Snapshot::builder()
4091            .with_snapshot_id(1)
4092            .with_timestamp_ms(first_addition.last_updated_ms + 1)
4093            .with_sequence_number(1)
4094            .with_schema_id(0)
4095            .with_manifest_list("foo")
4096            .with_parent_snapshot_id(Some(0))
4097            .with_summary(Summary {
4098                operation: Operation::Append,
4099                additional_properties: HashMap::new(),
4100            })
4101            .with_row_range(first_addition.next_row_id(), new_rows)
4102            .build();
4103
4104        let second_addition = first_addition
4105            .into_builder(None)
4106            .add_snapshot(add_more_rows)
4107            .unwrap()
4108            .build()
4109            .unwrap()
4110            .metadata;
4111        assert_eq!(second_addition.next_row_id(), new_rows * 2);
4112    }
4113
4114    #[test]
4115    fn test_row_lineage_invalid_snapshot() {
4116        let new_rows = 30;
4117        let base = builder_without_changes(FormatVersion::V3)
4118            .build()
4119            .unwrap()
4120            .metadata;
4121
4122        // add rows to check TableMetadata validation; Snapshot rejects negative next-row-id
4123        let add_rows = Snapshot::builder()
4124            .with_snapshot_id(0)
4125            .with_timestamp_ms(base.last_updated_ms + 1)
4126            .with_sequence_number(0)
4127            .with_schema_id(0)
4128            .with_manifest_list("foo")
4129            .with_parent_snapshot_id(None)
4130            .with_summary(Summary {
4131                operation: Operation::Append,
4132                additional_properties: HashMap::new(),
4133            })
4134            .with_row_range(base.next_row_id(), new_rows)
4135            .build();
4136
4137        let added = base
4138            .into_builder(None)
4139            .add_snapshot(add_rows)
4140            .unwrap()
4141            .build()
4142            .unwrap()
4143            .metadata;
4144
4145        let invalid_new_rows = Snapshot::builder()
4146            .with_snapshot_id(1)
4147            .with_timestamp_ms(added.last_updated_ms + 1)
4148            .with_sequence_number(1)
4149            .with_schema_id(0)
4150            .with_manifest_list("foo")
4151            .with_parent_snapshot_id(Some(0))
4152            .with_summary(Summary {
4153                operation: Operation::Append,
4154                additional_properties: HashMap::new(),
4155            })
4156            // first_row_id is behind table next_row_id
4157            .with_row_range(added.next_row_id() - 1, 10)
4158            .build();
4159
4160        let err = added
4161            .into_builder(None)
4162            .add_snapshot(invalid_new_rows)
4163            .unwrap_err();
4164        assert!(
4165            err.to_string().contains(
4166                "Cannot add a snapshot, first-row-id is behind table next-row-id: 29 < 30"
4167            )
4168        );
4169    }
4170
4171    #[test]
4172    fn test_row_lineage_append_branch() {
4173        // Appends to a branch should still change last-row-id even if not on main, these changes
4174        // should also affect commits to main
4175
4176        let branch = "some_branch";
4177
4178        // Start with V3 metadata to support row lineage
4179        let base = builder_without_changes(FormatVersion::V3)
4180            .build()
4181            .unwrap()
4182            .metadata;
4183
4184        // Initial next_row_id should be 0
4185        assert_eq!(base.next_row_id(), 0);
4186
4187        // Write to Branch - append 30 rows
4188        let branch_snapshot_1 = Snapshot::builder()
4189            .with_snapshot_id(1)
4190            .with_timestamp_ms(base.last_updated_ms + 1)
4191            .with_sequence_number(0)
4192            .with_schema_id(0)
4193            .with_manifest_list("foo")
4194            .with_parent_snapshot_id(None)
4195            .with_summary(Summary {
4196                operation: Operation::Append,
4197                additional_properties: HashMap::new(),
4198            })
4199            .with_row_range(base.next_row_id(), 30)
4200            .build();
4201
4202        let table_after_branch_1 = base
4203            .into_builder(None)
4204            .set_branch_snapshot(branch_snapshot_1.clone(), branch)
4205            .unwrap()
4206            .build()
4207            .unwrap()
4208            .metadata;
4209
4210        // Current snapshot should be null (no main branch snapshot yet)
4211        assert!(table_after_branch_1.current_snapshot().is_none());
4212
4213        // Branch snapshot should have first_row_id = 0
4214        let branch_ref = table_after_branch_1.refs.get(branch).unwrap();
4215        let branch_snap_1 = table_after_branch_1
4216            .snapshots
4217            .get(&branch_ref.snapshot_id)
4218            .unwrap();
4219        assert_eq!(branch_snap_1.first_row_id(), Some(0));
4220
4221        // Next row id should be 30
4222        assert_eq!(table_after_branch_1.next_row_id(), 30);
4223
4224        // Write to Main - append 28 rows
4225        let main_snapshot = Snapshot::builder()
4226            .with_snapshot_id(2)
4227            .with_timestamp_ms(table_after_branch_1.last_updated_ms + 1)
4228            .with_sequence_number(1)
4229            .with_schema_id(0)
4230            .with_manifest_list("bar")
4231            .with_parent_snapshot_id(None)
4232            .with_summary(Summary {
4233                operation: Operation::Append,
4234                additional_properties: HashMap::new(),
4235            })
4236            .with_row_range(table_after_branch_1.next_row_id(), 28)
4237            .build();
4238
4239        let table_after_main = table_after_branch_1
4240            .into_builder(None)
4241            .add_snapshot(main_snapshot.clone())
4242            .unwrap()
4243            .set_ref(MAIN_BRANCH, SnapshotReference {
4244                snapshot_id: main_snapshot.snapshot_id(),
4245                retention: SnapshotRetention::Branch {
4246                    min_snapshots_to_keep: None,
4247                    max_snapshot_age_ms: None,
4248                    max_ref_age_ms: None,
4249                },
4250            })
4251            .unwrap()
4252            .build()
4253            .unwrap()
4254            .metadata;
4255
4256        // Main snapshot should have first_row_id = 30
4257        let current_snapshot = table_after_main.current_snapshot().unwrap();
4258        assert_eq!(current_snapshot.first_row_id(), Some(30));
4259
4260        // Next row id should be 58 (30 + 28)
4261        assert_eq!(table_after_main.next_row_id(), 58);
4262
4263        // Write again to branch - append 21 rows
4264        let branch_snapshot_2 = Snapshot::builder()
4265            .with_snapshot_id(3)
4266            .with_timestamp_ms(table_after_main.last_updated_ms + 1)
4267            .with_sequence_number(2)
4268            .with_schema_id(0)
4269            .with_manifest_list("baz")
4270            .with_parent_snapshot_id(Some(branch_snapshot_1.snapshot_id()))
4271            .with_summary(Summary {
4272                operation: Operation::Append,
4273                additional_properties: HashMap::new(),
4274            })
4275            .with_row_range(table_after_main.next_row_id(), 21)
4276            .build();
4277
4278        let table_after_branch_2 = table_after_main
4279            .into_builder(None)
4280            .set_branch_snapshot(branch_snapshot_2.clone(), branch)
4281            .unwrap()
4282            .build()
4283            .unwrap()
4284            .metadata;
4285
4286        // Branch snapshot should have first_row_id = 58 (30 + 28)
4287        let branch_ref_2 = table_after_branch_2.refs.get(branch).unwrap();
4288        let branch_snap_2 = table_after_branch_2
4289            .snapshots
4290            .get(&branch_ref_2.snapshot_id)
4291            .unwrap();
4292        assert_eq!(branch_snap_2.first_row_id(), Some(58));
4293
4294        // Next row id should be 79 (30 + 28 + 21)
4295        assert_eq!(table_after_branch_2.next_row_id(), 79);
4296    }
4297
4298    #[test]
4299    fn test_encryption_keys() {
4300        let builder = builder_without_changes(FormatVersion::V3);
4301
4302        // Create test encryption keys
4303        let encryption_key_1 = EncryptedKey::builder()
4304            .key_id("key-1")
4305            .encrypted_key_metadata(vec![1, 2, 3, 4])
4306            .encrypted_by_id("encryption-service-1")
4307            .properties(HashMap::from_iter(vec![(
4308                "algorithm".to_string(),
4309                "AES-256".to_string(),
4310            )]))
4311            .build();
4312
4313        let encryption_key_2 = EncryptedKey::builder()
4314            .key_id("key-2")
4315            .encrypted_key_metadata(vec![5, 6, 7, 8])
4316            .encrypted_by_id("encryption-service-2")
4317            .properties(HashMap::new())
4318            .build();
4319
4320        // Add first encryption key
4321        let build_result = builder
4322            .add_encryption_key(encryption_key_1.clone())
4323            .build()
4324            .unwrap();
4325
4326        assert_eq!(build_result.changes.len(), 1);
4327        assert_eq!(build_result.metadata.encryption_keys.len(), 1);
4328        assert_eq!(
4329            build_result.metadata.encryption_key("key-1"),
4330            Some(&encryption_key_1)
4331        );
4332        assert_eq!(build_result.changes[0], TableUpdate::AddEncryptionKey {
4333            encryption_key: encryption_key_1.clone()
4334        });
4335
4336        // Add second encryption key
4337        let build_result = build_result
4338            .metadata
4339            .into_builder(Some(
4340                "s3://bucket/test/location/metadata/metadata1.json".to_string(),
4341            ))
4342            .add_encryption_key(encryption_key_2.clone())
4343            .build()
4344            .unwrap();
4345
4346        assert_eq!(build_result.changes.len(), 1);
4347        assert_eq!(build_result.metadata.encryption_keys.len(), 2);
4348        assert_eq!(
4349            build_result.metadata.encryption_key("key-1"),
4350            Some(&encryption_key_1)
4351        );
4352        assert_eq!(
4353            build_result.metadata.encryption_key("key-2"),
4354            Some(&encryption_key_2)
4355        );
4356        assert_eq!(build_result.changes[0], TableUpdate::AddEncryptionKey {
4357            encryption_key: encryption_key_2.clone()
4358        });
4359
4360        // Try to add duplicate key - should not create a change
4361        let build_result = build_result
4362            .metadata
4363            .into_builder(Some(
4364                "s3://bucket/test/location/metadata/metadata2.json".to_string(),
4365            ))
4366            .add_encryption_key(encryption_key_1.clone())
4367            .build()
4368            .unwrap();
4369
4370        assert_eq!(build_result.changes.len(), 0);
4371        assert_eq!(build_result.metadata.encryption_keys.len(), 2);
4372
4373        // Remove first encryption key
4374        let build_result = build_result
4375            .metadata
4376            .into_builder(Some(
4377                "s3://bucket/test/location/metadata/metadata3.json".to_string(),
4378            ))
4379            .remove_encryption_key("key-1")
4380            .build()
4381            .unwrap();
4382
4383        assert_eq!(build_result.changes.len(), 1);
4384        assert_eq!(build_result.metadata.encryption_keys.len(), 1);
4385        assert_eq!(build_result.metadata.encryption_key("key-1"), None);
4386        assert_eq!(
4387            build_result.metadata.encryption_key("key-2"),
4388            Some(&encryption_key_2)
4389        );
4390        assert_eq!(build_result.changes[0], TableUpdate::RemoveEncryptionKey {
4391            key_id: "key-1".to_string()
4392        });
4393
4394        // Try to remove non-existent key - should not create a change
4395        let build_result = build_result
4396            .metadata
4397            .into_builder(Some(
4398                "s3://bucket/test/location/metadata/metadata4.json".to_string(),
4399            ))
4400            .remove_encryption_key("non-existent-key")
4401            .build()
4402            .unwrap();
4403
4404        assert_eq!(build_result.changes.len(), 0);
4405        assert_eq!(build_result.metadata.encryption_keys.len(), 1);
4406
4407        // Test encryption_keys_iter()
4408        let keys = build_result
4409            .metadata
4410            .encryption_keys_iter()
4411            .collect::<Vec<_>>();
4412        assert_eq!(keys.len(), 1);
4413        assert_eq!(keys[0], &encryption_key_2);
4414
4415        // Remove last encryption key
4416        let build_result = build_result
4417            .metadata
4418            .into_builder(Some(
4419                "s3://bucket/test/location/metadata/metadata5.json".to_string(),
4420            ))
4421            .remove_encryption_key("key-2")
4422            .build()
4423            .unwrap();
4424
4425        assert_eq!(build_result.changes.len(), 1);
4426        assert_eq!(build_result.metadata.encryption_keys.len(), 0);
4427        assert_eq!(build_result.metadata.encryption_key("key-2"), None);
4428        assert_eq!(build_result.changes[0], TableUpdate::RemoveEncryptionKey {
4429            key_id: "key-2".to_string()
4430        });
4431
4432        // Verify empty encryption_keys_iter()
4433        let keys = build_result.metadata.encryption_keys_iter();
4434        assert_eq!(keys.len(), 0);
4435    }
4436
4437    #[test]
4438    fn test_partition_field_id_reuse_across_specs() {
4439        let schema = Schema::builder()
4440            .with_fields(vec![
4441                NestedField::required(1, "id", Type::Primitive(PrimitiveType::Long)).into(),
4442                NestedField::required(2, "data", Type::Primitive(PrimitiveType::String)).into(),
4443                NestedField::required(3, "timestamp", Type::Primitive(PrimitiveType::Timestamp))
4444                    .into(),
4445            ])
4446            .build()
4447            .unwrap();
4448
4449        // Create initial table with spec 0: identity(id) -> field_id = 1000
4450        let initial_spec = UnboundPartitionSpec::builder()
4451            .add_partition_field(
4452                UnboundPartitionField::builder()
4453                    .source_ids(vec![1])
4454                    .name("id")
4455                    .transform(Transform::Identity)
4456                    .build()
4457                    .unwrap(),
4458            )
4459            .unwrap()
4460            .build();
4461
4462        let mut metadata = TableMetadataBuilder::new(
4463            schema,
4464            initial_spec,
4465            SortOrder::unsorted_order(),
4466            "s3://bucket/table".to_string(),
4467            FormatVersion::V2,
4468            HashMap::new(),
4469        )
4470        .unwrap()
4471        .build()
4472        .unwrap()
4473        .metadata;
4474
4475        // Add spec 1: bucket(data) -> field_id = 1001
4476        let spec1 = UnboundPartitionSpec::builder()
4477            .add_partition_field(
4478                UnboundPartitionField::builder()
4479                    .source_ids(vec![2])
4480                    .name("data_bucket")
4481                    .transform(Transform::Bucket(10))
4482                    .build()
4483                    .unwrap(),
4484            )
4485            .unwrap()
4486            .build();
4487        let builder = metadata.into_builder(Some("s3://bucket/table/metadata/v1.json".to_string()));
4488        let result = builder.add_partition_spec(spec1).unwrap().build().unwrap();
4489        metadata = result.metadata;
4490
4491        // Add spec 2: identity(id) + bucket(data) + year(timestamp)
4492        // Should reuse field_id 1000 for identity(id) and 1001 for bucket(data)
4493        let spec2 = UnboundPartitionSpec::builder()
4494            .add_partition_field(
4495                UnboundPartitionField::builder()
4496                    .source_ids(vec![1])
4497                    .name("id")
4498                    .transform(Transform::Identity)
4499                    .build()
4500                    .unwrap(),
4501            ) // Should reuse 1000
4502            .unwrap()
4503            .add_partition_field(
4504                UnboundPartitionField::builder()
4505                    .source_ids(vec![2])
4506                    .name("data_bucket")
4507                    .transform(Transform::Bucket(10))
4508                    .build()
4509                    .unwrap(),
4510            ) // Should reuse 1001
4511            .unwrap()
4512            .add_partition_field(
4513                UnboundPartitionField::builder()
4514                    .source_ids(vec![3])
4515                    .name("year")
4516                    .transform(Transform::Year)
4517                    .build()
4518                    .unwrap(),
4519            ) // Should get new 1002
4520            .unwrap()
4521            .build();
4522        let builder = metadata.into_builder(Some("s3://bucket/table/metadata/v2.json".to_string()));
4523        let result = builder.add_partition_spec(spec2).unwrap().build().unwrap();
4524
4525        // Verify field ID reuse: spec 2 should reuse IDs from specs 0 and 1, assign new ID for new field
4526        let spec2 = result.metadata.partition_spec_by_id(2).unwrap();
4527        let field_ids: Vec<i32> = spec2.fields().iter().map(|f| f.field_id).collect();
4528        assert_eq!(field_ids, vec![1000, 1001, 1002]); // Reused 1000, 1001; new 1002
4529    }
4530
4531    fn unbound_field(source_id: i32, name: &str, transform: Transform) -> UnboundPartitionField {
4532        UnboundPartitionField::builder()
4533            .source_ids(vec![source_id])
4534            .name(name)
4535            .transform(transform)
4536            .build()
4537            .unwrap()
4538    }
4539
4540    fn replacement_schema() -> Schema {
4541        // `z` and `x` already exist in the table, `w` is new, `y` is dropped.
4542        Schema::builder()
4543            .with_fields(vec![
4544                NestedField::required(1, "z", Type::Primitive(PrimitiveType::Long)).into(),
4545                NestedField::required(2, "x", Type::Primitive(PrimitiveType::Long)).into(),
4546                NestedField::required(3, "w", Type::Primitive(PrimitiveType::String)).into(),
4547            ])
4548            .build()
4549            .unwrap()
4550    }
4551
4552    fn partition_replacement_builder(format_version: FormatVersion) -> TableMetadataBuilder {
4553        let schema = Schema::builder()
4554            .with_fields(vec![
4555                NestedField::required(1, "x", Type::Primitive(PrimitiveType::Long)).into(),
4556                NestedField::required(2, "y", Type::Primitive(PrimitiveType::Long)).into(),
4557            ])
4558            .build()
4559            .unwrap();
4560        let spec = UnboundPartitionSpec::builder()
4561            .add_partition_field(unbound_field(1, "x", Transform::Identity))
4562            .unwrap()
4563            .add_partition_field(unbound_field(2, "y", Transform::Identity))
4564            .unwrap()
4565            .build();
4566
4567        TableMetadataBuilder::new(
4568            schema,
4569            spec,
4570            SortOrder::unsorted_order(),
4571            TEST_LOCATION.to_string(),
4572            format_version,
4573            HashMap::new(),
4574        )
4575        .unwrap()
4576        .build()
4577        .unwrap()
4578        .metadata
4579        .into_builder(Some(
4580            "s3://bucket/test/location/metadata/metadata1.json".to_string(),
4581        ))
4582    }
4583
4584    fn populated_builder() -> TableMetadataBuilder {
4585        let builder = builder_without_changes(FormatVersion::V2);
4586        let snapshot = Snapshot::builder()
4587            .with_snapshot_id(1)
4588            .with_timestamp_ms(builder.metadata.last_updated_ms + 1)
4589            .with_sequence_number(0)
4590            .with_schema_id(0)
4591            .with_manifest_list("/snap-1.avro")
4592            .with_summary(Summary {
4593                operation: Operation::Append,
4594                additional_properties: HashMap::new(),
4595            })
4596            .build();
4597
4598        builder
4599            .add_snapshot(snapshot)
4600            .unwrap()
4601            .set_ref(MAIN_BRANCH, SnapshotReference {
4602                snapshot_id: 1,
4603                retention: SnapshotRetention::branch(None, None, None),
4604            })
4605            .unwrap()
4606            .build()
4607            .unwrap()
4608            .metadata
4609            .into_builder(Some(
4610                "s3://bucket/test/location/metadata/metadata2.json".to_string(),
4611            ))
4612    }
4613
4614    fn replaced_schema() -> Schema {
4615        Schema::builder()
4616            .with_schema_id(1)
4617            .with_fields(vec![
4618                NestedField::required(3, "z", Type::Primitive(PrimitiveType::Long)).into(),
4619                NestedField::required(1, "x", Type::Primitive(PrimitiveType::Long)).into(),
4620                NestedField::required(4, "w", Type::Primitive(PrimitiveType::String)).into(),
4621            ])
4622            .build()
4623            .unwrap()
4624    }
4625
4626    #[test]
4627    fn test_build_replacement_keeps_existing_field_ids_and_assigns_new_ones_above_last_column_id() {
4628        let metadata = builder_without_changes(FormatVersion::V2)
4629            .build_replacement(
4630                replacement_schema(),
4631                UnboundPartitionSpec::builder().build(),
4632                SortOrder::unsorted_order(),
4633                None,
4634                None,
4635                HashMap::new(),
4636            )
4637            .unwrap()
4638            .build()
4639            .unwrap()
4640            .metadata;
4641
4642        assert_eq!(metadata.current_schema().as_ref(), &replaced_schema());
4643        assert_eq!(metadata.last_column_id, 4);
4644    }
4645
4646    #[test]
4647    fn test_build_replacement_does_not_reuse_the_id_of_a_dropped_column() {
4648        let without_y = Schema::builder()
4649            .with_fields(vec![
4650                NestedField::required(1, "x", Type::Primitive(PrimitiveType::Long)).into(),
4651            ])
4652            .build()
4653            .unwrap();
4654        let replacement = Schema::builder()
4655            .with_fields(vec![
4656                NestedField::required(1, "x", Type::Primitive(PrimitiveType::Long)).into(),
4657                NestedField::required(2, "w", Type::Primitive(PrimitiveType::String)).into(),
4658            ])
4659            .build()
4660            .unwrap();
4661
4662        let metadata = partition_replacement_builder(FormatVersion::V2)
4663            .add_current_schema(without_y)
4664            .unwrap()
4665            .build_replacement(
4666                replacement,
4667                UnboundPartitionSpec::builder().build(),
4668                SortOrder::unsorted_order(),
4669                None,
4670                None,
4671                HashMap::new(),
4672            )
4673            .unwrap()
4674            .build()
4675            .unwrap()
4676            .metadata;
4677
4678        let expected = Schema::builder()
4679            .with_schema_id(2)
4680            .with_fields(vec![
4681                NestedField::required(1, "x", Type::Primitive(PrimitiveType::Long)).into(),
4682                NestedField::required(3, "w", Type::Primitive(PrimitiveType::String)).into(),
4683            ])
4684            .build()
4685            .unwrap();
4686        assert_eq!(metadata.current_schema().as_ref(), &expected);
4687        assert_eq!(metadata.last_column_id, 3);
4688    }
4689
4690    #[test]
4691    fn test_build_replacement_resolves_spec_and_sort_order_source_ids_against_the_replacement_schema()
4692     {
4693        let spec = UnboundPartitionSpec::builder()
4694            .add_partition_field(unbound_field(3, "w", Transform::Identity))
4695            .unwrap()
4696            .build();
4697        let sort_field = SortField {
4698            source_id: 3,
4699            transform: Transform::Identity,
4700            direction: SortDirection::Ascending,
4701            null_order: NullOrder::First,
4702        };
4703        let sort_order = SortOrder::builder()
4704            .with_sort_field(sort_field.clone())
4705            .build(&replacement_schema())
4706            .unwrap();
4707
4708        let metadata = builder_without_changes(FormatVersion::V2)
4709            .build_replacement(
4710                replacement_schema(),
4711                spec,
4712                sort_order,
4713                None,
4714                None,
4715                HashMap::new(),
4716            )
4717            .unwrap()
4718            .build()
4719            .unwrap()
4720            .metadata;
4721
4722        let expected_spec = PartitionSpec::builder(replaced_schema())
4723            .with_spec_id(1)
4724            .add_unbound_field(unbound_field(4, "w", Transform::Identity).with_field_id(1001))
4725            .unwrap()
4726            .build()
4727            .unwrap();
4728        let expected_sort_order = SortOrder::builder()
4729            .with_order_id(2)
4730            .with_sort_field(SortField {
4731                source_id: 4,
4732                ..sort_field
4733            })
4734            .build(&replaced_schema())
4735            .unwrap();
4736        assert_eq!(metadata.default_partition_spec().as_ref(), &expected_spec);
4737        assert_eq!(metadata.default_sort_order().as_ref(), &expected_sort_order);
4738    }
4739
4740    #[test]
4741    fn test_build_replacement_reassigns_partition_ids_for_v2() {
4742        let updated_schema = Schema::builder()
4743            .with_fields(vec![
4744                NestedField::required(1, "x", Type::Primitive(PrimitiveType::Long)).into(),
4745                NestedField::required(2, "z", Type::Primitive(PrimitiveType::String)).into(),
4746            ])
4747            .build()
4748            .unwrap();
4749        let updated_spec = UnboundPartitionSpec::builder()
4750            .add_partition_field(unbound_field(2, "z_bucket", Transform::Bucket(8)))
4751            .unwrap()
4752            .add_partition_field(unbound_field(1, "x", Transform::Identity))
4753            .unwrap()
4754            .build();
4755
4756        let result = partition_replacement_builder(FormatVersion::V2)
4757            .build_replacement(
4758                updated_schema,
4759                updated_spec,
4760                SortOrder::unsorted_order(),
4761                None,
4762                None,
4763                HashMap::new(),
4764            )
4765            .unwrap()
4766            .build()
4767            .unwrap();
4768        let metadata = &result.metadata;
4769
4770        let expected_spec = PartitionSpec::builder(metadata.current_schema().clone())
4771            .with_spec_id(1)
4772            .add_unbound_fields([
4773                unbound_field(3, "z_bucket", Transform::Bucket(8)).with_field_id(1002),
4774                unbound_field(1, "x", Transform::Identity).with_field_id(1000),
4775            ])
4776            .unwrap()
4777            .build()
4778            .unwrap();
4779        assert_eq!(metadata.default_partition_spec().as_ref(), &expected_spec);
4780        assert_eq!(metadata.last_partition_id, 1002);
4781        assert!(result.changes.contains(&TableUpdate::AddSpec {
4782            spec: expected_spec.into_unbound()
4783        }));
4784    }
4785
4786    #[test]
4787    fn test_build_replacement_rejects_a_new_partition_field_when_no_partition_id_is_left() {
4788        let mut builder = partition_replacement_builder(FormatVersion::V2);
4789        builder.metadata.last_partition_id = i32::MAX;
4790        let updated_schema = Schema::builder()
4791            .with_fields(vec![
4792                NestedField::required(1, "x", Type::Primitive(PrimitiveType::Long)).into(),
4793                NestedField::required(2, "z", Type::Primitive(PrimitiveType::String)).into(),
4794            ])
4795            .build()
4796            .unwrap();
4797        let updated_spec = UnboundPartitionSpec::builder()
4798            .add_partition_field(unbound_field(2, "z_bucket", Transform::Bucket(8)))
4799            .unwrap()
4800            .build();
4801
4802        let err = builder
4803            .build_replacement(
4804                updated_schema,
4805                updated_spec,
4806                SortOrder::unsorted_order(),
4807                None,
4808                None,
4809                HashMap::new(),
4810            )
4811            .unwrap_err();
4812
4813        assert_eq!(
4814            (err.kind(), err.message()),
4815            (
4816                ErrorKind::DataInvalid,
4817                "Cannot assign more partition ids. Overflow."
4818            )
4819        );
4820    }
4821
4822    #[test]
4823    fn test_build_replacement_keeps_the_table_and_its_snapshots_but_not_its_current_one() {
4824        let base = populated_builder();
4825        let previous = base.metadata.clone();
4826
4827        let metadata = base
4828            .build_replacement(
4829                replacement_schema(),
4830                UnboundPartitionSpec::builder().build(),
4831                SortOrder::unsorted_order(),
4832                None,
4833                None,
4834                HashMap::new(),
4835            )
4836            .unwrap()
4837            .build()
4838            .unwrap()
4839            .metadata;
4840
4841        assert_eq!(metadata.table_uuid, previous.table_uuid);
4842        assert_eq!(metadata.location, previous.location);
4843        assert_eq!(metadata.current_snapshot_id, None);
4844        assert_eq!(metadata.snapshots, previous.snapshots);
4845        assert_eq!(metadata.snapshot_log, previous.snapshot_log);
4846        assert_eq!(metadata.schema_by_id(0), previous.schema_by_id(0));
4847    }
4848
4849    #[test]
4850    fn test_build_replacement_then_adding_a_snapshot_makes_it_current_on_main() {
4851        let builder = populated_builder()
4852            .build_replacement(
4853                replacement_schema(),
4854                UnboundPartitionSpec::builder().build(),
4855                SortOrder::unsorted_order(),
4856                None,
4857                None,
4858                HashMap::new(),
4859            )
4860            .unwrap();
4861        let snapshot = Snapshot::builder()
4862            .with_snapshot_id(2)
4863            .with_timestamp_ms(builder.metadata.last_updated_ms + 1)
4864            .with_sequence_number(builder.metadata.next_sequence_number())
4865            .with_schema_id(builder.metadata.current_schema_id)
4866            .with_manifest_list("/snap-2.avro")
4867            .with_summary(Summary {
4868                operation: Operation::Overwrite,
4869                additional_properties: HashMap::new(),
4870            })
4871            .build();
4872        let main = SnapshotReference {
4873            snapshot_id: 2,
4874            retention: SnapshotRetention::branch(None, None, None),
4875        };
4876
4877        let metadata = builder
4878            .add_snapshot(snapshot.clone())
4879            .unwrap()
4880            .set_ref(MAIN_BRANCH, main.clone())
4881            .unwrap()
4882            .build()
4883            .unwrap()
4884            .metadata;
4885
4886        assert_eq!(metadata.current_snapshot(), Some(&Arc::new(snapshot)));
4887        assert_eq!(
4888            metadata.refs,
4889            HashMap::from([(MAIN_BRANCH.to_string(), main)])
4890        );
4891    }
4892
4893    #[test]
4894    fn test_build_replacement_changes_describe_the_replacement() {
4895        let changes = populated_builder()
4896            .build_replacement(
4897                replacement_schema(),
4898                UnboundPartitionSpec::builder().build(),
4899                SortOrder::unsorted_order(),
4900                None,
4901                None,
4902                HashMap::new(),
4903            )
4904            .unwrap()
4905            .build()
4906            .unwrap()
4907            .changes;
4908
4909        assert_eq!(changes, vec![
4910            TableUpdate::RemoveSnapshotRef {
4911                ref_name: MAIN_BRANCH.to_string()
4912            },
4913            TableUpdate::AddSchema {
4914                schema: replaced_schema()
4915            },
4916            TableUpdate::SetCurrentSchema {
4917                schema_id: TableMetadataBuilder::LAST_ADDED
4918            },
4919            TableUpdate::AddSpec {
4920                spec: UnboundPartitionSpec::builder().with_spec_id(1).build()
4921            },
4922            TableUpdate::SetDefaultSpec {
4923                spec_id: TableMetadataBuilder::LAST_ADDED
4924            },
4925            TableUpdate::AddSortOrder {
4926                sort_order: SortOrder::unsorted_order()
4927            },
4928            TableUpdate::SetDefaultSortOrder {
4929                sort_order_id: TableMetadataBuilder::LAST_ADDED as i64
4930            },
4931        ]);
4932    }
4933
4934    #[test]
4935    fn test_build_replacement_moves_the_table_when_given_a_location() {
4936        let metadata = builder_without_changes(FormatVersion::V2)
4937            .build_replacement(
4938                replacement_schema(),
4939                UnboundPartitionSpec::builder().build(),
4940                SortOrder::unsorted_order(),
4941                Some("s3://bucket/somewhere/else".to_string()),
4942                None,
4943                HashMap::new(),
4944            )
4945            .unwrap()
4946            .build()
4947            .unwrap()
4948            .metadata;
4949
4950        assert_eq!(metadata.location, "s3://bucket/somewhere/else");
4951    }
4952
4953    #[test]
4954    fn test_build_replacement_applies_format_version_and_properties() {
4955        let result = builder_without_changes(FormatVersion::V2)
4956            .set_properties(HashMap::from_iter([(
4957                "existing".to_string(),
4958                "value".to_string(),
4959            )]))
4960            .unwrap()
4961            .build_replacement(
4962                replacement_schema(),
4963                UnboundPartitionSpec::builder().build(),
4964                SortOrder::unsorted_order(),
4965                None,
4966                Some(FormatVersion::V3),
4967                HashMap::from_iter([("new".to_string(), "value".to_string())]),
4968            )
4969            .unwrap()
4970            .build()
4971            .unwrap();
4972
4973        assert_eq!(result.metadata.format_version, FormatVersion::V3);
4974        assert_eq!(
4975            result.metadata.properties,
4976            HashMap::from_iter([
4977                ("existing".to_string(), "value".to_string()),
4978                ("new".to_string(), "value".to_string()),
4979            ])
4980        );
4981        assert!(result.changes.contains(&TableUpdate::UpgradeFormatVersion {
4982            format_version: FormatVersion::V3
4983        }));
4984        assert!(result.changes.contains(&TableUpdate::SetProperties {
4985            updates: HashMap::from_iter([("new".to_string(), "value".to_string())])
4986        }));
4987    }
4988
4989    #[test]
4990    fn test_build_replacement_with_everything_unchanged_reuses_the_existing_ids() {
4991        let base = builder_without_changes(FormatVersion::V2)
4992            .build()
4993            .unwrap()
4994            .metadata;
4995        let schema = base.current_schema().as_ref().clone();
4996        let spec = base
4997            .default_partition_spec()
4998            .as_ref()
4999            .clone()
5000            .into_unbound();
5001        let sort_order = base.default_sort_order().as_ref().clone();
5002
5003        let result = base
5004            .clone()
5005            .into_builder(Some(
5006                "s3://bucket/test/location/metadata/metadata2.json".to_string(),
5007            ))
5008            .build_replacement(
5009                schema.clone(),
5010                spec.clone(),
5011                sort_order.clone(),
5012                None,
5013                Some(FormatVersion::V3),
5014                HashMap::new(),
5015            )
5016            .unwrap()
5017            .build()
5018            .unwrap();
5019
5020        let metadata = &result.metadata;
5021        assert_eq!(metadata.schemas, base.schemas);
5022        assert_eq!(metadata.current_schema_id, base.current_schema_id);
5023        assert_eq!(metadata.last_column_id, base.last_column_id);
5024        assert_eq!(metadata.partition_specs, base.partition_specs);
5025        assert_eq!(metadata.default_spec, base.default_spec);
5026        assert_eq!(metadata.last_partition_id, base.last_partition_id);
5027        assert_eq!(metadata.sort_orders, base.sort_orders);
5028        assert_eq!(metadata.default_sort_order_id, base.default_sort_order_id);
5029        assert_eq!(result.changes, vec![
5030            TableUpdate::UpgradeFormatVersion {
5031                format_version: FormatVersion::V3
5032            },
5033            TableUpdate::AddSchema { schema },
5034            TableUpdate::AddSpec { spec },
5035            TableUpdate::AddSortOrder { sort_order },
5036        ]);
5037    }
5038
5039    #[test]
5040    fn test_build_replacement_rejects_v1() {
5041        let err = builder_without_changes(FormatVersion::V1)
5042            .build_replacement(
5043                replacement_schema(),
5044                UnboundPartitionSpec::builder().build(),
5045                SortOrder::unsorted_order(),
5046                None,
5047                None,
5048                HashMap::new(),
5049            )
5050            .unwrap_err();
5051
5052        assert_eq!(
5053            (err.kind(), err.message()),
5054            (
5055                ErrorKind::FeatureUnsupported,
5056                "Table replacement is not supported for format version 1"
5057            )
5058        );
5059    }
5060
5061    #[test]
5062    fn test_build_replacement_rejects_v1_with_format_upgrade() {
5063        let err = builder_without_changes(FormatVersion::V1)
5064            .build_replacement(
5065                replacement_schema(),
5066                UnboundPartitionSpec::builder().build(),
5067                SortOrder::unsorted_order(),
5068                None,
5069                Some(FormatVersion::V2),
5070                HashMap::new(),
5071            )
5072            .unwrap_err();
5073
5074        assert_eq!(
5075            (err.kind(), err.message()),
5076            (
5077                ErrorKind::FeatureUnsupported,
5078                "Table replacement is not supported for format version 1"
5079            )
5080        );
5081    }
5082
5083    #[test]
5084    fn test_build_replacement_rejects_reserved_properties() {
5085        let err = builder_without_changes(FormatVersion::V2)
5086            .build_replacement(
5087                replacement_schema(),
5088                UnboundPartitionSpec::builder().build(),
5089                SortOrder::unsorted_order(),
5090                None,
5091                None,
5092                HashMap::from_iter([(
5093                    TableProperties::PROPERTY_UUID.to_string(),
5094                    "ignored".to_string(),
5095                )]),
5096            )
5097            .unwrap_err();
5098
5099        assert_eq!(
5100            (err.kind(), err.message()),
5101            (
5102                ErrorKind::DataInvalid,
5103                "Table properties should not contain reserved properties, but got: [uuid]"
5104            )
5105        );
5106    }
5107
5108    #[test]
5109    fn test_build_replacement_rejects_a_spec_on_a_missing_column() {
5110        let err = builder_without_changes(FormatVersion::V2)
5111            .build_replacement(
5112                replacement_schema(),
5113                UnboundPartitionSpec::builder()
5114                    .add_partition_field(unbound_field(99, "nope", Transform::Identity))
5115                    .unwrap()
5116                    .build(),
5117                SortOrder::unsorted_order(),
5118                None,
5119                None,
5120                HashMap::new(),
5121            )
5122            .unwrap_err();
5123
5124        assert_eq!(
5125            (err.kind(), err.message()),
5126            (
5127                ErrorKind::DataInvalid,
5128                "Cannot find source column with id 99 for partition column nope in schema."
5129            )
5130        );
5131    }
5132
5133    #[test]
5134    fn test_build_replacement_rejects_a_format_version_downgrade() {
5135        let err = builder_without_changes(FormatVersion::V3)
5136            .build_replacement(
5137                replacement_schema(),
5138                UnboundPartitionSpec::builder().build(),
5139                SortOrder::unsorted_order(),
5140                None,
5141                Some(FormatVersion::V2),
5142                HashMap::new(),
5143            )
5144            .unwrap_err();
5145
5146        assert_eq!(
5147            (err.kind(), err.message()),
5148            (
5149                ErrorKind::DataInvalid,
5150                "Cannot downgrade FormatVersion from v3 to v2"
5151            )
5152        );
5153    }
5154
5155    #[test]
5156    fn test_build_replacement_rejects_a_sort_order_on_a_missing_column() {
5157        let sort_order = SortOrder::builder()
5158            .with_sort_field(SortField {
5159                source_id: 99,
5160                transform: Transform::Identity,
5161                direction: SortDirection::Ascending,
5162                null_order: NullOrder::First,
5163            })
5164            .build_unbound()
5165            .unwrap();
5166
5167        let err = builder_without_changes(FormatVersion::V2)
5168            .build_replacement(
5169                replacement_schema(),
5170                UnboundPartitionSpec::builder().build(),
5171                sort_order,
5172                None,
5173                None,
5174                HashMap::new(),
5175            )
5176            .unwrap_err();
5177
5178        assert_eq!(
5179            (err.kind(), err.message()),
5180            (
5181                ErrorKind::DataInvalid,
5182                "Cannot find source column with id 99 for sort column in schema."
5183            )
5184        );
5185    }
5186
5187    #[test]
5188    fn test_build_replacement_rejects_a_column_named_after_a_partition_field() {
5189        let builder = TableMetadataBuilder::new(
5190            schema(),
5191            UnboundPartitionSpec::builder()
5192                .add_partition_field(unbound_field(1, "x_bucket", Transform::Bucket(8)))
5193                .unwrap()
5194                .build(),
5195            SortOrder::unsorted_order(),
5196            TEST_LOCATION.to_string(),
5197            FormatVersion::V2,
5198            HashMap::new(),
5199        )
5200        .unwrap()
5201        .build()
5202        .unwrap()
5203        .metadata
5204        .into_builder(Some(
5205            "s3://bucket/test/location/metadata/metadata1.json".to_string(),
5206        ));
5207        let replacement = Schema::builder()
5208            .with_fields(vec![
5209                NestedField::required(1, "x", Type::Primitive(PrimitiveType::Long)).into(),
5210                NestedField::required(2, "x_bucket", Type::Primitive(PrimitiveType::Long)).into(),
5211            ])
5212            .build()
5213            .unwrap();
5214
5215        let err = builder
5216            .build_replacement(
5217                replacement,
5218                UnboundPartitionSpec::builder().build(),
5219                SortOrder::unsorted_order(),
5220                None,
5221                None,
5222                HashMap::new(),
5223            )
5224            .unwrap_err();
5225
5226        assert_eq!(
5227            (err.kind(), err.message()),
5228            (
5229                ErrorKind::DataInvalid,
5230                "Cannot set current schema 1: field 'x_bucket' with id 4 conflicts with a \
5231                 partition field of that name in spec 0 sourced from field id 1"
5232            )
5233        );
5234    }
5235
5236    #[test]
5237    fn test_build_replacement_keeps_refs_other_than_main() {
5238        let audit = SnapshotReference {
5239            snapshot_id: 1,
5240            retention: SnapshotRetention::branch(None, None, None),
5241        };
5242        let release = SnapshotReference {
5243            snapshot_id: 1,
5244            retention: SnapshotRetention::Tag {
5245                max_ref_age_ms: None,
5246            },
5247        };
5248        let builder = populated_builder()
5249            .set_ref("audit", audit.clone())
5250            .unwrap()
5251            .set_ref("release", release.clone())
5252            .unwrap();
5253
5254        let metadata = builder
5255            .build_replacement(
5256                replacement_schema(),
5257                UnboundPartitionSpec::builder().build(),
5258                SortOrder::unsorted_order(),
5259                None,
5260                None,
5261                HashMap::new(),
5262            )
5263            .unwrap()
5264            .build()
5265            .unwrap()
5266            .metadata;
5267
5268        assert_eq!(
5269            metadata.refs,
5270            HashMap::from_iter([
5271                ("audit".to_string(), audit),
5272                ("release".to_string(), release),
5273            ])
5274        );
5275        assert_eq!(metadata.current_snapshot_id, None);
5276    }
5277
5278    #[test]
5279    fn test_build_replacement_keeps_the_id_of_a_column_whose_type_changes() {
5280        let replacement = Schema::builder()
5281            .with_fields(vec![
5282                NestedField::required(1, "x", Type::Primitive(PrimitiveType::String)).into(),
5283            ])
5284            .build()
5285            .unwrap();
5286
5287        let metadata = builder_without_changes(FormatVersion::V2)
5288            .build_replacement(
5289                replacement,
5290                UnboundPartitionSpec::builder().build(),
5291                SortOrder::unsorted_order(),
5292                None,
5293                None,
5294                HashMap::new(),
5295            )
5296            .unwrap()
5297            .build()
5298            .unwrap()
5299            .metadata;
5300
5301        let x = metadata.current_schema().field_by_name("x").unwrap();
5302        assert_eq!(x.id, 1);
5303        assert_eq!(*x.field_type, Type::Primitive(PrimitiveType::String));
5304    }
5305
5306    #[test]
5307    fn test_build_replacement_keeps_the_existing_compression_codec() {
5308        let codec = HashMap::from_iter([(
5309            TableProperties::PROPERTY_PARQUET_COMPRESSION_CODEC.to_string(),
5310            "gzip".to_string(),
5311        )]);
5312
5313        let metadata = builder_without_changes(FormatVersion::V2)
5314            .set_properties(codec.clone())
5315            .unwrap()
5316            .build_replacement(
5317                replacement_schema(),
5318                UnboundPartitionSpec::builder().build(),
5319                SortOrder::unsorted_order(),
5320                None,
5321                None,
5322                HashMap::new(),
5323            )
5324            .unwrap()
5325            .build()
5326            .unwrap()
5327            .metadata;
5328
5329        assert_eq!(metadata.properties, codec);
5330    }
5331
5332    #[test]
5333    fn test_highest_equivalent_field_ids_is_independent_of_field_order() {
5334        let field = |source_id, field_id, transform| PartitionField {
5335            source_id,
5336            field_id,
5337            name: format!("f{field_id}"),
5338            transform,
5339        };
5340        let fields = [
5341            field(1, 1000, Transform::Identity),
5342            field(2, 1003, Transform::Identity),
5343            field(1, 1006, Transform::Identity),
5344            field(1, 1005, Transform::Bucket(16)),
5345            field(2, 1001, Transform::Identity),
5346            field(1, 1002, Transform::Identity),
5347            field(1, 1004, Transform::Bucket(16)),
5348        ];
5349        let expected = HashMap::from([
5350            ((vec![1], Transform::Identity), 1006),
5351            ((vec![2], Transform::Identity), 1003),
5352            ((vec![1], Transform::Bucket(16)), 1005),
5353        ]);
5354
5355        assert_eq!(
5356            TableMetadataBuilder::highest_equivalent_field_ids(fields.iter()),
5357            expected
5358        );
5359        assert_eq!(
5360            TableMetadataBuilder::highest_equivalent_field_ids(fields.iter().rev()),
5361            expected
5362        );
5363    }
5364
5365    #[test]
5366    fn test_partition_field_id_reuse_picks_the_highest_of_several_candidates() {
5367        // These specs give identity(x) four different field ids, which a writer that does not
5368        // reuse ids can leave behind. `partition_specs` is a HashMap, so the id reused for a new
5369        // spec must not depend on which of them is visited first.
5370        let metadata: TableMetadata = serde_json::from_value(serde_json::json!({
5371            "format-version": 2,
5372            "table-uuid": "9c12d441-03fe-4693-9a96-a0705ddf69c1",
5373            "location": "s3://bucket/test/location",
5374            "last-sequence-number": 0,
5375            "last-updated-ms": 1602638573590i64,
5376            "last-column-id": 2,
5377            "current-schema-id": 0,
5378            "schemas": [{
5379                "type": "struct",
5380                "schema-id": 0,
5381                "fields": [
5382                    {"id": 1, "name": "x", "required": true, "type": "long"},
5383                    {"id": 2, "name": "y", "required": true, "type": "long"},
5384                ],
5385            }],
5386            "default-spec-id": 3,
5387            "last-partition-id": 1006,
5388            "partition-specs": [
5389                {"spec-id": 0, "fields": [
5390                    {"name": "x", "transform": "identity", "source-id": 1, "field-id": 1000},
5391                ]},
5392                {"spec-id": 1, "fields": [
5393                    {"name": "x", "transform": "identity", "source-id": 1, "field-id": 1002},
5394                ]},
5395                {"spec-id": 2, "fields": [
5396                    {"name": "x", "transform": "identity", "source-id": 1, "field-id": 1004},
5397                ]},
5398                {"spec-id": 3, "fields": [
5399                    {"name": "x", "transform": "identity", "source-id": 1, "field-id": 1006},
5400                ]},
5401            ],
5402            "default-sort-order-id": 0,
5403            "sort-orders": [{"order-id": 0, "fields": []}],
5404        }))
5405        .unwrap();
5406
5407        // Two fields, so this is not compatible with any existing spec and is added as a new
5408        // one rather than resolving to an existing spec id.
5409        let new_spec = UnboundPartitionSpec::builder()
5410            .add_partition_field(
5411                UnboundPartitionField::builder()
5412                    .source_ids(vec![1])
5413                    .name("x")
5414                    .transform(Transform::Identity)
5415                    .build()
5416                    .unwrap(),
5417            )
5418            .unwrap()
5419            .add_partition_field(
5420                UnboundPartitionField::builder()
5421                    .source_ids(vec![2])
5422                    .name("y")
5423                    .transform(Transform::Identity)
5424                    .build()
5425                    .unwrap(),
5426            )
5427            .unwrap()
5428            .build();
5429
5430        let result = metadata
5431            .into_builder(Some(
5432                "s3://bucket/test/location/metadata/v1.json".to_string(),
5433            ))
5434            .add_partition_spec(new_spec)
5435            .unwrap()
5436            .build()
5437            .unwrap();
5438
5439        let spec = result.metadata.partition_spec_by_id(4).unwrap();
5440        let fields: Vec<(i32, i32)> = spec
5441            .fields()
5442            .iter()
5443            .map(|f| (f.source_id, f.field_id))
5444            .collect();
5445        assert_eq!(fields, vec![(1, 1006), (2, 1007)]);
5446        assert_eq!(result.metadata.last_partition_id, 1007);
5447    }
5448}