Skip to main content

iceberg/spec/
table_metadata.rs

1// Licensed to the Apache Software Foundation (ASF) under one
2// or more contributor license agreements.  See the NOTICE file
3// distributed with this work for additional information
4// regarding copyright ownership.  The ASF licenses this file
5// to you under the Apache License, Version 2.0 (the
6// "License"); you may not use this file except in compliance
7// with the License.  You may obtain a copy of the License at
8//
9//   http://www.apache.org/licenses/LICENSE-2.0
10//
11// Unless required by applicable law or agreed to in writing,
12// software distributed under the License is distributed on an
13// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14// KIND, either express or implied.  See the License for the
15// specific language governing permissions and limitations
16// under the License.
17
18//! Defines the [table metadata](https://iceberg.apache.org/spec/#table-metadata).
19//! The main struct here is [TableMetadataV2] which defines the data for a table.
20
21use std::cmp::Ordering;
22use std::collections::HashMap;
23use std::fmt::{Display, Formatter};
24use std::hash::Hash;
25use std::sync::Arc;
26
27use _serde::TableMetadataEnum;
28use chrono::{DateTime, Utc};
29use serde::{Deserialize, Serialize};
30use serde_repr::{Deserialize_repr, Serialize_repr};
31use uuid::Uuid;
32
33use super::snapshot::SnapshotReference;
34pub use super::table_metadata_builder::{TableMetadataBuildResult, TableMetadataBuilder};
35use super::{
36    DEFAULT_PARTITION_SPEC_ID, PartitionSpecRef, PartitionStatisticsFile, Schema, SchemaId,
37    SchemaRef, SnapshotRef, SnapshotRetention, SortOrder, SortOrderRef, StatisticsFile, StructType,
38    TableProperties, Transform,
39};
40use crate::catalog::{METADATA_FOLDER_NAME, MetadataLocation};
41use crate::compression::CompressionCodec;
42use crate::error::{Result, invalid_data, timestamp_ms_to_utc};
43use crate::io::FileIO;
44use crate::partitioning::compute_unified_partition_type;
45use crate::spec::EncryptedKey;
46use crate::{Error, ErrorKind};
47
48static MAIN_BRANCH: &str = "main";
49pub(crate) static ONE_MINUTE_MS: i64 = 60_000;
50
51/// Sentinel value used by the Java implementation and older metadata files
52/// to represent a missing/empty current snapshot ID. During deserialization,
53/// this value is normalized to `None`.
54pub(crate) static EMPTY_SNAPSHOT_ID: i64 = -1;
55pub(crate) static INITIAL_SEQUENCE_NUMBER: i64 = 0;
56
57/// Initial row id for row lineage for new v3 tables and older tables upgrading to v3.
58pub const INITIAL_ROW_ID: u64 = 0;
59/// Minimum format version that supports row lineage (v3).
60pub const MIN_FORMAT_VERSION_ROW_LINEAGE: FormatVersion = FormatVersion::V3;
61/// Reference to [`TableMetadata`].
62pub type TableMetadataRef = Arc<TableMetadata>;
63
64#[derive(Debug, PartialEq, Deserialize, Eq, Clone)]
65#[serde(try_from = "TableMetadataEnum")]
66/// Fields for the version 2 of the table metadata.
67///
68/// We assume that this data structure is always valid, so we will panic when invalid error happens.
69/// We check the validity of this data structure when constructing.
70pub struct TableMetadata {
71    /// Integer Version for the format.
72    pub(crate) format_version: FormatVersion,
73    /// A UUID that identifies the table
74    pub(crate) table_uuid: Uuid,
75    /// Location tables base location
76    pub(crate) location: String,
77    /// The tables highest sequence number
78    pub(crate) last_sequence_number: i64,
79    /// Timestamp in milliseconds from the unix epoch when the table was last updated.
80    pub(crate) last_updated_ms: i64,
81    /// An integer; the highest assigned column ID for the table.
82    pub(crate) last_column_id: i32,
83    /// A list of schemas, stored as objects with schema-id.
84    pub(crate) schemas: HashMap<i32, SchemaRef>,
85    /// ID of the table’s current schema.
86    pub(crate) current_schema_id: i32,
87    /// A list of partition specs, stored as full partition spec objects.
88    pub(crate) partition_specs: HashMap<i32, PartitionSpecRef>,
89    /// ID of the “current” spec that writers should use by default.
90    pub(crate) default_spec: PartitionSpecRef,
91    /// Partition type of the default partition spec.
92    pub(crate) default_partition_type: StructType,
93    /// An integer; the highest assigned partition field ID across all partition specs for the table.
94    pub(crate) last_partition_id: i32,
95    ///A string to string map of table properties. This is used to control settings that
96    /// affect reading and writing and is not intended to be used for arbitrary metadata.
97    /// For example, commit.retry.num-retries is used to control the number of commit retries.
98    pub(crate) properties: HashMap<String, String>,
99    /// long ID of the current table snapshot; must be the same as the current
100    /// ID of the main branch in refs.
101    pub(crate) current_snapshot_id: Option<i64>,
102    ///A list of valid snapshots. Valid snapshots are snapshots for which all
103    /// data files exist in the file system. A data file must not be deleted
104    /// from the file system until the last snapshot in which it was listed is
105    /// garbage collected.
106    pub(crate) snapshots: HashMap<i64, SnapshotRef>,
107    /// A list (optional) of timestamp and snapshot ID pairs that encodes changes
108    /// to the current snapshot for the table. Each time the current-snapshot-id
109    /// is changed, a new entry should be added with the last-updated-ms
110    /// and the new current-snapshot-id. When snapshots are expired from
111    /// the list of valid snapshots, all entries before a snapshot that has
112    /// expired should be removed.
113    pub(crate) snapshot_log: Vec<SnapshotLog>,
114
115    /// A list (optional) of timestamp and metadata file location pairs
116    /// that encodes changes to the previous metadata files for the table.
117    /// Each time a new metadata file is created, a new entry of the
118    /// previous metadata file location should be added to the list.
119    /// Tables can be configured to remove the oldest metadata log entries and
120    /// keep a fixed-size log of the most recent entries after a commit.
121    pub(crate) metadata_log: Vec<MetadataLog>,
122
123    /// A list of sort orders, stored as full sort order objects.
124    pub(crate) sort_orders: HashMap<i64, SortOrderRef>,
125    /// Default sort order id of the table. Note that this could be used by
126    /// writers, but is not used when reading because reads use the specs
127    /// stored in manifest files.
128    pub(crate) default_sort_order_id: i64,
129    /// A map of snapshot references. The map keys are the unique snapshot reference
130    /// names in the table, and the map values are snapshot reference objects.
131    /// There is always a main branch reference pointing to the current-snapshot-id
132    /// even if the refs map is null.
133    pub(crate) refs: HashMap<String, SnapshotReference>,
134    /// Mapping of snapshot ids to statistics files.
135    pub(crate) statistics: HashMap<i64, StatisticsFile>,
136    /// Mapping of snapshot ids to partition statistics files.
137    pub(crate) partition_statistics: HashMap<i64, PartitionStatisticsFile>,
138    /// Encryption Keys - map of key id to the actual key
139    pub(crate) encryption_keys: HashMap<String, EncryptedKey>,
140    /// Next row id to be assigned for Row Lineage (v3)
141    pub(crate) next_row_id: u64,
142}
143
144impl TableMetadata {
145    /// Convert this Table Metadata into a builder for modification.
146    ///
147    /// `current_file_location` is the location where the current version
148    /// of the metadata file is stored. This is used to update the metadata log.
149    /// If `current_file_location` is `None`, the metadata log will not be updated.
150    /// This should only be used to stage-create tables.
151    #[must_use]
152    pub fn into_builder(self, current_file_location: Option<String>) -> TableMetadataBuilder {
153        TableMetadataBuilder::new_from_metadata(self, current_file_location)
154    }
155
156    /// Check if a field name exists in any schema.
157    #[inline]
158    pub(crate) fn name_exists_in_any_schema(&self, name: &str) -> bool {
159        self.schemas
160            .values()
161            .any(|schema| schema.field_by_name(name).is_some())
162    }
163
164    /// Returns format version of this metadata.
165    #[inline]
166    pub fn format_version(&self) -> FormatVersion {
167        self.format_version
168    }
169
170    /// Returns uuid of current table.
171    #[inline]
172    pub fn uuid(&self) -> Uuid {
173        self.table_uuid
174    }
175
176    /// Returns table location.
177    #[inline]
178    pub fn location(&self) -> &str {
179        self.location.as_str()
180    }
181
182    /// Returns last sequence number.
183    #[inline]
184    pub fn last_sequence_number(&self) -> i64 {
185        self.last_sequence_number
186    }
187
188    /// Returns the next sequence number for the table.
189    ///
190    /// For format version 1, it always returns the initial sequence number.
191    /// For other versions, it returns the last sequence number incremented by 1.
192    #[inline]
193    pub fn next_sequence_number(&self) -> i64 {
194        match self.format_version {
195            FormatVersion::V1 => INITIAL_SEQUENCE_NUMBER,
196            _ => self.last_sequence_number + 1,
197        }
198    }
199
200    /// Returns the last column id.
201    #[inline]
202    pub fn last_column_id(&self) -> i32 {
203        self.last_column_id
204    }
205
206    /// Returns the last partition_id
207    #[inline]
208    pub fn last_partition_id(&self) -> i32 {
209        self.last_partition_id
210    }
211
212    /// Returns last updated time.
213    #[inline]
214    pub fn last_updated_timestamp(&self) -> Result<DateTime<Utc>> {
215        timestamp_ms_to_utc(self.last_updated_ms)
216    }
217
218    /// Returns last updated time in milliseconds.
219    #[inline]
220    pub fn last_updated_ms(&self) -> i64 {
221        self.last_updated_ms
222    }
223
224    /// Returns schemas
225    #[inline]
226    pub fn schemas_iter(&self) -> impl ExactSizeIterator<Item = &SchemaRef> {
227        self.schemas.values()
228    }
229
230    /// Lookup schema by id.
231    #[inline]
232    pub fn schema_by_id(&self, schema_id: SchemaId) -> Option<&SchemaRef> {
233        self.schemas.get(&schema_id)
234    }
235
236    /// Get current schema
237    #[inline]
238    pub fn current_schema(&self) -> &SchemaRef {
239        self.schema_by_id(self.current_schema_id)
240            .expect("Current schema id set, but not found in table metadata")
241    }
242
243    /// Get the id of the current schema
244    #[inline]
245    pub fn current_schema_id(&self) -> SchemaId {
246        self.current_schema_id
247    }
248
249    /// Returns all partition specs.
250    #[inline]
251    pub fn partition_specs_iter(&self) -> impl ExactSizeIterator<Item = &PartitionSpecRef> {
252        self.partition_specs.values()
253    }
254
255    /// Lookup partition spec by id.
256    #[inline]
257    pub fn partition_spec_by_id(&self, spec_id: i32) -> Option<&PartitionSpecRef> {
258        self.partition_specs.get(&spec_id)
259    }
260
261    /// Get default partition spec
262    #[inline]
263    pub fn default_partition_spec(&self) -> &PartitionSpecRef {
264        &self.default_spec
265    }
266
267    /// Return the partition type of the default partition spec.
268    #[inline]
269    pub fn default_partition_type(&self) -> &StructType {
270        &self.default_partition_type
271    }
272
273    /// Returns the unified partition type across every partition spec in the table, resolved
274    /// against `schema`.
275    ///
276    /// Unlike [`Self::default_partition_type`], the result contains all partition fields ever
277    /// used by the table, so partition values stay readable across partition spec evolution.
278    /// See [`compute_unified_partition_type`] for the exact merge rules.
279    pub fn unified_partition_type(&self, schema: &Schema) -> Result<StructType> {
280        compute_unified_partition_type(
281            self.partition_specs_iter().map(|spec| spec.as_ref()),
282            schema,
283        )
284    }
285
286    #[inline]
287    /// Returns spec id of the "current" partition spec.
288    pub fn default_partition_spec_id(&self) -> i32 {
289        self.default_spec.spec_id()
290    }
291
292    /// Returns all snapshots
293    #[inline]
294    pub fn snapshots(&self) -> impl ExactSizeIterator<Item = &SnapshotRef> {
295        self.snapshots.values()
296    }
297
298    /// Lookup snapshot by id.
299    #[inline]
300    pub fn snapshot_by_id(&self, snapshot_id: i64) -> Option<&SnapshotRef> {
301        self.snapshots.get(&snapshot_id)
302    }
303
304    /// Returns snapshot history.
305    #[inline]
306    pub fn history(&self) -> &[SnapshotLog] {
307        &self.snapshot_log
308    }
309
310    /// Returns the metadata log.
311    #[inline]
312    pub fn metadata_log(&self) -> &[MetadataLog] {
313        &self.metadata_log
314    }
315
316    /// Get current snapshot
317    #[inline]
318    pub fn current_snapshot(&self) -> Option<&SnapshotRef> {
319        self.current_snapshot_id.map(|s| {
320            self.snapshot_by_id(s)
321                .expect("Current snapshot id has been set, but doesn't exist in metadata")
322        })
323    }
324
325    /// Get the current snapshot id
326    #[inline]
327    pub fn current_snapshot_id(&self) -> Option<i64> {
328        self.current_snapshot_id
329    }
330
331    /// Get the snapshot for a reference
332    /// Returns an option if the `ref_name` is not found
333    #[inline]
334    pub fn snapshot_for_ref(&self, ref_name: &str) -> Option<&SnapshotRef> {
335        self.refs.get(ref_name).map(|r| {
336            self.snapshot_by_id(r.snapshot_id)
337                .unwrap_or_else(|| panic!("Snapshot id of ref {ref_name} doesn't exist"))
338        })
339    }
340
341    /// Return all sort orders.
342    #[inline]
343    pub fn sort_orders_iter(&self) -> impl ExactSizeIterator<Item = &SortOrderRef> {
344        self.sort_orders.values()
345    }
346
347    /// Lookup sort order by id.
348    #[inline]
349    pub fn sort_order_by_id(&self, sort_order_id: i64) -> Option<&SortOrderRef> {
350        self.sort_orders.get(&sort_order_id)
351    }
352
353    /// Returns default sort order id.
354    #[inline]
355    pub fn default_sort_order(&self) -> &SortOrderRef {
356        self.sort_orders
357            .get(&self.default_sort_order_id)
358            .expect("Default order id has been set, but not found in table metadata!")
359    }
360
361    /// Returns default sort order id.
362    #[inline]
363    pub fn default_sort_order_id(&self) -> i64 {
364        self.default_sort_order_id
365    }
366
367    /// Returns properties of table.
368    #[inline]
369    pub fn properties(&self) -> &HashMap<String, String> {
370        &self.properties
371    }
372
373    /// Returns the base location for metadata files (manifests, manifest lists).
374    ///
375    /// Honors the `write.metadata.path` table property when set, otherwise defaults
376    /// to the `metadata` subdirectory under the table location.
377    pub fn metadata_location(&self) -> Result<String> {
378        Ok(self
379            .table_properties()
380            .write_metadata_path()?
381            .unwrap_or_else(|| format!("{}/{}", self.location(), METADATA_FOLDER_NAME)))
382    }
383
384    /// Returns the metadata compression codec from table properties.
385    ///
386    /// Returns `CompressionCodec::None` if compression is disabled or not configured.
387    /// Returns `CompressionCodec::Gzip` if gzip compression is enabled.
388    ///
389    /// # Errors
390    ///
391    /// Returns an error if the compression codec property has an invalid value.
392    pub fn metadata_compression_codec(&self) -> Result<CompressionCodec> {
393        self.table_properties().metadata_compression_codec()
394    }
395
396    /// Returns a typed view that parses each table property when its getter is called.
397    #[inline]
398    pub fn table_properties(&self) -> TableProperties<'_> {
399        TableProperties::new(&self.properties)
400    }
401
402    /// Return location of statistics files.
403    #[inline]
404    pub fn statistics_iter(&self) -> impl ExactSizeIterator<Item = &StatisticsFile> {
405        self.statistics.values()
406    }
407
408    /// Return location of partition statistics files.
409    #[inline]
410    pub fn partition_statistics_iter(
411        &self,
412    ) -> impl ExactSizeIterator<Item = &PartitionStatisticsFile> {
413        self.partition_statistics.values()
414    }
415
416    /// Get a statistics file for a snapshot id.
417    #[inline]
418    pub fn statistics_for_snapshot(&self, snapshot_id: i64) -> Option<&StatisticsFile> {
419        self.statistics.get(&snapshot_id)
420    }
421
422    /// Get a partition statistics file for a snapshot id.
423    #[inline]
424    pub fn partition_statistics_for_snapshot(
425        &self,
426        snapshot_id: i64,
427    ) -> Option<&PartitionStatisticsFile> {
428        self.partition_statistics.get(&snapshot_id)
429    }
430
431    fn construct_refs(&mut self) {
432        if let Some(current_snapshot_id) = self.current_snapshot_id
433            && !self.refs.contains_key(MAIN_BRANCH)
434        {
435            self.refs
436                .insert(MAIN_BRANCH.to_string(), SnapshotReference {
437                    snapshot_id: current_snapshot_id,
438                    retention: SnapshotRetention::Branch {
439                        min_snapshots_to_keep: None,
440                        max_snapshot_age_ms: None,
441                        max_ref_age_ms: None,
442                    },
443                });
444        }
445    }
446
447    /// Iterate over all encryption keys
448    #[inline]
449    pub fn encryption_keys_iter(&self) -> impl ExactSizeIterator<Item = &EncryptedKey> {
450        self.encryption_keys.values()
451    }
452
453    /// Get the encryption key for a given key id
454    #[inline]
455    pub fn encryption_key(&self, key_id: &str) -> Option<&EncryptedKey> {
456        self.encryption_keys.get(key_id)
457    }
458
459    /// Get the next row id to be assigned
460    #[inline]
461    pub fn next_row_id(&self) -> u64 {
462        self.next_row_id
463    }
464
465    /// Read table metadata from the given location.
466    pub async fn read_from(
467        file_io: &FileIO,
468        metadata_location: impl AsRef<str>,
469    ) -> Result<TableMetadata> {
470        let metadata_location = metadata_location.as_ref();
471        let input_file = file_io.new_input(metadata_location)?;
472        let metadata_content = input_file.read().await?;
473
474        // Check if the file is compressed by looking for the gzip "magic number".
475        let metadata = if metadata_content.len() > 2
476            && metadata_content[0] == 0x1F
477            && metadata_content[1] == 0x8B
478        {
479            let decompressed_data = CompressionCodec::gzip_default()
480                .decompress(metadata_content.to_vec())
481                .map_err(|e| {
482                    invalid_data!("Trying to read compressed metadata file")
483                        .with_context("file_path", metadata_location)
484                        .with_source(e)
485                })?;
486            serde_json::from_slice(&decompressed_data)?
487        } else {
488            serde_json::from_slice(&metadata_content)?
489        };
490
491        Ok(metadata)
492    }
493
494    /// Write table metadata to the given location.
495    pub async fn write_to(
496        &self,
497        file_io: &FileIO,
498        metadata_location: &MetadataLocation,
499    ) -> Result<()> {
500        let json_data = serde_json::to_vec(self)?;
501
502        // Check if compression codec from properties matches the one in metadata_location
503        let codec = self.table_properties().metadata_compression_codec()?;
504
505        if codec != metadata_location.compression_codec() {
506            return Err(invalid_data!(
507                "Compression codec mismatch: metadata_location has {:?}, but table properties specify {:?}",
508                metadata_location.compression_codec(),
509                codec
510            ));
511        }
512
513        // Apply compression based on codec
514        let data_to_write = match codec {
515            CompressionCodec::Gzip(_) => codec.compress(json_data)?,
516            CompressionCodec::None => json_data,
517            _ => {
518                return Err(invalid_data!(
519                    "Unsupported metadata compression codec: {codec:?}"
520                ));
521            }
522        };
523
524        file_io
525            .new_output(metadata_location.to_string())?
526            .write(data_to_write.into())
527            .await
528    }
529
530    /// Normalize this partition spec.
531    ///
532    /// This is an internal method
533    /// meant to be called after constructing table metadata from untrusted sources.
534    /// We run this method after json deserialization.
535    /// All constructors for `TableMetadata` which are part of `iceberg-rust`
536    /// should return normalized `TableMetadata`.
537    pub(super) fn try_normalize(&mut self) -> Result<&mut Self> {
538        self.validate_current_schema()?;
539        self.normalize_current_snapshot()?;
540        self.construct_refs();
541        self.validate_refs()?;
542        self.validate_chronological_snapshot_logs()?;
543        self.validate_chronological_metadata_logs()?;
544        // Normalize location (remove trailing slash)
545        self.location = self.location.trim_end_matches('/').to_string();
546        self.validate_snapshot_sequence_number()?;
547        self.validate_schema_format_compatibility()?;
548        self.try_normalize_partition_spec()?;
549        self.try_normalize_sort_order()?;
550        Ok(self)
551    }
552
553    /// Validate active default-spec sources and add the spec if it is not present.
554    fn try_normalize_partition_spec(&mut self) -> Result<()> {
555        for field in self.default_spec.fields() {
556            // Historical specs may reference dropped columns, but active default-spec
557            // fields need a source for new writes. Void fields do not read their source.
558            if field.transform != Transform::Void
559                && self.current_schema().field_by_id(field.source_id).is_none()
560            {
561                return Err(invalid_data!(
562                    "Default partition spec {} references missing source field {} in current schema {}",
563                    self.default_spec.spec_id(),
564                    field.source_id,
565                    self.current_schema_id
566                ));
567            }
568        }
569
570        if self
571            .partition_spec_by_id(self.default_spec.spec_id())
572            .is_none()
573        {
574            self.partition_specs.insert(
575                self.default_spec.spec_id(),
576                Arc::new(Arc::unwrap_or_clone(self.default_spec.clone())),
577            );
578        }
579
580        Ok(())
581    }
582
583    /// If the default sort order is unsorted but the sort order is not present, add it
584    fn try_normalize_sort_order(&mut self) -> Result<()> {
585        // Validate that sort order ID 0 (reserved for unsorted) has no fields
586        if let Some(sort_order) = self.sort_order_by_id(SortOrder::UNSORTED_ORDER_ID)
587            && !sort_order.fields.is_empty()
588        {
589            return Err(Error::new(
590                ErrorKind::Unexpected,
591                format!(
592                    "Sort order ID {} is reserved for unsorted order",
593                    SortOrder::UNSORTED_ORDER_ID
594                ),
595            ));
596        }
597
598        if self.sort_order_by_id(self.default_sort_order_id).is_some() {
599            return Ok(());
600        }
601
602        if self.default_sort_order_id != SortOrder::UNSORTED_ORDER_ID {
603            return Err(invalid_data!(
604                "No sort order exists with the default sort order id {}.",
605                self.default_sort_order_id
606            ));
607        }
608
609        let sort_order = SortOrder::unsorted_order();
610        self.sort_orders
611            .insert(SortOrder::UNSORTED_ORDER_ID, Arc::new(sort_order));
612        Ok(())
613    }
614
615    /// Validate the current schema is set and exists.
616    fn validate_current_schema(&self) -> Result<()> {
617        if self.schema_by_id(self.current_schema_id).is_none() {
618            return Err(invalid_data!(
619                "No schema exists with the current schema id {}.",
620                self.current_schema_id
621            ));
622        }
623        Ok(())
624    }
625
626    /// If current snapshot is Some(-1) then set it to None.
627    fn normalize_current_snapshot(&mut self) -> Result<()> {
628        if let Some(current_snapshot_id) = self.current_snapshot_id {
629            if current_snapshot_id == EMPTY_SNAPSHOT_ID {
630                self.current_snapshot_id = None;
631            } else if self.snapshot_by_id(current_snapshot_id).is_none() {
632                return Err(invalid_data!(
633                    "Snapshot for current snapshot id {current_snapshot_id} does not exist in the existing snapshots list"
634                ));
635            }
636        }
637        Ok(())
638    }
639
640    /// Validate that all refs are valid (snapshot exists)
641    fn validate_refs(&self) -> Result<()> {
642        for (name, snapshot_ref) in self.refs.iter() {
643            if self.snapshot_by_id(snapshot_ref.snapshot_id).is_none() {
644                return Err(invalid_data!(
645                    "Snapshot for reference {name} does not exist in the existing snapshots list"
646                ));
647            }
648        }
649
650        let main_ref = self.refs.get(MAIN_BRANCH);
651        if self.current_snapshot_id.is_some() {
652            if let Some(main_ref) = main_ref
653                && main_ref.snapshot_id != self.current_snapshot_id.unwrap_or_default()
654            {
655                return Err(invalid_data!(
656                    "Current snapshot id does not match main branch ({:?} != {:?})",
657                    self.current_snapshot_id.unwrap_or_default(),
658                    main_ref.snapshot_id
659                ));
660            }
661        } else if main_ref.is_some() {
662            return Err(invalid_data!(
663                "Current snapshot is not set, but main branch exists"
664            ));
665        }
666
667        Ok(())
668    }
669
670    /// Validate that for V1 Metadata the last_sequence_number is 0
671    fn validate_snapshot_sequence_number(&self) -> Result<()> {
672        if self.format_version < FormatVersion::V2 && self.last_sequence_number != 0 {
673            return Err(invalid_data!(
674                "Last sequence number must be 0 in v1. Found {}",
675                self.last_sequence_number
676            ));
677        }
678
679        if self.format_version >= FormatVersion::V2
680            && let Some(snapshot) = self
681                .snapshots
682                .values()
683                .find(|snapshot| snapshot.sequence_number() > self.last_sequence_number)
684        {
685            return Err(invalid_data!(
686                "Invalid snapshot with id {} and sequence number {} greater than last sequence number {}",
687                snapshot.snapshot_id(),
688                snapshot.sequence_number(),
689                self.last_sequence_number
690            ));
691        }
692
693        Ok(())
694    }
695
696    /// Validate snapshots logs are chronological and last updated is after the last snapshot log.
697    fn validate_chronological_snapshot_logs(&self) -> Result<()> {
698        for window in self.snapshot_log.windows(2) {
699            let (prev, curr) = (&window[0], &window[1]);
700            // commits can happen concurrently from different machines.
701            // A tolerance helps us avoid failure for small clock skew
702            if curr.timestamp_ms - prev.timestamp_ms < -ONE_MINUTE_MS {
703                return Err(invalid_data!("Expected sorted snapshot log entries"));
704            }
705        }
706
707        if let Some(last) = self.snapshot_log.last() {
708            // commits can happen concurrently from different machines.
709            // A tolerance helps us avoid failure for small clock skew
710            if self.last_updated_ms - last.timestamp_ms < -ONE_MINUTE_MS {
711                return Err(invalid_data!(
712                    "Invalid update timestamp {}: before last snapshot log entry at {}",
713                    self.last_updated_ms,
714                    last.timestamp_ms
715                ));
716            }
717        }
718        Ok(())
719    }
720
721    fn validate_chronological_metadata_logs(&self) -> Result<()> {
722        for window in self.metadata_log.windows(2) {
723            let (prev, curr) = (&window[0], &window[1]);
724            // commits can happen concurrently from different machines.
725            // A tolerance helps us avoid failure for small clock skew
726            if curr.timestamp_ms - prev.timestamp_ms < -ONE_MINUTE_MS {
727                return Err(invalid_data!("Expected sorted metadata log entries"));
728            }
729        }
730
731        if let Some(last) = self.metadata_log.last() {
732            // commits can happen concurrently from different machines.
733            // A tolerance helps us avoid failure for small clock skew
734            if self.last_updated_ms - last.timestamp_ms < -ONE_MINUTE_MS {
735                return Err(invalid_data!(
736                    "Invalid update timestamp {}: before last metadata log entry at {}",
737                    self.last_updated_ms,
738                    last.timestamp_ms
739                ));
740            }
741        }
742
743        Ok(())
744    }
745
746    /// Validates that every type used in the current schema is supported by the
747    /// table's format version.  Delegates to [`Schema::check_format_compatibility`].
748    fn validate_schema_format_compatibility(&self) -> Result<()> {
749        self.current_schema()
750            .check_format_compatibility(self.format_version)
751    }
752}
753
754pub(super) mod _serde {
755    use std::borrow::BorrowMut;
756    /// This is a helper module that defines types to help with serialization/deserialization.
757    /// For deserialization the input first gets read into either the [TableMetadataV1] or [TableMetadataV2] struct
758    /// and then converted into the [TableMetadata] struct. Serialization works the other way around.
759    /// [TableMetadataV1] and [TableMetadataV2] are internal struct that are only used for serialization and deserialization.
760    use std::collections::HashMap;
761    /// This is a helper module that defines types to help with serialization/deserialization.
762    /// For deserialization the input first gets read into either the [TableMetadataV1] or [TableMetadataV2] struct
763    /// and then converted into the [TableMetadata] struct. Serialization works the other way around.
764    /// [TableMetadataV1] and [TableMetadataV2] are internal struct that are only used for serialization and deserialization.
765    use std::sync::Arc;
766
767    use serde::{Deserialize, Serialize};
768    use uuid::Uuid;
769
770    use super::{
771        DEFAULT_PARTITION_SPEC_ID, EMPTY_SNAPSHOT_ID, FormatVersion, MAIN_BRANCH, MetadataLog,
772        SnapshotLog, TableMetadata,
773    };
774    use crate::error::invalid_data;
775    use crate::spec::schema::_serde::{SchemaV1, SchemaV2};
776    use crate::spec::snapshot::_serde::{SnapshotV1, SnapshotV2, SnapshotV3};
777    use crate::spec::{
778        EncryptedKey, INITIAL_ROW_ID, PartitionField, PartitionSpec, PartitionSpecRef,
779        PartitionStatisticsFile, Schema, SchemaRef, Snapshot, SnapshotReference, SnapshotRetention,
780        SortOrder, StatisticsFile, UnboundPartitionSpec,
781    };
782    use crate::{Error, ErrorKind};
783
784    #[derive(Debug, Serialize, Deserialize, PartialEq, Eq)]
785    #[serde(untagged)]
786    pub(super) enum TableMetadataEnum {
787        V3(TableMetadataV3),
788        V2(TableMetadataV2),
789        V1(TableMetadataV1),
790    }
791
792    #[derive(Debug, Serialize, Deserialize, PartialEq, Eq)]
793    #[serde(rename_all = "kebab-case")]
794    /// Defines the structure of a v2 table metadata for serialization/deserialization
795    pub(super) struct TableMetadataV3 {
796        pub format_version: VersionNumber<3>,
797        #[serde(flatten)]
798        pub shared: TableMetadataV2V3Shared,
799        pub next_row_id: u64,
800        #[serde(skip_serializing_if = "Option::is_none")]
801        pub encryption_keys: Option<Vec<EncryptedKey>>,
802        #[serde(skip_serializing_if = "Option::is_none")]
803        pub snapshots: Option<Vec<SnapshotV3>>,
804    }
805
806    #[derive(Debug, Serialize, Deserialize, PartialEq, Eq)]
807    #[serde(rename_all = "kebab-case")]
808    /// Defines the structure of a v2 table metadata for serialization/deserialization
809    pub(super) struct TableMetadataV2V3Shared {
810        pub table_uuid: Uuid,
811        pub location: String,
812        pub last_sequence_number: i64,
813        pub last_updated_ms: i64,
814        pub last_column_id: i32,
815        pub schemas: Vec<SchemaV2>,
816        pub current_schema_id: i32,
817        pub partition_specs: Vec<PartitionSpec>,
818        pub default_spec_id: i32,
819        pub last_partition_id: i32,
820        #[serde(skip_serializing_if = "Option::is_none")]
821        pub properties: Option<HashMap<String, String>>,
822        #[serde(skip_serializing_if = "Option::is_none")]
823        pub current_snapshot_id: Option<i64>,
824        #[serde(skip_serializing_if = "Option::is_none")]
825        pub snapshot_log: Option<Vec<SnapshotLog>>,
826        #[serde(skip_serializing_if = "Option::is_none")]
827        pub metadata_log: Option<Vec<MetadataLog>>,
828        pub sort_orders: Vec<SortOrder>,
829        pub default_sort_order_id: i64,
830        #[serde(skip_serializing_if = "Option::is_none")]
831        pub refs: Option<HashMap<String, SnapshotReference>>,
832        #[serde(default, skip_serializing_if = "Vec::is_empty")]
833        pub statistics: Vec<StatisticsFile>,
834        #[serde(default, skip_serializing_if = "Vec::is_empty")]
835        pub partition_statistics: Vec<PartitionStatisticsFile>,
836    }
837
838    #[derive(Debug, Serialize, Deserialize, PartialEq, Eq)]
839    #[serde(rename_all = "kebab-case")]
840    /// Defines the structure of a v2 table metadata for serialization/deserialization
841    pub(super) struct TableMetadataV2 {
842        pub format_version: VersionNumber<2>,
843        #[serde(flatten)]
844        pub shared: TableMetadataV2V3Shared,
845        #[serde(skip_serializing_if = "Option::is_none")]
846        pub snapshots: Option<Vec<SnapshotV2>>,
847    }
848
849    #[derive(Debug, Serialize, Deserialize, PartialEq, Eq)]
850    #[serde(rename_all = "kebab-case")]
851    /// Defines the structure of a v1 table metadata for serialization/deserialization
852    pub(super) struct TableMetadataV1 {
853        pub format_version: VersionNumber<1>,
854        #[serde(skip_serializing_if = "Option::is_none")]
855        pub table_uuid: Option<Uuid>,
856        pub location: String,
857        pub last_updated_ms: i64,
858        pub last_column_id: i32,
859        /// `schema` is optional to prioritize `schemas` and `current-schema-id`, allowing liberal reading of V1 metadata.
860        pub schema: Option<SchemaV1>,
861        #[serde(skip_serializing_if = "Option::is_none")]
862        pub schemas: Option<Vec<SchemaV1>>,
863        #[serde(skip_serializing_if = "Option::is_none")]
864        pub current_schema_id: Option<i32>,
865        /// `partition_spec` is optional to prioritize `partition_specs`, aligning with liberal reading of potentially invalid V1 metadata.
866        pub partition_spec: Option<Vec<PartitionField>>,
867        #[serde(skip_serializing_if = "Option::is_none")]
868        pub partition_specs: Option<Vec<PartitionSpec>>,
869        #[serde(skip_serializing_if = "Option::is_none")]
870        pub default_spec_id: Option<i32>,
871        #[serde(skip_serializing_if = "Option::is_none")]
872        pub last_partition_id: Option<i32>,
873        #[serde(skip_serializing_if = "Option::is_none")]
874        pub properties: Option<HashMap<String, String>>,
875        #[serde(skip_serializing_if = "Option::is_none")]
876        pub current_snapshot_id: Option<i64>,
877        #[serde(skip_serializing_if = "Option::is_none")]
878        pub snapshots: Option<Vec<SnapshotV1>>,
879        #[serde(skip_serializing_if = "Option::is_none")]
880        pub snapshot_log: Option<Vec<SnapshotLog>>,
881        #[serde(skip_serializing_if = "Option::is_none")]
882        pub metadata_log: Option<Vec<MetadataLog>>,
883        pub sort_orders: Option<Vec<SortOrder>>,
884        pub default_sort_order_id: Option<i64>,
885        #[serde(skip_serializing_if = "Option::is_none")]
886        pub refs: Option<HashMap<String, SnapshotReference>>,
887        #[serde(default, skip_serializing_if = "Vec::is_empty")]
888        pub statistics: Vec<StatisticsFile>,
889        #[serde(default, skip_serializing_if = "Vec::is_empty")]
890        pub partition_statistics: Vec<PartitionStatisticsFile>,
891    }
892
893    /// Helper to serialize and deserialize the format version.
894    #[derive(Debug, PartialEq, Eq)]
895    pub(crate) struct VersionNumber<const V: u8>;
896
897    impl Serialize for TableMetadata {
898        fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error>
899        where S: serde::Serializer {
900            // we must do a clone here
901            let table_metadata_enum: TableMetadataEnum =
902                self.clone().try_into().map_err(serde::ser::Error::custom)?;
903
904            table_metadata_enum.serialize(serializer)
905        }
906    }
907
908    impl<const V: u8> Serialize for VersionNumber<V> {
909        fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error>
910        where S: serde::Serializer {
911            serializer.serialize_u8(V)
912        }
913    }
914
915    impl<'de, const V: u8> Deserialize<'de> for VersionNumber<V> {
916        fn deserialize<D>(deserializer: D) -> Result<Self, D::Error>
917        where D: serde::Deserializer<'de> {
918            let value = u8::deserialize(deserializer)?;
919            if value == V {
920                Ok(VersionNumber::<V>)
921            } else {
922                Err(serde::de::Error::custom("Invalid Version"))
923            }
924        }
925    }
926
927    impl TryFrom<TableMetadataEnum> for TableMetadata {
928        type Error = Error;
929        fn try_from(value: TableMetadataEnum) -> Result<Self, Error> {
930            match value {
931                TableMetadataEnum::V3(value) => value.try_into(),
932                TableMetadataEnum::V2(value) => value.try_into(),
933                TableMetadataEnum::V1(value) => value.try_into(),
934            }
935        }
936    }
937
938    impl TryFrom<TableMetadata> for TableMetadataEnum {
939        type Error = Error;
940        fn try_from(value: TableMetadata) -> Result<Self, Error> {
941            Ok(match value.format_version {
942                FormatVersion::V3 => TableMetadataEnum::V3(value.try_into()?),
943                FormatVersion::V2 => TableMetadataEnum::V2(value.into()),
944                FormatVersion::V1 => TableMetadataEnum::V1(value.try_into()?),
945            })
946        }
947    }
948
949    impl TryFrom<TableMetadataV3> for TableMetadata {
950        type Error = Error;
951        fn try_from(value: TableMetadataV3) -> Result<Self, Error> {
952            let TableMetadataV3 {
953                format_version: _,
954                shared: value,
955                next_row_id,
956                encryption_keys,
957                snapshots,
958            } = value;
959            let current_snapshot_id = if value.current_snapshot_id == Some(EMPTY_SNAPSHOT_ID) {
960                None
961            } else {
962                value.current_snapshot_id
963            };
964            let schemas = HashMap::from_iter(
965                value
966                    .schemas
967                    .into_iter()
968                    .map(|schema| Ok((schema.schema_id, Arc::new(schema.try_into()?))))
969                    .collect::<Result<Vec<_>, Error>>()?,
970            );
971
972            let current_schema: &SchemaRef =
973                schemas.get(&value.current_schema_id).ok_or_else(|| {
974                    invalid_data!(
975                        "No schema exists with the current schema id {}.",
976                        value.current_schema_id
977                    )
978                })?;
979            let partition_specs = HashMap::from_iter(
980                value
981                    .partition_specs
982                    .into_iter()
983                    .map(|x| (x.spec_id(), Arc::new(x))),
984            );
985            let default_spec_id = value.default_spec_id;
986            let default_spec: PartitionSpecRef = partition_specs
987                .get(&value.default_spec_id)
988                .map(|spec| (**spec).clone())
989                .or_else(|| {
990                    (DEFAULT_PARTITION_SPEC_ID == default_spec_id)
991                        .then(PartitionSpec::unpartition_spec)
992                })
993                .ok_or_else(|| invalid_data!("Default partition spec {default_spec_id} not found"))?
994                .into();
995            let default_partition_type = default_spec.partition_type(current_schema)?;
996
997            let mut metadata = TableMetadata {
998                format_version: FormatVersion::V3,
999                table_uuid: value.table_uuid,
1000                location: value.location,
1001                last_sequence_number: value.last_sequence_number,
1002                last_updated_ms: value.last_updated_ms,
1003                last_column_id: value.last_column_id,
1004                current_schema_id: value.current_schema_id,
1005                schemas,
1006                partition_specs,
1007                default_partition_type,
1008                default_spec,
1009                last_partition_id: value.last_partition_id,
1010                properties: value.properties.unwrap_or_default(),
1011                current_snapshot_id,
1012                snapshots: snapshots
1013                    .map(|snapshots| {
1014                        HashMap::from_iter(
1015                            snapshots
1016                                .into_iter()
1017                                .map(|x| (x.snapshot_id, Arc::new(x.into()))),
1018                        )
1019                    })
1020                    .unwrap_or_default(),
1021                snapshot_log: value.snapshot_log.unwrap_or_default(),
1022                metadata_log: value.metadata_log.unwrap_or_default(),
1023                sort_orders: HashMap::from_iter(
1024                    value
1025                        .sort_orders
1026                        .into_iter()
1027                        .map(|x| (x.order_id, Arc::new(x))),
1028                ),
1029                default_sort_order_id: value.default_sort_order_id,
1030                refs: value.refs.unwrap_or_else(|| {
1031                    if let Some(snapshot_id) = current_snapshot_id {
1032                        HashMap::from_iter(vec![(MAIN_BRANCH.to_string(), SnapshotReference {
1033                            snapshot_id,
1034                            retention: SnapshotRetention::Branch {
1035                                min_snapshots_to_keep: None,
1036                                max_snapshot_age_ms: None,
1037                                max_ref_age_ms: None,
1038                            },
1039                        })])
1040                    } else {
1041                        HashMap::new()
1042                    }
1043                }),
1044                statistics: index_statistics(value.statistics),
1045                partition_statistics: index_partition_statistics(value.partition_statistics),
1046                encryption_keys: encryption_keys
1047                    .map(|keys| {
1048                        HashMap::from_iter(keys.into_iter().map(|key| (key.key_id.clone(), key)))
1049                    })
1050                    .unwrap_or_default(),
1051                next_row_id,
1052            };
1053
1054            metadata.borrow_mut().try_normalize()?;
1055            Ok(metadata)
1056        }
1057    }
1058
1059    impl TryFrom<TableMetadataV2> for TableMetadata {
1060        type Error = Error;
1061        fn try_from(value: TableMetadataV2) -> Result<Self, Error> {
1062            let snapshots = value.snapshots;
1063            let value = value.shared;
1064            let current_snapshot_id = if value.current_snapshot_id == Some(EMPTY_SNAPSHOT_ID) {
1065                None
1066            } else {
1067                value.current_snapshot_id
1068            };
1069            let schemas = HashMap::from_iter(
1070                value
1071                    .schemas
1072                    .into_iter()
1073                    .map(|schema| Ok((schema.schema_id, Arc::new(schema.try_into()?))))
1074                    .collect::<Result<Vec<_>, Error>>()?,
1075            );
1076
1077            let current_schema: &SchemaRef =
1078                schemas.get(&value.current_schema_id).ok_or_else(|| {
1079                    invalid_data!(
1080                        "No schema exists with the current schema id {}.",
1081                        value.current_schema_id
1082                    )
1083                })?;
1084            let partition_specs = HashMap::from_iter(
1085                value
1086                    .partition_specs
1087                    .into_iter()
1088                    .map(|x| (x.spec_id(), Arc::new(x))),
1089            );
1090            let default_spec_id = value.default_spec_id;
1091            let default_spec: PartitionSpecRef = partition_specs
1092                .get(&value.default_spec_id)
1093                .map(|spec| (**spec).clone())
1094                .or_else(|| {
1095                    (DEFAULT_PARTITION_SPEC_ID == default_spec_id)
1096                        .then(PartitionSpec::unpartition_spec)
1097                })
1098                .ok_or_else(|| invalid_data!("Default partition spec {default_spec_id} not found"))?
1099                .into();
1100            let default_partition_type = default_spec.partition_type(current_schema)?;
1101
1102            let mut metadata = TableMetadata {
1103                format_version: FormatVersion::V2,
1104                table_uuid: value.table_uuid,
1105                location: value.location,
1106                last_sequence_number: value.last_sequence_number,
1107                last_updated_ms: value.last_updated_ms,
1108                last_column_id: value.last_column_id,
1109                current_schema_id: value.current_schema_id,
1110                schemas,
1111                partition_specs,
1112                default_partition_type,
1113                default_spec,
1114                last_partition_id: value.last_partition_id,
1115                properties: value.properties.unwrap_or_default(),
1116                current_snapshot_id,
1117                snapshots: snapshots
1118                    .map(|snapshots| {
1119                        HashMap::from_iter(
1120                            snapshots
1121                                .into_iter()
1122                                .map(|x| (x.snapshot_id, Arc::new(x.into()))),
1123                        )
1124                    })
1125                    .unwrap_or_default(),
1126                snapshot_log: value.snapshot_log.unwrap_or_default(),
1127                metadata_log: value.metadata_log.unwrap_or_default(),
1128                sort_orders: HashMap::from_iter(
1129                    value
1130                        .sort_orders
1131                        .into_iter()
1132                        .map(|x| (x.order_id, Arc::new(x))),
1133                ),
1134                default_sort_order_id: value.default_sort_order_id,
1135                refs: value.refs.unwrap_or_else(|| {
1136                    if let Some(snapshot_id) = current_snapshot_id {
1137                        HashMap::from_iter(vec![(MAIN_BRANCH.to_string(), SnapshotReference {
1138                            snapshot_id,
1139                            retention: SnapshotRetention::Branch {
1140                                min_snapshots_to_keep: None,
1141                                max_snapshot_age_ms: None,
1142                                max_ref_age_ms: None,
1143                            },
1144                        })])
1145                    } else {
1146                        HashMap::new()
1147                    }
1148                }),
1149                statistics: index_statistics(value.statistics),
1150                partition_statistics: index_partition_statistics(value.partition_statistics),
1151                encryption_keys: HashMap::new(),
1152                next_row_id: INITIAL_ROW_ID,
1153            };
1154
1155            metadata.borrow_mut().try_normalize()?;
1156            Ok(metadata)
1157        }
1158    }
1159
1160    impl TryFrom<TableMetadataV1> for TableMetadata {
1161        type Error = Error;
1162        fn try_from(value: TableMetadataV1) -> Result<Self, Error> {
1163            let current_snapshot_id = if value.current_snapshot_id == Some(EMPTY_SNAPSHOT_ID) {
1164                None
1165            } else {
1166                value.current_snapshot_id
1167            };
1168
1169            let (schemas, current_schema_id, current_schema) =
1170                if let (Some(schemas_vec), Some(schema_id)) =
1171                    (&value.schemas, value.current_schema_id)
1172                {
1173                    // Option 1: Use 'schemas' + 'current_schema_id'
1174                    let schema_map = HashMap::from_iter(
1175                        schemas_vec
1176                            .clone()
1177                            .into_iter()
1178                            .map(|schema| {
1179                                let schema: Schema = schema.try_into()?;
1180                                Ok((schema.schema_id(), Arc::new(schema)))
1181                            })
1182                            .collect::<Result<Vec<_>, Error>>()?,
1183                    );
1184
1185                    let schema = schema_map
1186                        .get(&schema_id)
1187                        .ok_or_else(|| {
1188                            invalid_data!(
1189                                "No schema exists with the current schema id {schema_id}."
1190                            )
1191                        })?
1192                        .clone();
1193                    (schema_map, schema_id, schema)
1194                } else if let Some(schema) = value.schema {
1195                    // Option 2: Fall back to `schema`
1196                    let schema: Schema = schema.try_into()?;
1197                    let schema_id = schema.schema_id();
1198                    let schema_arc = Arc::new(schema);
1199                    let schema_map = HashMap::from_iter(vec![(schema_id, schema_arc.clone())]);
1200                    (schema_map, schema_id, schema_arc)
1201                } else {
1202                    // Option 3: No valid schema configuration found
1203                    return Err(invalid_data!(
1204                        "No valid schema configuration found in table metadata"
1205                    ));
1206                };
1207
1208            // Prioritize 'partition_specs' over 'partition_spec'
1209            let partition_specs = if let Some(specs_vec) = value.partition_specs {
1210                // Option 1: Use 'partition_specs'
1211                specs_vec
1212                    .into_iter()
1213                    .map(|x| (x.spec_id(), Arc::new(x)))
1214                    .collect::<HashMap<_, _>>()
1215            } else if let Some(partition_spec) = value.partition_spec {
1216                // Option 2: Fall back to 'partition_spec'
1217                let spec = UnboundPartitionSpec {
1218                    spec_id: Some(DEFAULT_PARTITION_SPEC_ID),
1219                    fields: partition_spec
1220                        .into_iter()
1221                        .map(|f| f.into_unbound())
1222                        .collect(),
1223                }
1224                .bind(current_schema.clone())?;
1225
1226                HashMap::from_iter(vec![(DEFAULT_PARTITION_SPEC_ID, Arc::new(spec))])
1227            } else {
1228                // Option 3: Create empty partition spec
1229                let spec = PartitionSpec::builder(current_schema.clone())
1230                    .with_spec_id(DEFAULT_PARTITION_SPEC_ID)
1231                    .build()?;
1232
1233                HashMap::from_iter(vec![(DEFAULT_PARTITION_SPEC_ID, Arc::new(spec))])
1234            };
1235
1236            // Get the default_spec_id, prioritizing the explicit value if provided
1237            let default_spec_id = value
1238                .default_spec_id
1239                .unwrap_or_else(|| partition_specs.keys().copied().max().unwrap_or_default());
1240
1241            // Get the default spec
1242            let default_spec: PartitionSpecRef = partition_specs
1243                .get(&default_spec_id)
1244                .map(|x| Arc::unwrap_or_clone(x.clone()))
1245                .ok_or_else(|| invalid_data!("Default partition spec {default_spec_id} not found"))?
1246                .into();
1247            let default_partition_type = default_spec.partition_type(&current_schema)?;
1248
1249            let mut metadata = TableMetadata {
1250                format_version: FormatVersion::V1,
1251                table_uuid: value.table_uuid.unwrap_or_default(),
1252                location: value.location,
1253                last_sequence_number: 0,
1254                last_updated_ms: value.last_updated_ms,
1255                last_column_id: value.last_column_id,
1256                current_schema_id,
1257                default_spec,
1258                default_partition_type,
1259                last_partition_id: value
1260                    .last_partition_id
1261                    .unwrap_or_else(|| partition_specs.keys().copied().max().unwrap_or_default()),
1262                partition_specs,
1263                schemas,
1264                properties: value.properties.unwrap_or_default(),
1265                current_snapshot_id,
1266                snapshots: value
1267                    .snapshots
1268                    .map(|snapshots| {
1269                        Ok::<_, Error>(HashMap::from_iter(
1270                            snapshots
1271                                .into_iter()
1272                                .map(|x| Ok((x.snapshot_id, Arc::new(x.try_into()?))))
1273                                .collect::<Result<Vec<_>, Error>>()?,
1274                        ))
1275                    })
1276                    .transpose()?
1277                    .unwrap_or_default(),
1278                snapshot_log: value.snapshot_log.unwrap_or_default(),
1279                metadata_log: value.metadata_log.unwrap_or_default(),
1280                sort_orders: match value.sort_orders {
1281                    Some(sort_orders) => HashMap::from_iter(
1282                        sort_orders.into_iter().map(|x| (x.order_id, Arc::new(x))),
1283                    ),
1284                    None => HashMap::new(),
1285                },
1286                default_sort_order_id: value
1287                    .default_sort_order_id
1288                    .unwrap_or(SortOrder::UNSORTED_ORDER_ID),
1289                // `try_normalize` adds `main` from `current-snapshot-id` if it is missing.
1290                refs: value.refs.unwrap_or_default(),
1291                statistics: index_statistics(value.statistics),
1292                partition_statistics: index_partition_statistics(value.partition_statistics),
1293                encryption_keys: HashMap::new(),
1294                next_row_id: INITIAL_ROW_ID, // v1 has no row lineage
1295            };
1296
1297            metadata.borrow_mut().try_normalize()?;
1298            Ok(metadata)
1299        }
1300    }
1301
1302    impl TryFrom<TableMetadata> for TableMetadataV3 {
1303        type Error = Error;
1304
1305        fn try_from(mut v: TableMetadata) -> Result<Self, Self::Error> {
1306            let next_row_id = v.next_row_id;
1307            let encryption_keys = std::mem::take(&mut v.encryption_keys);
1308            let snapshots = std::mem::take(&mut v.snapshots);
1309            let shared = v.into();
1310
1311            Ok(TableMetadataV3 {
1312                format_version: VersionNumber::<3>,
1313                shared,
1314                next_row_id,
1315                encryption_keys: if encryption_keys.is_empty() {
1316                    None
1317                } else {
1318                    Some(encryption_keys.into_values().collect())
1319                },
1320                snapshots: if snapshots.is_empty() {
1321                    None
1322                } else {
1323                    Some(
1324                        snapshots
1325                            .into_values()
1326                            .map(|s| SnapshotV3::try_from(Arc::unwrap_or_clone(s)))
1327                            .collect::<Result<_, _>>()?,
1328                    )
1329                },
1330            })
1331        }
1332    }
1333
1334    impl From<TableMetadata> for TableMetadataV2 {
1335        fn from(mut v: TableMetadata) -> Self {
1336            let snapshots = std::mem::take(&mut v.snapshots);
1337            let shared = v.into();
1338
1339            TableMetadataV2 {
1340                format_version: VersionNumber::<2>,
1341                shared,
1342                snapshots: if snapshots.is_empty() {
1343                    None
1344                } else {
1345                    Some(
1346                        snapshots
1347                            .into_values()
1348                            .map(|s| SnapshotV2::from(Arc::unwrap_or_clone(s)))
1349                            .collect(),
1350                    )
1351                },
1352            }
1353        }
1354    }
1355
1356    impl From<TableMetadata> for TableMetadataV2V3Shared {
1357        fn from(v: TableMetadata) -> Self {
1358            TableMetadataV2V3Shared {
1359                table_uuid: v.table_uuid,
1360                location: v.location,
1361                last_sequence_number: v.last_sequence_number,
1362                last_updated_ms: v.last_updated_ms,
1363                last_column_id: v.last_column_id,
1364                schemas: v
1365                    .schemas
1366                    .into_values()
1367                    .map(|x| {
1368                        Arc::try_unwrap(x)
1369                            .unwrap_or_else(|schema| schema.as_ref().clone())
1370                            .into()
1371                    })
1372                    .collect(),
1373                current_schema_id: v.current_schema_id,
1374                partition_specs: v
1375                    .partition_specs
1376                    .into_values()
1377                    .map(|x| Arc::try_unwrap(x).unwrap_or_else(|s| s.as_ref().clone()))
1378                    .collect(),
1379                default_spec_id: v.default_spec.spec_id(),
1380                last_partition_id: v.last_partition_id,
1381                properties: if v.properties.is_empty() {
1382                    None
1383                } else {
1384                    Some(v.properties)
1385                },
1386                current_snapshot_id: v.current_snapshot_id,
1387                snapshot_log: if v.snapshot_log.is_empty() {
1388                    None
1389                } else {
1390                    Some(v.snapshot_log)
1391                },
1392                metadata_log: if v.metadata_log.is_empty() {
1393                    None
1394                } else {
1395                    Some(v.metadata_log)
1396                },
1397                sort_orders: v
1398                    .sort_orders
1399                    .into_values()
1400                    .map(|x| Arc::try_unwrap(x).unwrap_or_else(|s| s.as_ref().clone()))
1401                    .collect(),
1402                default_sort_order_id: v.default_sort_order_id,
1403                refs: Some(v.refs),
1404                statistics: v.statistics.into_values().collect(),
1405                partition_statistics: v.partition_statistics.into_values().collect(),
1406            }
1407        }
1408    }
1409
1410    impl TryFrom<TableMetadata> for TableMetadataV1 {
1411        type Error = Error;
1412        fn try_from(v: TableMetadata) -> Result<Self, Error> {
1413            Ok(TableMetadataV1 {
1414                format_version: VersionNumber::<1>,
1415                table_uuid: Some(v.table_uuid),
1416                location: v.location,
1417                last_updated_ms: v.last_updated_ms,
1418                last_column_id: v.last_column_id,
1419                schema: Some(
1420                    v.schemas
1421                        .get(&v.current_schema_id)
1422                        .ok_or(Error::new(
1423                            ErrorKind::Unexpected,
1424                            "current_schema_id not found in schemas",
1425                        ))?
1426                        .as_ref()
1427                        .clone()
1428                        .into(),
1429                ),
1430                schemas: Some(
1431                    v.schemas
1432                        .into_values()
1433                        .map(|x| {
1434                            Arc::try_unwrap(x)
1435                                .unwrap_or_else(|schema| schema.as_ref().clone())
1436                                .into()
1437                        })
1438                        .collect(),
1439                ),
1440                current_schema_id: Some(v.current_schema_id),
1441                partition_spec: Some(v.default_spec.fields().to_vec()),
1442                partition_specs: Some(
1443                    v.partition_specs
1444                        .into_values()
1445                        .map(|x| Arc::try_unwrap(x).unwrap_or_else(|s| s.as_ref().clone()))
1446                        .collect(),
1447                ),
1448                default_spec_id: Some(v.default_spec.spec_id()),
1449                last_partition_id: Some(v.last_partition_id),
1450                properties: if v.properties.is_empty() {
1451                    None
1452                } else {
1453                    Some(v.properties)
1454                },
1455                current_snapshot_id: v.current_snapshot_id,
1456                snapshots: if v.snapshots.is_empty() {
1457                    None
1458                } else {
1459                    Some(
1460                        v.snapshots
1461                            .into_values()
1462                            .map(|x| Snapshot::clone(&x).into())
1463                            .collect(),
1464                    )
1465                },
1466                snapshot_log: if v.snapshot_log.is_empty() {
1467                    None
1468                } else {
1469                    Some(v.snapshot_log)
1470                },
1471                metadata_log: if v.metadata_log.is_empty() {
1472                    None
1473                } else {
1474                    Some(v.metadata_log)
1475                },
1476                sort_orders: Some(
1477                    v.sort_orders
1478                        .into_values()
1479                        .map(|s| Arc::try_unwrap(s).unwrap_or_else(|s| s.as_ref().clone()))
1480                        .collect(),
1481                ),
1482                default_sort_order_id: Some(v.default_sort_order_id),
1483                refs: Some(v.refs),
1484                statistics: v.statistics.into_values().collect(),
1485                partition_statistics: v.partition_statistics.into_values().collect(),
1486            })
1487        }
1488    }
1489
1490    fn index_statistics(statistics: Vec<StatisticsFile>) -> HashMap<i64, StatisticsFile> {
1491        statistics
1492            .into_iter()
1493            .rev()
1494            .map(|s| (s.snapshot_id, s))
1495            .collect()
1496    }
1497
1498    fn index_partition_statistics(
1499        statistics: Vec<PartitionStatisticsFile>,
1500    ) -> HashMap<i64, PartitionStatisticsFile> {
1501        statistics
1502            .into_iter()
1503            .rev()
1504            .map(|s| (s.snapshot_id, s))
1505            .collect()
1506    }
1507}
1508
1509#[derive(Debug, Serialize_repr, Deserialize_repr, PartialEq, Eq, Clone, Copy, Hash)]
1510#[repr(u8)]
1511/// Iceberg format version
1512pub enum FormatVersion {
1513    /// Iceberg spec version 1
1514    V1 = 1u8,
1515    /// Iceberg spec version 2
1516    V2 = 2u8,
1517    /// Iceberg spec version 3
1518    V3 = 3u8,
1519}
1520
1521impl PartialOrd for FormatVersion {
1522    fn partial_cmp(&self, other: &Self) -> Option<Ordering> {
1523        Some(self.cmp(other))
1524    }
1525}
1526
1527impl Ord for FormatVersion {
1528    fn cmp(&self, other: &Self) -> Ordering {
1529        (*self as u8).cmp(&(*other as u8))
1530    }
1531}
1532
1533impl Display for FormatVersion {
1534    fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result {
1535        match self {
1536            FormatVersion::V1 => write!(f, "v1"),
1537            FormatVersion::V2 => write!(f, "v2"),
1538            FormatVersion::V3 => write!(f, "v3"),
1539        }
1540    }
1541}
1542
1543#[derive(Debug, Serialize, Deserialize, PartialEq, Eq, Clone)]
1544#[serde(rename_all = "kebab-case")]
1545/// Encodes changes to the previous metadata files for the table
1546pub struct MetadataLog {
1547    /// The file for the log.
1548    pub metadata_file: String,
1549    /// Time new metadata was created
1550    pub timestamp_ms: i64,
1551}
1552
1553#[derive(Debug, Serialize, Deserialize, PartialEq, Eq, Clone)]
1554#[serde(rename_all = "kebab-case")]
1555/// A log of when each snapshot was made.
1556pub struct SnapshotLog {
1557    /// Id of the snapshot.
1558    pub snapshot_id: i64,
1559    /// Last updated timestamp
1560    pub timestamp_ms: i64,
1561}
1562
1563impl SnapshotLog {
1564    /// Returns the last updated timestamp as a [`DateTime<Utc>`] with millisecond precision
1565    pub fn timestamp(self) -> Result<DateTime<Utc>> {
1566        timestamp_ms_to_utc(self.timestamp_ms)
1567    }
1568
1569    /// Returns the timestamp in milliseconds
1570    #[inline]
1571    pub fn timestamp_ms(&self) -> i64 {
1572        self.timestamp_ms
1573    }
1574}
1575
1576#[cfg(test)]
1577mod tests {
1578    use std::collections::HashMap;
1579    use std::fs;
1580    use std::sync::Arc;
1581
1582    use anyhow::Result;
1583    use base64::Engine as _;
1584    use pretty_assertions::assert_eq;
1585    use tempfile::TempDir;
1586    use uuid::Uuid;
1587
1588    use super::{FormatVersion, MetadataLog, SnapshotLog, TableMetadataBuilder};
1589    use crate::catalog::MetadataLocation;
1590    use crate::compression::CompressionCodec;
1591    use crate::io::FileIO;
1592    use crate::spec::table_metadata::TableMetadata;
1593    use crate::spec::{
1594        BlobMetadata, EncryptedKey, INITIAL_ROW_ID, Literal, NestedField, NullOrder, Operation,
1595        PartitionSpec, PartitionStatisticsFile, PrimitiveLiteral, PrimitiveType, Schema, Snapshot,
1596        SnapshotReference, SnapshotRetention, SortDirection, SortField, SortOrder, StatisticsFile,
1597        Summary, TableProperties, Transform, Type, UnboundPartitionField, UnboundPartitionSpec,
1598    };
1599    use crate::{ErrorKind, TableCreation};
1600
1601    fn check_table_metadata_serde(json: &str, expected_type: TableMetadata) {
1602        let desered_type: TableMetadata = serde_json::from_str(json).unwrap();
1603        assert_eq!(desered_type, expected_type);
1604
1605        let sered_json = serde_json::to_string(&expected_type).unwrap();
1606        let parsed_json_value = serde_json::from_str::<TableMetadata>(&sered_json).unwrap();
1607
1608        assert_eq!(parsed_json_value, desered_type);
1609    }
1610
1611    fn get_test_table_metadata(file_name: &str) -> TableMetadata {
1612        let path = format!("testdata/table_metadata/{file_name}");
1613        let metadata: String = fs::read_to_string(path).unwrap();
1614
1615        serde_json::from_str(&metadata).unwrap()
1616    }
1617
1618    /// Loads a test table metadata and relocates it to `location`, so that derived
1619    /// metadata paths point at a writable (e.g. temp) directory.
1620    fn get_test_table_metadata_at(file_name: &str, location: &str) -> TableMetadata {
1621        TableMetadataBuilder::new_from_metadata(get_test_table_metadata(file_name), None)
1622            .set_location(location.to_string())
1623            .build()
1624            .unwrap()
1625            .metadata
1626    }
1627
1628    #[test]
1629    fn test_table_data_v2() {
1630        let data = r#"
1631            {
1632                "format-version" : 2,
1633                "table-uuid": "fb072c92-a02b-11e9-ae9c-1bb7bc9eca94",
1634                "location": "s3://b/wh/data.db/table",
1635                "last-sequence-number" : 1,
1636                "last-updated-ms": 1515100955770,
1637                "last-column-id": 1,
1638                "schemas": [
1639                    {
1640                        "schema-id" : 1,
1641                        "type" : "struct",
1642                        "fields" :[
1643                            {
1644                                "id": 1,
1645                                "name": "struct_name",
1646                                "required": true,
1647                                "type": "fixed[1]"
1648                            },
1649                            {
1650                                "id": 4,
1651                                "name": "ts",
1652                                "required": true,
1653                                "type": "timestamp"
1654                            }
1655                        ]
1656                    }
1657                ],
1658                "current-schema-id" : 1,
1659                "partition-specs": [
1660                    {
1661                        "spec-id": 0,
1662                        "fields": [
1663                            {
1664                                "source-id": 4,
1665                                "field-id": 1000,
1666                                "name": "ts_day",
1667                                "transform": "day"
1668                            }
1669                        ]
1670                    }
1671                ],
1672                "default-spec-id": 0,
1673                "last-partition-id": 1000,
1674                "properties": {
1675                    "commit.retry.num-retries": "1"
1676                },
1677                "metadata-log": [
1678                    {
1679                        "metadata-file": "s3://bucket/.../v1.json",
1680                        "timestamp-ms": 1515100
1681                    }
1682                ],
1683                "refs": {},
1684                "sort-orders": [
1685                    {
1686                    "order-id": 0,
1687                    "fields": []
1688                    }
1689                ],
1690                "default-sort-order-id": 0
1691            }
1692        "#;
1693
1694        let schema = Schema::builder()
1695            .with_schema_id(1)
1696            .with_fields(vec![
1697                Arc::new(NestedField::required(
1698                    1,
1699                    "struct_name",
1700                    Type::Primitive(PrimitiveType::Fixed(1)),
1701                )),
1702                Arc::new(NestedField::required(
1703                    4,
1704                    "ts",
1705                    Type::Primitive(PrimitiveType::Timestamp),
1706                )),
1707            ])
1708            .build()
1709            .unwrap();
1710
1711        let partition_spec = PartitionSpec::builder(schema.clone())
1712            .with_spec_id(0)
1713            .add_unbound_field(
1714                UnboundPartitionField::builder()
1715                    .source_ids(vec![4])
1716                    .field_id(1000)
1717                    .name("ts_day".to_string())
1718                    .transform(Transform::Day)
1719                    .build()
1720                    .unwrap(),
1721            )
1722            .unwrap()
1723            .build()
1724            .unwrap();
1725
1726        let default_partition_type = partition_spec.partition_type(&schema).unwrap();
1727        let expected = TableMetadata {
1728            format_version: FormatVersion::V2,
1729            table_uuid: Uuid::parse_str("fb072c92-a02b-11e9-ae9c-1bb7bc9eca94").unwrap(),
1730            location: "s3://b/wh/data.db/table".to_string(),
1731            last_updated_ms: 1515100955770,
1732            last_column_id: 1,
1733            schemas: HashMap::from_iter(vec![(1, Arc::new(schema))]),
1734            current_schema_id: 1,
1735            partition_specs: HashMap::from_iter(vec![(0, partition_spec.clone().into())]),
1736            default_partition_type,
1737            default_spec: partition_spec.into(),
1738            last_partition_id: 1000,
1739            default_sort_order_id: 0,
1740            sort_orders: HashMap::from_iter(vec![(0, SortOrder::unsorted_order().into())]),
1741            snapshots: HashMap::default(),
1742            current_snapshot_id: None,
1743            last_sequence_number: 1,
1744            properties: HashMap::from_iter(vec![(
1745                "commit.retry.num-retries".to_string(),
1746                "1".to_string(),
1747            )]),
1748            snapshot_log: Vec::new(),
1749            metadata_log: vec![MetadataLog {
1750                metadata_file: "s3://bucket/.../v1.json".to_string(),
1751                timestamp_ms: 1515100,
1752            }],
1753            refs: HashMap::new(),
1754            statistics: HashMap::new(),
1755            partition_statistics: HashMap::new(),
1756            encryption_keys: HashMap::new(),
1757            next_row_id: INITIAL_ROW_ID,
1758        };
1759
1760        let expected_json_value = serde_json::to_value(&expected).unwrap();
1761        check_table_metadata_serde(data, expected);
1762
1763        let json_value = serde_json::from_str::<serde_json::Value>(data).unwrap();
1764        assert_eq!(json_value, expected_json_value);
1765    }
1766
1767    #[test]
1768    fn test_table_data_v3() {
1769        let data = r#"
1770            {
1771                "format-version" : 3,
1772                "table-uuid": "fb072c92-a02b-11e9-ae9c-1bb7bc9eca94",
1773                "location": "s3://b/wh/data.db/table",
1774                "last-sequence-number" : 1,
1775                "last-updated-ms": 1515100955770,
1776                "last-column-id": 1,
1777                "next-row-id": 5,
1778                "schemas": [
1779                    {
1780                        "schema-id" : 1,
1781                        "type" : "struct",
1782                        "fields" :[
1783                            {
1784                                "id": 4,
1785                                "name": "ts",
1786                                "required": true,
1787                                "type": "timestamp"
1788                            }
1789                        ]
1790                    }
1791                ],
1792                "current-schema-id" : 1,
1793                "partition-specs": [
1794                    {
1795                        "spec-id": 0,
1796                        "fields": [
1797                            {
1798                                "source-id": 4,
1799                                "field-id": 1000,
1800                                "name": "ts_day",
1801                                "transform": "day"
1802                            }
1803                        ]
1804                    }
1805                ],
1806                "default-spec-id": 0,
1807                "last-partition-id": 1000,
1808                "properties": {
1809                    "commit.retry.num-retries": "1"
1810                },
1811                "metadata-log": [
1812                    {
1813                        "metadata-file": "s3://bucket/.../v1.json",
1814                        "timestamp-ms": 1515100
1815                    }
1816                ],
1817                "refs": {},
1818                "snapshots" : [ {
1819                    "snapshot-id" : 1,
1820                    "timestamp-ms" : 1662532818843,
1821                    "sequence-number" : 0,
1822                    "first-row-id" : 0,
1823                    "added-rows" : 4,
1824                    "key-id" : "key1",
1825                    "summary" : {
1826                        "operation" : "append"
1827                    },
1828                    "manifest-list" : "/home/iceberg/warehouse/nyc/taxis/metadata/snap-638933773299822130-1-7e6760f0-4f6c-4b23-b907-0a5a174e3863.avro",
1829                    "schema-id" : 0
1830                    }
1831                ],
1832                "encryption-keys": [
1833                    {
1834                        "key-id": "key1",
1835                        "encrypted-by-id": "KMS",
1836                        "encrypted-key-metadata": "c29tZS1lbmNyeXB0aW9uLWtleQ==",
1837                        "properties": {
1838                            "p1": "v1"
1839                        }
1840                    }
1841                ],
1842                "sort-orders": [
1843                    {
1844                    "order-id": 0,
1845                    "fields": []
1846                    }
1847                ],
1848                "default-sort-order-id": 0
1849            }
1850        "#;
1851
1852        let schema = Schema::builder()
1853            .with_schema_id(1)
1854            .with_fields(vec![Arc::new(NestedField::required(
1855                4,
1856                "ts",
1857                Type::Primitive(PrimitiveType::Timestamp),
1858            ))])
1859            .build()
1860            .unwrap();
1861
1862        let partition_spec = PartitionSpec::builder(schema.clone())
1863            .with_spec_id(0)
1864            .add_unbound_field(
1865                UnboundPartitionField::builder()
1866                    .source_ids(vec![4])
1867                    .field_id(1000)
1868                    .name("ts_day".to_string())
1869                    .transform(Transform::Day)
1870                    .build()
1871                    .unwrap(),
1872            )
1873            .unwrap()
1874            .build()
1875            .unwrap();
1876
1877        let snapshot = Snapshot::builder()
1878            .with_snapshot_id(1)
1879            .with_timestamp_ms(1662532818843)
1880            .with_sequence_number(0)
1881            .with_row_range(0, 4)
1882            .with_encryption_key_id(Some("key1".to_string()))
1883            .with_summary(Summary {
1884                operation: Operation::Append,
1885                additional_properties: HashMap::new(),
1886            })
1887            .with_manifest_list("/home/iceberg/warehouse/nyc/taxis/metadata/snap-638933773299822130-1-7e6760f0-4f6c-4b23-b907-0a5a174e3863.avro".to_string())
1888            .with_schema_id(0)
1889            .build();
1890
1891        let encryption_key = EncryptedKey::builder()
1892            .key_id("key1".to_string())
1893            .encrypted_by_id("KMS".to_string())
1894            .encrypted_key_metadata(
1895                base64::prelude::BASE64_STANDARD
1896                    .decode("c29tZS1lbmNyeXB0aW9uLWtleQ==")
1897                    .unwrap(),
1898            )
1899            .properties(HashMap::from_iter(vec![(
1900                "p1".to_string(),
1901                "v1".to_string(),
1902            )]))
1903            .build();
1904
1905        let default_partition_type = partition_spec.partition_type(&schema).unwrap();
1906        let expected = TableMetadata {
1907            format_version: FormatVersion::V3,
1908            table_uuid: Uuid::parse_str("fb072c92-a02b-11e9-ae9c-1bb7bc9eca94").unwrap(),
1909            location: "s3://b/wh/data.db/table".to_string(),
1910            last_updated_ms: 1515100955770,
1911            last_column_id: 1,
1912            schemas: HashMap::from_iter(vec![(1, Arc::new(schema))]),
1913            current_schema_id: 1,
1914            partition_specs: HashMap::from_iter(vec![(0, partition_spec.clone().into())]),
1915            default_partition_type,
1916            default_spec: partition_spec.into(),
1917            last_partition_id: 1000,
1918            default_sort_order_id: 0,
1919            sort_orders: HashMap::from_iter(vec![(0, SortOrder::unsorted_order().into())]),
1920            snapshots: HashMap::from_iter(vec![(1, snapshot.into())]),
1921            current_snapshot_id: None,
1922            last_sequence_number: 1,
1923            properties: HashMap::from_iter(vec![(
1924                "commit.retry.num-retries".to_string(),
1925                "1".to_string(),
1926            )]),
1927            snapshot_log: Vec::new(),
1928            metadata_log: vec![MetadataLog {
1929                metadata_file: "s3://bucket/.../v1.json".to_string(),
1930                timestamp_ms: 1515100,
1931            }],
1932            refs: HashMap::new(),
1933            statistics: HashMap::new(),
1934            partition_statistics: HashMap::new(),
1935            encryption_keys: HashMap::from_iter(vec![("key1".to_string(), encryption_key)]),
1936            next_row_id: 5,
1937        };
1938
1939        let expected_json_value = serde_json::to_value(&expected).unwrap();
1940        check_table_metadata_serde(data, expected);
1941
1942        let json_value = serde_json::from_str::<serde_json::Value>(data).unwrap();
1943        assert_eq!(json_value, expected_json_value);
1944    }
1945
1946    #[test]
1947    fn test_table_data_v1() {
1948        let data = r#"
1949        {
1950            "format-version" : 1,
1951            "table-uuid" : "df838b92-0b32-465d-a44e-d39936e538b7",
1952            "location" : "/home/iceberg/warehouse/nyc/taxis",
1953            "last-updated-ms" : 1662532818843,
1954            "last-column-id" : 5,
1955            "schema" : {
1956              "type" : "struct",
1957              "schema-id" : 0,
1958              "fields" : [ {
1959                "id" : 1,
1960                "name" : "vendor_id",
1961                "required" : false,
1962                "type" : "long"
1963              }, {
1964                "id" : 2,
1965                "name" : "trip_id",
1966                "required" : false,
1967                "type" : "long"
1968              }, {
1969                "id" : 3,
1970                "name" : "trip_distance",
1971                "required" : false,
1972                "type" : "float"
1973              }, {
1974                "id" : 4,
1975                "name" : "fare_amount",
1976                "required" : false,
1977                "type" : "double"
1978              }, {
1979                "id" : 5,
1980                "name" : "store_and_fwd_flag",
1981                "required" : false,
1982                "type" : "string"
1983              } ]
1984            },
1985            "partition-spec" : [ {
1986              "name" : "vendor_id",
1987              "transform" : "identity",
1988              "source-id" : 1,
1989              "field-id" : 1000
1990            } ],
1991            "last-partition-id" : 1000,
1992            "default-sort-order-id" : 0,
1993            "sort-orders" : [ {
1994              "order-id" : 0,
1995              "fields" : [ ]
1996            } ],
1997            "properties" : {
1998              "owner" : "root"
1999            },
2000            "current-snapshot-id" : 638933773299822130,
2001            "refs" : {
2002              "main" : {
2003                "snapshot-id" : 638933773299822130,
2004                "type" : "branch"
2005              }
2006            },
2007            "snapshots" : [ {
2008              "snapshot-id" : 638933773299822130,
2009              "timestamp-ms" : 1662532818843,
2010              "sequence-number" : 0,
2011              "summary" : {
2012                "operation" : "append",
2013                "spark.app.id" : "local-1662532784305",
2014                "added-data-files" : "4",
2015                "added-records" : "4",
2016                "added-files-size" : "6001"
2017              },
2018              "manifest-list" : "/home/iceberg/warehouse/nyc/taxis/metadata/snap-638933773299822130-1-7e6760f0-4f6c-4b23-b907-0a5a174e3863.avro",
2019              "schema-id" : 0
2020            } ],
2021            "snapshot-log" : [ {
2022              "timestamp-ms" : 1662532818843,
2023              "snapshot-id" : 638933773299822130
2024            } ],
2025            "metadata-log" : [ {
2026              "timestamp-ms" : 1662532805245,
2027              "metadata-file" : "/home/iceberg/warehouse/nyc/taxis/metadata/00000-8a62c37d-4573-4021-952a-c0baef7d21d0.metadata.json"
2028            } ]
2029          }
2030        "#;
2031
2032        let schema = Schema::builder()
2033            .with_fields(vec![
2034                Arc::new(NestedField::optional(
2035                    1,
2036                    "vendor_id",
2037                    Type::Primitive(PrimitiveType::Long),
2038                )),
2039                Arc::new(NestedField::optional(
2040                    2,
2041                    "trip_id",
2042                    Type::Primitive(PrimitiveType::Long),
2043                )),
2044                Arc::new(NestedField::optional(
2045                    3,
2046                    "trip_distance",
2047                    Type::Primitive(PrimitiveType::Float),
2048                )),
2049                Arc::new(NestedField::optional(
2050                    4,
2051                    "fare_amount",
2052                    Type::Primitive(PrimitiveType::Double),
2053                )),
2054                Arc::new(NestedField::optional(
2055                    5,
2056                    "store_and_fwd_flag",
2057                    Type::Primitive(PrimitiveType::String),
2058                )),
2059            ])
2060            .build()
2061            .unwrap();
2062
2063        let schema = Arc::new(schema);
2064        let partition_spec = PartitionSpec::builder(schema.clone())
2065            .with_spec_id(0)
2066            .add_partition_field("vendor_id", "vendor_id", Transform::Identity)
2067            .unwrap()
2068            .build()
2069            .unwrap();
2070
2071        let sort_order = SortOrder::builder()
2072            .with_order_id(0)
2073            .build_unbound()
2074            .unwrap();
2075
2076        let snapshot = Snapshot::builder()
2077            .with_snapshot_id(638933773299822130)
2078            .with_timestamp_ms(1662532818843)
2079            .with_sequence_number(0)
2080            .with_schema_id(0)
2081            .with_manifest_list("/home/iceberg/warehouse/nyc/taxis/metadata/snap-638933773299822130-1-7e6760f0-4f6c-4b23-b907-0a5a174e3863.avro")
2082            .with_summary(Summary { operation: Operation::Append, additional_properties: HashMap::from_iter(vec![("spark.app.id".to_string(), "local-1662532784305".to_string()), ("added-data-files".to_string(), "4".to_string()), ("added-records".to_string(), "4".to_string()), ("added-files-size".to_string(), "6001".to_string())]) })
2083            .build();
2084
2085        let default_partition_type = partition_spec.partition_type(&schema).unwrap();
2086        let expected = TableMetadata {
2087            format_version: FormatVersion::V1,
2088            table_uuid: Uuid::parse_str("df838b92-0b32-465d-a44e-d39936e538b7").unwrap(),
2089            location: "/home/iceberg/warehouse/nyc/taxis".to_string(),
2090            last_updated_ms: 1662532818843,
2091            last_column_id: 5,
2092            schemas: HashMap::from_iter(vec![(0, schema)]),
2093            current_schema_id: 0,
2094            partition_specs: HashMap::from_iter(vec![(0, partition_spec.clone().into())]),
2095            default_partition_type,
2096            default_spec: Arc::new(partition_spec),
2097            last_partition_id: 1000,
2098            default_sort_order_id: 0,
2099            sort_orders: HashMap::from_iter(vec![(0, sort_order.into())]),
2100            snapshots: HashMap::from_iter(vec![(638933773299822130, Arc::new(snapshot))]),
2101            current_snapshot_id: Some(638933773299822130),
2102            last_sequence_number: 0,
2103            properties: HashMap::from_iter(vec![("owner".to_string(), "root".to_string())]),
2104            snapshot_log: vec![SnapshotLog {
2105                snapshot_id: 638933773299822130,
2106                timestamp_ms: 1662532818843,
2107            }],
2108            metadata_log: vec![MetadataLog { metadata_file: "/home/iceberg/warehouse/nyc/taxis/metadata/00000-8a62c37d-4573-4021-952a-c0baef7d21d0.metadata.json".to_string(), timestamp_ms: 1662532805245 }],
2109            refs: HashMap::from_iter(vec![("main".to_string(), SnapshotReference { snapshot_id: 638933773299822130, retention: SnapshotRetention::Branch { min_snapshots_to_keep: None, max_snapshot_age_ms: None, max_ref_age_ms: None } })]),
2110            statistics: HashMap::new(),
2111            partition_statistics: HashMap::new(),
2112            encryption_keys: HashMap::new(),
2113            next_row_id: INITIAL_ROW_ID,
2114        };
2115
2116        check_table_metadata_serde(data, expected);
2117    }
2118
2119    #[test]
2120    fn test_table_data_v2_no_snapshots() {
2121        let data = r#"
2122        {
2123            "format-version" : 2,
2124            "table-uuid": "fb072c92-a02b-11e9-ae9c-1bb7bc9eca94",
2125            "location": "s3://b/wh/data.db/table",
2126            "last-sequence-number" : 1,
2127            "last-updated-ms": 1515100955770,
2128            "last-column-id": 1,
2129            "schemas": [
2130                {
2131                    "schema-id" : 1,
2132                    "type" : "struct",
2133                    "fields" :[
2134                        {
2135                            "id": 1,
2136                            "name": "struct_name",
2137                            "required": true,
2138                            "type": "fixed[1]"
2139                        }
2140                    ]
2141                }
2142            ],
2143            "current-schema-id" : 1,
2144            "partition-specs": [
2145                {
2146                    "spec-id": 0,
2147                    "fields": []
2148                }
2149            ],
2150            "refs": {},
2151            "default-spec-id": 0,
2152            "last-partition-id": 1000,
2153            "metadata-log": [
2154                {
2155                    "metadata-file": "s3://bucket/.../v1.json",
2156                    "timestamp-ms": 1515100
2157                }
2158            ],
2159            "sort-orders": [
2160                {
2161                "order-id": 0,
2162                "fields": []
2163                }
2164            ],
2165            "default-sort-order-id": 0
2166        }
2167        "#;
2168
2169        let schema = Schema::builder()
2170            .with_schema_id(1)
2171            .with_fields(vec![Arc::new(NestedField::required(
2172                1,
2173                "struct_name",
2174                Type::Primitive(PrimitiveType::Fixed(1)),
2175            ))])
2176            .build()
2177            .unwrap();
2178
2179        let partition_spec = PartitionSpec::builder(schema.clone())
2180            .with_spec_id(0)
2181            .build()
2182            .unwrap();
2183
2184        let default_partition_type = partition_spec.partition_type(&schema).unwrap();
2185        let expected = TableMetadata {
2186            format_version: FormatVersion::V2,
2187            table_uuid: Uuid::parse_str("fb072c92-a02b-11e9-ae9c-1bb7bc9eca94").unwrap(),
2188            location: "s3://b/wh/data.db/table".to_string(),
2189            last_updated_ms: 1515100955770,
2190            last_column_id: 1,
2191            schemas: HashMap::from_iter(vec![(1, Arc::new(schema))]),
2192            current_schema_id: 1,
2193            partition_specs: HashMap::from_iter(vec![(0, partition_spec.clone().into())]),
2194            default_partition_type,
2195            default_spec: partition_spec.into(),
2196            last_partition_id: 1000,
2197            default_sort_order_id: 0,
2198            sort_orders: HashMap::from_iter(vec![(0, SortOrder::unsorted_order().into())]),
2199            snapshots: HashMap::default(),
2200            current_snapshot_id: None,
2201            last_sequence_number: 1,
2202            properties: HashMap::new(),
2203            snapshot_log: Vec::new(),
2204            metadata_log: vec![MetadataLog {
2205                metadata_file: "s3://bucket/.../v1.json".to_string(),
2206                timestamp_ms: 1515100,
2207            }],
2208            refs: HashMap::new(),
2209            statistics: HashMap::new(),
2210            partition_statistics: HashMap::new(),
2211            encryption_keys: HashMap::new(),
2212            next_row_id: INITIAL_ROW_ID,
2213        };
2214
2215        let expected_json_value = serde_json::to_value(&expected).unwrap();
2216        check_table_metadata_serde(data, expected);
2217
2218        let json_value = serde_json::from_str::<serde_json::Value>(data).unwrap();
2219        assert_eq!(json_value, expected_json_value);
2220    }
2221
2222    #[test]
2223    fn test_current_snapshot_id_must_match_main_branch() {
2224        let data = r#"
2225        {
2226            "format-version" : 2,
2227            "table-uuid": "fb072c92-a02b-11e9-ae9c-1bb7bc9eca94",
2228            "location": "s3://b/wh/data.db/table",
2229            "last-sequence-number" : 1,
2230            "last-updated-ms": 1515100955770,
2231            "last-column-id": 1,
2232            "schemas": [
2233                {
2234                    "schema-id" : 1,
2235                    "type" : "struct",
2236                    "fields" :[
2237                        {
2238                            "id": 1,
2239                            "name": "struct_name",
2240                            "required": true,
2241                            "type": "fixed[1]"
2242                        },
2243                        {
2244                            "id": 4,
2245                            "name": "ts",
2246                            "required": true,
2247                            "type": "timestamp"
2248                        }
2249                    ]
2250                }
2251            ],
2252            "current-schema-id" : 1,
2253            "partition-specs": [
2254                {
2255                    "spec-id": 0,
2256                    "fields": [
2257                        {
2258                            "source-id": 4,
2259                            "field-id": 1000,
2260                            "name": "ts_day",
2261                            "transform": "day"
2262                        }
2263                    ]
2264                }
2265            ],
2266            "default-spec-id": 0,
2267            "last-partition-id": 1000,
2268            "properties": {
2269                "commit.retry.num-retries": "1"
2270            },
2271            "metadata-log": [
2272                {
2273                    "metadata-file": "s3://bucket/.../v1.json",
2274                    "timestamp-ms": 1515100
2275                }
2276            ],
2277            "sort-orders": [
2278                {
2279                "order-id": 0,
2280                "fields": []
2281                }
2282            ],
2283            "default-sort-order-id": 0,
2284            "current-snapshot-id" : 1,
2285            "refs" : {
2286              "main" : {
2287                "snapshot-id" : 2,
2288                "type" : "branch"
2289              }
2290            },
2291            "snapshots" : [ {
2292              "snapshot-id" : 1,
2293              "timestamp-ms" : 1662532818843,
2294              "sequence-number" : 0,
2295              "summary" : {
2296                "operation" : "append",
2297                "spark.app.id" : "local-1662532784305",
2298                "added-data-files" : "4",
2299                "added-records" : "4",
2300                "added-files-size" : "6001"
2301              },
2302              "manifest-list" : "/home/iceberg/warehouse/nyc/taxis/metadata/snap-638933773299822130-1-7e6760f0-4f6c-4b23-b907-0a5a174e3863.avro",
2303              "schema-id" : 0
2304            },
2305            {
2306              "snapshot-id" : 2,
2307              "timestamp-ms" : 1662532818844,
2308              "sequence-number" : 0,
2309              "summary" : {
2310                "operation" : "append",
2311                "spark.app.id" : "local-1662532784305",
2312                "added-data-files" : "4",
2313                "added-records" : "4",
2314                "added-files-size" : "6001"
2315              },
2316              "manifest-list" : "/home/iceberg/warehouse/nyc/taxis/metadata/snap-638933773299822130-1-7e6760f0-4f6c-4b23-b907-0a5a174e3863.avro",
2317              "schema-id" : 0
2318            } ]
2319        }
2320    "#;
2321
2322        let err = serde_json::from_str::<TableMetadata>(data).unwrap_err();
2323        assert!(
2324            err.to_string()
2325                .contains("Current snapshot id does not match main branch")
2326        );
2327    }
2328
2329    #[test]
2330    fn test_main_without_current() {
2331        let data = r#"
2332        {
2333            "format-version" : 2,
2334            "table-uuid": "fb072c92-a02b-11e9-ae9c-1bb7bc9eca94",
2335            "location": "s3://b/wh/data.db/table",
2336            "last-sequence-number" : 1,
2337            "last-updated-ms": 1515100955770,
2338            "last-column-id": 1,
2339            "schemas": [
2340                {
2341                    "schema-id" : 1,
2342                    "type" : "struct",
2343                    "fields" :[
2344                        {
2345                            "id": 1,
2346                            "name": "struct_name",
2347                            "required": true,
2348                            "type": "fixed[1]"
2349                        },
2350                        {
2351                            "id": 4,
2352                            "name": "ts",
2353                            "required": true,
2354                            "type": "timestamp"
2355                        }
2356                    ]
2357                }
2358            ],
2359            "current-schema-id" : 1,
2360            "partition-specs": [
2361                {
2362                    "spec-id": 0,
2363                    "fields": [
2364                        {
2365                            "source-id": 4,
2366                            "field-id": 1000,
2367                            "name": "ts_day",
2368                            "transform": "day"
2369                        }
2370                    ]
2371                }
2372            ],
2373            "default-spec-id": 0,
2374            "last-partition-id": 1000,
2375            "properties": {
2376                "commit.retry.num-retries": "1"
2377            },
2378            "metadata-log": [
2379                {
2380                    "metadata-file": "s3://bucket/.../v1.json",
2381                    "timestamp-ms": 1515100
2382                }
2383            ],
2384            "sort-orders": [
2385                {
2386                "order-id": 0,
2387                "fields": []
2388                }
2389            ],
2390            "default-sort-order-id": 0,
2391            "refs" : {
2392              "main" : {
2393                "snapshot-id" : 1,
2394                "type" : "branch"
2395              }
2396            },
2397            "snapshots" : [ {
2398              "snapshot-id" : 1,
2399              "timestamp-ms" : 1662532818843,
2400              "sequence-number" : 0,
2401              "summary" : {
2402                "operation" : "append",
2403                "spark.app.id" : "local-1662532784305",
2404                "added-data-files" : "4",
2405                "added-records" : "4",
2406                "added-files-size" : "6001"
2407              },
2408              "manifest-list" : "/home/iceberg/warehouse/nyc/taxis/metadata/snap-638933773299822130-1-7e6760f0-4f6c-4b23-b907-0a5a174e3863.avro",
2409              "schema-id" : 0
2410            } ]
2411        }
2412    "#;
2413
2414        let err = serde_json::from_str::<TableMetadata>(data).unwrap_err();
2415        assert!(
2416            err.to_string()
2417                .contains("Current snapshot is not set, but main branch exists")
2418        );
2419    }
2420
2421    #[test]
2422    fn test_branch_snapshot_missing() {
2423        let data = r#"
2424        {
2425            "format-version" : 2,
2426            "table-uuid": "fb072c92-a02b-11e9-ae9c-1bb7bc9eca94",
2427            "location": "s3://b/wh/data.db/table",
2428            "last-sequence-number" : 1,
2429            "last-updated-ms": 1515100955770,
2430            "last-column-id": 1,
2431            "schemas": [
2432                {
2433                    "schema-id" : 1,
2434                    "type" : "struct",
2435                    "fields" :[
2436                        {
2437                            "id": 1,
2438                            "name": "struct_name",
2439                            "required": true,
2440                            "type": "fixed[1]"
2441                        },
2442                        {
2443                            "id": 4,
2444                            "name": "ts",
2445                            "required": true,
2446                            "type": "timestamp"
2447                        }
2448                    ]
2449                }
2450            ],
2451            "current-schema-id" : 1,
2452            "partition-specs": [
2453                {
2454                    "spec-id": 0,
2455                    "fields": [
2456                        {
2457                            "source-id": 4,
2458                            "field-id": 1000,
2459                            "name": "ts_day",
2460                            "transform": "day"
2461                        }
2462                    ]
2463                }
2464            ],
2465            "default-spec-id": 0,
2466            "last-partition-id": 1000,
2467            "properties": {
2468                "commit.retry.num-retries": "1"
2469            },
2470            "metadata-log": [
2471                {
2472                    "metadata-file": "s3://bucket/.../v1.json",
2473                    "timestamp-ms": 1515100
2474                }
2475            ],
2476            "sort-orders": [
2477                {
2478                "order-id": 0,
2479                "fields": []
2480                }
2481            ],
2482            "default-sort-order-id": 0,
2483            "refs" : {
2484              "main" : {
2485                "snapshot-id" : 1,
2486                "type" : "branch"
2487              },
2488              "foo" : {
2489                "snapshot-id" : 2,
2490                "type" : "branch"
2491              }
2492            },
2493            "snapshots" : [ {
2494              "snapshot-id" : 1,
2495              "timestamp-ms" : 1662532818843,
2496              "sequence-number" : 0,
2497              "summary" : {
2498                "operation" : "append",
2499                "spark.app.id" : "local-1662532784305",
2500                "added-data-files" : "4",
2501                "added-records" : "4",
2502                "added-files-size" : "6001"
2503              },
2504              "manifest-list" : "/home/iceberg/warehouse/nyc/taxis/metadata/snap-638933773299822130-1-7e6760f0-4f6c-4b23-b907-0a5a174e3863.avro",
2505              "schema-id" : 0
2506            } ]
2507        }
2508    "#;
2509
2510        let err = serde_json::from_str::<TableMetadata>(data).unwrap_err();
2511        assert!(
2512            err.to_string().contains(
2513                "Snapshot for reference foo does not exist in the existing snapshots list"
2514            )
2515        );
2516    }
2517
2518    #[test]
2519    fn test_v2_wrong_max_snapshot_sequence_number() {
2520        let data = r#"
2521        {
2522            "format-version": 2,
2523            "table-uuid": "9c12d441-03fe-4693-9a96-a0705ddf69c1",
2524            "location": "s3://bucket/test/location",
2525            "last-sequence-number": 1,
2526            "last-updated-ms": 1602638573590,
2527            "last-column-id": 3,
2528            "current-schema-id": 0,
2529            "schemas": [
2530                {
2531                    "type": "struct",
2532                    "schema-id": 0,
2533                    "fields": [
2534                        {
2535                            "id": 1,
2536                            "name": "x",
2537                            "required": true,
2538                            "type": "long"
2539                        }
2540                    ]
2541                }
2542            ],
2543            "default-spec-id": 0,
2544            "partition-specs": [
2545                {
2546                    "spec-id": 0,
2547                    "fields": []
2548                }
2549            ],
2550            "last-partition-id": 1000,
2551            "default-sort-order-id": 0,
2552            "sort-orders": [
2553                {
2554                    "order-id": 0,
2555                    "fields": []
2556                }
2557            ],
2558            "properties": {},
2559            "current-snapshot-id": 3055729675574597004,
2560            "snapshots": [
2561                {
2562                    "snapshot-id": 3055729675574597004,
2563                    "timestamp-ms": 1555100955770,
2564                    "sequence-number": 4,
2565                    "summary": {
2566                        "operation": "append"
2567                    },
2568                    "manifest-list": "s3://a/b/2.avro",
2569                    "schema-id": 0
2570                }
2571            ],
2572            "statistics": [],
2573            "snapshot-log": [],
2574            "metadata-log": []
2575        }
2576    "#;
2577
2578        let err = serde_json::from_str::<TableMetadata>(data).unwrap_err();
2579        assert!(err.to_string().contains(
2580            "Invalid snapshot with id 3055729675574597004 and sequence number 4 greater than last sequence number 1"
2581        ));
2582
2583        // Change max sequence number to 4 - should work
2584        let data = data.replace(
2585            r#""last-sequence-number": 1,"#,
2586            r#""last-sequence-number": 4,"#,
2587        );
2588        let metadata = serde_json::from_str::<TableMetadata>(data.as_str()).unwrap();
2589        assert_eq!(metadata.last_sequence_number, 4);
2590
2591        // Change max sequence number to 5 - should work
2592        let data = data.replace(
2593            r#""last-sequence-number": 4,"#,
2594            r#""last-sequence-number": 5,"#,
2595        );
2596        let metadata = serde_json::from_str::<TableMetadata>(data.as_str()).unwrap();
2597        assert_eq!(metadata.last_sequence_number, 5);
2598    }
2599
2600    #[test]
2601    fn test_statistic_files() {
2602        let data = r#"
2603        {
2604            "format-version": 2,
2605            "table-uuid": "9c12d441-03fe-4693-9a96-a0705ddf69c1",
2606            "location": "s3://bucket/test/location",
2607            "last-sequence-number": 34,
2608            "last-updated-ms": 1602638573590,
2609            "last-column-id": 3,
2610            "current-schema-id": 0,
2611            "schemas": [
2612                {
2613                    "type": "struct",
2614                    "schema-id": 0,
2615                    "fields": [
2616                        {
2617                            "id": 1,
2618                            "name": "x",
2619                            "required": true,
2620                            "type": "long"
2621                        }
2622                    ]
2623                }
2624            ],
2625            "default-spec-id": 0,
2626            "partition-specs": [
2627                {
2628                    "spec-id": 0,
2629                    "fields": []
2630                }
2631            ],
2632            "last-partition-id": 1000,
2633            "default-sort-order-id": 0,
2634            "sort-orders": [
2635                {
2636                    "order-id": 0,
2637                    "fields": []
2638                }
2639            ],
2640            "properties": {},
2641            "current-snapshot-id": 3055729675574597004,
2642            "snapshots": [
2643                {
2644                    "snapshot-id": 3055729675574597004,
2645                    "timestamp-ms": 1555100955770,
2646                    "sequence-number": 1,
2647                    "summary": {
2648                        "operation": "append"
2649                    },
2650                    "manifest-list": "s3://a/b/2.avro",
2651                    "schema-id": 0
2652                }
2653            ],
2654            "statistics": [
2655                {
2656                    "snapshot-id": 3055729675574597004,
2657                    "statistics-path": "s3://a/b/stats.puffin",
2658                    "file-size-in-bytes": 413,
2659                    "file-footer-size-in-bytes": 42,
2660                    "blob-metadata": [
2661                        {
2662                            "type": "ndv",
2663                            "snapshot-id": 3055729675574597004,
2664                            "sequence-number": 1,
2665                            "fields": [
2666                                1
2667                            ]
2668                        }
2669                    ]
2670                }
2671            ],
2672            "snapshot-log": [],
2673            "metadata-log": []
2674        }
2675    "#;
2676
2677        let schema = Schema::builder()
2678            .with_schema_id(0)
2679            .with_fields(vec![Arc::new(NestedField::required(
2680                1,
2681                "x",
2682                Type::Primitive(PrimitiveType::Long),
2683            ))])
2684            .build()
2685            .unwrap();
2686        let partition_spec = PartitionSpec::builder(schema.clone())
2687            .with_spec_id(0)
2688            .build()
2689            .unwrap();
2690        let snapshot = Snapshot::builder()
2691            .with_snapshot_id(3055729675574597004)
2692            .with_timestamp_ms(1555100955770)
2693            .with_sequence_number(1)
2694            .with_manifest_list("s3://a/b/2.avro")
2695            .with_schema_id(0)
2696            .with_summary(Summary {
2697                operation: Operation::Append,
2698                additional_properties: HashMap::new(),
2699            })
2700            .build();
2701
2702        let default_partition_type = partition_spec.partition_type(&schema).unwrap();
2703        let expected = TableMetadata {
2704            format_version: FormatVersion::V2,
2705            table_uuid: Uuid::parse_str("9c12d441-03fe-4693-9a96-a0705ddf69c1").unwrap(),
2706            location: "s3://bucket/test/location".to_string(),
2707            last_updated_ms: 1602638573590,
2708            last_column_id: 3,
2709            schemas: HashMap::from_iter(vec![(0, Arc::new(schema))]),
2710            current_schema_id: 0,
2711            partition_specs: HashMap::from_iter(vec![(0, partition_spec.clone().into())]),
2712            default_partition_type,
2713            default_spec: Arc::new(partition_spec),
2714            last_partition_id: 1000,
2715            default_sort_order_id: 0,
2716            sort_orders: HashMap::from_iter(vec![(0, SortOrder::unsorted_order().into())]),
2717            snapshots: HashMap::from_iter(vec![(3055729675574597004, Arc::new(snapshot))]),
2718            current_snapshot_id: Some(3055729675574597004),
2719            last_sequence_number: 34,
2720            properties: HashMap::new(),
2721            snapshot_log: Vec::new(),
2722            metadata_log: Vec::new(),
2723            statistics: HashMap::from_iter(vec![(3055729675574597004, StatisticsFile {
2724                snapshot_id: 3055729675574597004,
2725                statistics_path: "s3://a/b/stats.puffin".to_string(),
2726                file_size_in_bytes: 413,
2727                file_footer_size_in_bytes: 42,
2728                key_metadata: None,
2729                blob_metadata: vec![BlobMetadata {
2730                    snapshot_id: 3055729675574597004,
2731                    sequence_number: 1,
2732                    fields: vec![1],
2733                    r#type: "ndv".to_string(),
2734                    properties: HashMap::new(),
2735                }],
2736            })]),
2737            partition_statistics: HashMap::new(),
2738            refs: HashMap::from_iter(vec![("main".to_string(), SnapshotReference {
2739                snapshot_id: 3055729675574597004,
2740                retention: SnapshotRetention::Branch {
2741                    min_snapshots_to_keep: None,
2742                    max_snapshot_age_ms: None,
2743                    max_ref_age_ms: None,
2744                },
2745            })]),
2746            encryption_keys: HashMap::new(),
2747            next_row_id: INITIAL_ROW_ID,
2748        };
2749
2750        check_table_metadata_serde(data, expected);
2751    }
2752
2753    #[test]
2754    fn test_partition_statistics_file() {
2755        let data = r#"
2756        {
2757            "format-version": 2,
2758            "table-uuid": "9c12d441-03fe-4693-9a96-a0705ddf69c1",
2759            "location": "s3://bucket/test/location",
2760            "last-sequence-number": 34,
2761            "last-updated-ms": 1602638573590,
2762            "last-column-id": 3,
2763            "current-schema-id": 0,
2764            "schemas": [
2765                {
2766                    "type": "struct",
2767                    "schema-id": 0,
2768                    "fields": [
2769                        {
2770                            "id": 1,
2771                            "name": "x",
2772                            "required": true,
2773                            "type": "long"
2774                        }
2775                    ]
2776                }
2777            ],
2778            "default-spec-id": 0,
2779            "partition-specs": [
2780                {
2781                    "spec-id": 0,
2782                    "fields": []
2783                }
2784            ],
2785            "last-partition-id": 1000,
2786            "default-sort-order-id": 0,
2787            "sort-orders": [
2788                {
2789                    "order-id": 0,
2790                    "fields": []
2791                }
2792            ],
2793            "properties": {},
2794            "current-snapshot-id": 3055729675574597004,
2795            "snapshots": [
2796                {
2797                    "snapshot-id": 3055729675574597004,
2798                    "timestamp-ms": 1555100955770,
2799                    "sequence-number": 1,
2800                    "summary": {
2801                        "operation": "append"
2802                    },
2803                    "manifest-list": "s3://a/b/2.avro",
2804                    "schema-id": 0
2805                }
2806            ],
2807            "partition-statistics": [
2808                {
2809                    "snapshot-id": 3055729675574597004,
2810                    "statistics-path": "s3://a/b/partition-stats.parquet",
2811                    "file-size-in-bytes": 43
2812                }
2813            ],
2814            "snapshot-log": [],
2815            "metadata-log": []
2816        }
2817        "#;
2818
2819        let schema = Schema::builder()
2820            .with_schema_id(0)
2821            .with_fields(vec![Arc::new(NestedField::required(
2822                1,
2823                "x",
2824                Type::Primitive(PrimitiveType::Long),
2825            ))])
2826            .build()
2827            .unwrap();
2828        let partition_spec = PartitionSpec::builder(schema.clone())
2829            .with_spec_id(0)
2830            .build()
2831            .unwrap();
2832        let snapshot = Snapshot::builder()
2833            .with_snapshot_id(3055729675574597004)
2834            .with_timestamp_ms(1555100955770)
2835            .with_sequence_number(1)
2836            .with_manifest_list("s3://a/b/2.avro")
2837            .with_schema_id(0)
2838            .with_summary(Summary {
2839                operation: Operation::Append,
2840                additional_properties: HashMap::new(),
2841            })
2842            .build();
2843
2844        let default_partition_type = partition_spec.partition_type(&schema).unwrap();
2845        let expected = TableMetadata {
2846            format_version: FormatVersion::V2,
2847            table_uuid: Uuid::parse_str("9c12d441-03fe-4693-9a96-a0705ddf69c1").unwrap(),
2848            location: "s3://bucket/test/location".to_string(),
2849            last_updated_ms: 1602638573590,
2850            last_column_id: 3,
2851            schemas: HashMap::from_iter(vec![(0, Arc::new(schema))]),
2852            current_schema_id: 0,
2853            partition_specs: HashMap::from_iter(vec![(0, partition_spec.clone().into())]),
2854            default_spec: Arc::new(partition_spec),
2855            default_partition_type,
2856            last_partition_id: 1000,
2857            default_sort_order_id: 0,
2858            sort_orders: HashMap::from_iter(vec![(0, SortOrder::unsorted_order().into())]),
2859            snapshots: HashMap::from_iter(vec![(3055729675574597004, Arc::new(snapshot))]),
2860            current_snapshot_id: Some(3055729675574597004),
2861            last_sequence_number: 34,
2862            properties: HashMap::new(),
2863            snapshot_log: Vec::new(),
2864            metadata_log: Vec::new(),
2865            statistics: HashMap::new(),
2866            partition_statistics: HashMap::from_iter(vec![(
2867                3055729675574597004,
2868                PartitionStatisticsFile {
2869                    snapshot_id: 3055729675574597004,
2870                    statistics_path: "s3://a/b/partition-stats.parquet".to_string(),
2871                    file_size_in_bytes: 43,
2872                },
2873            )]),
2874            refs: HashMap::from_iter(vec![("main".to_string(), SnapshotReference {
2875                snapshot_id: 3055729675574597004,
2876                retention: SnapshotRetention::Branch {
2877                    min_snapshots_to_keep: None,
2878                    max_snapshot_age_ms: None,
2879                    max_ref_age_ms: None,
2880                },
2881            })]),
2882            encryption_keys: HashMap::new(),
2883            next_row_id: INITIAL_ROW_ID,
2884        };
2885
2886        check_table_metadata_serde(data, expected);
2887    }
2888
2889    #[test]
2890    fn test_invalid_table_uuid() -> Result<()> {
2891        let data = r#"
2892            {
2893                "format-version" : 2,
2894                "table-uuid": "xxxx"
2895            }
2896        "#;
2897        assert!(serde_json::from_str::<TableMetadata>(data).is_err());
2898        Ok(())
2899    }
2900
2901    #[test]
2902    fn test_deserialize_table_data_v2_invalid_format_version() -> Result<()> {
2903        let data = r#"
2904            {
2905                "format-version" : 1
2906            }
2907        "#;
2908        assert!(serde_json::from_str::<TableMetadata>(data).is_err());
2909        Ok(())
2910    }
2911
2912    #[test]
2913    fn test_table_metadata_v3_valid_minimal() {
2914        let metadata_str =
2915            fs::read_to_string("testdata/table_metadata/TableMetadataV3ValidMinimal.json").unwrap();
2916
2917        let table_metadata = serde_json::from_str::<TableMetadata>(&metadata_str).unwrap();
2918        assert_eq!(table_metadata.format_version, FormatVersion::V3);
2919
2920        let schema = Schema::builder()
2921            .with_schema_id(0)
2922            .with_fields(vec![
2923                Arc::new(
2924                    NestedField::required(1, "x", Type::Primitive(PrimitiveType::Long))
2925                        .with_initial_default(Literal::Primitive(PrimitiveLiteral::Long(1)))
2926                        .with_write_default(Literal::Primitive(PrimitiveLiteral::Long(1))),
2927                ),
2928                Arc::new(
2929                    NestedField::required(2, "y", Type::Primitive(PrimitiveType::Long))
2930                        .with_doc("comment"),
2931                ),
2932                Arc::new(NestedField::required(
2933                    3,
2934                    "z",
2935                    Type::Primitive(PrimitiveType::Long),
2936                )),
2937            ])
2938            .build()
2939            .unwrap();
2940
2941        let partition_spec = PartitionSpec::builder(schema.clone())
2942            .with_spec_id(0)
2943            .add_unbound_field(
2944                UnboundPartitionField::builder()
2945                    .source_ids(vec![1])
2946                    .field_id(1000)
2947                    .name("x".to_string())
2948                    .transform(Transform::Identity)
2949                    .build()
2950                    .unwrap(),
2951            )
2952            .unwrap()
2953            .build()
2954            .unwrap();
2955
2956        let sort_order = SortOrder::builder()
2957            .with_order_id(3)
2958            .with_sort_field(SortField {
2959                source_id: 2,
2960                transform: Transform::Identity,
2961                direction: SortDirection::Ascending,
2962                null_order: NullOrder::First,
2963            })
2964            .with_sort_field(SortField {
2965                source_id: 3,
2966                transform: Transform::Bucket(4),
2967                direction: SortDirection::Descending,
2968                null_order: NullOrder::Last,
2969            })
2970            .build_unbound()
2971            .unwrap();
2972
2973        let default_partition_type = partition_spec.partition_type(&schema).unwrap();
2974        let expected = TableMetadata {
2975            format_version: FormatVersion::V3,
2976            table_uuid: Uuid::parse_str("9c12d441-03fe-4693-9a96-a0705ddf69c1").unwrap(),
2977            location: "s3://bucket/test/location".to_string(),
2978            last_updated_ms: 1602638573590,
2979            last_column_id: 3,
2980            schemas: HashMap::from_iter(vec![(0, Arc::new(schema))]),
2981            current_schema_id: 0,
2982            partition_specs: HashMap::from_iter(vec![(0, partition_spec.clone().into())]),
2983            default_spec: Arc::new(partition_spec),
2984            default_partition_type,
2985            last_partition_id: 1000,
2986            default_sort_order_id: 3,
2987            sort_orders: HashMap::from_iter(vec![(3, sort_order.into())]),
2988            snapshots: HashMap::default(),
2989            current_snapshot_id: None,
2990            last_sequence_number: 34,
2991            properties: HashMap::new(),
2992            snapshot_log: Vec::new(),
2993            metadata_log: Vec::new(),
2994            refs: HashMap::new(),
2995            statistics: HashMap::new(),
2996            partition_statistics: HashMap::new(),
2997            encryption_keys: HashMap::new(),
2998            next_row_id: 0, // V3 specific field from the JSON
2999        };
3000
3001        check_table_metadata_serde(&metadata_str, expected);
3002    }
3003
3004    #[test]
3005    fn test_table_metadata_v2_file_valid() {
3006        let metadata =
3007            fs::read_to_string("testdata/table_metadata/TableMetadataV2Valid.json").unwrap();
3008
3009        let schema1 = Schema::builder()
3010            .with_schema_id(0)
3011            .with_fields(vec![Arc::new(NestedField::required(
3012                1,
3013                "x",
3014                Type::Primitive(PrimitiveType::Long),
3015            ))])
3016            .build()
3017            .unwrap();
3018
3019        let schema2 = Schema::builder()
3020            .with_schema_id(1)
3021            .with_fields(vec![
3022                Arc::new(NestedField::required(
3023                    1,
3024                    "x",
3025                    Type::Primitive(PrimitiveType::Long),
3026                )),
3027                Arc::new(
3028                    NestedField::required(2, "y", Type::Primitive(PrimitiveType::Long))
3029                        .with_doc("comment"),
3030                ),
3031                Arc::new(NestedField::required(
3032                    3,
3033                    "z",
3034                    Type::Primitive(PrimitiveType::Long),
3035                )),
3036            ])
3037            .with_identifier_field_ids(vec![1, 2])
3038            .build()
3039            .unwrap();
3040
3041        let partition_spec = PartitionSpec::builder(schema2.clone())
3042            .with_spec_id(0)
3043            .add_unbound_field(
3044                UnboundPartitionField::builder()
3045                    .source_ids(vec![1])
3046                    .field_id(1000)
3047                    .name("x".to_string())
3048                    .transform(Transform::Identity)
3049                    .build()
3050                    .unwrap(),
3051            )
3052            .unwrap()
3053            .build()
3054            .unwrap();
3055
3056        let sort_order = SortOrder::builder()
3057            .with_order_id(3)
3058            .with_sort_field(SortField {
3059                source_id: 2,
3060                transform: Transform::Identity,
3061                direction: SortDirection::Ascending,
3062                null_order: NullOrder::First,
3063            })
3064            .with_sort_field(SortField {
3065                source_id: 3,
3066                transform: Transform::Bucket(4),
3067                direction: SortDirection::Descending,
3068                null_order: NullOrder::Last,
3069            })
3070            .build_unbound()
3071            .unwrap();
3072
3073        let snapshot1 = Snapshot::builder()
3074            .with_snapshot_id(3051729675574597004)
3075            .with_timestamp_ms(1515100955770)
3076            .with_sequence_number(0)
3077            .with_manifest_list("s3://a/b/1.avro")
3078            .with_summary(Summary {
3079                operation: Operation::Append,
3080                additional_properties: HashMap::new(),
3081            })
3082            .build();
3083
3084        let snapshot2 = Snapshot::builder()
3085            .with_snapshot_id(3055729675574597004)
3086            .with_parent_snapshot_id(Some(3051729675574597004))
3087            .with_timestamp_ms(1555100955770)
3088            .with_sequence_number(1)
3089            .with_schema_id(1)
3090            .with_manifest_list("s3://a/b/2.avro")
3091            .with_summary(Summary {
3092                operation: Operation::Append,
3093                additional_properties: HashMap::new(),
3094            })
3095            .build();
3096
3097        let default_partition_type = partition_spec.partition_type(&schema2).unwrap();
3098        let expected = TableMetadata {
3099            format_version: FormatVersion::V2,
3100            table_uuid: Uuid::parse_str("9c12d441-03fe-4693-9a96-a0705ddf69c1").unwrap(),
3101            location: "s3://bucket/test/location".to_string(),
3102            last_updated_ms: 1602638573590,
3103            last_column_id: 3,
3104            schemas: HashMap::from_iter(vec![(0, Arc::new(schema1)), (1, Arc::new(schema2))]),
3105            current_schema_id: 1,
3106            partition_specs: HashMap::from_iter(vec![(0, partition_spec.clone().into())]),
3107            default_spec: Arc::new(partition_spec),
3108            default_partition_type,
3109            last_partition_id: 1000,
3110            default_sort_order_id: 3,
3111            sort_orders: HashMap::from_iter(vec![(3, sort_order.into())]),
3112            snapshots: HashMap::from_iter(vec![
3113                (3051729675574597004, Arc::new(snapshot1)),
3114                (3055729675574597004, Arc::new(snapshot2)),
3115            ]),
3116            current_snapshot_id: Some(3055729675574597004),
3117            last_sequence_number: 34,
3118            properties: HashMap::new(),
3119            snapshot_log: vec![
3120                SnapshotLog {
3121                    snapshot_id: 3051729675574597004,
3122                    timestamp_ms: 1515100955770,
3123                },
3124                SnapshotLog {
3125                    snapshot_id: 3055729675574597004,
3126                    timestamp_ms: 1555100955770,
3127                },
3128            ],
3129            metadata_log: Vec::new(),
3130            refs: HashMap::from_iter(vec![("main".to_string(), SnapshotReference {
3131                snapshot_id: 3055729675574597004,
3132                retention: SnapshotRetention::Branch {
3133                    min_snapshots_to_keep: None,
3134                    max_snapshot_age_ms: None,
3135                    max_ref_age_ms: None,
3136                },
3137            })]),
3138            statistics: HashMap::new(),
3139            partition_statistics: HashMap::new(),
3140            encryption_keys: HashMap::new(),
3141            next_row_id: INITIAL_ROW_ID,
3142        };
3143
3144        check_table_metadata_serde(&metadata, expected);
3145    }
3146
3147    #[test]
3148    fn test_table_metadata_v2_file_valid_minimal() {
3149        let metadata =
3150            fs::read_to_string("testdata/table_metadata/TableMetadataV2ValidMinimal.json").unwrap();
3151
3152        let schema = Schema::builder()
3153            .with_schema_id(0)
3154            .with_fields(vec![
3155                Arc::new(NestedField::required(
3156                    1,
3157                    "x",
3158                    Type::Primitive(PrimitiveType::Long),
3159                )),
3160                Arc::new(
3161                    NestedField::required(2, "y", Type::Primitive(PrimitiveType::Long))
3162                        .with_doc("comment"),
3163                ),
3164                Arc::new(NestedField::required(
3165                    3,
3166                    "z",
3167                    Type::Primitive(PrimitiveType::Long),
3168                )),
3169            ])
3170            .build()
3171            .unwrap();
3172
3173        let partition_spec = PartitionSpec::builder(schema.clone())
3174            .with_spec_id(0)
3175            .add_unbound_field(
3176                UnboundPartitionField::builder()
3177                    .source_ids(vec![1])
3178                    .field_id(1000)
3179                    .name("x".to_string())
3180                    .transform(Transform::Identity)
3181                    .build()
3182                    .unwrap(),
3183            )
3184            .unwrap()
3185            .build()
3186            .unwrap();
3187
3188        let sort_order = SortOrder::builder()
3189            .with_order_id(3)
3190            .with_sort_field(SortField {
3191                source_id: 2,
3192                transform: Transform::Identity,
3193                direction: SortDirection::Ascending,
3194                null_order: NullOrder::First,
3195            })
3196            .with_sort_field(SortField {
3197                source_id: 3,
3198                transform: Transform::Bucket(4),
3199                direction: SortDirection::Descending,
3200                null_order: NullOrder::Last,
3201            })
3202            .build_unbound()
3203            .unwrap();
3204
3205        let default_partition_type = partition_spec.partition_type(&schema).unwrap();
3206        let expected = TableMetadata {
3207            format_version: FormatVersion::V2,
3208            table_uuid: Uuid::parse_str("9c12d441-03fe-4693-9a96-a0705ddf69c1").unwrap(),
3209            location: "s3://bucket/test/location".to_string(),
3210            last_updated_ms: 1602638573590,
3211            last_column_id: 3,
3212            schemas: HashMap::from_iter(vec![(0, Arc::new(schema))]),
3213            current_schema_id: 0,
3214            partition_specs: HashMap::from_iter(vec![(0, partition_spec.clone().into())]),
3215            default_partition_type,
3216            default_spec: Arc::new(partition_spec),
3217            last_partition_id: 1000,
3218            default_sort_order_id: 3,
3219            sort_orders: HashMap::from_iter(vec![(3, sort_order.into())]),
3220            snapshots: HashMap::default(),
3221            current_snapshot_id: None,
3222            last_sequence_number: 34,
3223            properties: HashMap::new(),
3224            snapshot_log: vec![],
3225            metadata_log: Vec::new(),
3226            refs: HashMap::new(),
3227            statistics: HashMap::new(),
3228            partition_statistics: HashMap::new(),
3229            encryption_keys: HashMap::new(),
3230            next_row_id: INITIAL_ROW_ID,
3231        };
3232
3233        check_table_metadata_serde(&metadata, expected);
3234    }
3235
3236    #[test]
3237    fn test_table_metadata_v1_file_valid() {
3238        let metadata =
3239            fs::read_to_string("testdata/table_metadata/TableMetadataV1Valid.json").unwrap();
3240
3241        let schema = Schema::builder()
3242            .with_schema_id(0)
3243            .with_fields(vec![
3244                Arc::new(NestedField::required(
3245                    1,
3246                    "x",
3247                    Type::Primitive(PrimitiveType::Long),
3248                )),
3249                Arc::new(
3250                    NestedField::required(2, "y", Type::Primitive(PrimitiveType::Long))
3251                        .with_doc("comment"),
3252                ),
3253                Arc::new(NestedField::required(
3254                    3,
3255                    "z",
3256                    Type::Primitive(PrimitiveType::Long),
3257                )),
3258            ])
3259            .build()
3260            .unwrap();
3261
3262        let partition_spec = PartitionSpec::builder(schema.clone())
3263            .with_spec_id(0)
3264            .add_unbound_field(
3265                UnboundPartitionField::builder()
3266                    .source_ids(vec![1])
3267                    .field_id(1000)
3268                    .name("x".to_string())
3269                    .transform(Transform::Identity)
3270                    .build()
3271                    .unwrap(),
3272            )
3273            .unwrap()
3274            .build()
3275            .unwrap();
3276
3277        let default_partition_type = partition_spec.partition_type(&schema).unwrap();
3278        let expected = TableMetadata {
3279            format_version: FormatVersion::V1,
3280            table_uuid: Uuid::parse_str("d20125c8-7284-442c-9aea-15fee620737c").unwrap(),
3281            location: "s3://bucket/test/location".to_string(),
3282            last_updated_ms: 1602638573874,
3283            last_column_id: 3,
3284            schemas: HashMap::from_iter(vec![(0, Arc::new(schema))]),
3285            current_schema_id: 0,
3286            partition_specs: HashMap::from_iter(vec![(0, partition_spec.clone().into())]),
3287            default_spec: Arc::new(partition_spec),
3288            default_partition_type,
3289            last_partition_id: 0,
3290            default_sort_order_id: 0,
3291            // Sort order is added during deserialization for V2 compatibility
3292            sort_orders: HashMap::from_iter(vec![(0, SortOrder::unsorted_order().into())]),
3293            snapshots: HashMap::new(),
3294            current_snapshot_id: None,
3295            last_sequence_number: 0,
3296            properties: HashMap::new(),
3297            snapshot_log: vec![],
3298            metadata_log: Vec::new(),
3299            refs: HashMap::new(),
3300            statistics: HashMap::new(),
3301            partition_statistics: HashMap::new(),
3302            encryption_keys: HashMap::new(),
3303            next_row_id: INITIAL_ROW_ID,
3304        };
3305
3306        check_table_metadata_serde(&metadata, expected);
3307    }
3308
3309    #[test]
3310    fn test_table_metadata_v1_legacy_partition_spec_with_void_fields() {
3311        let mut metadata: serde_json::Value = serde_json::from_str(
3312            &fs::read_to_string("testdata/table_metadata/TableMetadataV1Valid.json").unwrap(),
3313        )
3314        .unwrap();
3315        // What a format version 1 spec keeps after identity(x) was removed twice.
3316        metadata["partition-spec"] = serde_json::json!([
3317            {"name": "x", "transform": "void", "source-id": 1, "field-id": 1000},
3318            {"name": "x_null", "transform": "void", "source-id": 1, "field-id": 1001},
3319        ]);
3320
3321        let metadata: TableMetadata = serde_json::from_value(metadata).unwrap();
3322
3323        let fields = metadata.default_partition_spec().fields();
3324        assert_eq!(
3325            fields
3326                .iter()
3327                .map(|f| (f.field_id, f.name.as_str(), f.transform))
3328                .collect::<Vec<_>>(),
3329            vec![
3330                (1000, "x", Transform::Void),
3331                (1001, "x_null", Transform::Void)
3332            ]
3333        );
3334    }
3335
3336    #[test]
3337    fn test_empty_snapshot_id_is_normalized_to_none() {
3338        let metadata =
3339            fs::read_to_string("testdata/table_metadata/TableMetadataV1Valid.json").unwrap();
3340        let deserialized: TableMetadata = serde_json::from_str(&metadata).unwrap();
3341        assert_eq!(
3342            deserialized.current_snapshot_id(),
3343            None,
3344            "current_snapshot_id of -1 should be deserialized as None"
3345        );
3346    }
3347
3348    #[test]
3349    fn test_table_metadata_v1_compat() {
3350        let metadata =
3351            fs::read_to_string("testdata/table_metadata/TableMetadataV1Compat.json").unwrap();
3352
3353        // Deserialize the JSON to verify it works
3354        let desered_type: TableMetadata = serde_json::from_str(&metadata)
3355            .expect("Failed to deserialize TableMetadataV1Compat.json");
3356
3357        // Verify some key fields match
3358        assert_eq!(desered_type.format_version(), FormatVersion::V1);
3359        assert_eq!(
3360            desered_type.uuid(),
3361            Uuid::parse_str("3276010d-7b1d-488c-98d8-9025fc4fde6b").unwrap()
3362        );
3363        assert_eq!(
3364            desered_type.location(),
3365            "s3://bucket/warehouse/iceberg/glue.db/table_name"
3366        );
3367        assert_eq!(desered_type.last_updated_ms(), 1727773114005);
3368        assert_eq!(desered_type.current_schema_id(), 0);
3369    }
3370
3371    fn v1_main_ref() -> (String, SnapshotReference) {
3372        let retention = SnapshotRetention::branch(None, None, None);
3373        (
3374            "main".to_string(),
3375            SnapshotReference::new(6278926806555018392, retention),
3376        )
3377    }
3378
3379    #[test]
3380    fn test_table_metadata_v1_refs_round_trip() {
3381        // Written by Spark 3.5.6 with Iceberg 1.9.2, paths rewritten.
3382        let json =
3383            fs::read_to_string("testdata/table_metadata/TableMetadataV1WithRefs.json").unwrap();
3384        let metadata: TableMetadata = serde_json::from_str(&json).unwrap();
3385
3386        assert_eq!(metadata.format_version(), FormatVersion::V1);
3387        assert_eq!(
3388            metadata.refs,
3389            HashMap::from_iter([
3390                v1_main_ref(),
3391                (
3392                    "t1".to_string(),
3393                    SnapshotReference::new(7825560978372493301, SnapshotRetention::Tag {
3394                        max_ref_age_ms: Some(604800000),
3395                    }),
3396                ),
3397                (
3398                    "b1".to_string(),
3399                    SnapshotReference::new(
3400                        7825560978372493301,
3401                        SnapshotRetention::branch(Some(2), None, None),
3402                    ),
3403                ),
3404            ])
3405        );
3406
3407        let written = serde_json::to_value(&metadata).unwrap();
3408        let java_written = serde_json::from_str::<serde_json::Value>(&json).unwrap();
3409        assert_eq!(written["format-version"], 1);
3410        assert_eq!(written["refs"], java_written["refs"]);
3411        assert_eq!(
3412            serde_json::from_value::<TableMetadata>(written).unwrap(),
3413            metadata
3414        );
3415    }
3416
3417    #[test]
3418    fn test_table_metadata_v1_without_refs_has_main_branch() {
3419        let json =
3420            fs::read_to_string("testdata/table_metadata/TableMetadataV1WithRefs.json").unwrap();
3421        let mut value = serde_json::from_str::<serde_json::Value>(&json).unwrap();
3422        value.as_object_mut().unwrap().remove("refs").unwrap();
3423        let metadata: TableMetadata = serde_json::from_value(value).unwrap();
3424
3425        assert_eq!(metadata.refs, HashMap::from_iter([v1_main_ref()]));
3426        assert_eq!(
3427            serde_json::to_value(&metadata).unwrap()["refs"],
3428            serde_json::json!({"main": {"snapshot-id": 6278926806555018392_i64, "type": "branch"}})
3429        );
3430    }
3431
3432    #[test]
3433    fn test_table_metadata_v1_invalid_refs_fail_to_load() {
3434        let json =
3435            fs::read_to_string("testdata/table_metadata/TableMetadataV1WithRefs.json").unwrap();
3436        let load_error = |edit: fn(&mut serde_json::Value)| {
3437            let mut value = serde_json::from_str::<serde_json::Value>(&json).unwrap();
3438            edit(&mut value);
3439            serde_json::from_value::<TableMetadata>(value)
3440                .unwrap_err()
3441                .to_string()
3442        };
3443
3444        assert_eq!(
3445            load_error(|value| value["refs"]["t1"]["snapshot-id"] = 1.into()),
3446            "DataInvalid => Snapshot for reference t1 does not exist in the existing snapshots list"
3447        );
3448        assert_eq!(
3449            load_error(
3450                |value| value["refs"]["main"]["snapshot-id"] = 7825560978372493301_i64.into()
3451            ),
3452            "DataInvalid => Current snapshot id does not match main branch (6278926806555018392 != 7825560978372493301)"
3453        );
3454        assert_eq!(
3455            load_error(|value| value["current-snapshot-id"] = (-1).into()),
3456            "DataInvalid => Current snapshot is not set, but main branch exists"
3457        );
3458    }
3459
3460    #[test]
3461    fn test_table_metadata_v1_schemas_without_current_id() {
3462        let metadata = fs::read_to_string(
3463            "testdata/table_metadata/TableMetadataV1SchemasWithoutCurrentId.json",
3464        )
3465        .unwrap();
3466
3467        // Deserialize the JSON - this should succeed by using the 'schema' field instead of 'schemas'
3468        let desered_type: TableMetadata = serde_json::from_str(&metadata)
3469            .expect("Failed to deserialize TableMetadataV1SchemasWithoutCurrentId.json");
3470
3471        // Verify it used the 'schema' field
3472        assert_eq!(desered_type.format_version(), FormatVersion::V1);
3473        assert_eq!(
3474            desered_type.uuid(),
3475            Uuid::parse_str("d20125c8-7284-442c-9aea-15fee620737c").unwrap()
3476        );
3477
3478        // Get the schema and verify it has the expected fields
3479        let schema = desered_type.current_schema();
3480        assert_eq!(schema.as_struct().fields().len(), 3);
3481        assert_eq!(schema.as_struct().fields()[0].name, "x");
3482        assert_eq!(schema.as_struct().fields()[1].name, "y");
3483        assert_eq!(schema.as_struct().fields()[2].name, "z");
3484    }
3485
3486    #[test]
3487    fn test_table_metadata_v1_no_valid_schema() {
3488        let metadata =
3489            fs::read_to_string("testdata/table_metadata/TableMetadataV1NoValidSchema.json")
3490                .unwrap();
3491
3492        // Deserialize the JSON - this should fail because neither schemas + current_schema_id nor schema is valid
3493        let desered: Result<TableMetadata, serde_json::Error> = serde_json::from_str(&metadata);
3494
3495        assert!(desered.is_err());
3496        let error_message = desered.unwrap_err().to_string();
3497        assert!(
3498            error_message.contains("No valid schema configuration found"),
3499            "Expected error about no valid schema configuration, got: {error_message}"
3500        );
3501    }
3502
3503    #[test]
3504    fn test_table_metadata_v1_partition_specs_without_default_id() {
3505        let metadata = fs::read_to_string(
3506            "testdata/table_metadata/TableMetadataV1PartitionSpecsWithoutDefaultId.json",
3507        )
3508        .unwrap();
3509
3510        // Deserialize the JSON - this should succeed by inferring default_spec_id as the max spec ID
3511        let desered_type: TableMetadata = serde_json::from_str(&metadata)
3512            .expect("Failed to deserialize TableMetadataV1PartitionSpecsWithoutDefaultId.json");
3513
3514        // Verify basic metadata
3515        assert_eq!(desered_type.format_version(), FormatVersion::V1);
3516        assert_eq!(
3517            desered_type.uuid(),
3518            Uuid::parse_str("d20125c8-7284-442c-9aea-15fee620737c").unwrap()
3519        );
3520
3521        // Verify partition specs
3522        assert_eq!(desered_type.default_partition_spec_id(), 2); // Should pick the largest spec ID (2)
3523        assert_eq!(desered_type.partition_specs.len(), 2);
3524
3525        // Verify the default spec has the expected fields
3526        let default_spec = &desered_type.default_spec;
3527        assert_eq!(default_spec.spec_id(), 2);
3528        assert_eq!(default_spec.fields().len(), 1);
3529        assert_eq!(default_spec.fields()[0].name, "y");
3530        assert_eq!(default_spec.fields()[0].transform, Transform::Identity);
3531        assert_eq!(default_spec.fields()[0].source_id, 2);
3532    }
3533
3534    #[test]
3535    fn test_table_metadata_v2_schema_not_found() {
3536        let metadata =
3537            fs::read_to_string("testdata/table_metadata/TableMetadataV2CurrentSchemaNotFound.json")
3538                .unwrap();
3539
3540        let desered: Result<TableMetadata, serde_json::Error> = serde_json::from_str(&metadata);
3541
3542        assert_eq!(
3543            desered.unwrap_err().to_string(),
3544            "DataInvalid => No schema exists with the current schema id 2."
3545        )
3546    }
3547
3548    #[test]
3549    fn test_table_metadata_v2_missing_sort_order() {
3550        let metadata =
3551            fs::read_to_string("testdata/table_metadata/TableMetadataV2MissingSortOrder.json")
3552                .unwrap();
3553
3554        let desered: Result<TableMetadata, serde_json::Error> = serde_json::from_str(&metadata);
3555
3556        assert_eq!(
3557            desered.unwrap_err().to_string(),
3558            "data did not match any variant of untagged enum TableMetadataEnum"
3559        )
3560    }
3561
3562    #[test]
3563    fn test_table_metadata_v2_missing_partition_specs() {
3564        let metadata =
3565            fs::read_to_string("testdata/table_metadata/TableMetadataV2MissingPartitionSpecs.json")
3566                .unwrap();
3567
3568        let desered: Result<TableMetadata, serde_json::Error> = serde_json::from_str(&metadata);
3569
3570        assert_eq!(
3571            desered.unwrap_err().to_string(),
3572            "data did not match any variant of untagged enum TableMetadataEnum"
3573        )
3574    }
3575
3576    #[test]
3577    fn test_table_metadata_v2_missing_last_partition_id() {
3578        let metadata = fs::read_to_string(
3579            "testdata/table_metadata/TableMetadataV2MissingLastPartitionId.json",
3580        )
3581        .unwrap();
3582
3583        let desered: Result<TableMetadata, serde_json::Error> = serde_json::from_str(&metadata);
3584
3585        assert_eq!(
3586            desered.unwrap_err().to_string(),
3587            "data did not match any variant of untagged enum TableMetadataEnum"
3588        )
3589    }
3590
3591    #[test]
3592    fn test_table_metadata_v2_missing_schemas() {
3593        let metadata =
3594            fs::read_to_string("testdata/table_metadata/TableMetadataV2MissingSchemas.json")
3595                .unwrap();
3596
3597        let desered: Result<TableMetadata, serde_json::Error> = serde_json::from_str(&metadata);
3598
3599        assert_eq!(
3600            desered.unwrap_err().to_string(),
3601            "data did not match any variant of untagged enum TableMetadataEnum"
3602        )
3603    }
3604
3605    #[test]
3606    fn test_table_metadata_v2_unsupported_version() {
3607        let metadata =
3608            fs::read_to_string("testdata/table_metadata/TableMetadataUnsupportedVersion.json")
3609                .unwrap();
3610
3611        let desered: Result<TableMetadata, serde_json::Error> = serde_json::from_str(&metadata);
3612
3613        assert_eq!(
3614            desered.unwrap_err().to_string(),
3615            "data did not match any variant of untagged enum TableMetadataEnum"
3616        )
3617    }
3618
3619    #[test]
3620    fn test_order_of_format_version() {
3621        assert!(FormatVersion::V1 < FormatVersion::V2);
3622        assert_eq!(FormatVersion::V1, FormatVersion::V1);
3623        assert_eq!(FormatVersion::V2, FormatVersion::V2);
3624    }
3625
3626    #[test]
3627    fn test_deserialize_default_partition_spec_with_dropped_source() {
3628        for version in [1, 2, 3] {
3629            for transform in ["identity", "truncate[4]", "bucket[16]"] {
3630                let mut metadata = serde_json::json!({
3631                    "format-version": version,
3632                    "table-uuid": "9c12d441-03fe-4693-9a96-a0705ddf69c1",
3633                    "location": "s3://bucket/table",
3634                    "last-sequence-number": 0,
3635                    "last-updated-ms": 1602638573590_i64,
3636                    "last-column-id": 2,
3637                    "current-schema-id": 1,
3638                    "schemas": [
3639                        {
3640                            "schema-id": 0,
3641                            "type": "struct",
3642                            "fields": [
3643                                {"id": 1, "name": "x", "required": false, "type": "long"},
3644                                {"id": 2, "name": "y", "required": false, "type": "long"}
3645                            ]
3646                        },
3647                        {
3648                            "schema-id": 1,
3649                            "type": "struct",
3650                            "fields": [
3651                                {"id": 2, "name": "y", "required": false, "type": "long"}
3652                            ]
3653                        }
3654                    ],
3655                    "default-spec-id": 0,
3656                    "partition-specs": [{"spec-id": 0, "fields": [{
3657                        "source-id": 1, "field-id": 1000, "name": "x_partition",
3658                        "transform": transform
3659                    }]}],
3660                    "last-partition-id": 1000,
3661                    "default-sort-order-id": 0,
3662                    "sort-orders": [{"order-id": 0, "fields": []}],
3663                    "next-row-id": 0
3664                });
3665
3666                // Retaining the source in history does not make it writable using
3667                // the current schema and default spec.
3668                let err = serde_json::from_value::<TableMetadata>(metadata.clone()).unwrap_err();
3669                assert!(err.to_string().contains(
3670                    "Default partition spec 0 references missing source field 1 in current schema 1"
3671                ));
3672
3673                // The same spec is allowed once it becomes historical.
3674                metadata["partition-specs"]
3675                    .as_array_mut()
3676                    .unwrap()
3677                    .push(serde_json::json!({"spec-id": 1, "fields": []}));
3678                metadata["default-spec-id"] = serde_json::json!(1);
3679                serde_json::from_value::<TableMetadata>(metadata.clone()).unwrap();
3680
3681                // A voided field does not require its dropped source, even in the default spec.
3682                metadata["default-spec-id"] = serde_json::json!(0);
3683                metadata["partition-specs"][0]["fields"][0]["transform"] =
3684                    serde_json::json!("void");
3685                serde_json::from_value::<TableMetadata>(metadata).unwrap();
3686            }
3687        }
3688    }
3689
3690    #[test]
3691    fn test_default_partition_spec() {
3692        let default_spec_id = 1234;
3693        let mut table_meta_data = get_test_table_metadata("TableMetadataV2Valid.json");
3694        let partition_spec = PartitionSpec::unpartition_spec();
3695        table_meta_data.default_spec = partition_spec.clone().into();
3696        table_meta_data
3697            .partition_specs
3698            .insert(default_spec_id, Arc::new(partition_spec));
3699
3700        assert_eq!(
3701            (*table_meta_data.default_partition_spec().clone()).clone(),
3702            (*table_meta_data
3703                .partition_spec_by_id(default_spec_id)
3704                .unwrap()
3705                .clone())
3706            .clone()
3707        );
3708    }
3709    #[test]
3710    fn test_default_sort_order() {
3711        let default_sort_order_id = 1234;
3712        let mut table_meta_data = get_test_table_metadata("TableMetadataV2Valid.json");
3713        table_meta_data.default_sort_order_id = default_sort_order_id;
3714        table_meta_data
3715            .sort_orders
3716            .insert(default_sort_order_id, Arc::new(SortOrder::default()));
3717
3718        assert_eq!(
3719            table_meta_data.default_sort_order(),
3720            table_meta_data
3721                .sort_orders
3722                .get(&default_sort_order_id)
3723                .unwrap()
3724        )
3725    }
3726
3727    #[test]
3728    fn test_table_metadata_builder_from_table_creation() {
3729        let table_creation = TableCreation::builder()
3730            .location("s3://db/table".to_string())
3731            .name("table".to_string())
3732            .properties(HashMap::new())
3733            .schema(Schema::builder().build().unwrap())
3734            .build();
3735        let table_metadata = TableMetadataBuilder::from_table_creation(table_creation)
3736            .unwrap()
3737            .build()
3738            .unwrap()
3739            .metadata;
3740        assert_eq!(table_metadata.location, "s3://db/table");
3741        assert_eq!(table_metadata.schemas.len(), 1);
3742        assert_eq!(
3743            table_metadata
3744                .schemas
3745                .get(&0)
3746                .unwrap()
3747                .as_struct()
3748                .fields()
3749                .len(),
3750            0
3751        );
3752        assert_eq!(table_metadata.properties.len(), 0);
3753        assert_eq!(
3754            table_metadata.partition_specs,
3755            HashMap::from([(
3756                0,
3757                Arc::new(
3758                    PartitionSpec::builder(table_metadata.schemas.get(&0).unwrap().clone())
3759                        .with_spec_id(0)
3760                        .build()
3761                        .unwrap()
3762                )
3763            )])
3764        );
3765        assert_eq!(
3766            table_metadata.sort_orders,
3767            HashMap::from([(
3768                0,
3769                Arc::new(SortOrder {
3770                    order_id: 0,
3771                    fields: vec![]
3772                })
3773            )])
3774        );
3775    }
3776
3777    #[tokio::test]
3778    async fn test_table_metadata_read_write() {
3779        // Create a temporary directory for our test
3780        let temp_dir = TempDir::new().unwrap();
3781        let temp_path = temp_dir.path().to_str().unwrap();
3782
3783        // Create a FileIO instance
3784        let file_io = FileIO::new_with_fs();
3785
3786        // Use an existing test metadata from the test files
3787        let original_metadata: TableMetadata =
3788            get_test_table_metadata_at("TableMetadataV2Valid.json", temp_path);
3789
3790        // Define the metadata location
3791        let metadata_location =
3792            MetadataLocation::try_new_with_metadata(&original_metadata).unwrap();
3793        let metadata_location_str = metadata_location.to_string();
3794
3795        // Write the metadata
3796        original_metadata
3797            .write_to(&file_io, &metadata_location)
3798            .await
3799            .unwrap();
3800
3801        // Verify the file exists
3802        assert!(fs::metadata(&metadata_location_str).is_ok());
3803
3804        // Read the metadata back
3805        let read_metadata = TableMetadata::read_from(&file_io, &metadata_location_str)
3806            .await
3807            .unwrap();
3808
3809        // Verify the metadata matches
3810        assert_eq!(read_metadata, original_metadata);
3811    }
3812
3813    #[tokio::test]
3814    async fn test_table_metadata_read_compressed() {
3815        let temp_dir = TempDir::new().unwrap();
3816        let metadata_location = temp_dir.path().join("v1.gz.metadata.json");
3817
3818        let original_metadata: TableMetadata = get_test_table_metadata("TableMetadataV2Valid.json");
3819        let json = serde_json::to_string(&original_metadata).unwrap();
3820
3821        let compressed = CompressionCodec::gzip_default()
3822            .compress(json.into_bytes())
3823            .expect("failed to compress metadata");
3824        fs::write(&metadata_location, &compressed).expect("failed to write metadata");
3825
3826        // Read the metadata back
3827        let file_io = FileIO::new_with_fs();
3828        let metadata_location = metadata_location.to_str().unwrap();
3829        let read_metadata = TableMetadata::read_from(&file_io, metadata_location)
3830            .await
3831            .unwrap();
3832
3833        // Verify the metadata matches
3834        assert_eq!(read_metadata, original_metadata);
3835    }
3836
3837    #[tokio::test]
3838    async fn test_table_metadata_read_nonexistent_file() {
3839        // Create a FileIO instance
3840        let file_io = FileIO::new_with_fs();
3841
3842        // Try to read a non-existent file
3843        let result = TableMetadata::read_from(&file_io, "/nonexistent/path/metadata.json").await;
3844
3845        // Verify it returns an error
3846        assert!(result.is_err());
3847    }
3848
3849    #[tokio::test]
3850    async fn test_table_metadata_write_with_gzip_compression() {
3851        let temp_dir = TempDir::new().unwrap();
3852        let temp_path = temp_dir.path().to_str().unwrap();
3853        let file_io = FileIO::new_with_fs();
3854
3855        // Get a test metadata and add gzip compression property
3856        let original_metadata: TableMetadata =
3857            get_test_table_metadata_at("TableMetadataV2Valid.json", temp_path);
3858
3859        // Modify properties to enable gzip compression (using mixed case to test case-insensitive matching)
3860        let mut props = original_metadata.properties.clone();
3861        props.insert(
3862            TableProperties::PROPERTY_METADATA_COMPRESSION_CODEC.to_string(),
3863            "GziP".to_string(),
3864        );
3865        // Use builder to create new metadata with updated properties
3866        let compressed_metadata =
3867            TableMetadataBuilder::new_from_metadata(original_metadata.clone(), None)
3868                .assign_uuid(original_metadata.table_uuid)
3869                .set_properties(props.clone())
3870                .unwrap()
3871                .build()
3872                .unwrap()
3873                .metadata;
3874
3875        // Create MetadataLocation with compression codec from metadata
3876        let metadata_location =
3877            MetadataLocation::try_new_with_metadata(&compressed_metadata).unwrap();
3878        let metadata_location_str = metadata_location.to_string();
3879
3880        // Verify the location has the .gz extension
3881        assert!(metadata_location_str.contains(".gz.metadata.json"));
3882
3883        // Write the metadata with compression
3884        compressed_metadata
3885            .write_to(&file_io, &metadata_location)
3886            .await
3887            .unwrap();
3888
3889        // Verify the compressed file exists
3890        assert!(std::path::Path::new(&metadata_location_str).exists());
3891
3892        // Read the raw file and check it's gzip compressed
3893        let raw_content = fs::read(&metadata_location_str).unwrap();
3894        assert!(raw_content.len() > 2);
3895        assert_eq!(raw_content[0], 0x1F); // gzip magic number
3896        assert_eq!(raw_content[1], 0x8B); // gzip magic number
3897
3898        // Read the metadata back using the compressed location
3899        let read_metadata = TableMetadata::read_from(&file_io, &metadata_location_str)
3900            .await
3901            .unwrap();
3902
3903        // Verify the complete round-trip: read metadata should match what we wrote
3904        assert_eq!(read_metadata, compressed_metadata);
3905    }
3906
3907    #[test]
3908    fn test_name_exists_in_any_schema() {
3909        // Create multiple schemas with different fields
3910        let schema1 = Schema::builder()
3911            .with_schema_id(1)
3912            .with_fields(vec![
3913                NestedField::required(1, "field1", Type::Primitive(PrimitiveType::String)).into(),
3914                NestedField::required(2, "field2", Type::Primitive(PrimitiveType::Int)).into(),
3915            ])
3916            .build()
3917            .unwrap();
3918
3919        let schema2 = Schema::builder()
3920            .with_schema_id(2)
3921            .with_fields(vec![
3922                NestedField::required(1, "field1", Type::Primitive(PrimitiveType::String)).into(),
3923                NestedField::required(3, "field3", Type::Primitive(PrimitiveType::Long)).into(),
3924            ])
3925            .build()
3926            .unwrap();
3927
3928        let metadata = TableMetadataBuilder::new(
3929            schema1,
3930            PartitionSpec::unpartition_spec().into_unbound(),
3931            SortOrder::unsorted_order(),
3932            "s3://test/location".to_string(),
3933            FormatVersion::V2,
3934            HashMap::new(),
3935        )
3936        .unwrap()
3937        .add_current_schema(schema2)
3938        .unwrap()
3939        .build()
3940        .unwrap()
3941        .metadata;
3942
3943        assert!(metadata.name_exists_in_any_schema("field1")); // exists in both schemas
3944        assert!(metadata.name_exists_in_any_schema("field2")); // exists only in schema1 (historical)
3945        assert!(metadata.name_exists_in_any_schema("field3")); // exists only in schema2 (current)
3946
3947        assert!(!metadata.name_exists_in_any_schema("nonexistent_field"));
3948        assert!(!metadata.name_exists_in_any_schema("field4"));
3949        assert!(!metadata.name_exists_in_any_schema(""));
3950    }
3951
3952    #[test]
3953    fn test_name_exists_in_any_schema_empty_schemas() {
3954        let schema = Schema::builder().with_fields(vec![]).build().unwrap();
3955
3956        let metadata = TableMetadataBuilder::new(
3957            schema,
3958            PartitionSpec::unpartition_spec().into_unbound(),
3959            SortOrder::unsorted_order(),
3960            "s3://test/location".to_string(),
3961            FormatVersion::V2,
3962            HashMap::new(),
3963        )
3964        .unwrap()
3965        .build()
3966        .unwrap()
3967        .metadata;
3968
3969        assert!(!metadata.name_exists_in_any_schema("any_field"));
3970    }
3971
3972    #[test]
3973    fn test_helper_methods_multi_version_scenario() {
3974        // Test a realistic multi-version scenario
3975        let initial_schema = Schema::builder()
3976            .with_fields(vec![
3977                NestedField::required(1, "id", Type::Primitive(PrimitiveType::Long)).into(),
3978                NestedField::required(2, "name", Type::Primitive(PrimitiveType::String)).into(),
3979                NestedField::required(
3980                    3,
3981                    "deprecated_field",
3982                    Type::Primitive(PrimitiveType::String),
3983                )
3984                .into(),
3985            ])
3986            .build()
3987            .unwrap();
3988
3989        let metadata = TableMetadataBuilder::new(
3990            initial_schema,
3991            PartitionSpec::unpartition_spec().into_unbound(),
3992            SortOrder::unsorted_order(),
3993            "s3://test/location".to_string(),
3994            FormatVersion::V2,
3995            HashMap::new(),
3996        )
3997        .unwrap();
3998
3999        let evolved_schema = Schema::builder()
4000            .with_fields(vec![
4001                NestedField::required(1, "id", Type::Primitive(PrimitiveType::Long)).into(),
4002                NestedField::required(2, "name", Type::Primitive(PrimitiveType::String)).into(),
4003                NestedField::required(
4004                    3,
4005                    "deprecated_field",
4006                    Type::Primitive(PrimitiveType::String),
4007                )
4008                .into(),
4009                NestedField::required(4, "new_field", Type::Primitive(PrimitiveType::Double))
4010                    .into(),
4011            ])
4012            .build()
4013            .unwrap();
4014
4015        // Then add a third schema that removes the deprecated field
4016        let _final_schema = Schema::builder()
4017            .with_fields(vec![
4018                NestedField::required(1, "id", Type::Primitive(PrimitiveType::Long)).into(),
4019                NestedField::required(2, "name", Type::Primitive(PrimitiveType::String)).into(),
4020                NestedField::required(4, "new_field", Type::Primitive(PrimitiveType::Double))
4021                    .into(),
4022                NestedField::required(5, "latest_field", Type::Primitive(PrimitiveType::Boolean))
4023                    .into(),
4024            ])
4025            .build()
4026            .unwrap();
4027
4028        let final_metadata = metadata
4029            .add_current_schema(evolved_schema)
4030            .unwrap()
4031            .build()
4032            .unwrap()
4033            .metadata;
4034
4035        assert!(final_metadata.name_exists_in_any_schema("id")); // exists in both schemas
4036        assert!(final_metadata.name_exists_in_any_schema("name")); // exists in both schemas
4037        assert!(final_metadata.name_exists_in_any_schema("deprecated_field")); // exists in both schemas
4038        assert!(final_metadata.name_exists_in_any_schema("new_field")); // only in current schema
4039        assert!(!final_metadata.name_exists_in_any_schema("never_existed"));
4040    }
4041
4042    #[test]
4043    fn test_invalid_sort_order_id_zero_with_fields() {
4044        let metadata = r#"
4045        {
4046            "format-version": 2,
4047            "table-uuid": "9c12d441-03fe-4693-9a96-a0705ddf69c1",
4048            "location": "s3://bucket/test/location",
4049            "last-sequence-number": 111,
4050            "last-updated-ms": 1600000000000,
4051            "last-column-id": 3,
4052            "current-schema-id": 1,
4053            "schemas": [
4054                {
4055                    "type": "struct",
4056                    "schema-id": 1,
4057                    "fields": [
4058                        {"id": 1, "name": "x", "required": true, "type": "long"},
4059                        {"id": 2, "name": "y", "required": true, "type": "long"}
4060                    ]
4061                }
4062            ],
4063            "default-spec-id": 0,
4064            "partition-specs": [{"spec-id": 0, "fields": []}],
4065            "last-partition-id": 999,
4066            "default-sort-order-id": 0,
4067            "sort-orders": [
4068                {
4069                    "order-id": 0,
4070                    "fields": [
4071                        {
4072                            "transform": "identity",
4073                            "source-id": 1,
4074                            "direction": "asc",
4075                            "null-order": "nulls-first"
4076                        }
4077                    ]
4078                }
4079            ],
4080            "properties": {},
4081            "current-snapshot-id": -1,
4082            "snapshots": []
4083        }
4084        "#;
4085
4086        let result: Result<TableMetadata, serde_json::Error> = serde_json::from_str(metadata);
4087
4088        // Should fail because sort order ID 0 is reserved for unsorted order and cannot have fields
4089        assert!(
4090            result.is_err(),
4091            "Parsing should fail for sort order ID 0 with fields"
4092        );
4093    }
4094
4095    #[test]
4096    fn test_table_properties_with_defaults() {
4097        use crate::spec::TableProperties;
4098
4099        let schema = Schema::builder()
4100            .with_fields(vec![
4101                NestedField::required(1, "id", Type::Primitive(PrimitiveType::Long)).into(),
4102            ])
4103            .build()
4104            .unwrap();
4105
4106        let metadata = TableMetadataBuilder::new(
4107            schema,
4108            PartitionSpec::unpartition_spec().into_unbound(),
4109            SortOrder::unsorted_order(),
4110            "s3://test/location".to_string(),
4111            FormatVersion::V2,
4112            HashMap::new(),
4113        )
4114        .unwrap()
4115        .build()
4116        .unwrap()
4117        .metadata;
4118
4119        let props = metadata.table_properties();
4120
4121        assert_eq!(
4122            props.commit_num_retries().unwrap(),
4123            TableProperties::PROPERTY_COMMIT_NUM_RETRIES_DEFAULT
4124        );
4125        assert_eq!(
4126            props.write_target_file_size_bytes().unwrap(),
4127            TableProperties::PROPERTY_WRITE_TARGET_FILE_SIZE_BYTES_DEFAULT
4128        );
4129    }
4130
4131    #[test]
4132    fn test_table_properties_with_custom_values() {
4133        use crate::spec::TableProperties;
4134
4135        let schema = Schema::builder()
4136            .with_fields(vec![
4137                NestedField::required(1, "id", Type::Primitive(PrimitiveType::Long)).into(),
4138            ])
4139            .build()
4140            .unwrap();
4141
4142        let properties = HashMap::from([
4143            (
4144                TableProperties::PROPERTY_COMMIT_NUM_RETRIES.to_string(),
4145                "10".to_string(),
4146            ),
4147            (
4148                TableProperties::PROPERTY_WRITE_TARGET_FILE_SIZE_BYTES.to_string(),
4149                "1024".to_string(),
4150            ),
4151        ]);
4152
4153        let metadata = TableMetadataBuilder::new(
4154            schema,
4155            PartitionSpec::unpartition_spec().into_unbound(),
4156            SortOrder::unsorted_order(),
4157            "s3://test/location".to_string(),
4158            FormatVersion::V2,
4159            properties,
4160        )
4161        .unwrap()
4162        .build()
4163        .unwrap()
4164        .metadata;
4165
4166        let props = metadata.table_properties();
4167
4168        assert_eq!(props.commit_num_retries().unwrap(), 10);
4169        assert_eq!(props.write_target_file_size_bytes().unwrap(), 1024);
4170    }
4171
4172    #[test]
4173    fn test_deserialize_metadata_defers_invalid_table_property_errors() {
4174        let invalid_retries = "not_a_number";
4175        let invalid_codec = "unknown";
4176        let target_file_size = "1024";
4177
4178        for file_name in [
4179            "TableMetadataV1Valid.json",
4180            "TableMetadataV2ValidMinimal.json",
4181            "TableMetadataV3ValidMinimal.json",
4182        ] {
4183            let path = format!("testdata/table_metadata/{file_name}");
4184            let mut json: serde_json::Value =
4185                serde_json::from_str(&fs::read_to_string(path).unwrap()).unwrap();
4186            json["properties"] = serde_json::json!({
4187                (TableProperties::PROPERTY_COMMIT_NUM_RETRIES): invalid_retries,
4188                (TableProperties::PROPERTY_METADATA_COMPRESSION_CODEC): invalid_codec,
4189                (TableProperties::PROPERTY_WRITE_TARGET_FILE_SIZE_BYTES): target_file_size,
4190            });
4191
4192            let metadata: TableMetadata = serde_json::from_value(json).unwrap();
4193            assert_eq!(
4194                metadata
4195                    .properties()
4196                    .get(TableProperties::PROPERTY_COMMIT_NUM_RETRIES)
4197                    .map(String::as_str),
4198                Some(invalid_retries)
4199            );
4200
4201            let table_properties = metadata.table_properties();
4202            let error = table_properties.commit_num_retries().unwrap_err();
4203            assert!(
4204                error
4205                    .message()
4206                    .contains(TableProperties::PROPERTY_COMMIT_NUM_RETRIES)
4207            );
4208            assert_eq!(
4209                table_properties.write_target_file_size_bytes().unwrap(),
4210                1024
4211            );
4212            let error = table_properties.metadata_compression_codec().unwrap_err();
4213            assert!(
4214                format!("{error}").contains(TableProperties::PROPERTY_METADATA_COMPRESSION_CODEC)
4215            );
4216
4217            let serialized = serde_json::to_value(metadata).unwrap();
4218            assert_eq!(
4219                serialized["properties"][TableProperties::PROPERTY_COMMIT_NUM_RETRIES],
4220                invalid_retries
4221            );
4222            assert_eq!(
4223                serialized["properties"][TableProperties::PROPERTY_METADATA_COMPRESSION_CODEC],
4224                invalid_codec
4225            );
4226        }
4227    }
4228
4229    #[test]
4230    fn test_table_properties_with_invalid_value() {
4231        let schema = Schema::builder()
4232            .with_fields(vec![
4233                NestedField::required(1, "id", Type::Primitive(PrimitiveType::Long)).into(),
4234            ])
4235            .build()
4236            .unwrap();
4237
4238        let properties = HashMap::from([
4239            (
4240                TableProperties::PROPERTY_COMMIT_NUM_RETRIES.to_string(),
4241                "not_a_number".to_string(),
4242            ),
4243            (
4244                TableProperties::PROPERTY_WRITE_TARGET_FILE_SIZE_BYTES.to_string(),
4245                "1024".to_string(),
4246            ),
4247        ]);
4248
4249        let metadata = TableMetadataBuilder::new(
4250            schema,
4251            PartitionSpec::unpartition_spec().into_unbound(),
4252            SortOrder::unsorted_order(),
4253            "s3://test/location".to_string(),
4254            FormatVersion::V2,
4255            properties,
4256        )
4257        .unwrap()
4258        .build()
4259        .unwrap()
4260        .metadata;
4261
4262        let table_properties = metadata.table_properties();
4263        let err = table_properties.commit_num_retries().unwrap_err();
4264        assert_eq!(err.kind(), ErrorKind::DataInvalid);
4265        assert!(
4266            err.message()
4267                .contains(TableProperties::PROPERTY_COMMIT_NUM_RETRIES)
4268        );
4269        assert_eq!(
4270            table_properties.write_target_file_size_bytes().unwrap(),
4271            1024
4272        );
4273    }
4274
4275    #[test]
4276    fn test_v2_to_v3_upgrade_preserves_existing_snapshots_without_row_lineage() {
4277        // Create a v2 table metadata
4278        let schema = Schema::builder()
4279            .with_fields(vec![
4280                NestedField::required(1, "id", Type::Primitive(PrimitiveType::Long)).into(),
4281            ])
4282            .build()
4283            .unwrap();
4284
4285        let v2_metadata = TableMetadataBuilder::new(
4286            schema,
4287            PartitionSpec::unpartition_spec().into_unbound(),
4288            SortOrder::unsorted_order(),
4289            "s3://bucket/test/location".to_string(),
4290            FormatVersion::V2,
4291            HashMap::new(),
4292        )
4293        .unwrap()
4294        .build()
4295        .unwrap()
4296        .metadata;
4297
4298        // Add a v2 snapshot
4299        let snapshot = Snapshot::builder()
4300            .with_snapshot_id(1)
4301            .with_timestamp_ms(v2_metadata.last_updated_ms + 1)
4302            .with_sequence_number(1)
4303            .with_schema_id(0)
4304            .with_manifest_list("s3://bucket/test/metadata/snap-1.avro")
4305            .with_summary(Summary {
4306                operation: Operation::Append,
4307                additional_properties: HashMap::from([(
4308                    "added-data-files".to_string(),
4309                    "1".to_string(),
4310                )]),
4311            })
4312            .build();
4313
4314        let v2_with_snapshot = v2_metadata
4315            .into_builder(Some("s3://bucket/test/metadata/v00001.json".to_string()))
4316            .add_snapshot(snapshot)
4317            .unwrap()
4318            .set_ref("main", SnapshotReference {
4319                snapshot_id: 1,
4320                retention: SnapshotRetention::Branch {
4321                    min_snapshots_to_keep: None,
4322                    max_snapshot_age_ms: None,
4323                    max_ref_age_ms: None,
4324                },
4325            })
4326            .unwrap()
4327            .build()
4328            .unwrap()
4329            .metadata;
4330
4331        // Verify v2 serialization works fine
4332        let v2_json = serde_json::to_string(&v2_with_snapshot);
4333        assert!(v2_json.is_ok(), "v2 serialization should work");
4334
4335        // Upgrade to v3
4336        let v3_metadata = v2_with_snapshot
4337            .into_builder(Some("s3://bucket/test/metadata/v00002.json".to_string()))
4338            .upgrade_format_version(FormatVersion::V3)
4339            .unwrap()
4340            .build()
4341            .unwrap()
4342            .metadata;
4343
4344        assert_eq!(v3_metadata.format_version, FormatVersion::V3);
4345        assert_eq!(v3_metadata.next_row_id, INITIAL_ROW_ID);
4346        assert_eq!(v3_metadata.snapshots.len(), 1);
4347
4348        // Verify the snapshot has no row_range
4349        let snapshot = v3_metadata.snapshots.values().next().unwrap();
4350        assert!(
4351            snapshot.row_range().is_none(),
4352            "Snapshot should have no row_range after upgrade"
4353        );
4354
4355        // Try to serialize v3 metadata - this should now work
4356        let v3_json = serde_json::to_string(&v3_metadata);
4357        assert!(
4358            v3_json.is_ok(),
4359            "v3 serialization should work for upgraded tables"
4360        );
4361
4362        // Verify we can deserialize it back
4363        let deserialized: TableMetadata = serde_json::from_str(&v3_json.unwrap()).unwrap();
4364        assert_eq!(deserialized.format_version, FormatVersion::V3);
4365        assert_eq!(deserialized.snapshots.len(), 1);
4366
4367        // Verify the deserialized snapshot still has no row_range
4368        let deserialized_snapshot = deserialized.snapshots.values().next().unwrap();
4369        assert!(
4370            deserialized_snapshot.row_range().is_none(),
4371            "Deserialized snapshot should have no row_range"
4372        );
4373    }
4374
4375    #[test]
4376    fn test_v3_snapshot_with_row_lineage_serialization() {
4377        // Create a v3 table metadata
4378        let schema = Schema::builder()
4379            .with_fields(vec![
4380                NestedField::required(1, "id", Type::Primitive(PrimitiveType::Long)).into(),
4381            ])
4382            .build()
4383            .unwrap();
4384
4385        let v3_metadata = TableMetadataBuilder::new(
4386            schema,
4387            PartitionSpec::unpartition_spec().into_unbound(),
4388            SortOrder::unsorted_order(),
4389            "s3://bucket/test/location".to_string(),
4390            FormatVersion::V3,
4391            HashMap::new(),
4392        )
4393        .unwrap()
4394        .build()
4395        .unwrap()
4396        .metadata;
4397
4398        // Add a v3 snapshot with row lineage
4399        let snapshot = Snapshot::builder()
4400            .with_snapshot_id(1)
4401            .with_timestamp_ms(v3_metadata.last_updated_ms + 1)
4402            .with_sequence_number(1)
4403            .with_schema_id(0)
4404            .with_manifest_list("s3://bucket/test/metadata/snap-1.avro")
4405            .with_summary(Summary {
4406                operation: Operation::Append,
4407                additional_properties: HashMap::from([(
4408                    "added-data-files".to_string(),
4409                    "1".to_string(),
4410                )]),
4411            })
4412            .with_row_range(100, 50) // first_row_id=100, added_rows=50
4413            .build();
4414
4415        let v3_with_snapshot = v3_metadata
4416            .into_builder(Some("s3://bucket/test/metadata/v00001.json".to_string()))
4417            .add_snapshot(snapshot)
4418            .unwrap()
4419            .set_ref("main", SnapshotReference {
4420                snapshot_id: 1,
4421                retention: SnapshotRetention::Branch {
4422                    min_snapshots_to_keep: None,
4423                    max_snapshot_age_ms: None,
4424                    max_ref_age_ms: None,
4425                },
4426            })
4427            .unwrap()
4428            .build()
4429            .unwrap()
4430            .metadata;
4431
4432        // Verify the snapshot has row_range
4433        let snapshot = v3_with_snapshot.snapshots.values().next().unwrap();
4434        assert!(
4435            snapshot.row_range().is_some(),
4436            "Snapshot should have row_range"
4437        );
4438        let (first_row_id, added_rows) = snapshot.row_range().unwrap();
4439        assert_eq!(first_row_id, 100);
4440        assert_eq!(added_rows, 50);
4441
4442        // Serialize v3 metadata - this should work
4443        let v3_json = serde_json::to_string(&v3_with_snapshot);
4444        assert!(
4445            v3_json.is_ok(),
4446            "v3 serialization should work for snapshots with row lineage"
4447        );
4448
4449        // Verify we can deserialize it back
4450        let deserialized: TableMetadata = serde_json::from_str(&v3_json.unwrap()).unwrap();
4451        assert_eq!(deserialized.format_version, FormatVersion::V3);
4452        assert_eq!(deserialized.snapshots.len(), 1);
4453
4454        // Verify the deserialized snapshot has the correct row_range
4455        let deserialized_snapshot = deserialized.snapshots.values().next().unwrap();
4456        assert!(
4457            deserialized_snapshot.row_range().is_some(),
4458            "Deserialized snapshot should have row_range"
4459        );
4460        let (deserialized_first_row_id, deserialized_added_rows) =
4461            deserialized_snapshot.row_range().unwrap();
4462        assert_eq!(deserialized_first_row_id, 100);
4463        assert_eq!(deserialized_added_rows, 50);
4464    }
4465
4466    #[test]
4467    fn test_metadata_location_default() {
4468        // Verify metadata files go to `<location>/metadata` when `write.metadata.path` is not set
4469        let metadata = get_test_table_metadata("TableMetadataV2Valid.json");
4470        assert_eq!(metadata.location(), "s3://bucket/test/location");
4471        assert_eq!(
4472            metadata.metadata_location().unwrap(),
4473            "s3://bucket/test/location/metadata"
4474        );
4475    }
4476
4477    #[test]
4478    fn test_metadata_location_honors_write_metadata_path() {
4479        let metadata = get_test_table_metadata("TableMetadataV2Valid.json")
4480            .into_builder(None)
4481            .set_properties(HashMap::from([(
4482                TableProperties::PROPERTY_WRITE_METADATA_PATH.to_string(),
4483                "s3://other-bucket/custom-meta".to_string(),
4484            )]))
4485            .unwrap()
4486            .build()
4487            .unwrap()
4488            .metadata;
4489        assert_eq!(
4490            metadata.metadata_location().unwrap(),
4491            "s3://other-bucket/custom-meta"
4492        );
4493    }
4494
4495    #[test]
4496    fn test_metadata_location_trims_trailing_slash() {
4497        // A configured path with a trailing slash must not yield a doubled separator
4498        let metadata = get_test_table_metadata("TableMetadataV2Valid.json")
4499            .into_builder(None)
4500            .set_properties(HashMap::from([(
4501                TableProperties::PROPERTY_WRITE_METADATA_PATH.to_string(),
4502                "s3://other-bucket/custom-meta/".to_string(),
4503            )]))
4504            .unwrap()
4505            .build()
4506            .unwrap()
4507            .metadata;
4508        assert_eq!(
4509            metadata.metadata_location().unwrap(),
4510            "s3://other-bucket/custom-meta"
4511        );
4512    }
4513
4514    #[test]
4515    fn test_unified_partition_type_spans_all_specs() {
4516        let schema = Schema::builder()
4517            .with_fields(vec![
4518                NestedField::required(1, "x", Type::Primitive(PrimitiveType::Long)).into(),
4519                NestedField::required(2, "y", Type::Primitive(PrimitiveType::Long)).into(),
4520                NestedField::required(3, "z", Type::Primitive(PrimitiveType::Long)).into(),
4521            ])
4522            .build()
4523            .unwrap();
4524
4525        let metadata = TableMetadataBuilder::new(
4526            schema.clone(),
4527            UnboundPartitionSpec::builder()
4528                .with_spec_id(0)
4529                .add_partition_field(
4530                    UnboundPartitionField::builder()
4531                        .source_ids(vec![2])
4532                        .name("y")
4533                        .transform(Transform::Identity)
4534                        .build()
4535                        .unwrap(),
4536                )
4537                .unwrap()
4538                .build(),
4539            SortOrder::unsorted_order(),
4540            "s3://bucket/table".to_string(),
4541            FormatVersion::V2,
4542            HashMap::new(),
4543        )
4544        .unwrap()
4545        .build()
4546        .unwrap()
4547        .metadata
4548        .into_builder(None)
4549        .add_partition_spec(
4550            UnboundPartitionSpec::builder()
4551                .add_partition_field(
4552                    UnboundPartitionField::builder()
4553                        .source_ids(vec![3])
4554                        .name("z")
4555                        .transform(Transform::Identity)
4556                        .build()
4557                        .unwrap(),
4558                )
4559                .unwrap()
4560                .build(),
4561        )
4562        .unwrap()
4563        .build()
4564        .unwrap()
4565        .metadata;
4566
4567        // The default spec only knows about `y`, but `_partition` must expose both.
4568        assert_eq!(metadata.default_partition_type().fields().len(), 1);
4569
4570        let unified = metadata.unified_partition_type(&schema).unwrap();
4571        let names: Vec<&str> = unified
4572            .fields()
4573            .iter()
4574            .map(|field| field.name.as_str())
4575            .collect();
4576        assert_eq!(names, vec!["y", "z"]);
4577    }
4578}