1use std::cmp::Ordering;
22use std::collections::HashMap;
23use std::fmt::{Display, Formatter};
24use std::hash::Hash;
25use std::sync::Arc;
26
27use _serde::TableMetadataEnum;
28use chrono::{DateTime, Utc};
29use serde::{Deserialize, Serialize};
30use serde_repr::{Deserialize_repr, Serialize_repr};
31use uuid::Uuid;
32
33use super::snapshot::SnapshotReference;
34pub use super::table_metadata_builder::{TableMetadataBuildResult, TableMetadataBuilder};
35use super::{
36 DEFAULT_PARTITION_SPEC_ID, PartitionSpecRef, PartitionStatisticsFile, Schema, SchemaId,
37 SchemaRef, SnapshotRef, SnapshotRetention, SortOrder, SortOrderRef, StatisticsFile, StructType,
38 TableProperties, Transform,
39};
40use crate::catalog::{METADATA_FOLDER_NAME, MetadataLocation};
41use crate::compression::CompressionCodec;
42use crate::error::{Result, invalid_data, timestamp_ms_to_utc};
43use crate::io::FileIO;
44use crate::partitioning::compute_unified_partition_type;
45use crate::spec::EncryptedKey;
46use crate::{Error, ErrorKind};
47
48static MAIN_BRANCH: &str = "main";
49pub(crate) static ONE_MINUTE_MS: i64 = 60_000;
50
51pub(crate) static EMPTY_SNAPSHOT_ID: i64 = -1;
55pub(crate) static INITIAL_SEQUENCE_NUMBER: i64 = 0;
56
57pub const INITIAL_ROW_ID: u64 = 0;
59pub const MIN_FORMAT_VERSION_ROW_LINEAGE: FormatVersion = FormatVersion::V3;
61pub type TableMetadataRef = Arc<TableMetadata>;
63
64#[derive(Debug, PartialEq, Deserialize, Eq, Clone)]
65#[serde(try_from = "TableMetadataEnum")]
66pub struct TableMetadata {
71 pub(crate) format_version: FormatVersion,
73 pub(crate) table_uuid: Uuid,
75 pub(crate) location: String,
77 pub(crate) last_sequence_number: i64,
79 pub(crate) last_updated_ms: i64,
81 pub(crate) last_column_id: i32,
83 pub(crate) schemas: HashMap<i32, SchemaRef>,
85 pub(crate) current_schema_id: i32,
87 pub(crate) partition_specs: HashMap<i32, PartitionSpecRef>,
89 pub(crate) default_spec: PartitionSpecRef,
91 pub(crate) default_partition_type: StructType,
93 pub(crate) last_partition_id: i32,
95 pub(crate) properties: HashMap<String, String>,
99 pub(crate) current_snapshot_id: Option<i64>,
102 pub(crate) snapshots: HashMap<i64, SnapshotRef>,
107 pub(crate) snapshot_log: Vec<SnapshotLog>,
114
115 pub(crate) metadata_log: Vec<MetadataLog>,
122
123 pub(crate) sort_orders: HashMap<i64, SortOrderRef>,
125 pub(crate) default_sort_order_id: i64,
129 pub(crate) refs: HashMap<String, SnapshotReference>,
134 pub(crate) statistics: HashMap<i64, StatisticsFile>,
136 pub(crate) partition_statistics: HashMap<i64, PartitionStatisticsFile>,
138 pub(crate) encryption_keys: HashMap<String, EncryptedKey>,
140 pub(crate) next_row_id: u64,
142}
143
144impl TableMetadata {
145 #[must_use]
152 pub fn into_builder(self, current_file_location: Option<String>) -> TableMetadataBuilder {
153 TableMetadataBuilder::new_from_metadata(self, current_file_location)
154 }
155
156 #[inline]
158 pub(crate) fn name_exists_in_any_schema(&self, name: &str) -> bool {
159 self.schemas
160 .values()
161 .any(|schema| schema.field_by_name(name).is_some())
162 }
163
164 #[inline]
166 pub fn format_version(&self) -> FormatVersion {
167 self.format_version
168 }
169
170 #[inline]
172 pub fn uuid(&self) -> Uuid {
173 self.table_uuid
174 }
175
176 #[inline]
178 pub fn location(&self) -> &str {
179 self.location.as_str()
180 }
181
182 #[inline]
184 pub fn last_sequence_number(&self) -> i64 {
185 self.last_sequence_number
186 }
187
188 #[inline]
193 pub fn next_sequence_number(&self) -> i64 {
194 match self.format_version {
195 FormatVersion::V1 => INITIAL_SEQUENCE_NUMBER,
196 _ => self.last_sequence_number + 1,
197 }
198 }
199
200 #[inline]
202 pub fn last_column_id(&self) -> i32 {
203 self.last_column_id
204 }
205
206 #[inline]
208 pub fn last_partition_id(&self) -> i32 {
209 self.last_partition_id
210 }
211
212 #[inline]
214 pub fn last_updated_timestamp(&self) -> Result<DateTime<Utc>> {
215 timestamp_ms_to_utc(self.last_updated_ms)
216 }
217
218 #[inline]
220 pub fn last_updated_ms(&self) -> i64 {
221 self.last_updated_ms
222 }
223
224 #[inline]
226 pub fn schemas_iter(&self) -> impl ExactSizeIterator<Item = &SchemaRef> {
227 self.schemas.values()
228 }
229
230 #[inline]
232 pub fn schema_by_id(&self, schema_id: SchemaId) -> Option<&SchemaRef> {
233 self.schemas.get(&schema_id)
234 }
235
236 #[inline]
238 pub fn current_schema(&self) -> &SchemaRef {
239 self.schema_by_id(self.current_schema_id)
240 .expect("Current schema id set, but not found in table metadata")
241 }
242
243 #[inline]
245 pub fn current_schema_id(&self) -> SchemaId {
246 self.current_schema_id
247 }
248
249 #[inline]
251 pub fn partition_specs_iter(&self) -> impl ExactSizeIterator<Item = &PartitionSpecRef> {
252 self.partition_specs.values()
253 }
254
255 #[inline]
257 pub fn partition_spec_by_id(&self, spec_id: i32) -> Option<&PartitionSpecRef> {
258 self.partition_specs.get(&spec_id)
259 }
260
261 #[inline]
263 pub fn default_partition_spec(&self) -> &PartitionSpecRef {
264 &self.default_spec
265 }
266
267 #[inline]
269 pub fn default_partition_type(&self) -> &StructType {
270 &self.default_partition_type
271 }
272
273 pub fn unified_partition_type(&self, schema: &Schema) -> Result<StructType> {
280 compute_unified_partition_type(
281 self.partition_specs_iter().map(|spec| spec.as_ref()),
282 schema,
283 )
284 }
285
286 #[inline]
287 pub fn default_partition_spec_id(&self) -> i32 {
289 self.default_spec.spec_id()
290 }
291
292 #[inline]
294 pub fn snapshots(&self) -> impl ExactSizeIterator<Item = &SnapshotRef> {
295 self.snapshots.values()
296 }
297
298 #[inline]
300 pub fn snapshot_by_id(&self, snapshot_id: i64) -> Option<&SnapshotRef> {
301 self.snapshots.get(&snapshot_id)
302 }
303
304 #[inline]
306 pub fn history(&self) -> &[SnapshotLog] {
307 &self.snapshot_log
308 }
309
310 #[inline]
312 pub fn metadata_log(&self) -> &[MetadataLog] {
313 &self.metadata_log
314 }
315
316 #[inline]
318 pub fn current_snapshot(&self) -> Option<&SnapshotRef> {
319 self.current_snapshot_id.map(|s| {
320 self.snapshot_by_id(s)
321 .expect("Current snapshot id has been set, but doesn't exist in metadata")
322 })
323 }
324
325 #[inline]
327 pub fn current_snapshot_id(&self) -> Option<i64> {
328 self.current_snapshot_id
329 }
330
331 #[inline]
334 pub fn snapshot_for_ref(&self, ref_name: &str) -> Option<&SnapshotRef> {
335 self.refs.get(ref_name).map(|r| {
336 self.snapshot_by_id(r.snapshot_id)
337 .unwrap_or_else(|| panic!("Snapshot id of ref {ref_name} doesn't exist"))
338 })
339 }
340
341 #[inline]
343 pub fn sort_orders_iter(&self) -> impl ExactSizeIterator<Item = &SortOrderRef> {
344 self.sort_orders.values()
345 }
346
347 #[inline]
349 pub fn sort_order_by_id(&self, sort_order_id: i64) -> Option<&SortOrderRef> {
350 self.sort_orders.get(&sort_order_id)
351 }
352
353 #[inline]
355 pub fn default_sort_order(&self) -> &SortOrderRef {
356 self.sort_orders
357 .get(&self.default_sort_order_id)
358 .expect("Default order id has been set, but not found in table metadata!")
359 }
360
361 #[inline]
363 pub fn default_sort_order_id(&self) -> i64 {
364 self.default_sort_order_id
365 }
366
367 #[inline]
369 pub fn properties(&self) -> &HashMap<String, String> {
370 &self.properties
371 }
372
373 pub fn metadata_location(&self) -> Result<String> {
378 Ok(self
379 .table_properties()
380 .write_metadata_path()?
381 .unwrap_or_else(|| format!("{}/{}", self.location(), METADATA_FOLDER_NAME)))
382 }
383
384 pub fn metadata_compression_codec(&self) -> Result<CompressionCodec> {
393 self.table_properties().metadata_compression_codec()
394 }
395
396 #[inline]
398 pub fn table_properties(&self) -> TableProperties<'_> {
399 TableProperties::new(&self.properties)
400 }
401
402 #[inline]
404 pub fn statistics_iter(&self) -> impl ExactSizeIterator<Item = &StatisticsFile> {
405 self.statistics.values()
406 }
407
408 #[inline]
410 pub fn partition_statistics_iter(
411 &self,
412 ) -> impl ExactSizeIterator<Item = &PartitionStatisticsFile> {
413 self.partition_statistics.values()
414 }
415
416 #[inline]
418 pub fn statistics_for_snapshot(&self, snapshot_id: i64) -> Option<&StatisticsFile> {
419 self.statistics.get(&snapshot_id)
420 }
421
422 #[inline]
424 pub fn partition_statistics_for_snapshot(
425 &self,
426 snapshot_id: i64,
427 ) -> Option<&PartitionStatisticsFile> {
428 self.partition_statistics.get(&snapshot_id)
429 }
430
431 fn construct_refs(&mut self) {
432 if let Some(current_snapshot_id) = self.current_snapshot_id
433 && !self.refs.contains_key(MAIN_BRANCH)
434 {
435 self.refs
436 .insert(MAIN_BRANCH.to_string(), SnapshotReference {
437 snapshot_id: current_snapshot_id,
438 retention: SnapshotRetention::Branch {
439 min_snapshots_to_keep: None,
440 max_snapshot_age_ms: None,
441 max_ref_age_ms: None,
442 },
443 });
444 }
445 }
446
447 #[inline]
449 pub fn encryption_keys_iter(&self) -> impl ExactSizeIterator<Item = &EncryptedKey> {
450 self.encryption_keys.values()
451 }
452
453 #[inline]
455 pub fn encryption_key(&self, key_id: &str) -> Option<&EncryptedKey> {
456 self.encryption_keys.get(key_id)
457 }
458
459 #[inline]
461 pub fn next_row_id(&self) -> u64 {
462 self.next_row_id
463 }
464
465 pub async fn read_from(
467 file_io: &FileIO,
468 metadata_location: impl AsRef<str>,
469 ) -> Result<TableMetadata> {
470 let metadata_location = metadata_location.as_ref();
471 let input_file = file_io.new_input(metadata_location)?;
472 let metadata_content = input_file.read().await?;
473
474 let metadata = if metadata_content.len() > 2
476 && metadata_content[0] == 0x1F
477 && metadata_content[1] == 0x8B
478 {
479 let decompressed_data = CompressionCodec::gzip_default()
480 .decompress(metadata_content.to_vec())
481 .map_err(|e| {
482 invalid_data!("Trying to read compressed metadata file")
483 .with_context("file_path", metadata_location)
484 .with_source(e)
485 })?;
486 serde_json::from_slice(&decompressed_data)?
487 } else {
488 serde_json::from_slice(&metadata_content)?
489 };
490
491 Ok(metadata)
492 }
493
494 pub async fn write_to(
496 &self,
497 file_io: &FileIO,
498 metadata_location: &MetadataLocation,
499 ) -> Result<()> {
500 let json_data = serde_json::to_vec(self)?;
501
502 let codec = self.table_properties().metadata_compression_codec()?;
504
505 if codec != metadata_location.compression_codec() {
506 return Err(invalid_data!(
507 "Compression codec mismatch: metadata_location has {:?}, but table properties specify {:?}",
508 metadata_location.compression_codec(),
509 codec
510 ));
511 }
512
513 let data_to_write = match codec {
515 CompressionCodec::Gzip(_) => codec.compress(json_data)?,
516 CompressionCodec::None => json_data,
517 _ => {
518 return Err(invalid_data!(
519 "Unsupported metadata compression codec: {codec:?}"
520 ));
521 }
522 };
523
524 file_io
525 .new_output(metadata_location.to_string())?
526 .write(data_to_write.into())
527 .await
528 }
529
530 pub(super) fn try_normalize(&mut self) -> Result<&mut Self> {
538 self.validate_current_schema()?;
539 self.normalize_current_snapshot()?;
540 self.construct_refs();
541 self.validate_refs()?;
542 self.validate_chronological_snapshot_logs()?;
543 self.validate_chronological_metadata_logs()?;
544 self.location = self.location.trim_end_matches('/').to_string();
546 self.validate_snapshot_sequence_number()?;
547 self.validate_schema_format_compatibility()?;
548 self.try_normalize_partition_spec()?;
549 self.try_normalize_sort_order()?;
550 Ok(self)
551 }
552
553 fn try_normalize_partition_spec(&mut self) -> Result<()> {
555 for field in self.default_spec.fields() {
556 if field.transform != Transform::Void
559 && self.current_schema().field_by_id(field.source_id).is_none()
560 {
561 return Err(invalid_data!(
562 "Default partition spec {} references missing source field {} in current schema {}",
563 self.default_spec.spec_id(),
564 field.source_id,
565 self.current_schema_id
566 ));
567 }
568 }
569
570 if self
571 .partition_spec_by_id(self.default_spec.spec_id())
572 .is_none()
573 {
574 self.partition_specs.insert(
575 self.default_spec.spec_id(),
576 Arc::new(Arc::unwrap_or_clone(self.default_spec.clone())),
577 );
578 }
579
580 Ok(())
581 }
582
583 fn try_normalize_sort_order(&mut self) -> Result<()> {
585 if let Some(sort_order) = self.sort_order_by_id(SortOrder::UNSORTED_ORDER_ID)
587 && !sort_order.fields.is_empty()
588 {
589 return Err(Error::new(
590 ErrorKind::Unexpected,
591 format!(
592 "Sort order ID {} is reserved for unsorted order",
593 SortOrder::UNSORTED_ORDER_ID
594 ),
595 ));
596 }
597
598 if self.sort_order_by_id(self.default_sort_order_id).is_some() {
599 return Ok(());
600 }
601
602 if self.default_sort_order_id != SortOrder::UNSORTED_ORDER_ID {
603 return Err(invalid_data!(
604 "No sort order exists with the default sort order id {}.",
605 self.default_sort_order_id
606 ));
607 }
608
609 let sort_order = SortOrder::unsorted_order();
610 self.sort_orders
611 .insert(SortOrder::UNSORTED_ORDER_ID, Arc::new(sort_order));
612 Ok(())
613 }
614
615 fn validate_current_schema(&self) -> Result<()> {
617 if self.schema_by_id(self.current_schema_id).is_none() {
618 return Err(invalid_data!(
619 "No schema exists with the current schema id {}.",
620 self.current_schema_id
621 ));
622 }
623 Ok(())
624 }
625
626 fn normalize_current_snapshot(&mut self) -> Result<()> {
628 if let Some(current_snapshot_id) = self.current_snapshot_id {
629 if current_snapshot_id == EMPTY_SNAPSHOT_ID {
630 self.current_snapshot_id = None;
631 } else if self.snapshot_by_id(current_snapshot_id).is_none() {
632 return Err(invalid_data!(
633 "Snapshot for current snapshot id {current_snapshot_id} does not exist in the existing snapshots list"
634 ));
635 }
636 }
637 Ok(())
638 }
639
640 fn validate_refs(&self) -> Result<()> {
642 for (name, snapshot_ref) in self.refs.iter() {
643 if self.snapshot_by_id(snapshot_ref.snapshot_id).is_none() {
644 return Err(invalid_data!(
645 "Snapshot for reference {name} does not exist in the existing snapshots list"
646 ));
647 }
648 }
649
650 let main_ref = self.refs.get(MAIN_BRANCH);
651 if self.current_snapshot_id.is_some() {
652 if let Some(main_ref) = main_ref
653 && main_ref.snapshot_id != self.current_snapshot_id.unwrap_or_default()
654 {
655 return Err(invalid_data!(
656 "Current snapshot id does not match main branch ({:?} != {:?})",
657 self.current_snapshot_id.unwrap_or_default(),
658 main_ref.snapshot_id
659 ));
660 }
661 } else if main_ref.is_some() {
662 return Err(invalid_data!(
663 "Current snapshot is not set, but main branch exists"
664 ));
665 }
666
667 Ok(())
668 }
669
670 fn validate_snapshot_sequence_number(&self) -> Result<()> {
672 if self.format_version < FormatVersion::V2 && self.last_sequence_number != 0 {
673 return Err(invalid_data!(
674 "Last sequence number must be 0 in v1. Found {}",
675 self.last_sequence_number
676 ));
677 }
678
679 if self.format_version >= FormatVersion::V2
680 && let Some(snapshot) = self
681 .snapshots
682 .values()
683 .find(|snapshot| snapshot.sequence_number() > self.last_sequence_number)
684 {
685 return Err(invalid_data!(
686 "Invalid snapshot with id {} and sequence number {} greater than last sequence number {}",
687 snapshot.snapshot_id(),
688 snapshot.sequence_number(),
689 self.last_sequence_number
690 ));
691 }
692
693 Ok(())
694 }
695
696 fn validate_chronological_snapshot_logs(&self) -> Result<()> {
698 for window in self.snapshot_log.windows(2) {
699 let (prev, curr) = (&window[0], &window[1]);
700 if curr.timestamp_ms - prev.timestamp_ms < -ONE_MINUTE_MS {
703 return Err(invalid_data!("Expected sorted snapshot log entries"));
704 }
705 }
706
707 if let Some(last) = self.snapshot_log.last() {
708 if self.last_updated_ms - last.timestamp_ms < -ONE_MINUTE_MS {
711 return Err(invalid_data!(
712 "Invalid update timestamp {}: before last snapshot log entry at {}",
713 self.last_updated_ms,
714 last.timestamp_ms
715 ));
716 }
717 }
718 Ok(())
719 }
720
721 fn validate_chronological_metadata_logs(&self) -> Result<()> {
722 for window in self.metadata_log.windows(2) {
723 let (prev, curr) = (&window[0], &window[1]);
724 if curr.timestamp_ms - prev.timestamp_ms < -ONE_MINUTE_MS {
727 return Err(invalid_data!("Expected sorted metadata log entries"));
728 }
729 }
730
731 if let Some(last) = self.metadata_log.last() {
732 if self.last_updated_ms - last.timestamp_ms < -ONE_MINUTE_MS {
735 return Err(invalid_data!(
736 "Invalid update timestamp {}: before last metadata log entry at {}",
737 self.last_updated_ms,
738 last.timestamp_ms
739 ));
740 }
741 }
742
743 Ok(())
744 }
745
746 fn validate_schema_format_compatibility(&self) -> Result<()> {
749 self.current_schema()
750 .check_format_compatibility(self.format_version)
751 }
752}
753
754pub(super) mod _serde {
755 use std::borrow::BorrowMut;
756 use std::collections::HashMap;
761 use std::sync::Arc;
766
767 use serde::{Deserialize, Serialize};
768 use uuid::Uuid;
769
770 use super::{
771 DEFAULT_PARTITION_SPEC_ID, EMPTY_SNAPSHOT_ID, FormatVersion, MAIN_BRANCH, MetadataLog,
772 SnapshotLog, TableMetadata,
773 };
774 use crate::error::invalid_data;
775 use crate::spec::schema::_serde::{SchemaV1, SchemaV2};
776 use crate::spec::snapshot::_serde::{SnapshotV1, SnapshotV2, SnapshotV3};
777 use crate::spec::{
778 EncryptedKey, INITIAL_ROW_ID, PartitionField, PartitionSpec, PartitionSpecRef,
779 PartitionStatisticsFile, Schema, SchemaRef, Snapshot, SnapshotReference, SnapshotRetention,
780 SortOrder, StatisticsFile, UnboundPartitionSpec,
781 };
782 use crate::{Error, ErrorKind};
783
784 #[derive(Debug, Serialize, Deserialize, PartialEq, Eq)]
785 #[serde(untagged)]
786 pub(super) enum TableMetadataEnum {
787 V3(TableMetadataV3),
788 V2(TableMetadataV2),
789 V1(TableMetadataV1),
790 }
791
792 #[derive(Debug, Serialize, Deserialize, PartialEq, Eq)]
793 #[serde(rename_all = "kebab-case")]
794 pub(super) struct TableMetadataV3 {
796 pub format_version: VersionNumber<3>,
797 #[serde(flatten)]
798 pub shared: TableMetadataV2V3Shared,
799 pub next_row_id: u64,
800 #[serde(skip_serializing_if = "Option::is_none")]
801 pub encryption_keys: Option<Vec<EncryptedKey>>,
802 #[serde(skip_serializing_if = "Option::is_none")]
803 pub snapshots: Option<Vec<SnapshotV3>>,
804 }
805
806 #[derive(Debug, Serialize, Deserialize, PartialEq, Eq)]
807 #[serde(rename_all = "kebab-case")]
808 pub(super) struct TableMetadataV2V3Shared {
810 pub table_uuid: Uuid,
811 pub location: String,
812 pub last_sequence_number: i64,
813 pub last_updated_ms: i64,
814 pub last_column_id: i32,
815 pub schemas: Vec<SchemaV2>,
816 pub current_schema_id: i32,
817 pub partition_specs: Vec<PartitionSpec>,
818 pub default_spec_id: i32,
819 pub last_partition_id: i32,
820 #[serde(skip_serializing_if = "Option::is_none")]
821 pub properties: Option<HashMap<String, String>>,
822 #[serde(skip_serializing_if = "Option::is_none")]
823 pub current_snapshot_id: Option<i64>,
824 #[serde(skip_serializing_if = "Option::is_none")]
825 pub snapshot_log: Option<Vec<SnapshotLog>>,
826 #[serde(skip_serializing_if = "Option::is_none")]
827 pub metadata_log: Option<Vec<MetadataLog>>,
828 pub sort_orders: Vec<SortOrder>,
829 pub default_sort_order_id: i64,
830 #[serde(skip_serializing_if = "Option::is_none")]
831 pub refs: Option<HashMap<String, SnapshotReference>>,
832 #[serde(default, skip_serializing_if = "Vec::is_empty")]
833 pub statistics: Vec<StatisticsFile>,
834 #[serde(default, skip_serializing_if = "Vec::is_empty")]
835 pub partition_statistics: Vec<PartitionStatisticsFile>,
836 }
837
838 #[derive(Debug, Serialize, Deserialize, PartialEq, Eq)]
839 #[serde(rename_all = "kebab-case")]
840 pub(super) struct TableMetadataV2 {
842 pub format_version: VersionNumber<2>,
843 #[serde(flatten)]
844 pub shared: TableMetadataV2V3Shared,
845 #[serde(skip_serializing_if = "Option::is_none")]
846 pub snapshots: Option<Vec<SnapshotV2>>,
847 }
848
849 #[derive(Debug, Serialize, Deserialize, PartialEq, Eq)]
850 #[serde(rename_all = "kebab-case")]
851 pub(super) struct TableMetadataV1 {
853 pub format_version: VersionNumber<1>,
854 #[serde(skip_serializing_if = "Option::is_none")]
855 pub table_uuid: Option<Uuid>,
856 pub location: String,
857 pub last_updated_ms: i64,
858 pub last_column_id: i32,
859 pub schema: Option<SchemaV1>,
861 #[serde(skip_serializing_if = "Option::is_none")]
862 pub schemas: Option<Vec<SchemaV1>>,
863 #[serde(skip_serializing_if = "Option::is_none")]
864 pub current_schema_id: Option<i32>,
865 pub partition_spec: Option<Vec<PartitionField>>,
867 #[serde(skip_serializing_if = "Option::is_none")]
868 pub partition_specs: Option<Vec<PartitionSpec>>,
869 #[serde(skip_serializing_if = "Option::is_none")]
870 pub default_spec_id: Option<i32>,
871 #[serde(skip_serializing_if = "Option::is_none")]
872 pub last_partition_id: Option<i32>,
873 #[serde(skip_serializing_if = "Option::is_none")]
874 pub properties: Option<HashMap<String, String>>,
875 #[serde(skip_serializing_if = "Option::is_none")]
876 pub current_snapshot_id: Option<i64>,
877 #[serde(skip_serializing_if = "Option::is_none")]
878 pub snapshots: Option<Vec<SnapshotV1>>,
879 #[serde(skip_serializing_if = "Option::is_none")]
880 pub snapshot_log: Option<Vec<SnapshotLog>>,
881 #[serde(skip_serializing_if = "Option::is_none")]
882 pub metadata_log: Option<Vec<MetadataLog>>,
883 pub sort_orders: Option<Vec<SortOrder>>,
884 pub default_sort_order_id: Option<i64>,
885 #[serde(skip_serializing_if = "Option::is_none")]
886 pub refs: Option<HashMap<String, SnapshotReference>>,
887 #[serde(default, skip_serializing_if = "Vec::is_empty")]
888 pub statistics: Vec<StatisticsFile>,
889 #[serde(default, skip_serializing_if = "Vec::is_empty")]
890 pub partition_statistics: Vec<PartitionStatisticsFile>,
891 }
892
893 #[derive(Debug, PartialEq, Eq)]
895 pub(crate) struct VersionNumber<const V: u8>;
896
897 impl Serialize for TableMetadata {
898 fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error>
899 where S: serde::Serializer {
900 let table_metadata_enum: TableMetadataEnum =
902 self.clone().try_into().map_err(serde::ser::Error::custom)?;
903
904 table_metadata_enum.serialize(serializer)
905 }
906 }
907
908 impl<const V: u8> Serialize for VersionNumber<V> {
909 fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error>
910 where S: serde::Serializer {
911 serializer.serialize_u8(V)
912 }
913 }
914
915 impl<'de, const V: u8> Deserialize<'de> for VersionNumber<V> {
916 fn deserialize<D>(deserializer: D) -> Result<Self, D::Error>
917 where D: serde::Deserializer<'de> {
918 let value = u8::deserialize(deserializer)?;
919 if value == V {
920 Ok(VersionNumber::<V>)
921 } else {
922 Err(serde::de::Error::custom("Invalid Version"))
923 }
924 }
925 }
926
927 impl TryFrom<TableMetadataEnum> for TableMetadata {
928 type Error = Error;
929 fn try_from(value: TableMetadataEnum) -> Result<Self, Error> {
930 match value {
931 TableMetadataEnum::V3(value) => value.try_into(),
932 TableMetadataEnum::V2(value) => value.try_into(),
933 TableMetadataEnum::V1(value) => value.try_into(),
934 }
935 }
936 }
937
938 impl TryFrom<TableMetadata> for TableMetadataEnum {
939 type Error = Error;
940 fn try_from(value: TableMetadata) -> Result<Self, Error> {
941 Ok(match value.format_version {
942 FormatVersion::V3 => TableMetadataEnum::V3(value.try_into()?),
943 FormatVersion::V2 => TableMetadataEnum::V2(value.into()),
944 FormatVersion::V1 => TableMetadataEnum::V1(value.try_into()?),
945 })
946 }
947 }
948
949 impl TryFrom<TableMetadataV3> for TableMetadata {
950 type Error = Error;
951 fn try_from(value: TableMetadataV3) -> Result<Self, Error> {
952 let TableMetadataV3 {
953 format_version: _,
954 shared: value,
955 next_row_id,
956 encryption_keys,
957 snapshots,
958 } = value;
959 let current_snapshot_id = if value.current_snapshot_id == Some(EMPTY_SNAPSHOT_ID) {
960 None
961 } else {
962 value.current_snapshot_id
963 };
964 let schemas = HashMap::from_iter(
965 value
966 .schemas
967 .into_iter()
968 .map(|schema| Ok((schema.schema_id, Arc::new(schema.try_into()?))))
969 .collect::<Result<Vec<_>, Error>>()?,
970 );
971
972 let current_schema: &SchemaRef =
973 schemas.get(&value.current_schema_id).ok_or_else(|| {
974 invalid_data!(
975 "No schema exists with the current schema id {}.",
976 value.current_schema_id
977 )
978 })?;
979 let partition_specs = HashMap::from_iter(
980 value
981 .partition_specs
982 .into_iter()
983 .map(|x| (x.spec_id(), Arc::new(x))),
984 );
985 let default_spec_id = value.default_spec_id;
986 let default_spec: PartitionSpecRef = partition_specs
987 .get(&value.default_spec_id)
988 .map(|spec| (**spec).clone())
989 .or_else(|| {
990 (DEFAULT_PARTITION_SPEC_ID == default_spec_id)
991 .then(PartitionSpec::unpartition_spec)
992 })
993 .ok_or_else(|| invalid_data!("Default partition spec {default_spec_id} not found"))?
994 .into();
995 let default_partition_type = default_spec.partition_type(current_schema)?;
996
997 let mut metadata = TableMetadata {
998 format_version: FormatVersion::V3,
999 table_uuid: value.table_uuid,
1000 location: value.location,
1001 last_sequence_number: value.last_sequence_number,
1002 last_updated_ms: value.last_updated_ms,
1003 last_column_id: value.last_column_id,
1004 current_schema_id: value.current_schema_id,
1005 schemas,
1006 partition_specs,
1007 default_partition_type,
1008 default_spec,
1009 last_partition_id: value.last_partition_id,
1010 properties: value.properties.unwrap_or_default(),
1011 current_snapshot_id,
1012 snapshots: snapshots
1013 .map(|snapshots| {
1014 HashMap::from_iter(
1015 snapshots
1016 .into_iter()
1017 .map(|x| (x.snapshot_id, Arc::new(x.into()))),
1018 )
1019 })
1020 .unwrap_or_default(),
1021 snapshot_log: value.snapshot_log.unwrap_or_default(),
1022 metadata_log: value.metadata_log.unwrap_or_default(),
1023 sort_orders: HashMap::from_iter(
1024 value
1025 .sort_orders
1026 .into_iter()
1027 .map(|x| (x.order_id, Arc::new(x))),
1028 ),
1029 default_sort_order_id: value.default_sort_order_id,
1030 refs: value.refs.unwrap_or_else(|| {
1031 if let Some(snapshot_id) = current_snapshot_id {
1032 HashMap::from_iter(vec![(MAIN_BRANCH.to_string(), SnapshotReference {
1033 snapshot_id,
1034 retention: SnapshotRetention::Branch {
1035 min_snapshots_to_keep: None,
1036 max_snapshot_age_ms: None,
1037 max_ref_age_ms: None,
1038 },
1039 })])
1040 } else {
1041 HashMap::new()
1042 }
1043 }),
1044 statistics: index_statistics(value.statistics),
1045 partition_statistics: index_partition_statistics(value.partition_statistics),
1046 encryption_keys: encryption_keys
1047 .map(|keys| {
1048 HashMap::from_iter(keys.into_iter().map(|key| (key.key_id.clone(), key)))
1049 })
1050 .unwrap_or_default(),
1051 next_row_id,
1052 };
1053
1054 metadata.borrow_mut().try_normalize()?;
1055 Ok(metadata)
1056 }
1057 }
1058
1059 impl TryFrom<TableMetadataV2> for TableMetadata {
1060 type Error = Error;
1061 fn try_from(value: TableMetadataV2) -> Result<Self, Error> {
1062 let snapshots = value.snapshots;
1063 let value = value.shared;
1064 let current_snapshot_id = if value.current_snapshot_id == Some(EMPTY_SNAPSHOT_ID) {
1065 None
1066 } else {
1067 value.current_snapshot_id
1068 };
1069 let schemas = HashMap::from_iter(
1070 value
1071 .schemas
1072 .into_iter()
1073 .map(|schema| Ok((schema.schema_id, Arc::new(schema.try_into()?))))
1074 .collect::<Result<Vec<_>, Error>>()?,
1075 );
1076
1077 let current_schema: &SchemaRef =
1078 schemas.get(&value.current_schema_id).ok_or_else(|| {
1079 invalid_data!(
1080 "No schema exists with the current schema id {}.",
1081 value.current_schema_id
1082 )
1083 })?;
1084 let partition_specs = HashMap::from_iter(
1085 value
1086 .partition_specs
1087 .into_iter()
1088 .map(|x| (x.spec_id(), Arc::new(x))),
1089 );
1090 let default_spec_id = value.default_spec_id;
1091 let default_spec: PartitionSpecRef = partition_specs
1092 .get(&value.default_spec_id)
1093 .map(|spec| (**spec).clone())
1094 .or_else(|| {
1095 (DEFAULT_PARTITION_SPEC_ID == default_spec_id)
1096 .then(PartitionSpec::unpartition_spec)
1097 })
1098 .ok_or_else(|| invalid_data!("Default partition spec {default_spec_id} not found"))?
1099 .into();
1100 let default_partition_type = default_spec.partition_type(current_schema)?;
1101
1102 let mut metadata = TableMetadata {
1103 format_version: FormatVersion::V2,
1104 table_uuid: value.table_uuid,
1105 location: value.location,
1106 last_sequence_number: value.last_sequence_number,
1107 last_updated_ms: value.last_updated_ms,
1108 last_column_id: value.last_column_id,
1109 current_schema_id: value.current_schema_id,
1110 schemas,
1111 partition_specs,
1112 default_partition_type,
1113 default_spec,
1114 last_partition_id: value.last_partition_id,
1115 properties: value.properties.unwrap_or_default(),
1116 current_snapshot_id,
1117 snapshots: snapshots
1118 .map(|snapshots| {
1119 HashMap::from_iter(
1120 snapshots
1121 .into_iter()
1122 .map(|x| (x.snapshot_id, Arc::new(x.into()))),
1123 )
1124 })
1125 .unwrap_or_default(),
1126 snapshot_log: value.snapshot_log.unwrap_or_default(),
1127 metadata_log: value.metadata_log.unwrap_or_default(),
1128 sort_orders: HashMap::from_iter(
1129 value
1130 .sort_orders
1131 .into_iter()
1132 .map(|x| (x.order_id, Arc::new(x))),
1133 ),
1134 default_sort_order_id: value.default_sort_order_id,
1135 refs: value.refs.unwrap_or_else(|| {
1136 if let Some(snapshot_id) = current_snapshot_id {
1137 HashMap::from_iter(vec![(MAIN_BRANCH.to_string(), SnapshotReference {
1138 snapshot_id,
1139 retention: SnapshotRetention::Branch {
1140 min_snapshots_to_keep: None,
1141 max_snapshot_age_ms: None,
1142 max_ref_age_ms: None,
1143 },
1144 })])
1145 } else {
1146 HashMap::new()
1147 }
1148 }),
1149 statistics: index_statistics(value.statistics),
1150 partition_statistics: index_partition_statistics(value.partition_statistics),
1151 encryption_keys: HashMap::new(),
1152 next_row_id: INITIAL_ROW_ID,
1153 };
1154
1155 metadata.borrow_mut().try_normalize()?;
1156 Ok(metadata)
1157 }
1158 }
1159
1160 impl TryFrom<TableMetadataV1> for TableMetadata {
1161 type Error = Error;
1162 fn try_from(value: TableMetadataV1) -> Result<Self, Error> {
1163 let current_snapshot_id = if value.current_snapshot_id == Some(EMPTY_SNAPSHOT_ID) {
1164 None
1165 } else {
1166 value.current_snapshot_id
1167 };
1168
1169 let (schemas, current_schema_id, current_schema) =
1170 if let (Some(schemas_vec), Some(schema_id)) =
1171 (&value.schemas, value.current_schema_id)
1172 {
1173 let schema_map = HashMap::from_iter(
1175 schemas_vec
1176 .clone()
1177 .into_iter()
1178 .map(|schema| {
1179 let schema: Schema = schema.try_into()?;
1180 Ok((schema.schema_id(), Arc::new(schema)))
1181 })
1182 .collect::<Result<Vec<_>, Error>>()?,
1183 );
1184
1185 let schema = schema_map
1186 .get(&schema_id)
1187 .ok_or_else(|| {
1188 invalid_data!(
1189 "No schema exists with the current schema id {schema_id}."
1190 )
1191 })?
1192 .clone();
1193 (schema_map, schema_id, schema)
1194 } else if let Some(schema) = value.schema {
1195 let schema: Schema = schema.try_into()?;
1197 let schema_id = schema.schema_id();
1198 let schema_arc = Arc::new(schema);
1199 let schema_map = HashMap::from_iter(vec![(schema_id, schema_arc.clone())]);
1200 (schema_map, schema_id, schema_arc)
1201 } else {
1202 return Err(invalid_data!(
1204 "No valid schema configuration found in table metadata"
1205 ));
1206 };
1207
1208 let partition_specs = if let Some(specs_vec) = value.partition_specs {
1210 specs_vec
1212 .into_iter()
1213 .map(|x| (x.spec_id(), Arc::new(x)))
1214 .collect::<HashMap<_, _>>()
1215 } else if let Some(partition_spec) = value.partition_spec {
1216 let spec = UnboundPartitionSpec {
1218 spec_id: Some(DEFAULT_PARTITION_SPEC_ID),
1219 fields: partition_spec
1220 .into_iter()
1221 .map(|f| f.into_unbound())
1222 .collect(),
1223 }
1224 .bind(current_schema.clone())?;
1225
1226 HashMap::from_iter(vec![(DEFAULT_PARTITION_SPEC_ID, Arc::new(spec))])
1227 } else {
1228 let spec = PartitionSpec::builder(current_schema.clone())
1230 .with_spec_id(DEFAULT_PARTITION_SPEC_ID)
1231 .build()?;
1232
1233 HashMap::from_iter(vec![(DEFAULT_PARTITION_SPEC_ID, Arc::new(spec))])
1234 };
1235
1236 let default_spec_id = value
1238 .default_spec_id
1239 .unwrap_or_else(|| partition_specs.keys().copied().max().unwrap_or_default());
1240
1241 let default_spec: PartitionSpecRef = partition_specs
1243 .get(&default_spec_id)
1244 .map(|x| Arc::unwrap_or_clone(x.clone()))
1245 .ok_or_else(|| invalid_data!("Default partition spec {default_spec_id} not found"))?
1246 .into();
1247 let default_partition_type = default_spec.partition_type(¤t_schema)?;
1248
1249 let mut metadata = TableMetadata {
1250 format_version: FormatVersion::V1,
1251 table_uuid: value.table_uuid.unwrap_or_default(),
1252 location: value.location,
1253 last_sequence_number: 0,
1254 last_updated_ms: value.last_updated_ms,
1255 last_column_id: value.last_column_id,
1256 current_schema_id,
1257 default_spec,
1258 default_partition_type,
1259 last_partition_id: value
1260 .last_partition_id
1261 .unwrap_or_else(|| partition_specs.keys().copied().max().unwrap_or_default()),
1262 partition_specs,
1263 schemas,
1264 properties: value.properties.unwrap_or_default(),
1265 current_snapshot_id,
1266 snapshots: value
1267 .snapshots
1268 .map(|snapshots| {
1269 Ok::<_, Error>(HashMap::from_iter(
1270 snapshots
1271 .into_iter()
1272 .map(|x| Ok((x.snapshot_id, Arc::new(x.try_into()?))))
1273 .collect::<Result<Vec<_>, Error>>()?,
1274 ))
1275 })
1276 .transpose()?
1277 .unwrap_or_default(),
1278 snapshot_log: value.snapshot_log.unwrap_or_default(),
1279 metadata_log: value.metadata_log.unwrap_or_default(),
1280 sort_orders: match value.sort_orders {
1281 Some(sort_orders) => HashMap::from_iter(
1282 sort_orders.into_iter().map(|x| (x.order_id, Arc::new(x))),
1283 ),
1284 None => HashMap::new(),
1285 },
1286 default_sort_order_id: value
1287 .default_sort_order_id
1288 .unwrap_or(SortOrder::UNSORTED_ORDER_ID),
1289 refs: value.refs.unwrap_or_default(),
1291 statistics: index_statistics(value.statistics),
1292 partition_statistics: index_partition_statistics(value.partition_statistics),
1293 encryption_keys: HashMap::new(),
1294 next_row_id: INITIAL_ROW_ID, };
1296
1297 metadata.borrow_mut().try_normalize()?;
1298 Ok(metadata)
1299 }
1300 }
1301
1302 impl TryFrom<TableMetadata> for TableMetadataV3 {
1303 type Error = Error;
1304
1305 fn try_from(mut v: TableMetadata) -> Result<Self, Self::Error> {
1306 let next_row_id = v.next_row_id;
1307 let encryption_keys = std::mem::take(&mut v.encryption_keys);
1308 let snapshots = std::mem::take(&mut v.snapshots);
1309 let shared = v.into();
1310
1311 Ok(TableMetadataV3 {
1312 format_version: VersionNumber::<3>,
1313 shared,
1314 next_row_id,
1315 encryption_keys: if encryption_keys.is_empty() {
1316 None
1317 } else {
1318 Some(encryption_keys.into_values().collect())
1319 },
1320 snapshots: if snapshots.is_empty() {
1321 None
1322 } else {
1323 Some(
1324 snapshots
1325 .into_values()
1326 .map(|s| SnapshotV3::try_from(Arc::unwrap_or_clone(s)))
1327 .collect::<Result<_, _>>()?,
1328 )
1329 },
1330 })
1331 }
1332 }
1333
1334 impl From<TableMetadata> for TableMetadataV2 {
1335 fn from(mut v: TableMetadata) -> Self {
1336 let snapshots = std::mem::take(&mut v.snapshots);
1337 let shared = v.into();
1338
1339 TableMetadataV2 {
1340 format_version: VersionNumber::<2>,
1341 shared,
1342 snapshots: if snapshots.is_empty() {
1343 None
1344 } else {
1345 Some(
1346 snapshots
1347 .into_values()
1348 .map(|s| SnapshotV2::from(Arc::unwrap_or_clone(s)))
1349 .collect(),
1350 )
1351 },
1352 }
1353 }
1354 }
1355
1356 impl From<TableMetadata> for TableMetadataV2V3Shared {
1357 fn from(v: TableMetadata) -> Self {
1358 TableMetadataV2V3Shared {
1359 table_uuid: v.table_uuid,
1360 location: v.location,
1361 last_sequence_number: v.last_sequence_number,
1362 last_updated_ms: v.last_updated_ms,
1363 last_column_id: v.last_column_id,
1364 schemas: v
1365 .schemas
1366 .into_values()
1367 .map(|x| {
1368 Arc::try_unwrap(x)
1369 .unwrap_or_else(|schema| schema.as_ref().clone())
1370 .into()
1371 })
1372 .collect(),
1373 current_schema_id: v.current_schema_id,
1374 partition_specs: v
1375 .partition_specs
1376 .into_values()
1377 .map(|x| Arc::try_unwrap(x).unwrap_or_else(|s| s.as_ref().clone()))
1378 .collect(),
1379 default_spec_id: v.default_spec.spec_id(),
1380 last_partition_id: v.last_partition_id,
1381 properties: if v.properties.is_empty() {
1382 None
1383 } else {
1384 Some(v.properties)
1385 },
1386 current_snapshot_id: v.current_snapshot_id,
1387 snapshot_log: if v.snapshot_log.is_empty() {
1388 None
1389 } else {
1390 Some(v.snapshot_log)
1391 },
1392 metadata_log: if v.metadata_log.is_empty() {
1393 None
1394 } else {
1395 Some(v.metadata_log)
1396 },
1397 sort_orders: v
1398 .sort_orders
1399 .into_values()
1400 .map(|x| Arc::try_unwrap(x).unwrap_or_else(|s| s.as_ref().clone()))
1401 .collect(),
1402 default_sort_order_id: v.default_sort_order_id,
1403 refs: Some(v.refs),
1404 statistics: v.statistics.into_values().collect(),
1405 partition_statistics: v.partition_statistics.into_values().collect(),
1406 }
1407 }
1408 }
1409
1410 impl TryFrom<TableMetadata> for TableMetadataV1 {
1411 type Error = Error;
1412 fn try_from(v: TableMetadata) -> Result<Self, Error> {
1413 Ok(TableMetadataV1 {
1414 format_version: VersionNumber::<1>,
1415 table_uuid: Some(v.table_uuid),
1416 location: v.location,
1417 last_updated_ms: v.last_updated_ms,
1418 last_column_id: v.last_column_id,
1419 schema: Some(
1420 v.schemas
1421 .get(&v.current_schema_id)
1422 .ok_or(Error::new(
1423 ErrorKind::Unexpected,
1424 "current_schema_id not found in schemas",
1425 ))?
1426 .as_ref()
1427 .clone()
1428 .into(),
1429 ),
1430 schemas: Some(
1431 v.schemas
1432 .into_values()
1433 .map(|x| {
1434 Arc::try_unwrap(x)
1435 .unwrap_or_else(|schema| schema.as_ref().clone())
1436 .into()
1437 })
1438 .collect(),
1439 ),
1440 current_schema_id: Some(v.current_schema_id),
1441 partition_spec: Some(v.default_spec.fields().to_vec()),
1442 partition_specs: Some(
1443 v.partition_specs
1444 .into_values()
1445 .map(|x| Arc::try_unwrap(x).unwrap_or_else(|s| s.as_ref().clone()))
1446 .collect(),
1447 ),
1448 default_spec_id: Some(v.default_spec.spec_id()),
1449 last_partition_id: Some(v.last_partition_id),
1450 properties: if v.properties.is_empty() {
1451 None
1452 } else {
1453 Some(v.properties)
1454 },
1455 current_snapshot_id: v.current_snapshot_id,
1456 snapshots: if v.snapshots.is_empty() {
1457 None
1458 } else {
1459 Some(
1460 v.snapshots
1461 .into_values()
1462 .map(|x| Snapshot::clone(&x).into())
1463 .collect(),
1464 )
1465 },
1466 snapshot_log: if v.snapshot_log.is_empty() {
1467 None
1468 } else {
1469 Some(v.snapshot_log)
1470 },
1471 metadata_log: if v.metadata_log.is_empty() {
1472 None
1473 } else {
1474 Some(v.metadata_log)
1475 },
1476 sort_orders: Some(
1477 v.sort_orders
1478 .into_values()
1479 .map(|s| Arc::try_unwrap(s).unwrap_or_else(|s| s.as_ref().clone()))
1480 .collect(),
1481 ),
1482 default_sort_order_id: Some(v.default_sort_order_id),
1483 refs: Some(v.refs),
1484 statistics: v.statistics.into_values().collect(),
1485 partition_statistics: v.partition_statistics.into_values().collect(),
1486 })
1487 }
1488 }
1489
1490 fn index_statistics(statistics: Vec<StatisticsFile>) -> HashMap<i64, StatisticsFile> {
1491 statistics
1492 .into_iter()
1493 .rev()
1494 .map(|s| (s.snapshot_id, s))
1495 .collect()
1496 }
1497
1498 fn index_partition_statistics(
1499 statistics: Vec<PartitionStatisticsFile>,
1500 ) -> HashMap<i64, PartitionStatisticsFile> {
1501 statistics
1502 .into_iter()
1503 .rev()
1504 .map(|s| (s.snapshot_id, s))
1505 .collect()
1506 }
1507}
1508
1509#[derive(Debug, Serialize_repr, Deserialize_repr, PartialEq, Eq, Clone, Copy, Hash)]
1510#[repr(u8)]
1511pub enum FormatVersion {
1513 V1 = 1u8,
1515 V2 = 2u8,
1517 V3 = 3u8,
1519}
1520
1521impl PartialOrd for FormatVersion {
1522 fn partial_cmp(&self, other: &Self) -> Option<Ordering> {
1523 Some(self.cmp(other))
1524 }
1525}
1526
1527impl Ord for FormatVersion {
1528 fn cmp(&self, other: &Self) -> Ordering {
1529 (*self as u8).cmp(&(*other as u8))
1530 }
1531}
1532
1533impl Display for FormatVersion {
1534 fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result {
1535 match self {
1536 FormatVersion::V1 => write!(f, "v1"),
1537 FormatVersion::V2 => write!(f, "v2"),
1538 FormatVersion::V3 => write!(f, "v3"),
1539 }
1540 }
1541}
1542
1543#[derive(Debug, Serialize, Deserialize, PartialEq, Eq, Clone)]
1544#[serde(rename_all = "kebab-case")]
1545pub struct MetadataLog {
1547 pub metadata_file: String,
1549 pub timestamp_ms: i64,
1551}
1552
1553#[derive(Debug, Serialize, Deserialize, PartialEq, Eq, Clone)]
1554#[serde(rename_all = "kebab-case")]
1555pub struct SnapshotLog {
1557 pub snapshot_id: i64,
1559 pub timestamp_ms: i64,
1561}
1562
1563impl SnapshotLog {
1564 pub fn timestamp(self) -> Result<DateTime<Utc>> {
1566 timestamp_ms_to_utc(self.timestamp_ms)
1567 }
1568
1569 #[inline]
1571 pub fn timestamp_ms(&self) -> i64 {
1572 self.timestamp_ms
1573 }
1574}
1575
1576#[cfg(test)]
1577mod tests {
1578 use std::collections::HashMap;
1579 use std::fs;
1580 use std::sync::Arc;
1581
1582 use anyhow::Result;
1583 use base64::Engine as _;
1584 use pretty_assertions::assert_eq;
1585 use tempfile::TempDir;
1586 use uuid::Uuid;
1587
1588 use super::{FormatVersion, MetadataLog, SnapshotLog, TableMetadataBuilder};
1589 use crate::catalog::MetadataLocation;
1590 use crate::compression::CompressionCodec;
1591 use crate::io::FileIO;
1592 use crate::spec::table_metadata::TableMetadata;
1593 use crate::spec::{
1594 BlobMetadata, EncryptedKey, INITIAL_ROW_ID, Literal, NestedField, NullOrder, Operation,
1595 PartitionSpec, PartitionStatisticsFile, PrimitiveLiteral, PrimitiveType, Schema, Snapshot,
1596 SnapshotReference, SnapshotRetention, SortDirection, SortField, SortOrder, StatisticsFile,
1597 Summary, TableProperties, Transform, Type, UnboundPartitionField, UnboundPartitionSpec,
1598 };
1599 use crate::{ErrorKind, TableCreation};
1600
1601 fn check_table_metadata_serde(json: &str, expected_type: TableMetadata) {
1602 let desered_type: TableMetadata = serde_json::from_str(json).unwrap();
1603 assert_eq!(desered_type, expected_type);
1604
1605 let sered_json = serde_json::to_string(&expected_type).unwrap();
1606 let parsed_json_value = serde_json::from_str::<TableMetadata>(&sered_json).unwrap();
1607
1608 assert_eq!(parsed_json_value, desered_type);
1609 }
1610
1611 fn get_test_table_metadata(file_name: &str) -> TableMetadata {
1612 let path = format!("testdata/table_metadata/{file_name}");
1613 let metadata: String = fs::read_to_string(path).unwrap();
1614
1615 serde_json::from_str(&metadata).unwrap()
1616 }
1617
1618 fn get_test_table_metadata_at(file_name: &str, location: &str) -> TableMetadata {
1621 TableMetadataBuilder::new_from_metadata(get_test_table_metadata(file_name), None)
1622 .set_location(location.to_string())
1623 .build()
1624 .unwrap()
1625 .metadata
1626 }
1627
1628 #[test]
1629 fn test_table_data_v2() {
1630 let data = r#"
1631 {
1632 "format-version" : 2,
1633 "table-uuid": "fb072c92-a02b-11e9-ae9c-1bb7bc9eca94",
1634 "location": "s3://b/wh/data.db/table",
1635 "last-sequence-number" : 1,
1636 "last-updated-ms": 1515100955770,
1637 "last-column-id": 1,
1638 "schemas": [
1639 {
1640 "schema-id" : 1,
1641 "type" : "struct",
1642 "fields" :[
1643 {
1644 "id": 1,
1645 "name": "struct_name",
1646 "required": true,
1647 "type": "fixed[1]"
1648 },
1649 {
1650 "id": 4,
1651 "name": "ts",
1652 "required": true,
1653 "type": "timestamp"
1654 }
1655 ]
1656 }
1657 ],
1658 "current-schema-id" : 1,
1659 "partition-specs": [
1660 {
1661 "spec-id": 0,
1662 "fields": [
1663 {
1664 "source-id": 4,
1665 "field-id": 1000,
1666 "name": "ts_day",
1667 "transform": "day"
1668 }
1669 ]
1670 }
1671 ],
1672 "default-spec-id": 0,
1673 "last-partition-id": 1000,
1674 "properties": {
1675 "commit.retry.num-retries": "1"
1676 },
1677 "metadata-log": [
1678 {
1679 "metadata-file": "s3://bucket/.../v1.json",
1680 "timestamp-ms": 1515100
1681 }
1682 ],
1683 "refs": {},
1684 "sort-orders": [
1685 {
1686 "order-id": 0,
1687 "fields": []
1688 }
1689 ],
1690 "default-sort-order-id": 0
1691 }
1692 "#;
1693
1694 let schema = Schema::builder()
1695 .with_schema_id(1)
1696 .with_fields(vec![
1697 Arc::new(NestedField::required(
1698 1,
1699 "struct_name",
1700 Type::Primitive(PrimitiveType::Fixed(1)),
1701 )),
1702 Arc::new(NestedField::required(
1703 4,
1704 "ts",
1705 Type::Primitive(PrimitiveType::Timestamp),
1706 )),
1707 ])
1708 .build()
1709 .unwrap();
1710
1711 let partition_spec = PartitionSpec::builder(schema.clone())
1712 .with_spec_id(0)
1713 .add_unbound_field(
1714 UnboundPartitionField::builder()
1715 .source_ids(vec![4])
1716 .field_id(1000)
1717 .name("ts_day".to_string())
1718 .transform(Transform::Day)
1719 .build()
1720 .unwrap(),
1721 )
1722 .unwrap()
1723 .build()
1724 .unwrap();
1725
1726 let default_partition_type = partition_spec.partition_type(&schema).unwrap();
1727 let expected = TableMetadata {
1728 format_version: FormatVersion::V2,
1729 table_uuid: Uuid::parse_str("fb072c92-a02b-11e9-ae9c-1bb7bc9eca94").unwrap(),
1730 location: "s3://b/wh/data.db/table".to_string(),
1731 last_updated_ms: 1515100955770,
1732 last_column_id: 1,
1733 schemas: HashMap::from_iter(vec![(1, Arc::new(schema))]),
1734 current_schema_id: 1,
1735 partition_specs: HashMap::from_iter(vec![(0, partition_spec.clone().into())]),
1736 default_partition_type,
1737 default_spec: partition_spec.into(),
1738 last_partition_id: 1000,
1739 default_sort_order_id: 0,
1740 sort_orders: HashMap::from_iter(vec![(0, SortOrder::unsorted_order().into())]),
1741 snapshots: HashMap::default(),
1742 current_snapshot_id: None,
1743 last_sequence_number: 1,
1744 properties: HashMap::from_iter(vec![(
1745 "commit.retry.num-retries".to_string(),
1746 "1".to_string(),
1747 )]),
1748 snapshot_log: Vec::new(),
1749 metadata_log: vec![MetadataLog {
1750 metadata_file: "s3://bucket/.../v1.json".to_string(),
1751 timestamp_ms: 1515100,
1752 }],
1753 refs: HashMap::new(),
1754 statistics: HashMap::new(),
1755 partition_statistics: HashMap::new(),
1756 encryption_keys: HashMap::new(),
1757 next_row_id: INITIAL_ROW_ID,
1758 };
1759
1760 let expected_json_value = serde_json::to_value(&expected).unwrap();
1761 check_table_metadata_serde(data, expected);
1762
1763 let json_value = serde_json::from_str::<serde_json::Value>(data).unwrap();
1764 assert_eq!(json_value, expected_json_value);
1765 }
1766
1767 #[test]
1768 fn test_table_data_v3() {
1769 let data = r#"
1770 {
1771 "format-version" : 3,
1772 "table-uuid": "fb072c92-a02b-11e9-ae9c-1bb7bc9eca94",
1773 "location": "s3://b/wh/data.db/table",
1774 "last-sequence-number" : 1,
1775 "last-updated-ms": 1515100955770,
1776 "last-column-id": 1,
1777 "next-row-id": 5,
1778 "schemas": [
1779 {
1780 "schema-id" : 1,
1781 "type" : "struct",
1782 "fields" :[
1783 {
1784 "id": 4,
1785 "name": "ts",
1786 "required": true,
1787 "type": "timestamp"
1788 }
1789 ]
1790 }
1791 ],
1792 "current-schema-id" : 1,
1793 "partition-specs": [
1794 {
1795 "spec-id": 0,
1796 "fields": [
1797 {
1798 "source-id": 4,
1799 "field-id": 1000,
1800 "name": "ts_day",
1801 "transform": "day"
1802 }
1803 ]
1804 }
1805 ],
1806 "default-spec-id": 0,
1807 "last-partition-id": 1000,
1808 "properties": {
1809 "commit.retry.num-retries": "1"
1810 },
1811 "metadata-log": [
1812 {
1813 "metadata-file": "s3://bucket/.../v1.json",
1814 "timestamp-ms": 1515100
1815 }
1816 ],
1817 "refs": {},
1818 "snapshots" : [ {
1819 "snapshot-id" : 1,
1820 "timestamp-ms" : 1662532818843,
1821 "sequence-number" : 0,
1822 "first-row-id" : 0,
1823 "added-rows" : 4,
1824 "key-id" : "key1",
1825 "summary" : {
1826 "operation" : "append"
1827 },
1828 "manifest-list" : "/home/iceberg/warehouse/nyc/taxis/metadata/snap-638933773299822130-1-7e6760f0-4f6c-4b23-b907-0a5a174e3863.avro",
1829 "schema-id" : 0
1830 }
1831 ],
1832 "encryption-keys": [
1833 {
1834 "key-id": "key1",
1835 "encrypted-by-id": "KMS",
1836 "encrypted-key-metadata": "c29tZS1lbmNyeXB0aW9uLWtleQ==",
1837 "properties": {
1838 "p1": "v1"
1839 }
1840 }
1841 ],
1842 "sort-orders": [
1843 {
1844 "order-id": 0,
1845 "fields": []
1846 }
1847 ],
1848 "default-sort-order-id": 0
1849 }
1850 "#;
1851
1852 let schema = Schema::builder()
1853 .with_schema_id(1)
1854 .with_fields(vec![Arc::new(NestedField::required(
1855 4,
1856 "ts",
1857 Type::Primitive(PrimitiveType::Timestamp),
1858 ))])
1859 .build()
1860 .unwrap();
1861
1862 let partition_spec = PartitionSpec::builder(schema.clone())
1863 .with_spec_id(0)
1864 .add_unbound_field(
1865 UnboundPartitionField::builder()
1866 .source_ids(vec![4])
1867 .field_id(1000)
1868 .name("ts_day".to_string())
1869 .transform(Transform::Day)
1870 .build()
1871 .unwrap(),
1872 )
1873 .unwrap()
1874 .build()
1875 .unwrap();
1876
1877 let snapshot = Snapshot::builder()
1878 .with_snapshot_id(1)
1879 .with_timestamp_ms(1662532818843)
1880 .with_sequence_number(0)
1881 .with_row_range(0, 4)
1882 .with_encryption_key_id(Some("key1".to_string()))
1883 .with_summary(Summary {
1884 operation: Operation::Append,
1885 additional_properties: HashMap::new(),
1886 })
1887 .with_manifest_list("/home/iceberg/warehouse/nyc/taxis/metadata/snap-638933773299822130-1-7e6760f0-4f6c-4b23-b907-0a5a174e3863.avro".to_string())
1888 .with_schema_id(0)
1889 .build();
1890
1891 let encryption_key = EncryptedKey::builder()
1892 .key_id("key1".to_string())
1893 .encrypted_by_id("KMS".to_string())
1894 .encrypted_key_metadata(
1895 base64::prelude::BASE64_STANDARD
1896 .decode("c29tZS1lbmNyeXB0aW9uLWtleQ==")
1897 .unwrap(),
1898 )
1899 .properties(HashMap::from_iter(vec![(
1900 "p1".to_string(),
1901 "v1".to_string(),
1902 )]))
1903 .build();
1904
1905 let default_partition_type = partition_spec.partition_type(&schema).unwrap();
1906 let expected = TableMetadata {
1907 format_version: FormatVersion::V3,
1908 table_uuid: Uuid::parse_str("fb072c92-a02b-11e9-ae9c-1bb7bc9eca94").unwrap(),
1909 location: "s3://b/wh/data.db/table".to_string(),
1910 last_updated_ms: 1515100955770,
1911 last_column_id: 1,
1912 schemas: HashMap::from_iter(vec![(1, Arc::new(schema))]),
1913 current_schema_id: 1,
1914 partition_specs: HashMap::from_iter(vec![(0, partition_spec.clone().into())]),
1915 default_partition_type,
1916 default_spec: partition_spec.into(),
1917 last_partition_id: 1000,
1918 default_sort_order_id: 0,
1919 sort_orders: HashMap::from_iter(vec![(0, SortOrder::unsorted_order().into())]),
1920 snapshots: HashMap::from_iter(vec![(1, snapshot.into())]),
1921 current_snapshot_id: None,
1922 last_sequence_number: 1,
1923 properties: HashMap::from_iter(vec![(
1924 "commit.retry.num-retries".to_string(),
1925 "1".to_string(),
1926 )]),
1927 snapshot_log: Vec::new(),
1928 metadata_log: vec![MetadataLog {
1929 metadata_file: "s3://bucket/.../v1.json".to_string(),
1930 timestamp_ms: 1515100,
1931 }],
1932 refs: HashMap::new(),
1933 statistics: HashMap::new(),
1934 partition_statistics: HashMap::new(),
1935 encryption_keys: HashMap::from_iter(vec![("key1".to_string(), encryption_key)]),
1936 next_row_id: 5,
1937 };
1938
1939 let expected_json_value = serde_json::to_value(&expected).unwrap();
1940 check_table_metadata_serde(data, expected);
1941
1942 let json_value = serde_json::from_str::<serde_json::Value>(data).unwrap();
1943 assert_eq!(json_value, expected_json_value);
1944 }
1945
1946 #[test]
1947 fn test_table_data_v1() {
1948 let data = r#"
1949 {
1950 "format-version" : 1,
1951 "table-uuid" : "df838b92-0b32-465d-a44e-d39936e538b7",
1952 "location" : "/home/iceberg/warehouse/nyc/taxis",
1953 "last-updated-ms" : 1662532818843,
1954 "last-column-id" : 5,
1955 "schema" : {
1956 "type" : "struct",
1957 "schema-id" : 0,
1958 "fields" : [ {
1959 "id" : 1,
1960 "name" : "vendor_id",
1961 "required" : false,
1962 "type" : "long"
1963 }, {
1964 "id" : 2,
1965 "name" : "trip_id",
1966 "required" : false,
1967 "type" : "long"
1968 }, {
1969 "id" : 3,
1970 "name" : "trip_distance",
1971 "required" : false,
1972 "type" : "float"
1973 }, {
1974 "id" : 4,
1975 "name" : "fare_amount",
1976 "required" : false,
1977 "type" : "double"
1978 }, {
1979 "id" : 5,
1980 "name" : "store_and_fwd_flag",
1981 "required" : false,
1982 "type" : "string"
1983 } ]
1984 },
1985 "partition-spec" : [ {
1986 "name" : "vendor_id",
1987 "transform" : "identity",
1988 "source-id" : 1,
1989 "field-id" : 1000
1990 } ],
1991 "last-partition-id" : 1000,
1992 "default-sort-order-id" : 0,
1993 "sort-orders" : [ {
1994 "order-id" : 0,
1995 "fields" : [ ]
1996 } ],
1997 "properties" : {
1998 "owner" : "root"
1999 },
2000 "current-snapshot-id" : 638933773299822130,
2001 "refs" : {
2002 "main" : {
2003 "snapshot-id" : 638933773299822130,
2004 "type" : "branch"
2005 }
2006 },
2007 "snapshots" : [ {
2008 "snapshot-id" : 638933773299822130,
2009 "timestamp-ms" : 1662532818843,
2010 "sequence-number" : 0,
2011 "summary" : {
2012 "operation" : "append",
2013 "spark.app.id" : "local-1662532784305",
2014 "added-data-files" : "4",
2015 "added-records" : "4",
2016 "added-files-size" : "6001"
2017 },
2018 "manifest-list" : "/home/iceberg/warehouse/nyc/taxis/metadata/snap-638933773299822130-1-7e6760f0-4f6c-4b23-b907-0a5a174e3863.avro",
2019 "schema-id" : 0
2020 } ],
2021 "snapshot-log" : [ {
2022 "timestamp-ms" : 1662532818843,
2023 "snapshot-id" : 638933773299822130
2024 } ],
2025 "metadata-log" : [ {
2026 "timestamp-ms" : 1662532805245,
2027 "metadata-file" : "/home/iceberg/warehouse/nyc/taxis/metadata/00000-8a62c37d-4573-4021-952a-c0baef7d21d0.metadata.json"
2028 } ]
2029 }
2030 "#;
2031
2032 let schema = Schema::builder()
2033 .with_fields(vec![
2034 Arc::new(NestedField::optional(
2035 1,
2036 "vendor_id",
2037 Type::Primitive(PrimitiveType::Long),
2038 )),
2039 Arc::new(NestedField::optional(
2040 2,
2041 "trip_id",
2042 Type::Primitive(PrimitiveType::Long),
2043 )),
2044 Arc::new(NestedField::optional(
2045 3,
2046 "trip_distance",
2047 Type::Primitive(PrimitiveType::Float),
2048 )),
2049 Arc::new(NestedField::optional(
2050 4,
2051 "fare_amount",
2052 Type::Primitive(PrimitiveType::Double),
2053 )),
2054 Arc::new(NestedField::optional(
2055 5,
2056 "store_and_fwd_flag",
2057 Type::Primitive(PrimitiveType::String),
2058 )),
2059 ])
2060 .build()
2061 .unwrap();
2062
2063 let schema = Arc::new(schema);
2064 let partition_spec = PartitionSpec::builder(schema.clone())
2065 .with_spec_id(0)
2066 .add_partition_field("vendor_id", "vendor_id", Transform::Identity)
2067 .unwrap()
2068 .build()
2069 .unwrap();
2070
2071 let sort_order = SortOrder::builder()
2072 .with_order_id(0)
2073 .build_unbound()
2074 .unwrap();
2075
2076 let snapshot = Snapshot::builder()
2077 .with_snapshot_id(638933773299822130)
2078 .with_timestamp_ms(1662532818843)
2079 .with_sequence_number(0)
2080 .with_schema_id(0)
2081 .with_manifest_list("/home/iceberg/warehouse/nyc/taxis/metadata/snap-638933773299822130-1-7e6760f0-4f6c-4b23-b907-0a5a174e3863.avro")
2082 .with_summary(Summary { operation: Operation::Append, additional_properties: HashMap::from_iter(vec![("spark.app.id".to_string(), "local-1662532784305".to_string()), ("added-data-files".to_string(), "4".to_string()), ("added-records".to_string(), "4".to_string()), ("added-files-size".to_string(), "6001".to_string())]) })
2083 .build();
2084
2085 let default_partition_type = partition_spec.partition_type(&schema).unwrap();
2086 let expected = TableMetadata {
2087 format_version: FormatVersion::V1,
2088 table_uuid: Uuid::parse_str("df838b92-0b32-465d-a44e-d39936e538b7").unwrap(),
2089 location: "/home/iceberg/warehouse/nyc/taxis".to_string(),
2090 last_updated_ms: 1662532818843,
2091 last_column_id: 5,
2092 schemas: HashMap::from_iter(vec![(0, schema)]),
2093 current_schema_id: 0,
2094 partition_specs: HashMap::from_iter(vec![(0, partition_spec.clone().into())]),
2095 default_partition_type,
2096 default_spec: Arc::new(partition_spec),
2097 last_partition_id: 1000,
2098 default_sort_order_id: 0,
2099 sort_orders: HashMap::from_iter(vec![(0, sort_order.into())]),
2100 snapshots: HashMap::from_iter(vec![(638933773299822130, Arc::new(snapshot))]),
2101 current_snapshot_id: Some(638933773299822130),
2102 last_sequence_number: 0,
2103 properties: HashMap::from_iter(vec![("owner".to_string(), "root".to_string())]),
2104 snapshot_log: vec![SnapshotLog {
2105 snapshot_id: 638933773299822130,
2106 timestamp_ms: 1662532818843,
2107 }],
2108 metadata_log: vec![MetadataLog { metadata_file: "/home/iceberg/warehouse/nyc/taxis/metadata/00000-8a62c37d-4573-4021-952a-c0baef7d21d0.metadata.json".to_string(), timestamp_ms: 1662532805245 }],
2109 refs: HashMap::from_iter(vec![("main".to_string(), SnapshotReference { snapshot_id: 638933773299822130, retention: SnapshotRetention::Branch { min_snapshots_to_keep: None, max_snapshot_age_ms: None, max_ref_age_ms: None } })]),
2110 statistics: HashMap::new(),
2111 partition_statistics: HashMap::new(),
2112 encryption_keys: HashMap::new(),
2113 next_row_id: INITIAL_ROW_ID,
2114 };
2115
2116 check_table_metadata_serde(data, expected);
2117 }
2118
2119 #[test]
2120 fn test_table_data_v2_no_snapshots() {
2121 let data = r#"
2122 {
2123 "format-version" : 2,
2124 "table-uuid": "fb072c92-a02b-11e9-ae9c-1bb7bc9eca94",
2125 "location": "s3://b/wh/data.db/table",
2126 "last-sequence-number" : 1,
2127 "last-updated-ms": 1515100955770,
2128 "last-column-id": 1,
2129 "schemas": [
2130 {
2131 "schema-id" : 1,
2132 "type" : "struct",
2133 "fields" :[
2134 {
2135 "id": 1,
2136 "name": "struct_name",
2137 "required": true,
2138 "type": "fixed[1]"
2139 }
2140 ]
2141 }
2142 ],
2143 "current-schema-id" : 1,
2144 "partition-specs": [
2145 {
2146 "spec-id": 0,
2147 "fields": []
2148 }
2149 ],
2150 "refs": {},
2151 "default-spec-id": 0,
2152 "last-partition-id": 1000,
2153 "metadata-log": [
2154 {
2155 "metadata-file": "s3://bucket/.../v1.json",
2156 "timestamp-ms": 1515100
2157 }
2158 ],
2159 "sort-orders": [
2160 {
2161 "order-id": 0,
2162 "fields": []
2163 }
2164 ],
2165 "default-sort-order-id": 0
2166 }
2167 "#;
2168
2169 let schema = Schema::builder()
2170 .with_schema_id(1)
2171 .with_fields(vec![Arc::new(NestedField::required(
2172 1,
2173 "struct_name",
2174 Type::Primitive(PrimitiveType::Fixed(1)),
2175 ))])
2176 .build()
2177 .unwrap();
2178
2179 let partition_spec = PartitionSpec::builder(schema.clone())
2180 .with_spec_id(0)
2181 .build()
2182 .unwrap();
2183
2184 let default_partition_type = partition_spec.partition_type(&schema).unwrap();
2185 let expected = TableMetadata {
2186 format_version: FormatVersion::V2,
2187 table_uuid: Uuid::parse_str("fb072c92-a02b-11e9-ae9c-1bb7bc9eca94").unwrap(),
2188 location: "s3://b/wh/data.db/table".to_string(),
2189 last_updated_ms: 1515100955770,
2190 last_column_id: 1,
2191 schemas: HashMap::from_iter(vec![(1, Arc::new(schema))]),
2192 current_schema_id: 1,
2193 partition_specs: HashMap::from_iter(vec![(0, partition_spec.clone().into())]),
2194 default_partition_type,
2195 default_spec: partition_spec.into(),
2196 last_partition_id: 1000,
2197 default_sort_order_id: 0,
2198 sort_orders: HashMap::from_iter(vec![(0, SortOrder::unsorted_order().into())]),
2199 snapshots: HashMap::default(),
2200 current_snapshot_id: None,
2201 last_sequence_number: 1,
2202 properties: HashMap::new(),
2203 snapshot_log: Vec::new(),
2204 metadata_log: vec![MetadataLog {
2205 metadata_file: "s3://bucket/.../v1.json".to_string(),
2206 timestamp_ms: 1515100,
2207 }],
2208 refs: HashMap::new(),
2209 statistics: HashMap::new(),
2210 partition_statistics: HashMap::new(),
2211 encryption_keys: HashMap::new(),
2212 next_row_id: INITIAL_ROW_ID,
2213 };
2214
2215 let expected_json_value = serde_json::to_value(&expected).unwrap();
2216 check_table_metadata_serde(data, expected);
2217
2218 let json_value = serde_json::from_str::<serde_json::Value>(data).unwrap();
2219 assert_eq!(json_value, expected_json_value);
2220 }
2221
2222 #[test]
2223 fn test_current_snapshot_id_must_match_main_branch() {
2224 let data = r#"
2225 {
2226 "format-version" : 2,
2227 "table-uuid": "fb072c92-a02b-11e9-ae9c-1bb7bc9eca94",
2228 "location": "s3://b/wh/data.db/table",
2229 "last-sequence-number" : 1,
2230 "last-updated-ms": 1515100955770,
2231 "last-column-id": 1,
2232 "schemas": [
2233 {
2234 "schema-id" : 1,
2235 "type" : "struct",
2236 "fields" :[
2237 {
2238 "id": 1,
2239 "name": "struct_name",
2240 "required": true,
2241 "type": "fixed[1]"
2242 },
2243 {
2244 "id": 4,
2245 "name": "ts",
2246 "required": true,
2247 "type": "timestamp"
2248 }
2249 ]
2250 }
2251 ],
2252 "current-schema-id" : 1,
2253 "partition-specs": [
2254 {
2255 "spec-id": 0,
2256 "fields": [
2257 {
2258 "source-id": 4,
2259 "field-id": 1000,
2260 "name": "ts_day",
2261 "transform": "day"
2262 }
2263 ]
2264 }
2265 ],
2266 "default-spec-id": 0,
2267 "last-partition-id": 1000,
2268 "properties": {
2269 "commit.retry.num-retries": "1"
2270 },
2271 "metadata-log": [
2272 {
2273 "metadata-file": "s3://bucket/.../v1.json",
2274 "timestamp-ms": 1515100
2275 }
2276 ],
2277 "sort-orders": [
2278 {
2279 "order-id": 0,
2280 "fields": []
2281 }
2282 ],
2283 "default-sort-order-id": 0,
2284 "current-snapshot-id" : 1,
2285 "refs" : {
2286 "main" : {
2287 "snapshot-id" : 2,
2288 "type" : "branch"
2289 }
2290 },
2291 "snapshots" : [ {
2292 "snapshot-id" : 1,
2293 "timestamp-ms" : 1662532818843,
2294 "sequence-number" : 0,
2295 "summary" : {
2296 "operation" : "append",
2297 "spark.app.id" : "local-1662532784305",
2298 "added-data-files" : "4",
2299 "added-records" : "4",
2300 "added-files-size" : "6001"
2301 },
2302 "manifest-list" : "/home/iceberg/warehouse/nyc/taxis/metadata/snap-638933773299822130-1-7e6760f0-4f6c-4b23-b907-0a5a174e3863.avro",
2303 "schema-id" : 0
2304 },
2305 {
2306 "snapshot-id" : 2,
2307 "timestamp-ms" : 1662532818844,
2308 "sequence-number" : 0,
2309 "summary" : {
2310 "operation" : "append",
2311 "spark.app.id" : "local-1662532784305",
2312 "added-data-files" : "4",
2313 "added-records" : "4",
2314 "added-files-size" : "6001"
2315 },
2316 "manifest-list" : "/home/iceberg/warehouse/nyc/taxis/metadata/snap-638933773299822130-1-7e6760f0-4f6c-4b23-b907-0a5a174e3863.avro",
2317 "schema-id" : 0
2318 } ]
2319 }
2320 "#;
2321
2322 let err = serde_json::from_str::<TableMetadata>(data).unwrap_err();
2323 assert!(
2324 err.to_string()
2325 .contains("Current snapshot id does not match main branch")
2326 );
2327 }
2328
2329 #[test]
2330 fn test_main_without_current() {
2331 let data = r#"
2332 {
2333 "format-version" : 2,
2334 "table-uuid": "fb072c92-a02b-11e9-ae9c-1bb7bc9eca94",
2335 "location": "s3://b/wh/data.db/table",
2336 "last-sequence-number" : 1,
2337 "last-updated-ms": 1515100955770,
2338 "last-column-id": 1,
2339 "schemas": [
2340 {
2341 "schema-id" : 1,
2342 "type" : "struct",
2343 "fields" :[
2344 {
2345 "id": 1,
2346 "name": "struct_name",
2347 "required": true,
2348 "type": "fixed[1]"
2349 },
2350 {
2351 "id": 4,
2352 "name": "ts",
2353 "required": true,
2354 "type": "timestamp"
2355 }
2356 ]
2357 }
2358 ],
2359 "current-schema-id" : 1,
2360 "partition-specs": [
2361 {
2362 "spec-id": 0,
2363 "fields": [
2364 {
2365 "source-id": 4,
2366 "field-id": 1000,
2367 "name": "ts_day",
2368 "transform": "day"
2369 }
2370 ]
2371 }
2372 ],
2373 "default-spec-id": 0,
2374 "last-partition-id": 1000,
2375 "properties": {
2376 "commit.retry.num-retries": "1"
2377 },
2378 "metadata-log": [
2379 {
2380 "metadata-file": "s3://bucket/.../v1.json",
2381 "timestamp-ms": 1515100
2382 }
2383 ],
2384 "sort-orders": [
2385 {
2386 "order-id": 0,
2387 "fields": []
2388 }
2389 ],
2390 "default-sort-order-id": 0,
2391 "refs" : {
2392 "main" : {
2393 "snapshot-id" : 1,
2394 "type" : "branch"
2395 }
2396 },
2397 "snapshots" : [ {
2398 "snapshot-id" : 1,
2399 "timestamp-ms" : 1662532818843,
2400 "sequence-number" : 0,
2401 "summary" : {
2402 "operation" : "append",
2403 "spark.app.id" : "local-1662532784305",
2404 "added-data-files" : "4",
2405 "added-records" : "4",
2406 "added-files-size" : "6001"
2407 },
2408 "manifest-list" : "/home/iceberg/warehouse/nyc/taxis/metadata/snap-638933773299822130-1-7e6760f0-4f6c-4b23-b907-0a5a174e3863.avro",
2409 "schema-id" : 0
2410 } ]
2411 }
2412 "#;
2413
2414 let err = serde_json::from_str::<TableMetadata>(data).unwrap_err();
2415 assert!(
2416 err.to_string()
2417 .contains("Current snapshot is not set, but main branch exists")
2418 );
2419 }
2420
2421 #[test]
2422 fn test_branch_snapshot_missing() {
2423 let data = r#"
2424 {
2425 "format-version" : 2,
2426 "table-uuid": "fb072c92-a02b-11e9-ae9c-1bb7bc9eca94",
2427 "location": "s3://b/wh/data.db/table",
2428 "last-sequence-number" : 1,
2429 "last-updated-ms": 1515100955770,
2430 "last-column-id": 1,
2431 "schemas": [
2432 {
2433 "schema-id" : 1,
2434 "type" : "struct",
2435 "fields" :[
2436 {
2437 "id": 1,
2438 "name": "struct_name",
2439 "required": true,
2440 "type": "fixed[1]"
2441 },
2442 {
2443 "id": 4,
2444 "name": "ts",
2445 "required": true,
2446 "type": "timestamp"
2447 }
2448 ]
2449 }
2450 ],
2451 "current-schema-id" : 1,
2452 "partition-specs": [
2453 {
2454 "spec-id": 0,
2455 "fields": [
2456 {
2457 "source-id": 4,
2458 "field-id": 1000,
2459 "name": "ts_day",
2460 "transform": "day"
2461 }
2462 ]
2463 }
2464 ],
2465 "default-spec-id": 0,
2466 "last-partition-id": 1000,
2467 "properties": {
2468 "commit.retry.num-retries": "1"
2469 },
2470 "metadata-log": [
2471 {
2472 "metadata-file": "s3://bucket/.../v1.json",
2473 "timestamp-ms": 1515100
2474 }
2475 ],
2476 "sort-orders": [
2477 {
2478 "order-id": 0,
2479 "fields": []
2480 }
2481 ],
2482 "default-sort-order-id": 0,
2483 "refs" : {
2484 "main" : {
2485 "snapshot-id" : 1,
2486 "type" : "branch"
2487 },
2488 "foo" : {
2489 "snapshot-id" : 2,
2490 "type" : "branch"
2491 }
2492 },
2493 "snapshots" : [ {
2494 "snapshot-id" : 1,
2495 "timestamp-ms" : 1662532818843,
2496 "sequence-number" : 0,
2497 "summary" : {
2498 "operation" : "append",
2499 "spark.app.id" : "local-1662532784305",
2500 "added-data-files" : "4",
2501 "added-records" : "4",
2502 "added-files-size" : "6001"
2503 },
2504 "manifest-list" : "/home/iceberg/warehouse/nyc/taxis/metadata/snap-638933773299822130-1-7e6760f0-4f6c-4b23-b907-0a5a174e3863.avro",
2505 "schema-id" : 0
2506 } ]
2507 }
2508 "#;
2509
2510 let err = serde_json::from_str::<TableMetadata>(data).unwrap_err();
2511 assert!(
2512 err.to_string().contains(
2513 "Snapshot for reference foo does not exist in the existing snapshots list"
2514 )
2515 );
2516 }
2517
2518 #[test]
2519 fn test_v2_wrong_max_snapshot_sequence_number() {
2520 let data = r#"
2521 {
2522 "format-version": 2,
2523 "table-uuid": "9c12d441-03fe-4693-9a96-a0705ddf69c1",
2524 "location": "s3://bucket/test/location",
2525 "last-sequence-number": 1,
2526 "last-updated-ms": 1602638573590,
2527 "last-column-id": 3,
2528 "current-schema-id": 0,
2529 "schemas": [
2530 {
2531 "type": "struct",
2532 "schema-id": 0,
2533 "fields": [
2534 {
2535 "id": 1,
2536 "name": "x",
2537 "required": true,
2538 "type": "long"
2539 }
2540 ]
2541 }
2542 ],
2543 "default-spec-id": 0,
2544 "partition-specs": [
2545 {
2546 "spec-id": 0,
2547 "fields": []
2548 }
2549 ],
2550 "last-partition-id": 1000,
2551 "default-sort-order-id": 0,
2552 "sort-orders": [
2553 {
2554 "order-id": 0,
2555 "fields": []
2556 }
2557 ],
2558 "properties": {},
2559 "current-snapshot-id": 3055729675574597004,
2560 "snapshots": [
2561 {
2562 "snapshot-id": 3055729675574597004,
2563 "timestamp-ms": 1555100955770,
2564 "sequence-number": 4,
2565 "summary": {
2566 "operation": "append"
2567 },
2568 "manifest-list": "s3://a/b/2.avro",
2569 "schema-id": 0
2570 }
2571 ],
2572 "statistics": [],
2573 "snapshot-log": [],
2574 "metadata-log": []
2575 }
2576 "#;
2577
2578 let err = serde_json::from_str::<TableMetadata>(data).unwrap_err();
2579 assert!(err.to_string().contains(
2580 "Invalid snapshot with id 3055729675574597004 and sequence number 4 greater than last sequence number 1"
2581 ));
2582
2583 let data = data.replace(
2585 r#""last-sequence-number": 1,"#,
2586 r#""last-sequence-number": 4,"#,
2587 );
2588 let metadata = serde_json::from_str::<TableMetadata>(data.as_str()).unwrap();
2589 assert_eq!(metadata.last_sequence_number, 4);
2590
2591 let data = data.replace(
2593 r#""last-sequence-number": 4,"#,
2594 r#""last-sequence-number": 5,"#,
2595 );
2596 let metadata = serde_json::from_str::<TableMetadata>(data.as_str()).unwrap();
2597 assert_eq!(metadata.last_sequence_number, 5);
2598 }
2599
2600 #[test]
2601 fn test_statistic_files() {
2602 let data = r#"
2603 {
2604 "format-version": 2,
2605 "table-uuid": "9c12d441-03fe-4693-9a96-a0705ddf69c1",
2606 "location": "s3://bucket/test/location",
2607 "last-sequence-number": 34,
2608 "last-updated-ms": 1602638573590,
2609 "last-column-id": 3,
2610 "current-schema-id": 0,
2611 "schemas": [
2612 {
2613 "type": "struct",
2614 "schema-id": 0,
2615 "fields": [
2616 {
2617 "id": 1,
2618 "name": "x",
2619 "required": true,
2620 "type": "long"
2621 }
2622 ]
2623 }
2624 ],
2625 "default-spec-id": 0,
2626 "partition-specs": [
2627 {
2628 "spec-id": 0,
2629 "fields": []
2630 }
2631 ],
2632 "last-partition-id": 1000,
2633 "default-sort-order-id": 0,
2634 "sort-orders": [
2635 {
2636 "order-id": 0,
2637 "fields": []
2638 }
2639 ],
2640 "properties": {},
2641 "current-snapshot-id": 3055729675574597004,
2642 "snapshots": [
2643 {
2644 "snapshot-id": 3055729675574597004,
2645 "timestamp-ms": 1555100955770,
2646 "sequence-number": 1,
2647 "summary": {
2648 "operation": "append"
2649 },
2650 "manifest-list": "s3://a/b/2.avro",
2651 "schema-id": 0
2652 }
2653 ],
2654 "statistics": [
2655 {
2656 "snapshot-id": 3055729675574597004,
2657 "statistics-path": "s3://a/b/stats.puffin",
2658 "file-size-in-bytes": 413,
2659 "file-footer-size-in-bytes": 42,
2660 "blob-metadata": [
2661 {
2662 "type": "ndv",
2663 "snapshot-id": 3055729675574597004,
2664 "sequence-number": 1,
2665 "fields": [
2666 1
2667 ]
2668 }
2669 ]
2670 }
2671 ],
2672 "snapshot-log": [],
2673 "metadata-log": []
2674 }
2675 "#;
2676
2677 let schema = Schema::builder()
2678 .with_schema_id(0)
2679 .with_fields(vec![Arc::new(NestedField::required(
2680 1,
2681 "x",
2682 Type::Primitive(PrimitiveType::Long),
2683 ))])
2684 .build()
2685 .unwrap();
2686 let partition_spec = PartitionSpec::builder(schema.clone())
2687 .with_spec_id(0)
2688 .build()
2689 .unwrap();
2690 let snapshot = Snapshot::builder()
2691 .with_snapshot_id(3055729675574597004)
2692 .with_timestamp_ms(1555100955770)
2693 .with_sequence_number(1)
2694 .with_manifest_list("s3://a/b/2.avro")
2695 .with_schema_id(0)
2696 .with_summary(Summary {
2697 operation: Operation::Append,
2698 additional_properties: HashMap::new(),
2699 })
2700 .build();
2701
2702 let default_partition_type = partition_spec.partition_type(&schema).unwrap();
2703 let expected = TableMetadata {
2704 format_version: FormatVersion::V2,
2705 table_uuid: Uuid::parse_str("9c12d441-03fe-4693-9a96-a0705ddf69c1").unwrap(),
2706 location: "s3://bucket/test/location".to_string(),
2707 last_updated_ms: 1602638573590,
2708 last_column_id: 3,
2709 schemas: HashMap::from_iter(vec![(0, Arc::new(schema))]),
2710 current_schema_id: 0,
2711 partition_specs: HashMap::from_iter(vec![(0, partition_spec.clone().into())]),
2712 default_partition_type,
2713 default_spec: Arc::new(partition_spec),
2714 last_partition_id: 1000,
2715 default_sort_order_id: 0,
2716 sort_orders: HashMap::from_iter(vec![(0, SortOrder::unsorted_order().into())]),
2717 snapshots: HashMap::from_iter(vec![(3055729675574597004, Arc::new(snapshot))]),
2718 current_snapshot_id: Some(3055729675574597004),
2719 last_sequence_number: 34,
2720 properties: HashMap::new(),
2721 snapshot_log: Vec::new(),
2722 metadata_log: Vec::new(),
2723 statistics: HashMap::from_iter(vec![(3055729675574597004, StatisticsFile {
2724 snapshot_id: 3055729675574597004,
2725 statistics_path: "s3://a/b/stats.puffin".to_string(),
2726 file_size_in_bytes: 413,
2727 file_footer_size_in_bytes: 42,
2728 key_metadata: None,
2729 blob_metadata: vec![BlobMetadata {
2730 snapshot_id: 3055729675574597004,
2731 sequence_number: 1,
2732 fields: vec![1],
2733 r#type: "ndv".to_string(),
2734 properties: HashMap::new(),
2735 }],
2736 })]),
2737 partition_statistics: HashMap::new(),
2738 refs: HashMap::from_iter(vec![("main".to_string(), SnapshotReference {
2739 snapshot_id: 3055729675574597004,
2740 retention: SnapshotRetention::Branch {
2741 min_snapshots_to_keep: None,
2742 max_snapshot_age_ms: None,
2743 max_ref_age_ms: None,
2744 },
2745 })]),
2746 encryption_keys: HashMap::new(),
2747 next_row_id: INITIAL_ROW_ID,
2748 };
2749
2750 check_table_metadata_serde(data, expected);
2751 }
2752
2753 #[test]
2754 fn test_partition_statistics_file() {
2755 let data = r#"
2756 {
2757 "format-version": 2,
2758 "table-uuid": "9c12d441-03fe-4693-9a96-a0705ddf69c1",
2759 "location": "s3://bucket/test/location",
2760 "last-sequence-number": 34,
2761 "last-updated-ms": 1602638573590,
2762 "last-column-id": 3,
2763 "current-schema-id": 0,
2764 "schemas": [
2765 {
2766 "type": "struct",
2767 "schema-id": 0,
2768 "fields": [
2769 {
2770 "id": 1,
2771 "name": "x",
2772 "required": true,
2773 "type": "long"
2774 }
2775 ]
2776 }
2777 ],
2778 "default-spec-id": 0,
2779 "partition-specs": [
2780 {
2781 "spec-id": 0,
2782 "fields": []
2783 }
2784 ],
2785 "last-partition-id": 1000,
2786 "default-sort-order-id": 0,
2787 "sort-orders": [
2788 {
2789 "order-id": 0,
2790 "fields": []
2791 }
2792 ],
2793 "properties": {},
2794 "current-snapshot-id": 3055729675574597004,
2795 "snapshots": [
2796 {
2797 "snapshot-id": 3055729675574597004,
2798 "timestamp-ms": 1555100955770,
2799 "sequence-number": 1,
2800 "summary": {
2801 "operation": "append"
2802 },
2803 "manifest-list": "s3://a/b/2.avro",
2804 "schema-id": 0
2805 }
2806 ],
2807 "partition-statistics": [
2808 {
2809 "snapshot-id": 3055729675574597004,
2810 "statistics-path": "s3://a/b/partition-stats.parquet",
2811 "file-size-in-bytes": 43
2812 }
2813 ],
2814 "snapshot-log": [],
2815 "metadata-log": []
2816 }
2817 "#;
2818
2819 let schema = Schema::builder()
2820 .with_schema_id(0)
2821 .with_fields(vec![Arc::new(NestedField::required(
2822 1,
2823 "x",
2824 Type::Primitive(PrimitiveType::Long),
2825 ))])
2826 .build()
2827 .unwrap();
2828 let partition_spec = PartitionSpec::builder(schema.clone())
2829 .with_spec_id(0)
2830 .build()
2831 .unwrap();
2832 let snapshot = Snapshot::builder()
2833 .with_snapshot_id(3055729675574597004)
2834 .with_timestamp_ms(1555100955770)
2835 .with_sequence_number(1)
2836 .with_manifest_list("s3://a/b/2.avro")
2837 .with_schema_id(0)
2838 .with_summary(Summary {
2839 operation: Operation::Append,
2840 additional_properties: HashMap::new(),
2841 })
2842 .build();
2843
2844 let default_partition_type = partition_spec.partition_type(&schema).unwrap();
2845 let expected = TableMetadata {
2846 format_version: FormatVersion::V2,
2847 table_uuid: Uuid::parse_str("9c12d441-03fe-4693-9a96-a0705ddf69c1").unwrap(),
2848 location: "s3://bucket/test/location".to_string(),
2849 last_updated_ms: 1602638573590,
2850 last_column_id: 3,
2851 schemas: HashMap::from_iter(vec![(0, Arc::new(schema))]),
2852 current_schema_id: 0,
2853 partition_specs: HashMap::from_iter(vec![(0, partition_spec.clone().into())]),
2854 default_spec: Arc::new(partition_spec),
2855 default_partition_type,
2856 last_partition_id: 1000,
2857 default_sort_order_id: 0,
2858 sort_orders: HashMap::from_iter(vec![(0, SortOrder::unsorted_order().into())]),
2859 snapshots: HashMap::from_iter(vec![(3055729675574597004, Arc::new(snapshot))]),
2860 current_snapshot_id: Some(3055729675574597004),
2861 last_sequence_number: 34,
2862 properties: HashMap::new(),
2863 snapshot_log: Vec::new(),
2864 metadata_log: Vec::new(),
2865 statistics: HashMap::new(),
2866 partition_statistics: HashMap::from_iter(vec![(
2867 3055729675574597004,
2868 PartitionStatisticsFile {
2869 snapshot_id: 3055729675574597004,
2870 statistics_path: "s3://a/b/partition-stats.parquet".to_string(),
2871 file_size_in_bytes: 43,
2872 },
2873 )]),
2874 refs: HashMap::from_iter(vec![("main".to_string(), SnapshotReference {
2875 snapshot_id: 3055729675574597004,
2876 retention: SnapshotRetention::Branch {
2877 min_snapshots_to_keep: None,
2878 max_snapshot_age_ms: None,
2879 max_ref_age_ms: None,
2880 },
2881 })]),
2882 encryption_keys: HashMap::new(),
2883 next_row_id: INITIAL_ROW_ID,
2884 };
2885
2886 check_table_metadata_serde(data, expected);
2887 }
2888
2889 #[test]
2890 fn test_invalid_table_uuid() -> Result<()> {
2891 let data = r#"
2892 {
2893 "format-version" : 2,
2894 "table-uuid": "xxxx"
2895 }
2896 "#;
2897 assert!(serde_json::from_str::<TableMetadata>(data).is_err());
2898 Ok(())
2899 }
2900
2901 #[test]
2902 fn test_deserialize_table_data_v2_invalid_format_version() -> Result<()> {
2903 let data = r#"
2904 {
2905 "format-version" : 1
2906 }
2907 "#;
2908 assert!(serde_json::from_str::<TableMetadata>(data).is_err());
2909 Ok(())
2910 }
2911
2912 #[test]
2913 fn test_table_metadata_v3_valid_minimal() {
2914 let metadata_str =
2915 fs::read_to_string("testdata/table_metadata/TableMetadataV3ValidMinimal.json").unwrap();
2916
2917 let table_metadata = serde_json::from_str::<TableMetadata>(&metadata_str).unwrap();
2918 assert_eq!(table_metadata.format_version, FormatVersion::V3);
2919
2920 let schema = Schema::builder()
2921 .with_schema_id(0)
2922 .with_fields(vec![
2923 Arc::new(
2924 NestedField::required(1, "x", Type::Primitive(PrimitiveType::Long))
2925 .with_initial_default(Literal::Primitive(PrimitiveLiteral::Long(1)))
2926 .with_write_default(Literal::Primitive(PrimitiveLiteral::Long(1))),
2927 ),
2928 Arc::new(
2929 NestedField::required(2, "y", Type::Primitive(PrimitiveType::Long))
2930 .with_doc("comment"),
2931 ),
2932 Arc::new(NestedField::required(
2933 3,
2934 "z",
2935 Type::Primitive(PrimitiveType::Long),
2936 )),
2937 ])
2938 .build()
2939 .unwrap();
2940
2941 let partition_spec = PartitionSpec::builder(schema.clone())
2942 .with_spec_id(0)
2943 .add_unbound_field(
2944 UnboundPartitionField::builder()
2945 .source_ids(vec![1])
2946 .field_id(1000)
2947 .name("x".to_string())
2948 .transform(Transform::Identity)
2949 .build()
2950 .unwrap(),
2951 )
2952 .unwrap()
2953 .build()
2954 .unwrap();
2955
2956 let sort_order = SortOrder::builder()
2957 .with_order_id(3)
2958 .with_sort_field(SortField {
2959 source_id: 2,
2960 transform: Transform::Identity,
2961 direction: SortDirection::Ascending,
2962 null_order: NullOrder::First,
2963 })
2964 .with_sort_field(SortField {
2965 source_id: 3,
2966 transform: Transform::Bucket(4),
2967 direction: SortDirection::Descending,
2968 null_order: NullOrder::Last,
2969 })
2970 .build_unbound()
2971 .unwrap();
2972
2973 let default_partition_type = partition_spec.partition_type(&schema).unwrap();
2974 let expected = TableMetadata {
2975 format_version: FormatVersion::V3,
2976 table_uuid: Uuid::parse_str("9c12d441-03fe-4693-9a96-a0705ddf69c1").unwrap(),
2977 location: "s3://bucket/test/location".to_string(),
2978 last_updated_ms: 1602638573590,
2979 last_column_id: 3,
2980 schemas: HashMap::from_iter(vec![(0, Arc::new(schema))]),
2981 current_schema_id: 0,
2982 partition_specs: HashMap::from_iter(vec![(0, partition_spec.clone().into())]),
2983 default_spec: Arc::new(partition_spec),
2984 default_partition_type,
2985 last_partition_id: 1000,
2986 default_sort_order_id: 3,
2987 sort_orders: HashMap::from_iter(vec![(3, sort_order.into())]),
2988 snapshots: HashMap::default(),
2989 current_snapshot_id: None,
2990 last_sequence_number: 34,
2991 properties: HashMap::new(),
2992 snapshot_log: Vec::new(),
2993 metadata_log: Vec::new(),
2994 refs: HashMap::new(),
2995 statistics: HashMap::new(),
2996 partition_statistics: HashMap::new(),
2997 encryption_keys: HashMap::new(),
2998 next_row_id: 0, };
3000
3001 check_table_metadata_serde(&metadata_str, expected);
3002 }
3003
3004 #[test]
3005 fn test_table_metadata_v2_file_valid() {
3006 let metadata =
3007 fs::read_to_string("testdata/table_metadata/TableMetadataV2Valid.json").unwrap();
3008
3009 let schema1 = Schema::builder()
3010 .with_schema_id(0)
3011 .with_fields(vec![Arc::new(NestedField::required(
3012 1,
3013 "x",
3014 Type::Primitive(PrimitiveType::Long),
3015 ))])
3016 .build()
3017 .unwrap();
3018
3019 let schema2 = Schema::builder()
3020 .with_schema_id(1)
3021 .with_fields(vec![
3022 Arc::new(NestedField::required(
3023 1,
3024 "x",
3025 Type::Primitive(PrimitiveType::Long),
3026 )),
3027 Arc::new(
3028 NestedField::required(2, "y", Type::Primitive(PrimitiveType::Long))
3029 .with_doc("comment"),
3030 ),
3031 Arc::new(NestedField::required(
3032 3,
3033 "z",
3034 Type::Primitive(PrimitiveType::Long),
3035 )),
3036 ])
3037 .with_identifier_field_ids(vec![1, 2])
3038 .build()
3039 .unwrap();
3040
3041 let partition_spec = PartitionSpec::builder(schema2.clone())
3042 .with_spec_id(0)
3043 .add_unbound_field(
3044 UnboundPartitionField::builder()
3045 .source_ids(vec![1])
3046 .field_id(1000)
3047 .name("x".to_string())
3048 .transform(Transform::Identity)
3049 .build()
3050 .unwrap(),
3051 )
3052 .unwrap()
3053 .build()
3054 .unwrap();
3055
3056 let sort_order = SortOrder::builder()
3057 .with_order_id(3)
3058 .with_sort_field(SortField {
3059 source_id: 2,
3060 transform: Transform::Identity,
3061 direction: SortDirection::Ascending,
3062 null_order: NullOrder::First,
3063 })
3064 .with_sort_field(SortField {
3065 source_id: 3,
3066 transform: Transform::Bucket(4),
3067 direction: SortDirection::Descending,
3068 null_order: NullOrder::Last,
3069 })
3070 .build_unbound()
3071 .unwrap();
3072
3073 let snapshot1 = Snapshot::builder()
3074 .with_snapshot_id(3051729675574597004)
3075 .with_timestamp_ms(1515100955770)
3076 .with_sequence_number(0)
3077 .with_manifest_list("s3://a/b/1.avro")
3078 .with_summary(Summary {
3079 operation: Operation::Append,
3080 additional_properties: HashMap::new(),
3081 })
3082 .build();
3083
3084 let snapshot2 = Snapshot::builder()
3085 .with_snapshot_id(3055729675574597004)
3086 .with_parent_snapshot_id(Some(3051729675574597004))
3087 .with_timestamp_ms(1555100955770)
3088 .with_sequence_number(1)
3089 .with_schema_id(1)
3090 .with_manifest_list("s3://a/b/2.avro")
3091 .with_summary(Summary {
3092 operation: Operation::Append,
3093 additional_properties: HashMap::new(),
3094 })
3095 .build();
3096
3097 let default_partition_type = partition_spec.partition_type(&schema2).unwrap();
3098 let expected = TableMetadata {
3099 format_version: FormatVersion::V2,
3100 table_uuid: Uuid::parse_str("9c12d441-03fe-4693-9a96-a0705ddf69c1").unwrap(),
3101 location: "s3://bucket/test/location".to_string(),
3102 last_updated_ms: 1602638573590,
3103 last_column_id: 3,
3104 schemas: HashMap::from_iter(vec![(0, Arc::new(schema1)), (1, Arc::new(schema2))]),
3105 current_schema_id: 1,
3106 partition_specs: HashMap::from_iter(vec![(0, partition_spec.clone().into())]),
3107 default_spec: Arc::new(partition_spec),
3108 default_partition_type,
3109 last_partition_id: 1000,
3110 default_sort_order_id: 3,
3111 sort_orders: HashMap::from_iter(vec![(3, sort_order.into())]),
3112 snapshots: HashMap::from_iter(vec![
3113 (3051729675574597004, Arc::new(snapshot1)),
3114 (3055729675574597004, Arc::new(snapshot2)),
3115 ]),
3116 current_snapshot_id: Some(3055729675574597004),
3117 last_sequence_number: 34,
3118 properties: HashMap::new(),
3119 snapshot_log: vec![
3120 SnapshotLog {
3121 snapshot_id: 3051729675574597004,
3122 timestamp_ms: 1515100955770,
3123 },
3124 SnapshotLog {
3125 snapshot_id: 3055729675574597004,
3126 timestamp_ms: 1555100955770,
3127 },
3128 ],
3129 metadata_log: Vec::new(),
3130 refs: HashMap::from_iter(vec![("main".to_string(), SnapshotReference {
3131 snapshot_id: 3055729675574597004,
3132 retention: SnapshotRetention::Branch {
3133 min_snapshots_to_keep: None,
3134 max_snapshot_age_ms: None,
3135 max_ref_age_ms: None,
3136 },
3137 })]),
3138 statistics: HashMap::new(),
3139 partition_statistics: HashMap::new(),
3140 encryption_keys: HashMap::new(),
3141 next_row_id: INITIAL_ROW_ID,
3142 };
3143
3144 check_table_metadata_serde(&metadata, expected);
3145 }
3146
3147 #[test]
3148 fn test_table_metadata_v2_file_valid_minimal() {
3149 let metadata =
3150 fs::read_to_string("testdata/table_metadata/TableMetadataV2ValidMinimal.json").unwrap();
3151
3152 let schema = Schema::builder()
3153 .with_schema_id(0)
3154 .with_fields(vec![
3155 Arc::new(NestedField::required(
3156 1,
3157 "x",
3158 Type::Primitive(PrimitiveType::Long),
3159 )),
3160 Arc::new(
3161 NestedField::required(2, "y", Type::Primitive(PrimitiveType::Long))
3162 .with_doc("comment"),
3163 ),
3164 Arc::new(NestedField::required(
3165 3,
3166 "z",
3167 Type::Primitive(PrimitiveType::Long),
3168 )),
3169 ])
3170 .build()
3171 .unwrap();
3172
3173 let partition_spec = PartitionSpec::builder(schema.clone())
3174 .with_spec_id(0)
3175 .add_unbound_field(
3176 UnboundPartitionField::builder()
3177 .source_ids(vec![1])
3178 .field_id(1000)
3179 .name("x".to_string())
3180 .transform(Transform::Identity)
3181 .build()
3182 .unwrap(),
3183 )
3184 .unwrap()
3185 .build()
3186 .unwrap();
3187
3188 let sort_order = SortOrder::builder()
3189 .with_order_id(3)
3190 .with_sort_field(SortField {
3191 source_id: 2,
3192 transform: Transform::Identity,
3193 direction: SortDirection::Ascending,
3194 null_order: NullOrder::First,
3195 })
3196 .with_sort_field(SortField {
3197 source_id: 3,
3198 transform: Transform::Bucket(4),
3199 direction: SortDirection::Descending,
3200 null_order: NullOrder::Last,
3201 })
3202 .build_unbound()
3203 .unwrap();
3204
3205 let default_partition_type = partition_spec.partition_type(&schema).unwrap();
3206 let expected = TableMetadata {
3207 format_version: FormatVersion::V2,
3208 table_uuid: Uuid::parse_str("9c12d441-03fe-4693-9a96-a0705ddf69c1").unwrap(),
3209 location: "s3://bucket/test/location".to_string(),
3210 last_updated_ms: 1602638573590,
3211 last_column_id: 3,
3212 schemas: HashMap::from_iter(vec![(0, Arc::new(schema))]),
3213 current_schema_id: 0,
3214 partition_specs: HashMap::from_iter(vec![(0, partition_spec.clone().into())]),
3215 default_partition_type,
3216 default_spec: Arc::new(partition_spec),
3217 last_partition_id: 1000,
3218 default_sort_order_id: 3,
3219 sort_orders: HashMap::from_iter(vec![(3, sort_order.into())]),
3220 snapshots: HashMap::default(),
3221 current_snapshot_id: None,
3222 last_sequence_number: 34,
3223 properties: HashMap::new(),
3224 snapshot_log: vec![],
3225 metadata_log: Vec::new(),
3226 refs: HashMap::new(),
3227 statistics: HashMap::new(),
3228 partition_statistics: HashMap::new(),
3229 encryption_keys: HashMap::new(),
3230 next_row_id: INITIAL_ROW_ID,
3231 };
3232
3233 check_table_metadata_serde(&metadata, expected);
3234 }
3235
3236 #[test]
3237 fn test_table_metadata_v1_file_valid() {
3238 let metadata =
3239 fs::read_to_string("testdata/table_metadata/TableMetadataV1Valid.json").unwrap();
3240
3241 let schema = Schema::builder()
3242 .with_schema_id(0)
3243 .with_fields(vec![
3244 Arc::new(NestedField::required(
3245 1,
3246 "x",
3247 Type::Primitive(PrimitiveType::Long),
3248 )),
3249 Arc::new(
3250 NestedField::required(2, "y", Type::Primitive(PrimitiveType::Long))
3251 .with_doc("comment"),
3252 ),
3253 Arc::new(NestedField::required(
3254 3,
3255 "z",
3256 Type::Primitive(PrimitiveType::Long),
3257 )),
3258 ])
3259 .build()
3260 .unwrap();
3261
3262 let partition_spec = PartitionSpec::builder(schema.clone())
3263 .with_spec_id(0)
3264 .add_unbound_field(
3265 UnboundPartitionField::builder()
3266 .source_ids(vec![1])
3267 .field_id(1000)
3268 .name("x".to_string())
3269 .transform(Transform::Identity)
3270 .build()
3271 .unwrap(),
3272 )
3273 .unwrap()
3274 .build()
3275 .unwrap();
3276
3277 let default_partition_type = partition_spec.partition_type(&schema).unwrap();
3278 let expected = TableMetadata {
3279 format_version: FormatVersion::V1,
3280 table_uuid: Uuid::parse_str("d20125c8-7284-442c-9aea-15fee620737c").unwrap(),
3281 location: "s3://bucket/test/location".to_string(),
3282 last_updated_ms: 1602638573874,
3283 last_column_id: 3,
3284 schemas: HashMap::from_iter(vec![(0, Arc::new(schema))]),
3285 current_schema_id: 0,
3286 partition_specs: HashMap::from_iter(vec![(0, partition_spec.clone().into())]),
3287 default_spec: Arc::new(partition_spec),
3288 default_partition_type,
3289 last_partition_id: 0,
3290 default_sort_order_id: 0,
3291 sort_orders: HashMap::from_iter(vec![(0, SortOrder::unsorted_order().into())]),
3293 snapshots: HashMap::new(),
3294 current_snapshot_id: None,
3295 last_sequence_number: 0,
3296 properties: HashMap::new(),
3297 snapshot_log: vec![],
3298 metadata_log: Vec::new(),
3299 refs: HashMap::new(),
3300 statistics: HashMap::new(),
3301 partition_statistics: HashMap::new(),
3302 encryption_keys: HashMap::new(),
3303 next_row_id: INITIAL_ROW_ID,
3304 };
3305
3306 check_table_metadata_serde(&metadata, expected);
3307 }
3308
3309 #[test]
3310 fn test_table_metadata_v1_legacy_partition_spec_with_void_fields() {
3311 let mut metadata: serde_json::Value = serde_json::from_str(
3312 &fs::read_to_string("testdata/table_metadata/TableMetadataV1Valid.json").unwrap(),
3313 )
3314 .unwrap();
3315 metadata["partition-spec"] = serde_json::json!([
3317 {"name": "x", "transform": "void", "source-id": 1, "field-id": 1000},
3318 {"name": "x_null", "transform": "void", "source-id": 1, "field-id": 1001},
3319 ]);
3320
3321 let metadata: TableMetadata = serde_json::from_value(metadata).unwrap();
3322
3323 let fields = metadata.default_partition_spec().fields();
3324 assert_eq!(
3325 fields
3326 .iter()
3327 .map(|f| (f.field_id, f.name.as_str(), f.transform))
3328 .collect::<Vec<_>>(),
3329 vec![
3330 (1000, "x", Transform::Void),
3331 (1001, "x_null", Transform::Void)
3332 ]
3333 );
3334 }
3335
3336 #[test]
3337 fn test_empty_snapshot_id_is_normalized_to_none() {
3338 let metadata =
3339 fs::read_to_string("testdata/table_metadata/TableMetadataV1Valid.json").unwrap();
3340 let deserialized: TableMetadata = serde_json::from_str(&metadata).unwrap();
3341 assert_eq!(
3342 deserialized.current_snapshot_id(),
3343 None,
3344 "current_snapshot_id of -1 should be deserialized as None"
3345 );
3346 }
3347
3348 #[test]
3349 fn test_table_metadata_v1_compat() {
3350 let metadata =
3351 fs::read_to_string("testdata/table_metadata/TableMetadataV1Compat.json").unwrap();
3352
3353 let desered_type: TableMetadata = serde_json::from_str(&metadata)
3355 .expect("Failed to deserialize TableMetadataV1Compat.json");
3356
3357 assert_eq!(desered_type.format_version(), FormatVersion::V1);
3359 assert_eq!(
3360 desered_type.uuid(),
3361 Uuid::parse_str("3276010d-7b1d-488c-98d8-9025fc4fde6b").unwrap()
3362 );
3363 assert_eq!(
3364 desered_type.location(),
3365 "s3://bucket/warehouse/iceberg/glue.db/table_name"
3366 );
3367 assert_eq!(desered_type.last_updated_ms(), 1727773114005);
3368 assert_eq!(desered_type.current_schema_id(), 0);
3369 }
3370
3371 fn v1_main_ref() -> (String, SnapshotReference) {
3372 let retention = SnapshotRetention::branch(None, None, None);
3373 (
3374 "main".to_string(),
3375 SnapshotReference::new(6278926806555018392, retention),
3376 )
3377 }
3378
3379 #[test]
3380 fn test_table_metadata_v1_refs_round_trip() {
3381 let json =
3383 fs::read_to_string("testdata/table_metadata/TableMetadataV1WithRefs.json").unwrap();
3384 let metadata: TableMetadata = serde_json::from_str(&json).unwrap();
3385
3386 assert_eq!(metadata.format_version(), FormatVersion::V1);
3387 assert_eq!(
3388 metadata.refs,
3389 HashMap::from_iter([
3390 v1_main_ref(),
3391 (
3392 "t1".to_string(),
3393 SnapshotReference::new(7825560978372493301, SnapshotRetention::Tag {
3394 max_ref_age_ms: Some(604800000),
3395 }),
3396 ),
3397 (
3398 "b1".to_string(),
3399 SnapshotReference::new(
3400 7825560978372493301,
3401 SnapshotRetention::branch(Some(2), None, None),
3402 ),
3403 ),
3404 ])
3405 );
3406
3407 let written = serde_json::to_value(&metadata).unwrap();
3408 let java_written = serde_json::from_str::<serde_json::Value>(&json).unwrap();
3409 assert_eq!(written["format-version"], 1);
3410 assert_eq!(written["refs"], java_written["refs"]);
3411 assert_eq!(
3412 serde_json::from_value::<TableMetadata>(written).unwrap(),
3413 metadata
3414 );
3415 }
3416
3417 #[test]
3418 fn test_table_metadata_v1_without_refs_has_main_branch() {
3419 let json =
3420 fs::read_to_string("testdata/table_metadata/TableMetadataV1WithRefs.json").unwrap();
3421 let mut value = serde_json::from_str::<serde_json::Value>(&json).unwrap();
3422 value.as_object_mut().unwrap().remove("refs").unwrap();
3423 let metadata: TableMetadata = serde_json::from_value(value).unwrap();
3424
3425 assert_eq!(metadata.refs, HashMap::from_iter([v1_main_ref()]));
3426 assert_eq!(
3427 serde_json::to_value(&metadata).unwrap()["refs"],
3428 serde_json::json!({"main": {"snapshot-id": 6278926806555018392_i64, "type": "branch"}})
3429 );
3430 }
3431
3432 #[test]
3433 fn test_table_metadata_v1_invalid_refs_fail_to_load() {
3434 let json =
3435 fs::read_to_string("testdata/table_metadata/TableMetadataV1WithRefs.json").unwrap();
3436 let load_error = |edit: fn(&mut serde_json::Value)| {
3437 let mut value = serde_json::from_str::<serde_json::Value>(&json).unwrap();
3438 edit(&mut value);
3439 serde_json::from_value::<TableMetadata>(value)
3440 .unwrap_err()
3441 .to_string()
3442 };
3443
3444 assert_eq!(
3445 load_error(|value| value["refs"]["t1"]["snapshot-id"] = 1.into()),
3446 "DataInvalid => Snapshot for reference t1 does not exist in the existing snapshots list"
3447 );
3448 assert_eq!(
3449 load_error(
3450 |value| value["refs"]["main"]["snapshot-id"] = 7825560978372493301_i64.into()
3451 ),
3452 "DataInvalid => Current snapshot id does not match main branch (6278926806555018392 != 7825560978372493301)"
3453 );
3454 assert_eq!(
3455 load_error(|value| value["current-snapshot-id"] = (-1).into()),
3456 "DataInvalid => Current snapshot is not set, but main branch exists"
3457 );
3458 }
3459
3460 #[test]
3461 fn test_table_metadata_v1_schemas_without_current_id() {
3462 let metadata = fs::read_to_string(
3463 "testdata/table_metadata/TableMetadataV1SchemasWithoutCurrentId.json",
3464 )
3465 .unwrap();
3466
3467 let desered_type: TableMetadata = serde_json::from_str(&metadata)
3469 .expect("Failed to deserialize TableMetadataV1SchemasWithoutCurrentId.json");
3470
3471 assert_eq!(desered_type.format_version(), FormatVersion::V1);
3473 assert_eq!(
3474 desered_type.uuid(),
3475 Uuid::parse_str("d20125c8-7284-442c-9aea-15fee620737c").unwrap()
3476 );
3477
3478 let schema = desered_type.current_schema();
3480 assert_eq!(schema.as_struct().fields().len(), 3);
3481 assert_eq!(schema.as_struct().fields()[0].name, "x");
3482 assert_eq!(schema.as_struct().fields()[1].name, "y");
3483 assert_eq!(schema.as_struct().fields()[2].name, "z");
3484 }
3485
3486 #[test]
3487 fn test_table_metadata_v1_no_valid_schema() {
3488 let metadata =
3489 fs::read_to_string("testdata/table_metadata/TableMetadataV1NoValidSchema.json")
3490 .unwrap();
3491
3492 let desered: Result<TableMetadata, serde_json::Error> = serde_json::from_str(&metadata);
3494
3495 assert!(desered.is_err());
3496 let error_message = desered.unwrap_err().to_string();
3497 assert!(
3498 error_message.contains("No valid schema configuration found"),
3499 "Expected error about no valid schema configuration, got: {error_message}"
3500 );
3501 }
3502
3503 #[test]
3504 fn test_table_metadata_v1_partition_specs_without_default_id() {
3505 let metadata = fs::read_to_string(
3506 "testdata/table_metadata/TableMetadataV1PartitionSpecsWithoutDefaultId.json",
3507 )
3508 .unwrap();
3509
3510 let desered_type: TableMetadata = serde_json::from_str(&metadata)
3512 .expect("Failed to deserialize TableMetadataV1PartitionSpecsWithoutDefaultId.json");
3513
3514 assert_eq!(desered_type.format_version(), FormatVersion::V1);
3516 assert_eq!(
3517 desered_type.uuid(),
3518 Uuid::parse_str("d20125c8-7284-442c-9aea-15fee620737c").unwrap()
3519 );
3520
3521 assert_eq!(desered_type.default_partition_spec_id(), 2); assert_eq!(desered_type.partition_specs.len(), 2);
3524
3525 let default_spec = &desered_type.default_spec;
3527 assert_eq!(default_spec.spec_id(), 2);
3528 assert_eq!(default_spec.fields().len(), 1);
3529 assert_eq!(default_spec.fields()[0].name, "y");
3530 assert_eq!(default_spec.fields()[0].transform, Transform::Identity);
3531 assert_eq!(default_spec.fields()[0].source_id, 2);
3532 }
3533
3534 #[test]
3535 fn test_table_metadata_v2_schema_not_found() {
3536 let metadata =
3537 fs::read_to_string("testdata/table_metadata/TableMetadataV2CurrentSchemaNotFound.json")
3538 .unwrap();
3539
3540 let desered: Result<TableMetadata, serde_json::Error> = serde_json::from_str(&metadata);
3541
3542 assert_eq!(
3543 desered.unwrap_err().to_string(),
3544 "DataInvalid => No schema exists with the current schema id 2."
3545 )
3546 }
3547
3548 #[test]
3549 fn test_table_metadata_v2_missing_sort_order() {
3550 let metadata =
3551 fs::read_to_string("testdata/table_metadata/TableMetadataV2MissingSortOrder.json")
3552 .unwrap();
3553
3554 let desered: Result<TableMetadata, serde_json::Error> = serde_json::from_str(&metadata);
3555
3556 assert_eq!(
3557 desered.unwrap_err().to_string(),
3558 "data did not match any variant of untagged enum TableMetadataEnum"
3559 )
3560 }
3561
3562 #[test]
3563 fn test_table_metadata_v2_missing_partition_specs() {
3564 let metadata =
3565 fs::read_to_string("testdata/table_metadata/TableMetadataV2MissingPartitionSpecs.json")
3566 .unwrap();
3567
3568 let desered: Result<TableMetadata, serde_json::Error> = serde_json::from_str(&metadata);
3569
3570 assert_eq!(
3571 desered.unwrap_err().to_string(),
3572 "data did not match any variant of untagged enum TableMetadataEnum"
3573 )
3574 }
3575
3576 #[test]
3577 fn test_table_metadata_v2_missing_last_partition_id() {
3578 let metadata = fs::read_to_string(
3579 "testdata/table_metadata/TableMetadataV2MissingLastPartitionId.json",
3580 )
3581 .unwrap();
3582
3583 let desered: Result<TableMetadata, serde_json::Error> = serde_json::from_str(&metadata);
3584
3585 assert_eq!(
3586 desered.unwrap_err().to_string(),
3587 "data did not match any variant of untagged enum TableMetadataEnum"
3588 )
3589 }
3590
3591 #[test]
3592 fn test_table_metadata_v2_missing_schemas() {
3593 let metadata =
3594 fs::read_to_string("testdata/table_metadata/TableMetadataV2MissingSchemas.json")
3595 .unwrap();
3596
3597 let desered: Result<TableMetadata, serde_json::Error> = serde_json::from_str(&metadata);
3598
3599 assert_eq!(
3600 desered.unwrap_err().to_string(),
3601 "data did not match any variant of untagged enum TableMetadataEnum"
3602 )
3603 }
3604
3605 #[test]
3606 fn test_table_metadata_v2_unsupported_version() {
3607 let metadata =
3608 fs::read_to_string("testdata/table_metadata/TableMetadataUnsupportedVersion.json")
3609 .unwrap();
3610
3611 let desered: Result<TableMetadata, serde_json::Error> = serde_json::from_str(&metadata);
3612
3613 assert_eq!(
3614 desered.unwrap_err().to_string(),
3615 "data did not match any variant of untagged enum TableMetadataEnum"
3616 )
3617 }
3618
3619 #[test]
3620 fn test_order_of_format_version() {
3621 assert!(FormatVersion::V1 < FormatVersion::V2);
3622 assert_eq!(FormatVersion::V1, FormatVersion::V1);
3623 assert_eq!(FormatVersion::V2, FormatVersion::V2);
3624 }
3625
3626 #[test]
3627 fn test_deserialize_default_partition_spec_with_dropped_source() {
3628 for version in [1, 2, 3] {
3629 for transform in ["identity", "truncate[4]", "bucket[16]"] {
3630 let mut metadata = serde_json::json!({
3631 "format-version": version,
3632 "table-uuid": "9c12d441-03fe-4693-9a96-a0705ddf69c1",
3633 "location": "s3://bucket/table",
3634 "last-sequence-number": 0,
3635 "last-updated-ms": 1602638573590_i64,
3636 "last-column-id": 2,
3637 "current-schema-id": 1,
3638 "schemas": [
3639 {
3640 "schema-id": 0,
3641 "type": "struct",
3642 "fields": [
3643 {"id": 1, "name": "x", "required": false, "type": "long"},
3644 {"id": 2, "name": "y", "required": false, "type": "long"}
3645 ]
3646 },
3647 {
3648 "schema-id": 1,
3649 "type": "struct",
3650 "fields": [
3651 {"id": 2, "name": "y", "required": false, "type": "long"}
3652 ]
3653 }
3654 ],
3655 "default-spec-id": 0,
3656 "partition-specs": [{"spec-id": 0, "fields": [{
3657 "source-id": 1, "field-id": 1000, "name": "x_partition",
3658 "transform": transform
3659 }]}],
3660 "last-partition-id": 1000,
3661 "default-sort-order-id": 0,
3662 "sort-orders": [{"order-id": 0, "fields": []}],
3663 "next-row-id": 0
3664 });
3665
3666 let err = serde_json::from_value::<TableMetadata>(metadata.clone()).unwrap_err();
3669 assert!(err.to_string().contains(
3670 "Default partition spec 0 references missing source field 1 in current schema 1"
3671 ));
3672
3673 metadata["partition-specs"]
3675 .as_array_mut()
3676 .unwrap()
3677 .push(serde_json::json!({"spec-id": 1, "fields": []}));
3678 metadata["default-spec-id"] = serde_json::json!(1);
3679 serde_json::from_value::<TableMetadata>(metadata.clone()).unwrap();
3680
3681 metadata["default-spec-id"] = serde_json::json!(0);
3683 metadata["partition-specs"][0]["fields"][0]["transform"] =
3684 serde_json::json!("void");
3685 serde_json::from_value::<TableMetadata>(metadata).unwrap();
3686 }
3687 }
3688 }
3689
3690 #[test]
3691 fn test_default_partition_spec() {
3692 let default_spec_id = 1234;
3693 let mut table_meta_data = get_test_table_metadata("TableMetadataV2Valid.json");
3694 let partition_spec = PartitionSpec::unpartition_spec();
3695 table_meta_data.default_spec = partition_spec.clone().into();
3696 table_meta_data
3697 .partition_specs
3698 .insert(default_spec_id, Arc::new(partition_spec));
3699
3700 assert_eq!(
3701 (*table_meta_data.default_partition_spec().clone()).clone(),
3702 (*table_meta_data
3703 .partition_spec_by_id(default_spec_id)
3704 .unwrap()
3705 .clone())
3706 .clone()
3707 );
3708 }
3709 #[test]
3710 fn test_default_sort_order() {
3711 let default_sort_order_id = 1234;
3712 let mut table_meta_data = get_test_table_metadata("TableMetadataV2Valid.json");
3713 table_meta_data.default_sort_order_id = default_sort_order_id;
3714 table_meta_data
3715 .sort_orders
3716 .insert(default_sort_order_id, Arc::new(SortOrder::default()));
3717
3718 assert_eq!(
3719 table_meta_data.default_sort_order(),
3720 table_meta_data
3721 .sort_orders
3722 .get(&default_sort_order_id)
3723 .unwrap()
3724 )
3725 }
3726
3727 #[test]
3728 fn test_table_metadata_builder_from_table_creation() {
3729 let table_creation = TableCreation::builder()
3730 .location("s3://db/table".to_string())
3731 .name("table".to_string())
3732 .properties(HashMap::new())
3733 .schema(Schema::builder().build().unwrap())
3734 .build();
3735 let table_metadata = TableMetadataBuilder::from_table_creation(table_creation)
3736 .unwrap()
3737 .build()
3738 .unwrap()
3739 .metadata;
3740 assert_eq!(table_metadata.location, "s3://db/table");
3741 assert_eq!(table_metadata.schemas.len(), 1);
3742 assert_eq!(
3743 table_metadata
3744 .schemas
3745 .get(&0)
3746 .unwrap()
3747 .as_struct()
3748 .fields()
3749 .len(),
3750 0
3751 );
3752 assert_eq!(table_metadata.properties.len(), 0);
3753 assert_eq!(
3754 table_metadata.partition_specs,
3755 HashMap::from([(
3756 0,
3757 Arc::new(
3758 PartitionSpec::builder(table_metadata.schemas.get(&0).unwrap().clone())
3759 .with_spec_id(0)
3760 .build()
3761 .unwrap()
3762 )
3763 )])
3764 );
3765 assert_eq!(
3766 table_metadata.sort_orders,
3767 HashMap::from([(
3768 0,
3769 Arc::new(SortOrder {
3770 order_id: 0,
3771 fields: vec![]
3772 })
3773 )])
3774 );
3775 }
3776
3777 #[tokio::test]
3778 async fn test_table_metadata_read_write() {
3779 let temp_dir = TempDir::new().unwrap();
3781 let temp_path = temp_dir.path().to_str().unwrap();
3782
3783 let file_io = FileIO::new_with_fs();
3785
3786 let original_metadata: TableMetadata =
3788 get_test_table_metadata_at("TableMetadataV2Valid.json", temp_path);
3789
3790 let metadata_location =
3792 MetadataLocation::try_new_with_metadata(&original_metadata).unwrap();
3793 let metadata_location_str = metadata_location.to_string();
3794
3795 original_metadata
3797 .write_to(&file_io, &metadata_location)
3798 .await
3799 .unwrap();
3800
3801 assert!(fs::metadata(&metadata_location_str).is_ok());
3803
3804 let read_metadata = TableMetadata::read_from(&file_io, &metadata_location_str)
3806 .await
3807 .unwrap();
3808
3809 assert_eq!(read_metadata, original_metadata);
3811 }
3812
3813 #[tokio::test]
3814 async fn test_table_metadata_read_compressed() {
3815 let temp_dir = TempDir::new().unwrap();
3816 let metadata_location = temp_dir.path().join("v1.gz.metadata.json");
3817
3818 let original_metadata: TableMetadata = get_test_table_metadata("TableMetadataV2Valid.json");
3819 let json = serde_json::to_string(&original_metadata).unwrap();
3820
3821 let compressed = CompressionCodec::gzip_default()
3822 .compress(json.into_bytes())
3823 .expect("failed to compress metadata");
3824 fs::write(&metadata_location, &compressed).expect("failed to write metadata");
3825
3826 let file_io = FileIO::new_with_fs();
3828 let metadata_location = metadata_location.to_str().unwrap();
3829 let read_metadata = TableMetadata::read_from(&file_io, metadata_location)
3830 .await
3831 .unwrap();
3832
3833 assert_eq!(read_metadata, original_metadata);
3835 }
3836
3837 #[tokio::test]
3838 async fn test_table_metadata_read_nonexistent_file() {
3839 let file_io = FileIO::new_with_fs();
3841
3842 let result = TableMetadata::read_from(&file_io, "/nonexistent/path/metadata.json").await;
3844
3845 assert!(result.is_err());
3847 }
3848
3849 #[tokio::test]
3850 async fn test_table_metadata_write_with_gzip_compression() {
3851 let temp_dir = TempDir::new().unwrap();
3852 let temp_path = temp_dir.path().to_str().unwrap();
3853 let file_io = FileIO::new_with_fs();
3854
3855 let original_metadata: TableMetadata =
3857 get_test_table_metadata_at("TableMetadataV2Valid.json", temp_path);
3858
3859 let mut props = original_metadata.properties.clone();
3861 props.insert(
3862 TableProperties::PROPERTY_METADATA_COMPRESSION_CODEC.to_string(),
3863 "GziP".to_string(),
3864 );
3865 let compressed_metadata =
3867 TableMetadataBuilder::new_from_metadata(original_metadata.clone(), None)
3868 .assign_uuid(original_metadata.table_uuid)
3869 .set_properties(props.clone())
3870 .unwrap()
3871 .build()
3872 .unwrap()
3873 .metadata;
3874
3875 let metadata_location =
3877 MetadataLocation::try_new_with_metadata(&compressed_metadata).unwrap();
3878 let metadata_location_str = metadata_location.to_string();
3879
3880 assert!(metadata_location_str.contains(".gz.metadata.json"));
3882
3883 compressed_metadata
3885 .write_to(&file_io, &metadata_location)
3886 .await
3887 .unwrap();
3888
3889 assert!(std::path::Path::new(&metadata_location_str).exists());
3891
3892 let raw_content = fs::read(&metadata_location_str).unwrap();
3894 assert!(raw_content.len() > 2);
3895 assert_eq!(raw_content[0], 0x1F); assert_eq!(raw_content[1], 0x8B); let read_metadata = TableMetadata::read_from(&file_io, &metadata_location_str)
3900 .await
3901 .unwrap();
3902
3903 assert_eq!(read_metadata, compressed_metadata);
3905 }
3906
3907 #[test]
3908 fn test_name_exists_in_any_schema() {
3909 let schema1 = Schema::builder()
3911 .with_schema_id(1)
3912 .with_fields(vec![
3913 NestedField::required(1, "field1", Type::Primitive(PrimitiveType::String)).into(),
3914 NestedField::required(2, "field2", Type::Primitive(PrimitiveType::Int)).into(),
3915 ])
3916 .build()
3917 .unwrap();
3918
3919 let schema2 = Schema::builder()
3920 .with_schema_id(2)
3921 .with_fields(vec![
3922 NestedField::required(1, "field1", Type::Primitive(PrimitiveType::String)).into(),
3923 NestedField::required(3, "field3", Type::Primitive(PrimitiveType::Long)).into(),
3924 ])
3925 .build()
3926 .unwrap();
3927
3928 let metadata = TableMetadataBuilder::new(
3929 schema1,
3930 PartitionSpec::unpartition_spec().into_unbound(),
3931 SortOrder::unsorted_order(),
3932 "s3://test/location".to_string(),
3933 FormatVersion::V2,
3934 HashMap::new(),
3935 )
3936 .unwrap()
3937 .add_current_schema(schema2)
3938 .unwrap()
3939 .build()
3940 .unwrap()
3941 .metadata;
3942
3943 assert!(metadata.name_exists_in_any_schema("field1")); assert!(metadata.name_exists_in_any_schema("field2")); assert!(metadata.name_exists_in_any_schema("field3")); assert!(!metadata.name_exists_in_any_schema("nonexistent_field"));
3948 assert!(!metadata.name_exists_in_any_schema("field4"));
3949 assert!(!metadata.name_exists_in_any_schema(""));
3950 }
3951
3952 #[test]
3953 fn test_name_exists_in_any_schema_empty_schemas() {
3954 let schema = Schema::builder().with_fields(vec![]).build().unwrap();
3955
3956 let metadata = TableMetadataBuilder::new(
3957 schema,
3958 PartitionSpec::unpartition_spec().into_unbound(),
3959 SortOrder::unsorted_order(),
3960 "s3://test/location".to_string(),
3961 FormatVersion::V2,
3962 HashMap::new(),
3963 )
3964 .unwrap()
3965 .build()
3966 .unwrap()
3967 .metadata;
3968
3969 assert!(!metadata.name_exists_in_any_schema("any_field"));
3970 }
3971
3972 #[test]
3973 fn test_helper_methods_multi_version_scenario() {
3974 let initial_schema = Schema::builder()
3976 .with_fields(vec![
3977 NestedField::required(1, "id", Type::Primitive(PrimitiveType::Long)).into(),
3978 NestedField::required(2, "name", Type::Primitive(PrimitiveType::String)).into(),
3979 NestedField::required(
3980 3,
3981 "deprecated_field",
3982 Type::Primitive(PrimitiveType::String),
3983 )
3984 .into(),
3985 ])
3986 .build()
3987 .unwrap();
3988
3989 let metadata = TableMetadataBuilder::new(
3990 initial_schema,
3991 PartitionSpec::unpartition_spec().into_unbound(),
3992 SortOrder::unsorted_order(),
3993 "s3://test/location".to_string(),
3994 FormatVersion::V2,
3995 HashMap::new(),
3996 )
3997 .unwrap();
3998
3999 let evolved_schema = Schema::builder()
4000 .with_fields(vec![
4001 NestedField::required(1, "id", Type::Primitive(PrimitiveType::Long)).into(),
4002 NestedField::required(2, "name", Type::Primitive(PrimitiveType::String)).into(),
4003 NestedField::required(
4004 3,
4005 "deprecated_field",
4006 Type::Primitive(PrimitiveType::String),
4007 )
4008 .into(),
4009 NestedField::required(4, "new_field", Type::Primitive(PrimitiveType::Double))
4010 .into(),
4011 ])
4012 .build()
4013 .unwrap();
4014
4015 let _final_schema = Schema::builder()
4017 .with_fields(vec![
4018 NestedField::required(1, "id", Type::Primitive(PrimitiveType::Long)).into(),
4019 NestedField::required(2, "name", Type::Primitive(PrimitiveType::String)).into(),
4020 NestedField::required(4, "new_field", Type::Primitive(PrimitiveType::Double))
4021 .into(),
4022 NestedField::required(5, "latest_field", Type::Primitive(PrimitiveType::Boolean))
4023 .into(),
4024 ])
4025 .build()
4026 .unwrap();
4027
4028 let final_metadata = metadata
4029 .add_current_schema(evolved_schema)
4030 .unwrap()
4031 .build()
4032 .unwrap()
4033 .metadata;
4034
4035 assert!(final_metadata.name_exists_in_any_schema("id")); assert!(final_metadata.name_exists_in_any_schema("name")); assert!(final_metadata.name_exists_in_any_schema("deprecated_field")); assert!(final_metadata.name_exists_in_any_schema("new_field")); assert!(!final_metadata.name_exists_in_any_schema("never_existed"));
4040 }
4041
4042 #[test]
4043 fn test_invalid_sort_order_id_zero_with_fields() {
4044 let metadata = r#"
4045 {
4046 "format-version": 2,
4047 "table-uuid": "9c12d441-03fe-4693-9a96-a0705ddf69c1",
4048 "location": "s3://bucket/test/location",
4049 "last-sequence-number": 111,
4050 "last-updated-ms": 1600000000000,
4051 "last-column-id": 3,
4052 "current-schema-id": 1,
4053 "schemas": [
4054 {
4055 "type": "struct",
4056 "schema-id": 1,
4057 "fields": [
4058 {"id": 1, "name": "x", "required": true, "type": "long"},
4059 {"id": 2, "name": "y", "required": true, "type": "long"}
4060 ]
4061 }
4062 ],
4063 "default-spec-id": 0,
4064 "partition-specs": [{"spec-id": 0, "fields": []}],
4065 "last-partition-id": 999,
4066 "default-sort-order-id": 0,
4067 "sort-orders": [
4068 {
4069 "order-id": 0,
4070 "fields": [
4071 {
4072 "transform": "identity",
4073 "source-id": 1,
4074 "direction": "asc",
4075 "null-order": "nulls-first"
4076 }
4077 ]
4078 }
4079 ],
4080 "properties": {},
4081 "current-snapshot-id": -1,
4082 "snapshots": []
4083 }
4084 "#;
4085
4086 let result: Result<TableMetadata, serde_json::Error> = serde_json::from_str(metadata);
4087
4088 assert!(
4090 result.is_err(),
4091 "Parsing should fail for sort order ID 0 with fields"
4092 );
4093 }
4094
4095 #[test]
4096 fn test_table_properties_with_defaults() {
4097 use crate::spec::TableProperties;
4098
4099 let schema = Schema::builder()
4100 .with_fields(vec![
4101 NestedField::required(1, "id", Type::Primitive(PrimitiveType::Long)).into(),
4102 ])
4103 .build()
4104 .unwrap();
4105
4106 let metadata = TableMetadataBuilder::new(
4107 schema,
4108 PartitionSpec::unpartition_spec().into_unbound(),
4109 SortOrder::unsorted_order(),
4110 "s3://test/location".to_string(),
4111 FormatVersion::V2,
4112 HashMap::new(),
4113 )
4114 .unwrap()
4115 .build()
4116 .unwrap()
4117 .metadata;
4118
4119 let props = metadata.table_properties();
4120
4121 assert_eq!(
4122 props.commit_num_retries().unwrap(),
4123 TableProperties::PROPERTY_COMMIT_NUM_RETRIES_DEFAULT
4124 );
4125 assert_eq!(
4126 props.write_target_file_size_bytes().unwrap(),
4127 TableProperties::PROPERTY_WRITE_TARGET_FILE_SIZE_BYTES_DEFAULT
4128 );
4129 }
4130
4131 #[test]
4132 fn test_table_properties_with_custom_values() {
4133 use crate::spec::TableProperties;
4134
4135 let schema = Schema::builder()
4136 .with_fields(vec![
4137 NestedField::required(1, "id", Type::Primitive(PrimitiveType::Long)).into(),
4138 ])
4139 .build()
4140 .unwrap();
4141
4142 let properties = HashMap::from([
4143 (
4144 TableProperties::PROPERTY_COMMIT_NUM_RETRIES.to_string(),
4145 "10".to_string(),
4146 ),
4147 (
4148 TableProperties::PROPERTY_WRITE_TARGET_FILE_SIZE_BYTES.to_string(),
4149 "1024".to_string(),
4150 ),
4151 ]);
4152
4153 let metadata = TableMetadataBuilder::new(
4154 schema,
4155 PartitionSpec::unpartition_spec().into_unbound(),
4156 SortOrder::unsorted_order(),
4157 "s3://test/location".to_string(),
4158 FormatVersion::V2,
4159 properties,
4160 )
4161 .unwrap()
4162 .build()
4163 .unwrap()
4164 .metadata;
4165
4166 let props = metadata.table_properties();
4167
4168 assert_eq!(props.commit_num_retries().unwrap(), 10);
4169 assert_eq!(props.write_target_file_size_bytes().unwrap(), 1024);
4170 }
4171
4172 #[test]
4173 fn test_deserialize_metadata_defers_invalid_table_property_errors() {
4174 let invalid_retries = "not_a_number";
4175 let invalid_codec = "unknown";
4176 let target_file_size = "1024";
4177
4178 for file_name in [
4179 "TableMetadataV1Valid.json",
4180 "TableMetadataV2ValidMinimal.json",
4181 "TableMetadataV3ValidMinimal.json",
4182 ] {
4183 let path = format!("testdata/table_metadata/{file_name}");
4184 let mut json: serde_json::Value =
4185 serde_json::from_str(&fs::read_to_string(path).unwrap()).unwrap();
4186 json["properties"] = serde_json::json!({
4187 (TableProperties::PROPERTY_COMMIT_NUM_RETRIES): invalid_retries,
4188 (TableProperties::PROPERTY_METADATA_COMPRESSION_CODEC): invalid_codec,
4189 (TableProperties::PROPERTY_WRITE_TARGET_FILE_SIZE_BYTES): target_file_size,
4190 });
4191
4192 let metadata: TableMetadata = serde_json::from_value(json).unwrap();
4193 assert_eq!(
4194 metadata
4195 .properties()
4196 .get(TableProperties::PROPERTY_COMMIT_NUM_RETRIES)
4197 .map(String::as_str),
4198 Some(invalid_retries)
4199 );
4200
4201 let table_properties = metadata.table_properties();
4202 let error = table_properties.commit_num_retries().unwrap_err();
4203 assert!(
4204 error
4205 .message()
4206 .contains(TableProperties::PROPERTY_COMMIT_NUM_RETRIES)
4207 );
4208 assert_eq!(
4209 table_properties.write_target_file_size_bytes().unwrap(),
4210 1024
4211 );
4212 let error = table_properties.metadata_compression_codec().unwrap_err();
4213 assert!(
4214 format!("{error}").contains(TableProperties::PROPERTY_METADATA_COMPRESSION_CODEC)
4215 );
4216
4217 let serialized = serde_json::to_value(metadata).unwrap();
4218 assert_eq!(
4219 serialized["properties"][TableProperties::PROPERTY_COMMIT_NUM_RETRIES],
4220 invalid_retries
4221 );
4222 assert_eq!(
4223 serialized["properties"][TableProperties::PROPERTY_METADATA_COMPRESSION_CODEC],
4224 invalid_codec
4225 );
4226 }
4227 }
4228
4229 #[test]
4230 fn test_table_properties_with_invalid_value() {
4231 let schema = Schema::builder()
4232 .with_fields(vec![
4233 NestedField::required(1, "id", Type::Primitive(PrimitiveType::Long)).into(),
4234 ])
4235 .build()
4236 .unwrap();
4237
4238 let properties = HashMap::from([
4239 (
4240 TableProperties::PROPERTY_COMMIT_NUM_RETRIES.to_string(),
4241 "not_a_number".to_string(),
4242 ),
4243 (
4244 TableProperties::PROPERTY_WRITE_TARGET_FILE_SIZE_BYTES.to_string(),
4245 "1024".to_string(),
4246 ),
4247 ]);
4248
4249 let metadata = TableMetadataBuilder::new(
4250 schema,
4251 PartitionSpec::unpartition_spec().into_unbound(),
4252 SortOrder::unsorted_order(),
4253 "s3://test/location".to_string(),
4254 FormatVersion::V2,
4255 properties,
4256 )
4257 .unwrap()
4258 .build()
4259 .unwrap()
4260 .metadata;
4261
4262 let table_properties = metadata.table_properties();
4263 let err = table_properties.commit_num_retries().unwrap_err();
4264 assert_eq!(err.kind(), ErrorKind::DataInvalid);
4265 assert!(
4266 err.message()
4267 .contains(TableProperties::PROPERTY_COMMIT_NUM_RETRIES)
4268 );
4269 assert_eq!(
4270 table_properties.write_target_file_size_bytes().unwrap(),
4271 1024
4272 );
4273 }
4274
4275 #[test]
4276 fn test_v2_to_v3_upgrade_preserves_existing_snapshots_without_row_lineage() {
4277 let schema = Schema::builder()
4279 .with_fields(vec![
4280 NestedField::required(1, "id", Type::Primitive(PrimitiveType::Long)).into(),
4281 ])
4282 .build()
4283 .unwrap();
4284
4285 let v2_metadata = TableMetadataBuilder::new(
4286 schema,
4287 PartitionSpec::unpartition_spec().into_unbound(),
4288 SortOrder::unsorted_order(),
4289 "s3://bucket/test/location".to_string(),
4290 FormatVersion::V2,
4291 HashMap::new(),
4292 )
4293 .unwrap()
4294 .build()
4295 .unwrap()
4296 .metadata;
4297
4298 let snapshot = Snapshot::builder()
4300 .with_snapshot_id(1)
4301 .with_timestamp_ms(v2_metadata.last_updated_ms + 1)
4302 .with_sequence_number(1)
4303 .with_schema_id(0)
4304 .with_manifest_list("s3://bucket/test/metadata/snap-1.avro")
4305 .with_summary(Summary {
4306 operation: Operation::Append,
4307 additional_properties: HashMap::from([(
4308 "added-data-files".to_string(),
4309 "1".to_string(),
4310 )]),
4311 })
4312 .build();
4313
4314 let v2_with_snapshot = v2_metadata
4315 .into_builder(Some("s3://bucket/test/metadata/v00001.json".to_string()))
4316 .add_snapshot(snapshot)
4317 .unwrap()
4318 .set_ref("main", SnapshotReference {
4319 snapshot_id: 1,
4320 retention: SnapshotRetention::Branch {
4321 min_snapshots_to_keep: None,
4322 max_snapshot_age_ms: None,
4323 max_ref_age_ms: None,
4324 },
4325 })
4326 .unwrap()
4327 .build()
4328 .unwrap()
4329 .metadata;
4330
4331 let v2_json = serde_json::to_string(&v2_with_snapshot);
4333 assert!(v2_json.is_ok(), "v2 serialization should work");
4334
4335 let v3_metadata = v2_with_snapshot
4337 .into_builder(Some("s3://bucket/test/metadata/v00002.json".to_string()))
4338 .upgrade_format_version(FormatVersion::V3)
4339 .unwrap()
4340 .build()
4341 .unwrap()
4342 .metadata;
4343
4344 assert_eq!(v3_metadata.format_version, FormatVersion::V3);
4345 assert_eq!(v3_metadata.next_row_id, INITIAL_ROW_ID);
4346 assert_eq!(v3_metadata.snapshots.len(), 1);
4347
4348 let snapshot = v3_metadata.snapshots.values().next().unwrap();
4350 assert!(
4351 snapshot.row_range().is_none(),
4352 "Snapshot should have no row_range after upgrade"
4353 );
4354
4355 let v3_json = serde_json::to_string(&v3_metadata);
4357 assert!(
4358 v3_json.is_ok(),
4359 "v3 serialization should work for upgraded tables"
4360 );
4361
4362 let deserialized: TableMetadata = serde_json::from_str(&v3_json.unwrap()).unwrap();
4364 assert_eq!(deserialized.format_version, FormatVersion::V3);
4365 assert_eq!(deserialized.snapshots.len(), 1);
4366
4367 let deserialized_snapshot = deserialized.snapshots.values().next().unwrap();
4369 assert!(
4370 deserialized_snapshot.row_range().is_none(),
4371 "Deserialized snapshot should have no row_range"
4372 );
4373 }
4374
4375 #[test]
4376 fn test_v3_snapshot_with_row_lineage_serialization() {
4377 let schema = Schema::builder()
4379 .with_fields(vec![
4380 NestedField::required(1, "id", Type::Primitive(PrimitiveType::Long)).into(),
4381 ])
4382 .build()
4383 .unwrap();
4384
4385 let v3_metadata = TableMetadataBuilder::new(
4386 schema,
4387 PartitionSpec::unpartition_spec().into_unbound(),
4388 SortOrder::unsorted_order(),
4389 "s3://bucket/test/location".to_string(),
4390 FormatVersion::V3,
4391 HashMap::new(),
4392 )
4393 .unwrap()
4394 .build()
4395 .unwrap()
4396 .metadata;
4397
4398 let snapshot = Snapshot::builder()
4400 .with_snapshot_id(1)
4401 .with_timestamp_ms(v3_metadata.last_updated_ms + 1)
4402 .with_sequence_number(1)
4403 .with_schema_id(0)
4404 .with_manifest_list("s3://bucket/test/metadata/snap-1.avro")
4405 .with_summary(Summary {
4406 operation: Operation::Append,
4407 additional_properties: HashMap::from([(
4408 "added-data-files".to_string(),
4409 "1".to_string(),
4410 )]),
4411 })
4412 .with_row_range(100, 50) .build();
4414
4415 let v3_with_snapshot = v3_metadata
4416 .into_builder(Some("s3://bucket/test/metadata/v00001.json".to_string()))
4417 .add_snapshot(snapshot)
4418 .unwrap()
4419 .set_ref("main", SnapshotReference {
4420 snapshot_id: 1,
4421 retention: SnapshotRetention::Branch {
4422 min_snapshots_to_keep: None,
4423 max_snapshot_age_ms: None,
4424 max_ref_age_ms: None,
4425 },
4426 })
4427 .unwrap()
4428 .build()
4429 .unwrap()
4430 .metadata;
4431
4432 let snapshot = v3_with_snapshot.snapshots.values().next().unwrap();
4434 assert!(
4435 snapshot.row_range().is_some(),
4436 "Snapshot should have row_range"
4437 );
4438 let (first_row_id, added_rows) = snapshot.row_range().unwrap();
4439 assert_eq!(first_row_id, 100);
4440 assert_eq!(added_rows, 50);
4441
4442 let v3_json = serde_json::to_string(&v3_with_snapshot);
4444 assert!(
4445 v3_json.is_ok(),
4446 "v3 serialization should work for snapshots with row lineage"
4447 );
4448
4449 let deserialized: TableMetadata = serde_json::from_str(&v3_json.unwrap()).unwrap();
4451 assert_eq!(deserialized.format_version, FormatVersion::V3);
4452 assert_eq!(deserialized.snapshots.len(), 1);
4453
4454 let deserialized_snapshot = deserialized.snapshots.values().next().unwrap();
4456 assert!(
4457 deserialized_snapshot.row_range().is_some(),
4458 "Deserialized snapshot should have row_range"
4459 );
4460 let (deserialized_first_row_id, deserialized_added_rows) =
4461 deserialized_snapshot.row_range().unwrap();
4462 assert_eq!(deserialized_first_row_id, 100);
4463 assert_eq!(deserialized_added_rows, 50);
4464 }
4465
4466 #[test]
4467 fn test_metadata_location_default() {
4468 let metadata = get_test_table_metadata("TableMetadataV2Valid.json");
4470 assert_eq!(metadata.location(), "s3://bucket/test/location");
4471 assert_eq!(
4472 metadata.metadata_location().unwrap(),
4473 "s3://bucket/test/location/metadata"
4474 );
4475 }
4476
4477 #[test]
4478 fn test_metadata_location_honors_write_metadata_path() {
4479 let metadata = get_test_table_metadata("TableMetadataV2Valid.json")
4480 .into_builder(None)
4481 .set_properties(HashMap::from([(
4482 TableProperties::PROPERTY_WRITE_METADATA_PATH.to_string(),
4483 "s3://other-bucket/custom-meta".to_string(),
4484 )]))
4485 .unwrap()
4486 .build()
4487 .unwrap()
4488 .metadata;
4489 assert_eq!(
4490 metadata.metadata_location().unwrap(),
4491 "s3://other-bucket/custom-meta"
4492 );
4493 }
4494
4495 #[test]
4496 fn test_metadata_location_trims_trailing_slash() {
4497 let metadata = get_test_table_metadata("TableMetadataV2Valid.json")
4499 .into_builder(None)
4500 .set_properties(HashMap::from([(
4501 TableProperties::PROPERTY_WRITE_METADATA_PATH.to_string(),
4502 "s3://other-bucket/custom-meta/".to_string(),
4503 )]))
4504 .unwrap()
4505 .build()
4506 .unwrap()
4507 .metadata;
4508 assert_eq!(
4509 metadata.metadata_location().unwrap(),
4510 "s3://other-bucket/custom-meta"
4511 );
4512 }
4513
4514 #[test]
4515 fn test_unified_partition_type_spans_all_specs() {
4516 let schema = Schema::builder()
4517 .with_fields(vec![
4518 NestedField::required(1, "x", Type::Primitive(PrimitiveType::Long)).into(),
4519 NestedField::required(2, "y", Type::Primitive(PrimitiveType::Long)).into(),
4520 NestedField::required(3, "z", Type::Primitive(PrimitiveType::Long)).into(),
4521 ])
4522 .build()
4523 .unwrap();
4524
4525 let metadata = TableMetadataBuilder::new(
4526 schema.clone(),
4527 UnboundPartitionSpec::builder()
4528 .with_spec_id(0)
4529 .add_partition_field(
4530 UnboundPartitionField::builder()
4531 .source_ids(vec![2])
4532 .name("y")
4533 .transform(Transform::Identity)
4534 .build()
4535 .unwrap(),
4536 )
4537 .unwrap()
4538 .build(),
4539 SortOrder::unsorted_order(),
4540 "s3://bucket/table".to_string(),
4541 FormatVersion::V2,
4542 HashMap::new(),
4543 )
4544 .unwrap()
4545 .build()
4546 .unwrap()
4547 .metadata
4548 .into_builder(None)
4549 .add_partition_spec(
4550 UnboundPartitionSpec::builder()
4551 .add_partition_field(
4552 UnboundPartitionField::builder()
4553 .source_ids(vec![3])
4554 .name("z")
4555 .transform(Transform::Identity)
4556 .build()
4557 .unwrap(),
4558 )
4559 .unwrap()
4560 .build(),
4561 )
4562 .unwrap()
4563 .build()
4564 .unwrap()
4565 .metadata;
4566
4567 assert_eq!(metadata.default_partition_type().fields().len(), 1);
4569
4570 let unified = metadata.unified_partition_type(&schema).unwrap();
4571 let names: Vec<&str> = unified
4572 .fields()
4573 .iter()
4574 .map(|field| field.name.as_str())
4575 .collect();
4576 assert_eq!(names, vec!["y", "z"]);
4577 }
4578}