1use std::collections::{HashMap, HashSet};
19use std::sync::Arc;
20
21use uuid::Uuid;
22
23use super::schema::assign_fresh_ids;
24use super::{
25 DEFAULT_PARTITION_SPEC_ID, DEFAULT_SCHEMA_ID, FormatVersion, MAIN_BRANCH, MetadataLog,
26 ONE_MINUTE_MS, PartitionField, PartitionSpec, PartitionSpecBuilder, PartitionStatisticsFile,
27 Schema, SchemaRef, Snapshot, SnapshotLog, SnapshotReference, SnapshotRetention, SortOrder,
28 SortOrderRef, StatisticsFile, StructType, TableMetadata, TableProperties, Transform,
29 UNPARTITIONED_LAST_ASSIGNED_ID, UnboundPartitionField, UnboundPartitionSpec,
30};
31use crate::error::{Error, ErrorKind, Result, invalid_data};
32use crate::spec::{EncryptedKey, INITIAL_ROW_ID, MIN_FORMAT_VERSION_ROW_LINEAGE};
33use crate::{TableCreation, TableUpdate};
34
35pub(crate) const FIRST_FIELD_ID: i32 = 1;
36
37#[derive(Debug, Clone)]
50pub struct TableMetadataBuilder {
51 metadata: TableMetadata,
52 changes: Vec<TableUpdate>,
53 last_added_schema_id: Option<i32>,
54 last_added_spec_id: Option<i32>,
55 last_added_order_id: Option<i64>,
56 previous_history_entry: Option<MetadataLog>,
58 last_updated_ms: Option<i64>,
59}
60
61#[derive(Debug, Clone, PartialEq)]
62pub struct TableMetadataBuildResult {
64 pub metadata: TableMetadata,
66 pub changes: Vec<TableUpdate>,
68 pub expired_metadata_logs: Vec<MetadataLog>,
70}
71
72impl TableMetadataBuilder {
73 pub const LAST_ADDED: i32 = -1;
75
76 pub fn new(
81 schema: Schema,
82 spec: impl Into<UnboundPartitionSpec>,
83 sort_order: SortOrder,
84 location: String,
85 format_version: FormatVersion,
86 properties: HashMap<String, String>,
87 ) -> Result<Self> {
88 let (fresh_schema, fresh_spec, fresh_sort_order) =
90 Self::reassign_ids(schema, spec.into(), sort_order)?;
91 let schema_id = fresh_schema.schema_id();
92
93 let builder = Self {
94 metadata: TableMetadata {
95 format_version,
96 table_uuid: Uuid::now_v7(),
97 location: "".to_string(), last_sequence_number: 0,
99 last_updated_ms: 0, last_column_id: -1, current_schema_id: -1, schemas: HashMap::new(),
103 partition_specs: HashMap::new(),
104 default_spec: Arc::new(
105 PartitionSpec::unpartition_spec().with_spec_id(-1),
111 ), default_partition_type: StructType::new(vec![]),
113 last_partition_id: UNPARTITIONED_LAST_ASSIGNED_ID,
114 properties: HashMap::new(),
115 current_snapshot_id: None,
116 snapshots: HashMap::new(),
117 snapshot_log: vec![],
118 sort_orders: HashMap::new(),
119 metadata_log: vec![],
120 default_sort_order_id: -1, refs: HashMap::default(),
122 statistics: HashMap::new(),
123 partition_statistics: HashMap::new(),
124 encryption_keys: HashMap::new(),
125 next_row_id: INITIAL_ROW_ID,
126 },
127 last_updated_ms: None,
128 changes: vec![],
129 last_added_schema_id: Some(schema_id),
130 last_added_spec_id: None,
131 last_added_order_id: None,
132 previous_history_entry: None,
133 };
134
135 builder
136 .set_location(location)
137 .add_current_schema(fresh_schema)?
138 .add_default_partition_spec(fresh_spec.into_unbound())?
139 .add_default_sort_order(fresh_sort_order)?
140 .set_properties(properties)
141 }
142
143 #[must_use]
149 pub fn new_from_metadata(
150 previous: TableMetadata,
151 current_file_location: Option<String>,
152 ) -> Self {
153 Self {
154 previous_history_entry: current_file_location.map(|l| MetadataLog {
155 metadata_file: l,
156 timestamp_ms: previous.last_updated_ms,
157 }),
158 metadata: previous,
159 changes: Vec::default(),
160 last_added_schema_id: None,
161 last_added_spec_id: None,
162 last_added_order_id: None,
163 last_updated_ms: None,
164 }
165 }
166
167 pub fn from_table_creation(table_creation: TableCreation) -> Result<Self> {
169 let TableCreation {
170 name: _,
171 location,
172 schema,
173 partition_spec,
174 sort_order,
175 properties,
176 format_version,
177 } = table_creation;
178
179 let location =
180 location.ok_or_else(|| invalid_data!("Can't create table without location"))?;
181 let partition_spec = partition_spec.unwrap_or(UnboundPartitionSpec {
182 spec_id: None,
183 fields: vec![],
184 });
185
186 Self::new(
187 schema,
188 partition_spec,
189 sort_order.unwrap_or(SortOrder::unsorted_order()),
190 location,
191 format_version,
192 properties,
193 )
194 }
195
196 #[allow(dead_code)]
224 pub(crate) fn build_replacement(
225 self,
226 schema: Schema,
227 spec: impl Into<UnboundPartitionSpec>,
228 sort_order: SortOrder,
229 location: Option<String>,
230 format_version: Option<FormatVersion>,
231 properties: HashMap<String, String>,
232 ) -> Result<Self> {
233 if self.metadata.format_version == FormatVersion::V1 {
234 return Err(Error::new(
235 ErrorKind::FeatureUnsupported,
236 "Table replacement is not supported for format version 1",
237 ));
238 }
239
240 let format_version = format_version.unwrap_or(self.metadata.format_version);
241 let spec = spec.into();
242
243 let previous_id_to_name = schema.field_id_to_name_map().clone();
244 let fresh_schema = assign_fresh_ids(
245 schema,
246 self.get_current_schema()?,
247 self.metadata.last_column_id,
248 )?;
249
250 let (fresh_spec, fresh_sort_order) =
251 Self::rebind_to_schema(&previous_id_to_name, &fresh_schema, spec, sort_order)?;
252 let fresh_spec = self.reassign_partition_field_ids(fresh_spec, &fresh_schema)?;
253
254 let mut builder = self.upgrade_format_version(format_version)?;
255 if builder.metadata.refs.contains_key(MAIN_BRANCH) {
256 builder = builder.remove_ref(MAIN_BRANCH);
257 }
258
259 let builder = builder
260 .add_current_schema(fresh_schema)?
261 .add_default_partition_spec(fresh_spec)?
262 .add_default_sort_order(fresh_sort_order)?;
263
264 let builder = match location {
265 Some(location) => builder.set_location(location),
266 None => builder,
267 };
268
269 builder.set_properties(properties)
270 }
271
272 pub fn assign_uuid(mut self, uuid: Uuid) -> Self {
274 if self.metadata.table_uuid != uuid {
275 self.metadata.table_uuid = uuid;
276 self.changes.push(TableUpdate::AssignUuid { uuid });
277 }
278
279 self
280 }
281
282 pub fn upgrade_format_version(mut self, format_version: FormatVersion) -> Result<Self> {
287 if format_version < self.metadata.format_version {
288 return Err(invalid_data!(
289 "Cannot downgrade FormatVersion from {} to {}",
290 self.metadata.format_version,
291 format_version
292 ));
293 }
294
295 if format_version != self.metadata.format_version {
296 match format_version {
297 FormatVersion::V1 => {
298 }
300 FormatVersion::V2 => {
301 self.metadata.format_version = format_version;
302 self.changes
303 .push(TableUpdate::UpgradeFormatVersion { format_version });
304 }
305 FormatVersion::V3 => {
306 self.metadata.format_version = format_version;
307 self.metadata.next_row_id = INITIAL_ROW_ID;
309 self.changes
310 .push(TableUpdate::UpgradeFormatVersion { format_version });
311 }
312 }
313 }
314
315 Ok(self)
316 }
317
318 pub fn set_properties(mut self, properties: HashMap<String, String>) -> Result<Self> {
327 let reserved_properties = properties
329 .keys()
330 .filter(|key| TableProperties::RESERVED_PROPERTIES.contains(&key.as_str()))
331 .map(ToString::to_string)
332 .collect::<Vec<_>>();
333
334 if !reserved_properties.is_empty() {
335 return Err(invalid_data!(
336 "Table properties should not contain reserved properties, but got: [{}]",
337 reserved_properties.join(", ")
338 ));
339 }
340
341 if properties.is_empty() {
342 return Ok(self);
343 }
344
345 self.metadata.properties.extend(properties.clone());
346 self.changes.push(TableUpdate::SetProperties {
347 updates: properties,
348 });
349
350 Ok(self)
351 }
352
353 pub fn remove_properties(mut self, properties: &[String]) -> Result<Self> {
359 let properties = properties.iter().cloned().collect::<HashSet<_>>();
361
362 let reserved_properties = properties
364 .iter()
365 .filter(|key| TableProperties::RESERVED_PROPERTIES.contains(&key.as_str()))
366 .map(ToString::to_string)
367 .collect::<Vec<_>>();
368
369 if !reserved_properties.is_empty() {
370 return Err(invalid_data!(
371 "Table properties to remove contain reserved properties: [{}]",
372 reserved_properties.join(", ")
373 ));
374 }
375
376 for property in &properties {
377 self.metadata.properties.remove(property);
378 }
379
380 if !properties.is_empty() {
381 self.changes.push(TableUpdate::RemoveProperties {
382 removals: properties.into_iter().collect(),
383 });
384 }
385
386 Ok(self)
387 }
388
389 pub fn set_location(mut self, location: String) -> Self {
391 let location = location.trim_end_matches('/').to_string();
392 if self.metadata.location != location {
393 self.changes.push(TableUpdate::SetLocation {
394 location: location.clone(),
395 });
396 self.metadata.location = location;
397 }
398
399 self
400 }
401
402 pub fn add_snapshot(mut self, snapshot: Snapshot) -> Result<Self> {
411 if self
412 .metadata
413 .snapshots
414 .contains_key(&snapshot.snapshot_id())
415 {
416 return Err(invalid_data!(
417 "Snapshot already exists for: '{}'",
418 snapshot.snapshot_id()
419 ));
420 }
421
422 if self.metadata.format_version != FormatVersion::V1
423 && snapshot.sequence_number() <= self.metadata.last_sequence_number
424 && snapshot.parent_snapshot_id().is_some()
425 {
426 return Err(invalid_data!(
427 "Cannot add snapshot with sequence number {} older than last sequence number {}",
428 snapshot.sequence_number(),
429 self.metadata.last_sequence_number
430 ));
431 }
432
433 if let Some(last) = self.metadata.snapshot_log.last() {
434 if snapshot.timestamp_ms() - last.timestamp_ms < -ONE_MINUTE_MS {
437 return Err(invalid_data!(
438 "Invalid snapshot timestamp {}: before last snapshot timestamp {}",
439 snapshot.timestamp_ms(),
440 last.timestamp_ms
441 ));
442 }
443 }
444
445 let max_last_updated = self
446 .last_updated_ms
447 .unwrap_or_default()
448 .max(self.metadata.last_updated_ms);
449 if snapshot.timestamp_ms() - max_last_updated < -ONE_MINUTE_MS {
450 return Err(invalid_data!(
451 "Invalid snapshot timestamp {}: before last updated timestamp {}",
452 snapshot.timestamp_ms(),
453 max_last_updated
454 ));
455 }
456
457 let mut added_rows = None;
458 if self.metadata.format_version >= MIN_FORMAT_VERSION_ROW_LINEAGE {
459 if let Some((first_row_id, added_rows_count)) = snapshot.row_range() {
460 if first_row_id < self.metadata.next_row_id {
461 return Err(invalid_data!(
462 "Cannot add a snapshot, first-row-id is behind table next-row-id: {first_row_id} < {}",
463 self.metadata.next_row_id
464 ));
465 }
466
467 added_rows = Some(added_rows_count);
468 } else {
469 return Err(invalid_data!(
470 "Cannot add a snapshot: first-row-id is null. first-row-id must be set for format version >= {MIN_FORMAT_VERSION_ROW_LINEAGE}",
471 ));
472 }
473 }
474
475 if let Some(added_rows) = added_rows {
476 self.metadata.next_row_id = self
477 .metadata
478 .next_row_id
479 .checked_add(added_rows)
480 .ok_or_else(|| {
481 invalid_data!(
482 "Cannot add snapshot: next-row-id overflowed when adding added-rows"
483 )
484 })?;
485 }
486
487 self.changes.push(TableUpdate::AddSnapshot {
489 snapshot: snapshot.clone(),
490 });
491
492 self.last_updated_ms = Some(snapshot.timestamp_ms());
493 self.metadata.last_sequence_number = snapshot.sequence_number();
494 self.metadata
495 .snapshots
496 .insert(snapshot.snapshot_id(), snapshot.into());
497
498 Ok(self)
499 }
500
501 pub fn set_branch_snapshot(self, snapshot: Snapshot, branch: &str) -> Result<Self> {
507 let reference = self.metadata.refs.get(branch).cloned();
508
509 let reference = if let Some(mut reference) = reference {
510 if !reference.is_branch() {
511 return Err(invalid_data!(
512 "Cannot append snapshot to non-branch reference '{branch}'",
513 ));
514 }
515
516 reference.snapshot_id = snapshot.snapshot_id();
517 reference
518 } else {
519 SnapshotReference {
520 snapshot_id: snapshot.snapshot_id(),
521 retention: SnapshotRetention::Branch {
522 min_snapshots_to_keep: None,
523 max_snapshot_age_ms: None,
524 max_ref_age_ms: None,
525 },
526 }
527 };
528
529 self.add_snapshot(snapshot)?.set_ref(branch, reference)
530 }
531
532 pub fn remove_snapshots(mut self, snapshot_ids: &[i64]) -> Self {
538 let mut removed_snapshots = Vec::with_capacity(snapshot_ids.len());
539
540 for &snapshot_id in snapshot_ids {
541 if self.metadata.snapshots.remove(&snapshot_id).is_some() {
542 removed_snapshots.push(snapshot_id);
543 }
544 }
545
546 if !removed_snapshots.is_empty() {
547 self.changes.push(TableUpdate::RemoveSnapshots {
548 snapshot_ids: removed_snapshots,
549 });
550 }
551
552 let mut dangling_refs: Vec<String> = self
554 .metadata
555 .refs
556 .iter()
557 .filter(|(_, v)| !self.metadata.snapshots.contains_key(&v.snapshot_id))
558 .map(|(name, _)| name.clone())
559 .collect();
560 dangling_refs.sort();
561
562 for ref_name in dangling_refs {
563 self = self.remove_ref(&ref_name);
564 }
565
566 self
567 }
568
569 pub fn set_ref(mut self, ref_name: &str, reference: SnapshotReference) -> Result<Self> {
574 if self
575 .metadata
576 .refs
577 .get(ref_name)
578 .is_some_and(|snap_ref| snap_ref.eq(&reference))
579 {
580 return Ok(self);
581 }
582
583 let Some(snapshot) = self.metadata.snapshots.get(&reference.snapshot_id) else {
584 return Err(invalid_data!(
585 "Cannot set '{ref_name}' to unknown snapshot: '{}'",
586 reference.snapshot_id
587 ));
588 };
589
590 let is_added_snapshot = self.changes.iter().any(|update| {
592 matches!(update, TableUpdate::AddSnapshot { snapshot: snap } if snap.snapshot_id() == snapshot.snapshot_id())
593 });
594 if is_added_snapshot {
595 self.last_updated_ms = Some(snapshot.timestamp_ms());
596 }
597
598 if ref_name == MAIN_BRANCH {
600 self.metadata.current_snapshot_id = Some(snapshot.snapshot_id());
601 let timestamp_ms = if let Some(last_updated_ms) = self.last_updated_ms {
602 last_updated_ms
603 } else {
604 let last_updated_ms = chrono::Utc::now().timestamp_millis();
605 self.last_updated_ms = Some(last_updated_ms);
606 last_updated_ms
607 };
608
609 self.metadata.snapshot_log.push(SnapshotLog {
610 snapshot_id: snapshot.snapshot_id(),
611 timestamp_ms,
612 });
613 }
614
615 self.changes.push(TableUpdate::SetSnapshotRef {
616 ref_name: ref_name.to_string(),
617 reference: reference.clone(),
618 });
619 self.metadata.refs.insert(ref_name.to_string(), reference);
620
621 Ok(self)
622 }
623
624 pub fn remove_ref(mut self, ref_name: &str) -> Self {
628 if ref_name == MAIN_BRANCH {
629 self.metadata.current_snapshot_id = None;
630 }
631
632 if self.metadata.refs.remove(ref_name).is_some() || ref_name == MAIN_BRANCH {
633 self.changes.push(TableUpdate::RemoveSnapshotRef {
634 ref_name: ref_name.to_string(),
635 });
636 }
637
638 self
639 }
640
641 pub fn set_statistics(mut self, statistics: StatisticsFile) -> Self {
643 self.metadata
644 .statistics
645 .insert(statistics.snapshot_id, statistics.clone());
646 self.changes.push(TableUpdate::SetStatistics {
647 statistics: statistics.clone(),
648 });
649 self
650 }
651
652 pub fn remove_statistics(mut self, snapshot_id: i64) -> Self {
654 let previous = self.metadata.statistics.remove(&snapshot_id);
655 if previous.is_some() {
656 self.changes
657 .push(TableUpdate::RemoveStatistics { snapshot_id });
658 }
659 self
660 }
661
662 pub fn set_partition_statistics(
664 mut self,
665 partition_statistics_file: PartitionStatisticsFile,
666 ) -> Self {
667 self.metadata.partition_statistics.insert(
668 partition_statistics_file.snapshot_id,
669 partition_statistics_file.clone(),
670 );
671 self.changes.push(TableUpdate::SetPartitionStatistics {
672 partition_statistics: partition_statistics_file,
673 });
674 self
675 }
676
677 pub fn remove_partition_statistics(mut self, snapshot_id: i64) -> Self {
679 let previous = self.metadata.partition_statistics.remove(&snapshot_id);
680 if previous.is_some() {
681 self.changes
682 .push(TableUpdate::RemovePartitionStatistics { snapshot_id });
683 }
684 self
685 }
686
687 pub fn add_schema(mut self, schema: Schema) -> Result<Self> {
694 let new_schema_id = self.reuse_or_create_new_schema_id(&schema);
695 let schema_found = self.metadata.schemas.contains_key(&new_schema_id);
696
697 if schema_found {
698 if self.last_added_schema_id != Some(new_schema_id) {
699 self.changes.push(TableUpdate::AddSchema {
700 schema: schema.clone(),
701 });
702 self.last_added_schema_id = Some(new_schema_id);
703 }
704
705 return Ok(self);
706 }
707
708 self.metadata.last_column_id =
711 std::cmp::max(self.metadata.last_column_id, schema.highest_field_id());
712
713 let schema = match new_schema_id == schema.schema_id() {
715 true => schema,
716 false => schema.with_schema_id(new_schema_id),
717 };
718
719 self.metadata
720 .schemas
721 .insert(new_schema_id, schema.clone().into());
722
723 self.changes.push(TableUpdate::AddSchema { schema });
724
725 self.last_added_schema_id = Some(new_schema_id);
726
727 Ok(self)
728 }
729
730 pub fn set_current_schema(mut self, mut schema_id: i32) -> Result<Self> {
739 if schema_id == Self::LAST_ADDED {
740 schema_id = self.last_added_schema_id.ok_or_else(|| {
741 invalid_data!(
742 "Cannot set current schema to last added schema: no schema has been added."
743 )
744 })?;
745 };
746 let schema_id = schema_id; if schema_id == self.metadata.current_schema_id {
749 return Ok(self);
750 }
751
752 let schema = self.metadata.schemas.get(&schema_id).ok_or_else(|| {
753 invalid_data!("Cannot set current schema to unknown schema with id: '{schema_id}'")
754 })?;
755 self.validate_schema_field_names(schema)?;
756
757 self.metadata.current_schema_id = schema_id;
763
764 if self.last_added_schema_id == Some(schema_id) {
765 self.changes.push(TableUpdate::SetCurrentSchema {
766 schema_id: Self::LAST_ADDED,
767 });
768 } else {
769 self.changes
770 .push(TableUpdate::SetCurrentSchema { schema_id });
771 }
772
773 Ok(self)
774 }
775
776 pub fn add_current_schema(self, schema: Schema) -> Result<Self> {
778 self.add_schema(schema)?
779 .set_current_schema(Self::LAST_ADDED)
780 }
781
782 fn validate_schema_field_names(&self, schema: &Schema) -> Result<()> {
792 for (spec_id, spec) in &self.metadata.partition_specs {
793 for partition_field in spec.fields() {
794 if schema.field_by_id(partition_field.source_id).is_none() {
795 continue;
796 }
797
798 let Some(schema_field) = schema.field_by_name(&partition_field.name) else {
799 continue;
800 };
801
802 if schema_field.id != partition_field.source_id {
803 return Err(invalid_data!(
804 "Cannot set current schema {}: field '{}' with id {} conflicts with a \
805 partition field of that name in spec {} sourced from field id {}",
806 schema.schema_id(),
807 partition_field.name,
808 schema_field.id,
809 spec_id,
810 partition_field.source_id
811 ));
812 }
813 }
814 }
815
816 Ok(())
817 }
818
819 fn validate_partition_field_names(&self, unbound_spec: &UnboundPartitionSpec) -> Result<()> {
830 if self.metadata.schemas.is_empty() {
831 return Ok(());
832 }
833
834 let current_schema = self.get_current_schema()?;
835 for partition_field in unbound_spec.fields() {
836 let exists_in_any_schema = self
837 .metadata
838 .name_exists_in_any_schema(partition_field.name());
839
840 if !exists_in_any_schema {
842 continue;
843 }
844
845 if let Some(schema_field) = current_schema.field_by_name(partition_field.name()) {
847 let has_matching_source_id = schema_field.id == partition_field.source_id()?;
848
849 if !partition_field.transform().may_share_name_with_source() {
850 return Err(invalid_data!(
851 "Cannot create partition with name '{}' that conflicts with schema field and is not an identity or void transform.",
852 partition_field.name()
853 ));
854 }
855
856 if !has_matching_source_id {
857 return Err(invalid_data!(
858 "Cannot create identity or void partition sourced from different field in schema. \
859 Field name '{}' has id `{}` in schema but partition source id is `{}`",
860 partition_field.name(),
861 schema_field.id,
862 partition_field.source_id()?
863 ));
864 }
865 }
866 }
867
868 Ok(())
869 }
870
871 pub fn add_partition_spec(mut self, unbound_spec: UnboundPartitionSpec) -> Result<Self> {
882 let schema = self.get_current_schema()?.clone();
883
884 self.validate_partition_field_names(&unbound_spec)?;
886
887 let unbound_spec = self.reuse_partition_field_ids(unbound_spec)?;
889
890 let spec = PartitionSpecBuilder::new_from_unbound(unbound_spec.clone(), schema)?
891 .with_last_assigned_field_id(self.metadata.last_partition_id)
892 .build()?;
893
894 let new_spec_id = self.reuse_or_create_new_spec_id(&spec);
895 let spec_found = self.metadata.partition_specs.contains_key(&new_spec_id);
896 let spec = spec.with_spec_id(new_spec_id);
897 let unbound_spec = unbound_spec.with_spec_id(new_spec_id);
898
899 if spec_found {
900 if self.last_added_spec_id != Some(new_spec_id) {
901 self.changes
902 .push(TableUpdate::AddSpec { spec: unbound_spec });
903 self.last_added_spec_id = Some(new_spec_id);
904 }
905
906 return Ok(self);
907 }
908
909 if self.metadata.format_version <= FormatVersion::V1 && !spec.has_sequential_ids() {
910 return Err(invalid_data!(
911 "Cannot add partition spec with non-sequential field ids to format version 1 table"
912 ));
913 }
914
915 let highest_field_id = spec
916 .highest_field_id()
917 .unwrap_or(UNPARTITIONED_LAST_ASSIGNED_ID);
918 self.metadata
919 .partition_specs
920 .insert(new_spec_id, Arc::new(spec));
921 self.changes
922 .push(TableUpdate::AddSpec { spec: unbound_spec });
923
924 self.last_added_spec_id = Some(new_spec_id);
925 self.metadata.last_partition_id =
926 std::cmp::max(self.metadata.last_partition_id, highest_field_id);
927
928 Ok(self)
929 }
930
931 fn reuse_partition_field_ids(
936 &self,
937 unbound_spec: UnboundPartitionSpec,
938 ) -> Result<UnboundPartitionSpec> {
939 let equivalent_field_ids = Self::highest_equivalent_field_ids(
940 self.metadata
941 .partition_specs
942 .values()
943 .flat_map(|spec| spec.fields()),
944 );
945
946 let fields = unbound_spec
948 .fields
949 .into_iter()
950 .map(|field| {
951 if field.field_id().is_none()
952 && let Some(&existing_field_id) =
953 equivalent_field_ids.get(&(field.source_ids().to_vec(), field.transform()))
954 {
955 field.with_field_id(existing_field_id)
956 } else {
957 field
958 }
959 })
960 .collect();
961
962 Ok(UnboundPartitionSpec {
963 spec_id: unbound_spec.spec_id,
964 fields,
965 })
966 }
967
968 fn highest_equivalent_field_ids<'a>(
973 fields: impl Iterator<Item = &'a PartitionField>,
974 ) -> HashMap<(Vec<i32>, Transform), i32> {
975 let mut equivalent_field_ids = HashMap::new();
976 for field in fields {
977 equivalent_field_ids
978 .entry((vec![field.source_id], field.transform))
979 .and_modify(|id: &mut i32| *id = (*id).max(field.field_id))
980 .or_insert(field.field_id);
981 }
982 equivalent_field_ids
983 }
984
985 pub fn set_default_partition_spec(mut self, mut spec_id: i32) -> Result<Self> {
991 if spec_id == Self::LAST_ADDED {
992 spec_id = self.last_added_spec_id.ok_or_else(|| {
993 invalid_data!(
994 "Cannot set default partition spec to last added spec: no spec has been added."
995 )
996 })?;
997 }
998
999 if self.metadata.default_spec.spec_id() == spec_id {
1000 return Ok(self);
1001 }
1002
1003 if !self.metadata.partition_specs.contains_key(&spec_id) {
1004 return Err(invalid_data!(
1005 "Cannot set default partition spec to unknown spec with id: '{spec_id}'",
1006 ));
1007 }
1008
1009 let schemaless_spec = self
1010 .metadata
1011 .partition_specs
1012 .get(&spec_id)
1013 .ok_or_else(|| {
1014 invalid_data!(
1015 "Cannot set default partition spec to unknown spec with id: '{spec_id}'",
1016 )
1017 })?
1018 .clone();
1019 let spec = Arc::unwrap_or_clone(schemaless_spec);
1020 let spec_type = spec.partition_type(self.get_current_schema()?)?;
1021 self.metadata.default_spec = Arc::new(spec);
1022 self.metadata.default_partition_type = spec_type;
1023
1024 if self.last_added_spec_id == Some(spec_id) {
1025 self.changes.push(TableUpdate::SetDefaultSpec {
1026 spec_id: Self::LAST_ADDED,
1027 });
1028 } else {
1029 self.changes.push(TableUpdate::SetDefaultSpec { spec_id });
1030 }
1031
1032 Ok(self)
1033 }
1034
1035 pub fn add_default_partition_spec(self, unbound_spec: UnboundPartitionSpec) -> Result<Self> {
1037 self.add_partition_spec(unbound_spec)?
1038 .set_default_partition_spec(Self::LAST_ADDED)
1039 }
1040
1041 pub fn remove_partition_specs(mut self, spec_ids: &[i32]) -> Result<Self> {
1048 if spec_ids.contains(&self.metadata.default_spec.spec_id()) {
1049 return Err(invalid_data!("Cannot remove default partition spec"));
1050 }
1051
1052 let mut removed_specs = Vec::with_capacity(spec_ids.len());
1053 spec_ids.iter().for_each(|id| {
1054 if self.metadata.partition_specs.remove(id).is_some() {
1055 removed_specs.push(*id);
1056 }
1057 });
1058
1059 if !removed_specs.is_empty() {
1060 self.changes.push(TableUpdate::RemovePartitionSpecs {
1061 spec_ids: removed_specs,
1062 });
1063 }
1064
1065 Ok(self)
1066 }
1067
1068 pub fn add_sort_order(mut self, sort_order: SortOrder) -> Result<Self> {
1079 let new_order_id = self.reuse_or_create_new_sort_id(&sort_order);
1080 let sort_order_found = self.metadata.sort_orders.contains_key(&new_order_id);
1081
1082 if sort_order_found {
1083 if self.last_added_order_id != Some(new_order_id) {
1084 self.changes.push(TableUpdate::AddSortOrder {
1085 sort_order: sort_order.clone().with_order_id(new_order_id),
1086 });
1087 self.last_added_order_id = Some(new_order_id);
1088 }
1089
1090 return Ok(self);
1091 }
1092
1093 let schema = self.get_current_schema()?.clone().as_ref().clone();
1094 let sort_order = SortOrder::builder()
1095 .with_order_id(new_order_id)
1096 .with_fields(sort_order.fields)
1097 .build(&schema)
1098 .map_err(|e| {
1099 invalid_data!("Sort order to add is incompatible with current schema: {e}")
1100 .with_source(e)
1101 })?;
1102
1103 self.last_added_order_id = Some(new_order_id);
1104 self.metadata
1105 .sort_orders
1106 .insert(new_order_id, sort_order.clone().into());
1107 self.changes.push(TableUpdate::AddSortOrder { sort_order });
1108
1109 Ok(self)
1110 }
1111
1112 pub fn set_default_sort_order(mut self, mut sort_order_id: i64) -> Result<Self> {
1118 if sort_order_id == Self::LAST_ADDED as i64 {
1119 sort_order_id = self.last_added_order_id.ok_or_else(|| {
1120 invalid_data!(
1121 "Cannot set default sort order to last added order: no order has been added."
1122 )
1123 })?;
1124 }
1125
1126 if self.metadata.default_sort_order_id == sort_order_id {
1127 return Ok(self);
1128 }
1129
1130 if !self.metadata.sort_orders.contains_key(&sort_order_id) {
1131 return Err(invalid_data!(
1132 "Cannot set default sort order to unknown order with id: '{sort_order_id}'"
1133 ));
1134 }
1135
1136 self.metadata.default_sort_order_id = sort_order_id;
1137
1138 if self.last_added_order_id == Some(sort_order_id) {
1139 self.changes.push(TableUpdate::SetDefaultSortOrder {
1140 sort_order_id: Self::LAST_ADDED as i64,
1141 });
1142 } else {
1143 self.changes
1144 .push(TableUpdate::SetDefaultSortOrder { sort_order_id });
1145 }
1146
1147 Ok(self)
1148 }
1149
1150 fn add_default_sort_order(self, sort_order: SortOrder) -> Result<Self> {
1152 self.add_sort_order(sort_order)?
1153 .set_default_sort_order(Self::LAST_ADDED as i64)
1154 }
1155
1156 pub fn add_encryption_key(mut self, key: EncryptedKey) -> Self {
1158 let key_id = key.key_id().to_string();
1159 if self.metadata.encryption_keys.contains_key(&key_id) {
1160 return self;
1162 }
1163
1164 self.metadata.encryption_keys.insert(key_id, key.clone());
1165 self.changes.push(TableUpdate::AddEncryptionKey {
1166 encryption_key: key,
1167 });
1168 self
1169 }
1170
1171 pub fn remove_encryption_key(mut self, key_id: &str) -> Self {
1173 if self.metadata.encryption_keys.remove(key_id).is_some() {
1174 self.changes.push(TableUpdate::RemoveEncryptionKey {
1175 key_id: key_id.to_string(),
1176 });
1177 }
1178 self
1179 }
1180
1181 pub fn build(mut self) -> Result<TableMetadataBuildResult> {
1183 self.metadata.last_updated_ms = self
1184 .last_updated_ms
1185 .unwrap_or_else(|| chrono::Utc::now().timestamp_millis());
1186
1187 self.check_format_compatibility()?;
1188
1189 let schema = self.get_current_schema()?.clone();
1193 let sort_order = Arc::unwrap_or_clone(self.get_default_sort_order()?);
1194
1195 self.metadata.default_spec = Arc::new(
1196 Arc::unwrap_or_clone(self.metadata.default_spec)
1197 .into_unbound()
1198 .bind(schema.clone())?,
1199 );
1200 self.metadata.default_partition_type =
1201 self.metadata.default_spec.partition_type(&schema)?;
1202 SortOrder::builder()
1203 .with_fields(sort_order.fields)
1204 .build(&schema)?;
1205
1206 self.update_snapshot_log()?;
1207
1208 if let Some(hist_entry) = self.previous_history_entry.take() {
1210 self.metadata.metadata_log.push(hist_entry);
1211 }
1212 let expired_metadata_logs = self.expire_metadata_log();
1213 self.metadata.try_normalize()?;
1214
1215 Ok(TableMetadataBuildResult {
1216 metadata: self.metadata,
1217 changes: self.changes,
1218 expired_metadata_logs,
1219 })
1220 }
1221
1222 fn expire_metadata_log(&mut self) -> Vec<MetadataLog> {
1223 let max_size = self
1224 .metadata
1225 .properties
1226 .get(TableProperties::PROPERTY_METADATA_PREVIOUS_VERSIONS_MAX)
1227 .and_then(|v| v.parse::<usize>().ok())
1228 .unwrap_or(TableProperties::PROPERTY_METADATA_PREVIOUS_VERSIONS_MAX_DEFAULT)
1229 .max(1);
1230
1231 if self.metadata.metadata_log.len() > max_size {
1232 self.metadata
1233 .metadata_log
1234 .drain(0..self.metadata.metadata_log.len() - max_size)
1235 .collect()
1236 } else {
1237 Vec::new()
1238 }
1239 }
1240
1241 fn update_snapshot_log(&mut self) -> Result<()> {
1242 let intermediate_snapshots = self.get_intermediate_snapshots();
1243 let has_removed_snapshots = self
1244 .changes
1245 .iter()
1246 .any(|update| matches!(update, TableUpdate::RemoveSnapshots { .. }));
1247
1248 if intermediate_snapshots.is_empty() && !has_removed_snapshots {
1249 return Ok(());
1250 }
1251
1252 let mut new_snapshot_log = Vec::new();
1253 for log_entry in &self.metadata.snapshot_log {
1254 let snapshot_id = log_entry.snapshot_id;
1255 if self.metadata.snapshots.contains_key(&snapshot_id) {
1256 if !intermediate_snapshots.contains(&snapshot_id) {
1257 new_snapshot_log.push(log_entry.clone());
1258 }
1259 } else if has_removed_snapshots {
1260 new_snapshot_log.clear();
1266 }
1267 }
1268
1269 if let Some(current_snapshot_id) = self.metadata.current_snapshot_id {
1270 let last_id = new_snapshot_log.last().map(|entry| entry.snapshot_id);
1271 if last_id != Some(current_snapshot_id) {
1272 return Err(invalid_data!(
1273 "Cannot set invalid snapshot log: latest entry is not the current snapshot"
1274 ));
1275 }
1276 };
1277
1278 self.metadata.snapshot_log = new_snapshot_log;
1279 Ok(())
1280 }
1281
1282 fn get_intermediate_snapshots(&self) -> HashSet<i64> {
1292 let added_snapshot_ids = self
1293 .changes
1294 .iter()
1295 .filter_map(|update| match update {
1296 TableUpdate::AddSnapshot { snapshot } => Some(snapshot.snapshot_id()),
1297 _ => None,
1298 })
1299 .collect::<HashSet<_>>();
1300
1301 self.changes
1302 .iter()
1303 .filter_map(|update| match update {
1304 TableUpdate::SetSnapshotRef {
1305 ref_name,
1306 reference,
1307 } => {
1308 if added_snapshot_ids.contains(&reference.snapshot_id)
1309 && ref_name == MAIN_BRANCH
1310 && reference.snapshot_id
1311 != self
1312 .metadata
1313 .current_snapshot_id
1314 .unwrap_or(i64::from(Self::LAST_ADDED))
1315 {
1316 Some(reference.snapshot_id)
1317 } else {
1318 None
1319 }
1320 }
1321 _ => None,
1322 })
1323 .collect()
1324 }
1325
1326 fn reassign_partition_field_ids(
1329 &self,
1330 spec: PartitionSpec,
1331 schema: &Schema,
1332 ) -> Result<UnboundPartitionSpec> {
1333 let fields = spec
1334 .fields()
1335 .iter()
1336 .map(|field| {
1337 UnboundPartitionField::builder()
1338 .source_ids(vec![field.source_id])
1339 .name(field.name.clone())
1340 .transform(field.transform)
1341 .build()
1342 })
1343 .collect::<Result<Vec<_>>>()?;
1344 let spec = self.reuse_partition_field_ids(
1345 UnboundPartitionSpec::builder()
1346 .add_partition_fields(fields)?
1347 .build(),
1348 )?;
1349
1350 Ok(
1351 PartitionSpecBuilder::new_from_unbound(spec, schema.clone())?
1352 .with_last_assigned_field_id(self.metadata.last_partition_id)
1353 .build()?
1354 .into_unbound(),
1355 )
1356 }
1357
1358 fn reassign_ids(
1359 schema: Schema,
1360 spec: UnboundPartitionSpec,
1361 sort_order: SortOrder,
1362 ) -> Result<(Schema, PartitionSpec, SortOrder)> {
1363 let previous_id_to_name = schema.field_id_to_name_map().clone();
1365 let fresh_schema = schema
1366 .into_builder()
1367 .with_schema_id(DEFAULT_SCHEMA_ID)
1368 .with_reassigned_field_ids(FIRST_FIELD_ID)
1369 .build()?;
1370
1371 let (fresh_spec, fresh_sort_order) =
1372 Self::rebind_to_schema(&previous_id_to_name, &fresh_schema, spec, sort_order)?;
1373
1374 Ok((fresh_schema, fresh_spec, fresh_sort_order))
1375 }
1376
1377 fn rebind_to_schema(
1380 previous_id_to_name: &HashMap<i32, String>,
1381 fresh_schema: &Schema,
1382 spec: UnboundPartitionSpec,
1383 sort_order: SortOrder,
1384 ) -> Result<(PartitionSpec, SortOrder)> {
1385 let mut fresh_spec = PartitionSpecBuilder::new(fresh_schema.clone());
1387 for field in spec.fields() {
1388 let source_id = field.source_id()?;
1389 let source_field_name = previous_id_to_name.get(&source_id).ok_or_else(|| {
1390 invalid_data!(
1391 "Cannot find source column with id {} for partition column {} in schema.",
1392 source_id,
1393 field.name()
1394 )
1395 })?;
1396 fresh_spec = fresh_spec.add_partition_field(
1397 source_field_name,
1398 field.name(),
1399 field.transform(),
1400 )?;
1401 }
1402 let fresh_spec = fresh_spec.build()?;
1403
1404 let mut fresh_order = SortOrder::builder();
1406 for mut field in sort_order.fields {
1407 let source_field_name = previous_id_to_name.get(&field.source_id).ok_or_else(|| {
1408 invalid_data!(
1409 "Cannot find source column with id {} for sort column in schema.",
1410 field.source_id
1411 )
1412 })?;
1413 let new_field_id = fresh_schema
1414 .field_by_name(source_field_name)
1415 .ok_or_else(|| {
1416 Error::new(
1417 ErrorKind::Unexpected,
1418 format!(
1419 "Cannot find source column with name {source_field_name} for sort column in re-assigned schema."
1420 ),
1421 )
1422 })?.id;
1423 field.source_id = new_field_id;
1424 fresh_order.with_sort_field(field);
1425 }
1426 let fresh_sort_order = fresh_order.build(fresh_schema)?;
1427
1428 Ok((fresh_spec, fresh_sort_order))
1429 }
1430
1431 fn check_format_compatibility(&self) -> Result<()> {
1432 let format_version = self.metadata.format_version;
1433 if format_version < FormatVersion::V3 {
1434 let encryption_properties = [
1435 TableProperties::PROPERTY_ENCRYPTION_KEY_ID,
1436 TableProperties::PROPERTY_ENCRYPTION_DATA_KEY_LENGTH,
1437 ]
1438 .into_iter()
1439 .filter(|key| self.metadata.properties.contains_key(*key))
1440 .collect::<Vec<_>>();
1441 if !encryption_properties.is_empty() {
1442 return Err(invalid_data!(
1443 "Invalid properties for {format_version}: [{}]",
1444 encryption_properties.join(", ")
1445 ));
1446 }
1447
1448 if !self.metadata.encryption_keys.is_empty() {
1449 return Err(invalid_data!(
1450 "Encryption keys are not supported for {format_version} tables"
1451 ));
1452 }
1453 }
1454
1455 Ok(())
1456 }
1457
1458 fn reuse_or_create_new_schema_id(&self, new_schema: &Schema) -> i32 {
1459 self.metadata
1460 .schemas
1461 .iter()
1462 .find_map(|(id, schema)| new_schema.is_same_schema(schema).then_some(*id))
1463 .unwrap_or_else(|| self.get_highest_schema_id() + 1)
1464 }
1465
1466 fn get_highest_schema_id(&self) -> i32 {
1467 *self
1468 .metadata
1469 .schemas
1470 .keys()
1471 .max()
1472 .unwrap_or(&self.metadata.current_schema_id)
1473 }
1474
1475 fn get_current_schema(&self) -> Result<&SchemaRef> {
1476 self.metadata
1477 .schemas
1478 .get(&self.metadata.current_schema_id)
1479 .ok_or_else(|| {
1480 invalid_data!(
1481 "Current schema with id '{}' not found in table metadata.",
1482 self.metadata.current_schema_id
1483 )
1484 })
1485 }
1486
1487 fn get_default_sort_order(&self) -> Result<SortOrderRef> {
1488 self.metadata
1489 .sort_orders
1490 .get(&self.metadata.default_sort_order_id)
1491 .cloned()
1492 .ok_or_else(|| {
1493 invalid_data!(
1494 "Default sort order with id '{}' not found in table metadata.",
1495 self.metadata.default_sort_order_id
1496 )
1497 })
1498 }
1499
1500 fn reuse_or_create_new_spec_id(&self, new_spec: &PartitionSpec) -> i32 {
1502 self.metadata
1503 .partition_specs
1504 .iter()
1505 .find_map(|(id, old_spec)| new_spec.is_compatible_with(old_spec).then_some(*id))
1506 .unwrap_or_else(|| {
1507 self.get_highest_spec_id()
1508 .map(|id| id + 1)
1509 .unwrap_or(DEFAULT_PARTITION_SPEC_ID)
1510 })
1511 }
1512
1513 fn get_highest_spec_id(&self) -> Option<i32> {
1514 self.metadata.partition_specs.keys().max().copied()
1515 }
1516
1517 fn reuse_or_create_new_sort_id(&self, new_sort_order: &SortOrder) -> i64 {
1519 if new_sort_order.is_unsorted() {
1520 return SortOrder::unsorted_order().order_id;
1521 }
1522
1523 self.metadata
1524 .sort_orders
1525 .iter()
1526 .find_map(|(id, sort_order)| {
1527 sort_order.fields.eq(&new_sort_order.fields).then_some(*id)
1528 })
1529 .unwrap_or_else(|| {
1530 self.highest_sort_order_id()
1531 .unwrap_or(SortOrder::unsorted_order().order_id)
1532 + 1
1533 })
1534 }
1535
1536 fn highest_sort_order_id(&self) -> Option<i64> {
1537 self.metadata.sort_orders.keys().max().copied()
1538 }
1539
1540 pub fn remove_schemas(mut self, schema_id_to_remove: &[i32]) -> Result<Self> {
1543 if schema_id_to_remove.contains(&self.metadata.current_schema_id) {
1544 return Err(invalid_data!("Cannot remove current schema"));
1545 }
1546
1547 if schema_id_to_remove.is_empty() {
1548 return Ok(self);
1549 }
1550
1551 let mut removed_schemas = Vec::with_capacity(schema_id_to_remove.len());
1552 self.metadata.schemas.retain(|id, _schema| {
1553 if schema_id_to_remove.contains(id) {
1554 removed_schemas.push(*id);
1555 false
1556 } else {
1557 true
1558 }
1559 });
1560
1561 self.changes.push(TableUpdate::RemoveSchemas {
1562 schema_ids: removed_schemas,
1563 });
1564
1565 Ok(self)
1566 }
1567}
1568
1569impl From<TableMetadataBuildResult> for TableMetadata {
1570 fn from(result: TableMetadataBuildResult) -> Self {
1571 result.metadata
1572 }
1573}
1574
1575#[cfg(test)]
1576mod tests {
1577 use std::fs::File;
1578 use std::io::BufReader;
1579 use std::thread::sleep;
1580
1581 use super::*;
1582 use crate::TableIdent;
1583 use crate::io::FileIO;
1584 use crate::spec::{
1585 BlobMetadata, NestedField, NullOrder, Operation, PartitionSpec, PrimitiveType, Schema,
1586 SnapshotRetention, SortDirection, SortField, StructType, Summary, TableProperties,
1587 Transform, Type, UnboundPartitionField,
1588 };
1589 use crate::table::Table;
1590 use crate::test_utils::scan::TableTestFixture;
1591 use crate::test_utils::test_runtime;
1592
1593 const TEST_LOCATION: &str = "s3://bucket/test/location";
1594 const LAST_ASSIGNED_COLUMN_ID: i32 = 3;
1595
1596 fn schema() -> Schema {
1597 Schema::builder()
1598 .with_fields(vec![
1599 NestedField::required(1, "x", Type::Primitive(PrimitiveType::Long)).into(),
1600 NestedField::required(2, "y", Type::Primitive(PrimitiveType::Long)).into(),
1601 NestedField::required(3, "z", Type::Primitive(PrimitiveType::Long)).into(),
1602 ])
1603 .build()
1604 .unwrap()
1605 }
1606
1607 fn sort_order() -> SortOrder {
1608 let schema = schema();
1609 SortOrder::builder()
1610 .with_order_id(1)
1611 .with_sort_field(SortField {
1612 source_id: 3,
1613 transform: Transform::Bucket(4),
1614 direction: SortDirection::Descending,
1615 null_order: NullOrder::First,
1616 })
1617 .build(&schema)
1618 .unwrap()
1619 }
1620
1621 fn partition_spec() -> UnboundPartitionSpec {
1622 UnboundPartitionSpec::builder()
1623 .with_spec_id(0)
1624 .add_partition_field(
1625 UnboundPartitionField::builder()
1626 .source_ids(vec![2])
1627 .name("y")
1628 .transform(Transform::Identity)
1629 .build()
1630 .unwrap(),
1631 )
1632 .unwrap()
1633 .build()
1634 }
1635
1636 fn builder_without_changes(format_version: FormatVersion) -> TableMetadataBuilder {
1637 TableMetadataBuilder::new(
1638 schema(),
1639 partition_spec(),
1640 sort_order(),
1641 TEST_LOCATION.to_string(),
1642 format_version,
1643 HashMap::new(),
1644 )
1645 .unwrap()
1646 .build()
1647 .unwrap()
1648 .metadata
1649 .into_builder(Some(
1650 "s3://bucket/test/location/metadata/metadata1.json".to_string(),
1651 ))
1652 }
1653
1654 #[test]
1655 fn test_minimal_build() {
1656 let metadata = TableMetadataBuilder::new(
1657 schema(),
1658 partition_spec(),
1659 sort_order(),
1660 TEST_LOCATION.to_string(),
1661 FormatVersion::V1,
1662 HashMap::new(),
1663 )
1664 .unwrap()
1665 .build()
1666 .unwrap()
1667 .metadata;
1668
1669 assert_eq!(metadata.format_version, FormatVersion::V1);
1670 assert_eq!(metadata.location, TEST_LOCATION);
1671 assert_eq!(metadata.current_schema_id, 0);
1672 assert_eq!(metadata.default_spec.spec_id(), 0);
1673 assert_eq!(metadata.default_sort_order_id, 1);
1674 assert_eq!(metadata.last_partition_id, 1000);
1675 assert_eq!(metadata.last_column_id, 3);
1676 assert_eq!(metadata.snapshots.len(), 0);
1677 assert_eq!(metadata.current_snapshot_id, None);
1678 assert_eq!(metadata.refs.len(), 0);
1679 assert_eq!(metadata.properties.len(), 0);
1680 assert_eq!(metadata.metadata_log.len(), 0);
1681 assert_eq!(metadata.last_sequence_number, 0);
1682 assert_eq!(metadata.last_column_id, LAST_ASSIGNED_COLUMN_ID);
1683
1684 let _ = serde_json::to_string(&metadata).unwrap();
1686
1687 let metadata = metadata
1689 .into_builder(Some(
1690 "s3://bucket/test/location/metadata/metadata1.json".to_string(),
1691 ))
1692 .upgrade_format_version(FormatVersion::V2)
1693 .unwrap()
1694 .build()
1695 .unwrap()
1696 .metadata;
1697
1698 assert_eq!(metadata.format_version, FormatVersion::V2);
1699 let _ = serde_json::to_string(&metadata).unwrap();
1700 }
1701
1702 #[test]
1703 fn test_build_unpartitioned_unsorted() {
1704 let schema = Schema::builder().build().unwrap();
1705 let metadata = TableMetadataBuilder::new(
1706 schema.clone(),
1707 PartitionSpec::unpartition_spec(),
1708 SortOrder::unsorted_order(),
1709 TEST_LOCATION.to_string(),
1710 FormatVersion::V2,
1711 HashMap::new(),
1712 )
1713 .unwrap()
1714 .build()
1715 .unwrap()
1716 .metadata;
1717
1718 assert_eq!(metadata.format_version, FormatVersion::V2);
1719 assert_eq!(metadata.location, TEST_LOCATION);
1720 assert_eq!(metadata.current_schema_id, 0);
1721 assert_eq!(metadata.default_spec.spec_id(), 0);
1722 assert_eq!(metadata.default_sort_order_id, 0);
1723 assert_eq!(metadata.last_partition_id, UNPARTITIONED_LAST_ASSIGNED_ID);
1724 assert_eq!(metadata.last_column_id, 0);
1725 assert_eq!(metadata.snapshots.len(), 0);
1726 assert_eq!(metadata.current_snapshot_id, None);
1727 assert_eq!(metadata.refs.len(), 0);
1728 assert_eq!(metadata.properties.len(), 0);
1729 assert_eq!(metadata.metadata_log.len(), 0);
1730 assert_eq!(metadata.last_sequence_number, 0);
1731 }
1732
1733 #[test]
1734 fn test_reassigns_ids() {
1735 let schema = Schema::builder()
1736 .with_schema_id(10)
1737 .with_fields(vec![
1738 NestedField::required(11, "a", Type::Primitive(PrimitiveType::Long)).into(),
1739 NestedField::required(12, "b", Type::Primitive(PrimitiveType::Long)).into(),
1740 NestedField::required(
1741 13,
1742 "struct",
1743 Type::Struct(StructType::new(vec![
1744 NestedField::required(14, "nested", Type::Primitive(PrimitiveType::Long))
1745 .into(),
1746 ])),
1747 )
1748 .into(),
1749 NestedField::required(15, "c", Type::Primitive(PrimitiveType::Long)).into(),
1750 ])
1751 .build()
1752 .unwrap();
1753 let spec = PartitionSpec::builder(schema.clone())
1754 .with_spec_id(20)
1755 .add_partition_field("a", "a", Transform::Identity)
1756 .unwrap()
1757 .add_partition_field("struct.nested", "nested_partition", Transform::Identity)
1758 .unwrap()
1759 .build()
1760 .unwrap();
1761 let sort_order = SortOrder::builder()
1762 .with_fields(vec![SortField {
1763 source_id: 11,
1764 transform: Transform::Identity,
1765 direction: SortDirection::Ascending,
1766 null_order: NullOrder::First,
1767 }])
1768 .with_order_id(10)
1769 .build(&schema)
1770 .unwrap();
1771
1772 let (fresh_schema, fresh_spec, fresh_sort_order) =
1773 TableMetadataBuilder::reassign_ids(schema, spec.into_unbound(), sort_order).unwrap();
1774
1775 let expected_schema = Schema::builder()
1776 .with_fields(vec![
1777 NestedField::required(1, "a", Type::Primitive(PrimitiveType::Long)).into(),
1778 NestedField::required(2, "b", Type::Primitive(PrimitiveType::Long)).into(),
1779 NestedField::required(
1780 3,
1781 "struct",
1782 Type::Struct(StructType::new(vec![
1783 NestedField::required(5, "nested", Type::Primitive(PrimitiveType::Long))
1784 .into(),
1785 ])),
1786 )
1787 .into(),
1788 NestedField::required(4, "c", Type::Primitive(PrimitiveType::Long)).into(),
1789 ])
1790 .build()
1791 .unwrap();
1792
1793 let expected_spec = PartitionSpec::builder(expected_schema.clone())
1794 .with_spec_id(0)
1795 .add_partition_field("a", "a", Transform::Identity)
1796 .unwrap()
1797 .add_partition_field("struct.nested", "nested_partition", Transform::Identity)
1798 .unwrap()
1799 .build()
1800 .unwrap();
1801
1802 let expected_sort_order = SortOrder::builder()
1803 .with_fields(vec![SortField {
1804 source_id: 1,
1805 transform: Transform::Identity,
1806 direction: SortDirection::Ascending,
1807 null_order: NullOrder::First,
1808 }])
1809 .with_order_id(1)
1810 .build(&expected_schema)
1811 .unwrap();
1812
1813 assert_eq!(fresh_schema, expected_schema);
1814 assert_eq!(fresh_spec, expected_spec);
1815 assert_eq!(fresh_sort_order, expected_sort_order);
1816 }
1817
1818 #[test]
1819 fn test_ids_are_reassigned_for_new_metadata() {
1820 let schema = schema().into_builder().with_schema_id(10).build().unwrap();
1821
1822 let metadata = TableMetadataBuilder::new(
1823 schema,
1824 partition_spec(),
1825 sort_order(),
1826 TEST_LOCATION.to_string(),
1827 FormatVersion::V1,
1828 HashMap::new(),
1829 )
1830 .unwrap()
1831 .build()
1832 .unwrap()
1833 .metadata;
1834
1835 assert_eq!(metadata.current_schema_id, 0);
1836 assert_eq!(metadata.current_schema().schema_id(), 0);
1837 }
1838
1839 #[test]
1840 fn test_new_metadata_changes() {
1841 let changes = TableMetadataBuilder::new(
1842 schema(),
1843 partition_spec(),
1844 sort_order(),
1845 TEST_LOCATION.to_string(),
1846 FormatVersion::V1,
1847 HashMap::from_iter(vec![("property 1".to_string(), "value 1".to_string())]),
1848 )
1849 .unwrap()
1850 .build()
1851 .unwrap()
1852 .changes;
1853
1854 pretty_assertions::assert_eq!(changes, vec![
1855 TableUpdate::SetLocation {
1856 location: TEST_LOCATION.to_string()
1857 },
1858 TableUpdate::AddSchema { schema: schema() },
1859 TableUpdate::SetCurrentSchema { schema_id: -1 },
1860 TableUpdate::AddSpec {
1861 spec: PartitionSpec::builder(schema())
1864 .with_spec_id(0)
1865 .add_unbound_field(
1866 UnboundPartitionField::builder()
1867 .source_ids(vec![2])
1868 .field_id(1000)
1869 .name("y".to_string())
1870 .transform(Transform::Identity)
1871 .build()
1872 .unwrap()
1873 )
1874 .unwrap()
1875 .build()
1876 .unwrap()
1877 .into_unbound(),
1878 },
1879 TableUpdate::SetDefaultSpec { spec_id: -1 },
1880 TableUpdate::AddSortOrder {
1881 sort_order: sort_order(),
1882 },
1883 TableUpdate::SetDefaultSortOrder { sort_order_id: -1 },
1884 TableUpdate::SetProperties {
1885 updates: HashMap::from_iter(vec![(
1886 "property 1".to_string(),
1887 "value 1".to_string()
1888 )]),
1889 }
1890 ]);
1891 }
1892
1893 #[test]
1894 fn test_new_metadata_changes_unpartitioned_unsorted() {
1895 let schema = Schema::builder().build().unwrap();
1896 let changes = TableMetadataBuilder::new(
1897 schema.clone(),
1898 PartitionSpec::unpartition_spec().into_unbound(),
1899 SortOrder::unsorted_order(),
1900 TEST_LOCATION.to_string(),
1901 FormatVersion::V1,
1902 HashMap::new(),
1903 )
1904 .unwrap()
1905 .build()
1906 .unwrap()
1907 .changes;
1908
1909 pretty_assertions::assert_eq!(changes, vec![
1910 TableUpdate::SetLocation {
1911 location: TEST_LOCATION.to_string()
1912 },
1913 TableUpdate::AddSchema {
1914 schema: Schema::builder().build().unwrap(),
1915 },
1916 TableUpdate::SetCurrentSchema { schema_id: -1 },
1917 TableUpdate::AddSpec {
1918 spec: PartitionSpec::builder(schema)
1921 .with_spec_id(0)
1922 .build()
1923 .unwrap()
1924 .into_unbound(),
1925 },
1926 TableUpdate::SetDefaultSpec { spec_id: -1 },
1927 TableUpdate::AddSortOrder {
1928 sort_order: SortOrder::unsorted_order(),
1929 },
1930 TableUpdate::SetDefaultSortOrder { sort_order_id: -1 },
1931 ]);
1932 }
1933
1934 #[test]
1935 fn test_add_partition_spec() {
1936 let builder = builder_without_changes(FormatVersion::V2);
1937
1938 let added_spec = UnboundPartitionSpec::builder()
1939 .with_spec_id(10)
1940 .add_partition_fields(vec![
1941 UnboundPartitionField::builder()
1943 .source_ids(vec![2])
1944 .field_id(1000)
1945 .name("y".to_string())
1946 .transform(Transform::Identity)
1947 .build()
1948 .unwrap(),
1949 UnboundPartitionField::builder()
1951 .source_ids(vec![3])
1952 .name("z".to_string())
1953 .transform(Transform::Identity)
1954 .build()
1955 .unwrap(),
1956 ])
1957 .unwrap()
1958 .build();
1959
1960 let build_result = builder
1961 .add_partition_spec(added_spec.clone())
1962 .unwrap()
1963 .build()
1964 .unwrap();
1965
1966 let expected_change = added_spec.with_spec_id(1);
1968 let expected_spec = PartitionSpec::builder(schema())
1969 .with_spec_id(1)
1970 .add_unbound_field(
1971 UnboundPartitionField::builder()
1972 .source_ids(vec![2])
1973 .field_id(1000)
1974 .name("y".to_string())
1975 .transform(Transform::Identity)
1976 .build()
1977 .unwrap(),
1978 )
1979 .unwrap()
1980 .add_unbound_field(
1981 UnboundPartitionField::builder()
1982 .source_ids(vec![3])
1983 .field_id(1001)
1984 .name("z".to_string())
1985 .transform(Transform::Identity)
1986 .build()
1987 .unwrap(),
1988 )
1989 .unwrap()
1990 .build()
1991 .unwrap();
1992
1993 assert_eq!(build_result.changes.len(), 1);
1994 assert_eq!(
1995 build_result.metadata.partition_spec_by_id(1),
1996 Some(&Arc::new(expected_spec))
1997 );
1998 assert_eq!(build_result.metadata.default_spec.spec_id(), 0);
1999 assert_eq!(build_result.metadata.last_partition_id, 1001);
2000 pretty_assertions::assert_eq!(build_result.changes[0], TableUpdate::AddSpec {
2001 spec: expected_change
2002 });
2003
2004 let build_result = build_result
2006 .metadata
2007 .into_builder(Some(
2008 "s3://bucket/test/location/metadata/metadata1.json".to_string(),
2009 ))
2010 .remove_partition_specs(&[1])
2011 .unwrap()
2012 .build()
2013 .unwrap();
2014
2015 assert_eq!(build_result.changes.len(), 1);
2016 assert_eq!(build_result.metadata.partition_specs.len(), 1);
2017 assert!(build_result.metadata.partition_spec_by_id(1).is_none());
2018 }
2019
2020 #[test]
2021 fn test_set_default_partition_spec() {
2022 let builder = builder_without_changes(FormatVersion::V2);
2023 let schema = builder.get_current_schema().unwrap().clone();
2024 let added_spec = UnboundPartitionSpec::builder()
2025 .with_spec_id(10)
2026 .add_partition_field(
2027 UnboundPartitionField::builder()
2028 .source_ids(vec![1])
2029 .name("y_bucket[2]")
2030 .transform(Transform::Bucket(2))
2031 .build()
2032 .unwrap(),
2033 )
2034 .unwrap()
2035 .build();
2036
2037 let build_result = builder
2038 .add_partition_spec(added_spec.clone())
2039 .unwrap()
2040 .set_default_partition_spec(-1)
2041 .unwrap()
2042 .build()
2043 .unwrap();
2044
2045 let expected_spec = PartitionSpec::builder(schema)
2046 .with_spec_id(1)
2047 .add_unbound_field(
2048 UnboundPartitionField::builder()
2049 .source_ids(vec![1])
2050 .field_id(1001)
2051 .name("y_bucket[2]".to_string())
2052 .transform(Transform::Bucket(2))
2053 .build()
2054 .unwrap(),
2055 )
2056 .unwrap()
2057 .build()
2058 .unwrap();
2059
2060 assert_eq!(build_result.changes.len(), 2);
2061 assert_eq!(build_result.metadata.default_spec, Arc::new(expected_spec));
2062 assert_eq!(build_result.changes, vec![
2063 TableUpdate::AddSpec {
2064 spec: added_spec.with_spec_id(1)
2066 },
2067 TableUpdate::SetDefaultSpec { spec_id: -1 }
2068 ]);
2069 }
2070
2071 #[test]
2072 fn test_set_existing_default_partition_spec() {
2073 let builder = builder_without_changes(FormatVersion::V2);
2074 let unbound_spec = UnboundPartitionSpec::builder().with_spec_id(1).build();
2076 let build_result = builder
2077 .add_partition_spec(unbound_spec.clone())
2078 .unwrap()
2079 .set_default_partition_spec(-1)
2080 .unwrap()
2081 .build()
2082 .unwrap();
2083
2084 assert_eq!(build_result.changes.len(), 2);
2085 assert_eq!(build_result.changes[0], TableUpdate::AddSpec {
2086 spec: unbound_spec.clone()
2087 });
2088 assert_eq!(build_result.changes[1], TableUpdate::SetDefaultSpec {
2089 spec_id: -1
2090 });
2091 assert_eq!(
2092 build_result.metadata.default_spec,
2093 Arc::new(
2094 unbound_spec
2095 .bind(build_result.metadata.current_schema().clone())
2096 .unwrap()
2097 )
2098 );
2099
2100 let build_result = build_result
2102 .metadata
2103 .into_builder(Some(
2104 "s3://bucket/test/location/metadata/metadata1.json".to_string(),
2105 ))
2106 .set_default_partition_spec(0)
2107 .unwrap()
2108 .build()
2109 .unwrap();
2110
2111 assert_eq!(build_result.changes.len(), 1);
2112 assert_eq!(build_result.changes[0], TableUpdate::SetDefaultSpec {
2113 spec_id: 0
2114 });
2115 assert_eq!(
2116 build_result.metadata.default_spec,
2117 Arc::new(
2118 partition_spec()
2119 .bind(build_result.metadata.current_schema().clone())
2120 .unwrap()
2121 )
2122 );
2123 }
2124
2125 #[test]
2126 fn test_add_sort_order() {
2127 let builder = builder_without_changes(FormatVersion::V2);
2128
2129 let added_sort_order = SortOrder::builder()
2130 .with_order_id(10)
2131 .with_fields(vec![SortField {
2132 source_id: 1,
2133 transform: Transform::Identity,
2134 direction: SortDirection::Ascending,
2135 null_order: NullOrder::First,
2136 }])
2137 .build(&schema())
2138 .unwrap();
2139
2140 let build_result = builder
2141 .add_sort_order(added_sort_order.clone())
2142 .unwrap()
2143 .build()
2144 .unwrap();
2145
2146 let expected_sort_order = added_sort_order.with_order_id(2);
2147
2148 assert_eq!(build_result.changes.len(), 1);
2149 assert_eq!(build_result.metadata.sort_orders.keys().max(), Some(&2));
2150 pretty_assertions::assert_eq!(
2151 build_result.metadata.sort_order_by_id(2),
2152 Some(&Arc::new(expected_sort_order.clone()))
2153 );
2154 pretty_assertions::assert_eq!(build_result.changes[0], TableUpdate::AddSortOrder {
2155 sort_order: expected_sort_order
2156 });
2157 }
2158
2159 #[test]
2160 fn test_add_compatible_schema() {
2161 let builder = builder_without_changes(FormatVersion::V2);
2162
2163 let added_schema = Schema::builder()
2164 .with_schema_id(1)
2165 .with_fields(vec![
2166 NestedField::required(1, "x", Type::Primitive(PrimitiveType::Long)).into(),
2167 NestedField::required(2, "y", Type::Primitive(PrimitiveType::Long)).into(),
2168 NestedField::required(3, "z", Type::Primitive(PrimitiveType::Long)).into(),
2169 NestedField::required(4, "a", Type::Primitive(PrimitiveType::Long)).into(),
2170 ])
2171 .build()
2172 .unwrap();
2173
2174 let build_result = builder
2175 .add_current_schema(added_schema.clone())
2176 .unwrap()
2177 .build()
2178 .unwrap();
2179
2180 assert_eq!(build_result.changes.len(), 2);
2181 assert_eq!(build_result.metadata.schemas.keys().max(), Some(&1));
2182 pretty_assertions::assert_eq!(
2183 build_result.metadata.schema_by_id(1),
2184 Some(&Arc::new(added_schema.clone()))
2185 );
2186 pretty_assertions::assert_eq!(build_result.changes[0], TableUpdate::AddSchema {
2187 schema: added_schema
2188 });
2189 assert_eq!(build_result.changes[1], TableUpdate::SetCurrentSchema {
2190 schema_id: -1
2191 });
2192 }
2193
2194 #[test]
2195 fn test_set_current_schema_change_is_minus_one_if_schema_was_added_in_this_change() {
2196 let builder = builder_without_changes(FormatVersion::V2);
2197
2198 let added_schema = Schema::builder()
2199 .with_schema_id(1)
2200 .with_fields(vec![
2201 NestedField::required(1, "x", Type::Primitive(PrimitiveType::Long)).into(),
2202 NestedField::required(2, "y", Type::Primitive(PrimitiveType::Long)).into(),
2203 NestedField::required(3, "z", Type::Primitive(PrimitiveType::Long)).into(),
2204 NestedField::required(4, "a", Type::Primitive(PrimitiveType::Long)).into(),
2205 ])
2206 .build()
2207 .unwrap();
2208
2209 let build_result = builder
2210 .add_schema(added_schema.clone())
2211 .unwrap()
2212 .set_current_schema(1)
2213 .unwrap()
2214 .build()
2215 .unwrap();
2216
2217 assert_eq!(build_result.changes.len(), 2);
2218 assert_eq!(build_result.changes[1], TableUpdate::SetCurrentSchema {
2219 schema_id: -1
2220 });
2221 }
2222
2223 #[test]
2224 fn test_no_metadata_log_for_create_table() {
2225 let build_result = TableMetadataBuilder::new(
2226 schema(),
2227 partition_spec(),
2228 sort_order(),
2229 TEST_LOCATION.to_string(),
2230 FormatVersion::V2,
2231 HashMap::new(),
2232 )
2233 .unwrap()
2234 .build()
2235 .unwrap();
2236
2237 assert_eq!(build_result.metadata.metadata_log.len(), 0);
2238 }
2239
2240 const KEY_ID: (&str, &str) = (TableProperties::PROPERTY_ENCRYPTION_KEY_ID, "master-1");
2241 const KEY_LENGTH: (&str, &str) = (TableProperties::PROPERTY_ENCRYPTION_DATA_KEY_LENGTH, "16");
2242
2243 fn string_map(entries: &[(&str, &str)]) -> HashMap<String, String> {
2244 entries
2245 .iter()
2246 .map(|(key, value)| (key.to_string(), value.to_string()))
2247 .collect()
2248 }
2249
2250 fn build_error(builder: TableMetadataBuilder) -> (ErrorKind, String) {
2251 let err = builder.build().unwrap_err();
2252 (err.kind(), err.message().to_string())
2253 }
2254
2255 #[test]
2256 fn test_build_rejects_encryption_properties_before_v3() {
2257 for version in [FormatVersion::V1, FormatVersion::V2] {
2258 for (properties, names) in [
2259 (vec![KEY_ID], "encryption.key-id"),
2260 (vec![KEY_LENGTH], "encryption.data-key-length"),
2261 (
2262 vec![KEY_ID, KEY_LENGTH],
2263 "encryption.key-id, encryption.data-key-length",
2264 ),
2265 ] {
2266 let builder = builder_without_changes(version)
2267 .set_properties(string_map(&properties))
2268 .unwrap();
2269 assert_eq!(
2270 build_error(builder),
2271 (
2272 ErrorKind::DataInvalid,
2273 format!("Invalid properties for {version}: [{names}]")
2274 )
2275 );
2276 }
2277 }
2278 }
2279
2280 #[test]
2281 fn test_build_rejects_encryption_keys_before_v3() {
2282 let key = EncryptedKey::builder()
2283 .key_id("key-1")
2284 .encrypted_key_metadata(vec![1, 2, 3, 4])
2285 .build();
2286
2287 for version in [FormatVersion::V1, FormatVersion::V2] {
2288 let builder = builder_without_changes(version).add_encryption_key(key.clone());
2289 assert_eq!(
2290 build_error(builder),
2291 (
2292 ErrorKind::DataInvalid,
2293 format!("Encryption keys are not supported for {version} tables")
2294 )
2295 );
2296 }
2297 }
2298
2299 #[test]
2300 fn test_build_accepts_encryption_properties_on_v3() {
2301 let properties = string_map(&[KEY_ID, KEY_LENGTH]);
2302
2303 let metadata = builder_without_changes(FormatVersion::V3)
2304 .set_properties(properties.clone())
2305 .unwrap()
2306 .build()
2307 .unwrap()
2308 .metadata;
2309
2310 assert_eq!(metadata.properties, properties);
2311 }
2312
2313 #[test]
2314 fn test_pre_v3_encryption_property_blocks_updates_until_removed() {
2315 let legacy = || {
2316 let mut builder = builder_without_changes(FormatVersion::V2);
2317 builder.metadata.properties = string_map(&[KEY_ID]);
2318 builder
2319 };
2320
2321 let other_update = legacy()
2322 .set_properties(string_map(&[("key", "value")]))
2323 .unwrap();
2324 assert_eq!(
2325 build_error(other_update),
2326 (
2327 ErrorKind::DataInvalid,
2328 "Invalid properties for v2: [encryption.key-id]".to_string()
2329 )
2330 );
2331
2332 let repaired = legacy()
2333 .remove_properties(&[KEY_ID.0.to_string()])
2334 .unwrap()
2335 .build()
2336 .unwrap();
2337 assert_eq!(repaired.metadata.properties, HashMap::new());
2338 }
2339
2340 #[test]
2341 fn test_table_properties_view_reflects_metadata_updates() {
2342 let property = TableProperties::PROPERTY_COMMIT_NUM_RETRIES.to_string();
2343 let metadata = builder_without_changes(FormatVersion::V2)
2344 .set_properties(HashMap::from([(property.clone(), "7".to_string())]))
2345 .unwrap()
2346 .build()
2347 .unwrap()
2348 .metadata;
2349
2350 assert_eq!(metadata.table_properties().commit_num_retries().unwrap(), 7);
2351
2352 let metadata = metadata
2353 .into_builder(None)
2354 .remove_properties(&[property])
2355 .unwrap()
2356 .build()
2357 .unwrap()
2358 .metadata;
2359
2360 assert_eq!(
2361 metadata.table_properties().commit_num_retries().unwrap(),
2362 TableProperties::PROPERTY_COMMIT_NUM_RETRIES_DEFAULT
2363 );
2364 }
2365
2366 #[test]
2367 fn test_no_metadata_log_entry_for_no_previous_location() {
2368 let metadata = builder_without_changes(FormatVersion::V2)
2370 .build()
2371 .unwrap()
2372 .metadata;
2373 assert_eq!(metadata.metadata_log.len(), 1);
2374
2375 let build_result = metadata
2376 .into_builder(None)
2377 .set_properties(HashMap::from_iter(vec![(
2378 "foo".to_string(),
2379 "bar".to_string(),
2380 )]))
2381 .unwrap()
2382 .build()
2383 .unwrap();
2384
2385 assert_eq!(build_result.metadata.metadata_log.len(), 1);
2386 }
2387
2388 #[test]
2389 fn test_from_metadata_generates_metadata_log() {
2390 let metadata_path = "s3://bucket/test/location/metadata/metadata1.json";
2391 let builder = TableMetadataBuilder::new(
2392 schema(),
2393 partition_spec(),
2394 sort_order(),
2395 TEST_LOCATION.to_string(),
2396 FormatVersion::V2,
2397 HashMap::new(),
2398 )
2399 .unwrap()
2400 .build()
2401 .unwrap()
2402 .metadata
2403 .into_builder(Some(metadata_path.to_string()));
2404
2405 let builder = builder
2406 .add_default_sort_order(SortOrder::unsorted_order())
2407 .unwrap();
2408
2409 let build_result = builder.build().unwrap();
2410
2411 assert_eq!(build_result.metadata.metadata_log.len(), 1);
2412 assert_eq!(
2413 build_result.metadata.metadata_log[0].metadata_file,
2414 metadata_path
2415 );
2416 }
2417
2418 #[test]
2419 fn test_set_ref() {
2420 let builder = builder_without_changes(FormatVersion::V2);
2421
2422 let snapshot = Snapshot::builder()
2423 .with_snapshot_id(1)
2424 .with_timestamp_ms(builder.metadata.last_updated_ms + 1)
2425 .with_sequence_number(0)
2426 .with_schema_id(0)
2427 .with_manifest_list("/snap-1.avro")
2428 .with_summary(Summary {
2429 operation: Operation::Append,
2430 additional_properties: HashMap::from_iter(vec![
2431 (
2432 "spark.app.id".to_string(),
2433 "local-1662532784305".to_string(),
2434 ),
2435 ("added-data-files".to_string(), "4".to_string()),
2436 ("added-records".to_string(), "4".to_string()),
2437 ("added-files-size".to_string(), "6001".to_string()),
2438 ]),
2439 })
2440 .build();
2441
2442 let builder = builder.add_snapshot(snapshot.clone()).unwrap();
2443
2444 assert!(
2445 builder
2446 .clone()
2447 .set_ref(MAIN_BRANCH, SnapshotReference {
2448 snapshot_id: 10,
2449 retention: SnapshotRetention::Branch {
2450 min_snapshots_to_keep: Some(10),
2451 max_snapshot_age_ms: None,
2452 max_ref_age_ms: None,
2453 },
2454 })
2455 .unwrap_err()
2456 .to_string()
2457 .contains("Cannot set 'main' to unknown snapshot: '10'")
2458 );
2459
2460 let build_result = builder
2461 .set_ref(MAIN_BRANCH, SnapshotReference {
2462 snapshot_id: 1,
2463 retention: SnapshotRetention::Branch {
2464 min_snapshots_to_keep: Some(10),
2465 max_snapshot_age_ms: None,
2466 max_ref_age_ms: None,
2467 },
2468 })
2469 .unwrap()
2470 .build()
2471 .unwrap();
2472 assert_eq!(build_result.metadata.snapshots.len(), 1);
2473 assert_eq!(
2474 build_result.metadata.snapshot_by_id(1),
2475 Some(&Arc::new(snapshot.clone()))
2476 );
2477 assert_eq!(build_result.metadata.snapshot_log, vec![SnapshotLog {
2478 snapshot_id: 1,
2479 timestamp_ms: snapshot.timestamp_ms()
2480 }])
2481 }
2482
2483 #[test]
2484 fn test_snapshot_log_skips_intermediates() {
2485 let builder = builder_without_changes(FormatVersion::V2);
2486
2487 let snapshot_1 = Snapshot::builder()
2488 .with_snapshot_id(1)
2489 .with_timestamp_ms(builder.metadata.last_updated_ms + 1)
2490 .with_sequence_number(0)
2491 .with_schema_id(0)
2492 .with_manifest_list("/snap-1.avro")
2493 .with_summary(Summary {
2494 operation: Operation::Append,
2495 additional_properties: HashMap::from_iter(vec![
2496 (
2497 "spark.app.id".to_string(),
2498 "local-1662532784305".to_string(),
2499 ),
2500 ("added-data-files".to_string(), "4".to_string()),
2501 ("added-records".to_string(), "4".to_string()),
2502 ("added-files-size".to_string(), "6001".to_string()),
2503 ]),
2504 })
2505 .build();
2506
2507 let snapshot_2 = Snapshot::builder()
2508 .with_snapshot_id(2)
2509 .with_timestamp_ms(builder.metadata.last_updated_ms + 1)
2510 .with_sequence_number(0)
2511 .with_schema_id(0)
2512 .with_manifest_list("/snap-1.avro")
2513 .with_summary(Summary {
2514 operation: Operation::Append,
2515 additional_properties: HashMap::from_iter(vec![
2516 (
2517 "spark.app.id".to_string(),
2518 "local-1662532784305".to_string(),
2519 ),
2520 ("added-data-files".to_string(), "4".to_string()),
2521 ("added-records".to_string(), "4".to_string()),
2522 ("added-files-size".to_string(), "6001".to_string()),
2523 ]),
2524 })
2525 .build();
2526
2527 let result = builder
2528 .add_snapshot(snapshot_1)
2529 .unwrap()
2530 .set_ref(MAIN_BRANCH, SnapshotReference {
2531 snapshot_id: 1,
2532 retention: SnapshotRetention::Branch {
2533 min_snapshots_to_keep: Some(10),
2534 max_snapshot_age_ms: None,
2535 max_ref_age_ms: None,
2536 },
2537 })
2538 .unwrap()
2539 .set_branch_snapshot(snapshot_2.clone(), MAIN_BRANCH)
2540 .unwrap()
2541 .build()
2542 .unwrap();
2543
2544 assert_eq!(result.metadata.snapshot_log, vec![SnapshotLog {
2545 snapshot_id: 2,
2546 timestamp_ms: snapshot_2.timestamp_ms()
2547 }]);
2548 assert_eq!(result.metadata.current_snapshot().unwrap().snapshot_id(), 2);
2549 }
2550
2551 #[test]
2552 fn test_remove_main_ref_keeps_snapshot_log() {
2553 let builder = builder_without_changes(FormatVersion::V2);
2554
2555 let snapshot = Snapshot::builder()
2556 .with_snapshot_id(1)
2557 .with_timestamp_ms(builder.metadata.last_updated_ms + 1)
2558 .with_sequence_number(0)
2559 .with_schema_id(0)
2560 .with_manifest_list("/snap-1.avro")
2561 .with_summary(Summary {
2562 operation: Operation::Append,
2563 additional_properties: HashMap::from_iter(vec![
2564 (
2565 "spark.app.id".to_string(),
2566 "local-1662532784305".to_string(),
2567 ),
2568 ("added-data-files".to_string(), "4".to_string()),
2569 ("added-records".to_string(), "4".to_string()),
2570 ("added-files-size".to_string(), "6001".to_string()),
2571 ]),
2572 })
2573 .build();
2574
2575 let result = builder
2576 .add_snapshot(snapshot.clone())
2577 .unwrap()
2578 .set_ref(MAIN_BRANCH, SnapshotReference {
2579 snapshot_id: 1,
2580 retention: SnapshotRetention::Branch {
2581 min_snapshots_to_keep: Some(10),
2582 max_snapshot_age_ms: None,
2583 max_ref_age_ms: None,
2584 },
2585 })
2586 .unwrap()
2587 .build()
2588 .unwrap();
2589
2590 assert_eq!(result.metadata.snapshot_log.len(), 1);
2592 assert_eq!(result.metadata.snapshot_log[0].snapshot_id, 1);
2593 assert_eq!(result.metadata.current_snapshot_id, Some(1));
2594
2595 let result_after_remove = result
2597 .metadata
2598 .into_builder(Some(
2599 "s3://bucket/test/location/metadata/metadata2.json".to_string(),
2600 ))
2601 .remove_ref(MAIN_BRANCH)
2602 .build()
2603 .unwrap();
2604
2605 assert_eq!(result_after_remove.metadata.snapshot_log.len(), 1);
2607 assert_eq!(result_after_remove.metadata.snapshot_log[0].snapshot_id, 1);
2608 assert_eq!(result_after_remove.metadata.current_snapshot_id, None);
2609 assert_eq!(result_after_remove.changes.len(), 1);
2610 assert_eq!(
2611 result_after_remove.changes[0],
2612 TableUpdate::RemoveSnapshotRef {
2613 ref_name: MAIN_BRANCH.to_string()
2614 }
2615 );
2616 }
2617
2618 #[test]
2619 fn test_set_branch_snapshot_creates_branch_if_not_exists() {
2620 let builder = builder_without_changes(FormatVersion::V2);
2621
2622 let snapshot = Snapshot::builder()
2623 .with_snapshot_id(2)
2624 .with_timestamp_ms(builder.metadata.last_updated_ms + 1)
2625 .with_sequence_number(0)
2626 .with_schema_id(0)
2627 .with_manifest_list("/snap-1.avro")
2628 .with_summary(Summary {
2629 operation: Operation::Append,
2630 additional_properties: HashMap::new(),
2631 })
2632 .build();
2633
2634 let build_result = builder
2635 .set_branch_snapshot(snapshot.clone(), "new_branch")
2636 .unwrap()
2637 .build()
2638 .unwrap();
2639
2640 let reference = SnapshotReference {
2641 snapshot_id: 2,
2642 retention: SnapshotRetention::Branch {
2643 min_snapshots_to_keep: None,
2644 max_snapshot_age_ms: None,
2645 max_ref_age_ms: None,
2646 },
2647 };
2648
2649 assert_eq!(build_result.metadata.refs.len(), 1);
2650 assert_eq!(
2651 build_result.metadata.refs.get("new_branch"),
2652 Some(&reference)
2653 );
2654 assert_eq!(build_result.changes, vec![
2655 TableUpdate::AddSnapshot { snapshot },
2656 TableUpdate::SetSnapshotRef {
2657 ref_name: "new_branch".to_string(),
2658 reference
2659 }
2660 ]);
2661 }
2662
2663 fn metadata_with_snapshots_and_refs(
2666 snapshot_ids: &[i64],
2667 refs: &[(&str, SnapshotReference)],
2668 ) -> TableMetadata {
2669 let mut builder = builder_without_changes(FormatVersion::V2);
2670 let timestamp_ms = builder.metadata.last_updated_ms + 1;
2671 for (i, snapshot_id) in snapshot_ids.iter().enumerate() {
2672 let snapshot = Snapshot::builder()
2673 .with_snapshot_id(*snapshot_id)
2674 .with_timestamp_ms(timestamp_ms + i as i64)
2675 .with_sequence_number(i as i64 + 1)
2676 .with_schema_id(0)
2677 .with_manifest_list(format!("/snap-{snapshot_id}.avro"))
2678 .with_summary(Summary {
2679 operation: Operation::Append,
2680 additional_properties: HashMap::new(),
2681 })
2682 .build();
2683 builder = builder.add_snapshot(snapshot).unwrap();
2684 }
2685 for (ref_name, reference) in refs {
2686 builder = builder.set_ref(ref_name, reference.clone()).unwrap();
2687 }
2688 builder.build().unwrap().metadata
2689 }
2690
2691 fn branch_ref(snapshot_id: i64) -> SnapshotReference {
2692 SnapshotReference {
2693 snapshot_id,
2694 retention: SnapshotRetention::Branch {
2695 min_snapshots_to_keep: None,
2696 max_snapshot_age_ms: None,
2697 max_ref_age_ms: None,
2698 },
2699 }
2700 }
2701
2702 fn tag_ref(snapshot_id: i64) -> SnapshotReference {
2703 SnapshotReference {
2704 snapshot_id,
2705 retention: SnapshotRetention::Tag {
2706 max_ref_age_ms: None,
2707 },
2708 }
2709 }
2710
2711 #[test]
2712 fn test_remove_snapshots_records_removal_of_refs_to_removed_snapshots() {
2713 let metadata = metadata_with_snapshots_and_refs(&[1, 2], &[
2714 (MAIN_BRANCH, branch_ref(2)),
2715 ("branch-on-1", branch_ref(1)),
2716 ("tag-on-1", tag_ref(1)),
2717 ("branch-on-2", branch_ref(2)),
2718 ]);
2719
2720 let result = metadata
2721 .into_builder(Some(
2722 "s3://bucket/test/location/metadata/metadata2.json".to_string(),
2723 ))
2724 .remove_snapshots(&[1])
2725 .build()
2726 .unwrap();
2727
2728 assert_eq!(result.changes, vec![
2729 TableUpdate::RemoveSnapshots {
2730 snapshot_ids: vec![1]
2731 },
2732 TableUpdate::RemoveSnapshotRef {
2733 ref_name: "branch-on-1".to_string()
2734 },
2735 TableUpdate::RemoveSnapshotRef {
2736 ref_name: "tag-on-1".to_string()
2737 },
2738 ]);
2739 assert_eq!(
2740 result.metadata.refs,
2741 HashMap::from([
2742 (MAIN_BRANCH.to_string(), branch_ref(2)),
2743 ("branch-on-2".to_string(), branch_ref(2)),
2744 ])
2745 );
2746 assert_eq!(result.metadata.current_snapshot_id, Some(2));
2747 }
2748
2749 #[test]
2750 fn test_remove_snapshots_clears_current_snapshot_when_main_snapshot_removed() {
2751 let metadata = metadata_with_snapshots_and_refs(&[1, 2], &[
2752 (MAIN_BRANCH, branch_ref(2)),
2753 ("branch-on-1", branch_ref(1)),
2754 ]);
2755
2756 let result = metadata
2757 .into_builder(Some(
2758 "s3://bucket/test/location/metadata/metadata2.json".to_string(),
2759 ))
2760 .remove_snapshots(&[2])
2761 .build()
2762 .unwrap();
2763
2764 assert_eq!(result.changes, vec![
2765 TableUpdate::RemoveSnapshots {
2766 snapshot_ids: vec![2]
2767 },
2768 TableUpdate::RemoveSnapshotRef {
2769 ref_name: MAIN_BRANCH.to_string()
2770 },
2771 ]);
2772 assert_eq!(result.metadata.current_snapshot_id, None);
2773 assert_eq!(
2774 result.metadata.refs,
2775 HashMap::from([("branch-on-1".to_string(), branch_ref(1))])
2776 );
2777 assert!(result.metadata.snapshot_log.is_empty());
2778 }
2779
2780 #[test]
2781 fn test_remove_snapshots_keeps_refs_to_retained_snapshots() {
2782 let metadata = metadata_with_snapshots_and_refs(&[1, 2, 3], &[
2783 (MAIN_BRANCH, branch_ref(3)),
2784 ("branch-on-2", branch_ref(2)),
2785 ("tag-on-3", tag_ref(3)),
2786 ]);
2787 let refs_before = metadata.refs.clone();
2788
2789 let result = metadata
2790 .into_builder(Some(
2791 "s3://bucket/test/location/metadata/metadata2.json".to_string(),
2792 ))
2793 .remove_snapshots(&[1])
2794 .build()
2795 .unwrap();
2796
2797 assert_eq!(result.changes, vec![TableUpdate::RemoveSnapshots {
2798 snapshot_ids: vec![1]
2799 }]);
2800 assert_eq!(result.metadata.refs, refs_before);
2801 assert_eq!(result.metadata.current_snapshot_id, Some(3));
2802 }
2803
2804 #[test]
2805 fn test_cannot_add_duplicate_snapshot_id() {
2806 let builder = builder_without_changes(FormatVersion::V2);
2807
2808 let snapshot = Snapshot::builder()
2809 .with_snapshot_id(2)
2810 .with_timestamp_ms(builder.metadata.last_updated_ms + 1)
2811 .with_sequence_number(0)
2812 .with_schema_id(0)
2813 .with_manifest_list("/snap-1.avro")
2814 .with_summary(Summary {
2815 operation: Operation::Append,
2816 additional_properties: HashMap::from_iter(vec![
2817 (
2818 "spark.app.id".to_string(),
2819 "local-1662532784305".to_string(),
2820 ),
2821 ("added-data-files".to_string(), "4".to_string()),
2822 ("added-records".to_string(), "4".to_string()),
2823 ("added-files-size".to_string(), "6001".to_string()),
2824 ]),
2825 })
2826 .build();
2827
2828 let builder = builder.add_snapshot(snapshot.clone()).unwrap();
2829 builder.add_snapshot(snapshot).unwrap_err();
2830 }
2831
2832 #[test]
2833 fn test_remove_snapshots_records_existing_ids_once() {
2834 let fixture = TableTestFixture::new_with_deep_history();
2835 let metadata = fixture.table.metadata().clone();
2836 let first_id = metadata.snapshot_log[0].snapshot_id;
2837 let third_id = metadata.snapshot_log[2].snapshot_id;
2838 let mut expected_metadata = metadata.clone();
2839 expected_metadata.snapshots = [
2840 3055729675574597004,
2841 3057729675574597004,
2842 3059729675574597004,
2843 ]
2844 .into_iter()
2845 .map(|id| (id, metadata.snapshot_by_id(id).unwrap().clone()))
2846 .collect();
2847 expected_metadata.snapshot_log = metadata.snapshot_log[3..].to_vec();
2848 let expected = TableMetadataBuildResult {
2849 metadata: expected_metadata,
2850 changes: vec![TableUpdate::RemoveSnapshots {
2851 snapshot_ids: vec![third_id, first_id],
2852 }],
2853 expired_metadata_logs: vec![],
2854 };
2855 let mut builder = metadata.into_builder(None);
2856 builder.last_updated_ms = Some(expected.metadata.last_updated_ms);
2857
2858 let result = builder
2859 .remove_snapshots(&[third_id, 0, first_id, third_id, 0, first_id])
2860 .build()
2861 .unwrap();
2862
2863 assert_eq!(result, expected);
2864 }
2865
2866 #[test]
2867 fn test_add_incompatible_current_schema_fails() {
2868 let builder = builder_without_changes(FormatVersion::V2);
2869
2870 let added_schema = Schema::builder()
2871 .with_schema_id(1)
2872 .with_fields(vec![])
2873 .build()
2874 .unwrap();
2875
2876 let err = builder
2877 .add_current_schema(added_schema)
2878 .unwrap()
2879 .build()
2880 .unwrap_err();
2881
2882 assert!(
2883 err.to_string()
2884 .contains("Cannot find partition source field")
2885 );
2886 }
2887
2888 #[test]
2889 fn test_add_partition_spec_for_v1_requires_sequential_ids() {
2890 let builder = builder_without_changes(FormatVersion::V1);
2891
2892 let added_spec = UnboundPartitionSpec::builder()
2893 .with_spec_id(10)
2894 .add_partition_fields(vec![
2895 UnboundPartitionField::builder()
2896 .source_ids(vec![2])
2897 .field_id(1000)
2898 .name("y".to_string())
2899 .transform(Transform::Identity)
2900 .build()
2901 .unwrap(),
2902 UnboundPartitionField::builder()
2903 .source_ids(vec![3])
2904 .field_id(1002)
2905 .name("z".to_string())
2906 .transform(Transform::Identity)
2907 .build()
2908 .unwrap(),
2909 ])
2910 .unwrap()
2911 .build();
2912
2913 let err = builder.add_partition_spec(added_spec).unwrap_err();
2914 assert!(err.to_string().contains(
2915 "Cannot add partition spec with non-sequential field ids to format version 1 table"
2916 ));
2917 }
2918
2919 #[test]
2920 fn test_add_partition_spec_accepts_void_fields_of_removed_fields() {
2921 let builder = builder_without_changes(FormatVersion::V1);
2922 let void_field = |field_id: i32, name: &str| {
2923 UnboundPartitionField::builder()
2924 .source_ids(vec![2])
2925 .field_id(field_id)
2926 .name(name)
2927 .transform(Transform::Void)
2928 .build()
2929 .unwrap()
2930 };
2931 let added_spec = UnboundPartitionSpec {
2932 spec_id: None,
2933 fields: vec![void_field(1000, "y"), void_field(1001, "y_null")],
2934 };
2935
2936 let metadata = builder
2937 .add_default_partition_spec(added_spec.clone())
2938 .unwrap()
2939 .build()
2940 .unwrap()
2941 .metadata;
2942
2943 let expected = added_spec.with_spec_id(1).bind(schema()).unwrap();
2944 assert_eq!(metadata.default_partition_spec().as_ref(), &expected);
2945 assert_eq!(metadata.last_partition_id, 1001);
2946 }
2947
2948 #[test]
2949 fn test_add_partition_spec_accepts_two_time_granularities_of_one_column() {
2950 let schema = Schema::builder()
2951 .with_fields(vec![
2952 NestedField::required(1, "ts", Type::Primitive(PrimitiveType::Timestamptz)).into(),
2953 ])
2954 .build()
2955 .unwrap();
2956 let time_field = |field_id: i32, name: &str, transform: Transform| {
2957 UnboundPartitionField::builder()
2958 .source_ids(vec![1])
2959 .field_id(field_id)
2960 .name(name)
2961 .transform(transform)
2962 .build()
2963 .unwrap()
2964 };
2965 let added_spec = UnboundPartitionSpec {
2966 spec_id: None,
2967 fields: vec![
2968 time_field(1000, "ts_day", Transform::Day),
2969 time_field(1001, "ts_hour", Transform::Hour),
2970 ],
2971 };
2972 let metadata = TableMetadataBuilder::new(
2973 schema.clone(),
2974 PartitionSpec::unpartition_spec(),
2975 SortOrder::unsorted_order(),
2976 TEST_LOCATION.to_string(),
2977 FormatVersion::V2,
2978 HashMap::new(),
2979 )
2980 .unwrap()
2981 .build()
2982 .unwrap()
2983 .metadata;
2984
2985 let metadata = metadata
2986 .into_builder(None)
2987 .add_default_partition_spec(added_spec.clone())
2988 .unwrap()
2989 .build()
2990 .unwrap()
2991 .metadata;
2992 let metadata = metadata
2994 .into_builder(None)
2995 .set_properties(HashMap::from([("key".to_string(), "value".to_string())]))
2996 .unwrap()
2997 .build()
2998 .unwrap()
2999 .metadata;
3000
3001 let expected = added_spec.with_spec_id(1).bind(schema).unwrap();
3002 assert_eq!(metadata.default_partition_spec().as_ref(), &expected);
3003 }
3004
3005 fn build_after_fast_branch_snapshot(skew_ms: i64) -> (i64, Result<TableMetadataBuildResult>) {
3008 let builder = builder_without_changes(FormatVersion::V2);
3009 let fast = builder.metadata.last_updated_ms + skew_ms;
3010 let snapshot = Snapshot::builder()
3011 .with_snapshot_id(2)
3012 .with_timestamp_ms(fast)
3013 .with_sequence_number(1)
3014 .with_schema_id(0)
3015 .with_manifest_list("/snap-2.avro")
3016 .with_summary(Summary {
3017 operation: Operation::Append,
3018 additional_properties: HashMap::new(),
3019 })
3020 .build();
3021 let on_branch = builder
3022 .set_branch_snapshot(snapshot, "b")
3023 .unwrap()
3024 .build()
3025 .unwrap()
3026 .metadata;
3027 assert_eq!(on_branch.last_updated_ms, fast);
3028
3029 let result = on_branch
3030 .into_builder(Some(
3031 "s3://bucket/test/location/metadata/metadata2.json".to_string(),
3032 ))
3033 .set_properties(HashMap::from([("k".to_string(), "v".to_string())]))
3034 .unwrap()
3035 .build();
3036 (fast, result)
3037 }
3038
3039 #[test]
3040 fn test_metadata_change_after_a_fast_branch_snapshot_is_refused_in_build() {
3041 let (fast, result) = build_after_fast_branch_snapshot(2 * 60 * 60_000);
3044 let err = result.unwrap_err();
3045 assert_eq!(err.kind(), ErrorKind::DataInvalid);
3046 assert!(
3047 err.message()
3048 .ends_with(&format!(": before last metadata log entry at {fast}")),
3049 "{}",
3050 err.message()
3051 );
3052 }
3053
3054 #[test]
3055 fn test_metadata_change_after_a_slightly_fast_branch_snapshot_builds() {
3056 let (fast, result) = build_after_fast_branch_snapshot(30_000);
3058 assert_eq!(
3059 result.unwrap().metadata.metadata_log.last(),
3060 Some(&MetadataLog {
3061 metadata_file: "s3://bucket/test/location/metadata/metadata2.json".to_string(),
3062 timestamp_ms: fast,
3063 })
3064 );
3065 }
3066
3067 #[test]
3068 fn test_expire_metadata_log() {
3069 let builder = builder_without_changes(FormatVersion::V2);
3070 let metadata = builder
3071 .set_properties(HashMap::from_iter(vec![(
3072 TableProperties::PROPERTY_METADATA_PREVIOUS_VERSIONS_MAX.to_string(),
3073 "2".to_string(),
3074 )]))
3075 .unwrap()
3076 .build()
3077 .unwrap();
3078 assert_eq!(metadata.metadata.metadata_log.len(), 1);
3079 assert_eq!(metadata.expired_metadata_logs.len(), 0);
3080
3081 let metadata = metadata
3082 .metadata
3083 .into_builder(Some("path2".to_string()))
3084 .set_properties(HashMap::from_iter(vec![(
3085 "change_nr".to_string(),
3086 "1".to_string(),
3087 )]))
3088 .unwrap()
3089 .build()
3090 .unwrap();
3091
3092 assert_eq!(metadata.metadata.metadata_log.len(), 2);
3093 assert_eq!(metadata.expired_metadata_logs.len(), 0);
3094
3095 let metadata = metadata
3096 .metadata
3097 .into_builder(Some("path2".to_string()))
3098 .set_properties(HashMap::from_iter(vec![(
3099 "change_nr".to_string(),
3100 "2".to_string(),
3101 )]))
3102 .unwrap()
3103 .build()
3104 .unwrap();
3105 assert_eq!(metadata.metadata.metadata_log.len(), 2);
3106 assert_eq!(metadata.expired_metadata_logs.len(), 1);
3107 }
3108
3109 #[test]
3110 fn test_v2_sequence_number_cannot_decrease() {
3111 let builder = builder_without_changes(FormatVersion::V2);
3112
3113 let snapshot = Snapshot::builder()
3114 .with_snapshot_id(1)
3115 .with_timestamp_ms(builder.metadata.last_updated_ms + 1)
3116 .with_sequence_number(1)
3117 .with_schema_id(0)
3118 .with_manifest_list("/snap-1")
3119 .with_summary(Summary {
3120 operation: Operation::Append,
3121 additional_properties: HashMap::new(),
3122 })
3123 .build();
3124
3125 let builder = builder
3126 .add_snapshot(snapshot.clone())
3127 .unwrap()
3128 .set_ref(MAIN_BRANCH, SnapshotReference {
3129 snapshot_id: 1,
3130 retention: SnapshotRetention::Branch {
3131 min_snapshots_to_keep: Some(10),
3132 max_snapshot_age_ms: None,
3133 max_ref_age_ms: None,
3134 },
3135 })
3136 .unwrap();
3137
3138 let snapshot = Snapshot::builder()
3139 .with_snapshot_id(2)
3140 .with_timestamp_ms(builder.metadata.last_updated_ms + 1)
3141 .with_sequence_number(0)
3142 .with_schema_id(0)
3143 .with_manifest_list("/snap-0")
3144 .with_parent_snapshot_id(Some(1))
3145 .with_summary(Summary {
3146 operation: Operation::Append,
3147 additional_properties: HashMap::new(),
3148 })
3149 .build();
3150
3151 let err = builder
3152 .set_branch_snapshot(snapshot, MAIN_BRANCH)
3153 .unwrap_err();
3154 assert!(
3155 err.to_string()
3156 .contains("Cannot add snapshot with sequence number")
3157 );
3158 }
3159
3160 #[test]
3161 fn test_default_spec_cannot_be_removed() {
3162 let builder = builder_without_changes(FormatVersion::V2);
3163
3164 builder.remove_partition_specs(&[0]).unwrap_err();
3165 }
3166
3167 #[test]
3168 fn test_statistics() {
3169 let builder = builder_without_changes(FormatVersion::V2);
3170
3171 let statistics = StatisticsFile {
3172 snapshot_id: 3055729675574597004,
3173 statistics_path: "s3://a/b/stats.puffin".to_string(),
3174 file_size_in_bytes: 413,
3175 file_footer_size_in_bytes: 42,
3176 key_metadata: None,
3177 blob_metadata: vec![BlobMetadata {
3178 snapshot_id: 3055729675574597004,
3179 sequence_number: 1,
3180 fields: vec![1],
3181 r#type: "ndv".to_string(),
3182 properties: HashMap::new(),
3183 }],
3184 };
3185 let build_result = builder.set_statistics(statistics.clone()).build().unwrap();
3186
3187 assert_eq!(
3188 build_result.metadata.statistics,
3189 HashMap::from_iter(vec![(3055729675574597004, statistics.clone())])
3190 );
3191 assert_eq!(build_result.changes, vec![TableUpdate::SetStatistics {
3192 statistics: statistics.clone()
3193 }]);
3194
3195 let builder = build_result.metadata.into_builder(None);
3197 let build_result = builder
3198 .remove_statistics(statistics.snapshot_id)
3199 .build()
3200 .unwrap();
3201
3202 assert_eq!(build_result.metadata.statistics.len(), 0);
3203 assert_eq!(build_result.changes, vec![TableUpdate::RemoveStatistics {
3204 snapshot_id: statistics.snapshot_id
3205 }]);
3206
3207 let builder = build_result.metadata.into_builder(None);
3209 let build_result = builder
3210 .remove_statistics(statistics.snapshot_id)
3211 .build()
3212 .unwrap();
3213 assert_eq!(build_result.metadata.statistics.len(), 0);
3214 assert_eq!(build_result.changes.len(), 0);
3215 }
3216
3217 #[test]
3218 fn test_add_partition_statistics() {
3219 let builder = builder_without_changes(FormatVersion::V2);
3220
3221 let statistics = PartitionStatisticsFile {
3222 snapshot_id: 3055729675574597004,
3223 statistics_path: "s3://a/b/partition-stats.parquet".to_string(),
3224 file_size_in_bytes: 43,
3225 };
3226
3227 let build_result = builder
3228 .set_partition_statistics(statistics.clone())
3229 .build()
3230 .unwrap();
3231 assert_eq!(
3232 build_result.metadata.partition_statistics,
3233 HashMap::from_iter(vec![(3055729675574597004, statistics.clone())])
3234 );
3235 assert_eq!(build_result.changes, vec![
3236 TableUpdate::SetPartitionStatistics {
3237 partition_statistics: statistics.clone()
3238 }
3239 ]);
3240
3241 let builder = build_result.metadata.into_builder(None);
3243 let build_result = builder
3244 .remove_partition_statistics(statistics.snapshot_id)
3245 .build()
3246 .unwrap();
3247 assert_eq!(build_result.metadata.partition_statistics.len(), 0);
3248 assert_eq!(build_result.changes, vec![
3249 TableUpdate::RemovePartitionStatistics {
3250 snapshot_id: statistics.snapshot_id
3251 }
3252 ]);
3253
3254 let builder = build_result.metadata.into_builder(None);
3256 let build_result = builder
3257 .remove_partition_statistics(statistics.snapshot_id)
3258 .build()
3259 .unwrap();
3260 assert_eq!(build_result.metadata.partition_statistics.len(), 0);
3261 assert_eq!(build_result.changes.len(), 0);
3262 }
3263
3264 #[test]
3265 fn last_update_increased_for_property_only_update() {
3266 let builder = builder_without_changes(FormatVersion::V2);
3267
3268 let metadata = builder.build().unwrap().metadata;
3269 let last_updated_ms = metadata.last_updated_ms;
3270 sleep(std::time::Duration::from_millis(2));
3271
3272 let build_result = metadata
3273 .into_builder(Some(
3274 "s3://bucket/test/location/metadata/metadata1.json".to_string(),
3275 ))
3276 .set_properties(HashMap::from_iter(vec![(
3277 "foo".to_string(),
3278 "bar".to_string(),
3279 )]))
3280 .unwrap()
3281 .build()
3282 .unwrap();
3283
3284 assert!(
3285 build_result.metadata.last_updated_ms > last_updated_ms,
3286 "{} > {}",
3287 build_result.metadata.last_updated_ms,
3288 last_updated_ms
3289 );
3290 }
3291
3292 #[test]
3293 fn test_construct_default_main_branch() {
3294 let file = File::open(format!(
3296 "{}/testdata/table_metadata/{}",
3297 env!("CARGO_MANIFEST_DIR"),
3298 "TableMetadataV2Valid.json"
3299 ))
3300 .unwrap();
3301 let reader = BufReader::new(file);
3302 let resp = serde_json::from_reader::<_, TableMetadata>(reader).unwrap();
3303
3304 let table = Table::builder()
3305 .metadata(resp)
3306 .metadata_location("s3://bucket/test/location/metadata/v1.json")
3307 .identifier(TableIdent::from_strs(["ns1", "test1"]).unwrap())
3308 .file_io(FileIO::new_with_memory())
3309 .runtime(test_runtime())
3310 .build()
3311 .unwrap();
3312
3313 assert_eq!(
3314 table.metadata().refs.get(MAIN_BRANCH).unwrap().snapshot_id,
3315 table.metadata().current_snapshot_id().unwrap()
3316 );
3317 }
3318
3319 #[test]
3320 fn test_active_schema_cannot_be_removed() {
3321 let builder = builder_without_changes(FormatVersion::V2);
3322 builder.remove_schemas(&[0]).unwrap_err();
3323 }
3324
3325 #[test]
3326 fn test_remove_schemas() {
3327 let file = File::open(format!(
3328 "{}/testdata/table_metadata/{}",
3329 env!("CARGO_MANIFEST_DIR"),
3330 "TableMetadataV2Valid.json"
3331 ))
3332 .unwrap();
3333 let reader = BufReader::new(file);
3334 let resp = serde_json::from_reader::<_, TableMetadata>(reader).unwrap();
3335
3336 let table = Table::builder()
3337 .metadata(resp)
3338 .metadata_location("s3://bucket/test/location/metadata/v1.json")
3339 .identifier(TableIdent::from_strs(["ns1", "test1"]).unwrap())
3340 .file_io(FileIO::new_with_memory())
3341 .runtime(test_runtime())
3342 .build()
3343 .unwrap();
3344
3345 assert_eq!(2, table.metadata().schemas.len());
3346
3347 {
3348 let meta_data_builder = table.metadata().clone().into_builder(None);
3350 meta_data_builder.remove_schemas(&[1]).unwrap_err();
3351 }
3352
3353 let mut meta_data_builder = table.metadata().clone().into_builder(None);
3354 meta_data_builder = meta_data_builder.remove_schemas(&[0]).unwrap();
3355 let build_result = meta_data_builder.build().unwrap();
3356 assert_eq!(1, build_result.metadata.schemas.len());
3357 assert_eq!(1, build_result.metadata.current_schema_id);
3358 assert_eq!(1, build_result.metadata.current_schema().schema_id());
3359 assert_eq!(1, build_result.changes.len());
3360
3361 let remove_schema_ids =
3362 if let TableUpdate::RemoveSchemas { schema_ids } = &build_result.changes[0] {
3363 schema_ids
3364 } else {
3365 unreachable!("Expected RemoveSchema change")
3366 };
3367 assert_eq!(remove_schema_ids, &[0]);
3368 }
3369
3370 #[test]
3371 fn test_schema_evolution_now_correctly_validates_partition_field_name_conflicts() {
3372 let initial_schema = Schema::builder()
3373 .with_fields(vec![
3374 NestedField::required(1, "data", Type::Primitive(PrimitiveType::String)).into(),
3375 ])
3376 .build()
3377 .unwrap();
3378
3379 let partition_spec_with_bucket = UnboundPartitionSpec::builder()
3380 .with_spec_id(0)
3381 .add_partition_field(
3382 UnboundPartitionField::builder()
3383 .source_ids(vec![1])
3384 .name("bucket_data")
3385 .transform(Transform::Bucket(16))
3386 .build()
3387 .unwrap(),
3388 )
3389 .unwrap()
3390 .build();
3391
3392 let metadata = TableMetadataBuilder::new(
3393 initial_schema,
3394 partition_spec_with_bucket,
3395 SortOrder::unsorted_order(),
3396 TEST_LOCATION.to_string(),
3397 FormatVersion::V2,
3398 HashMap::new(),
3399 )
3400 .unwrap()
3401 .build()
3402 .unwrap()
3403 .metadata;
3404
3405 let partition_field_names: Vec<String> = metadata
3406 .default_partition_spec()
3407 .fields()
3408 .iter()
3409 .map(|f| f.name.clone())
3410 .collect();
3411 assert!(partition_field_names.contains(&"bucket_data".to_string()));
3412
3413 let evolved_schema = Schema::builder()
3414 .with_fields(vec![
3415 NestedField::required(1, "data", Type::Primitive(PrimitiveType::String)).into(),
3416 NestedField::required(2, "bucket_data", Type::Primitive(PrimitiveType::Int)).into(),
3418 ])
3419 .build()
3420 .unwrap();
3421
3422 let builder = metadata.into_builder(Some(
3423 "s3://bucket/test/location/metadata/metadata1.json".to_string(),
3424 ));
3425
3426 let result = builder.add_current_schema(evolved_schema);
3428
3429 assert!(result.is_err());
3430 let error = result.unwrap_err();
3431 let error_message = error.message();
3432 assert!(error_message.contains("Cannot set current schema"));
3433 assert!(error_message.contains("sourced from field id 1"));
3434 }
3435
3436 #[test]
3437 fn test_schema_evolution_should_validate_on_schema_add_not_metadata_build() {
3438 let initial_schema = Schema::builder()
3439 .with_fields(vec![
3440 NestedField::required(1, "data", Type::Primitive(PrimitiveType::String)).into(),
3441 ])
3442 .build()
3443 .unwrap();
3444
3445 let partition_spec = UnboundPartitionSpec::builder()
3446 .with_spec_id(0)
3447 .add_partition_field(
3448 UnboundPartitionField::builder()
3449 .source_ids(vec![1])
3450 .name("partition_col")
3451 .transform(Transform::Bucket(16))
3452 .build()
3453 .unwrap(),
3454 )
3455 .unwrap()
3456 .build();
3457
3458 let metadata = TableMetadataBuilder::new(
3459 initial_schema,
3460 partition_spec,
3461 SortOrder::unsorted_order(),
3462 TEST_LOCATION.to_string(),
3463 FormatVersion::V2,
3464 HashMap::new(),
3465 )
3466 .unwrap()
3467 .build()
3468 .unwrap()
3469 .metadata;
3470
3471 let non_conflicting_schema = Schema::builder()
3472 .with_fields(vec![
3473 NestedField::required(1, "data", Type::Primitive(PrimitiveType::String)).into(),
3474 NestedField::required(2, "new_field", Type::Primitive(PrimitiveType::Int)).into(),
3475 ])
3476 .build()
3477 .unwrap();
3478
3479 let result = metadata
3481 .clone()
3482 .into_builder(Some("test_location".to_string()))
3483 .add_current_schema(non_conflicting_schema)
3484 .unwrap()
3485 .build();
3486
3487 assert!(result.is_ok());
3488 }
3489
3490 #[test]
3491 fn test_partition_spec_evolution_validates_schema_field_name_conflicts() {
3492 let initial_schema = Schema::builder()
3493 .with_fields(vec![
3494 NestedField::required(1, "data", Type::Primitive(PrimitiveType::String)).into(),
3495 NestedField::required(2, "existing_field", Type::Primitive(PrimitiveType::Int))
3496 .into(),
3497 ])
3498 .build()
3499 .unwrap();
3500
3501 let partition_spec = UnboundPartitionSpec::builder()
3502 .with_spec_id(0)
3503 .add_partition_field(
3504 UnboundPartitionField::builder()
3505 .source_ids(vec![1])
3506 .name("data_bucket")
3507 .transform(Transform::Bucket(16))
3508 .build()
3509 .unwrap(),
3510 )
3511 .unwrap()
3512 .build();
3513
3514 let metadata = TableMetadataBuilder::new(
3515 initial_schema,
3516 partition_spec,
3517 SortOrder::unsorted_order(),
3518 TEST_LOCATION.to_string(),
3519 FormatVersion::V2,
3520 HashMap::new(),
3521 )
3522 .unwrap()
3523 .build()
3524 .unwrap()
3525 .metadata;
3526
3527 let builder = metadata.into_builder(Some(
3528 "s3://bucket/test/location/metadata/metadata1.json".to_string(),
3529 ));
3530
3531 let conflicting_partition_spec = UnboundPartitionSpec::builder()
3532 .with_spec_id(1)
3533 .add_partition_field(
3534 UnboundPartitionField::builder()
3535 .source_ids(vec![1])
3536 .name("existing_field")
3537 .transform(Transform::Bucket(8))
3538 .build()
3539 .unwrap(),
3540 )
3541 .unwrap()
3542 .build();
3543
3544 let result = builder.add_partition_spec(conflicting_partition_spec);
3545
3546 assert!(result.is_err());
3547 let error = result.unwrap_err();
3548 let error_message = error.message();
3549 assert!(error_message.contains(
3551 "Cannot create partition with name 'existing_field' that conflicts with schema field"
3552 ));
3553 assert!(error_message.contains("and is not an identity or void transform"));
3554 }
3555
3556 #[test]
3557 fn test_schema_evolution_validates_against_all_historical_schemas() {
3558 let initial_schema = Schema::builder()
3560 .with_fields(vec![
3561 NestedField::required(1, "data", Type::Primitive(PrimitiveType::String)).into(),
3562 NestedField::required(2, "existing_field", Type::Primitive(PrimitiveType::Int))
3563 .into(),
3564 ])
3565 .build()
3566 .unwrap();
3567
3568 let partition_spec = UnboundPartitionSpec::builder()
3569 .with_spec_id(0)
3570 .add_partition_field(
3571 UnboundPartitionField::builder()
3572 .source_ids(vec![1])
3573 .name("bucket_data")
3574 .transform(Transform::Bucket(16))
3575 .build()
3576 .unwrap(),
3577 )
3578 .unwrap()
3579 .build();
3580
3581 let metadata = TableMetadataBuilder::new(
3582 initial_schema,
3583 partition_spec,
3584 SortOrder::unsorted_order(),
3585 TEST_LOCATION.to_string(),
3586 FormatVersion::V2,
3587 HashMap::new(),
3588 )
3589 .unwrap()
3590 .build()
3591 .unwrap()
3592 .metadata;
3593
3594 let second_schema = Schema::builder()
3596 .with_fields(vec![
3597 NestedField::required(1, "data", Type::Primitive(PrimitiveType::String)).into(),
3598 NestedField::required(3, "new_field", Type::Primitive(PrimitiveType::String))
3599 .into(),
3600 ])
3601 .build()
3602 .unwrap();
3603
3604 let metadata = metadata
3605 .into_builder(Some("test_location".to_string()))
3606 .add_current_schema(second_schema)
3607 .unwrap()
3608 .build()
3609 .unwrap()
3610 .metadata;
3611
3612 let third_schema = Schema::builder()
3616 .with_fields(vec![
3617 NestedField::required(1, "data", Type::Primitive(PrimitiveType::String)).into(),
3618 NestedField::required(3, "new_field", Type::Primitive(PrimitiveType::String))
3619 .into(),
3620 NestedField::required(4, "existing_field", Type::Primitive(PrimitiveType::Int))
3621 .into(),
3622 ])
3623 .build()
3624 .unwrap();
3625
3626 let builder = metadata
3627 .clone()
3628 .into_builder(Some("test_location".to_string()));
3629
3630 let result = builder.add_current_schema(third_schema);
3632 assert!(result.is_ok());
3633
3634 let conflicting_schema = Schema::builder()
3637 .with_fields(vec![
3638 NestedField::required(1, "data", Type::Primitive(PrimitiveType::String)).into(),
3639 NestedField::required(3, "new_field", Type::Primitive(PrimitiveType::String))
3640 .into(),
3641 NestedField::required(4, "existing_field", Type::Primitive(PrimitiveType::Int))
3642 .into(),
3643 NestedField::required(5, "bucket_data", Type::Primitive(PrimitiveType::String))
3644 .into(), ])
3646 .build()
3647 .unwrap();
3648
3649 let builder2 = metadata.into_builder(Some("test_location".to_string()));
3650 let result2 = builder2.add_current_schema(conflicting_schema);
3651
3652 assert!(result2.is_err());
3655 let error = result2.unwrap_err();
3656 assert!(error.message().contains("Cannot set current schema"));
3657 assert!(error.message().contains("sourced from field id 1"));
3658 }
3659
3660 #[test]
3661 fn test_schema_evolution_allows_existing_partition_field_if_exists_in_historical_schema() {
3662 let initial_schema = Schema::builder()
3664 .with_fields(vec![
3665 NestedField::required(1, "data", Type::Primitive(PrimitiveType::String)).into(),
3666 NestedField::required(2, "partition_data", Type::Primitive(PrimitiveType::Int))
3667 .into(),
3668 ])
3669 .build()
3670 .unwrap();
3671
3672 let partition_spec = UnboundPartitionSpec::builder()
3673 .with_spec_id(0)
3674 .add_partition_field(
3675 UnboundPartitionField::builder()
3676 .source_ids(vec![2])
3677 .name("partition_data")
3678 .transform(Transform::Identity)
3679 .build()
3680 .unwrap(),
3681 )
3682 .unwrap()
3683 .build();
3684
3685 let metadata = TableMetadataBuilder::new(
3686 initial_schema,
3687 partition_spec,
3688 SortOrder::unsorted_order(),
3689 TEST_LOCATION.to_string(),
3690 FormatVersion::V2,
3691 HashMap::new(),
3692 )
3693 .unwrap()
3694 .build()
3695 .unwrap()
3696 .metadata;
3697
3698 let evolved_schema = Schema::builder()
3700 .with_fields(vec![
3701 NestedField::required(1, "data", Type::Primitive(PrimitiveType::String)).into(),
3702 NestedField::required(2, "partition_data", Type::Primitive(PrimitiveType::Int))
3703 .into(),
3704 NestedField::required(3, "new_field", Type::Primitive(PrimitiveType::String))
3705 .into(),
3706 ])
3707 .build()
3708 .unwrap();
3709
3710 let result = metadata
3712 .into_builder(Some("test_location".to_string()))
3713 .add_current_schema(evolved_schema);
3714
3715 assert!(result.is_ok());
3716 }
3717
3718 fn bucket_partitioned_metadata() -> TableMetadata {
3720 let schema = Schema::builder()
3721 .with_fields(vec![
3722 NestedField::required(1, "data", Type::Primitive(PrimitiveType::String)).into(),
3723 ])
3724 .build()
3725 .unwrap();
3726 let spec = UnboundPartitionSpec::builder()
3727 .with_spec_id(0)
3728 .add_partition_field(
3729 UnboundPartitionField::builder()
3730 .source_ids(vec![1])
3731 .name("bucket_data")
3732 .transform(Transform::Bucket(16))
3733 .build()
3734 .unwrap(),
3735 )
3736 .unwrap()
3737 .build();
3738
3739 TableMetadataBuilder::new(
3740 schema,
3741 spec,
3742 SortOrder::unsorted_order(),
3743 TEST_LOCATION.to_string(),
3744 FormatVersion::V2,
3745 HashMap::new(),
3746 )
3747 .unwrap()
3748 .build()
3749 .unwrap()
3750 .metadata
3751 }
3752
3753 fn unpartition(metadata: TableMetadata, file: &str) -> TableMetadata {
3755 let metadata = metadata
3756 .into_builder(Some(file.to_string()))
3757 .add_default_partition_spec(UnboundPartitionSpec::builder().build())
3758 .unwrap()
3759 .build()
3760 .unwrap()
3761 .metadata;
3762 assert_ne!(metadata.default_partition_spec().spec_id(), 0);
3763 assert!(metadata.partition_specs.contains_key(&0));
3764 metadata
3765 }
3766
3767 #[test]
3768 fn test_schema_evolution_allows_a_superseded_partition_field_name_once_its_source_is_dropped() {
3769 let schema = Schema::builder()
3772 .with_fields(vec![
3773 NestedField::required(2, "bucket_data", Type::Primitive(PrimitiveType::Int)).into(),
3774 ])
3775 .build()
3776 .unwrap();
3777
3778 let metadata = unpartition(
3779 bucket_partitioned_metadata(),
3780 "s3://bucket/t/metadata/m1.json",
3781 )
3782 .into_builder(Some("s3://bucket/t/metadata/m2.json".to_string()))
3783 .add_current_schema(schema.clone())
3784 .unwrap()
3785 .build()
3786 .unwrap()
3787 .metadata;
3788
3789 assert_eq!(metadata.current_schema().as_struct(), schema.as_struct());
3790 assert!(metadata.current_schema().field_by_name("data").is_none());
3791 assert_eq!(
3793 metadata.partition_spec_by_id(0).unwrap().fields()[0].name,
3794 "bucket_data"
3795 );
3796 }
3797
3798 #[test]
3799 fn test_schema_evolution_rejects_a_partition_field_name_while_its_source_remains() {
3800 let schema = Schema::builder()
3803 .with_fields(vec![
3804 NestedField::required(1, "data", Type::Primitive(PrimitiveType::String)).into(),
3805 NestedField::required(2, "bucket_data", Type::Primitive(PrimitiveType::Int)).into(),
3806 ])
3807 .build()
3808 .unwrap();
3809
3810 let error = bucket_partitioned_metadata()
3811 .into_builder(Some("s3://bucket/t/metadata/m1.json".to_string()))
3812 .add_current_schema(schema.clone())
3813 .unwrap_err();
3814 assert_eq!(error.kind(), ErrorKind::DataInvalid);
3815 assert!(
3816 error.message().contains("sourced from field id 1"),
3817 "{error}"
3818 );
3819
3820 let error = unpartition(
3821 bucket_partitioned_metadata(),
3822 "s3://bucket/t/metadata/m1.json",
3823 )
3824 .into_builder(Some("s3://bucket/t/metadata/m2.json".to_string()))
3825 .add_current_schema(schema)
3826 .unwrap_err();
3827 assert_eq!(error.kind(), ErrorKind::DataInvalid);
3828 assert!(
3829 error.message().contains("sourced from field id 1"),
3830 "{error}"
3831 );
3832 }
3833
3834 #[test]
3835 fn test_schema_evolution_partition_name_check_is_order_independent() {
3836 let schema = Schema::builder()
3839 .with_fields(vec![
3840 NestedField::required(2, "bucket_data", Type::Primitive(PrimitiveType::Int)).into(),
3841 ])
3842 .build()
3843 .unwrap();
3844 let unpartitioned = UnboundPartitionSpec::builder().build();
3845
3846 let schema_first = bucket_partitioned_metadata()
3847 .into_builder(Some("s3://bucket/t/metadata/m1.json".to_string()))
3848 .add_current_schema(schema.clone())
3849 .and_then(|b| b.add_default_partition_spec(unpartitioned.clone()))
3850 .and_then(|b| b.build());
3851 let spec_first = bucket_partitioned_metadata()
3852 .into_builder(Some("s3://bucket/t/metadata/m1.json".to_string()))
3853 .add_default_partition_spec(unpartitioned)
3854 .and_then(|b| b.add_current_schema(schema))
3855 .and_then(|b| b.build());
3856
3857 assert!(schema_first.is_ok(), "{:?}", schema_first.err());
3858 assert!(spec_first.is_ok(), "{:?}", spec_first.err());
3859 }
3860
3861 #[test]
3862 fn test_set_current_schema_rejects_a_partition_field_name_from_a_spec_added_after_the_schema() {
3863 let schema = Schema::builder()
3864 .with_fields(vec![
3865 NestedField::required(1, "data", Type::Primitive(PrimitiveType::String)).into(),
3866 ])
3867 .build()
3868 .unwrap();
3869 let conflicting_schema = Schema::builder()
3870 .with_fields(vec![
3871 NestedField::required(1, "data", Type::Primitive(PrimitiveType::String)).into(),
3872 NestedField::required(2, "bucket_data", Type::Primitive(PrimitiveType::Int)).into(),
3873 ])
3874 .build()
3875 .unwrap();
3876 let spec = UnboundPartitionSpec::builder()
3877 .add_partition_field(
3878 UnboundPartitionField::builder()
3879 .source_ids(vec![1])
3880 .name("bucket_data")
3881 .transform(Transform::Bucket(16))
3882 .build()
3883 .unwrap(),
3884 )
3885 .unwrap()
3886 .build();
3887
3888 let error = TableMetadataBuilder::new(
3889 schema,
3890 UnboundPartitionSpec::builder().build(),
3891 SortOrder::unsorted_order(),
3892 TEST_LOCATION.to_string(),
3893 FormatVersion::V2,
3894 HashMap::new(),
3895 )
3896 .unwrap()
3897 .add_schema(conflicting_schema)
3898 .and_then(|b| b.add_partition_spec(spec))
3899 .and_then(|b| b.set_current_schema(TableMetadataBuilder::LAST_ADDED))
3900 .unwrap_err();
3901 assert_eq!(error.kind(), ErrorKind::DataInvalid);
3902 assert!(
3903 error.message().contains("sourced from field id 1"),
3904 "{error}"
3905 );
3906 }
3907
3908 #[test]
3909 fn test_schema_with_a_conflicting_partition_field_name_can_be_added_but_not_made_current() {
3910 let conflicting_schema = Schema::builder()
3911 .with_fields(vec![
3912 NestedField::required(1, "data", Type::Primitive(PrimitiveType::String)).into(),
3913 NestedField::required(2, "bucket_data", Type::Primitive(PrimitiveType::Int)).into(),
3914 ])
3915 .build()
3916 .unwrap();
3917
3918 let metadata = bucket_partitioned_metadata()
3919 .into_builder(Some("s3://bucket/t/metadata/m1.json".to_string()))
3920 .add_schema(conflicting_schema.clone())
3921 .unwrap()
3922 .build()
3923 .unwrap()
3924 .metadata;
3925 assert_eq!(metadata.current_schema_id(), 0);
3926 assert_eq!(
3927 metadata.schema_by_id(1).unwrap().as_struct(),
3928 conflicting_schema.as_struct()
3929 );
3930
3931 let error = metadata
3932 .into_builder(Some("s3://bucket/t/metadata/m2.json".to_string()))
3933 .set_current_schema(1)
3934 .unwrap_err();
3935 assert_eq!(error.kind(), ErrorKind::DataInvalid);
3936 assert!(
3937 error.message().contains("sourced from field id 1"),
3938 "{error}"
3939 );
3940 }
3941
3942 #[test]
3943 fn test_schema_evolution_prevents_new_field_conflicting_with_partition_field() {
3944 let initial_schema = Schema::builder()
3946 .with_fields(vec![
3947 NestedField::required(1, "data", Type::Primitive(PrimitiveType::String)).into(),
3948 ])
3949 .build()
3950 .unwrap();
3951
3952 let partition_spec = UnboundPartitionSpec::builder()
3953 .with_spec_id(0)
3954 .add_partition_field(
3955 UnboundPartitionField::builder()
3956 .source_ids(vec![1])
3957 .name("bucket_data")
3958 .transform(Transform::Bucket(16))
3959 .build()
3960 .unwrap(),
3961 )
3962 .unwrap()
3963 .build();
3964
3965 let metadata = TableMetadataBuilder::new(
3966 initial_schema,
3967 partition_spec,
3968 SortOrder::unsorted_order(),
3969 TEST_LOCATION.to_string(),
3970 FormatVersion::V2,
3971 HashMap::new(),
3972 )
3973 .unwrap()
3974 .build()
3975 .unwrap()
3976 .metadata;
3977
3978 let conflicting_schema = Schema::builder()
3980 .with_fields(vec![
3981 NestedField::required(1, "data", Type::Primitive(PrimitiveType::String)).into(),
3982 NestedField::required(2, "bucket_data", Type::Primitive(PrimitiveType::Int)).into(),
3984 ])
3985 .build()
3986 .unwrap();
3987
3988 let builder = metadata.into_builder(Some("test_location".to_string()));
3989 let result = builder.add_current_schema(conflicting_schema);
3990
3991 assert!(result.is_err());
3994 let error = result.unwrap_err();
3995 assert!(error.message().contains("Cannot set current schema"));
3996 assert!(error.message().contains("sourced from field id 1"));
3997 }
3998
3999 #[test]
4000 fn test_partition_spec_evolution_allows_non_conflicting_names() {
4001 let initial_schema = Schema::builder()
4002 .with_fields(vec![
4003 NestedField::required(1, "data", Type::Primitive(PrimitiveType::String)).into(),
4004 NestedField::required(2, "existing_field", Type::Primitive(PrimitiveType::Int))
4005 .into(),
4006 ])
4007 .build()
4008 .unwrap();
4009
4010 let partition_spec = UnboundPartitionSpec::builder()
4011 .with_spec_id(0)
4012 .add_partition_field(
4013 UnboundPartitionField::builder()
4014 .source_ids(vec![1])
4015 .name("data_bucket")
4016 .transform(Transform::Bucket(16))
4017 .build()
4018 .unwrap(),
4019 )
4020 .unwrap()
4021 .build();
4022
4023 let metadata = TableMetadataBuilder::new(
4024 initial_schema,
4025 partition_spec,
4026 SortOrder::unsorted_order(),
4027 TEST_LOCATION.to_string(),
4028 FormatVersion::V2,
4029 HashMap::new(),
4030 )
4031 .unwrap()
4032 .build()
4033 .unwrap()
4034 .metadata;
4035
4036 let builder = metadata.into_builder(Some(
4037 "s3://bucket/test/location/metadata/metadata1.json".to_string(),
4038 ));
4039
4040 let non_conflicting_partition_spec = UnboundPartitionSpec::builder()
4042 .with_spec_id(1)
4043 .add_partition_field(
4044 UnboundPartitionField::builder()
4045 .source_ids(vec![2])
4046 .name("new_partition_field")
4047 .transform(Transform::Bucket(8))
4048 .build()
4049 .unwrap(),
4050 )
4051 .unwrap()
4052 .build();
4053
4054 let result = builder.add_partition_spec(non_conflicting_partition_spec);
4055
4056 assert!(result.is_ok());
4057 }
4058
4059 #[test]
4060 fn test_row_lineage_addition() {
4061 let new_rows = 30;
4062 let base = builder_without_changes(FormatVersion::V3)
4063 .build()
4064 .unwrap()
4065 .metadata;
4066 let add_rows = Snapshot::builder()
4067 .with_snapshot_id(0)
4068 .with_timestamp_ms(base.last_updated_ms + 1)
4069 .with_sequence_number(0)
4070 .with_schema_id(0)
4071 .with_manifest_list("foo")
4072 .with_parent_snapshot_id(None)
4073 .with_summary(Summary {
4074 operation: Operation::Append,
4075 additional_properties: HashMap::new(),
4076 })
4077 .with_row_range(base.next_row_id(), new_rows)
4078 .build();
4079
4080 let first_addition = base
4081 .into_builder(None)
4082 .add_snapshot(add_rows.clone())
4083 .unwrap()
4084 .build()
4085 .unwrap()
4086 .metadata;
4087
4088 assert_eq!(first_addition.next_row_id(), new_rows);
4089
4090 let add_more_rows = Snapshot::builder()
4091 .with_snapshot_id(1)
4092 .with_timestamp_ms(first_addition.last_updated_ms + 1)
4093 .with_sequence_number(1)
4094 .with_schema_id(0)
4095 .with_manifest_list("foo")
4096 .with_parent_snapshot_id(Some(0))
4097 .with_summary(Summary {
4098 operation: Operation::Append,
4099 additional_properties: HashMap::new(),
4100 })
4101 .with_row_range(first_addition.next_row_id(), new_rows)
4102 .build();
4103
4104 let second_addition = first_addition
4105 .into_builder(None)
4106 .add_snapshot(add_more_rows)
4107 .unwrap()
4108 .build()
4109 .unwrap()
4110 .metadata;
4111 assert_eq!(second_addition.next_row_id(), new_rows * 2);
4112 }
4113
4114 #[test]
4115 fn test_row_lineage_invalid_snapshot() {
4116 let new_rows = 30;
4117 let base = builder_without_changes(FormatVersion::V3)
4118 .build()
4119 .unwrap()
4120 .metadata;
4121
4122 let add_rows = Snapshot::builder()
4124 .with_snapshot_id(0)
4125 .with_timestamp_ms(base.last_updated_ms + 1)
4126 .with_sequence_number(0)
4127 .with_schema_id(0)
4128 .with_manifest_list("foo")
4129 .with_parent_snapshot_id(None)
4130 .with_summary(Summary {
4131 operation: Operation::Append,
4132 additional_properties: HashMap::new(),
4133 })
4134 .with_row_range(base.next_row_id(), new_rows)
4135 .build();
4136
4137 let added = base
4138 .into_builder(None)
4139 .add_snapshot(add_rows)
4140 .unwrap()
4141 .build()
4142 .unwrap()
4143 .metadata;
4144
4145 let invalid_new_rows = Snapshot::builder()
4146 .with_snapshot_id(1)
4147 .with_timestamp_ms(added.last_updated_ms + 1)
4148 .with_sequence_number(1)
4149 .with_schema_id(0)
4150 .with_manifest_list("foo")
4151 .with_parent_snapshot_id(Some(0))
4152 .with_summary(Summary {
4153 operation: Operation::Append,
4154 additional_properties: HashMap::new(),
4155 })
4156 .with_row_range(added.next_row_id() - 1, 10)
4158 .build();
4159
4160 let err = added
4161 .into_builder(None)
4162 .add_snapshot(invalid_new_rows)
4163 .unwrap_err();
4164 assert!(
4165 err.to_string().contains(
4166 "Cannot add a snapshot, first-row-id is behind table next-row-id: 29 < 30"
4167 )
4168 );
4169 }
4170
4171 #[test]
4172 fn test_row_lineage_append_branch() {
4173 let branch = "some_branch";
4177
4178 let base = builder_without_changes(FormatVersion::V3)
4180 .build()
4181 .unwrap()
4182 .metadata;
4183
4184 assert_eq!(base.next_row_id(), 0);
4186
4187 let branch_snapshot_1 = Snapshot::builder()
4189 .with_snapshot_id(1)
4190 .with_timestamp_ms(base.last_updated_ms + 1)
4191 .with_sequence_number(0)
4192 .with_schema_id(0)
4193 .with_manifest_list("foo")
4194 .with_parent_snapshot_id(None)
4195 .with_summary(Summary {
4196 operation: Operation::Append,
4197 additional_properties: HashMap::new(),
4198 })
4199 .with_row_range(base.next_row_id(), 30)
4200 .build();
4201
4202 let table_after_branch_1 = base
4203 .into_builder(None)
4204 .set_branch_snapshot(branch_snapshot_1.clone(), branch)
4205 .unwrap()
4206 .build()
4207 .unwrap()
4208 .metadata;
4209
4210 assert!(table_after_branch_1.current_snapshot().is_none());
4212
4213 let branch_ref = table_after_branch_1.refs.get(branch).unwrap();
4215 let branch_snap_1 = table_after_branch_1
4216 .snapshots
4217 .get(&branch_ref.snapshot_id)
4218 .unwrap();
4219 assert_eq!(branch_snap_1.first_row_id(), Some(0));
4220
4221 assert_eq!(table_after_branch_1.next_row_id(), 30);
4223
4224 let main_snapshot = Snapshot::builder()
4226 .with_snapshot_id(2)
4227 .with_timestamp_ms(table_after_branch_1.last_updated_ms + 1)
4228 .with_sequence_number(1)
4229 .with_schema_id(0)
4230 .with_manifest_list("bar")
4231 .with_parent_snapshot_id(None)
4232 .with_summary(Summary {
4233 operation: Operation::Append,
4234 additional_properties: HashMap::new(),
4235 })
4236 .with_row_range(table_after_branch_1.next_row_id(), 28)
4237 .build();
4238
4239 let table_after_main = table_after_branch_1
4240 .into_builder(None)
4241 .add_snapshot(main_snapshot.clone())
4242 .unwrap()
4243 .set_ref(MAIN_BRANCH, SnapshotReference {
4244 snapshot_id: main_snapshot.snapshot_id(),
4245 retention: SnapshotRetention::Branch {
4246 min_snapshots_to_keep: None,
4247 max_snapshot_age_ms: None,
4248 max_ref_age_ms: None,
4249 },
4250 })
4251 .unwrap()
4252 .build()
4253 .unwrap()
4254 .metadata;
4255
4256 let current_snapshot = table_after_main.current_snapshot().unwrap();
4258 assert_eq!(current_snapshot.first_row_id(), Some(30));
4259
4260 assert_eq!(table_after_main.next_row_id(), 58);
4262
4263 let branch_snapshot_2 = Snapshot::builder()
4265 .with_snapshot_id(3)
4266 .with_timestamp_ms(table_after_main.last_updated_ms + 1)
4267 .with_sequence_number(2)
4268 .with_schema_id(0)
4269 .with_manifest_list("baz")
4270 .with_parent_snapshot_id(Some(branch_snapshot_1.snapshot_id()))
4271 .with_summary(Summary {
4272 operation: Operation::Append,
4273 additional_properties: HashMap::new(),
4274 })
4275 .with_row_range(table_after_main.next_row_id(), 21)
4276 .build();
4277
4278 let table_after_branch_2 = table_after_main
4279 .into_builder(None)
4280 .set_branch_snapshot(branch_snapshot_2.clone(), branch)
4281 .unwrap()
4282 .build()
4283 .unwrap()
4284 .metadata;
4285
4286 let branch_ref_2 = table_after_branch_2.refs.get(branch).unwrap();
4288 let branch_snap_2 = table_after_branch_2
4289 .snapshots
4290 .get(&branch_ref_2.snapshot_id)
4291 .unwrap();
4292 assert_eq!(branch_snap_2.first_row_id(), Some(58));
4293
4294 assert_eq!(table_after_branch_2.next_row_id(), 79);
4296 }
4297
4298 #[test]
4299 fn test_encryption_keys() {
4300 let builder = builder_without_changes(FormatVersion::V3);
4301
4302 let encryption_key_1 = EncryptedKey::builder()
4304 .key_id("key-1")
4305 .encrypted_key_metadata(vec![1, 2, 3, 4])
4306 .encrypted_by_id("encryption-service-1")
4307 .properties(HashMap::from_iter(vec![(
4308 "algorithm".to_string(),
4309 "AES-256".to_string(),
4310 )]))
4311 .build();
4312
4313 let encryption_key_2 = EncryptedKey::builder()
4314 .key_id("key-2")
4315 .encrypted_key_metadata(vec![5, 6, 7, 8])
4316 .encrypted_by_id("encryption-service-2")
4317 .properties(HashMap::new())
4318 .build();
4319
4320 let build_result = builder
4322 .add_encryption_key(encryption_key_1.clone())
4323 .build()
4324 .unwrap();
4325
4326 assert_eq!(build_result.changes.len(), 1);
4327 assert_eq!(build_result.metadata.encryption_keys.len(), 1);
4328 assert_eq!(
4329 build_result.metadata.encryption_key("key-1"),
4330 Some(&encryption_key_1)
4331 );
4332 assert_eq!(build_result.changes[0], TableUpdate::AddEncryptionKey {
4333 encryption_key: encryption_key_1.clone()
4334 });
4335
4336 let build_result = build_result
4338 .metadata
4339 .into_builder(Some(
4340 "s3://bucket/test/location/metadata/metadata1.json".to_string(),
4341 ))
4342 .add_encryption_key(encryption_key_2.clone())
4343 .build()
4344 .unwrap();
4345
4346 assert_eq!(build_result.changes.len(), 1);
4347 assert_eq!(build_result.metadata.encryption_keys.len(), 2);
4348 assert_eq!(
4349 build_result.metadata.encryption_key("key-1"),
4350 Some(&encryption_key_1)
4351 );
4352 assert_eq!(
4353 build_result.metadata.encryption_key("key-2"),
4354 Some(&encryption_key_2)
4355 );
4356 assert_eq!(build_result.changes[0], TableUpdate::AddEncryptionKey {
4357 encryption_key: encryption_key_2.clone()
4358 });
4359
4360 let build_result = build_result
4362 .metadata
4363 .into_builder(Some(
4364 "s3://bucket/test/location/metadata/metadata2.json".to_string(),
4365 ))
4366 .add_encryption_key(encryption_key_1.clone())
4367 .build()
4368 .unwrap();
4369
4370 assert_eq!(build_result.changes.len(), 0);
4371 assert_eq!(build_result.metadata.encryption_keys.len(), 2);
4372
4373 let build_result = build_result
4375 .metadata
4376 .into_builder(Some(
4377 "s3://bucket/test/location/metadata/metadata3.json".to_string(),
4378 ))
4379 .remove_encryption_key("key-1")
4380 .build()
4381 .unwrap();
4382
4383 assert_eq!(build_result.changes.len(), 1);
4384 assert_eq!(build_result.metadata.encryption_keys.len(), 1);
4385 assert_eq!(build_result.metadata.encryption_key("key-1"), None);
4386 assert_eq!(
4387 build_result.metadata.encryption_key("key-2"),
4388 Some(&encryption_key_2)
4389 );
4390 assert_eq!(build_result.changes[0], TableUpdate::RemoveEncryptionKey {
4391 key_id: "key-1".to_string()
4392 });
4393
4394 let build_result = build_result
4396 .metadata
4397 .into_builder(Some(
4398 "s3://bucket/test/location/metadata/metadata4.json".to_string(),
4399 ))
4400 .remove_encryption_key("non-existent-key")
4401 .build()
4402 .unwrap();
4403
4404 assert_eq!(build_result.changes.len(), 0);
4405 assert_eq!(build_result.metadata.encryption_keys.len(), 1);
4406
4407 let keys = build_result
4409 .metadata
4410 .encryption_keys_iter()
4411 .collect::<Vec<_>>();
4412 assert_eq!(keys.len(), 1);
4413 assert_eq!(keys[0], &encryption_key_2);
4414
4415 let build_result = build_result
4417 .metadata
4418 .into_builder(Some(
4419 "s3://bucket/test/location/metadata/metadata5.json".to_string(),
4420 ))
4421 .remove_encryption_key("key-2")
4422 .build()
4423 .unwrap();
4424
4425 assert_eq!(build_result.changes.len(), 1);
4426 assert_eq!(build_result.metadata.encryption_keys.len(), 0);
4427 assert_eq!(build_result.metadata.encryption_key("key-2"), None);
4428 assert_eq!(build_result.changes[0], TableUpdate::RemoveEncryptionKey {
4429 key_id: "key-2".to_string()
4430 });
4431
4432 let keys = build_result.metadata.encryption_keys_iter();
4434 assert_eq!(keys.len(), 0);
4435 }
4436
4437 #[test]
4438 fn test_partition_field_id_reuse_across_specs() {
4439 let schema = Schema::builder()
4440 .with_fields(vec![
4441 NestedField::required(1, "id", Type::Primitive(PrimitiveType::Long)).into(),
4442 NestedField::required(2, "data", Type::Primitive(PrimitiveType::String)).into(),
4443 NestedField::required(3, "timestamp", Type::Primitive(PrimitiveType::Timestamp))
4444 .into(),
4445 ])
4446 .build()
4447 .unwrap();
4448
4449 let initial_spec = UnboundPartitionSpec::builder()
4451 .add_partition_field(
4452 UnboundPartitionField::builder()
4453 .source_ids(vec![1])
4454 .name("id")
4455 .transform(Transform::Identity)
4456 .build()
4457 .unwrap(),
4458 )
4459 .unwrap()
4460 .build();
4461
4462 let mut metadata = TableMetadataBuilder::new(
4463 schema,
4464 initial_spec,
4465 SortOrder::unsorted_order(),
4466 "s3://bucket/table".to_string(),
4467 FormatVersion::V2,
4468 HashMap::new(),
4469 )
4470 .unwrap()
4471 .build()
4472 .unwrap()
4473 .metadata;
4474
4475 let spec1 = UnboundPartitionSpec::builder()
4477 .add_partition_field(
4478 UnboundPartitionField::builder()
4479 .source_ids(vec![2])
4480 .name("data_bucket")
4481 .transform(Transform::Bucket(10))
4482 .build()
4483 .unwrap(),
4484 )
4485 .unwrap()
4486 .build();
4487 let builder = metadata.into_builder(Some("s3://bucket/table/metadata/v1.json".to_string()));
4488 let result = builder.add_partition_spec(spec1).unwrap().build().unwrap();
4489 metadata = result.metadata;
4490
4491 let spec2 = UnboundPartitionSpec::builder()
4494 .add_partition_field(
4495 UnboundPartitionField::builder()
4496 .source_ids(vec![1])
4497 .name("id")
4498 .transform(Transform::Identity)
4499 .build()
4500 .unwrap(),
4501 ) .unwrap()
4503 .add_partition_field(
4504 UnboundPartitionField::builder()
4505 .source_ids(vec![2])
4506 .name("data_bucket")
4507 .transform(Transform::Bucket(10))
4508 .build()
4509 .unwrap(),
4510 ) .unwrap()
4512 .add_partition_field(
4513 UnboundPartitionField::builder()
4514 .source_ids(vec![3])
4515 .name("year")
4516 .transform(Transform::Year)
4517 .build()
4518 .unwrap(),
4519 ) .unwrap()
4521 .build();
4522 let builder = metadata.into_builder(Some("s3://bucket/table/metadata/v2.json".to_string()));
4523 let result = builder.add_partition_spec(spec2).unwrap().build().unwrap();
4524
4525 let spec2 = result.metadata.partition_spec_by_id(2).unwrap();
4527 let field_ids: Vec<i32> = spec2.fields().iter().map(|f| f.field_id).collect();
4528 assert_eq!(field_ids, vec![1000, 1001, 1002]); }
4530
4531 fn unbound_field(source_id: i32, name: &str, transform: Transform) -> UnboundPartitionField {
4532 UnboundPartitionField::builder()
4533 .source_ids(vec![source_id])
4534 .name(name)
4535 .transform(transform)
4536 .build()
4537 .unwrap()
4538 }
4539
4540 fn replacement_schema() -> Schema {
4541 Schema::builder()
4543 .with_fields(vec![
4544 NestedField::required(1, "z", Type::Primitive(PrimitiveType::Long)).into(),
4545 NestedField::required(2, "x", Type::Primitive(PrimitiveType::Long)).into(),
4546 NestedField::required(3, "w", Type::Primitive(PrimitiveType::String)).into(),
4547 ])
4548 .build()
4549 .unwrap()
4550 }
4551
4552 fn partition_replacement_builder(format_version: FormatVersion) -> TableMetadataBuilder {
4553 let schema = Schema::builder()
4554 .with_fields(vec![
4555 NestedField::required(1, "x", Type::Primitive(PrimitiveType::Long)).into(),
4556 NestedField::required(2, "y", Type::Primitive(PrimitiveType::Long)).into(),
4557 ])
4558 .build()
4559 .unwrap();
4560 let spec = UnboundPartitionSpec::builder()
4561 .add_partition_field(unbound_field(1, "x", Transform::Identity))
4562 .unwrap()
4563 .add_partition_field(unbound_field(2, "y", Transform::Identity))
4564 .unwrap()
4565 .build();
4566
4567 TableMetadataBuilder::new(
4568 schema,
4569 spec,
4570 SortOrder::unsorted_order(),
4571 TEST_LOCATION.to_string(),
4572 format_version,
4573 HashMap::new(),
4574 )
4575 .unwrap()
4576 .build()
4577 .unwrap()
4578 .metadata
4579 .into_builder(Some(
4580 "s3://bucket/test/location/metadata/metadata1.json".to_string(),
4581 ))
4582 }
4583
4584 fn populated_builder() -> TableMetadataBuilder {
4585 let builder = builder_without_changes(FormatVersion::V2);
4586 let snapshot = Snapshot::builder()
4587 .with_snapshot_id(1)
4588 .with_timestamp_ms(builder.metadata.last_updated_ms + 1)
4589 .with_sequence_number(0)
4590 .with_schema_id(0)
4591 .with_manifest_list("/snap-1.avro")
4592 .with_summary(Summary {
4593 operation: Operation::Append,
4594 additional_properties: HashMap::new(),
4595 })
4596 .build();
4597
4598 builder
4599 .add_snapshot(snapshot)
4600 .unwrap()
4601 .set_ref(MAIN_BRANCH, SnapshotReference {
4602 snapshot_id: 1,
4603 retention: SnapshotRetention::branch(None, None, None),
4604 })
4605 .unwrap()
4606 .build()
4607 .unwrap()
4608 .metadata
4609 .into_builder(Some(
4610 "s3://bucket/test/location/metadata/metadata2.json".to_string(),
4611 ))
4612 }
4613
4614 fn replaced_schema() -> Schema {
4615 Schema::builder()
4616 .with_schema_id(1)
4617 .with_fields(vec![
4618 NestedField::required(3, "z", Type::Primitive(PrimitiveType::Long)).into(),
4619 NestedField::required(1, "x", Type::Primitive(PrimitiveType::Long)).into(),
4620 NestedField::required(4, "w", Type::Primitive(PrimitiveType::String)).into(),
4621 ])
4622 .build()
4623 .unwrap()
4624 }
4625
4626 #[test]
4627 fn test_build_replacement_keeps_existing_field_ids_and_assigns_new_ones_above_last_column_id() {
4628 let metadata = builder_without_changes(FormatVersion::V2)
4629 .build_replacement(
4630 replacement_schema(),
4631 UnboundPartitionSpec::builder().build(),
4632 SortOrder::unsorted_order(),
4633 None,
4634 None,
4635 HashMap::new(),
4636 )
4637 .unwrap()
4638 .build()
4639 .unwrap()
4640 .metadata;
4641
4642 assert_eq!(metadata.current_schema().as_ref(), &replaced_schema());
4643 assert_eq!(metadata.last_column_id, 4);
4644 }
4645
4646 #[test]
4647 fn test_build_replacement_does_not_reuse_the_id_of_a_dropped_column() {
4648 let without_y = Schema::builder()
4649 .with_fields(vec![
4650 NestedField::required(1, "x", Type::Primitive(PrimitiveType::Long)).into(),
4651 ])
4652 .build()
4653 .unwrap();
4654 let replacement = Schema::builder()
4655 .with_fields(vec![
4656 NestedField::required(1, "x", Type::Primitive(PrimitiveType::Long)).into(),
4657 NestedField::required(2, "w", Type::Primitive(PrimitiveType::String)).into(),
4658 ])
4659 .build()
4660 .unwrap();
4661
4662 let metadata = partition_replacement_builder(FormatVersion::V2)
4663 .add_current_schema(without_y)
4664 .unwrap()
4665 .build_replacement(
4666 replacement,
4667 UnboundPartitionSpec::builder().build(),
4668 SortOrder::unsorted_order(),
4669 None,
4670 None,
4671 HashMap::new(),
4672 )
4673 .unwrap()
4674 .build()
4675 .unwrap()
4676 .metadata;
4677
4678 let expected = Schema::builder()
4679 .with_schema_id(2)
4680 .with_fields(vec![
4681 NestedField::required(1, "x", Type::Primitive(PrimitiveType::Long)).into(),
4682 NestedField::required(3, "w", Type::Primitive(PrimitiveType::String)).into(),
4683 ])
4684 .build()
4685 .unwrap();
4686 assert_eq!(metadata.current_schema().as_ref(), &expected);
4687 assert_eq!(metadata.last_column_id, 3);
4688 }
4689
4690 #[test]
4691 fn test_build_replacement_resolves_spec_and_sort_order_source_ids_against_the_replacement_schema()
4692 {
4693 let spec = UnboundPartitionSpec::builder()
4694 .add_partition_field(unbound_field(3, "w", Transform::Identity))
4695 .unwrap()
4696 .build();
4697 let sort_field = SortField {
4698 source_id: 3,
4699 transform: Transform::Identity,
4700 direction: SortDirection::Ascending,
4701 null_order: NullOrder::First,
4702 };
4703 let sort_order = SortOrder::builder()
4704 .with_sort_field(sort_field.clone())
4705 .build(&replacement_schema())
4706 .unwrap();
4707
4708 let metadata = builder_without_changes(FormatVersion::V2)
4709 .build_replacement(
4710 replacement_schema(),
4711 spec,
4712 sort_order,
4713 None,
4714 None,
4715 HashMap::new(),
4716 )
4717 .unwrap()
4718 .build()
4719 .unwrap()
4720 .metadata;
4721
4722 let expected_spec = PartitionSpec::builder(replaced_schema())
4723 .with_spec_id(1)
4724 .add_unbound_field(unbound_field(4, "w", Transform::Identity).with_field_id(1001))
4725 .unwrap()
4726 .build()
4727 .unwrap();
4728 let expected_sort_order = SortOrder::builder()
4729 .with_order_id(2)
4730 .with_sort_field(SortField {
4731 source_id: 4,
4732 ..sort_field
4733 })
4734 .build(&replaced_schema())
4735 .unwrap();
4736 assert_eq!(metadata.default_partition_spec().as_ref(), &expected_spec);
4737 assert_eq!(metadata.default_sort_order().as_ref(), &expected_sort_order);
4738 }
4739
4740 #[test]
4741 fn test_build_replacement_reassigns_partition_ids_for_v2() {
4742 let updated_schema = Schema::builder()
4743 .with_fields(vec![
4744 NestedField::required(1, "x", Type::Primitive(PrimitiveType::Long)).into(),
4745 NestedField::required(2, "z", Type::Primitive(PrimitiveType::String)).into(),
4746 ])
4747 .build()
4748 .unwrap();
4749 let updated_spec = UnboundPartitionSpec::builder()
4750 .add_partition_field(unbound_field(2, "z_bucket", Transform::Bucket(8)))
4751 .unwrap()
4752 .add_partition_field(unbound_field(1, "x", Transform::Identity))
4753 .unwrap()
4754 .build();
4755
4756 let result = partition_replacement_builder(FormatVersion::V2)
4757 .build_replacement(
4758 updated_schema,
4759 updated_spec,
4760 SortOrder::unsorted_order(),
4761 None,
4762 None,
4763 HashMap::new(),
4764 )
4765 .unwrap()
4766 .build()
4767 .unwrap();
4768 let metadata = &result.metadata;
4769
4770 let expected_spec = PartitionSpec::builder(metadata.current_schema().clone())
4771 .with_spec_id(1)
4772 .add_unbound_fields([
4773 unbound_field(3, "z_bucket", Transform::Bucket(8)).with_field_id(1002),
4774 unbound_field(1, "x", Transform::Identity).with_field_id(1000),
4775 ])
4776 .unwrap()
4777 .build()
4778 .unwrap();
4779 assert_eq!(metadata.default_partition_spec().as_ref(), &expected_spec);
4780 assert_eq!(metadata.last_partition_id, 1002);
4781 assert!(result.changes.contains(&TableUpdate::AddSpec {
4782 spec: expected_spec.into_unbound()
4783 }));
4784 }
4785
4786 #[test]
4787 fn test_build_replacement_rejects_a_new_partition_field_when_no_partition_id_is_left() {
4788 let mut builder = partition_replacement_builder(FormatVersion::V2);
4789 builder.metadata.last_partition_id = i32::MAX;
4790 let updated_schema = Schema::builder()
4791 .with_fields(vec![
4792 NestedField::required(1, "x", Type::Primitive(PrimitiveType::Long)).into(),
4793 NestedField::required(2, "z", Type::Primitive(PrimitiveType::String)).into(),
4794 ])
4795 .build()
4796 .unwrap();
4797 let updated_spec = UnboundPartitionSpec::builder()
4798 .add_partition_field(unbound_field(2, "z_bucket", Transform::Bucket(8)))
4799 .unwrap()
4800 .build();
4801
4802 let err = builder
4803 .build_replacement(
4804 updated_schema,
4805 updated_spec,
4806 SortOrder::unsorted_order(),
4807 None,
4808 None,
4809 HashMap::new(),
4810 )
4811 .unwrap_err();
4812
4813 assert_eq!(
4814 (err.kind(), err.message()),
4815 (
4816 ErrorKind::DataInvalid,
4817 "Cannot assign more partition ids. Overflow."
4818 )
4819 );
4820 }
4821
4822 #[test]
4823 fn test_build_replacement_keeps_the_table_and_its_snapshots_but_not_its_current_one() {
4824 let base = populated_builder();
4825 let previous = base.metadata.clone();
4826
4827 let metadata = base
4828 .build_replacement(
4829 replacement_schema(),
4830 UnboundPartitionSpec::builder().build(),
4831 SortOrder::unsorted_order(),
4832 None,
4833 None,
4834 HashMap::new(),
4835 )
4836 .unwrap()
4837 .build()
4838 .unwrap()
4839 .metadata;
4840
4841 assert_eq!(metadata.table_uuid, previous.table_uuid);
4842 assert_eq!(metadata.location, previous.location);
4843 assert_eq!(metadata.current_snapshot_id, None);
4844 assert_eq!(metadata.snapshots, previous.snapshots);
4845 assert_eq!(metadata.snapshot_log, previous.snapshot_log);
4846 assert_eq!(metadata.schema_by_id(0), previous.schema_by_id(0));
4847 }
4848
4849 #[test]
4850 fn test_build_replacement_then_adding_a_snapshot_makes_it_current_on_main() {
4851 let builder = populated_builder()
4852 .build_replacement(
4853 replacement_schema(),
4854 UnboundPartitionSpec::builder().build(),
4855 SortOrder::unsorted_order(),
4856 None,
4857 None,
4858 HashMap::new(),
4859 )
4860 .unwrap();
4861 let snapshot = Snapshot::builder()
4862 .with_snapshot_id(2)
4863 .with_timestamp_ms(builder.metadata.last_updated_ms + 1)
4864 .with_sequence_number(builder.metadata.next_sequence_number())
4865 .with_schema_id(builder.metadata.current_schema_id)
4866 .with_manifest_list("/snap-2.avro")
4867 .with_summary(Summary {
4868 operation: Operation::Overwrite,
4869 additional_properties: HashMap::new(),
4870 })
4871 .build();
4872 let main = SnapshotReference {
4873 snapshot_id: 2,
4874 retention: SnapshotRetention::branch(None, None, None),
4875 };
4876
4877 let metadata = builder
4878 .add_snapshot(snapshot.clone())
4879 .unwrap()
4880 .set_ref(MAIN_BRANCH, main.clone())
4881 .unwrap()
4882 .build()
4883 .unwrap()
4884 .metadata;
4885
4886 assert_eq!(metadata.current_snapshot(), Some(&Arc::new(snapshot)));
4887 assert_eq!(
4888 metadata.refs,
4889 HashMap::from([(MAIN_BRANCH.to_string(), main)])
4890 );
4891 }
4892
4893 #[test]
4894 fn test_build_replacement_changes_describe_the_replacement() {
4895 let changes = populated_builder()
4896 .build_replacement(
4897 replacement_schema(),
4898 UnboundPartitionSpec::builder().build(),
4899 SortOrder::unsorted_order(),
4900 None,
4901 None,
4902 HashMap::new(),
4903 )
4904 .unwrap()
4905 .build()
4906 .unwrap()
4907 .changes;
4908
4909 assert_eq!(changes, vec![
4910 TableUpdate::RemoveSnapshotRef {
4911 ref_name: MAIN_BRANCH.to_string()
4912 },
4913 TableUpdate::AddSchema {
4914 schema: replaced_schema()
4915 },
4916 TableUpdate::SetCurrentSchema {
4917 schema_id: TableMetadataBuilder::LAST_ADDED
4918 },
4919 TableUpdate::AddSpec {
4920 spec: UnboundPartitionSpec::builder().with_spec_id(1).build()
4921 },
4922 TableUpdate::SetDefaultSpec {
4923 spec_id: TableMetadataBuilder::LAST_ADDED
4924 },
4925 TableUpdate::AddSortOrder {
4926 sort_order: SortOrder::unsorted_order()
4927 },
4928 TableUpdate::SetDefaultSortOrder {
4929 sort_order_id: TableMetadataBuilder::LAST_ADDED as i64
4930 },
4931 ]);
4932 }
4933
4934 #[test]
4935 fn test_build_replacement_moves_the_table_when_given_a_location() {
4936 let metadata = builder_without_changes(FormatVersion::V2)
4937 .build_replacement(
4938 replacement_schema(),
4939 UnboundPartitionSpec::builder().build(),
4940 SortOrder::unsorted_order(),
4941 Some("s3://bucket/somewhere/else".to_string()),
4942 None,
4943 HashMap::new(),
4944 )
4945 .unwrap()
4946 .build()
4947 .unwrap()
4948 .metadata;
4949
4950 assert_eq!(metadata.location, "s3://bucket/somewhere/else");
4951 }
4952
4953 #[test]
4954 fn test_build_replacement_applies_format_version_and_properties() {
4955 let result = builder_without_changes(FormatVersion::V2)
4956 .set_properties(HashMap::from_iter([(
4957 "existing".to_string(),
4958 "value".to_string(),
4959 )]))
4960 .unwrap()
4961 .build_replacement(
4962 replacement_schema(),
4963 UnboundPartitionSpec::builder().build(),
4964 SortOrder::unsorted_order(),
4965 None,
4966 Some(FormatVersion::V3),
4967 HashMap::from_iter([("new".to_string(), "value".to_string())]),
4968 )
4969 .unwrap()
4970 .build()
4971 .unwrap();
4972
4973 assert_eq!(result.metadata.format_version, FormatVersion::V3);
4974 assert_eq!(
4975 result.metadata.properties,
4976 HashMap::from_iter([
4977 ("existing".to_string(), "value".to_string()),
4978 ("new".to_string(), "value".to_string()),
4979 ])
4980 );
4981 assert!(result.changes.contains(&TableUpdate::UpgradeFormatVersion {
4982 format_version: FormatVersion::V3
4983 }));
4984 assert!(result.changes.contains(&TableUpdate::SetProperties {
4985 updates: HashMap::from_iter([("new".to_string(), "value".to_string())])
4986 }));
4987 }
4988
4989 #[test]
4990 fn test_build_replacement_with_everything_unchanged_reuses_the_existing_ids() {
4991 let base = builder_without_changes(FormatVersion::V2)
4992 .build()
4993 .unwrap()
4994 .metadata;
4995 let schema = base.current_schema().as_ref().clone();
4996 let spec = base
4997 .default_partition_spec()
4998 .as_ref()
4999 .clone()
5000 .into_unbound();
5001 let sort_order = base.default_sort_order().as_ref().clone();
5002
5003 let result = base
5004 .clone()
5005 .into_builder(Some(
5006 "s3://bucket/test/location/metadata/metadata2.json".to_string(),
5007 ))
5008 .build_replacement(
5009 schema.clone(),
5010 spec.clone(),
5011 sort_order.clone(),
5012 None,
5013 Some(FormatVersion::V3),
5014 HashMap::new(),
5015 )
5016 .unwrap()
5017 .build()
5018 .unwrap();
5019
5020 let metadata = &result.metadata;
5021 assert_eq!(metadata.schemas, base.schemas);
5022 assert_eq!(metadata.current_schema_id, base.current_schema_id);
5023 assert_eq!(metadata.last_column_id, base.last_column_id);
5024 assert_eq!(metadata.partition_specs, base.partition_specs);
5025 assert_eq!(metadata.default_spec, base.default_spec);
5026 assert_eq!(metadata.last_partition_id, base.last_partition_id);
5027 assert_eq!(metadata.sort_orders, base.sort_orders);
5028 assert_eq!(metadata.default_sort_order_id, base.default_sort_order_id);
5029 assert_eq!(result.changes, vec![
5030 TableUpdate::UpgradeFormatVersion {
5031 format_version: FormatVersion::V3
5032 },
5033 TableUpdate::AddSchema { schema },
5034 TableUpdate::AddSpec { spec },
5035 TableUpdate::AddSortOrder { sort_order },
5036 ]);
5037 }
5038
5039 #[test]
5040 fn test_build_replacement_rejects_v1() {
5041 let err = builder_without_changes(FormatVersion::V1)
5042 .build_replacement(
5043 replacement_schema(),
5044 UnboundPartitionSpec::builder().build(),
5045 SortOrder::unsorted_order(),
5046 None,
5047 None,
5048 HashMap::new(),
5049 )
5050 .unwrap_err();
5051
5052 assert_eq!(
5053 (err.kind(), err.message()),
5054 (
5055 ErrorKind::FeatureUnsupported,
5056 "Table replacement is not supported for format version 1"
5057 )
5058 );
5059 }
5060
5061 #[test]
5062 fn test_build_replacement_rejects_v1_with_format_upgrade() {
5063 let err = builder_without_changes(FormatVersion::V1)
5064 .build_replacement(
5065 replacement_schema(),
5066 UnboundPartitionSpec::builder().build(),
5067 SortOrder::unsorted_order(),
5068 None,
5069 Some(FormatVersion::V2),
5070 HashMap::new(),
5071 )
5072 .unwrap_err();
5073
5074 assert_eq!(
5075 (err.kind(), err.message()),
5076 (
5077 ErrorKind::FeatureUnsupported,
5078 "Table replacement is not supported for format version 1"
5079 )
5080 );
5081 }
5082
5083 #[test]
5084 fn test_build_replacement_rejects_reserved_properties() {
5085 let err = builder_without_changes(FormatVersion::V2)
5086 .build_replacement(
5087 replacement_schema(),
5088 UnboundPartitionSpec::builder().build(),
5089 SortOrder::unsorted_order(),
5090 None,
5091 None,
5092 HashMap::from_iter([(
5093 TableProperties::PROPERTY_UUID.to_string(),
5094 "ignored".to_string(),
5095 )]),
5096 )
5097 .unwrap_err();
5098
5099 assert_eq!(
5100 (err.kind(), err.message()),
5101 (
5102 ErrorKind::DataInvalid,
5103 "Table properties should not contain reserved properties, but got: [uuid]"
5104 )
5105 );
5106 }
5107
5108 #[test]
5109 fn test_build_replacement_rejects_a_spec_on_a_missing_column() {
5110 let err = builder_without_changes(FormatVersion::V2)
5111 .build_replacement(
5112 replacement_schema(),
5113 UnboundPartitionSpec::builder()
5114 .add_partition_field(unbound_field(99, "nope", Transform::Identity))
5115 .unwrap()
5116 .build(),
5117 SortOrder::unsorted_order(),
5118 None,
5119 None,
5120 HashMap::new(),
5121 )
5122 .unwrap_err();
5123
5124 assert_eq!(
5125 (err.kind(), err.message()),
5126 (
5127 ErrorKind::DataInvalid,
5128 "Cannot find source column with id 99 for partition column nope in schema."
5129 )
5130 );
5131 }
5132
5133 #[test]
5134 fn test_build_replacement_rejects_a_format_version_downgrade() {
5135 let err = builder_without_changes(FormatVersion::V3)
5136 .build_replacement(
5137 replacement_schema(),
5138 UnboundPartitionSpec::builder().build(),
5139 SortOrder::unsorted_order(),
5140 None,
5141 Some(FormatVersion::V2),
5142 HashMap::new(),
5143 )
5144 .unwrap_err();
5145
5146 assert_eq!(
5147 (err.kind(), err.message()),
5148 (
5149 ErrorKind::DataInvalid,
5150 "Cannot downgrade FormatVersion from v3 to v2"
5151 )
5152 );
5153 }
5154
5155 #[test]
5156 fn test_build_replacement_rejects_a_sort_order_on_a_missing_column() {
5157 let sort_order = SortOrder::builder()
5158 .with_sort_field(SortField {
5159 source_id: 99,
5160 transform: Transform::Identity,
5161 direction: SortDirection::Ascending,
5162 null_order: NullOrder::First,
5163 })
5164 .build_unbound()
5165 .unwrap();
5166
5167 let err = builder_without_changes(FormatVersion::V2)
5168 .build_replacement(
5169 replacement_schema(),
5170 UnboundPartitionSpec::builder().build(),
5171 sort_order,
5172 None,
5173 None,
5174 HashMap::new(),
5175 )
5176 .unwrap_err();
5177
5178 assert_eq!(
5179 (err.kind(), err.message()),
5180 (
5181 ErrorKind::DataInvalid,
5182 "Cannot find source column with id 99 for sort column in schema."
5183 )
5184 );
5185 }
5186
5187 #[test]
5188 fn test_build_replacement_rejects_a_column_named_after_a_partition_field() {
5189 let builder = TableMetadataBuilder::new(
5190 schema(),
5191 UnboundPartitionSpec::builder()
5192 .add_partition_field(unbound_field(1, "x_bucket", Transform::Bucket(8)))
5193 .unwrap()
5194 .build(),
5195 SortOrder::unsorted_order(),
5196 TEST_LOCATION.to_string(),
5197 FormatVersion::V2,
5198 HashMap::new(),
5199 )
5200 .unwrap()
5201 .build()
5202 .unwrap()
5203 .metadata
5204 .into_builder(Some(
5205 "s3://bucket/test/location/metadata/metadata1.json".to_string(),
5206 ));
5207 let replacement = Schema::builder()
5208 .with_fields(vec![
5209 NestedField::required(1, "x", Type::Primitive(PrimitiveType::Long)).into(),
5210 NestedField::required(2, "x_bucket", Type::Primitive(PrimitiveType::Long)).into(),
5211 ])
5212 .build()
5213 .unwrap();
5214
5215 let err = builder
5216 .build_replacement(
5217 replacement,
5218 UnboundPartitionSpec::builder().build(),
5219 SortOrder::unsorted_order(),
5220 None,
5221 None,
5222 HashMap::new(),
5223 )
5224 .unwrap_err();
5225
5226 assert_eq!(
5227 (err.kind(), err.message()),
5228 (
5229 ErrorKind::DataInvalid,
5230 "Cannot set current schema 1: field 'x_bucket' with id 4 conflicts with a \
5231 partition field of that name in spec 0 sourced from field id 1"
5232 )
5233 );
5234 }
5235
5236 #[test]
5237 fn test_build_replacement_keeps_refs_other_than_main() {
5238 let audit = SnapshotReference {
5239 snapshot_id: 1,
5240 retention: SnapshotRetention::branch(None, None, None),
5241 };
5242 let release = SnapshotReference {
5243 snapshot_id: 1,
5244 retention: SnapshotRetention::Tag {
5245 max_ref_age_ms: None,
5246 },
5247 };
5248 let builder = populated_builder()
5249 .set_ref("audit", audit.clone())
5250 .unwrap()
5251 .set_ref("release", release.clone())
5252 .unwrap();
5253
5254 let metadata = builder
5255 .build_replacement(
5256 replacement_schema(),
5257 UnboundPartitionSpec::builder().build(),
5258 SortOrder::unsorted_order(),
5259 None,
5260 None,
5261 HashMap::new(),
5262 )
5263 .unwrap()
5264 .build()
5265 .unwrap()
5266 .metadata;
5267
5268 assert_eq!(
5269 metadata.refs,
5270 HashMap::from_iter([
5271 ("audit".to_string(), audit),
5272 ("release".to_string(), release),
5273 ])
5274 );
5275 assert_eq!(metadata.current_snapshot_id, None);
5276 }
5277
5278 #[test]
5279 fn test_build_replacement_keeps_the_id_of_a_column_whose_type_changes() {
5280 let replacement = Schema::builder()
5281 .with_fields(vec![
5282 NestedField::required(1, "x", Type::Primitive(PrimitiveType::String)).into(),
5283 ])
5284 .build()
5285 .unwrap();
5286
5287 let metadata = builder_without_changes(FormatVersion::V2)
5288 .build_replacement(
5289 replacement,
5290 UnboundPartitionSpec::builder().build(),
5291 SortOrder::unsorted_order(),
5292 None,
5293 None,
5294 HashMap::new(),
5295 )
5296 .unwrap()
5297 .build()
5298 .unwrap()
5299 .metadata;
5300
5301 let x = metadata.current_schema().field_by_name("x").unwrap();
5302 assert_eq!(x.id, 1);
5303 assert_eq!(*x.field_type, Type::Primitive(PrimitiveType::String));
5304 }
5305
5306 #[test]
5307 fn test_build_replacement_keeps_the_existing_compression_codec() {
5308 let codec = HashMap::from_iter([(
5309 TableProperties::PROPERTY_PARQUET_COMPRESSION_CODEC.to_string(),
5310 "gzip".to_string(),
5311 )]);
5312
5313 let metadata = builder_without_changes(FormatVersion::V2)
5314 .set_properties(codec.clone())
5315 .unwrap()
5316 .build_replacement(
5317 replacement_schema(),
5318 UnboundPartitionSpec::builder().build(),
5319 SortOrder::unsorted_order(),
5320 None,
5321 None,
5322 HashMap::new(),
5323 )
5324 .unwrap()
5325 .build()
5326 .unwrap()
5327 .metadata;
5328
5329 assert_eq!(metadata.properties, codec);
5330 }
5331
5332 #[test]
5333 fn test_highest_equivalent_field_ids_is_independent_of_field_order() {
5334 let field = |source_id, field_id, transform| PartitionField {
5335 source_id,
5336 field_id,
5337 name: format!("f{field_id}"),
5338 transform,
5339 };
5340 let fields = [
5341 field(1, 1000, Transform::Identity),
5342 field(2, 1003, Transform::Identity),
5343 field(1, 1006, Transform::Identity),
5344 field(1, 1005, Transform::Bucket(16)),
5345 field(2, 1001, Transform::Identity),
5346 field(1, 1002, Transform::Identity),
5347 field(1, 1004, Transform::Bucket(16)),
5348 ];
5349 let expected = HashMap::from([
5350 ((vec![1], Transform::Identity), 1006),
5351 ((vec![2], Transform::Identity), 1003),
5352 ((vec![1], Transform::Bucket(16)), 1005),
5353 ]);
5354
5355 assert_eq!(
5356 TableMetadataBuilder::highest_equivalent_field_ids(fields.iter()),
5357 expected
5358 );
5359 assert_eq!(
5360 TableMetadataBuilder::highest_equivalent_field_ids(fields.iter().rev()),
5361 expected
5362 );
5363 }
5364
5365 #[test]
5366 fn test_partition_field_id_reuse_picks_the_highest_of_several_candidates() {
5367 let metadata: TableMetadata = serde_json::from_value(serde_json::json!({
5371 "format-version": 2,
5372 "table-uuid": "9c12d441-03fe-4693-9a96-a0705ddf69c1",
5373 "location": "s3://bucket/test/location",
5374 "last-sequence-number": 0,
5375 "last-updated-ms": 1602638573590i64,
5376 "last-column-id": 2,
5377 "current-schema-id": 0,
5378 "schemas": [{
5379 "type": "struct",
5380 "schema-id": 0,
5381 "fields": [
5382 {"id": 1, "name": "x", "required": true, "type": "long"},
5383 {"id": 2, "name": "y", "required": true, "type": "long"},
5384 ],
5385 }],
5386 "default-spec-id": 3,
5387 "last-partition-id": 1006,
5388 "partition-specs": [
5389 {"spec-id": 0, "fields": [
5390 {"name": "x", "transform": "identity", "source-id": 1, "field-id": 1000},
5391 ]},
5392 {"spec-id": 1, "fields": [
5393 {"name": "x", "transform": "identity", "source-id": 1, "field-id": 1002},
5394 ]},
5395 {"spec-id": 2, "fields": [
5396 {"name": "x", "transform": "identity", "source-id": 1, "field-id": 1004},
5397 ]},
5398 {"spec-id": 3, "fields": [
5399 {"name": "x", "transform": "identity", "source-id": 1, "field-id": 1006},
5400 ]},
5401 ],
5402 "default-sort-order-id": 0,
5403 "sort-orders": [{"order-id": 0, "fields": []}],
5404 }))
5405 .unwrap();
5406
5407 let new_spec = UnboundPartitionSpec::builder()
5410 .add_partition_field(
5411 UnboundPartitionField::builder()
5412 .source_ids(vec![1])
5413 .name("x")
5414 .transform(Transform::Identity)
5415 .build()
5416 .unwrap(),
5417 )
5418 .unwrap()
5419 .add_partition_field(
5420 UnboundPartitionField::builder()
5421 .source_ids(vec![2])
5422 .name("y")
5423 .transform(Transform::Identity)
5424 .build()
5425 .unwrap(),
5426 )
5427 .unwrap()
5428 .build();
5429
5430 let result = metadata
5431 .into_builder(Some(
5432 "s3://bucket/test/location/metadata/v1.json".to_string(),
5433 ))
5434 .add_partition_spec(new_spec)
5435 .unwrap()
5436 .build()
5437 .unwrap();
5438
5439 let spec = result.metadata.partition_spec_by_id(4).unwrap();
5440 let fields: Vec<(i32, i32)> = spec
5441 .fields()
5442 .iter()
5443 .map(|f| (f.source_id, f.field_id))
5444 .collect();
5445 assert_eq!(fields, vec![(1, 1006), (2, 1007)]);
5446 assert_eq!(result.metadata.last_partition_id, 1007);
5447 }
5448}