1use std::collections::{HashMap, HashSet};
19use std::sync::Arc;
20
21use async_trait::async_trait;
22use typed_builder::TypedBuilder;
23
24use crate::spec::{
25 LIST_FIELD_NAME, ListType, Literal, MAP_KEY_FIELD_NAME, MAP_VALUE_FIELD_NAME, MapType,
26 MappedField, NameMapping, NestedField, NestedFieldRef, SCHEMA_NAME_DELIMITER, Schema,
27 StructType, TableProperties, Type,
28};
29use crate::table::Table;
30use crate::transaction::action::{ActionCommit, StatelessAction};
31use crate::{Error, ErrorKind, Result, TableRequirement, TableUpdate};
32
33const DEFAULT_FIELD_ID: i32 = 0;
35
36#[derive(Clone, TypedBuilder)]
42pub struct AddColumn {
43 #[builder(default = None, setter(strip_option, into))]
44 parent: Option<String>,
45 #[builder(setter(into))]
46 name: String,
47 #[builder(default = false)]
48 required: bool,
49 field_type: Type,
50 #[builder(default = None, setter(strip_option, into))]
51 doc: Option<String>,
52 #[builder(default = None, setter(strip_option))]
53 initial_default: Option<Literal>,
54 #[builder(default = None, setter(strip_option))]
55 write_default: Option<Literal>,
56}
57
58impl AddColumn {
59 pub fn optional(name: impl ToString, field_type: Type) -> Self {
61 Self::builder()
62 .name(name.to_string())
63 .field_type(field_type)
64 .required(false)
65 .build()
66 }
67
68 pub fn required(name: impl ToString, field_type: Type, initial_default: Literal) -> Self {
70 Self::builder()
71 .name(name.to_string())
72 .field_type(field_type)
73 .required(true)
74 .initial_default(initial_default.clone())
75 .write_default(initial_default)
76 .build()
77 }
78
79 fn to_nested_field(&self) -> NestedFieldRef {
80 let mut field = NestedField::new(
81 DEFAULT_FIELD_ID,
82 self.name.clone(),
83 self.field_type.clone(),
84 self.required,
85 );
86
87 field.doc = self.doc.clone();
88 field.initial_default = self.initial_default.clone();
89 field.write_default = self.write_default.clone();
90 Arc::new(field)
91 }
92}
93
94#[derive(Clone)]
117pub struct UpdateSchemaAction {
118 additions: Vec<AddColumn>,
119 deletes: Vec<String>,
120}
121
122impl UpdateSchemaAction {
123 pub(crate) fn new() -> Self {
125 Self {
126 additions: Vec::new(),
127 deletes: Vec::new(),
128 }
129 }
130
131 pub fn add_column(mut self, add_column: AddColumn) -> Self {
139 self.additions.push(add_column);
140 self
141 }
142
143 pub fn delete_column(mut self, name: impl ToString) -> Self {
149 self.deletes.push(name.to_string());
150 self
151 }
152}
153
154fn assign_fresh_ids(field: &NestedField, next_id: &mut i32) -> NestedFieldRef {
166 *next_id += 1;
167 let new_id = *next_id;
168 let new_type = assign_fresh_ids_to_type(&field.field_type, next_id);
169
170 Arc::new(NestedField {
171 id: new_id,
172 name: field.name.clone(),
173 required: field.required,
174 field_type: Box::new(new_type),
175 doc: field.doc.clone(),
176 initial_default: field.initial_default.clone(),
177 write_default: field.write_default.clone(),
178 })
179}
180
181fn assign_fresh_ids_to_type(field_type: &Type, next_id: &mut i32) -> Type {
183 match field_type {
184 Type::Primitive(_) => field_type.clone(),
185 Type::Variant(v) => Type::Variant(*v),
188 Type::Struct(struct_type) => {
189 let new_fields: Vec<NestedFieldRef> = struct_type
190 .fields()
191 .iter()
192 .map(|f| assign_fresh_ids(f, next_id))
193 .collect();
194 Type::Struct(StructType::new(new_fields))
195 }
196 Type::List(list_type) => {
197 let new_element = assign_fresh_ids(&list_type.element_field, next_id);
198 Type::List(ListType {
199 element_field: new_element,
200 })
201 }
202 Type::Map(map_type) => {
203 let new_key = assign_fresh_ids(&map_type.key_field, next_id);
204 let new_value = assign_fresh_ids(&map_type.value_field, next_id);
205 Type::Map(MapType {
206 key_field: new_key,
207 value_field: new_value,
208 })
209 }
210 }
211}
212
213fn resolve_parent_target<'a>(
223 base_schema: &'a Schema,
224 parent: &str,
225) -> Result<(i32, &'a StructType)> {
226 base_schema
227 .field_by_name(parent)
228 .ok_or_else(|| {
229 Error::new(
230 ErrorKind::PreconditionFailed,
231 format!("Cannot add column: parent '{parent}' not found"),
232 )
233 })
234 .and_then(|parent_field| match parent_field.field_type.as_ref() {
235 Type::Struct(s) => Ok((parent_field.id, s)),
236 Type::Map(m) => match m.value_field.field_type.as_ref() {
237 Type::Struct(s) => Ok((m.value_field.id, s)),
238 _ => Err(Error::new(
239 ErrorKind::PreconditionFailed,
240 format!("Cannot add column: map value of '{parent}' is not a struct"),
241 )),
242 },
243 Type::List(l) => match l.element_field.field_type.as_ref() {
244 Type::Struct(s) => Ok((l.element_field.id, s)),
245 _ => Err(Error::new(
246 ErrorKind::PreconditionFailed,
247 format!("Cannot add column: list element of '{parent}' is not a struct"),
248 )),
249 },
250 _ => Err(Error::new(
251 ErrorKind::PreconditionFailed,
252 format!("Cannot add column: parent '{parent}' is not a struct, map, or list"),
253 )),
254 })
255}
256
257fn rebuild_fields(
264 fields: &[NestedFieldRef],
265 adds: &HashMap<Option<i32>, Vec<NestedFieldRef>>,
266 delete_ids: &HashSet<i32>,
267 parent_id: Option<i32>,
268) -> Vec<NestedFieldRef> {
269 fields
270 .iter()
271 .filter(|f| !delete_ids.contains(&f.id))
272 .map(|f| rebuild_field(f, adds, delete_ids))
273 .chain(adds.get(&parent_id).into_iter().flatten().cloned())
274 .collect()
275}
276
277fn rebuild_field(
281 field: &NestedFieldRef,
282 adds: &HashMap<Option<i32>, Vec<NestedFieldRef>>,
283 delete_ids: &HashSet<i32>,
284) -> NestedFieldRef {
285 match field.field_type.as_ref() {
286 Type::Primitive(_) | Type::Variant(_) => field.clone(),
287 Type::Struct(s) => {
288 let new_fields = rebuild_fields(s.fields(), adds, delete_ids, Some(field.id));
289 Arc::new(NestedField {
290 id: field.id,
291 name: field.name.clone(),
292 required: field.required,
293 field_type: Box::new(Type::Struct(StructType::new(new_fields))),
294 doc: field.doc.clone(),
295 initial_default: field.initial_default.clone(),
296 write_default: field.write_default.clone(),
297 })
298 }
299 Type::List(l) => {
300 let new_element = rebuild_field(&l.element_field, adds, delete_ids);
301 Arc::new(NestedField {
302 id: field.id,
303 name: field.name.clone(),
304 required: field.required,
305 field_type: Box::new(Type::List(ListType {
306 element_field: new_element,
307 })),
308 doc: field.doc.clone(),
309 initial_default: field.initial_default.clone(),
310 write_default: field.write_default.clone(),
311 })
312 }
313 Type::Map(m) => {
314 let new_key = rebuild_field(&m.key_field, adds, delete_ids);
315 let new_value = rebuild_field(&m.value_field, adds, delete_ids);
316 Arc::new(NestedField {
317 id: field.id,
318 name: field.name.clone(),
319 required: field.required,
320 field_type: Box::new(Type::Map(MapType {
321 key_field: new_key,
322 value_field: new_value,
323 })),
324 doc: field.doc.clone(),
325 initial_default: field.initial_default.clone(),
326 write_default: field.write_default.clone(),
327 })
328 }
329 }
330}
331
332fn add_columns_to_name_mapping(
342 mapping: &NameMapping,
343 adds: &HashMap<Option<i32>, Vec<NestedFieldRef>>,
344) -> NameMapping {
345 let root_adds = adds.get(&None).map_or(&[][..], Vec::as_slice);
346 NameMapping::new(add_columns_to_mapped_fields(
347 mapping.fields().iter(),
348 root_adds,
349 adds,
350 ))
351}
352
353fn add_columns_to_mapped_fields<'a>(
354 fields: impl Iterator<Item = &'a MappedField>,
355 added: &[NestedFieldRef],
356 adds: &HashMap<Option<i32>, Vec<NestedFieldRef>>,
357) -> Vec<MappedField> {
358 let added_ids: HashMap<&str, i32> = added
359 .iter()
360 .map(|add| (add.name.as_str(), add.id))
361 .collect();
362 fields
363 .map(|field| {
364 let nested_added = field
365 .field_id()
366 .and_then(|id| adds.get(&Some(id)))
367 .map_or(&[][..], Vec::as_slice);
368 let nested = add_columns_to_mapped_fields(
369 field.fields().iter().map(Arc::as_ref),
370 nested_added,
371 adds,
372 );
373 let names = field
374 .names()
375 .iter()
376 .filter(|name| {
377 added_ids
378 .get(name.as_str())
379 .is_none_or(|id| Some(*id) == field.field_id())
380 })
381 .cloned()
382 .collect();
383 MappedField::new(field.field_id(), names, nested)
384 })
385 .chain(added.iter().map(|add| create_mapped_field(add, &add.name)))
386 .collect()
387}
388
389fn create_mapped_field(field: &NestedField, name: &str) -> MappedField {
391 let nested = match field.field_type.as_ref() {
392 Type::Primitive(_) | Type::Variant(_) => vec![],
393 Type::Struct(s) => s
394 .fields()
395 .iter()
396 .map(|f| create_mapped_field(f, &f.name))
397 .collect(),
398 Type::List(l) => vec![create_mapped_field(&l.element_field, LIST_FIELD_NAME)],
399 Type::Map(m) => vec![
400 create_mapped_field(&m.key_field, MAP_KEY_FIELD_NAME),
401 create_mapped_field(&m.value_field, MAP_VALUE_FIELD_NAME),
402 ],
403 };
404 MappedField::new(Some(field.id), vec![name.to_string()], nested)
405}
406
407#[async_trait]
412impl StatelessAction for UpdateSchemaAction {
413 async fn commit(&self, table: &Table) -> Result<ActionCommit> {
414 let base_schema = table.metadata().current_schema();
415 let mut last_column_id = table.metadata().last_column_id();
416
417 let delete_ids = self
419 .deletes
420 .iter()
421 .map(|name: &String| {
422 base_schema
423 .field_by_name(name)
424 .ok_or_else(|| {
425 Error::new(
426 ErrorKind::PreconditionFailed,
427 format!("Cannot delete missing column: {name}"),
428 )
429 })
430 .and_then(|field| {
431 match base_schema
432 .identifier_field_ids()
433 .find(|id| *id == field.id)
434 {
435 Some(_) => Err(Error::new(
436 ErrorKind::PreconditionFailed,
437 format!("Cannot delete identifier field: {name}"),
438 )),
439 None => Ok(field.id),
440 }
441 })
442 })
443 .collect::<Result<HashSet<i32>>>()?;
444
445 let mut additions_by_parent: HashMap<Option<i32>, Vec<NestedFieldRef>> = HashMap::new();
449
450 for add in &self.additions {
451 let pending_field = add.to_nested_field();
452
453 if pending_field.name.contains(SCHEMA_NAME_DELIMITER) {
455 return Err(Error::new(
456 ErrorKind::PreconditionFailed,
457 format!(
458 "Cannot add column with ambiguous name: {}. Use `AddColumn::with_parent` to add a column to a nested struct.",
459 pending_field.name
460 ),
461 ));
462 }
463
464 if pending_field.required && pending_field.initial_default.is_none() {
466 return Err(Error::new(
467 ErrorKind::PreconditionFailed,
468 format!(
469 "Incompatible change: cannot add required column without an initial default: {}",
470 pending_field.name
471 ),
472 ));
473 }
474
475 let parent_id = match &add.parent {
476 None => {
477 if let Some(existing) = base_schema.field_by_name(&pending_field.name)
479 && !delete_ids.contains(&existing.id)
480 {
481 return Err(Error::new(
482 ErrorKind::PreconditionFailed,
483 format!(
484 "Cannot add column, name already exists: {}",
485 pending_field.name
486 ),
487 ));
488 }
489 None
490 }
491 Some(parent_path) => {
492 let (resolved_parent_id, parent_struct) =
494 resolve_parent_target(base_schema, parent_path)?;
495
496 if parent_struct.fields().iter().any(|f| {
497 f.name == pending_field.name
498 && !delete_ids.contains(&f.id)
499 && !delete_ids.contains(&resolved_parent_id)
500 }) {
501 return Err(Error::new(
502 ErrorKind::PreconditionFailed,
503 format!(
504 "Cannot add column, name already exists in '{}': {}",
505 parent_path, pending_field.name
506 ),
507 ));
508 }
509
510 Some(resolved_parent_id)
511 }
512 };
513
514 let field = assign_fresh_ids(&pending_field, &mut last_column_id);
516
517 additions_by_parent
518 .entry(parent_id)
519 .or_default()
520 .push(field);
521 }
522
523 let new_fields = rebuild_fields(
525 base_schema.as_struct().fields(),
526 &additions_by_parent,
527 &delete_ids,
528 None,
529 );
530
531 let schema = Schema::builder()
533 .with_fields(new_fields)
534 .with_identifier_field_ids(base_schema.identifier_field_ids())
535 .build()?;
536
537 additions_by_parent
540 .retain(|parent_id, _| parent_id.is_none_or(|id| schema.field_by_id(id).is_some()));
541
542 let mut updates = vec![
543 TableUpdate::AddSchema { schema },
544 TableUpdate::SetCurrentSchema { schema_id: -1 },
545 ];
546
547 if !additions_by_parent.is_empty()
549 && let Some(mapping_json) = table
550 .metadata()
551 .properties()
552 .get(TableProperties::PROPERTY_DEFAULT_NAME_MAPPING)
553 {
554 match mapping_json.parse::<NameMapping>().and_then(|mapping| {
556 serde_json::to_string(&add_columns_to_name_mapping(&mapping, &additions_by_parent))
557 .map_err(Error::from)
558 }) {
559 Ok(updated_json) => updates.push(TableUpdate::SetProperties {
560 updates: HashMap::from([(
561 TableProperties::PROPERTY_DEFAULT_NAME_MAPPING.to_string(),
562 updated_json,
563 )]),
564 }),
565 Err(error) => {
566 tracing::warn!("Failed to update name mapping {mapping_json}: {error}")
567 }
568 }
569 }
570
571 let requirements = vec![TableRequirement::CurrentSchemaIdMatch {
572 current_schema_id: base_schema.schema_id(),
573 }];
574
575 Ok(ActionCommit::new(updates, requirements))
576 }
577}
578
579#[cfg(test)]
580mod tests {
581 use std::collections::HashMap;
582 use std::io::BufReader;
583 use std::sync::Arc;
584
585 use as_any::Downcast;
586
587 use crate::spec::{
588 DEFAULT_SCHEMA_ID, ListType, Literal, MapType, MappedField, NameMapping, NestedField,
589 PrimitiveType, StructType, TableMetadata, TableProperties, Type, VariantType,
590 };
591 use crate::table::Table;
592 use crate::test_utils::commit_with_fresh_state;
593 use crate::transaction::Transaction;
594 use crate::transaction::action::{ActionEntry, ApplyTransactionAction};
595 use crate::transaction::tests::make_v2_table;
596 use crate::transaction::update_schema::{AddColumn, DEFAULT_FIELD_ID, UpdateSchemaAction};
597 use crate::{ErrorKind, TableIdent, TableRequirement, TableUpdate};
598
599 fn make_v2_table_with_nested() -> Table {
623 let json = r#"{
624 "format-version": 2,
625 "table-uuid": "9c12d441-03fe-4693-9a96-a0705ddf69c2",
626 "location": "s3://bucket/test/location",
627 "last-sequence-number": 0,
628 "last-updated-ms": 1602638573590,
629 "last-column-id": 14,
630 "current-schema-id": 0,
631 "schemas": [
632 {
633 "type": "struct",
634 "schema-id": 0,
635 "identifier-field-ids": [1, 2],
636 "fields": [
637 {"id": 1, "name": "x", "required": true, "type": "long"},
638 {"id": 2, "name": "y", "required": true, "type": "long"},
639 {"id": 3, "name": "z", "required": true, "type": "long"},
640 {"id": 4, "name": "person", "required": false, "type": {
641 "type": "struct",
642 "fields": [
643 {"id": 5, "name": "name", "required": false, "type": "string"},
644 {"id": 6, "name": "age", "required": true, "type": "int"}
645 ]
646 }},
647 {"id": 7, "name": "tags", "required": false, "type": {
648 "type": "list",
649 "element-id": 8,
650 "element": {
651 "type": "struct",
652 "fields": [
653 {"id": 9, "name": "key", "required": false, "type": "string"},
654 {"id": 10, "name": "value", "required": false, "type": "string"}
655 ]
656 },
657 "element-required": true
658 }},
659 {"id": 11, "name": "props", "required": false, "type": {
660 "type": "map",
661 "key-id": 12,
662 "key": "string",
663 "value-id": 13,
664 "value": {
665 "type": "struct",
666 "fields": [
667 {"id": 14, "name": "data", "required": false, "type": "string"}
668 ]
669 },
670 "value-required": true
671 }}
672 ]
673 }
674 ],
675 "default-spec-id": 0,
676 "partition-specs": [
677 {"spec-id": 0, "fields": []}
678 ],
679 "last-partition-id": 999,
680 "default-sort-order-id": 0,
681 "sort-orders": [
682 {"order-id": 0, "fields": []}
683 ],
684 "properties": {},
685 "current-snapshot-id": -1,
686 "snapshots": []
687 }"#;
688
689 let reader = BufReader::new(json.as_bytes());
690 let metadata = serde_json::from_reader::<_, TableMetadata>(reader).unwrap();
691
692 Table::builder()
693 .metadata(metadata)
694 .metadata_location("s3://bucket/test/location/metadata/v1.json".to_string())
695 .identifier(TableIdent::from_strs(["ns1", "test1"]).unwrap())
696 .file_io(crate::io::FileIO::new_with_memory())
697 .runtime(crate::test_utils::test_runtime())
698 .build()
699 .unwrap()
700 }
701
702 #[test]
707 fn test_assign_fresh_ids_variant() {
708 let mut next_id = 10;
711 let field = NestedField::optional(1, "data", Type::Variant(VariantType));
712 let assigned = super::assign_fresh_ids(&field, &mut next_id);
713
714 assert_eq!(assigned.id, 11);
715 assert_eq!(*assigned.field_type, Type::Variant(VariantType));
716 assert_eq!(next_id, 11);
717 }
718
719 #[tokio::test]
720 async fn test_add_column() {
721 let table = make_v2_table();
722 let tx = Transaction::new(&table);
723
724 let action = tx.update_schema().add_column(AddColumn::optional(
725 "new_col",
726 Type::Primitive(PrimitiveType::Int),
727 ));
728
729 let mut action_commit = commit_with_fresh_state(action, &table).await.unwrap();
730 let updates = action_commit.take_updates();
731 let requirements = action_commit.take_requirements();
732
733 assert_eq!(updates.len(), 2);
734
735 let new_schema = match &updates[0] {
737 TableUpdate::AddSchema { schema } => schema,
738 other => panic!("expected AddSchema, got {other:?}"),
739 };
740
741 let expected_schema = table
742 .metadata()
743 .current_schema()
744 .as_ref()
745 .clone()
746 .into_builder()
747 .with_schema_id(DEFAULT_SCHEMA_ID)
748 .with_fields([
749 NestedField::optional(4, "new_col", Type::Primitive(PrimitiveType::Int)).into(),
750 ])
751 .build()
752 .unwrap();
753 assert_eq!(new_schema, &expected_schema);
754
755 assert_eq!(updates[1], TableUpdate::SetCurrentSchema { schema_id: -1 });
756
757 assert_eq!(requirements.len(), 1);
759 assert_eq!(requirements[0], TableRequirement::CurrentSchemaIdMatch {
760 current_schema_id: table.metadata().current_schema().schema_id()
761 });
762 }
763
764 #[tokio::test]
765 async fn test_add_column_with_doc() {
766 let table = make_v2_table();
767 let tx = Transaction::new(&table);
768
769 let action = tx.update_schema().add_column(
770 AddColumn::builder()
771 .name("documented_col")
772 .field_type(Type::Primitive(PrimitiveType::String))
773 .doc("A documented column")
774 .build(),
775 );
776
777 let mut action_commit = commit_with_fresh_state(action, &table).await.unwrap();
778 let updates = action_commit.take_updates();
779
780 let new_schema = match &updates[0] {
781 TableUpdate::AddSchema { schema } => schema,
782 other => panic!("expected AddSchema, got {other:?}"),
783 };
784
785 let field = new_schema
786 .field_by_name("documented_col")
787 .expect("documented_col should exist");
788 assert_eq!(field.id, 4);
789 assert!(!field.required);
790 assert_eq!(field.doc.as_deref(), Some("A documented column"));
791 }
792
793 #[tokio::test]
794 async fn test_add_required_column_with_initial_default() {
795 let table = make_v2_table();
796 let tx = Transaction::new(&table);
797
798 let action = tx.update_schema().add_column(AddColumn::required(
799 "req_col",
800 Type::Primitive(PrimitiveType::Int),
801 Literal::int(0),
802 ));
803
804 let mut action_commit = commit_with_fresh_state(action, &table).await.unwrap();
805 let updates = action_commit.take_updates();
806
807 let new_schema = match &updates[0] {
808 TableUpdate::AddSchema { schema } => schema,
809 other => panic!("expected AddSchema, got {other:?}"),
810 };
811
812 let field = new_schema
813 .field_by_name("req_col")
814 .expect("req_col should exist");
815 assert_eq!(field.id, 4);
816 assert!(field.required);
817 assert_eq!(field.initial_default, Some(Literal::int(0)));
818 assert_eq!(field.write_default, Some(Literal::int(0)));
819 }
820
821 #[tokio::test]
822 async fn test_add_column_name_conflict_fails() {
823 let table = make_v2_table();
824 let tx = Transaction::new(&table);
825
826 let action = tx.update_schema().add_column(AddColumn::optional(
828 "x",
829 Type::Primitive(PrimitiveType::Int),
830 ));
831
832 let result = commit_with_fresh_state(action, &table).await;
833 let err = match result {
834 Err(e) => e,
835 Ok(_) => panic!("should reject adding a column with an existing name"),
836 };
837 assert_eq!(err.kind(), ErrorKind::PreconditionFailed);
838 assert!(
839 err.message().contains("already exists"),
840 "error should mention name conflict, got: {}",
841 err.message()
842 );
843 }
844
845 #[tokio::test]
846 async fn test_delete_column() {
847 let table = make_v2_table();
848 let tx = Transaction::new(&table);
849
850 let action = tx.update_schema().delete_column("z");
852
853 let mut action_commit = commit_with_fresh_state(action, &table).await.unwrap();
854 let updates = action_commit.take_updates();
855
856 let new_schema = match &updates[0] {
857 TableUpdate::AddSchema { schema } => schema,
858 other => panic!("expected AddSchema, got {other:?}"),
859 };
860
861 assert!(
862 new_schema.field_by_name("z").is_none(),
863 "z should be deleted"
864 );
865 assert!(new_schema.field_by_name("x").is_some());
866 assert!(new_schema.field_by_name("y").is_some());
867 }
868
869 #[tokio::test]
870 async fn test_delete_missing_column_fails() {
871 let table = make_v2_table();
872 let tx = Transaction::new(&table);
873
874 let action = tx.update_schema().delete_column("nonexistent");
875
876 let result = commit_with_fresh_state(action, &table).await;
877 let err = match result {
878 Err(e) => e,
879 Ok(_) => panic!("should reject deleting a non-existent column"),
880 };
881 assert_eq!(err.kind(), ErrorKind::PreconditionFailed);
882 assert!(
883 err.message().contains("nonexistent"),
884 "error should mention the missing column, got: {}",
885 err.message()
886 );
887 }
888
889 #[tokio::test]
890 async fn test_add_and_delete_combined() {
891 let table = make_v2_table();
892 let tx = Transaction::new(&table);
893
894 let action = tx
896 .update_schema()
897 .delete_column("z")
898 .add_column(AddColumn::optional(
899 "w",
900 Type::Primitive(PrimitiveType::Boolean),
901 ));
902
903 let mut action_commit = commit_with_fresh_state(action, &table).await.unwrap();
904 let updates = action_commit.take_updates();
905
906 let new_schema = match &updates[0] {
907 TableUpdate::AddSchema { schema } => schema,
908 other => panic!("expected AddSchema, got {other:?}"),
909 };
910
911 assert!(
912 new_schema.field_by_name("z").is_none(),
913 "z should be deleted"
914 );
915 let w = new_schema.field_by_name("w").expect("w should exist");
916 assert_eq!(w.id, 4);
917 assert!(!w.required);
918 }
919
920 #[tokio::test]
921 async fn test_delete_and_readd_same_name() {
922 let table = make_v2_table();
923 let tx = Transaction::new(&table);
924
925 let action = tx
927 .update_schema()
928 .delete_column("z")
929 .add_column(AddColumn::optional(
930 "z",
931 Type::Primitive(PrimitiveType::Boolean),
932 ));
933
934 let mut action_commit = commit_with_fresh_state(action, &table).await.unwrap();
935 let updates = action_commit.take_updates();
936
937 let new_schema = match &updates[0] {
938 TableUpdate::AddSchema { schema } => schema,
939 other => panic!("expected AddSchema, got {other:?}"),
940 };
941
942 let z = new_schema
943 .field_by_name("z")
944 .expect("z should exist with new type");
945 assert_eq!(z.id, 4); assert_eq!(*z.field_type, Type::Primitive(PrimitiveType::Boolean));
947 }
948
949 #[test]
950 fn test_apply() {
951 let table = make_v2_table();
952 let tx = Transaction::new(&table);
953
954 let tx = tx
955 .update_schema()
956 .add_column(AddColumn::optional(
957 "new_col",
958 Type::Primitive(PrimitiveType::Int),
959 ))
960 .apply(tx)
961 .unwrap();
962
963 assert_eq!(tx.actions.len(), 1);
964 (*tx.actions[0])
965 .downcast_ref::<ActionEntry<UpdateSchemaAction>>()
966 .expect("UpdateSchemaAction was not applied to Transaction!");
967 }
968
969 #[tokio::test]
974 async fn test_add_column_to_struct() {
975 let table = make_v2_table_with_nested();
976 let tx = Transaction::new(&table);
977
978 let action = tx.update_schema().add_column(
980 AddColumn::builder()
981 .name("email")
982 .field_type(Type::Primitive(PrimitiveType::String))
983 .parent("person")
984 .build(),
985 );
986
987 let mut action_commit = commit_with_fresh_state(action, &table).await.unwrap();
988 let updates = action_commit.take_updates();
989
990 let new_schema = match &updates[0] {
991 TableUpdate::AddSchema { schema } => schema,
992 other => panic!("expected AddSchema, got {other:?}"),
993 };
994
995 let email = new_schema
997 .field_by_name("person.email")
998 .expect("person.email should exist");
999 assert_eq!(email.id, 15);
1000 assert!(!email.required);
1001 assert_eq!(*email.field_type, Type::Primitive(PrimitiveType::String));
1002
1003 assert!(new_schema.field_by_name("person.name").is_some());
1005 assert!(new_schema.field_by_name("person.age").is_some());
1006 }
1007
1008 #[tokio::test]
1009 async fn test_add_column_to_struct_with_doc() {
1010 let table = make_v2_table_with_nested();
1011 let tx = Transaction::new(&table);
1012
1013 let action = tx.update_schema().add_column(
1014 AddColumn::builder()
1015 .name("phone")
1016 .field_type(Type::Primitive(PrimitiveType::String))
1017 .parent("person")
1018 .doc("Phone number")
1019 .build(),
1020 );
1021
1022 let mut action_commit = commit_with_fresh_state(action, &table).await.unwrap();
1023 let updates = action_commit.take_updates();
1024
1025 let new_schema = match &updates[0] {
1026 TableUpdate::AddSchema { schema } => schema,
1027 other => panic!("expected AddSchema, got {other:?}"),
1028 };
1029
1030 let phone = new_schema
1031 .field_by_name("person.phone")
1032 .expect("person.phone should exist");
1033 assert_eq!(phone.id, 15);
1034 assert_eq!(phone.doc.as_deref(), Some("Phone number"));
1035 }
1036
1037 #[tokio::test]
1038 async fn test_add_column_to_list_element_struct() {
1039 let table = make_v2_table_with_nested();
1040 let tx = Transaction::new(&table);
1041
1042 let action = tx.update_schema().add_column(
1045 AddColumn::builder()
1046 .name("score")
1047 .field_type(Type::Primitive(PrimitiveType::Double))
1048 .parent("tags")
1049 .build(),
1050 );
1051
1052 let mut action_commit = commit_with_fresh_state(action, &table).await.unwrap();
1053 let updates = action_commit.take_updates();
1054
1055 let new_schema = match &updates[0] {
1056 TableUpdate::AddSchema { schema } => schema,
1057 other => panic!("expected AddSchema, got {other:?}"),
1058 };
1059
1060 let score = new_schema
1062 .field_by_name("tags.element.score")
1063 .expect("tags.element.score should exist");
1064 assert_eq!(score.id, 15);
1065 assert!(!score.required);
1066
1067 assert!(new_schema.field_by_name("tags.element.key").is_some());
1069 assert!(new_schema.field_by_name("tags.element.value").is_some());
1070 }
1071
1072 #[tokio::test]
1073 async fn test_add_column_to_map_value_struct() {
1074 let table = make_v2_table_with_nested();
1075 let tx = Transaction::new(&table);
1076
1077 let action = tx.update_schema().add_column(
1080 AddColumn::builder()
1081 .name("version")
1082 .field_type(Type::Primitive(PrimitiveType::Int))
1083 .parent("props")
1084 .build(),
1085 );
1086
1087 let mut action_commit = commit_with_fresh_state(action, &table).await.unwrap();
1088 let updates = action_commit.take_updates();
1089
1090 let new_schema = match &updates[0] {
1091 TableUpdate::AddSchema { schema } => schema,
1092 other => panic!("expected AddSchema, got {other:?}"),
1093 };
1094
1095 let version = new_schema
1096 .field_by_name("props.value.version")
1097 .expect("props.value.version should exist");
1098 assert_eq!(version.id, 15);
1099
1100 assert!(new_schema.field_by_name("props.value.data").is_some());
1102 }
1103
1104 #[tokio::test]
1105 async fn test_add_column_to_nonexistent_parent_fails() {
1106 let table = make_v2_table_with_nested();
1107 let tx = Transaction::new(&table);
1108
1109 let action = tx.update_schema().add_column(
1110 AddColumn::builder()
1111 .name("col")
1112 .field_type(Type::Primitive(PrimitiveType::Int))
1113 .parent("nonexistent")
1114 .build(),
1115 );
1116
1117 let err = match commit_with_fresh_state(action, &table).await {
1118 Err(e) => e,
1119 Ok(_) => panic!("should reject adding to a nonexistent parent"),
1120 };
1121 assert_eq!(err.kind(), ErrorKind::PreconditionFailed);
1122 assert!(
1123 err.message().contains("nonexistent"),
1124 "error should mention the missing parent, got: {}",
1125 err.message()
1126 );
1127 }
1128
1129 #[tokio::test]
1130 async fn test_add_column_to_primitive_parent_fails() {
1131 let table = make_v2_table_with_nested();
1132 let tx = Transaction::new(&table);
1133
1134 let action = tx.update_schema().add_column(
1136 AddColumn::builder()
1137 .name("col")
1138 .field_type(Type::Primitive(PrimitiveType::Int))
1139 .parent("x")
1140 .build(),
1141 );
1142
1143 let err = match commit_with_fresh_state(action, &table).await {
1144 Err(e) => e,
1145 Ok(_) => panic!("should reject adding to a primitive parent"),
1146 };
1147 assert_eq!(err.kind(), ErrorKind::PreconditionFailed);
1148 assert!(
1149 err.message().contains("not a struct"),
1150 "error should mention type mismatch, got: {}",
1151 err.message()
1152 );
1153 }
1154
1155 #[tokio::test]
1156 async fn test_add_column_to_nested_name_conflict_fails() {
1157 let table = make_v2_table_with_nested();
1158 let tx = Transaction::new(&table);
1159
1160 let action = tx.update_schema().add_column(
1162 AddColumn::builder()
1163 .name("name")
1164 .field_type(Type::Primitive(PrimitiveType::String))
1165 .parent("person")
1166 .build(),
1167 );
1168
1169 let err = match commit_with_fresh_state(action, &table).await {
1170 Err(e) => e,
1171 Ok(_) => panic!("should reject adding a column with conflicting name"),
1172 };
1173 assert_eq!(err.kind(), ErrorKind::PreconditionFailed);
1174 assert!(
1175 err.message().contains("already exists"),
1176 "error should mention name conflict, got: {}",
1177 err.message()
1178 );
1179 }
1180
1181 #[tokio::test]
1182 async fn test_root_and_nested_add_combined() {
1183 let table = make_v2_table_with_nested();
1184 let tx = Transaction::new(&table);
1185
1186 let action = tx
1188 .update_schema()
1189 .add_column(AddColumn::optional(
1190 "root_col",
1191 Type::Primitive(PrimitiveType::Boolean),
1192 ))
1193 .add_column(
1194 AddColumn::builder()
1195 .name("email")
1196 .field_type(Type::Primitive(PrimitiveType::String))
1197 .parent("person")
1198 .build(),
1199 );
1200
1201 let mut action_commit = commit_with_fresh_state(action, &table).await.unwrap();
1202 let updates = action_commit.take_updates();
1203
1204 let new_schema = match &updates[0] {
1205 TableUpdate::AddSchema { schema } => schema,
1206 other => panic!("expected AddSchema, got {other:?}"),
1207 };
1208
1209 let root_col = new_schema
1211 .field_by_name("root_col")
1212 .expect("root_col should exist");
1213 assert_eq!(root_col.id, 15);
1214
1215 let email = new_schema
1217 .field_by_name("person.email")
1218 .expect("person.email should exist");
1219 assert_eq!(email.id, 16);
1220 }
1221
1222 #[tokio::test]
1223 async fn test_add_nested_struct_type_with_fresh_ids() {
1224 let table = make_v2_table();
1227 let tx = Transaction::new(&table);
1228
1229 let action = tx.update_schema().add_column(AddColumn::optional(
1230 "address",
1231 Type::Struct(StructType::new(vec![
1232 NestedField::optional(
1233 DEFAULT_FIELD_ID,
1234 "street",
1235 Type::Primitive(PrimitiveType::String),
1236 )
1237 .into(),
1238 NestedField::optional(
1239 DEFAULT_FIELD_ID,
1240 "city",
1241 Type::Primitive(PrimitiveType::String),
1242 )
1243 .into(),
1244 ])),
1245 ));
1246
1247 let mut action_commit = commit_with_fresh_state(action, &table).await.unwrap();
1248 let updates = action_commit.take_updates();
1249
1250 let new_schema = match &updates[0] {
1251 TableUpdate::AddSchema { schema } => schema,
1252 other => panic!("expected AddSchema, got {other:?}"),
1253 };
1254
1255 let address = new_schema
1257 .field_by_name("address")
1258 .expect("address should exist");
1259 assert_eq!(address.id, 4);
1260
1261 let street = new_schema
1263 .field_by_name("address.street")
1264 .expect("address.street should exist");
1265 assert_eq!(street.id, 5);
1266
1267 let city = new_schema
1268 .field_by_name("address.city")
1269 .expect("address.city should exist");
1270 assert_eq!(city.id, 6);
1271 }
1272
1273 fn with_name_mapping_property(table: Table, mapping_json: String) -> Table {
1278 let metadata = table
1279 .metadata()
1280 .clone()
1281 .into_builder(None)
1282 .set_properties(HashMap::from([(
1283 TableProperties::PROPERTY_DEFAULT_NAME_MAPPING.to_string(),
1284 mapping_json,
1285 )]))
1286 .unwrap()
1287 .build()
1288 .unwrap()
1289 .metadata;
1290 table.with_metadata(Arc::new(metadata))
1291 }
1292
1293 fn mapped(id: i32, name: &str, fields: Vec<MappedField>) -> MappedField {
1294 MappedField::new(Some(id), vec![name.to_string()], fields)
1295 }
1296
1297 fn name_mapping_property_update(mapping: &NameMapping) -> TableUpdate {
1298 TableUpdate::SetProperties {
1299 updates: HashMap::from([(
1300 TableProperties::PROPERTY_DEFAULT_NAME_MAPPING.to_string(),
1301 serde_json::to_string(mapping).unwrap(),
1302 )]),
1303 }
1304 }
1305
1306 fn v2_table_name_mapping() -> NameMapping {
1307 NameMapping::new(vec![
1308 mapped(1, "x", vec![]),
1309 mapped(2, "y", vec![]),
1310 mapped(3, "z", vec![]),
1311 ])
1312 }
1313
1314 fn nested_table_name_mapping() -> NameMapping {
1315 NameMapping::new(vec![
1316 mapped(1, "x", vec![]),
1317 mapped(2, "y", vec![]),
1318 mapped(3, "z", vec![]),
1319 mapped(4, "person", vec![
1320 mapped(5, "name", vec![]),
1321 mapped(6, "age", vec![]),
1322 ]),
1323 mapped(7, "tags", vec![mapped(8, "element", vec![
1324 mapped(9, "key", vec![]),
1325 mapped(10, "value", vec![]),
1326 ])]),
1327 mapped(11, "props", vec![
1328 mapped(12, "key", vec![]),
1329 mapped(13, "value", vec![mapped(14, "data", vec![])]),
1330 ]),
1331 ])
1332 }
1333
1334 #[tokio::test]
1335 async fn test_add_root_column_adds_name_mapping_entry() {
1336 let table = with_name_mapping_property(
1337 make_v2_table(),
1338 serde_json::to_string(&v2_table_name_mapping()).unwrap(),
1339 );
1340 let action = Transaction::new(&table)
1341 .update_schema()
1342 .add_column(AddColumn::optional(
1343 "new_col",
1344 Type::Primitive(PrimitiveType::Int),
1345 ));
1346
1347 let updates = commit_with_fresh_state(action, &table)
1348 .await
1349 .unwrap()
1350 .take_updates();
1351
1352 let expected = NameMapping::new(vec![
1353 mapped(1, "x", vec![]),
1354 mapped(2, "y", vec![]),
1355 mapped(3, "z", vec![]),
1356 mapped(4, "new_col", vec![]),
1357 ]);
1358 assert_eq!(updates[2..], [name_mapping_property_update(&expected)]);
1359 }
1360
1361 #[tokio::test]
1362 async fn test_add_nested_columns_adds_name_mapping_entries_under_parents() {
1363 let mapping = nested_table_name_mapping();
1364 let table = with_name_mapping_property(
1365 make_v2_table_with_nested(),
1366 serde_json::to_string(&mapping).unwrap(),
1367 );
1368 let action = Transaction::new(&table)
1369 .update_schema()
1370 .add_column(
1371 AddColumn::builder()
1372 .name("email")
1373 .field_type(Type::Primitive(PrimitiveType::String))
1374 .parent("person")
1375 .build(),
1376 )
1377 .add_column(
1378 AddColumn::builder()
1379 .name("version")
1380 .field_type(Type::Primitive(PrimitiveType::Int))
1381 .parent("props")
1382 .build(),
1383 )
1384 .add_column(AddColumn::optional(
1385 "attrs",
1386 Type::Map(MapType::new(
1387 NestedField::map_key_element(
1388 DEFAULT_FIELD_ID,
1389 Type::Primitive(PrimitiveType::String),
1390 )
1391 .into(),
1392 NestedField::map_value_element(
1393 DEFAULT_FIELD_ID,
1394 Type::Primitive(PrimitiveType::Long),
1395 false,
1396 )
1397 .into(),
1398 )),
1399 ))
1400 .add_column(AddColumn::optional(
1401 "points",
1402 Type::List(ListType::new(
1403 NestedField::list_element(
1404 DEFAULT_FIELD_ID,
1405 Type::Struct(StructType::new(vec![
1406 NestedField::optional(
1407 DEFAULT_FIELD_ID,
1408 "px",
1409 Type::Primitive(PrimitiveType::Double),
1410 )
1411 .into(),
1412 ])),
1413 true,
1414 )
1415 .into(),
1416 )),
1417 ));
1418
1419 let updates = commit_with_fresh_state(action, &table)
1420 .await
1421 .unwrap()
1422 .take_updates();
1423
1424 let expected = NameMapping::new(vec![
1425 mapped(1, "x", vec![]),
1426 mapped(2, "y", vec![]),
1427 mapped(3, "z", vec![]),
1428 mapped(4, "person", vec![
1429 mapped(5, "name", vec![]),
1430 mapped(6, "age", vec![]),
1431 mapped(15, "email", vec![]),
1432 ]),
1433 mapped(7, "tags", vec![mapped(8, "element", vec![
1434 mapped(9, "key", vec![]),
1435 mapped(10, "value", vec![]),
1436 ])]),
1437 mapped(11, "props", vec![
1438 mapped(12, "key", vec![]),
1439 mapped(13, "value", vec![
1440 mapped(14, "data", vec![]),
1441 mapped(16, "version", vec![]),
1442 ]),
1443 ]),
1444 mapped(17, "attrs", vec![
1445 mapped(18, "key", vec![]),
1446 mapped(19, "value", vec![]),
1447 ]),
1448 mapped(20, "points", vec![mapped(21, "element", vec![mapped(
1449 22,
1450 "px",
1451 vec![],
1452 )])]),
1453 ]);
1454 assert_eq!(updates[2..], [name_mapping_property_update(&expected)]);
1455 }
1456
1457 #[tokio::test]
1458 async fn test_add_column_under_deleted_parent_adds_no_name_mapping_entry() {
1459 let table = with_name_mapping_property(
1460 make_v2_table_with_nested(),
1461 serde_json::to_string(&nested_table_name_mapping()).unwrap(),
1462 );
1463 let action = Transaction::new(&table)
1464 .update_schema()
1465 .delete_column("person")
1466 .add_column(
1467 AddColumn::builder()
1468 .name("name")
1469 .field_type(Type::Primitive(PrimitiveType::String))
1470 .parent("person")
1471 .build(),
1472 )
1473 .add_column(AddColumn::optional(
1474 "new_col",
1475 Type::Primitive(PrimitiveType::Int),
1476 ));
1477
1478 let updates = commit_with_fresh_state(action, &table)
1479 .await
1480 .unwrap()
1481 .take_updates();
1482
1483 let mut expected_fields = nested_table_name_mapping().fields().to_vec();
1484 expected_fields.push(mapped(16, "new_col", vec![]));
1485 assert_eq!(updates[2..], [name_mapping_property_update(
1486 &NameMapping::new(expected_fields)
1487 )]);
1488 }
1489
1490 #[tokio::test]
1491 async fn test_readd_deleted_column_moves_name_to_new_name_mapping_entry() {
1492 let table = with_name_mapping_property(
1493 make_v2_table(),
1494 serde_json::to_string(&v2_table_name_mapping()).unwrap(),
1495 );
1496 let action = Transaction::new(&table)
1497 .update_schema()
1498 .delete_column("z")
1499 .add_column(AddColumn::optional(
1500 "z",
1501 Type::Primitive(PrimitiveType::Boolean),
1502 ));
1503
1504 let updates = commit_with_fresh_state(action, &table)
1505 .await
1506 .unwrap()
1507 .take_updates();
1508
1509 let expected = NameMapping::new(vec![
1510 mapped(1, "x", vec![]),
1511 mapped(2, "y", vec![]),
1512 MappedField::new(Some(3), vec![], vec![]),
1513 mapped(4, "z", vec![]),
1514 ]);
1515 assert_eq!(updates[2..], [name_mapping_property_update(&expected)]);
1516 }
1517
1518 #[tokio::test]
1519 async fn test_delete_column_leaves_name_mapping_unchanged() {
1520 let table = with_name_mapping_property(
1521 make_v2_table(),
1522 serde_json::to_string(&v2_table_name_mapping()).unwrap(),
1523 );
1524 let action = Transaction::new(&table).update_schema().delete_column("z");
1525
1526 let updates = commit_with_fresh_state(action, &table)
1527 .await
1528 .unwrap()
1529 .take_updates();
1530
1531 assert_eq!(updates.len(), 2);
1532 }
1533
1534 #[tokio::test]
1535 async fn test_add_column_with_invalid_name_mapping_leaves_it_unchanged() {
1536 let table = with_name_mapping_property(make_v2_table(), "not json".to_string());
1537 let action = Transaction::new(&table)
1538 .update_schema()
1539 .add_column(AddColumn::optional(
1540 "new_col",
1541 Type::Primitive(PrimitiveType::Int),
1542 ));
1543
1544 let updates = commit_with_fresh_state(action, &table)
1545 .await
1546 .unwrap()
1547 .take_updates();
1548
1549 assert_eq!(updates.len(), 2);
1550 }
1551}