1use std::collections::HashMap;
21use std::str::FromStr;
22use std::sync::Arc;
23
24use async_trait::async_trait;
25use futures::lock::{Mutex, MutexGuard};
26use iceberg_property_macro::Properties;
27use itertools::Itertools;
28
29use super::namespace_state::NamespaceState;
30use crate::encryption::kms::{KeyManagementClient, KmsClientFactory};
31use crate::error::invalid_data;
32use crate::io::{FileIO, FileIOBuilder, MemoryStorageFactory, StorageFactory};
33use crate::runtime::Runtime;
34use crate::spec::{TableMetadata, TableMetadataBuilder};
35use crate::table::Table;
36use crate::{
37 Catalog, CatalogBuilder, MetadataLocation, Namespace, NamespaceIdent, Result, TableCommit,
38 TableCreation, TableIdent, delete_removed_metadata_files,
39};
40
41pub const MEMORY_CATALOG_WAREHOUSE: &str = "warehouse";
43
44const LOCATION: &str = "location";
46
47#[derive(Debug, Default)]
49pub struct MemoryCatalogBuilder {
50 storage_factory: Option<Arc<dyn StorageFactory>>,
51 kms_client_factory: Option<Arc<dyn KmsClientFactory>>,
52 runtime: Option<Runtime>,
53}
54
55impl CatalogBuilder for MemoryCatalogBuilder {
56 type C = MemoryCatalog;
57
58 fn with_storage_factory(mut self, storage_factory: Arc<dyn StorageFactory>) -> Self {
59 self.storage_factory = Some(storage_factory);
60 self
61 }
62
63 fn with_kms_client_factory(mut self, kms_client_factory: Arc<dyn KmsClientFactory>) -> Self {
64 self.kms_client_factory = Some(kms_client_factory);
65 self
66 }
67
68 fn with_runtime(mut self, runtime: Runtime) -> Self {
69 self.runtime = Some(runtime);
70 self
71 }
72
73 fn load(
74 self,
75 name: impl Into<String>,
76 props: HashMap<String, String>,
77 ) -> impl Future<Output = Result<Self::C>> + Send {
78 let name = name.into();
79
80 async move {
81 let catalog_properties = MemoryCatalogProperties::from_properties(&props)?;
82 if catalog_properties.warehouse.is_empty() {
83 return Err(invalid_data!("Catalog warehouse is required"));
84 }
85
86 let runtime = self.runtime.unwrap_or_else(Runtime::current);
87 let kms_client = match self.kms_client_factory {
88 Some(factory) => Some(factory.create_kms_client(&props).await?),
89 None => None,
90 };
91 MemoryCatalog::new(
92 name,
93 catalog_properties,
94 props,
95 self.storage_factory,
96 runtime,
97 kms_client,
98 )
99 }
100 }
101}
102
103#[derive(Debug, Properties)]
105pub(crate) struct MemoryCatalogProperties {
106 #[property(key = MEMORY_CATALOG_WAREHOUSE, default = "")]
107 warehouse: String,
108}
109
110pub struct MemoryCatalog {
112 name: String,
113 properties: MemoryCatalogProperties,
114 root_namespace_state: Mutex<NamespaceState>,
115 file_io: FileIO,
116 runtime: Runtime,
117 kms_client: Option<Arc<dyn KeyManagementClient>>,
118}
119
120impl std::fmt::Debug for MemoryCatalog {
121 fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
122 f.debug_struct("MemoryCatalog")
123 .field("name", &self.name)
124 .field("properties", &self.properties)
125 .finish_non_exhaustive()
126 }
127}
128
129impl MemoryCatalog {
130 fn new(
132 name: String,
133 properties: MemoryCatalogProperties,
134 props: HashMap<String, String>,
135 storage_factory: Option<Arc<dyn StorageFactory>>,
136 runtime: Runtime,
137 kms_client: Option<Arc<dyn KeyManagementClient>>,
138 ) -> Result<Self> {
139 let factory = storage_factory.unwrap_or_else(|| Arc::new(MemoryStorageFactory));
141
142 Ok(Self {
143 name,
144 properties,
145 file_io: FileIOBuilder::new(factory).with_props(props).build(),
146 root_namespace_state: Mutex::new(NamespaceState::default()),
147 runtime,
148 kms_client,
149 })
150 }
151
152 async fn load_table_from_locked_state(
154 &self,
155 table_ident: &TableIdent,
156 root_namespace_state: &MutexGuard<'_, NamespaceState>,
157 ) -> Result<Table> {
158 let metadata_location = root_namespace_state.get_existing_table_location(table_ident)?;
159 let metadata = TableMetadata::read_from(&self.file_io, metadata_location).await?;
160
161 let mut builder = Table::builder()
162 .identifier(table_ident.clone())
163 .metadata(metadata)
164 .metadata_location(metadata_location.to_string())
165 .file_io(self.file_io.clone())
166 .runtime(self.runtime.clone());
167 if let Some(kms_client) = self.kms_client.clone() {
168 builder = builder.kms_client(kms_client);
169 }
170 builder.build()
171 }
172}
173
174#[async_trait]
175impl Catalog for MemoryCatalog {
176 async fn list_namespaces(
178 &self,
179 maybe_parent: Option<&NamespaceIdent>,
180 ) -> Result<Vec<NamespaceIdent>> {
181 let root_namespace_state = self.root_namespace_state.lock().await;
182
183 match maybe_parent {
184 None => {
185 let namespaces = root_namespace_state
186 .list_top_level_namespaces()
187 .into_iter()
188 .map(|str| NamespaceIdent::new(str.to_string()))
189 .collect_vec();
190
191 Ok(namespaces)
192 }
193 Some(parent_namespace_ident) => {
194 let namespaces = root_namespace_state
195 .list_namespaces_under(parent_namespace_ident)?
196 .into_iter()
197 .map(|name| {
198 let mut names = parent_namespace_ident.iter().cloned().collect::<Vec<_>>();
199 names.push(name.to_string());
200 NamespaceIdent::from_vec(names)
201 })
202 .collect::<Result<Vec<_>>>()?;
203
204 Ok(namespaces)
205 }
206 }
207 }
208
209 async fn create_namespace(
211 &self,
212 namespace_ident: &NamespaceIdent,
213 properties: HashMap<String, String>,
214 ) -> Result<Namespace> {
215 let mut root_namespace_state = self.root_namespace_state.lock().await;
216
217 root_namespace_state.insert_new_namespace(namespace_ident, properties.clone())?;
218 let namespace = Namespace::with_properties(namespace_ident.clone(), properties);
219
220 Ok(namespace)
221 }
222
223 async fn get_namespace(&self, namespace_ident: &NamespaceIdent) -> Result<Namespace> {
225 let root_namespace_state = self.root_namespace_state.lock().await;
226
227 let namespace = Namespace::with_properties(
228 namespace_ident.clone(),
229 root_namespace_state
230 .get_properties(namespace_ident)?
231 .clone(),
232 );
233
234 Ok(namespace)
235 }
236
237 async fn namespace_exists(&self, namespace_ident: &NamespaceIdent) -> Result<bool> {
239 let guarded_namespaces = self.root_namespace_state.lock().await;
240
241 Ok(guarded_namespaces.namespace_exists(namespace_ident))
242 }
243
244 async fn update_namespace(
250 &self,
251 namespace_ident: &NamespaceIdent,
252 properties: HashMap<String, String>,
253 ) -> Result<()> {
254 let mut root_namespace_state = self.root_namespace_state.lock().await;
255
256 root_namespace_state.replace_properties(namespace_ident, properties)
257 }
258
259 async fn drop_namespace(&self, namespace_ident: &NamespaceIdent) -> Result<()> {
261 let mut root_namespace_state = self.root_namespace_state.lock().await;
262
263 root_namespace_state.remove_existing_namespace(namespace_ident)
264 }
265
266 async fn list_tables(&self, namespace_ident: &NamespaceIdent) -> Result<Vec<TableIdent>> {
268 let root_namespace_state = self.root_namespace_state.lock().await;
269
270 let table_names = root_namespace_state.list_tables(namespace_ident)?;
271 let table_idents = table_names
272 .into_iter()
273 .map(|table_name| TableIdent::new(namespace_ident.clone(), table_name.clone()))
274 .collect_vec();
275
276 Ok(table_idents)
277 }
278
279 async fn create_table(
281 &self,
282 namespace_ident: &NamespaceIdent,
283 table_creation: TableCreation,
284 ) -> Result<Table> {
285 let mut root_namespace_state = self.root_namespace_state.lock().await;
286
287 let table_name = table_creation.name.clone();
288 let table_ident = TableIdent::new(namespace_ident.clone(), table_name);
289
290 let table_creation = if table_creation.location.is_some() {
291 table_creation
292 } else {
293 let namespace_properties = root_namespace_state.get_properties(namespace_ident)?;
294 let location_prefix = match namespace_properties.get(LOCATION) {
295 Some(namespace_location) => namespace_location.clone(),
296 None => format!(
297 "{}/{}",
298 self.properties.warehouse,
299 namespace_ident.join("/")
300 ),
301 };
302
303 let location = format!("{}/{}", location_prefix, table_ident.name());
304
305 TableCreation {
306 location: Some(location),
307 ..table_creation
308 }
309 };
310
311 let metadata = TableMetadataBuilder::from_table_creation(table_creation)?
312 .build()?
313 .metadata;
314 let metadata_location = MetadataLocation::try_new_with_metadata(&metadata)?;
315
316 metadata.write_to(&self.file_io, &metadata_location).await?;
317
318 root_namespace_state.insert_new_table(&table_ident, metadata_location.to_string())?;
319
320 let mut builder = Table::builder()
321 .file_io(self.file_io.clone())
322 .metadata_location(metadata_location.to_string())
323 .metadata(metadata)
324 .identifier(table_ident)
325 .runtime(self.runtime.clone());
326 if let Some(kms_client) = self.kms_client.clone() {
327 builder = builder.kms_client(kms_client);
328 }
329 builder.build()
330 }
331
332 async fn load_table(&self, table_ident: &TableIdent) -> Result<Table> {
334 let root_namespace_state = self.root_namespace_state.lock().await;
335
336 self.load_table_from_locked_state(table_ident, &root_namespace_state)
337 .await
338 }
339
340 async fn drop_table(&self, table_ident: &TableIdent) -> Result<()> {
342 let mut root_namespace_state = self.root_namespace_state.lock().await;
343
344 root_namespace_state.remove_existing_table(table_ident)?;
345 Ok(())
346 }
347
348 async fn purge_table(&self, table_ident: &TableIdent) -> Result<()> {
349 let table_info = self.load_table(table_ident).await?;
350 self.drop_table(table_ident).await?;
351 crate::catalog::utils::drop_table_data(&table_info).await
352 }
353
354 async fn table_exists(&self, table_ident: &TableIdent) -> Result<bool> {
356 let root_namespace_state = self.root_namespace_state.lock().await;
357
358 root_namespace_state.table_exists(table_ident)
359 }
360
361 async fn rename_table(
363 &self,
364 src_table_ident: &TableIdent,
365 dst_table_ident: &TableIdent,
366 ) -> Result<()> {
367 let mut root_namespace_state = self.root_namespace_state.lock().await;
368
369 let mut new_root_namespace_state = root_namespace_state.clone();
370 let metadata_location = new_root_namespace_state
371 .get_existing_table_location(src_table_ident)?
372 .clone();
373 new_root_namespace_state.remove_existing_table(src_table_ident)?;
374 new_root_namespace_state.insert_new_table(dst_table_ident, metadata_location)?;
375 *root_namespace_state = new_root_namespace_state;
376
377 Ok(())
378 }
379
380 async fn register_table(
381 &self,
382 table_ident: &TableIdent,
383 metadata_location: String,
384 ) -> Result<Table> {
385 let mut root_namespace_state = self.root_namespace_state.lock().await;
386 root_namespace_state.insert_new_table(&table_ident.clone(), metadata_location.clone())?;
387
388 let metadata = TableMetadata::read_from(&self.file_io, &metadata_location).await?;
389
390 let mut builder = Table::builder()
391 .file_io(self.file_io.clone())
392 .metadata_location(metadata_location)
393 .metadata(metadata)
394 .identifier(table_ident.clone())
395 .runtime(self.runtime.clone());
396 if let Some(kms_client) = self.kms_client.clone() {
397 builder = builder.kms_client(kms_client);
398 }
399 builder.build()
400 }
401
402 async fn update_table(&self, commit: TableCommit) -> Result<Table> {
404 let mut root_namespace_state = self.root_namespace_state.lock().await;
405
406 let current_table = self
407 .load_table_from_locked_state(commit.identifier(), &root_namespace_state)
408 .await?;
409 let base_metadata = current_table.metadata_ref();
410
411 let staged_table = commit.apply(current_table)?;
413
414 let metadata_location =
416 MetadataLocation::from_str(staged_table.metadata_location_result()?)?;
417 staged_table
418 .metadata()
419 .write_to(staged_table.file_io(), &metadata_location)
420 .await?;
421
422 let updated_table = root_namespace_state.commit_table_update(staged_table)?;
424 drop(root_namespace_state);
425
426 delete_removed_metadata_files(
427 updated_table.file_io(),
428 &base_metadata,
429 updated_table.metadata(),
430 )
431 .await;
432
433 Ok(updated_table)
434 }
435}
436
437#[cfg(test)]
438pub(crate) mod tests {
439 use std::collections::HashSet;
440 use std::hash::Hash;
441 use std::iter::FromIterator;
442 use std::vec;
443
444 use regex::Regex;
445 use tempfile::TempDir;
446
447 use super::*;
448 use crate::ErrorKind;
449 use crate::encryption::kms::MemoryKmsClientFactory;
450 use crate::io::{FileIO, LocalFsStorageFactory};
451 use crate::spec::{
452 NestedField, PartitionSpec, PrimitiveType, Schema, SortOrder, TableProperties, Type,
453 };
454 use crate::test_utils::test_runtime;
455 use crate::transaction::{ApplyTransactionAction, Transaction};
456
457 fn temp_path() -> String {
458 let temp_dir = TempDir::new().unwrap();
459 temp_dir.path().to_str().unwrap().to_string()
460 }
461
462 #[test]
463 fn test_catalog_properties() {
464 let properties = MemoryCatalogProperties::from_properties(&HashMap::from([(
465 MEMORY_CATALOG_WAREHOUSE.to_string(),
466 "memory:///warehouse".to_string(),
467 )]))
468 .unwrap();
469
470 assert_eq!(properties.warehouse, "memory:///warehouse");
471 }
472
473 #[test]
474 fn test_catalog_properties_warehouse_defaults_to_empty() {
475 let missing = MemoryCatalogProperties::from_properties(&HashMap::new()).unwrap();
476 let explicitly_empty = MemoryCatalogProperties::from_properties(&HashMap::from([(
477 MEMORY_CATALOG_WAREHOUSE.to_string(),
478 String::new(),
479 )]))
480 .unwrap();
481
482 assert_eq!(missing.warehouse, "");
483 assert_eq!(explicitly_empty.warehouse, "");
484 }
485
486 #[tokio::test]
487 async fn test_catalog_forwards_properties_to_file_io() {
488 let catalog = MemoryCatalogBuilder::default()
489 .load(
490 "memory",
491 HashMap::from([
492 (
493 MEMORY_CATALOG_WAREHOUSE.to_string(),
494 "memory:///warehouse".to_string(),
495 ),
496 ("custom.property".to_string(), "value".to_string()),
497 ]),
498 )
499 .await
500 .unwrap();
501 let file_io_props = catalog.file_io.config().props();
502
503 assert_eq!(
504 file_io_props.get("custom.property"),
505 Some(&"value".to_string())
506 );
507 assert_eq!(
508 file_io_props.get(MEMORY_CATALOG_WAREHOUSE),
509 Some(&"memory:///warehouse".to_string())
510 );
511 }
512
513 pub(crate) async fn new_memory_catalog() -> impl Catalog {
514 let warehouse_location = temp_path();
515 MemoryCatalogBuilder::default()
516 .load(
517 "memory",
518 HashMap::from([(MEMORY_CATALOG_WAREHOUSE.to_string(), warehouse_location)]),
519 )
520 .await
521 .unwrap()
522 }
523
524 async fn create_namespace<C: Catalog>(catalog: &C, namespace_ident: &NamespaceIdent) {
525 let _ = catalog
526 .create_namespace(namespace_ident, HashMap::new())
527 .await
528 .unwrap();
529 }
530
531 async fn create_namespaces<C: Catalog>(catalog: &C, namespace_idents: &Vec<&NamespaceIdent>) {
532 for namespace_ident in namespace_idents {
533 let _ = create_namespace(catalog, namespace_ident).await;
534 }
535 }
536
537 fn to_set<T: Eq + Hash>(vec: Vec<T>) -> HashSet<T> {
538 HashSet::from_iter(vec)
539 }
540
541 fn simple_table_schema() -> Schema {
542 Schema::builder()
543 .with_fields(vec![
544 NestedField::required(1, "foo", Type::Primitive(PrimitiveType::Int)).into(),
545 ])
546 .build()
547 .unwrap()
548 }
549
550 async fn create_table<C: Catalog>(catalog: &C, table_ident: &TableIdent) -> Table {
551 catalog
552 .create_table(
553 &table_ident.namespace,
554 TableCreation::builder()
555 .name(table_ident.name().into())
556 .schema(simple_table_schema())
557 .build(),
558 )
559 .await
560 .unwrap()
561 }
562
563 async fn create_tables<C: Catalog>(catalog: &C, table_idents: Vec<&TableIdent>) {
564 for table_ident in table_idents {
565 create_table(catalog, table_ident).await;
566 }
567 }
568
569 async fn create_table_with_namespace<C: Catalog>(catalog: &C) -> Table {
570 let namespace_ident = NamespaceIdent::new("abc".into());
571 create_namespace(catalog, &namespace_ident).await;
572
573 let table_ident = TableIdent::new(namespace_ident, "test".to_string());
574 create_table(catalog, &table_ident).await
575 }
576
577 fn assert_table_eq(table: &Table, expected_table_ident: &TableIdent, expected_schema: &Schema) {
578 assert_eq!(table.identifier(), expected_table_ident);
579
580 let metadata = table.metadata();
581
582 assert_eq!(metadata.current_schema().as_ref(), expected_schema);
583
584 let expected_partition_spec = PartitionSpec::builder((*expected_schema).clone())
585 .with_spec_id(0)
586 .build()
587 .unwrap();
588
589 assert_eq!(
590 metadata
591 .partition_specs_iter()
592 .map(|p| p.as_ref())
593 .collect_vec(),
594 vec![&expected_partition_spec]
595 );
596
597 let expected_sorted_order = SortOrder::builder()
598 .with_order_id(0)
599 .with_fields(vec![])
600 .build(expected_schema)
601 .unwrap();
602
603 assert_eq!(
604 metadata
605 .sort_orders_iter()
606 .map(|s| s.as_ref())
607 .collect_vec(),
608 vec![&expected_sorted_order]
609 );
610
611 assert_eq!(metadata.properties(), &HashMap::new());
612
613 assert!(!table.readonly());
614 }
615
616 const UUID_REGEX_STR: &str = "[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}";
617
618 fn assert_table_metadata_location_matches(table: &Table, regex_str: &str) {
619 let actual = table.metadata_location().unwrap().to_string();
620 let regex = Regex::new(regex_str).unwrap();
621 assert!(
622 regex.is_match(&actual),
623 "Expected metadata location to match regex, but got location: {actual} and regex: {regex}"
624 )
625 }
626
627 #[tokio::test]
628 async fn test_list_namespaces_returns_empty_vector() {
629 let catalog = new_memory_catalog().await;
630
631 assert_eq!(catalog.list_namespaces(None).await.unwrap(), vec![]);
632 }
633
634 #[tokio::test]
635 async fn test_list_namespaces_returns_single_namespace() {
636 let catalog = new_memory_catalog().await;
637 let namespace_ident = NamespaceIdent::new("abc".into());
638 create_namespace(&catalog, &namespace_ident).await;
639
640 assert_eq!(catalog.list_namespaces(None).await.unwrap(), vec![
641 namespace_ident
642 ]);
643 }
644
645 #[tokio::test]
646 async fn test_list_namespaces_returns_multiple_namespaces() {
647 let catalog = new_memory_catalog().await;
648 let namespace_ident_1 = NamespaceIdent::new("a".into());
649 let namespace_ident_2 = NamespaceIdent::new("b".into());
650 create_namespaces(&catalog, &vec![&namespace_ident_1, &namespace_ident_2]).await;
651
652 assert_eq!(
653 to_set(catalog.list_namespaces(None).await.unwrap()),
654 to_set(vec![namespace_ident_1, namespace_ident_2])
655 );
656 }
657
658 #[tokio::test]
659 async fn test_list_namespaces_returns_only_top_level_namespaces() {
660 let catalog = new_memory_catalog().await;
661 let namespace_ident_1 = NamespaceIdent::new("a".into());
662 let namespace_ident_2 = NamespaceIdent::from_strs(vec!["a", "b"]).unwrap();
663 let namespace_ident_3 = NamespaceIdent::new("b".into());
664 create_namespaces(&catalog, &vec![
665 &namespace_ident_1,
666 &namespace_ident_2,
667 &namespace_ident_3,
668 ])
669 .await;
670
671 assert_eq!(
672 to_set(catalog.list_namespaces(None).await.unwrap()),
673 to_set(vec![namespace_ident_1, namespace_ident_3])
674 );
675 }
676
677 #[tokio::test]
678 async fn test_list_namespaces_returns_no_namespaces_under_parent() {
679 let catalog = new_memory_catalog().await;
680 let namespace_ident_1 = NamespaceIdent::new("a".into());
681 let namespace_ident_2 = NamespaceIdent::new("b".into());
682 create_namespaces(&catalog, &vec![&namespace_ident_1, &namespace_ident_2]).await;
683
684 assert_eq!(
685 catalog
686 .list_namespaces(Some(&namespace_ident_1))
687 .await
688 .unwrap(),
689 vec![]
690 );
691 }
692
693 #[tokio::test]
694 async fn test_list_namespaces_returns_namespace_under_parent() {
695 let catalog = new_memory_catalog().await;
696 let namespace_ident_1 = NamespaceIdent::new("a".into());
697 let namespace_ident_2 = NamespaceIdent::from_strs(vec!["a", "b"]).unwrap();
698 let namespace_ident_3 = NamespaceIdent::new("c".into());
699 create_namespaces(&catalog, &vec![
700 &namespace_ident_1,
701 &namespace_ident_2,
702 &namespace_ident_3,
703 ])
704 .await;
705
706 assert_eq!(
707 to_set(catalog.list_namespaces(None).await.unwrap()),
708 to_set(vec![namespace_ident_1.clone(), namespace_ident_3])
709 );
710
711 assert_eq!(
712 catalog
713 .list_namespaces(Some(&namespace_ident_1))
714 .await
715 .unwrap(),
716 vec![namespace_ident_2]
717 );
718 }
719
720 #[tokio::test]
721 async fn test_list_namespaces_returns_multiple_namespaces_under_parent() {
722 let catalog = new_memory_catalog().await;
723 let namespace_ident_1 = NamespaceIdent::new("a".to_string());
724 let namespace_ident_2 = NamespaceIdent::from_strs(vec!["a", "a"]).unwrap();
725 let namespace_ident_3 = NamespaceIdent::from_strs(vec!["a", "b"]).unwrap();
726 let namespace_ident_4 = NamespaceIdent::from_strs(vec!["a", "c"]).unwrap();
727 let namespace_ident_5 = NamespaceIdent::new("b".into());
728 create_namespaces(&catalog, &vec![
729 &namespace_ident_1,
730 &namespace_ident_2,
731 &namespace_ident_3,
732 &namespace_ident_4,
733 &namespace_ident_5,
734 ])
735 .await;
736
737 assert_eq!(
738 to_set(
739 catalog
740 .list_namespaces(Some(&namespace_ident_1))
741 .await
742 .unwrap()
743 ),
744 to_set(vec![
745 namespace_ident_2,
746 namespace_ident_3,
747 namespace_ident_4,
748 ])
749 );
750 }
751
752 #[tokio::test]
753 async fn test_namespace_exists_returns_false() {
754 let catalog = new_memory_catalog().await;
755 let namespace_ident = NamespaceIdent::new("a".into());
756 create_namespace(&catalog, &namespace_ident).await;
757
758 assert!(
759 !catalog
760 .namespace_exists(&NamespaceIdent::new("b".into()))
761 .await
762 .unwrap()
763 );
764 }
765
766 #[tokio::test]
767 async fn test_namespace_exists_returns_true() {
768 let catalog = new_memory_catalog().await;
769 let namespace_ident = NamespaceIdent::new("a".into());
770 create_namespace(&catalog, &namespace_ident).await;
771
772 assert!(catalog.namespace_exists(&namespace_ident).await.unwrap());
773 }
774
775 #[tokio::test]
776 async fn test_create_namespace_with_empty_properties() {
777 let catalog = new_memory_catalog().await;
778 let namespace_ident = NamespaceIdent::new("a".into());
779
780 assert_eq!(
781 catalog
782 .create_namespace(&namespace_ident, HashMap::new())
783 .await
784 .unwrap(),
785 Namespace::new(namespace_ident.clone())
786 );
787
788 assert_eq!(
789 catalog.get_namespace(&namespace_ident).await.unwrap(),
790 Namespace::with_properties(namespace_ident, HashMap::new())
791 );
792 }
793
794 #[tokio::test]
795 async fn test_create_namespace_with_properties() {
796 let catalog = new_memory_catalog().await;
797 let namespace_ident = NamespaceIdent::new("abc".into());
798
799 let mut properties: HashMap<String, String> = HashMap::new();
800 properties.insert("k".into(), "v".into());
801
802 assert_eq!(
803 catalog
804 .create_namespace(&namespace_ident, properties.clone())
805 .await
806 .unwrap(),
807 Namespace::with_properties(namespace_ident.clone(), properties.clone())
808 );
809
810 assert_eq!(
811 catalog.get_namespace(&namespace_ident).await.unwrap(),
812 Namespace::with_properties(namespace_ident, properties)
813 );
814 }
815
816 #[tokio::test]
817 async fn test_create_namespace_throws_error_if_namespace_already_exists() {
818 let catalog = new_memory_catalog().await;
819 let namespace_ident = NamespaceIdent::new("a".into());
820 create_namespace(&catalog, &namespace_ident).await;
821
822 assert_eq!(
823 catalog
824 .create_namespace(&namespace_ident, HashMap::new())
825 .await
826 .unwrap_err()
827 .to_string(),
828 format!(
829 "NamespaceAlreadyExists => Cannot create namespace {:?}. Namespace already exists.",
830 &namespace_ident
831 )
832 );
833
834 assert_eq!(
835 catalog.get_namespace(&namespace_ident).await.unwrap(),
836 Namespace::with_properties(namespace_ident, HashMap::new())
837 );
838 }
839
840 #[tokio::test]
841 async fn test_create_nested_namespace() {
842 let catalog = new_memory_catalog().await;
843 let parent_namespace_ident = NamespaceIdent::new("a".into());
844 create_namespace(&catalog, &parent_namespace_ident).await;
845
846 let child_namespace_ident = NamespaceIdent::from_strs(vec!["a", "b"]).unwrap();
847
848 assert_eq!(
849 catalog
850 .create_namespace(&child_namespace_ident, HashMap::new())
851 .await
852 .unwrap(),
853 Namespace::new(child_namespace_ident.clone())
854 );
855
856 assert_eq!(
857 catalog.get_namespace(&child_namespace_ident).await.unwrap(),
858 Namespace::with_properties(child_namespace_ident, HashMap::new())
859 );
860 }
861
862 #[tokio::test]
863 async fn test_create_deeply_nested_namespace() {
864 let catalog = new_memory_catalog().await;
865 let namespace_ident_a = NamespaceIdent::new("a".into());
866 let namespace_ident_a_b = NamespaceIdent::from_strs(vec!["a", "b"]).unwrap();
867 create_namespaces(&catalog, &vec![&namespace_ident_a, &namespace_ident_a_b]).await;
868
869 let namespace_ident_a_b_c = NamespaceIdent::from_strs(vec!["a", "b", "c"]).unwrap();
870
871 assert_eq!(
872 catalog
873 .create_namespace(&namespace_ident_a_b_c, HashMap::new())
874 .await
875 .unwrap(),
876 Namespace::new(namespace_ident_a_b_c.clone())
877 );
878
879 assert_eq!(
880 catalog.get_namespace(&namespace_ident_a_b_c).await.unwrap(),
881 Namespace::with_properties(namespace_ident_a_b_c, HashMap::new())
882 );
883 }
884
885 #[tokio::test]
886 async fn test_create_nested_namespace_throws_error_if_top_level_namespace_doesnt_exist() {
887 let catalog = new_memory_catalog().await;
888
889 let nested_namespace_ident = NamespaceIdent::from_strs(vec!["a", "b"]).unwrap();
890
891 assert_eq!(
892 catalog
893 .create_namespace(&nested_namespace_ident, HashMap::new())
894 .await
895 .unwrap_err()
896 .to_string(),
897 format!(
898 "NamespaceNotFound => No such namespace: {:?}",
899 NamespaceIdent::new("a".into())
900 )
901 );
902
903 assert_eq!(catalog.list_namespaces(None).await.unwrap(), vec![]);
904 }
905
906 #[tokio::test]
907 async fn test_create_deeply_nested_namespace_throws_error_if_intermediate_namespace_doesnt_exist()
908 {
909 let catalog = new_memory_catalog().await;
910
911 let namespace_ident_a = NamespaceIdent::new("a".into());
912 create_namespace(&catalog, &namespace_ident_a).await;
913
914 let namespace_ident_a_b_c = NamespaceIdent::from_strs(vec!["a", "b", "c"]).unwrap();
915
916 assert_eq!(
917 catalog
918 .create_namespace(&namespace_ident_a_b_c, HashMap::new())
919 .await
920 .unwrap_err()
921 .to_string(),
922 format!(
923 "NamespaceNotFound => No such namespace: {:?}",
924 NamespaceIdent::from_strs(vec!["a", "b"]).unwrap()
925 )
926 );
927
928 assert_eq!(catalog.list_namespaces(None).await.unwrap(), vec![
929 namespace_ident_a.clone()
930 ]);
931
932 assert_eq!(
933 catalog
934 .list_namespaces(Some(&namespace_ident_a))
935 .await
936 .unwrap(),
937 vec![]
938 );
939 }
940
941 #[tokio::test]
942 async fn test_get_namespace() {
943 let catalog = new_memory_catalog().await;
944 let namespace_ident = NamespaceIdent::new("abc".into());
945
946 let mut properties: HashMap<String, String> = HashMap::new();
947 properties.insert("k".into(), "v".into());
948 let _ = catalog
949 .create_namespace(&namespace_ident, properties.clone())
950 .await
951 .unwrap();
952
953 assert_eq!(
954 catalog.get_namespace(&namespace_ident).await.unwrap(),
955 Namespace::with_properties(namespace_ident, properties)
956 )
957 }
958
959 #[tokio::test]
960 async fn test_get_nested_namespace() {
961 let catalog = new_memory_catalog().await;
962 let namespace_ident_a = NamespaceIdent::new("a".into());
963 let namespace_ident_a_b = NamespaceIdent::from_strs(vec!["a", "b"]).unwrap();
964 create_namespaces(&catalog, &vec![&namespace_ident_a, &namespace_ident_a_b]).await;
965
966 assert_eq!(
967 catalog.get_namespace(&namespace_ident_a_b).await.unwrap(),
968 Namespace::with_properties(namespace_ident_a_b, HashMap::new())
969 );
970 }
971
972 #[tokio::test]
973 async fn test_get_deeply_nested_namespace() {
974 let catalog = new_memory_catalog().await;
975 let namespace_ident_a = NamespaceIdent::new("a".into());
976 let namespace_ident_a_b = NamespaceIdent::from_strs(vec!["a", "b"]).unwrap();
977 let namespace_ident_a_b_c = NamespaceIdent::from_strs(vec!["a", "b", "c"]).unwrap();
978 create_namespaces(&catalog, &vec![
979 &namespace_ident_a,
980 &namespace_ident_a_b,
981 &namespace_ident_a_b_c,
982 ])
983 .await;
984
985 assert_eq!(
986 catalog.get_namespace(&namespace_ident_a_b_c).await.unwrap(),
987 Namespace::with_properties(namespace_ident_a_b_c, HashMap::new())
988 );
989 }
990
991 #[tokio::test]
992 async fn test_get_namespace_throws_error_if_namespace_doesnt_exist() {
993 let catalog = new_memory_catalog().await;
994 create_namespace(&catalog, &NamespaceIdent::new("a".into())).await;
995
996 let non_existent_namespace_ident = NamespaceIdent::new("b".into());
997 assert_eq!(
998 catalog
999 .get_namespace(&non_existent_namespace_ident)
1000 .await
1001 .unwrap_err()
1002 .to_string(),
1003 format!("NamespaceNotFound => No such namespace: {non_existent_namespace_ident:?}")
1004 )
1005 }
1006
1007 #[tokio::test]
1008 async fn test_update_namespace() {
1009 let catalog = new_memory_catalog().await;
1010 let namespace_ident = NamespaceIdent::new("abc".into());
1011 create_namespace(&catalog, &namespace_ident).await;
1012
1013 let mut new_properties: HashMap<String, String> = HashMap::new();
1014 new_properties.insert("k".into(), "v".into());
1015
1016 catalog
1017 .update_namespace(&namespace_ident, new_properties.clone())
1018 .await
1019 .unwrap();
1020
1021 assert_eq!(
1022 catalog.get_namespace(&namespace_ident).await.unwrap(),
1023 Namespace::with_properties(namespace_ident, new_properties)
1024 )
1025 }
1026
1027 #[tokio::test]
1028 async fn test_update_nested_namespace() {
1029 let catalog = new_memory_catalog().await;
1030 let namespace_ident_a = NamespaceIdent::new("a".into());
1031 let namespace_ident_a_b = NamespaceIdent::from_strs(vec!["a", "b"]).unwrap();
1032 create_namespaces(&catalog, &vec![&namespace_ident_a, &namespace_ident_a_b]).await;
1033
1034 let mut new_properties = HashMap::new();
1035 new_properties.insert("k".into(), "v".into());
1036
1037 catalog
1038 .update_namespace(&namespace_ident_a_b, new_properties.clone())
1039 .await
1040 .unwrap();
1041
1042 assert_eq!(
1043 catalog.get_namespace(&namespace_ident_a_b).await.unwrap(),
1044 Namespace::with_properties(namespace_ident_a_b, new_properties)
1045 );
1046 }
1047
1048 #[tokio::test]
1049 async fn test_update_deeply_nested_namespace() {
1050 let catalog = new_memory_catalog().await;
1051 let namespace_ident_a = NamespaceIdent::new("a".into());
1052 let namespace_ident_a_b = NamespaceIdent::from_strs(vec!["a", "b"]).unwrap();
1053 let namespace_ident_a_b_c = NamespaceIdent::from_strs(vec!["a", "b", "c"]).unwrap();
1054 create_namespaces(&catalog, &vec![
1055 &namespace_ident_a,
1056 &namespace_ident_a_b,
1057 &namespace_ident_a_b_c,
1058 ])
1059 .await;
1060
1061 let mut new_properties = HashMap::new();
1062 new_properties.insert("k".into(), "v".into());
1063
1064 catalog
1065 .update_namespace(&namespace_ident_a_b_c, new_properties.clone())
1066 .await
1067 .unwrap();
1068
1069 assert_eq!(
1070 catalog.get_namespace(&namespace_ident_a_b_c).await.unwrap(),
1071 Namespace::with_properties(namespace_ident_a_b_c, new_properties)
1072 );
1073 }
1074
1075 #[tokio::test]
1076 async fn test_update_namespace_throws_error_if_namespace_doesnt_exist() {
1077 let catalog = new_memory_catalog().await;
1078 create_namespace(&catalog, &NamespaceIdent::new("abc".into())).await;
1079
1080 let non_existent_namespace_ident = NamespaceIdent::new("def".into());
1081 assert_eq!(
1082 catalog
1083 .update_namespace(&non_existent_namespace_ident, HashMap::new())
1084 .await
1085 .unwrap_err()
1086 .to_string(),
1087 format!("NamespaceNotFound => No such namespace: {non_existent_namespace_ident:?}")
1088 )
1089 }
1090
1091 #[tokio::test]
1092 async fn test_drop_namespace() {
1093 let catalog = new_memory_catalog().await;
1094 let namespace_ident = NamespaceIdent::new("abc".into());
1095 create_namespace(&catalog, &namespace_ident).await;
1096
1097 catalog.drop_namespace(&namespace_ident).await.unwrap();
1098
1099 assert!(!catalog.namespace_exists(&namespace_ident).await.unwrap())
1100 }
1101
1102 #[tokio::test]
1103 async fn test_drop_nested_namespace() {
1104 let catalog = new_memory_catalog().await;
1105 let namespace_ident_a = NamespaceIdent::new("a".into());
1106 let namespace_ident_a_b = NamespaceIdent::from_strs(vec!["a", "b"]).unwrap();
1107 create_namespaces(&catalog, &vec![&namespace_ident_a, &namespace_ident_a_b]).await;
1108
1109 catalog.drop_namespace(&namespace_ident_a_b).await.unwrap();
1110
1111 assert!(
1112 !catalog
1113 .namespace_exists(&namespace_ident_a_b)
1114 .await
1115 .unwrap()
1116 );
1117
1118 assert!(catalog.namespace_exists(&namespace_ident_a).await.unwrap());
1119 }
1120
1121 #[tokio::test]
1122 async fn test_drop_deeply_nested_namespace() {
1123 let catalog = new_memory_catalog().await;
1124 let namespace_ident_a = NamespaceIdent::new("a".into());
1125 let namespace_ident_a_b = NamespaceIdent::from_strs(vec!["a", "b"]).unwrap();
1126 let namespace_ident_a_b_c = NamespaceIdent::from_strs(vec!["a", "b", "c"]).unwrap();
1127 create_namespaces(&catalog, &vec![
1128 &namespace_ident_a,
1129 &namespace_ident_a_b,
1130 &namespace_ident_a_b_c,
1131 ])
1132 .await;
1133
1134 catalog
1135 .drop_namespace(&namespace_ident_a_b_c)
1136 .await
1137 .unwrap();
1138
1139 assert!(
1140 !catalog
1141 .namespace_exists(&namespace_ident_a_b_c)
1142 .await
1143 .unwrap()
1144 );
1145
1146 assert!(
1147 catalog
1148 .namespace_exists(&namespace_ident_a_b)
1149 .await
1150 .unwrap()
1151 );
1152
1153 assert!(catalog.namespace_exists(&namespace_ident_a).await.unwrap());
1154 }
1155
1156 #[tokio::test]
1157 async fn test_drop_namespace_throws_error_if_namespace_doesnt_exist() {
1158 let catalog = new_memory_catalog().await;
1159
1160 let non_existent_namespace_ident = NamespaceIdent::new("abc".into());
1161 assert_eq!(
1162 catalog
1163 .drop_namespace(&non_existent_namespace_ident)
1164 .await
1165 .unwrap_err()
1166 .to_string(),
1167 format!("NamespaceNotFound => No such namespace: {non_existent_namespace_ident:?}")
1168 )
1169 }
1170
1171 #[tokio::test]
1172 async fn test_drop_namespace_throws_error_if_nested_namespace_doesnt_exist() {
1173 let catalog = new_memory_catalog().await;
1174 create_namespace(&catalog, &NamespaceIdent::new("a".into())).await;
1175
1176 let non_existent_namespace_ident =
1177 NamespaceIdent::from_vec(vec!["a".into(), "b".into()]).unwrap();
1178 assert_eq!(
1179 catalog
1180 .drop_namespace(&non_existent_namespace_ident)
1181 .await
1182 .unwrap_err()
1183 .to_string(),
1184 format!("NamespaceNotFound => No such namespace: {non_existent_namespace_ident:?}")
1185 )
1186 }
1187
1188 #[tokio::test]
1189 async fn test_dropping_a_namespace_also_drops_namespaces_nested_under_that_one() {
1190 let catalog = new_memory_catalog().await;
1191 let namespace_ident_a = NamespaceIdent::new("a".into());
1192 let namespace_ident_a_b = NamespaceIdent::from_strs(vec!["a", "b"]).unwrap();
1193 create_namespaces(&catalog, &vec![&namespace_ident_a, &namespace_ident_a_b]).await;
1194
1195 catalog.drop_namespace(&namespace_ident_a).await.unwrap();
1196
1197 assert!(!catalog.namespace_exists(&namespace_ident_a).await.unwrap());
1198
1199 assert!(
1200 !catalog
1201 .namespace_exists(&namespace_ident_a_b)
1202 .await
1203 .unwrap()
1204 );
1205 }
1206
1207 #[tokio::test]
1208 async fn test_create_table_with_location() {
1209 let tmp_dir = TempDir::new().unwrap();
1210 let catalog = new_memory_catalog().await;
1211 let namespace_ident = NamespaceIdent::new("a".into());
1212 create_namespace(&catalog, &namespace_ident).await;
1213
1214 let table_name = "abc";
1215 let location = tmp_dir.path().to_str().unwrap().to_string();
1216 let table_creation = TableCreation::builder()
1217 .name(table_name.into())
1218 .location(location.clone())
1219 .schema(simple_table_schema())
1220 .build();
1221
1222 let expected_table_ident = TableIdent::new(namespace_ident.clone(), table_name.into());
1223
1224 assert_table_eq(
1225 &catalog
1226 .create_table(&namespace_ident, table_creation)
1227 .await
1228 .unwrap(),
1229 &expected_table_ident,
1230 &simple_table_schema(),
1231 );
1232
1233 let table = catalog.load_table(&expected_table_ident).await.unwrap();
1234
1235 assert_table_eq(&table, &expected_table_ident, &simple_table_schema());
1236
1237 assert!(
1238 table
1239 .metadata_location()
1240 .unwrap()
1241 .to_string()
1242 .starts_with(&location)
1243 )
1244 }
1245
1246 #[tokio::test]
1247 async fn test_create_table_falls_back_to_namespace_location_if_table_location_is_missing() {
1248 let warehouse_location = temp_path();
1249 let catalog = MemoryCatalogBuilder::default()
1250 .load(
1251 "memory",
1252 HashMap::from([(
1253 MEMORY_CATALOG_WAREHOUSE.to_string(),
1254 warehouse_location.clone(),
1255 )]),
1256 )
1257 .await
1258 .unwrap();
1259
1260 let namespace_ident = NamespaceIdent::new("a".into());
1261 let mut namespace_properties = HashMap::new();
1262 let namespace_location = temp_path();
1263 namespace_properties.insert(LOCATION.to_string(), namespace_location.to_string());
1264 catalog
1265 .create_namespace(&namespace_ident, namespace_properties)
1266 .await
1267 .unwrap();
1268
1269 let table_name = "tbl1";
1270 let expected_table_ident = TableIdent::new(namespace_ident.clone(), table_name.into());
1271 let expected_table_metadata_location_regex =
1272 format!("^{namespace_location}/tbl1/metadata/00000-{UUID_REGEX_STR}.metadata.json$",);
1273
1274 let table = catalog
1275 .create_table(
1276 &namespace_ident,
1277 TableCreation::builder()
1278 .name(table_name.into())
1279 .schema(simple_table_schema())
1280 .build(),
1282 )
1283 .await
1284 .unwrap();
1285 assert_table_eq(&table, &expected_table_ident, &simple_table_schema());
1286 assert_table_metadata_location_matches(&table, &expected_table_metadata_location_regex);
1287
1288 let table = catalog.load_table(&expected_table_ident).await.unwrap();
1289 assert_table_eq(&table, &expected_table_ident, &simple_table_schema());
1290 assert_table_metadata_location_matches(&table, &expected_table_metadata_location_regex);
1291 }
1292
1293 #[tokio::test]
1294 async fn test_create_table_in_nested_namespace_falls_back_to_nested_namespace_location_if_table_location_is_missing()
1295 {
1296 let warehouse_location = temp_path();
1297 let catalog = MemoryCatalogBuilder::default()
1298 .load(
1299 "memory",
1300 HashMap::from([(
1301 MEMORY_CATALOG_WAREHOUSE.to_string(),
1302 warehouse_location.clone(),
1303 )]),
1304 )
1305 .await
1306 .unwrap();
1307
1308 let namespace_ident = NamespaceIdent::new("a".into());
1309 let mut namespace_properties = HashMap::new();
1310 let namespace_location = temp_path();
1311 namespace_properties.insert(LOCATION.to_string(), namespace_location.to_string());
1312 catalog
1313 .create_namespace(&namespace_ident, namespace_properties)
1314 .await
1315 .unwrap();
1316
1317 let nested_namespace_ident = NamespaceIdent::from_strs(vec!["a", "b"]).unwrap();
1318 let mut nested_namespace_properties = HashMap::new();
1319 let nested_namespace_location = temp_path();
1320 nested_namespace_properties
1321 .insert(LOCATION.to_string(), nested_namespace_location.to_string());
1322 catalog
1323 .create_namespace(&nested_namespace_ident, nested_namespace_properties)
1324 .await
1325 .unwrap();
1326
1327 let table_name = "tbl1";
1328 let expected_table_ident =
1329 TableIdent::new(nested_namespace_ident.clone(), table_name.into());
1330 let expected_table_metadata_location_regex = format!(
1331 "^{nested_namespace_location}/tbl1/metadata/00000-{UUID_REGEX_STR}.metadata.json$",
1332 );
1333
1334 let table = catalog
1335 .create_table(
1336 &nested_namespace_ident,
1337 TableCreation::builder()
1338 .name(table_name.into())
1339 .schema(simple_table_schema())
1340 .build(),
1342 )
1343 .await
1344 .unwrap();
1345 assert_table_eq(&table, &expected_table_ident, &simple_table_schema());
1346 assert_table_metadata_location_matches(&table, &expected_table_metadata_location_regex);
1347
1348 let table = catalog.load_table(&expected_table_ident).await.unwrap();
1349 assert_table_eq(&table, &expected_table_ident, &simple_table_schema());
1350 assert_table_metadata_location_matches(&table, &expected_table_metadata_location_regex);
1351 }
1352
1353 #[tokio::test]
1354 async fn test_create_table_falls_back_to_warehouse_location_if_both_table_location_and_namespace_location_are_missing()
1355 {
1356 let warehouse_location = temp_path();
1357 let catalog = MemoryCatalogBuilder::default()
1358 .load(
1359 "memory",
1360 HashMap::from([(
1361 MEMORY_CATALOG_WAREHOUSE.to_string(),
1362 warehouse_location.clone(),
1363 )]),
1364 )
1365 .await
1366 .unwrap();
1367
1368 let namespace_ident = NamespaceIdent::new("a".into());
1369 let namespace_properties = HashMap::new();
1371 catalog
1372 .create_namespace(&namespace_ident, namespace_properties)
1373 .await
1374 .unwrap();
1375
1376 let table_name = "tbl1";
1377 let expected_table_ident = TableIdent::new(namespace_ident.clone(), table_name.into());
1378 let expected_table_metadata_location_regex =
1379 format!("^{warehouse_location}/a/tbl1/metadata/00000-{UUID_REGEX_STR}.metadata.json$");
1380
1381 let table = catalog
1382 .create_table(
1383 &namespace_ident,
1384 TableCreation::builder()
1385 .name(table_name.into())
1386 .schema(simple_table_schema())
1387 .build(),
1389 )
1390 .await
1391 .unwrap();
1392 assert_table_eq(&table, &expected_table_ident, &simple_table_schema());
1393 assert_table_metadata_location_matches(&table, &expected_table_metadata_location_regex);
1394
1395 let table = catalog.load_table(&expected_table_ident).await.unwrap();
1396 assert_table_eq(&table, &expected_table_ident, &simple_table_schema());
1397 assert_table_metadata_location_matches(&table, &expected_table_metadata_location_regex);
1398 }
1399
1400 #[tokio::test]
1401 async fn test_create_table_in_nested_namespace_falls_back_to_warehouse_location_if_both_table_location_and_namespace_location_are_missing()
1402 {
1403 let warehouse_location = temp_path();
1404 let catalog = MemoryCatalogBuilder::default()
1405 .load(
1406 "memory",
1407 HashMap::from([(
1408 MEMORY_CATALOG_WAREHOUSE.to_string(),
1409 warehouse_location.clone(),
1410 )]),
1411 )
1412 .await
1413 .unwrap();
1414
1415 let namespace_ident = NamespaceIdent::new("a".into());
1416 catalog
1417 .create_namespace(&namespace_ident, HashMap::new())
1419 .await
1420 .unwrap();
1421
1422 let nested_namespace_ident = NamespaceIdent::from_strs(vec!["a", "b"]).unwrap();
1423 catalog
1424 .create_namespace(&nested_namespace_ident, HashMap::new())
1426 .await
1427 .unwrap();
1428
1429 let table_name = "tbl1";
1430 let expected_table_ident =
1431 TableIdent::new(nested_namespace_ident.clone(), table_name.into());
1432 let expected_table_metadata_location_regex = format!(
1433 "^{warehouse_location}/a/b/tbl1/metadata/00000-{UUID_REGEX_STR}.metadata.json$"
1434 );
1435
1436 let table = catalog
1437 .create_table(
1438 &nested_namespace_ident,
1439 TableCreation::builder()
1440 .name(table_name.into())
1441 .schema(simple_table_schema())
1442 .build(),
1444 )
1445 .await
1446 .unwrap();
1447 assert_table_eq(&table, &expected_table_ident, &simple_table_schema());
1448 assert_table_metadata_location_matches(&table, &expected_table_metadata_location_regex);
1449
1450 let table = catalog.load_table(&expected_table_ident).await.unwrap();
1451 assert_table_eq(&table, &expected_table_ident, &simple_table_schema());
1452 assert_table_metadata_location_matches(&table, &expected_table_metadata_location_regex);
1453 }
1454
1455 #[tokio::test]
1456 async fn test_load_throws_error_if_warehouse_is_missing() {
1457 let error = MemoryCatalogBuilder::default()
1458 .load("memory", HashMap::from([]))
1459 .await
1460 .unwrap_err();
1461
1462 assert_eq!(error.kind(), ErrorKind::DataInvalid);
1463 assert_eq!(error.message(), "Catalog warehouse is required");
1464 }
1465
1466 #[tokio::test]
1467 async fn test_load_throws_error_if_warehouse_is_empty() {
1468 let error = MemoryCatalogBuilder::default()
1469 .load(
1470 "memory",
1471 HashMap::from([(MEMORY_CATALOG_WAREHOUSE.to_string(), String::new())]),
1472 )
1473 .await
1474 .unwrap_err();
1475
1476 assert_eq!(error.kind(), ErrorKind::DataInvalid);
1477 assert_eq!(error.message(), "Catalog warehouse is required");
1478 }
1479
1480 #[tokio::test]
1481 async fn test_create_table_throws_error_if_table_with_same_name_already_exists() {
1482 let catalog = new_memory_catalog().await;
1483 let namespace_ident = NamespaceIdent::new("a".into());
1484 create_namespace(&catalog, &namespace_ident).await;
1485 let table_name = "tbl1";
1486 let table_ident = TableIdent::new(namespace_ident.clone(), table_name.into());
1487 create_table(&catalog, &table_ident).await;
1488
1489 let tmp_dir = TempDir::new().unwrap();
1490 let location = tmp_dir.path().to_str().unwrap().to_string();
1491
1492 assert_eq!(
1493 catalog
1494 .create_table(
1495 &namespace_ident,
1496 TableCreation::builder()
1497 .name(table_name.into())
1498 .schema(simple_table_schema())
1499 .location(location)
1500 .build()
1501 )
1502 .await
1503 .unwrap_err()
1504 .to_string(),
1505 format!(
1506 "TableAlreadyExists => Cannot create table {:?}. Table already exists.",
1507 &table_ident
1508 )
1509 );
1510 }
1511
1512 #[tokio::test]
1513 async fn test_list_tables_returns_empty_vector() {
1514 let catalog = new_memory_catalog().await;
1515 let namespace_ident = NamespaceIdent::new("a".into());
1516 create_namespace(&catalog, &namespace_ident).await;
1517
1518 assert_eq!(catalog.list_tables(&namespace_ident).await.unwrap(), vec![]);
1519 }
1520
1521 #[tokio::test]
1522 async fn test_list_tables_returns_a_single_table() {
1523 let catalog = new_memory_catalog().await;
1524 let namespace_ident = NamespaceIdent::new("n1".into());
1525 create_namespace(&catalog, &namespace_ident).await;
1526
1527 let table_ident = TableIdent::new(namespace_ident.clone(), "tbl1".into());
1528 create_table(&catalog, &table_ident).await;
1529
1530 assert_eq!(catalog.list_tables(&namespace_ident).await.unwrap(), vec![
1531 table_ident
1532 ]);
1533 }
1534
1535 #[tokio::test]
1536 async fn test_list_tables_returns_multiple_tables() {
1537 let catalog = new_memory_catalog().await;
1538 let namespace_ident = NamespaceIdent::new("n1".into());
1539 create_namespace(&catalog, &namespace_ident).await;
1540
1541 let table_ident_1 = TableIdent::new(namespace_ident.clone(), "tbl1".into());
1542 let table_ident_2 = TableIdent::new(namespace_ident.clone(), "tbl2".into());
1543 let _ = create_tables(&catalog, vec![&table_ident_1, &table_ident_2]).await;
1544
1545 assert_eq!(
1546 to_set(catalog.list_tables(&namespace_ident).await.unwrap()),
1547 to_set(vec![table_ident_1, table_ident_2])
1548 );
1549 }
1550
1551 #[tokio::test]
1552 async fn test_list_tables_returns_tables_from_correct_namespace() {
1553 let catalog = new_memory_catalog().await;
1554 let namespace_ident_1 = NamespaceIdent::new("n1".into());
1555 let namespace_ident_2 = NamespaceIdent::new("n2".into());
1556 create_namespaces(&catalog, &vec![&namespace_ident_1, &namespace_ident_2]).await;
1557
1558 let table_ident_1 = TableIdent::new(namespace_ident_1.clone(), "tbl1".into());
1559 let table_ident_2 = TableIdent::new(namespace_ident_1.clone(), "tbl2".into());
1560 let table_ident_3 = TableIdent::new(namespace_ident_2.clone(), "tbl1".into());
1561 let _ = create_tables(&catalog, vec![
1562 &table_ident_1,
1563 &table_ident_2,
1564 &table_ident_3,
1565 ])
1566 .await;
1567
1568 assert_eq!(
1569 to_set(catalog.list_tables(&namespace_ident_1).await.unwrap()),
1570 to_set(vec![table_ident_1, table_ident_2])
1571 );
1572
1573 assert_eq!(
1574 to_set(catalog.list_tables(&namespace_ident_2).await.unwrap()),
1575 to_set(vec![table_ident_3])
1576 );
1577 }
1578
1579 #[tokio::test]
1580 async fn test_list_tables_returns_table_under_nested_namespace() {
1581 let catalog = new_memory_catalog().await;
1582 let namespace_ident_a = NamespaceIdent::new("a".into());
1583 let namespace_ident_a_b = NamespaceIdent::from_strs(vec!["a", "b"]).unwrap();
1584 create_namespaces(&catalog, &vec![&namespace_ident_a, &namespace_ident_a_b]).await;
1585
1586 let table_ident = TableIdent::new(namespace_ident_a_b.clone(), "tbl1".into());
1587 create_table(&catalog, &table_ident).await;
1588
1589 assert_eq!(
1590 catalog.list_tables(&namespace_ident_a_b).await.unwrap(),
1591 vec![table_ident]
1592 );
1593 }
1594
1595 #[tokio::test]
1596 async fn test_list_tables_throws_error_if_namespace_doesnt_exist() {
1597 let catalog = new_memory_catalog().await;
1598
1599 let non_existent_namespace_ident = NamespaceIdent::new("n1".into());
1600
1601 assert_eq!(
1602 catalog
1603 .list_tables(&non_existent_namespace_ident)
1604 .await
1605 .unwrap_err()
1606 .to_string(),
1607 format!("NamespaceNotFound => No such namespace: {non_existent_namespace_ident:?}"),
1608 );
1609 }
1610
1611 #[tokio::test]
1612 async fn test_drop_table() {
1613 let catalog = new_memory_catalog().await;
1614 let namespace_ident = NamespaceIdent::new("n1".into());
1615 create_namespace(&catalog, &namespace_ident).await;
1616 let table_ident = TableIdent::new(namespace_ident.clone(), "tbl1".into());
1617 create_table(&catalog, &table_ident).await;
1618
1619 catalog.drop_table(&table_ident).await.unwrap();
1620 }
1621
1622 #[tokio::test]
1623 async fn test_drop_table_drops_table_under_nested_namespace() {
1624 let catalog = new_memory_catalog().await;
1625 let namespace_ident_a = NamespaceIdent::new("a".into());
1626 let namespace_ident_a_b = NamespaceIdent::from_strs(vec!["a", "b"]).unwrap();
1627 create_namespaces(&catalog, &vec![&namespace_ident_a, &namespace_ident_a_b]).await;
1628
1629 let table_ident = TableIdent::new(namespace_ident_a_b.clone(), "tbl1".into());
1630 create_table(&catalog, &table_ident).await;
1631
1632 catalog.drop_table(&table_ident).await.unwrap();
1633
1634 assert_eq!(
1635 catalog.list_tables(&namespace_ident_a_b).await.unwrap(),
1636 vec![]
1637 );
1638 }
1639
1640 #[tokio::test]
1641 async fn test_drop_table_throws_error_if_namespace_doesnt_exist() {
1642 let catalog = new_memory_catalog().await;
1643
1644 let non_existent_namespace_ident = NamespaceIdent::new("n1".into());
1645 let non_existent_table_ident =
1646 TableIdent::new(non_existent_namespace_ident.clone(), "tbl1".into());
1647
1648 assert_eq!(
1649 catalog
1650 .drop_table(&non_existent_table_ident)
1651 .await
1652 .unwrap_err()
1653 .to_string(),
1654 format!("NamespaceNotFound => No such namespace: {non_existent_namespace_ident:?}"),
1655 );
1656 }
1657
1658 #[tokio::test]
1659 async fn test_drop_table_throws_error_if_table_doesnt_exist() {
1660 let catalog = new_memory_catalog().await;
1661 let namespace_ident = NamespaceIdent::new("n1".into());
1662 create_namespace(&catalog, &namespace_ident).await;
1663
1664 let non_existent_table_ident = TableIdent::new(namespace_ident.clone(), "tbl1".into());
1665
1666 assert_eq!(
1667 catalog
1668 .drop_table(&non_existent_table_ident)
1669 .await
1670 .unwrap_err()
1671 .to_string(),
1672 format!("TableNotFound => No such table: {non_existent_table_ident:?}"),
1673 );
1674 }
1675
1676 #[tokio::test]
1677 async fn test_table_exists_returns_true() {
1678 let catalog = new_memory_catalog().await;
1679 let namespace_ident = NamespaceIdent::new("n1".into());
1680 create_namespace(&catalog, &namespace_ident).await;
1681 let table_ident = TableIdent::new(namespace_ident.clone(), "tbl1".into());
1682 create_table(&catalog, &table_ident).await;
1683
1684 assert!(catalog.table_exists(&table_ident).await.unwrap());
1685 }
1686
1687 #[tokio::test]
1688 async fn test_table_exists_returns_false() {
1689 let catalog = new_memory_catalog().await;
1690 let namespace_ident = NamespaceIdent::new("n1".into());
1691 create_namespace(&catalog, &namespace_ident).await;
1692 let non_existent_table_ident = TableIdent::new(namespace_ident.clone(), "tbl1".into());
1693
1694 assert!(
1695 !catalog
1696 .table_exists(&non_existent_table_ident)
1697 .await
1698 .unwrap()
1699 );
1700 }
1701
1702 #[tokio::test]
1703 async fn test_table_exists_under_nested_namespace() {
1704 let catalog = new_memory_catalog().await;
1705 let namespace_ident_a = NamespaceIdent::new("a".into());
1706 let namespace_ident_a_b = NamespaceIdent::from_strs(vec!["a", "b"]).unwrap();
1707 create_namespaces(&catalog, &vec![&namespace_ident_a, &namespace_ident_a_b]).await;
1708
1709 let table_ident = TableIdent::new(namespace_ident_a_b.clone(), "tbl1".into());
1710 create_table(&catalog, &table_ident).await;
1711
1712 assert!(catalog.table_exists(&table_ident).await.unwrap());
1713
1714 let non_existent_table_ident = TableIdent::new(namespace_ident_a_b.clone(), "tbl2".into());
1715 assert!(
1716 !catalog
1717 .table_exists(&non_existent_table_ident)
1718 .await
1719 .unwrap()
1720 );
1721 }
1722
1723 #[tokio::test]
1724 async fn test_table_exists_throws_error_if_namespace_doesnt_exist() {
1725 let catalog = new_memory_catalog().await;
1726
1727 let non_existent_namespace_ident = NamespaceIdent::new("n1".into());
1728 let non_existent_table_ident =
1729 TableIdent::new(non_existent_namespace_ident.clone(), "tbl1".into());
1730
1731 assert_eq!(
1732 catalog
1733 .table_exists(&non_existent_table_ident)
1734 .await
1735 .unwrap_err()
1736 .to_string(),
1737 format!("NamespaceNotFound => No such namespace: {non_existent_namespace_ident:?}"),
1738 );
1739 }
1740
1741 #[tokio::test]
1742 async fn test_rename_table_in_same_namespace() {
1743 let catalog = new_memory_catalog().await;
1744 let namespace_ident = NamespaceIdent::new("n1".into());
1745 create_namespace(&catalog, &namespace_ident).await;
1746 let src_table_ident = TableIdent::new(namespace_ident.clone(), "tbl1".into());
1747 let dst_table_ident = TableIdent::new(namespace_ident.clone(), "tbl2".into());
1748 create_table(&catalog, &src_table_ident).await;
1749
1750 catalog
1751 .rename_table(&src_table_ident, &dst_table_ident)
1752 .await
1753 .unwrap();
1754
1755 assert_eq!(catalog.list_tables(&namespace_ident).await.unwrap(), vec![
1756 dst_table_ident
1757 ],);
1758 }
1759
1760 #[tokio::test]
1761 async fn test_rename_table_across_namespaces() {
1762 let catalog = new_memory_catalog().await;
1763 let src_namespace_ident = NamespaceIdent::new("a".into());
1764 let dst_namespace_ident = NamespaceIdent::new("b".into());
1765 create_namespaces(&catalog, &vec![&src_namespace_ident, &dst_namespace_ident]).await;
1766 let src_table_ident = TableIdent::new(src_namespace_ident.clone(), "tbl1".into());
1767 let dst_table_ident = TableIdent::new(dst_namespace_ident.clone(), "tbl2".into());
1768 create_table(&catalog, &src_table_ident).await;
1769
1770 catalog
1771 .rename_table(&src_table_ident, &dst_table_ident)
1772 .await
1773 .unwrap();
1774
1775 assert_eq!(
1776 catalog.list_tables(&src_namespace_ident).await.unwrap(),
1777 vec![],
1778 );
1779
1780 assert_eq!(
1781 catalog.list_tables(&dst_namespace_ident).await.unwrap(),
1782 vec![dst_table_ident],
1783 );
1784 }
1785
1786 #[tokio::test]
1787 async fn test_rename_table_src_table_is_same_as_dst_table() {
1788 let catalog = new_memory_catalog().await;
1789 let namespace_ident = NamespaceIdent::new("n1".into());
1790 create_namespace(&catalog, &namespace_ident).await;
1791 let table_ident = TableIdent::new(namespace_ident.clone(), "tbl".into());
1792 create_table(&catalog, &table_ident).await;
1793
1794 catalog
1795 .rename_table(&table_ident, &table_ident)
1796 .await
1797 .unwrap();
1798
1799 assert_eq!(catalog.list_tables(&namespace_ident).await.unwrap(), vec![
1800 table_ident
1801 ],);
1802 }
1803
1804 #[tokio::test]
1805 async fn test_rename_table_across_nested_namespaces() {
1806 let catalog = new_memory_catalog().await;
1807 let namespace_ident_a = NamespaceIdent::new("a".into());
1808 let namespace_ident_a_b = NamespaceIdent::from_strs(vec!["a", "b"]).unwrap();
1809 let namespace_ident_a_b_c = NamespaceIdent::from_strs(vec!["a", "b", "c"]).unwrap();
1810 create_namespaces(&catalog, &vec![
1811 &namespace_ident_a,
1812 &namespace_ident_a_b,
1813 &namespace_ident_a_b_c,
1814 ])
1815 .await;
1816
1817 let src_table_ident = TableIdent::new(namespace_ident_a_b_c.clone(), "tbl1".into());
1818 create_tables(&catalog, vec![&src_table_ident]).await;
1819
1820 let dst_table_ident = TableIdent::new(namespace_ident_a_b.clone(), "tbl1".into());
1821 catalog
1822 .rename_table(&src_table_ident, &dst_table_ident)
1823 .await
1824 .unwrap();
1825
1826 assert!(!catalog.table_exists(&src_table_ident).await.unwrap());
1827
1828 assert!(catalog.table_exists(&dst_table_ident).await.unwrap());
1829 }
1830
1831 #[tokio::test]
1832 async fn test_rename_table_throws_error_if_src_namespace_doesnt_exist() {
1833 let catalog = new_memory_catalog().await;
1834
1835 let non_existent_src_namespace_ident = NamespaceIdent::new("n1".into());
1836 let src_table_ident =
1837 TableIdent::new(non_existent_src_namespace_ident.clone(), "tbl1".into());
1838
1839 let dst_namespace_ident = NamespaceIdent::new("n2".into());
1840 create_namespace(&catalog, &dst_namespace_ident).await;
1841 let dst_table_ident = TableIdent::new(dst_namespace_ident.clone(), "tbl1".into());
1842
1843 assert_eq!(
1844 catalog
1845 .rename_table(&src_table_ident, &dst_table_ident)
1846 .await
1847 .unwrap_err()
1848 .to_string(),
1849 format!("NamespaceNotFound => No such namespace: {non_existent_src_namespace_ident:?}"),
1850 );
1851 }
1852
1853 #[tokio::test]
1854 async fn test_rename_table_throws_error_if_dst_namespace_doesnt_exist() {
1855 let catalog = new_memory_catalog().await;
1856 let src_namespace_ident = NamespaceIdent::new("n1".into());
1857 let src_table_ident = TableIdent::new(src_namespace_ident.clone(), "tbl1".into());
1858 create_namespace(&catalog, &src_namespace_ident).await;
1859 create_table(&catalog, &src_table_ident).await;
1860
1861 let non_existent_dst_namespace_ident = NamespaceIdent::new("n2".into());
1862 let dst_table_ident =
1863 TableIdent::new(non_existent_dst_namespace_ident.clone(), "tbl1".into());
1864 assert_eq!(
1865 catalog
1866 .rename_table(&src_table_ident, &dst_table_ident)
1867 .await
1868 .unwrap_err()
1869 .to_string(),
1870 format!("NamespaceNotFound => No such namespace: {non_existent_dst_namespace_ident:?}"),
1871 );
1872 }
1873
1874 #[tokio::test]
1875 async fn test_rename_table_throws_error_if_src_table_doesnt_exist() {
1876 let catalog = new_memory_catalog().await;
1877 let namespace_ident = NamespaceIdent::new("n1".into());
1878 create_namespace(&catalog, &namespace_ident).await;
1879 let src_table_ident = TableIdent::new(namespace_ident.clone(), "tbl1".into());
1880 let dst_table_ident = TableIdent::new(namespace_ident.clone(), "tbl2".into());
1881
1882 assert_eq!(
1883 catalog
1884 .rename_table(&src_table_ident, &dst_table_ident)
1885 .await
1886 .unwrap_err()
1887 .to_string(),
1888 format!("TableNotFound => No such table: {src_table_ident:?}"),
1889 );
1890 }
1891
1892 #[tokio::test]
1893 async fn test_rename_table_throws_error_if_dst_table_already_exists() {
1894 let catalog = new_memory_catalog().await;
1895 let namespace_ident = NamespaceIdent::new("n1".into());
1896 create_namespace(&catalog, &namespace_ident).await;
1897 let src_table_ident = TableIdent::new(namespace_ident.clone(), "tbl1".into());
1898 let dst_table_ident = TableIdent::new(namespace_ident.clone(), "tbl2".into());
1899 create_tables(&catalog, vec![&src_table_ident, &dst_table_ident]).await;
1900
1901 assert_eq!(
1902 catalog
1903 .rename_table(&src_table_ident, &dst_table_ident)
1904 .await
1905 .unwrap_err()
1906 .to_string(),
1907 format!(
1908 "TableAlreadyExists => Cannot create table {:?}. Table already exists.",
1909 &dst_table_ident
1910 ),
1911 );
1912 }
1913
1914 #[tokio::test]
1915 async fn test_register_table() {
1916 let catalog = new_memory_catalog().await;
1918 let namespace_ident = NamespaceIdent::new("test_namespace".into());
1919 create_namespace(&catalog, &namespace_ident).await;
1920
1921 let source_table_ident = TableIdent::new(namespace_ident.clone(), "source_table".into());
1923 create_table(&catalog, &source_table_ident).await;
1924
1925 let source_table = catalog.load_table(&source_table_ident).await.unwrap();
1927 let metadata_location = source_table.metadata_location().unwrap().to_string();
1928
1929 let register_table_ident =
1931 TableIdent::new(namespace_ident.clone(), "register_table".into());
1932 let registered_table = catalog
1933 .register_table(®ister_table_ident, metadata_location.clone())
1934 .await
1935 .unwrap();
1936
1937 assert_eq!(registered_table.identifier(), ®ister_table_ident);
1939
1940 assert_eq!(
1942 registered_table.metadata_location().unwrap().to_string(),
1943 metadata_location
1944 );
1945
1946 assert!(catalog.table_exists(®ister_table_ident).await.unwrap());
1948
1949 let loaded_table = catalog.load_table(®ister_table_ident).await.unwrap();
1951 assert_eq!(loaded_table.identifier(), ®ister_table_ident);
1952 assert_eq!(
1953 loaded_table.metadata_location().unwrap().to_string(),
1954 metadata_location
1955 );
1956 }
1957
1958 #[tokio::test]
1959 async fn test_update_table() {
1960 let catalog = new_memory_catalog().await;
1961
1962 let table = create_table_with_namespace(&catalog).await;
1963
1964 assert!(!table.metadata().properties().contains_key("key"));
1966
1967 tokio::time::sleep(std::time::Duration::from_millis(2)).await;
1971
1972 let tx = Transaction::new(&table);
1974 let updated_table = tx
1975 .update_table_properties()
1976 .set("key".to_string(), "value".to_string())
1977 .apply(tx)
1978 .unwrap()
1979 .commit(&catalog)
1980 .await
1981 .unwrap();
1982
1983 assert_eq!(
1984 updated_table.metadata().properties().get("key").unwrap(),
1985 "value"
1986 );
1987
1988 assert_eq!(table.identifier(), updated_table.identifier());
1989 assert_eq!(table.metadata().uuid(), updated_table.metadata().uuid());
1990 assert_ne!(table.metadata_location(), updated_table.metadata_location());
1991
1992 assert!(
1993 table.metadata().metadata_log().len() < updated_table.metadata().metadata_log().len()
1994 );
1995 }
1996
1997 async fn create_table_with_properties<C: Catalog>(
1998 catalog: &C,
1999 properties: HashMap<String, String>,
2000 ) -> Table {
2001 let namespace_ident = NamespaceIdent::new("abc".into());
2002 create_namespace(catalog, &namespace_ident).await;
2003 catalog
2004 .create_table(
2005 &namespace_ident,
2006 TableCreation::builder()
2007 .name("test".into())
2008 .schema(simple_table_schema())
2009 .properties(properties)
2010 .build(),
2011 )
2012 .await
2013 .unwrap()
2014 }
2015
2016 async fn commit_property_update<C: Catalog>(catalog: &C, table: &Table, value: &str) -> Table {
2017 let tx = Transaction::new(table);
2018 tx.update_table_properties()
2019 .set("key".to_string(), value.to_string())
2020 .apply(tx)
2021 .unwrap()
2022 .commit(catalog)
2023 .await
2024 .unwrap()
2025 }
2026
2027 async fn commit_metadata_versions<C: Catalog>(
2030 catalog: &C,
2031 mut table: Table,
2032 num_commits: usize,
2033 ) -> Vec<String> {
2034 let mut locations = vec![table.metadata_location().unwrap().to_string()];
2035 for i in 0..num_commits {
2036 table = commit_property_update(catalog, &table, &i.to_string()).await;
2037 locations.push(table.metadata_location().unwrap().to_string());
2038 }
2039 locations
2040 }
2041
2042 async fn existing_files(file_io: &FileIO, locations: &[String]) -> Vec<String> {
2043 let mut existing = Vec::new();
2044 for location in locations {
2045 if file_io.exists(location).await.unwrap() {
2046 existing.push(location.clone());
2047 }
2048 }
2049 existing
2050 }
2051
2052 #[tokio::test]
2053 async fn test_update_table_deletes_metadata_files_removed_from_log_when_enabled() {
2054 let catalog = new_memory_catalog().await;
2055 let table = create_table_with_properties(
2056 &catalog,
2057 HashMap::from([
2058 (
2059 TableProperties::PROPERTY_METADATA_DELETE_AFTER_COMMIT_ENABLED.to_string(),
2060 "true".to_string(),
2061 ),
2062 (
2063 TableProperties::PROPERTY_METADATA_PREVIOUS_VERSIONS_MAX.to_string(),
2064 "1".to_string(),
2065 ),
2066 ]),
2067 )
2068 .await;
2069 let file_io = table.file_io().clone();
2070
2071 let locations = commit_metadata_versions(&catalog, table, 3).await;
2072
2073 assert_eq!(
2075 existing_files(&file_io, &locations).await,
2076 locations[2..].to_vec()
2077 );
2078 }
2079
2080 #[tokio::test]
2081 async fn test_update_table_deletes_removed_metadata_files_on_commit_that_enables_cleanup() {
2082 let catalog = new_memory_catalog().await;
2083 let table = create_table_with_properties(
2084 &catalog,
2085 HashMap::from([(
2086 TableProperties::PROPERTY_METADATA_PREVIOUS_VERSIONS_MAX.to_string(),
2087 "1".to_string(),
2088 )]),
2089 )
2090 .await;
2091 let file_io = table.file_io().clone();
2092 let v0 = table.metadata_location().unwrap().to_string();
2093 let v1_table = commit_property_update(&catalog, &table, "1").await;
2094 let v1 = v1_table.metadata_location().unwrap().to_string();
2095 let v2_table = commit_property_update(&catalog, &v1_table, "2").await;
2096 let v2 = v2_table.metadata_location().unwrap().to_string();
2097
2098 let tx = Transaction::new(&v2_table);
2099 let v3_table = tx
2100 .update_table_properties()
2101 .set(
2102 TableProperties::PROPERTY_METADATA_DELETE_AFTER_COMMIT_ENABLED.to_string(),
2103 "true".to_string(),
2104 )
2105 .apply(tx)
2106 .unwrap()
2107 .commit(&catalog)
2108 .await
2109 .unwrap();
2110 let v3 = v3_table.metadata_location().unwrap().to_string();
2111
2112 assert_eq!(
2114 existing_files(&file_io, &[v0.clone(), v1, v2.clone(), v3.clone()]).await,
2115 vec![v0, v2, v3]
2116 );
2117 }
2118
2119 #[tokio::test]
2120 async fn test_update_table_keeps_metadata_files_removed_from_log_when_not_enabled() {
2121 for delete_after_commit in [None, Some("false")] {
2122 let catalog = new_memory_catalog().await;
2123 let mut properties = HashMap::from([(
2124 TableProperties::PROPERTY_METADATA_PREVIOUS_VERSIONS_MAX.to_string(),
2125 "1".to_string(),
2126 )]);
2127 if let Some(value) = delete_after_commit {
2128 properties.insert(
2129 TableProperties::PROPERTY_METADATA_DELETE_AFTER_COMMIT_ENABLED.to_string(),
2130 value.to_string(),
2131 );
2132 }
2133 let table = create_table_with_properties(&catalog, properties).await;
2134 let file_io = table.file_io().clone();
2135
2136 let locations = commit_metadata_versions(&catalog, table, 3).await;
2137
2138 assert_eq!(existing_files(&file_io, &locations).await, locations);
2139 }
2140 }
2141
2142 #[tokio::test]
2143 async fn test_update_table_succeeds_when_deleting_removed_metadata_file_fails() {
2144 let warehouse = TempDir::new().unwrap();
2145 let catalog = MemoryCatalogBuilder::default()
2146 .with_storage_factory(Arc::new(LocalFsStorageFactory))
2147 .load(
2148 "memory",
2149 HashMap::from([(
2150 MEMORY_CATALOG_WAREHOUSE.to_string(),
2151 warehouse.path().to_str().unwrap().to_string(),
2152 )]),
2153 )
2154 .await
2155 .unwrap();
2156 let table = create_table_with_properties(
2157 &catalog,
2158 HashMap::from([
2159 (
2160 TableProperties::PROPERTY_METADATA_DELETE_AFTER_COMMIT_ENABLED.to_string(),
2161 "true".to_string(),
2162 ),
2163 (
2164 TableProperties::PROPERTY_METADATA_PREVIOUS_VERSIONS_MAX.to_string(),
2165 "1".to_string(),
2166 ),
2167 ]),
2168 )
2169 .await;
2170 let file_io = table.file_io().clone();
2171 let v0 = table.metadata_location().unwrap().to_string();
2172 let v1_table = commit_property_update(&catalog, &table, "1").await;
2173 let v1 = v1_table.metadata_location().unwrap().to_string();
2174
2175 std::fs::remove_file(&v0).unwrap();
2177 std::fs::create_dir(&v0).unwrap();
2178 std::fs::write(format!("{v0}/blocker"), b"").unwrap();
2179
2180 let v2_table = commit_property_update(&catalog, &v1_table, "2").await;
2181 let v2 = v2_table.metadata_location().unwrap().to_string();
2182 let v3_table = commit_property_update(&catalog, &v2_table, "3").await;
2183 let v3 = v3_table.metadata_location().unwrap().to_string();
2184
2185 assert_eq!(
2186 existing_files(&file_io, &[v0.clone(), v1, v2.clone(), v3.clone()]).await,
2187 vec![v0, v2, v3]
2188 );
2189 }
2190
2191 #[tokio::test]
2192 async fn test_update_table_fails_if_table_doesnt_exist() {
2193 let catalog = new_memory_catalog().await;
2194
2195 let namespace_ident = NamespaceIdent::new("a".into());
2196 create_namespace(&catalog, &namespace_ident).await;
2197
2198 let table_ident = TableIdent::new(namespace_ident, "test".to_string());
2200 let table = build_table(table_ident);
2201
2202 let tx = Transaction::new(&table);
2203 let err = tx
2204 .update_table_properties()
2205 .set("key".to_string(), "value".to_string())
2206 .apply(tx)
2207 .unwrap()
2208 .commit(&catalog)
2209 .await
2210 .unwrap_err();
2211 assert_eq!(err.kind(), ErrorKind::TableNotFound);
2212 }
2213
2214 const FIXTURE_MASTER_KEY_ID: &str = "master-1";
2217 const FIXTURE_MASTER_KEY_BYTES: [u8; 16] = [
2218 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e,
2219 0x0f,
2220 ];
2221
2222 fn fixture_kms_factory() -> MemoryKmsClientFactory {
2224 use crate::encryption::SensitiveBytes;
2225
2226 let factory = MemoryKmsClientFactory::new();
2227 factory
2228 .add_master_key_bytes(
2229 FIXTURE_MASTER_KEY_ID,
2230 SensitiveBytes::new(FIXTURE_MASTER_KEY_BYTES),
2231 )
2232 .unwrap();
2233 factory
2234 }
2235
2236 fn load_encrypted_fixture_metadata() -> TableMetadata {
2239 let manifest_dir = env!("CARGO_MANIFEST_DIR");
2240 let metadata_json = std::fs::read_to_string(format!(
2241 "{manifest_dir}/testdata/table_metadata/TableMetadataV3ValidEncryption.json"
2242 ))
2243 .unwrap();
2244 let mut metadata: TableMetadata = serde_json::from_str(&metadata_json).unwrap();
2245
2246 let manifest_list_path =
2247 format!("{manifest_dir}/testdata/manifests_lists/manifest-list-v3-encrypted.avro");
2248 let snapshot = metadata.snapshots.get_mut(&1).unwrap();
2249 let mut patched = snapshot.as_ref().clone();
2250 patched.manifest_list = manifest_list_path;
2251 *snapshot = Arc::new(patched);
2252
2253 metadata
2254 }
2255
2256 #[tokio::test]
2257 async fn catalog_kms_factory_client_reaches_table_encryption_manager() {
2258 let warehouse = temp_path();
2259 let catalog = MemoryCatalogBuilder::default()
2260 .with_storage_factory(Arc::new(LocalFsStorageFactory))
2261 .with_kms_client_factory(Arc::new(fixture_kms_factory()))
2262 .load(
2263 "memory",
2264 HashMap::from([(MEMORY_CATALOG_WAREHOUSE.to_string(), warehouse)]),
2265 )
2266 .await
2267 .unwrap();
2268
2269 let namespace_ident = NamespaceIdent::new("enc_ns".into());
2270 create_namespace(&catalog, &namespace_ident).await;
2271
2272 let metadata = load_encrypted_fixture_metadata();
2273 let metadata_dir = TempDir::new().unwrap();
2274 let metadata_location =
2275 format!("{}/v1.metadata.json", metadata_dir.path().to_str().unwrap());
2276 std::fs::write(&metadata_location, serde_json::to_vec(&metadata).unwrap()).unwrap();
2277
2278 let table_ident = TableIdent::new(namespace_ident, "enc".to_string());
2279 catalog
2280 .register_table(&table_ident, metadata_location)
2281 .await
2282 .unwrap();
2283
2284 let table = catalog.load_table(&table_ident).await.unwrap();
2285 assert!(
2286 table.encryption_manager().is_some(),
2287 "factory-built KMS client should have reached the table's EncryptionManager"
2288 );
2289
2290 let snapshot_ref = table.metadata().current_snapshot().unwrap();
2291 let manifest_list = table
2292 .object_cache()
2293 .get_manifest_list(snapshot_ref, &table.metadata_ref())
2294 .await
2295 .unwrap();
2296 assert_eq!(manifest_list.entries().len(), 0);
2297 }
2298
2299 fn build_table(ident: TableIdent) -> Table {
2300 let file_io = FileIO::new_with_fs();
2301
2302 let temp_dir = TempDir::new().unwrap();
2303 let location = temp_dir.path().to_str().unwrap().to_string();
2304
2305 let table_creation = TableCreation::builder()
2306 .name(ident.name().to_string())
2307 .schema(simple_table_schema())
2308 .location(location)
2309 .build();
2310 let metadata = TableMetadataBuilder::from_table_creation(table_creation)
2311 .unwrap()
2312 .build()
2313 .unwrap()
2314 .metadata;
2315
2316 Table::builder()
2317 .identifier(ident)
2318 .metadata(metadata)
2319 .file_io(file_io)
2320 .runtime(test_runtime())
2321 .build()
2322 .unwrap()
2323 }
2324}