Skip to main content

iceberg/catalog/memory/
catalog.rs

1// Licensed to the Apache Software Foundation (ASF) under one
2// or more contributor license agreements.  See the NOTICE file
3// distributed with this work for additional information
4// regarding copyright ownership.  The ASF licenses this file
5// to you under the Apache License, Version 2.0 (the
6// "License"); you may not use this file except in compliance
7// with the License.  You may obtain a copy of the License at
8//
9//   http://www.apache.org/licenses/LICENSE-2.0
10//
11// Unless required by applicable law or agreed to in writing,
12// software distributed under the License is distributed on an
13// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14// KIND, either express or implied.  See the License for the
15// specific language governing permissions and limitations
16// under the License.
17
18//! This module contains memory catalog implementation.
19
20use std::collections::HashMap;
21use std::str::FromStr;
22use std::sync::Arc;
23
24use async_trait::async_trait;
25use futures::lock::{Mutex, MutexGuard};
26use iceberg_property_macro::Properties;
27use itertools::Itertools;
28
29use super::namespace_state::NamespaceState;
30use crate::encryption::kms::{KeyManagementClient, KmsClientFactory};
31use crate::error::invalid_data;
32use crate::io::{FileIO, FileIOBuilder, MemoryStorageFactory, StorageFactory};
33use crate::runtime::Runtime;
34use crate::spec::{TableMetadata, TableMetadataBuilder};
35use crate::table::Table;
36use crate::{
37    Catalog, CatalogBuilder, MetadataLocation, Namespace, NamespaceIdent, Result, TableCommit,
38    TableCreation, TableIdent, delete_removed_metadata_files,
39};
40
41/// Memory catalog warehouse location
42pub const MEMORY_CATALOG_WAREHOUSE: &str = "warehouse";
43
44/// namespace `location` property
45const LOCATION: &str = "location";
46
47/// Builder for [`MemoryCatalog`].
48#[derive(Debug, Default)]
49pub struct MemoryCatalogBuilder {
50    storage_factory: Option<Arc<dyn StorageFactory>>,
51    kms_client_factory: Option<Arc<dyn KmsClientFactory>>,
52    runtime: Option<Runtime>,
53}
54
55impl CatalogBuilder for MemoryCatalogBuilder {
56    type C = MemoryCatalog;
57
58    fn with_storage_factory(mut self, storage_factory: Arc<dyn StorageFactory>) -> Self {
59        self.storage_factory = Some(storage_factory);
60        self
61    }
62
63    fn with_kms_client_factory(mut self, kms_client_factory: Arc<dyn KmsClientFactory>) -> Self {
64        self.kms_client_factory = Some(kms_client_factory);
65        self
66    }
67
68    fn with_runtime(mut self, runtime: Runtime) -> Self {
69        self.runtime = Some(runtime);
70        self
71    }
72
73    fn load(
74        self,
75        name: impl Into<String>,
76        props: HashMap<String, String>,
77    ) -> impl Future<Output = Result<Self::C>> + Send {
78        let name = name.into();
79
80        async move {
81            let catalog_properties = MemoryCatalogProperties::from_properties(&props)?;
82            if catalog_properties.warehouse.is_empty() {
83                return Err(invalid_data!("Catalog warehouse is required"));
84            }
85
86            let runtime = self.runtime.unwrap_or_else(Runtime::current);
87            let kms_client = match self.kms_client_factory {
88                Some(factory) => Some(factory.create_kms_client(&props).await?),
89                None => None,
90            };
91            MemoryCatalog::new(
92                name,
93                catalog_properties,
94                props,
95                self.storage_factory,
96                runtime,
97                kms_client,
98            )
99        }
100    }
101}
102
103/// Memory catalog properties parsed from a catalog property map.
104#[derive(Debug, Properties)]
105pub(crate) struct MemoryCatalogProperties {
106    #[property(key = MEMORY_CATALOG_WAREHOUSE, default = "")]
107    warehouse: String,
108}
109
110/// Memory catalog implementation.
111pub struct MemoryCatalog {
112    name: String,
113    properties: MemoryCatalogProperties,
114    root_namespace_state: Mutex<NamespaceState>,
115    file_io: FileIO,
116    runtime: Runtime,
117    kms_client: Option<Arc<dyn KeyManagementClient>>,
118}
119
120impl std::fmt::Debug for MemoryCatalog {
121    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
122        f.debug_struct("MemoryCatalog")
123            .field("name", &self.name)
124            .field("properties", &self.properties)
125            .finish_non_exhaustive()
126    }
127}
128
129impl MemoryCatalog {
130    /// Creates a memory catalog.
131    fn new(
132        name: String,
133        properties: MemoryCatalogProperties,
134        props: HashMap<String, String>,
135        storage_factory: Option<Arc<dyn StorageFactory>>,
136        runtime: Runtime,
137        kms_client: Option<Arc<dyn KeyManagementClient>>,
138    ) -> Result<Self> {
139        // Use provided factory or default to MemoryStorageFactory
140        let factory = storage_factory.unwrap_or_else(|| Arc::new(MemoryStorageFactory));
141
142        Ok(Self {
143            name,
144            properties,
145            file_io: FileIOBuilder::new(factory).with_props(props).build(),
146            root_namespace_state: Mutex::new(NamespaceState::default()),
147            runtime,
148            kms_client,
149        })
150    }
151
152    /// Loads a table from the locked namespace state.
153    async fn load_table_from_locked_state(
154        &self,
155        table_ident: &TableIdent,
156        root_namespace_state: &MutexGuard<'_, NamespaceState>,
157    ) -> Result<Table> {
158        let metadata_location = root_namespace_state.get_existing_table_location(table_ident)?;
159        let metadata = TableMetadata::read_from(&self.file_io, metadata_location).await?;
160
161        let mut builder = Table::builder()
162            .identifier(table_ident.clone())
163            .metadata(metadata)
164            .metadata_location(metadata_location.to_string())
165            .file_io(self.file_io.clone())
166            .runtime(self.runtime.clone());
167        if let Some(kms_client) = self.kms_client.clone() {
168            builder = builder.kms_client(kms_client);
169        }
170        builder.build()
171    }
172}
173
174#[async_trait]
175impl Catalog for MemoryCatalog {
176    /// List namespaces inside the catalog.
177    async fn list_namespaces(
178        &self,
179        maybe_parent: Option<&NamespaceIdent>,
180    ) -> Result<Vec<NamespaceIdent>> {
181        let root_namespace_state = self.root_namespace_state.lock().await;
182
183        match maybe_parent {
184            None => {
185                let namespaces = root_namespace_state
186                    .list_top_level_namespaces()
187                    .into_iter()
188                    .map(|str| NamespaceIdent::new(str.to_string()))
189                    .collect_vec();
190
191                Ok(namespaces)
192            }
193            Some(parent_namespace_ident) => {
194                let namespaces = root_namespace_state
195                    .list_namespaces_under(parent_namespace_ident)?
196                    .into_iter()
197                    .map(|name| {
198                        let mut names = parent_namespace_ident.iter().cloned().collect::<Vec<_>>();
199                        names.push(name.to_string());
200                        NamespaceIdent::from_vec(names)
201                    })
202                    .collect::<Result<Vec<_>>>()?;
203
204                Ok(namespaces)
205            }
206        }
207    }
208
209    /// Create a new namespace inside the catalog.
210    async fn create_namespace(
211        &self,
212        namespace_ident: &NamespaceIdent,
213        properties: HashMap<String, String>,
214    ) -> Result<Namespace> {
215        let mut root_namespace_state = self.root_namespace_state.lock().await;
216
217        root_namespace_state.insert_new_namespace(namespace_ident, properties.clone())?;
218        let namespace = Namespace::with_properties(namespace_ident.clone(), properties);
219
220        Ok(namespace)
221    }
222
223    /// Get a namespace information from the catalog.
224    async fn get_namespace(&self, namespace_ident: &NamespaceIdent) -> Result<Namespace> {
225        let root_namespace_state = self.root_namespace_state.lock().await;
226
227        let namespace = Namespace::with_properties(
228            namespace_ident.clone(),
229            root_namespace_state
230                .get_properties(namespace_ident)?
231                .clone(),
232        );
233
234        Ok(namespace)
235    }
236
237    /// Check if namespace exists in catalog.
238    async fn namespace_exists(&self, namespace_ident: &NamespaceIdent) -> Result<bool> {
239        let guarded_namespaces = self.root_namespace_state.lock().await;
240
241        Ok(guarded_namespaces.namespace_exists(namespace_ident))
242    }
243
244    /// Update a namespace inside the catalog.
245    ///
246    /// # Behavior
247    ///
248    /// The properties must be the full set of namespace.
249    async fn update_namespace(
250        &self,
251        namespace_ident: &NamespaceIdent,
252        properties: HashMap<String, String>,
253    ) -> Result<()> {
254        let mut root_namespace_state = self.root_namespace_state.lock().await;
255
256        root_namespace_state.replace_properties(namespace_ident, properties)
257    }
258
259    /// Drop a namespace from the catalog.
260    async fn drop_namespace(&self, namespace_ident: &NamespaceIdent) -> Result<()> {
261        let mut root_namespace_state = self.root_namespace_state.lock().await;
262
263        root_namespace_state.remove_existing_namespace(namespace_ident)
264    }
265
266    /// List tables from namespace.
267    async fn list_tables(&self, namespace_ident: &NamespaceIdent) -> Result<Vec<TableIdent>> {
268        let root_namespace_state = self.root_namespace_state.lock().await;
269
270        let table_names = root_namespace_state.list_tables(namespace_ident)?;
271        let table_idents = table_names
272            .into_iter()
273            .map(|table_name| TableIdent::new(namespace_ident.clone(), table_name.clone()))
274            .collect_vec();
275
276        Ok(table_idents)
277    }
278
279    /// Create a new table inside the namespace.
280    async fn create_table(
281        &self,
282        namespace_ident: &NamespaceIdent,
283        table_creation: TableCreation,
284    ) -> Result<Table> {
285        let mut root_namespace_state = self.root_namespace_state.lock().await;
286
287        let table_name = table_creation.name.clone();
288        let table_ident = TableIdent::new(namespace_ident.clone(), table_name);
289
290        let table_creation = if table_creation.location.is_some() {
291            table_creation
292        } else {
293            let namespace_properties = root_namespace_state.get_properties(namespace_ident)?;
294            let location_prefix = match namespace_properties.get(LOCATION) {
295                Some(namespace_location) => namespace_location.clone(),
296                None => format!(
297                    "{}/{}",
298                    self.properties.warehouse,
299                    namespace_ident.join("/")
300                ),
301            };
302
303            let location = format!("{}/{}", location_prefix, table_ident.name());
304
305            TableCreation {
306                location: Some(location),
307                ..table_creation
308            }
309        };
310
311        let metadata = TableMetadataBuilder::from_table_creation(table_creation)?
312            .build()?
313            .metadata;
314        let metadata_location = MetadataLocation::try_new_with_metadata(&metadata)?;
315
316        metadata.write_to(&self.file_io, &metadata_location).await?;
317
318        root_namespace_state.insert_new_table(&table_ident, metadata_location.to_string())?;
319
320        let mut builder = Table::builder()
321            .file_io(self.file_io.clone())
322            .metadata_location(metadata_location.to_string())
323            .metadata(metadata)
324            .identifier(table_ident)
325            .runtime(self.runtime.clone());
326        if let Some(kms_client) = self.kms_client.clone() {
327            builder = builder.kms_client(kms_client);
328        }
329        builder.build()
330    }
331
332    /// Load table from the catalog.
333    async fn load_table(&self, table_ident: &TableIdent) -> Result<Table> {
334        let root_namespace_state = self.root_namespace_state.lock().await;
335
336        self.load_table_from_locked_state(table_ident, &root_namespace_state)
337            .await
338    }
339
340    /// Drop a table from the catalog.
341    async fn drop_table(&self, table_ident: &TableIdent) -> Result<()> {
342        let mut root_namespace_state = self.root_namespace_state.lock().await;
343
344        root_namespace_state.remove_existing_table(table_ident)?;
345        Ok(())
346    }
347
348    async fn purge_table(&self, table_ident: &TableIdent) -> Result<()> {
349        let table_info = self.load_table(table_ident).await?;
350        self.drop_table(table_ident).await?;
351        crate::catalog::utils::drop_table_data(&table_info).await
352    }
353
354    /// Check if a table exists in the catalog.
355    async fn table_exists(&self, table_ident: &TableIdent) -> Result<bool> {
356        let root_namespace_state = self.root_namespace_state.lock().await;
357
358        root_namespace_state.table_exists(table_ident)
359    }
360
361    /// Rename a table in the catalog.
362    async fn rename_table(
363        &self,
364        src_table_ident: &TableIdent,
365        dst_table_ident: &TableIdent,
366    ) -> Result<()> {
367        let mut root_namespace_state = self.root_namespace_state.lock().await;
368
369        let mut new_root_namespace_state = root_namespace_state.clone();
370        let metadata_location = new_root_namespace_state
371            .get_existing_table_location(src_table_ident)?
372            .clone();
373        new_root_namespace_state.remove_existing_table(src_table_ident)?;
374        new_root_namespace_state.insert_new_table(dst_table_ident, metadata_location)?;
375        *root_namespace_state = new_root_namespace_state;
376
377        Ok(())
378    }
379
380    async fn register_table(
381        &self,
382        table_ident: &TableIdent,
383        metadata_location: String,
384    ) -> Result<Table> {
385        let mut root_namespace_state = self.root_namespace_state.lock().await;
386        root_namespace_state.insert_new_table(&table_ident.clone(), metadata_location.clone())?;
387
388        let metadata = TableMetadata::read_from(&self.file_io, &metadata_location).await?;
389
390        let mut builder = Table::builder()
391            .file_io(self.file_io.clone())
392            .metadata_location(metadata_location)
393            .metadata(metadata)
394            .identifier(table_ident.clone())
395            .runtime(self.runtime.clone());
396        if let Some(kms_client) = self.kms_client.clone() {
397            builder = builder.kms_client(kms_client);
398        }
399        builder.build()
400    }
401
402    /// Update a table in the catalog.
403    async fn update_table(&self, commit: TableCommit) -> Result<Table> {
404        let mut root_namespace_state = self.root_namespace_state.lock().await;
405
406        let current_table = self
407            .load_table_from_locked_state(commit.identifier(), &root_namespace_state)
408            .await?;
409        let base_metadata = current_table.metadata_ref();
410
411        // Apply TableCommit to get staged table
412        let staged_table = commit.apply(current_table)?;
413
414        // Write table metadata to the new location
415        let metadata_location =
416            MetadataLocation::from_str(staged_table.metadata_location_result()?)?;
417        staged_table
418            .metadata()
419            .write_to(staged_table.file_io(), &metadata_location)
420            .await?;
421
422        // Flip the pointer to reference the new metadata file.
423        let updated_table = root_namespace_state.commit_table_update(staged_table)?;
424        drop(root_namespace_state);
425
426        delete_removed_metadata_files(
427            updated_table.file_io(),
428            &base_metadata,
429            updated_table.metadata(),
430        )
431        .await;
432
433        Ok(updated_table)
434    }
435}
436
437#[cfg(test)]
438pub(crate) mod tests {
439    use std::collections::HashSet;
440    use std::hash::Hash;
441    use std::iter::FromIterator;
442    use std::vec;
443
444    use regex::Regex;
445    use tempfile::TempDir;
446
447    use super::*;
448    use crate::ErrorKind;
449    use crate::encryption::kms::MemoryKmsClientFactory;
450    use crate::io::{FileIO, LocalFsStorageFactory};
451    use crate::spec::{
452        NestedField, PartitionSpec, PrimitiveType, Schema, SortOrder, TableProperties, Type,
453    };
454    use crate::test_utils::test_runtime;
455    use crate::transaction::{ApplyTransactionAction, Transaction};
456
457    fn temp_path() -> String {
458        let temp_dir = TempDir::new().unwrap();
459        temp_dir.path().to_str().unwrap().to_string()
460    }
461
462    #[test]
463    fn test_catalog_properties() {
464        let properties = MemoryCatalogProperties::from_properties(&HashMap::from([(
465            MEMORY_CATALOG_WAREHOUSE.to_string(),
466            "memory:///warehouse".to_string(),
467        )]))
468        .unwrap();
469
470        assert_eq!(properties.warehouse, "memory:///warehouse");
471    }
472
473    #[test]
474    fn test_catalog_properties_warehouse_defaults_to_empty() {
475        let missing = MemoryCatalogProperties::from_properties(&HashMap::new()).unwrap();
476        let explicitly_empty = MemoryCatalogProperties::from_properties(&HashMap::from([(
477            MEMORY_CATALOG_WAREHOUSE.to_string(),
478            String::new(),
479        )]))
480        .unwrap();
481
482        assert_eq!(missing.warehouse, "");
483        assert_eq!(explicitly_empty.warehouse, "");
484    }
485
486    #[tokio::test]
487    async fn test_catalog_forwards_properties_to_file_io() {
488        let catalog = MemoryCatalogBuilder::default()
489            .load(
490                "memory",
491                HashMap::from([
492                    (
493                        MEMORY_CATALOG_WAREHOUSE.to_string(),
494                        "memory:///warehouse".to_string(),
495                    ),
496                    ("custom.property".to_string(), "value".to_string()),
497                ]),
498            )
499            .await
500            .unwrap();
501        let file_io_props = catalog.file_io.config().props();
502
503        assert_eq!(
504            file_io_props.get("custom.property"),
505            Some(&"value".to_string())
506        );
507        assert_eq!(
508            file_io_props.get(MEMORY_CATALOG_WAREHOUSE),
509            Some(&"memory:///warehouse".to_string())
510        );
511    }
512
513    pub(crate) async fn new_memory_catalog() -> impl Catalog {
514        let warehouse_location = temp_path();
515        MemoryCatalogBuilder::default()
516            .load(
517                "memory",
518                HashMap::from([(MEMORY_CATALOG_WAREHOUSE.to_string(), warehouse_location)]),
519            )
520            .await
521            .unwrap()
522    }
523
524    async fn create_namespace<C: Catalog>(catalog: &C, namespace_ident: &NamespaceIdent) {
525        let _ = catalog
526            .create_namespace(namespace_ident, HashMap::new())
527            .await
528            .unwrap();
529    }
530
531    async fn create_namespaces<C: Catalog>(catalog: &C, namespace_idents: &Vec<&NamespaceIdent>) {
532        for namespace_ident in namespace_idents {
533            let _ = create_namespace(catalog, namespace_ident).await;
534        }
535    }
536
537    fn to_set<T: Eq + Hash>(vec: Vec<T>) -> HashSet<T> {
538        HashSet::from_iter(vec)
539    }
540
541    fn simple_table_schema() -> Schema {
542        Schema::builder()
543            .with_fields(vec![
544                NestedField::required(1, "foo", Type::Primitive(PrimitiveType::Int)).into(),
545            ])
546            .build()
547            .unwrap()
548    }
549
550    async fn create_table<C: Catalog>(catalog: &C, table_ident: &TableIdent) -> Table {
551        catalog
552            .create_table(
553                &table_ident.namespace,
554                TableCreation::builder()
555                    .name(table_ident.name().into())
556                    .schema(simple_table_schema())
557                    .build(),
558            )
559            .await
560            .unwrap()
561    }
562
563    async fn create_tables<C: Catalog>(catalog: &C, table_idents: Vec<&TableIdent>) {
564        for table_ident in table_idents {
565            create_table(catalog, table_ident).await;
566        }
567    }
568
569    async fn create_table_with_namespace<C: Catalog>(catalog: &C) -> Table {
570        let namespace_ident = NamespaceIdent::new("abc".into());
571        create_namespace(catalog, &namespace_ident).await;
572
573        let table_ident = TableIdent::new(namespace_ident, "test".to_string());
574        create_table(catalog, &table_ident).await
575    }
576
577    fn assert_table_eq(table: &Table, expected_table_ident: &TableIdent, expected_schema: &Schema) {
578        assert_eq!(table.identifier(), expected_table_ident);
579
580        let metadata = table.metadata();
581
582        assert_eq!(metadata.current_schema().as_ref(), expected_schema);
583
584        let expected_partition_spec = PartitionSpec::builder((*expected_schema).clone())
585            .with_spec_id(0)
586            .build()
587            .unwrap();
588
589        assert_eq!(
590            metadata
591                .partition_specs_iter()
592                .map(|p| p.as_ref())
593                .collect_vec(),
594            vec![&expected_partition_spec]
595        );
596
597        let expected_sorted_order = SortOrder::builder()
598            .with_order_id(0)
599            .with_fields(vec![])
600            .build(expected_schema)
601            .unwrap();
602
603        assert_eq!(
604            metadata
605                .sort_orders_iter()
606                .map(|s| s.as_ref())
607                .collect_vec(),
608            vec![&expected_sorted_order]
609        );
610
611        assert_eq!(metadata.properties(), &HashMap::new());
612
613        assert!(!table.readonly());
614    }
615
616    const UUID_REGEX_STR: &str = "[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}";
617
618    fn assert_table_metadata_location_matches(table: &Table, regex_str: &str) {
619        let actual = table.metadata_location().unwrap().to_string();
620        let regex = Regex::new(regex_str).unwrap();
621        assert!(
622            regex.is_match(&actual),
623            "Expected metadata location to match regex, but got location: {actual} and regex: {regex}"
624        )
625    }
626
627    #[tokio::test]
628    async fn test_list_namespaces_returns_empty_vector() {
629        let catalog = new_memory_catalog().await;
630
631        assert_eq!(catalog.list_namespaces(None).await.unwrap(), vec![]);
632    }
633
634    #[tokio::test]
635    async fn test_list_namespaces_returns_single_namespace() {
636        let catalog = new_memory_catalog().await;
637        let namespace_ident = NamespaceIdent::new("abc".into());
638        create_namespace(&catalog, &namespace_ident).await;
639
640        assert_eq!(catalog.list_namespaces(None).await.unwrap(), vec![
641            namespace_ident
642        ]);
643    }
644
645    #[tokio::test]
646    async fn test_list_namespaces_returns_multiple_namespaces() {
647        let catalog = new_memory_catalog().await;
648        let namespace_ident_1 = NamespaceIdent::new("a".into());
649        let namespace_ident_2 = NamespaceIdent::new("b".into());
650        create_namespaces(&catalog, &vec![&namespace_ident_1, &namespace_ident_2]).await;
651
652        assert_eq!(
653            to_set(catalog.list_namespaces(None).await.unwrap()),
654            to_set(vec![namespace_ident_1, namespace_ident_2])
655        );
656    }
657
658    #[tokio::test]
659    async fn test_list_namespaces_returns_only_top_level_namespaces() {
660        let catalog = new_memory_catalog().await;
661        let namespace_ident_1 = NamespaceIdent::new("a".into());
662        let namespace_ident_2 = NamespaceIdent::from_strs(vec!["a", "b"]).unwrap();
663        let namespace_ident_3 = NamespaceIdent::new("b".into());
664        create_namespaces(&catalog, &vec![
665            &namespace_ident_1,
666            &namespace_ident_2,
667            &namespace_ident_3,
668        ])
669        .await;
670
671        assert_eq!(
672            to_set(catalog.list_namespaces(None).await.unwrap()),
673            to_set(vec![namespace_ident_1, namespace_ident_3])
674        );
675    }
676
677    #[tokio::test]
678    async fn test_list_namespaces_returns_no_namespaces_under_parent() {
679        let catalog = new_memory_catalog().await;
680        let namespace_ident_1 = NamespaceIdent::new("a".into());
681        let namespace_ident_2 = NamespaceIdent::new("b".into());
682        create_namespaces(&catalog, &vec![&namespace_ident_1, &namespace_ident_2]).await;
683
684        assert_eq!(
685            catalog
686                .list_namespaces(Some(&namespace_ident_1))
687                .await
688                .unwrap(),
689            vec![]
690        );
691    }
692
693    #[tokio::test]
694    async fn test_list_namespaces_returns_namespace_under_parent() {
695        let catalog = new_memory_catalog().await;
696        let namespace_ident_1 = NamespaceIdent::new("a".into());
697        let namespace_ident_2 = NamespaceIdent::from_strs(vec!["a", "b"]).unwrap();
698        let namespace_ident_3 = NamespaceIdent::new("c".into());
699        create_namespaces(&catalog, &vec![
700            &namespace_ident_1,
701            &namespace_ident_2,
702            &namespace_ident_3,
703        ])
704        .await;
705
706        assert_eq!(
707            to_set(catalog.list_namespaces(None).await.unwrap()),
708            to_set(vec![namespace_ident_1.clone(), namespace_ident_3])
709        );
710
711        assert_eq!(
712            catalog
713                .list_namespaces(Some(&namespace_ident_1))
714                .await
715                .unwrap(),
716            vec![namespace_ident_2]
717        );
718    }
719
720    #[tokio::test]
721    async fn test_list_namespaces_returns_multiple_namespaces_under_parent() {
722        let catalog = new_memory_catalog().await;
723        let namespace_ident_1 = NamespaceIdent::new("a".to_string());
724        let namespace_ident_2 = NamespaceIdent::from_strs(vec!["a", "a"]).unwrap();
725        let namespace_ident_3 = NamespaceIdent::from_strs(vec!["a", "b"]).unwrap();
726        let namespace_ident_4 = NamespaceIdent::from_strs(vec!["a", "c"]).unwrap();
727        let namespace_ident_5 = NamespaceIdent::new("b".into());
728        create_namespaces(&catalog, &vec![
729            &namespace_ident_1,
730            &namespace_ident_2,
731            &namespace_ident_3,
732            &namespace_ident_4,
733            &namespace_ident_5,
734        ])
735        .await;
736
737        assert_eq!(
738            to_set(
739                catalog
740                    .list_namespaces(Some(&namespace_ident_1))
741                    .await
742                    .unwrap()
743            ),
744            to_set(vec![
745                namespace_ident_2,
746                namespace_ident_3,
747                namespace_ident_4,
748            ])
749        );
750    }
751
752    #[tokio::test]
753    async fn test_namespace_exists_returns_false() {
754        let catalog = new_memory_catalog().await;
755        let namespace_ident = NamespaceIdent::new("a".into());
756        create_namespace(&catalog, &namespace_ident).await;
757
758        assert!(
759            !catalog
760                .namespace_exists(&NamespaceIdent::new("b".into()))
761                .await
762                .unwrap()
763        );
764    }
765
766    #[tokio::test]
767    async fn test_namespace_exists_returns_true() {
768        let catalog = new_memory_catalog().await;
769        let namespace_ident = NamespaceIdent::new("a".into());
770        create_namespace(&catalog, &namespace_ident).await;
771
772        assert!(catalog.namespace_exists(&namespace_ident).await.unwrap());
773    }
774
775    #[tokio::test]
776    async fn test_create_namespace_with_empty_properties() {
777        let catalog = new_memory_catalog().await;
778        let namespace_ident = NamespaceIdent::new("a".into());
779
780        assert_eq!(
781            catalog
782                .create_namespace(&namespace_ident, HashMap::new())
783                .await
784                .unwrap(),
785            Namespace::new(namespace_ident.clone())
786        );
787
788        assert_eq!(
789            catalog.get_namespace(&namespace_ident).await.unwrap(),
790            Namespace::with_properties(namespace_ident, HashMap::new())
791        );
792    }
793
794    #[tokio::test]
795    async fn test_create_namespace_with_properties() {
796        let catalog = new_memory_catalog().await;
797        let namespace_ident = NamespaceIdent::new("abc".into());
798
799        let mut properties: HashMap<String, String> = HashMap::new();
800        properties.insert("k".into(), "v".into());
801
802        assert_eq!(
803            catalog
804                .create_namespace(&namespace_ident, properties.clone())
805                .await
806                .unwrap(),
807            Namespace::with_properties(namespace_ident.clone(), properties.clone())
808        );
809
810        assert_eq!(
811            catalog.get_namespace(&namespace_ident).await.unwrap(),
812            Namespace::with_properties(namespace_ident, properties)
813        );
814    }
815
816    #[tokio::test]
817    async fn test_create_namespace_throws_error_if_namespace_already_exists() {
818        let catalog = new_memory_catalog().await;
819        let namespace_ident = NamespaceIdent::new("a".into());
820        create_namespace(&catalog, &namespace_ident).await;
821
822        assert_eq!(
823            catalog
824                .create_namespace(&namespace_ident, HashMap::new())
825                .await
826                .unwrap_err()
827                .to_string(),
828            format!(
829                "NamespaceAlreadyExists => Cannot create namespace {:?}. Namespace already exists.",
830                &namespace_ident
831            )
832        );
833
834        assert_eq!(
835            catalog.get_namespace(&namespace_ident).await.unwrap(),
836            Namespace::with_properties(namespace_ident, HashMap::new())
837        );
838    }
839
840    #[tokio::test]
841    async fn test_create_nested_namespace() {
842        let catalog = new_memory_catalog().await;
843        let parent_namespace_ident = NamespaceIdent::new("a".into());
844        create_namespace(&catalog, &parent_namespace_ident).await;
845
846        let child_namespace_ident = NamespaceIdent::from_strs(vec!["a", "b"]).unwrap();
847
848        assert_eq!(
849            catalog
850                .create_namespace(&child_namespace_ident, HashMap::new())
851                .await
852                .unwrap(),
853            Namespace::new(child_namespace_ident.clone())
854        );
855
856        assert_eq!(
857            catalog.get_namespace(&child_namespace_ident).await.unwrap(),
858            Namespace::with_properties(child_namespace_ident, HashMap::new())
859        );
860    }
861
862    #[tokio::test]
863    async fn test_create_deeply_nested_namespace() {
864        let catalog = new_memory_catalog().await;
865        let namespace_ident_a = NamespaceIdent::new("a".into());
866        let namespace_ident_a_b = NamespaceIdent::from_strs(vec!["a", "b"]).unwrap();
867        create_namespaces(&catalog, &vec![&namespace_ident_a, &namespace_ident_a_b]).await;
868
869        let namespace_ident_a_b_c = NamespaceIdent::from_strs(vec!["a", "b", "c"]).unwrap();
870
871        assert_eq!(
872            catalog
873                .create_namespace(&namespace_ident_a_b_c, HashMap::new())
874                .await
875                .unwrap(),
876            Namespace::new(namespace_ident_a_b_c.clone())
877        );
878
879        assert_eq!(
880            catalog.get_namespace(&namespace_ident_a_b_c).await.unwrap(),
881            Namespace::with_properties(namespace_ident_a_b_c, HashMap::new())
882        );
883    }
884
885    #[tokio::test]
886    async fn test_create_nested_namespace_throws_error_if_top_level_namespace_doesnt_exist() {
887        let catalog = new_memory_catalog().await;
888
889        let nested_namespace_ident = NamespaceIdent::from_strs(vec!["a", "b"]).unwrap();
890
891        assert_eq!(
892            catalog
893                .create_namespace(&nested_namespace_ident, HashMap::new())
894                .await
895                .unwrap_err()
896                .to_string(),
897            format!(
898                "NamespaceNotFound => No such namespace: {:?}",
899                NamespaceIdent::new("a".into())
900            )
901        );
902
903        assert_eq!(catalog.list_namespaces(None).await.unwrap(), vec![]);
904    }
905
906    #[tokio::test]
907    async fn test_create_deeply_nested_namespace_throws_error_if_intermediate_namespace_doesnt_exist()
908     {
909        let catalog = new_memory_catalog().await;
910
911        let namespace_ident_a = NamespaceIdent::new("a".into());
912        create_namespace(&catalog, &namespace_ident_a).await;
913
914        let namespace_ident_a_b_c = NamespaceIdent::from_strs(vec!["a", "b", "c"]).unwrap();
915
916        assert_eq!(
917            catalog
918                .create_namespace(&namespace_ident_a_b_c, HashMap::new())
919                .await
920                .unwrap_err()
921                .to_string(),
922            format!(
923                "NamespaceNotFound => No such namespace: {:?}",
924                NamespaceIdent::from_strs(vec!["a", "b"]).unwrap()
925            )
926        );
927
928        assert_eq!(catalog.list_namespaces(None).await.unwrap(), vec![
929            namespace_ident_a.clone()
930        ]);
931
932        assert_eq!(
933            catalog
934                .list_namespaces(Some(&namespace_ident_a))
935                .await
936                .unwrap(),
937            vec![]
938        );
939    }
940
941    #[tokio::test]
942    async fn test_get_namespace() {
943        let catalog = new_memory_catalog().await;
944        let namespace_ident = NamespaceIdent::new("abc".into());
945
946        let mut properties: HashMap<String, String> = HashMap::new();
947        properties.insert("k".into(), "v".into());
948        let _ = catalog
949            .create_namespace(&namespace_ident, properties.clone())
950            .await
951            .unwrap();
952
953        assert_eq!(
954            catalog.get_namespace(&namespace_ident).await.unwrap(),
955            Namespace::with_properties(namespace_ident, properties)
956        )
957    }
958
959    #[tokio::test]
960    async fn test_get_nested_namespace() {
961        let catalog = new_memory_catalog().await;
962        let namespace_ident_a = NamespaceIdent::new("a".into());
963        let namespace_ident_a_b = NamespaceIdent::from_strs(vec!["a", "b"]).unwrap();
964        create_namespaces(&catalog, &vec![&namespace_ident_a, &namespace_ident_a_b]).await;
965
966        assert_eq!(
967            catalog.get_namespace(&namespace_ident_a_b).await.unwrap(),
968            Namespace::with_properties(namespace_ident_a_b, HashMap::new())
969        );
970    }
971
972    #[tokio::test]
973    async fn test_get_deeply_nested_namespace() {
974        let catalog = new_memory_catalog().await;
975        let namespace_ident_a = NamespaceIdent::new("a".into());
976        let namespace_ident_a_b = NamespaceIdent::from_strs(vec!["a", "b"]).unwrap();
977        let namespace_ident_a_b_c = NamespaceIdent::from_strs(vec!["a", "b", "c"]).unwrap();
978        create_namespaces(&catalog, &vec![
979            &namespace_ident_a,
980            &namespace_ident_a_b,
981            &namespace_ident_a_b_c,
982        ])
983        .await;
984
985        assert_eq!(
986            catalog.get_namespace(&namespace_ident_a_b_c).await.unwrap(),
987            Namespace::with_properties(namespace_ident_a_b_c, HashMap::new())
988        );
989    }
990
991    #[tokio::test]
992    async fn test_get_namespace_throws_error_if_namespace_doesnt_exist() {
993        let catalog = new_memory_catalog().await;
994        create_namespace(&catalog, &NamespaceIdent::new("a".into())).await;
995
996        let non_existent_namespace_ident = NamespaceIdent::new("b".into());
997        assert_eq!(
998            catalog
999                .get_namespace(&non_existent_namespace_ident)
1000                .await
1001                .unwrap_err()
1002                .to_string(),
1003            format!("NamespaceNotFound => No such namespace: {non_existent_namespace_ident:?}")
1004        )
1005    }
1006
1007    #[tokio::test]
1008    async fn test_update_namespace() {
1009        let catalog = new_memory_catalog().await;
1010        let namespace_ident = NamespaceIdent::new("abc".into());
1011        create_namespace(&catalog, &namespace_ident).await;
1012
1013        let mut new_properties: HashMap<String, String> = HashMap::new();
1014        new_properties.insert("k".into(), "v".into());
1015
1016        catalog
1017            .update_namespace(&namespace_ident, new_properties.clone())
1018            .await
1019            .unwrap();
1020
1021        assert_eq!(
1022            catalog.get_namespace(&namespace_ident).await.unwrap(),
1023            Namespace::with_properties(namespace_ident, new_properties)
1024        )
1025    }
1026
1027    #[tokio::test]
1028    async fn test_update_nested_namespace() {
1029        let catalog = new_memory_catalog().await;
1030        let namespace_ident_a = NamespaceIdent::new("a".into());
1031        let namespace_ident_a_b = NamespaceIdent::from_strs(vec!["a", "b"]).unwrap();
1032        create_namespaces(&catalog, &vec![&namespace_ident_a, &namespace_ident_a_b]).await;
1033
1034        let mut new_properties = HashMap::new();
1035        new_properties.insert("k".into(), "v".into());
1036
1037        catalog
1038            .update_namespace(&namespace_ident_a_b, new_properties.clone())
1039            .await
1040            .unwrap();
1041
1042        assert_eq!(
1043            catalog.get_namespace(&namespace_ident_a_b).await.unwrap(),
1044            Namespace::with_properties(namespace_ident_a_b, new_properties)
1045        );
1046    }
1047
1048    #[tokio::test]
1049    async fn test_update_deeply_nested_namespace() {
1050        let catalog = new_memory_catalog().await;
1051        let namespace_ident_a = NamespaceIdent::new("a".into());
1052        let namespace_ident_a_b = NamespaceIdent::from_strs(vec!["a", "b"]).unwrap();
1053        let namespace_ident_a_b_c = NamespaceIdent::from_strs(vec!["a", "b", "c"]).unwrap();
1054        create_namespaces(&catalog, &vec![
1055            &namespace_ident_a,
1056            &namespace_ident_a_b,
1057            &namespace_ident_a_b_c,
1058        ])
1059        .await;
1060
1061        let mut new_properties = HashMap::new();
1062        new_properties.insert("k".into(), "v".into());
1063
1064        catalog
1065            .update_namespace(&namespace_ident_a_b_c, new_properties.clone())
1066            .await
1067            .unwrap();
1068
1069        assert_eq!(
1070            catalog.get_namespace(&namespace_ident_a_b_c).await.unwrap(),
1071            Namespace::with_properties(namespace_ident_a_b_c, new_properties)
1072        );
1073    }
1074
1075    #[tokio::test]
1076    async fn test_update_namespace_throws_error_if_namespace_doesnt_exist() {
1077        let catalog = new_memory_catalog().await;
1078        create_namespace(&catalog, &NamespaceIdent::new("abc".into())).await;
1079
1080        let non_existent_namespace_ident = NamespaceIdent::new("def".into());
1081        assert_eq!(
1082            catalog
1083                .update_namespace(&non_existent_namespace_ident, HashMap::new())
1084                .await
1085                .unwrap_err()
1086                .to_string(),
1087            format!("NamespaceNotFound => No such namespace: {non_existent_namespace_ident:?}")
1088        )
1089    }
1090
1091    #[tokio::test]
1092    async fn test_drop_namespace() {
1093        let catalog = new_memory_catalog().await;
1094        let namespace_ident = NamespaceIdent::new("abc".into());
1095        create_namespace(&catalog, &namespace_ident).await;
1096
1097        catalog.drop_namespace(&namespace_ident).await.unwrap();
1098
1099        assert!(!catalog.namespace_exists(&namespace_ident).await.unwrap())
1100    }
1101
1102    #[tokio::test]
1103    async fn test_drop_nested_namespace() {
1104        let catalog = new_memory_catalog().await;
1105        let namespace_ident_a = NamespaceIdent::new("a".into());
1106        let namespace_ident_a_b = NamespaceIdent::from_strs(vec!["a", "b"]).unwrap();
1107        create_namespaces(&catalog, &vec![&namespace_ident_a, &namespace_ident_a_b]).await;
1108
1109        catalog.drop_namespace(&namespace_ident_a_b).await.unwrap();
1110
1111        assert!(
1112            !catalog
1113                .namespace_exists(&namespace_ident_a_b)
1114                .await
1115                .unwrap()
1116        );
1117
1118        assert!(catalog.namespace_exists(&namespace_ident_a).await.unwrap());
1119    }
1120
1121    #[tokio::test]
1122    async fn test_drop_deeply_nested_namespace() {
1123        let catalog = new_memory_catalog().await;
1124        let namespace_ident_a = NamespaceIdent::new("a".into());
1125        let namespace_ident_a_b = NamespaceIdent::from_strs(vec!["a", "b"]).unwrap();
1126        let namespace_ident_a_b_c = NamespaceIdent::from_strs(vec!["a", "b", "c"]).unwrap();
1127        create_namespaces(&catalog, &vec![
1128            &namespace_ident_a,
1129            &namespace_ident_a_b,
1130            &namespace_ident_a_b_c,
1131        ])
1132        .await;
1133
1134        catalog
1135            .drop_namespace(&namespace_ident_a_b_c)
1136            .await
1137            .unwrap();
1138
1139        assert!(
1140            !catalog
1141                .namespace_exists(&namespace_ident_a_b_c)
1142                .await
1143                .unwrap()
1144        );
1145
1146        assert!(
1147            catalog
1148                .namespace_exists(&namespace_ident_a_b)
1149                .await
1150                .unwrap()
1151        );
1152
1153        assert!(catalog.namespace_exists(&namespace_ident_a).await.unwrap());
1154    }
1155
1156    #[tokio::test]
1157    async fn test_drop_namespace_throws_error_if_namespace_doesnt_exist() {
1158        let catalog = new_memory_catalog().await;
1159
1160        let non_existent_namespace_ident = NamespaceIdent::new("abc".into());
1161        assert_eq!(
1162            catalog
1163                .drop_namespace(&non_existent_namespace_ident)
1164                .await
1165                .unwrap_err()
1166                .to_string(),
1167            format!("NamespaceNotFound => No such namespace: {non_existent_namespace_ident:?}")
1168        )
1169    }
1170
1171    #[tokio::test]
1172    async fn test_drop_namespace_throws_error_if_nested_namespace_doesnt_exist() {
1173        let catalog = new_memory_catalog().await;
1174        create_namespace(&catalog, &NamespaceIdent::new("a".into())).await;
1175
1176        let non_existent_namespace_ident =
1177            NamespaceIdent::from_vec(vec!["a".into(), "b".into()]).unwrap();
1178        assert_eq!(
1179            catalog
1180                .drop_namespace(&non_existent_namespace_ident)
1181                .await
1182                .unwrap_err()
1183                .to_string(),
1184            format!("NamespaceNotFound => No such namespace: {non_existent_namespace_ident:?}")
1185        )
1186    }
1187
1188    #[tokio::test]
1189    async fn test_dropping_a_namespace_also_drops_namespaces_nested_under_that_one() {
1190        let catalog = new_memory_catalog().await;
1191        let namespace_ident_a = NamespaceIdent::new("a".into());
1192        let namespace_ident_a_b = NamespaceIdent::from_strs(vec!["a", "b"]).unwrap();
1193        create_namespaces(&catalog, &vec![&namespace_ident_a, &namespace_ident_a_b]).await;
1194
1195        catalog.drop_namespace(&namespace_ident_a).await.unwrap();
1196
1197        assert!(!catalog.namespace_exists(&namespace_ident_a).await.unwrap());
1198
1199        assert!(
1200            !catalog
1201                .namespace_exists(&namespace_ident_a_b)
1202                .await
1203                .unwrap()
1204        );
1205    }
1206
1207    #[tokio::test]
1208    async fn test_create_table_with_location() {
1209        let tmp_dir = TempDir::new().unwrap();
1210        let catalog = new_memory_catalog().await;
1211        let namespace_ident = NamespaceIdent::new("a".into());
1212        create_namespace(&catalog, &namespace_ident).await;
1213
1214        let table_name = "abc";
1215        let location = tmp_dir.path().to_str().unwrap().to_string();
1216        let table_creation = TableCreation::builder()
1217            .name(table_name.into())
1218            .location(location.clone())
1219            .schema(simple_table_schema())
1220            .build();
1221
1222        let expected_table_ident = TableIdent::new(namespace_ident.clone(), table_name.into());
1223
1224        assert_table_eq(
1225            &catalog
1226                .create_table(&namespace_ident, table_creation)
1227                .await
1228                .unwrap(),
1229            &expected_table_ident,
1230            &simple_table_schema(),
1231        );
1232
1233        let table = catalog.load_table(&expected_table_ident).await.unwrap();
1234
1235        assert_table_eq(&table, &expected_table_ident, &simple_table_schema());
1236
1237        assert!(
1238            table
1239                .metadata_location()
1240                .unwrap()
1241                .to_string()
1242                .starts_with(&location)
1243        )
1244    }
1245
1246    #[tokio::test]
1247    async fn test_create_table_falls_back_to_namespace_location_if_table_location_is_missing() {
1248        let warehouse_location = temp_path();
1249        let catalog = MemoryCatalogBuilder::default()
1250            .load(
1251                "memory",
1252                HashMap::from([(
1253                    MEMORY_CATALOG_WAREHOUSE.to_string(),
1254                    warehouse_location.clone(),
1255                )]),
1256            )
1257            .await
1258            .unwrap();
1259
1260        let namespace_ident = NamespaceIdent::new("a".into());
1261        let mut namespace_properties = HashMap::new();
1262        let namespace_location = temp_path();
1263        namespace_properties.insert(LOCATION.to_string(), namespace_location.to_string());
1264        catalog
1265            .create_namespace(&namespace_ident, namespace_properties)
1266            .await
1267            .unwrap();
1268
1269        let table_name = "tbl1";
1270        let expected_table_ident = TableIdent::new(namespace_ident.clone(), table_name.into());
1271        let expected_table_metadata_location_regex =
1272            format!("^{namespace_location}/tbl1/metadata/00000-{UUID_REGEX_STR}.metadata.json$",);
1273
1274        let table = catalog
1275            .create_table(
1276                &namespace_ident,
1277                TableCreation::builder()
1278                    .name(table_name.into())
1279                    .schema(simple_table_schema())
1280                    // no location specified for table
1281                    .build(),
1282            )
1283            .await
1284            .unwrap();
1285        assert_table_eq(&table, &expected_table_ident, &simple_table_schema());
1286        assert_table_metadata_location_matches(&table, &expected_table_metadata_location_regex);
1287
1288        let table = catalog.load_table(&expected_table_ident).await.unwrap();
1289        assert_table_eq(&table, &expected_table_ident, &simple_table_schema());
1290        assert_table_metadata_location_matches(&table, &expected_table_metadata_location_regex);
1291    }
1292
1293    #[tokio::test]
1294    async fn test_create_table_in_nested_namespace_falls_back_to_nested_namespace_location_if_table_location_is_missing()
1295     {
1296        let warehouse_location = temp_path();
1297        let catalog = MemoryCatalogBuilder::default()
1298            .load(
1299                "memory",
1300                HashMap::from([(
1301                    MEMORY_CATALOG_WAREHOUSE.to_string(),
1302                    warehouse_location.clone(),
1303                )]),
1304            )
1305            .await
1306            .unwrap();
1307
1308        let namespace_ident = NamespaceIdent::new("a".into());
1309        let mut namespace_properties = HashMap::new();
1310        let namespace_location = temp_path();
1311        namespace_properties.insert(LOCATION.to_string(), namespace_location.to_string());
1312        catalog
1313            .create_namespace(&namespace_ident, namespace_properties)
1314            .await
1315            .unwrap();
1316
1317        let nested_namespace_ident = NamespaceIdent::from_strs(vec!["a", "b"]).unwrap();
1318        let mut nested_namespace_properties = HashMap::new();
1319        let nested_namespace_location = temp_path();
1320        nested_namespace_properties
1321            .insert(LOCATION.to_string(), nested_namespace_location.to_string());
1322        catalog
1323            .create_namespace(&nested_namespace_ident, nested_namespace_properties)
1324            .await
1325            .unwrap();
1326
1327        let table_name = "tbl1";
1328        let expected_table_ident =
1329            TableIdent::new(nested_namespace_ident.clone(), table_name.into());
1330        let expected_table_metadata_location_regex = format!(
1331            "^{nested_namespace_location}/tbl1/metadata/00000-{UUID_REGEX_STR}.metadata.json$",
1332        );
1333
1334        let table = catalog
1335            .create_table(
1336                &nested_namespace_ident,
1337                TableCreation::builder()
1338                    .name(table_name.into())
1339                    .schema(simple_table_schema())
1340                    // no location specified for table
1341                    .build(),
1342            )
1343            .await
1344            .unwrap();
1345        assert_table_eq(&table, &expected_table_ident, &simple_table_schema());
1346        assert_table_metadata_location_matches(&table, &expected_table_metadata_location_regex);
1347
1348        let table = catalog.load_table(&expected_table_ident).await.unwrap();
1349        assert_table_eq(&table, &expected_table_ident, &simple_table_schema());
1350        assert_table_metadata_location_matches(&table, &expected_table_metadata_location_regex);
1351    }
1352
1353    #[tokio::test]
1354    async fn test_create_table_falls_back_to_warehouse_location_if_both_table_location_and_namespace_location_are_missing()
1355     {
1356        let warehouse_location = temp_path();
1357        let catalog = MemoryCatalogBuilder::default()
1358            .load(
1359                "memory",
1360                HashMap::from([(
1361                    MEMORY_CATALOG_WAREHOUSE.to_string(),
1362                    warehouse_location.clone(),
1363                )]),
1364            )
1365            .await
1366            .unwrap();
1367
1368        let namespace_ident = NamespaceIdent::new("a".into());
1369        // note: no location specified in namespace_properties
1370        let namespace_properties = HashMap::new();
1371        catalog
1372            .create_namespace(&namespace_ident, namespace_properties)
1373            .await
1374            .unwrap();
1375
1376        let table_name = "tbl1";
1377        let expected_table_ident = TableIdent::new(namespace_ident.clone(), table_name.into());
1378        let expected_table_metadata_location_regex =
1379            format!("^{warehouse_location}/a/tbl1/metadata/00000-{UUID_REGEX_STR}.metadata.json$");
1380
1381        let table = catalog
1382            .create_table(
1383                &namespace_ident,
1384                TableCreation::builder()
1385                    .name(table_name.into())
1386                    .schema(simple_table_schema())
1387                    // no location specified for table
1388                    .build(),
1389            )
1390            .await
1391            .unwrap();
1392        assert_table_eq(&table, &expected_table_ident, &simple_table_schema());
1393        assert_table_metadata_location_matches(&table, &expected_table_metadata_location_regex);
1394
1395        let table = catalog.load_table(&expected_table_ident).await.unwrap();
1396        assert_table_eq(&table, &expected_table_ident, &simple_table_schema());
1397        assert_table_metadata_location_matches(&table, &expected_table_metadata_location_regex);
1398    }
1399
1400    #[tokio::test]
1401    async fn test_create_table_in_nested_namespace_falls_back_to_warehouse_location_if_both_table_location_and_namespace_location_are_missing()
1402     {
1403        let warehouse_location = temp_path();
1404        let catalog = MemoryCatalogBuilder::default()
1405            .load(
1406                "memory",
1407                HashMap::from([(
1408                    MEMORY_CATALOG_WAREHOUSE.to_string(),
1409                    warehouse_location.clone(),
1410                )]),
1411            )
1412            .await
1413            .unwrap();
1414
1415        let namespace_ident = NamespaceIdent::new("a".into());
1416        catalog
1417            // note: no location specified in namespace_properties
1418            .create_namespace(&namespace_ident, HashMap::new())
1419            .await
1420            .unwrap();
1421
1422        let nested_namespace_ident = NamespaceIdent::from_strs(vec!["a", "b"]).unwrap();
1423        catalog
1424            // note: no location specified in namespace_properties
1425            .create_namespace(&nested_namespace_ident, HashMap::new())
1426            .await
1427            .unwrap();
1428
1429        let table_name = "tbl1";
1430        let expected_table_ident =
1431            TableIdent::new(nested_namespace_ident.clone(), table_name.into());
1432        let expected_table_metadata_location_regex = format!(
1433            "^{warehouse_location}/a/b/tbl1/metadata/00000-{UUID_REGEX_STR}.metadata.json$"
1434        );
1435
1436        let table = catalog
1437            .create_table(
1438                &nested_namespace_ident,
1439                TableCreation::builder()
1440                    .name(table_name.into())
1441                    .schema(simple_table_schema())
1442                    // no location specified for table
1443                    .build(),
1444            )
1445            .await
1446            .unwrap();
1447        assert_table_eq(&table, &expected_table_ident, &simple_table_schema());
1448        assert_table_metadata_location_matches(&table, &expected_table_metadata_location_regex);
1449
1450        let table = catalog.load_table(&expected_table_ident).await.unwrap();
1451        assert_table_eq(&table, &expected_table_ident, &simple_table_schema());
1452        assert_table_metadata_location_matches(&table, &expected_table_metadata_location_regex);
1453    }
1454
1455    #[tokio::test]
1456    async fn test_load_throws_error_if_warehouse_is_missing() {
1457        let error = MemoryCatalogBuilder::default()
1458            .load("memory", HashMap::from([]))
1459            .await
1460            .unwrap_err();
1461
1462        assert_eq!(error.kind(), ErrorKind::DataInvalid);
1463        assert_eq!(error.message(), "Catalog warehouse is required");
1464    }
1465
1466    #[tokio::test]
1467    async fn test_load_throws_error_if_warehouse_is_empty() {
1468        let error = MemoryCatalogBuilder::default()
1469            .load(
1470                "memory",
1471                HashMap::from([(MEMORY_CATALOG_WAREHOUSE.to_string(), String::new())]),
1472            )
1473            .await
1474            .unwrap_err();
1475
1476        assert_eq!(error.kind(), ErrorKind::DataInvalid);
1477        assert_eq!(error.message(), "Catalog warehouse is required");
1478    }
1479
1480    #[tokio::test]
1481    async fn test_create_table_throws_error_if_table_with_same_name_already_exists() {
1482        let catalog = new_memory_catalog().await;
1483        let namespace_ident = NamespaceIdent::new("a".into());
1484        create_namespace(&catalog, &namespace_ident).await;
1485        let table_name = "tbl1";
1486        let table_ident = TableIdent::new(namespace_ident.clone(), table_name.into());
1487        create_table(&catalog, &table_ident).await;
1488
1489        let tmp_dir = TempDir::new().unwrap();
1490        let location = tmp_dir.path().to_str().unwrap().to_string();
1491
1492        assert_eq!(
1493            catalog
1494                .create_table(
1495                    &namespace_ident,
1496                    TableCreation::builder()
1497                        .name(table_name.into())
1498                        .schema(simple_table_schema())
1499                        .location(location)
1500                        .build()
1501                )
1502                .await
1503                .unwrap_err()
1504                .to_string(),
1505            format!(
1506                "TableAlreadyExists => Cannot create table {:?}. Table already exists.",
1507                &table_ident
1508            )
1509        );
1510    }
1511
1512    #[tokio::test]
1513    async fn test_list_tables_returns_empty_vector() {
1514        let catalog = new_memory_catalog().await;
1515        let namespace_ident = NamespaceIdent::new("a".into());
1516        create_namespace(&catalog, &namespace_ident).await;
1517
1518        assert_eq!(catalog.list_tables(&namespace_ident).await.unwrap(), vec![]);
1519    }
1520
1521    #[tokio::test]
1522    async fn test_list_tables_returns_a_single_table() {
1523        let catalog = new_memory_catalog().await;
1524        let namespace_ident = NamespaceIdent::new("n1".into());
1525        create_namespace(&catalog, &namespace_ident).await;
1526
1527        let table_ident = TableIdent::new(namespace_ident.clone(), "tbl1".into());
1528        create_table(&catalog, &table_ident).await;
1529
1530        assert_eq!(catalog.list_tables(&namespace_ident).await.unwrap(), vec![
1531            table_ident
1532        ]);
1533    }
1534
1535    #[tokio::test]
1536    async fn test_list_tables_returns_multiple_tables() {
1537        let catalog = new_memory_catalog().await;
1538        let namespace_ident = NamespaceIdent::new("n1".into());
1539        create_namespace(&catalog, &namespace_ident).await;
1540
1541        let table_ident_1 = TableIdent::new(namespace_ident.clone(), "tbl1".into());
1542        let table_ident_2 = TableIdent::new(namespace_ident.clone(), "tbl2".into());
1543        let _ = create_tables(&catalog, vec![&table_ident_1, &table_ident_2]).await;
1544
1545        assert_eq!(
1546            to_set(catalog.list_tables(&namespace_ident).await.unwrap()),
1547            to_set(vec![table_ident_1, table_ident_2])
1548        );
1549    }
1550
1551    #[tokio::test]
1552    async fn test_list_tables_returns_tables_from_correct_namespace() {
1553        let catalog = new_memory_catalog().await;
1554        let namespace_ident_1 = NamespaceIdent::new("n1".into());
1555        let namespace_ident_2 = NamespaceIdent::new("n2".into());
1556        create_namespaces(&catalog, &vec![&namespace_ident_1, &namespace_ident_2]).await;
1557
1558        let table_ident_1 = TableIdent::new(namespace_ident_1.clone(), "tbl1".into());
1559        let table_ident_2 = TableIdent::new(namespace_ident_1.clone(), "tbl2".into());
1560        let table_ident_3 = TableIdent::new(namespace_ident_2.clone(), "tbl1".into());
1561        let _ = create_tables(&catalog, vec![
1562            &table_ident_1,
1563            &table_ident_2,
1564            &table_ident_3,
1565        ])
1566        .await;
1567
1568        assert_eq!(
1569            to_set(catalog.list_tables(&namespace_ident_1).await.unwrap()),
1570            to_set(vec![table_ident_1, table_ident_2])
1571        );
1572
1573        assert_eq!(
1574            to_set(catalog.list_tables(&namespace_ident_2).await.unwrap()),
1575            to_set(vec![table_ident_3])
1576        );
1577    }
1578
1579    #[tokio::test]
1580    async fn test_list_tables_returns_table_under_nested_namespace() {
1581        let catalog = new_memory_catalog().await;
1582        let namespace_ident_a = NamespaceIdent::new("a".into());
1583        let namespace_ident_a_b = NamespaceIdent::from_strs(vec!["a", "b"]).unwrap();
1584        create_namespaces(&catalog, &vec![&namespace_ident_a, &namespace_ident_a_b]).await;
1585
1586        let table_ident = TableIdent::new(namespace_ident_a_b.clone(), "tbl1".into());
1587        create_table(&catalog, &table_ident).await;
1588
1589        assert_eq!(
1590            catalog.list_tables(&namespace_ident_a_b).await.unwrap(),
1591            vec![table_ident]
1592        );
1593    }
1594
1595    #[tokio::test]
1596    async fn test_list_tables_throws_error_if_namespace_doesnt_exist() {
1597        let catalog = new_memory_catalog().await;
1598
1599        let non_existent_namespace_ident = NamespaceIdent::new("n1".into());
1600
1601        assert_eq!(
1602            catalog
1603                .list_tables(&non_existent_namespace_ident)
1604                .await
1605                .unwrap_err()
1606                .to_string(),
1607            format!("NamespaceNotFound => No such namespace: {non_existent_namespace_ident:?}"),
1608        );
1609    }
1610
1611    #[tokio::test]
1612    async fn test_drop_table() {
1613        let catalog = new_memory_catalog().await;
1614        let namespace_ident = NamespaceIdent::new("n1".into());
1615        create_namespace(&catalog, &namespace_ident).await;
1616        let table_ident = TableIdent::new(namespace_ident.clone(), "tbl1".into());
1617        create_table(&catalog, &table_ident).await;
1618
1619        catalog.drop_table(&table_ident).await.unwrap();
1620    }
1621
1622    #[tokio::test]
1623    async fn test_drop_table_drops_table_under_nested_namespace() {
1624        let catalog = new_memory_catalog().await;
1625        let namespace_ident_a = NamespaceIdent::new("a".into());
1626        let namespace_ident_a_b = NamespaceIdent::from_strs(vec!["a", "b"]).unwrap();
1627        create_namespaces(&catalog, &vec![&namespace_ident_a, &namespace_ident_a_b]).await;
1628
1629        let table_ident = TableIdent::new(namespace_ident_a_b.clone(), "tbl1".into());
1630        create_table(&catalog, &table_ident).await;
1631
1632        catalog.drop_table(&table_ident).await.unwrap();
1633
1634        assert_eq!(
1635            catalog.list_tables(&namespace_ident_a_b).await.unwrap(),
1636            vec![]
1637        );
1638    }
1639
1640    #[tokio::test]
1641    async fn test_drop_table_throws_error_if_namespace_doesnt_exist() {
1642        let catalog = new_memory_catalog().await;
1643
1644        let non_existent_namespace_ident = NamespaceIdent::new("n1".into());
1645        let non_existent_table_ident =
1646            TableIdent::new(non_existent_namespace_ident.clone(), "tbl1".into());
1647
1648        assert_eq!(
1649            catalog
1650                .drop_table(&non_existent_table_ident)
1651                .await
1652                .unwrap_err()
1653                .to_string(),
1654            format!("NamespaceNotFound => No such namespace: {non_existent_namespace_ident:?}"),
1655        );
1656    }
1657
1658    #[tokio::test]
1659    async fn test_drop_table_throws_error_if_table_doesnt_exist() {
1660        let catalog = new_memory_catalog().await;
1661        let namespace_ident = NamespaceIdent::new("n1".into());
1662        create_namespace(&catalog, &namespace_ident).await;
1663
1664        let non_existent_table_ident = TableIdent::new(namespace_ident.clone(), "tbl1".into());
1665
1666        assert_eq!(
1667            catalog
1668                .drop_table(&non_existent_table_ident)
1669                .await
1670                .unwrap_err()
1671                .to_string(),
1672            format!("TableNotFound => No such table: {non_existent_table_ident:?}"),
1673        );
1674    }
1675
1676    #[tokio::test]
1677    async fn test_table_exists_returns_true() {
1678        let catalog = new_memory_catalog().await;
1679        let namespace_ident = NamespaceIdent::new("n1".into());
1680        create_namespace(&catalog, &namespace_ident).await;
1681        let table_ident = TableIdent::new(namespace_ident.clone(), "tbl1".into());
1682        create_table(&catalog, &table_ident).await;
1683
1684        assert!(catalog.table_exists(&table_ident).await.unwrap());
1685    }
1686
1687    #[tokio::test]
1688    async fn test_table_exists_returns_false() {
1689        let catalog = new_memory_catalog().await;
1690        let namespace_ident = NamespaceIdent::new("n1".into());
1691        create_namespace(&catalog, &namespace_ident).await;
1692        let non_existent_table_ident = TableIdent::new(namespace_ident.clone(), "tbl1".into());
1693
1694        assert!(
1695            !catalog
1696                .table_exists(&non_existent_table_ident)
1697                .await
1698                .unwrap()
1699        );
1700    }
1701
1702    #[tokio::test]
1703    async fn test_table_exists_under_nested_namespace() {
1704        let catalog = new_memory_catalog().await;
1705        let namespace_ident_a = NamespaceIdent::new("a".into());
1706        let namespace_ident_a_b = NamespaceIdent::from_strs(vec!["a", "b"]).unwrap();
1707        create_namespaces(&catalog, &vec![&namespace_ident_a, &namespace_ident_a_b]).await;
1708
1709        let table_ident = TableIdent::new(namespace_ident_a_b.clone(), "tbl1".into());
1710        create_table(&catalog, &table_ident).await;
1711
1712        assert!(catalog.table_exists(&table_ident).await.unwrap());
1713
1714        let non_existent_table_ident = TableIdent::new(namespace_ident_a_b.clone(), "tbl2".into());
1715        assert!(
1716            !catalog
1717                .table_exists(&non_existent_table_ident)
1718                .await
1719                .unwrap()
1720        );
1721    }
1722
1723    #[tokio::test]
1724    async fn test_table_exists_throws_error_if_namespace_doesnt_exist() {
1725        let catalog = new_memory_catalog().await;
1726
1727        let non_existent_namespace_ident = NamespaceIdent::new("n1".into());
1728        let non_existent_table_ident =
1729            TableIdent::new(non_existent_namespace_ident.clone(), "tbl1".into());
1730
1731        assert_eq!(
1732            catalog
1733                .table_exists(&non_existent_table_ident)
1734                .await
1735                .unwrap_err()
1736                .to_string(),
1737            format!("NamespaceNotFound => No such namespace: {non_existent_namespace_ident:?}"),
1738        );
1739    }
1740
1741    #[tokio::test]
1742    async fn test_rename_table_in_same_namespace() {
1743        let catalog = new_memory_catalog().await;
1744        let namespace_ident = NamespaceIdent::new("n1".into());
1745        create_namespace(&catalog, &namespace_ident).await;
1746        let src_table_ident = TableIdent::new(namespace_ident.clone(), "tbl1".into());
1747        let dst_table_ident = TableIdent::new(namespace_ident.clone(), "tbl2".into());
1748        create_table(&catalog, &src_table_ident).await;
1749
1750        catalog
1751            .rename_table(&src_table_ident, &dst_table_ident)
1752            .await
1753            .unwrap();
1754
1755        assert_eq!(catalog.list_tables(&namespace_ident).await.unwrap(), vec![
1756            dst_table_ident
1757        ],);
1758    }
1759
1760    #[tokio::test]
1761    async fn test_rename_table_across_namespaces() {
1762        let catalog = new_memory_catalog().await;
1763        let src_namespace_ident = NamespaceIdent::new("a".into());
1764        let dst_namespace_ident = NamespaceIdent::new("b".into());
1765        create_namespaces(&catalog, &vec![&src_namespace_ident, &dst_namespace_ident]).await;
1766        let src_table_ident = TableIdent::new(src_namespace_ident.clone(), "tbl1".into());
1767        let dst_table_ident = TableIdent::new(dst_namespace_ident.clone(), "tbl2".into());
1768        create_table(&catalog, &src_table_ident).await;
1769
1770        catalog
1771            .rename_table(&src_table_ident, &dst_table_ident)
1772            .await
1773            .unwrap();
1774
1775        assert_eq!(
1776            catalog.list_tables(&src_namespace_ident).await.unwrap(),
1777            vec![],
1778        );
1779
1780        assert_eq!(
1781            catalog.list_tables(&dst_namespace_ident).await.unwrap(),
1782            vec![dst_table_ident],
1783        );
1784    }
1785
1786    #[tokio::test]
1787    async fn test_rename_table_src_table_is_same_as_dst_table() {
1788        let catalog = new_memory_catalog().await;
1789        let namespace_ident = NamespaceIdent::new("n1".into());
1790        create_namespace(&catalog, &namespace_ident).await;
1791        let table_ident = TableIdent::new(namespace_ident.clone(), "tbl".into());
1792        create_table(&catalog, &table_ident).await;
1793
1794        catalog
1795            .rename_table(&table_ident, &table_ident)
1796            .await
1797            .unwrap();
1798
1799        assert_eq!(catalog.list_tables(&namespace_ident).await.unwrap(), vec![
1800            table_ident
1801        ],);
1802    }
1803
1804    #[tokio::test]
1805    async fn test_rename_table_across_nested_namespaces() {
1806        let catalog = new_memory_catalog().await;
1807        let namespace_ident_a = NamespaceIdent::new("a".into());
1808        let namespace_ident_a_b = NamespaceIdent::from_strs(vec!["a", "b"]).unwrap();
1809        let namespace_ident_a_b_c = NamespaceIdent::from_strs(vec!["a", "b", "c"]).unwrap();
1810        create_namespaces(&catalog, &vec![
1811            &namespace_ident_a,
1812            &namespace_ident_a_b,
1813            &namespace_ident_a_b_c,
1814        ])
1815        .await;
1816
1817        let src_table_ident = TableIdent::new(namespace_ident_a_b_c.clone(), "tbl1".into());
1818        create_tables(&catalog, vec![&src_table_ident]).await;
1819
1820        let dst_table_ident = TableIdent::new(namespace_ident_a_b.clone(), "tbl1".into());
1821        catalog
1822            .rename_table(&src_table_ident, &dst_table_ident)
1823            .await
1824            .unwrap();
1825
1826        assert!(!catalog.table_exists(&src_table_ident).await.unwrap());
1827
1828        assert!(catalog.table_exists(&dst_table_ident).await.unwrap());
1829    }
1830
1831    #[tokio::test]
1832    async fn test_rename_table_throws_error_if_src_namespace_doesnt_exist() {
1833        let catalog = new_memory_catalog().await;
1834
1835        let non_existent_src_namespace_ident = NamespaceIdent::new("n1".into());
1836        let src_table_ident =
1837            TableIdent::new(non_existent_src_namespace_ident.clone(), "tbl1".into());
1838
1839        let dst_namespace_ident = NamespaceIdent::new("n2".into());
1840        create_namespace(&catalog, &dst_namespace_ident).await;
1841        let dst_table_ident = TableIdent::new(dst_namespace_ident.clone(), "tbl1".into());
1842
1843        assert_eq!(
1844            catalog
1845                .rename_table(&src_table_ident, &dst_table_ident)
1846                .await
1847                .unwrap_err()
1848                .to_string(),
1849            format!("NamespaceNotFound => No such namespace: {non_existent_src_namespace_ident:?}"),
1850        );
1851    }
1852
1853    #[tokio::test]
1854    async fn test_rename_table_throws_error_if_dst_namespace_doesnt_exist() {
1855        let catalog = new_memory_catalog().await;
1856        let src_namespace_ident = NamespaceIdent::new("n1".into());
1857        let src_table_ident = TableIdent::new(src_namespace_ident.clone(), "tbl1".into());
1858        create_namespace(&catalog, &src_namespace_ident).await;
1859        create_table(&catalog, &src_table_ident).await;
1860
1861        let non_existent_dst_namespace_ident = NamespaceIdent::new("n2".into());
1862        let dst_table_ident =
1863            TableIdent::new(non_existent_dst_namespace_ident.clone(), "tbl1".into());
1864        assert_eq!(
1865            catalog
1866                .rename_table(&src_table_ident, &dst_table_ident)
1867                .await
1868                .unwrap_err()
1869                .to_string(),
1870            format!("NamespaceNotFound => No such namespace: {non_existent_dst_namespace_ident:?}"),
1871        );
1872    }
1873
1874    #[tokio::test]
1875    async fn test_rename_table_throws_error_if_src_table_doesnt_exist() {
1876        let catalog = new_memory_catalog().await;
1877        let namespace_ident = NamespaceIdent::new("n1".into());
1878        create_namespace(&catalog, &namespace_ident).await;
1879        let src_table_ident = TableIdent::new(namespace_ident.clone(), "tbl1".into());
1880        let dst_table_ident = TableIdent::new(namespace_ident.clone(), "tbl2".into());
1881
1882        assert_eq!(
1883            catalog
1884                .rename_table(&src_table_ident, &dst_table_ident)
1885                .await
1886                .unwrap_err()
1887                .to_string(),
1888            format!("TableNotFound => No such table: {src_table_ident:?}"),
1889        );
1890    }
1891
1892    #[tokio::test]
1893    async fn test_rename_table_throws_error_if_dst_table_already_exists() {
1894        let catalog = new_memory_catalog().await;
1895        let namespace_ident = NamespaceIdent::new("n1".into());
1896        create_namespace(&catalog, &namespace_ident).await;
1897        let src_table_ident = TableIdent::new(namespace_ident.clone(), "tbl1".into());
1898        let dst_table_ident = TableIdent::new(namespace_ident.clone(), "tbl2".into());
1899        create_tables(&catalog, vec![&src_table_ident, &dst_table_ident]).await;
1900
1901        assert_eq!(
1902            catalog
1903                .rename_table(&src_table_ident, &dst_table_ident)
1904                .await
1905                .unwrap_err()
1906                .to_string(),
1907            format!(
1908                "TableAlreadyExists => Cannot create table {:?}. Table already exists.",
1909                &dst_table_ident
1910            ),
1911        );
1912    }
1913
1914    #[tokio::test]
1915    async fn test_register_table() {
1916        // Create a catalog and namespace
1917        let catalog = new_memory_catalog().await;
1918        let namespace_ident = NamespaceIdent::new("test_namespace".into());
1919        create_namespace(&catalog, &namespace_ident).await;
1920
1921        // Create a table to get a valid metadata file
1922        let source_table_ident = TableIdent::new(namespace_ident.clone(), "source_table".into());
1923        create_table(&catalog, &source_table_ident).await;
1924
1925        // Get the metadata location from the source table
1926        let source_table = catalog.load_table(&source_table_ident).await.unwrap();
1927        let metadata_location = source_table.metadata_location().unwrap().to_string();
1928
1929        // Register a new table using the same metadata location
1930        let register_table_ident =
1931            TableIdent::new(namespace_ident.clone(), "register_table".into());
1932        let registered_table = catalog
1933            .register_table(&register_table_ident, metadata_location.clone())
1934            .await
1935            .unwrap();
1936
1937        // Verify the registered table has the correct identifier
1938        assert_eq!(registered_table.identifier(), &register_table_ident);
1939
1940        // Verify the registered table has the correct metadata location
1941        assert_eq!(
1942            registered_table.metadata_location().unwrap().to_string(),
1943            metadata_location
1944        );
1945
1946        // Verify the table exists in the catalog
1947        assert!(catalog.table_exists(&register_table_ident).await.unwrap());
1948
1949        // Verify we can load the registered table
1950        let loaded_table = catalog.load_table(&register_table_ident).await.unwrap();
1951        assert_eq!(loaded_table.identifier(), &register_table_ident);
1952        assert_eq!(
1953            loaded_table.metadata_location().unwrap().to_string(),
1954            metadata_location
1955        );
1956    }
1957
1958    #[tokio::test]
1959    async fn test_update_table() {
1960        let catalog = new_memory_catalog().await;
1961
1962        let table = create_table_with_namespace(&catalog).await;
1963
1964        // Assert the table doesn't contain the update yet
1965        assert!(!table.metadata().properties().contains_key("key"));
1966
1967        // `last_updated_ms` is millisecond-resolution `chrono::Utc::now()`.
1968        // Without this sleep, create and commit can land in the same
1969        // millisecond and the `<` assertion below flakes.
1970        tokio::time::sleep(std::time::Duration::from_millis(2)).await;
1971
1972        // Update table metadata
1973        let tx = Transaction::new(&table);
1974        let updated_table = tx
1975            .update_table_properties()
1976            .set("key".to_string(), "value".to_string())
1977            .apply(tx)
1978            .unwrap()
1979            .commit(&catalog)
1980            .await
1981            .unwrap();
1982
1983        assert_eq!(
1984            updated_table.metadata().properties().get("key").unwrap(),
1985            "value"
1986        );
1987
1988        assert_eq!(table.identifier(), updated_table.identifier());
1989        assert_eq!(table.metadata().uuid(), updated_table.metadata().uuid());
1990        assert_ne!(table.metadata_location(), updated_table.metadata_location());
1991
1992        assert!(
1993            table.metadata().metadata_log().len() < updated_table.metadata().metadata_log().len()
1994        );
1995    }
1996
1997    async fn create_table_with_properties<C: Catalog>(
1998        catalog: &C,
1999        properties: HashMap<String, String>,
2000    ) -> Table {
2001        let namespace_ident = NamespaceIdent::new("abc".into());
2002        create_namespace(catalog, &namespace_ident).await;
2003        catalog
2004            .create_table(
2005                &namespace_ident,
2006                TableCreation::builder()
2007                    .name("test".into())
2008                    .schema(simple_table_schema())
2009                    .properties(properties)
2010                    .build(),
2011            )
2012            .await
2013            .unwrap()
2014    }
2015
2016    async fn commit_property_update<C: Catalog>(catalog: &C, table: &Table, value: &str) -> Table {
2017        let tx = Transaction::new(table);
2018        tx.update_table_properties()
2019            .set("key".to_string(), value.to_string())
2020            .apply(tx)
2021            .unwrap()
2022            .commit(catalog)
2023            .await
2024            .unwrap()
2025    }
2026
2027    /// Commits `num_commits` property updates and returns the metadata location of every
2028    /// table version, oldest first.
2029    async fn commit_metadata_versions<C: Catalog>(
2030        catalog: &C,
2031        mut table: Table,
2032        num_commits: usize,
2033    ) -> Vec<String> {
2034        let mut locations = vec![table.metadata_location().unwrap().to_string()];
2035        for i in 0..num_commits {
2036            table = commit_property_update(catalog, &table, &i.to_string()).await;
2037            locations.push(table.metadata_location().unwrap().to_string());
2038        }
2039        locations
2040    }
2041
2042    async fn existing_files(file_io: &FileIO, locations: &[String]) -> Vec<String> {
2043        let mut existing = Vec::new();
2044        for location in locations {
2045            if file_io.exists(location).await.unwrap() {
2046                existing.push(location.clone());
2047            }
2048        }
2049        existing
2050    }
2051
2052    #[tokio::test]
2053    async fn test_update_table_deletes_metadata_files_removed_from_log_when_enabled() {
2054        let catalog = new_memory_catalog().await;
2055        let table = create_table_with_properties(
2056            &catalog,
2057            HashMap::from([
2058                (
2059                    TableProperties::PROPERTY_METADATA_DELETE_AFTER_COMMIT_ENABLED.to_string(),
2060                    "true".to_string(),
2061                ),
2062                (
2063                    TableProperties::PROPERTY_METADATA_PREVIOUS_VERSIONS_MAX.to_string(),
2064                    "1".to_string(),
2065                ),
2066            ]),
2067        )
2068        .await;
2069        let file_io = table.file_io().clone();
2070
2071        let locations = commit_metadata_versions(&catalog, table, 3).await;
2072
2073        // The current metadata file and the one entry left in the metadata log remain.
2074        assert_eq!(
2075            existing_files(&file_io, &locations).await,
2076            locations[2..].to_vec()
2077        );
2078    }
2079
2080    #[tokio::test]
2081    async fn test_update_table_deletes_removed_metadata_files_on_commit_that_enables_cleanup() {
2082        let catalog = new_memory_catalog().await;
2083        let table = create_table_with_properties(
2084            &catalog,
2085            HashMap::from([(
2086                TableProperties::PROPERTY_METADATA_PREVIOUS_VERSIONS_MAX.to_string(),
2087                "1".to_string(),
2088            )]),
2089        )
2090        .await;
2091        let file_io = table.file_io().clone();
2092        let v0 = table.metadata_location().unwrap().to_string();
2093        let v1_table = commit_property_update(&catalog, &table, "1").await;
2094        let v1 = v1_table.metadata_location().unwrap().to_string();
2095        let v2_table = commit_property_update(&catalog, &v1_table, "2").await;
2096        let v2 = v2_table.metadata_location().unwrap().to_string();
2097
2098        let tx = Transaction::new(&v2_table);
2099        let v3_table = tx
2100            .update_table_properties()
2101            .set(
2102                TableProperties::PROPERTY_METADATA_DELETE_AFTER_COMMIT_ENABLED.to_string(),
2103                "true".to_string(),
2104            )
2105            .apply(tx)
2106            .unwrap()
2107            .commit(&catalog)
2108            .await
2109            .unwrap();
2110        let v3 = v3_table.metadata_location().unwrap().to_string();
2111
2112        // v0 fell out of the log while cleanup was disabled, so only v1 is deleted.
2113        assert_eq!(
2114            existing_files(&file_io, &[v0.clone(), v1, v2.clone(), v3.clone()]).await,
2115            vec![v0, v2, v3]
2116        );
2117    }
2118
2119    #[tokio::test]
2120    async fn test_update_table_keeps_metadata_files_removed_from_log_when_not_enabled() {
2121        for delete_after_commit in [None, Some("false")] {
2122            let catalog = new_memory_catalog().await;
2123            let mut properties = HashMap::from([(
2124                TableProperties::PROPERTY_METADATA_PREVIOUS_VERSIONS_MAX.to_string(),
2125                "1".to_string(),
2126            )]);
2127            if let Some(value) = delete_after_commit {
2128                properties.insert(
2129                    TableProperties::PROPERTY_METADATA_DELETE_AFTER_COMMIT_ENABLED.to_string(),
2130                    value.to_string(),
2131                );
2132            }
2133            let table = create_table_with_properties(&catalog, properties).await;
2134            let file_io = table.file_io().clone();
2135
2136            let locations = commit_metadata_versions(&catalog, table, 3).await;
2137
2138            assert_eq!(existing_files(&file_io, &locations).await, locations);
2139        }
2140    }
2141
2142    #[tokio::test]
2143    async fn test_update_table_succeeds_when_deleting_removed_metadata_file_fails() {
2144        let warehouse = TempDir::new().unwrap();
2145        let catalog = MemoryCatalogBuilder::default()
2146            .with_storage_factory(Arc::new(LocalFsStorageFactory))
2147            .load(
2148                "memory",
2149                HashMap::from([(
2150                    MEMORY_CATALOG_WAREHOUSE.to_string(),
2151                    warehouse.path().to_str().unwrap().to_string(),
2152                )]),
2153            )
2154            .await
2155            .unwrap();
2156        let table = create_table_with_properties(
2157            &catalog,
2158            HashMap::from([
2159                (
2160                    TableProperties::PROPERTY_METADATA_DELETE_AFTER_COMMIT_ENABLED.to_string(),
2161                    "true".to_string(),
2162                ),
2163                (
2164                    TableProperties::PROPERTY_METADATA_PREVIOUS_VERSIONS_MAX.to_string(),
2165                    "1".to_string(),
2166                ),
2167            ]),
2168        )
2169        .await;
2170        let file_io = table.file_io().clone();
2171        let v0 = table.metadata_location().unwrap().to_string();
2172        let v1_table = commit_property_update(&catalog, &table, "1").await;
2173        let v1 = v1_table.metadata_location().unwrap().to_string();
2174
2175        // A non-empty directory in place of v0 makes its deletion fail.
2176        std::fs::remove_file(&v0).unwrap();
2177        std::fs::create_dir(&v0).unwrap();
2178        std::fs::write(format!("{v0}/blocker"), b"").unwrap();
2179
2180        let v2_table = commit_property_update(&catalog, &v1_table, "2").await;
2181        let v2 = v2_table.metadata_location().unwrap().to_string();
2182        let v3_table = commit_property_update(&catalog, &v2_table, "3").await;
2183        let v3 = v3_table.metadata_location().unwrap().to_string();
2184
2185        assert_eq!(
2186            existing_files(&file_io, &[v0.clone(), v1, v2.clone(), v3.clone()]).await,
2187            vec![v0, v2, v3]
2188        );
2189    }
2190
2191    #[tokio::test]
2192    async fn test_update_table_fails_if_table_doesnt_exist() {
2193        let catalog = new_memory_catalog().await;
2194
2195        let namespace_ident = NamespaceIdent::new("a".into());
2196        create_namespace(&catalog, &namespace_ident).await;
2197
2198        // This table is not known to the catalog.
2199        let table_ident = TableIdent::new(namespace_ident, "test".to_string());
2200        let table = build_table(table_ident);
2201
2202        let tx = Transaction::new(&table);
2203        let err = tx
2204            .update_table_properties()
2205            .set("key".to_string(), "value".to_string())
2206            .apply(tx)
2207            .unwrap()
2208            .commit(&catalog)
2209            .await
2210            .unwrap_err();
2211        assert_eq!(err.kind(), ErrorKind::TableNotFound);
2212    }
2213
2214    /// Master key bytes used to generate the encrypted testdata fixtures.
2215    /// See `testdata/manifests_lists/README.md`.
2216    const FIXTURE_MASTER_KEY_ID: &str = "master-1";
2217    const FIXTURE_MASTER_KEY_BYTES: [u8; 16] = [
2218        0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e,
2219        0x0f,
2220    ];
2221
2222    /// Builds a `MemoryKmsClientFactory` seeded with the fixture master key.
2223    fn fixture_kms_factory() -> MemoryKmsClientFactory {
2224        use crate::encryption::SensitiveBytes;
2225
2226        let factory = MemoryKmsClientFactory::new();
2227        factory
2228            .add_master_key_bytes(
2229                FIXTURE_MASTER_KEY_ID,
2230                SensitiveBytes::new(FIXTURE_MASTER_KEY_BYTES),
2231            )
2232            .unwrap();
2233        factory
2234    }
2235
2236    /// Loads the encrypted V3 metadata fixture and patches its snapshot's
2237    /// manifest-list to point at the on-disk encrypted testdata file.
2238    fn load_encrypted_fixture_metadata() -> TableMetadata {
2239        let manifest_dir = env!("CARGO_MANIFEST_DIR");
2240        let metadata_json = std::fs::read_to_string(format!(
2241            "{manifest_dir}/testdata/table_metadata/TableMetadataV3ValidEncryption.json"
2242        ))
2243        .unwrap();
2244        let mut metadata: TableMetadata = serde_json::from_str(&metadata_json).unwrap();
2245
2246        let manifest_list_path =
2247            format!("{manifest_dir}/testdata/manifests_lists/manifest-list-v3-encrypted.avro");
2248        let snapshot = metadata.snapshots.get_mut(&1).unwrap();
2249        let mut patched = snapshot.as_ref().clone();
2250        patched.manifest_list = manifest_list_path;
2251        *snapshot = Arc::new(patched);
2252
2253        metadata
2254    }
2255
2256    #[tokio::test]
2257    async fn catalog_kms_factory_client_reaches_table_encryption_manager() {
2258        let warehouse = temp_path();
2259        let catalog = MemoryCatalogBuilder::default()
2260            .with_storage_factory(Arc::new(LocalFsStorageFactory))
2261            .with_kms_client_factory(Arc::new(fixture_kms_factory()))
2262            .load(
2263                "memory",
2264                HashMap::from([(MEMORY_CATALOG_WAREHOUSE.to_string(), warehouse)]),
2265            )
2266            .await
2267            .unwrap();
2268
2269        let namespace_ident = NamespaceIdent::new("enc_ns".into());
2270        create_namespace(&catalog, &namespace_ident).await;
2271
2272        let metadata = load_encrypted_fixture_metadata();
2273        let metadata_dir = TempDir::new().unwrap();
2274        let metadata_location =
2275            format!("{}/v1.metadata.json", metadata_dir.path().to_str().unwrap());
2276        std::fs::write(&metadata_location, serde_json::to_vec(&metadata).unwrap()).unwrap();
2277
2278        let table_ident = TableIdent::new(namespace_ident, "enc".to_string());
2279        catalog
2280            .register_table(&table_ident, metadata_location)
2281            .await
2282            .unwrap();
2283
2284        let table = catalog.load_table(&table_ident).await.unwrap();
2285        assert!(
2286            table.encryption_manager().is_some(),
2287            "factory-built KMS client should have reached the table's EncryptionManager"
2288        );
2289
2290        let snapshot_ref = table.metadata().current_snapshot().unwrap();
2291        let manifest_list = table
2292            .object_cache()
2293            .get_manifest_list(snapshot_ref, &table.metadata_ref())
2294            .await
2295            .unwrap();
2296        assert_eq!(manifest_list.entries().len(), 0);
2297    }
2298
2299    fn build_table(ident: TableIdent) -> Table {
2300        let file_io = FileIO::new_with_fs();
2301
2302        let temp_dir = TempDir::new().unwrap();
2303        let location = temp_dir.path().to_str().unwrap().to_string();
2304
2305        let table_creation = TableCreation::builder()
2306            .name(ident.name().to_string())
2307            .schema(simple_table_schema())
2308            .location(location)
2309            .build();
2310        let metadata = TableMetadataBuilder::from_table_creation(table_creation)
2311            .unwrap()
2312            .build()
2313            .unwrap()
2314            .metadata;
2315
2316        Table::builder()
2317            .identifier(ident)
2318            .metadata(metadata)
2319            .file_io(file_io)
2320            .runtime(test_runtime())
2321            .build()
2322            .unwrap()
2323    }
2324}