Skip to main content

iceberg_catalog_sql/
catalog.rs

1// Licensed to the Apache Software Foundation (ASF) under one
2// or more contributor license agreements.  See the NOTICE file
3// distributed with this work for additional information
4// regarding copyright ownership.  The ASF licenses this file
5// to you under the Apache License, Version 2.0 (the
6// "License"); you may not use this file except in compliance
7// with the License.  You may obtain a copy of the License at
8//
9//   http://www.apache.org/licenses/LICENSE-2.0
10//
11// Unless required by applicable law or agreed to in writing,
12// software distributed under the License is distributed on an
13// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14// KIND, either express or implied.  See the License for the
15// specific language governing permissions and limitations
16// under the License.
17
18use std::collections::{HashMap, HashSet};
19use std::str::FromStr;
20use std::sync::Arc;
21use std::time::Duration;
22
23use async_trait::async_trait;
24use iceberg::encryption::kms::{KeyManagementClient, KmsClientFactory};
25use iceberg::io::{FileIO, FileIOBuilder, StorageFactory};
26use iceberg::spec::{TableMetadata, TableMetadataBuilder};
27use iceberg::table::Table;
28use iceberg::{
29    Catalog, CatalogBuilder, Error, ErrorKind, MetadataLocation, Namespace, NamespaceIdent, Result,
30    Runtime, TableCommit, TableCreation, TableIdent, delete_removed_metadata_files,
31};
32use iceberg_property_macro::Properties;
33use sqlx::any::{AnyPoolOptions, AnyQueryResult, AnyRow, install_default_drivers};
34use sqlx::{Any, AnyPool, Column, Executor, Row, Transaction};
35
36use crate::error::{
37    from_sqlx_error, no_such_namespace_err, no_such_table_err, table_already_exists_err,
38};
39
40/// catalog URI
41pub const SQL_CATALOG_PROP_URI: &str = "uri";
42/// catalog warehouse location
43pub const SQL_CATALOG_PROP_WAREHOUSE: &str = "warehouse";
44/// catalog sql bind style
45pub const SQL_CATALOG_PROP_BIND_STYLE: &str = "sql.bind-style";
46/// Legacy (pre-`sql.bind-style`) key for [`SQL_CATALOG_PROP_BIND_STYLE`], still accepted for
47/// backward compatibility.
48const SQL_CATALOG_PROP_BIND_STYLE_LEGACY: &str = "sql_bind_style";
49/// Expected catalog schema version.
50///
51/// If this property is set and it is newer than the detected schema version,
52/// a migration will be attempted.
53/// If it is older, it is ignored with a warning.
54/// If the catalog table didn't already exist, this value is ignored and it will be created with `V1`.
55///
56/// `V0` is a compatibility mode for catalog tables created before the `iceberg_type` column
57/// existed; it cannot be requested for a new catalog table, since table creation and
58/// registration are unsupported on `V0`.
59pub const SQL_CATALOG_PROP_SCHEMA_VERSION: &str = "sql.schema-version";
60
61static CATALOG_TABLE_NAME: &str = "iceberg_tables";
62static CATALOG_FIELD_CATALOG_NAME: &str = "catalog_name";
63static CATALOG_FIELD_TABLE_NAME: &str = "table_name";
64static CATALOG_FIELD_TABLE_NAMESPACE: &str = "table_namespace";
65static CATALOG_FIELD_METADATA_LOCATION_PROP: &str = "metadata_location";
66static CATALOG_FIELD_PREVIOUS_METADATA_LOCATION_PROP: &str = "previous_metadata_location";
67static CATALOG_FIELD_RECORD_TYPE: &str = "iceberg_type";
68static CATALOG_FIELD_TABLE_RECORD_TYPE: &str = "TABLE";
69
70static NAMESPACE_TABLE_NAME: &str = "iceberg_namespace_properties";
71static NAMESPACE_FIELD_NAME: &str = "namespace";
72static NAMESPACE_FIELD_PROPERTY_KEY: &str = "property_key";
73static NAMESPACE_FIELD_PROPERTY_VALUE: &str = "property_value";
74
75static NAMESPACE_LOCATION_PROPERTY_KEY: &str = "location";
76
77static MAX_CONNECTIONS: u32 = 10; // Default the SQL pool to 10 connections if not provided
78static IDLE_TIMEOUT: u64 = 10; // Default the maximum idle timeout per connection to 10s before it is closed
79static TEST_BEFORE_ACQUIRE: bool = true; // Default the health-check of each connection to enabled prior to returning
80
81/// Parses one pool-property value; the `Properties` derive adds the property-key context.
82fn parse_pool_property<T>(value: &str) -> Result<T>
83where
84    T: FromStr,
85    T::Err: std::error::Error + Send + Sync + 'static,
86{
87    value.parse().map_err(|error| {
88        Error::new(
89            ErrorKind::DataInvalid,
90            "Failed to parse SQL catalog pool property",
91        )
92        .with_context("value", value)
93        .with_source(error)
94    })
95}
96
97/// Builder for [`SqlCatalog`]
98#[derive(Debug, Default)]
99pub struct SqlCatalogBuilder {
100    props: HashMap<String, String>,
101    storage_factory: Option<Arc<dyn StorageFactory>>,
102    kms_client_factory: Option<Arc<dyn KmsClientFactory>>,
103    runtime: Option<Runtime>,
104}
105
106impl SqlCatalogBuilder {
107    /// Configure the database URI
108    ///
109    /// If `SQL_CATALOG_PROP_URI` has a value set in `props` during `SqlCatalogBuilder::load`,
110    /// that value takes precedence, and the value specified by this method will not be used.
111    pub fn uri(mut self, uri: impl Into<String>) -> Self {
112        self.props
113            .insert(SQL_CATALOG_PROP_URI.to_string(), uri.into());
114        self
115    }
116
117    /// Configure the warehouse location
118    ///
119    /// If `SQL_CATALOG_PROP_WAREHOUSE` has a value set in `props` during `SqlCatalogBuilder::load`,
120    /// that value takes precedence, and the value specified by this method will not be used.
121    pub fn warehouse_location(mut self, location: impl Into<String>) -> Self {
122        self.props
123            .insert(SQL_CATALOG_PROP_WAREHOUSE.to_string(), location.into());
124        self
125    }
126
127    /// Configure the bound SQL Statement
128    ///
129    /// If either `SQL_CATALOG_PROP_BIND_STYLE` or the legacy `sql_bind_style` property has a value
130    /// set in `props` during `SqlCatalogBuilder::load`, that load-time value takes precedence.
131    /// When both load-time keys are present, `SQL_CATALOG_PROP_BIND_STYLE` takes precedence.
132    pub fn sql_bind_style(mut self, sql_bind_style: SqlBindStyle) -> Self {
133        self.props.insert(
134            SQL_CATALOG_PROP_BIND_STYLE.to_string(),
135            sql_bind_style.to_string(),
136        );
137        self
138    }
139
140    /// Configure the any properties
141    ///
142    /// If the same key has values set in `props` during `SqlCatalogBuilder::load`,
143    /// those values will take precedence.
144    pub fn props(mut self, props: HashMap<String, String>) -> Self {
145        for (k, v) in props {
146            self.props.insert(k, v);
147        }
148        self
149    }
150
151    /// Set a new property on the property to be configured.
152    /// When multiple methods are executed with the same key,
153    /// the later-set value takes precedence.
154    ///
155    /// If the same key has values set in `props` during `SqlCatalogBuilder::load`,
156    /// those values will take precedence.
157    pub fn prop(mut self, key: impl Into<String>, value: impl Into<String>) -> Self {
158        self.props.insert(key.into(), value.into());
159        self
160    }
161}
162
163impl CatalogBuilder for SqlCatalogBuilder {
164    type C = SqlCatalog;
165
166    fn with_storage_factory(mut self, storage_factory: Arc<dyn StorageFactory>) -> Self {
167        self.storage_factory = Some(storage_factory);
168        self
169    }
170
171    fn with_kms_client_factory(mut self, kms_client_factory: Arc<dyn KmsClientFactory>) -> Self {
172        self.kms_client_factory = Some(kms_client_factory);
173        self
174    }
175
176    fn with_runtime(mut self, runtime: Runtime) -> Self {
177        self.runtime = Some(runtime);
178        self
179    }
180
181    fn load(
182        self,
183        name: impl Into<String>,
184        props: HashMap<String, String>,
185    ) -> impl Future<Output = Result<Self::C>> + Send {
186        let name = name.into();
187
188        async move {
189            if name.trim().is_empty() {
190                return Err(Error::new(
191                    ErrorKind::DataInvalid,
192                    "Catalog name cannot be empty",
193                ));
194            }
195
196            let load_overrides_bind_style = props.contains_key(SQL_CATALOG_PROP_BIND_STYLE)
197                || props.contains_key(SQL_CATALOG_PROP_BIND_STYLE_LEGACY);
198            let mut merged_props = self.props;
199            if load_overrides_bind_style {
200                // Treat the preferred and legacy names as aliases so either load-time spelling
201                // overrides a value configured on the builder.
202                merged_props.remove(SQL_CATALOG_PROP_BIND_STYLE);
203                merged_props.remove(SQL_CATALOG_PROP_BIND_STYLE_LEGACY);
204            }
205            merged_props.extend(props);
206            let catalog_properties = SqlCatalogProperties::from_properties(&merged_props)?;
207
208            // The catalog URI is consumed above and may contain database credentials. Do not
209            // expose this security-sensitive value to FileIO or KMS extension points.
210            merged_props.remove(SQL_CATALOG_PROP_URI);
211
212            let runtime = match self.runtime {
213                Some(rt) => rt,
214                None => Runtime::try_current()?,
215            };
216            let kms_client = match self.kms_client_factory {
217                Some(factory) => Some(factory.create_kms_client(&merged_props).await?),
218                None => None,
219            };
220            SqlCatalog::new(
221                name,
222                catalog_properties,
223                merged_props,
224                self.storage_factory,
225                runtime,
226                kms_client,
227            )
228            .await
229        }
230    }
231}
232
233fn parse_sql_bind_style(
234    properties: &HashMap<String, String>,
235    key: &str,
236    additional_keys: &[&str],
237    default: SqlBindStyle,
238) -> Result<SqlBindStyle> {
239    let configured_value = properties.get(key).map(|value| (key, value)).or_else(|| {
240        additional_keys
241            .iter()
242            .find_map(|alt_key| properties.get(*alt_key).map(|value| (*alt_key, value)))
243    });
244
245    configured_value.map_or(Ok(default), |(configured_key, value)| {
246        SqlBindStyle::from_str(value).map_err(|_| {
247            Error::new(
248                ErrorKind::DataInvalid,
249                format!(
250                    "`{}` values are valid only if they're `{}` or `{}`",
251                    configured_key,
252                    SqlBindStyle::DollarNumeric,
253                    SqlBindStyle::QMark
254                ),
255            )
256        })
257    })
258}
259
260fn parse_schema_version(value: &str) -> Result<SchemaVersion> {
261    SchemaVersion::from_str(value).map_err(|_| {
262        Error::new(
263            ErrorKind::DataInvalid,
264            format!(
265                "`{}` values are valid only if they're `{}` or `{}`",
266                SQL_CATALOG_PROP_SCHEMA_VERSION,
267                SchemaVersion::V0,
268                SchemaVersion::V1
269            ),
270        )
271    })
272}
273
274#[derive(Debug, Properties)]
275pub(crate) struct SqlCatalogProperties {
276    #[property(key = SQL_CATALOG_PROP_URI, default = "")]
277    uri: String,
278    #[property(key = SQL_CATALOG_PROP_WAREHOUSE, default = "")]
279    warehouse_location: String,
280    #[property(
281        key = SQL_CATALOG_PROP_BIND_STYLE,
282        additional_keys = [SQL_CATALOG_PROP_BIND_STYLE_LEGACY],
283        default = SqlBindStyle::DollarNumeric,
284        parse_properties_with = parse_sql_bind_style
285    )]
286    sql_bind_style: SqlBindStyle,
287    #[property(
288        key = SQL_CATALOG_PROP_SCHEMA_VERSION,
289        default = None,
290        parse_with = parse_schema_version
291    )]
292    schema_version: Option<SchemaVersion>,
293    #[property(
294        key = "pool.max-connections",
295        default = MAX_CONNECTIONS,
296        parse_with = parse_pool_property
297    )]
298    max_connections: u32,
299    #[property(
300        key = "pool.idle-timeout",
301        default = IDLE_TIMEOUT,
302        parse_with = parse_pool_property
303    )]
304    idle_timeout: u64,
305    #[property(
306        key = "pool.test-before-acquire",
307        default = TEST_BEFORE_ACQUIRE,
308        parse_with = parse_pool_property
309    )]
310    test_before_acquire: bool,
311}
312
313#[derive(Debug)]
314/// SQL catalog implementation.
315///
316/// The catalog supports SQL catalog schema V1, as well as limited support for V0.
317/// Catalogs can opt-in to automatic migration by configuring the `sql.schema-version` catalog property.
318pub struct SqlCatalog {
319    name: String,
320    properties: SqlCatalogProperties,
321    connection: AnyPool,
322    fileio: FileIO,
323    runtime: Runtime,
324    kms_client: Option<Arc<dyn KeyManagementClient>>,
325    schema_version: SchemaVersion,
326}
327
328#[derive(Debug, Clone, Copy, PartialEq, strum::EnumString, strum::Display)]
329#[strum(ascii_case_insensitive)]
330/// Schema version of the `iceberg_tables` catalog table.
331pub enum SchemaVersion {
332    /// Original schema without the `iceberg_type` column.
333    V0,
334    /// Extended schema with the `iceberg_type` column for view support.
335    V1,
336}
337
338impl SchemaVersion {
339    /// Detect the schema version of an existing catalog table by introspecting its columns.
340    async fn detect(pool: &AnyPool) -> Result<Self> {
341        let catalog_table_description = pool
342            .describe(&format!("SELECT * FROM {CATALOG_TABLE_NAME}"))
343            .await
344            .map_err(from_sqlx_error)?;
345
346        let has_type_column = catalog_table_description.columns().iter().any(|column| {
347            column
348                .name()
349                .eq_ignore_ascii_case(CATALOG_FIELD_RECORD_TYPE)
350        });
351
352        Ok(if has_type_column {
353            SchemaVersion::V1
354        } else {
355            SchemaVersion::V0
356        })
357    }
358
359    /// The trailing SQL `AND` clause used to exclude view rows when querying for tables.
360    ///
361    /// `V1` schemas carry an `iceberg_type` column, so table rows are those tagged `TABLE`
362    /// (or `NULL`, for rows written before the column existed). `V0` schemas have no such
363    /// column, so no filter is applied.
364    fn record_type_filter(self) -> String {
365        match self {
366            SchemaVersion::V1 => format!(
367                "AND ({CATALOG_FIELD_RECORD_TYPE} = '{CATALOG_FIELD_TABLE_RECORD_TYPE}' \
368                 OR {CATALOG_FIELD_RECORD_TYPE} IS NULL)"
369            ),
370            SchemaVersion::V0 => String::new(),
371        }
372    }
373
374    /// The SQL needed to migrate a `V0` catalog table up to this schema version.
375    ///
376    /// Returns `None` when the target version requires no migration (i.e. `V0`).
377    fn migration_sql(self) -> Option<String> {
378        match self {
379            SchemaVersion::V1 => Some(format!(
380                "ALTER TABLE {CATALOG_TABLE_NAME} ADD COLUMN {CATALOG_FIELD_RECORD_TYPE} VARCHAR(5)"
381            )),
382            SchemaVersion::V0 => None,
383        }
384    }
385}
386
387#[derive(Debug, PartialEq, strum::EnumString, strum::Display)]
388/// Set the SQL parameter bind style to either $1..$N (Postgres style) or ? (SQLite/MySQL/MariaDB)
389pub enum SqlBindStyle {
390    /// DollarNumeric uses parameters of the form `$1..$N``, which is the Postgres style
391    DollarNumeric,
392    /// QMark uses parameters of the form `?` which is the style for other dialects (SQLite/MySQL/MariaDB)
393    QMark,
394}
395
396impl SqlCatalog {
397    /// Create new sql catalog instance
398    async fn new(
399        name: String,
400        properties: SqlCatalogProperties,
401        props: HashMap<String, String>,
402        storage_factory: Option<Arc<dyn StorageFactory>>,
403        runtime: Runtime,
404        kms_client: Option<Arc<dyn KeyManagementClient>>,
405    ) -> Result<Self> {
406        let factory = storage_factory.ok_or_else(|| {
407            Error::new(
408                ErrorKind::Unexpected,
409                "StorageFactory must be provided for SqlCatalog. Use `with_storage_factory` to configure it.",
410            )
411        })?;
412        install_default_drivers();
413        // Forward the complete property map so storage-backend keys reach FileIO.
414        // Unrecognized keys are ignored by backends.
415        let fileio = FileIOBuilder::new(factory).with_props(props).build();
416
417        let pool = AnyPoolOptions::new()
418            .max_connections(properties.max_connections)
419            .idle_timeout(Duration::from_secs(properties.idle_timeout))
420            .test_before_acquire(properties.test_before_acquire)
421            .connect(&properties.uri)
422            .await
423            .map_err(from_sqlx_error)?;
424
425        sqlx::query(&format!(
426            "CREATE TABLE IF NOT EXISTS {CATALOG_TABLE_NAME} (
427                {CATALOG_FIELD_CATALOG_NAME} VARCHAR(255) NOT NULL,
428                {CATALOG_FIELD_TABLE_NAMESPACE} VARCHAR(255) NOT NULL,
429                {CATALOG_FIELD_TABLE_NAME} VARCHAR(255) NOT NULL,
430                {CATALOG_FIELD_METADATA_LOCATION_PROP} VARCHAR(1000),
431                {CATALOG_FIELD_PREVIOUS_METADATA_LOCATION_PROP} VARCHAR(1000),
432                {CATALOG_FIELD_RECORD_TYPE} VARCHAR(5),
433                PRIMARY KEY ({CATALOG_FIELD_CATALOG_NAME}, {CATALOG_FIELD_TABLE_NAMESPACE}, {CATALOG_FIELD_TABLE_NAME}))"
434        ))
435        .execute(&pool)
436        .await
437        .map_err(from_sqlx_error)?;
438
439        sqlx::query(&format!(
440            "CREATE TABLE IF NOT EXISTS {NAMESPACE_TABLE_NAME} (
441                {CATALOG_FIELD_CATALOG_NAME} VARCHAR(255) NOT NULL,
442                {NAMESPACE_FIELD_NAME} VARCHAR(255) NOT NULL,
443                {NAMESPACE_FIELD_PROPERTY_KEY} VARCHAR(255),
444                {NAMESPACE_FIELD_PROPERTY_VALUE} VARCHAR(1000),
445                PRIMARY KEY ({CATALOG_FIELD_CATALOG_NAME}, {NAMESPACE_FIELD_NAME}, {NAMESPACE_FIELD_PROPERTY_KEY}))"
446        ))
447        .execute(&pool)
448        .await
449        .map_err(from_sqlx_error)?;
450
451        let detected_schema_version = SchemaVersion::detect(&pool).await?;
452        let expected_schema_version = properties.schema_version;
453
454        // Detect schema by describing columns. If expected is configured then automigrate, otherwise gracefully support older schemas.
455        let schema_version = match (detected_schema_version, expected_schema_version) {
456            (SchemaVersion::V1, Some(SchemaVersion::V1) | None) => {
457                tracing::debug!(
458                    "detected {CATALOG_TABLE_NAME} schema {} which already supports views",
459                    detected_schema_version,
460                );
461                SchemaVersion::V1
462            }
463            (SchemaVersion::V0, Some(expected_schema_version @ SchemaVersion::V1)) => {
464                tracing::warn!(
465                    "table {CATALOG_TABLE_NAME} has inferred schema {} but expected schema {}, performing migration",
466                    detected_schema_version,
467                    expected_schema_version,
468                );
469                if let Some(migration_sql) = SchemaVersion::V1.migration_sql() {
470                    sqlx::query(&migration_sql)
471                        .execute(&pool)
472                        .await
473                        .map_err(from_sqlx_error)?;
474                }
475                SchemaVersion::V1
476            }
477            (SchemaVersion::V0, Some(SchemaVersion::V0) | None) => {
478                tracing::warn!(
479                    "table {CATALOG_TABLE_NAME} has inferred schema {}; SQL catalog is initialized without view support, table creation, and table registration. \
480                    To auto-migrate the database schema, set {}=V1",
481                    detected_schema_version,
482                    SQL_CATALOG_PROP_SCHEMA_VERSION,
483                );
484                SchemaVersion::V0
485            }
486            (SchemaVersion::V1, Some(expected_schema_version @ SchemaVersion::V0)) => {
487                tracing::warn!(
488                    "ignoring expected schema {} for table {CATALOG_TABLE_NAME}: the table is \
489                    already at schema {}, and downgrade migration is not supported",
490                    expected_schema_version,
491                    detected_schema_version,
492                );
493                SchemaVersion::V1
494            }
495        };
496
497        Ok(SqlCatalog {
498            name,
499            properties,
500            connection: pool,
501            fileio,
502            runtime,
503            kms_client,
504            schema_version,
505        })
506    }
507
508    /// SQLX Any does not implement PostgresSQL bindings, so we have to do this.
509    fn replace_placeholders(&self, query: &str) -> String {
510        match self.properties.sql_bind_style {
511            SqlBindStyle::DollarNumeric => {
512                let mut count = 1;
513                query
514                    .chars()
515                    .fold(String::with_capacity(query.len()), |mut acc, c| {
516                        if c == '?' {
517                            acc.push('$');
518                            acc.push_str(&count.to_string());
519                            count += 1;
520                        } else {
521                            acc.push(c);
522                        }
523                        acc
524                    })
525            }
526            _ => query.to_owned(),
527        }
528    }
529
530    /// Fetch a vec of AnyRows from a given query
531    async fn fetch_rows(&self, query: &str, args: Vec<Option<&str>>) -> Result<Vec<AnyRow>> {
532        let query_with_placeholders = self.replace_placeholders(query);
533
534        let mut sqlx_query = sqlx::query(&query_with_placeholders);
535        for arg in args {
536            sqlx_query = sqlx_query.bind(arg);
537        }
538
539        sqlx_query
540            .fetch_all(&self.connection)
541            .await
542            .map_err(from_sqlx_error)
543    }
544
545    /// Execute statements in a transaction, provided or not
546    async fn execute(
547        &self,
548        query: &str,
549        args: Vec<Option<&str>>,
550        transaction: Option<&mut Transaction<'_, Any>>,
551    ) -> Result<AnyQueryResult> {
552        let query_with_placeholders = self.replace_placeholders(query);
553
554        let mut sqlx_query = sqlx::query(&query_with_placeholders);
555        for arg in args {
556            sqlx_query = sqlx_query.bind(arg);
557        }
558
559        match transaction {
560            Some(t) => sqlx_query.execute(&mut **t).await.map_err(from_sqlx_error),
561            None => {
562                let mut tx = self.connection.begin().await.map_err(from_sqlx_error)?;
563                let result = sqlx_query
564                    .execute(&mut *tx)
565                    .await
566                    .map_err(from_sqlx_error)?;
567                tx.commit().await.map_err(from_sqlx_error)?;
568                Ok(result)
569            }
570        }
571    }
572}
573
574#[async_trait]
575impl Catalog for SqlCatalog {
576    async fn list_namespaces(
577        &self,
578        parent: Option<&NamespaceIdent>,
579    ) -> Result<Vec<NamespaceIdent>> {
580        // UNION will remove duplicates.
581        let all_namespaces_stmt = format!(
582            "SELECT {CATALOG_FIELD_TABLE_NAMESPACE}
583             FROM {CATALOG_TABLE_NAME}
584             WHERE {CATALOG_FIELD_CATALOG_NAME} = ?
585             UNION
586             SELECT {NAMESPACE_FIELD_NAME}
587             FROM {NAMESPACE_TABLE_NAME}
588             WHERE {CATALOG_FIELD_CATALOG_NAME} = ?"
589        );
590
591        let namespace_rows = self
592            .fetch_rows(&all_namespaces_stmt, vec![
593                Some(&self.name),
594                Some(&self.name),
595            ])
596            .await?;
597
598        let mut namespaces = HashSet::<NamespaceIdent>::with_capacity(namespace_rows.len());
599
600        if let Some(parent) = parent {
601            if self.namespace_exists(parent).await? {
602                let parent_str = parent.join(".");
603
604                for row in namespace_rows.iter() {
605                    let nsp = row.try_get::<String, _>(0).map_err(from_sqlx_error)?;
606                    // if parent = a, then we only want to see a.b, a.c returned.
607                    if nsp != parent_str && nsp.starts_with(&parent_str) {
608                        namespaces.insert(NamespaceIdent::from_strs(nsp.split("."))?);
609                    }
610                }
611
612                Ok(namespaces.into_iter().collect::<Vec<NamespaceIdent>>())
613            } else {
614                no_such_namespace_err(parent)
615            }
616        } else {
617            for row in namespace_rows.iter() {
618                let nsp = row.try_get::<String, _>(0).map_err(from_sqlx_error)?;
619                let mut levels = nsp.split(".").collect::<Vec<&str>>();
620                if !levels.is_empty() {
621                    let first_level = levels.drain(..1).collect::<Vec<&str>>();
622                    namespaces.insert(NamespaceIdent::from_strs(first_level)?);
623                }
624            }
625
626            Ok(namespaces.into_iter().collect::<Vec<NamespaceIdent>>())
627        }
628    }
629
630    async fn create_namespace(
631        &self,
632        namespace: &NamespaceIdent,
633        properties: HashMap<String, String>,
634    ) -> Result<Namespace> {
635        let exists = self.namespace_exists(namespace).await?;
636
637        if exists {
638            return Err(Error::new(
639                ErrorKind::NamespaceAlreadyExists,
640                format!("Namespace {namespace:?} already exists"),
641            ));
642        }
643
644        let namespace_str = namespace.join(".");
645        let insert = format!(
646            "INSERT INTO {NAMESPACE_TABLE_NAME} ({CATALOG_FIELD_CATALOG_NAME}, {NAMESPACE_FIELD_NAME}, {NAMESPACE_FIELD_PROPERTY_KEY}, {NAMESPACE_FIELD_PROPERTY_VALUE})
647             VALUES (?, ?, ?, ?)");
648        if !properties.is_empty() {
649            let mut insert_properties = properties.clone();
650            insert_properties.insert("exists".to_string(), "true".to_string());
651
652            let mut query_args = Vec::with_capacity(insert_properties.len() * 4);
653            let mut insert_stmt = insert.clone();
654            for (index, (key, value)) in insert_properties.iter().enumerate() {
655                query_args.extend_from_slice(&[
656                    Some(self.name.as_str()),
657                    Some(namespace_str.as_str()),
658                    Some(key.as_str()),
659                    Some(value.as_str()),
660                ]);
661                if index > 0 {
662                    insert_stmt.push_str(", (?, ?, ?, ?)");
663                }
664            }
665
666            self.execute(&insert_stmt, query_args, None).await?;
667
668            Ok(Namespace::with_properties(
669                namespace.clone(),
670                insert_properties,
671            ))
672        } else {
673            // set a default property of exists = true
674            self.execute(
675                &insert,
676                vec![
677                    Some(&self.name),
678                    Some(&namespace_str),
679                    Some("exists"),
680                    Some("true"),
681                ],
682                None,
683            )
684            .await?;
685            Ok(Namespace::with_properties(namespace.clone(), properties))
686        }
687    }
688
689    async fn get_namespace(&self, namespace: &NamespaceIdent) -> Result<Namespace> {
690        let exists = self.namespace_exists(namespace).await?;
691        if exists {
692            let namespace_props = self
693                .fetch_rows(
694                    &format!(
695                        "SELECT
696                            {NAMESPACE_FIELD_NAME},
697                            {NAMESPACE_FIELD_PROPERTY_KEY},
698                            {NAMESPACE_FIELD_PROPERTY_VALUE}
699                            FROM {NAMESPACE_TABLE_NAME}
700                            WHERE {CATALOG_FIELD_CATALOG_NAME} = ?
701                            AND {NAMESPACE_FIELD_NAME} = ?"
702                    ),
703                    vec![Some(&self.name), Some(&namespace.join("."))],
704                )
705                .await?;
706
707            let mut properties = HashMap::with_capacity(namespace_props.len());
708
709            for row in namespace_props {
710                let key = row
711                    .try_get::<String, _>(NAMESPACE_FIELD_PROPERTY_KEY)
712                    .map_err(from_sqlx_error)?;
713                let value = row
714                    .try_get::<String, _>(NAMESPACE_FIELD_PROPERTY_VALUE)
715                    .map_err(from_sqlx_error)?;
716
717                properties.insert(key, value);
718            }
719
720            Ok(Namespace::with_properties(namespace.clone(), properties))
721        } else {
722            no_such_namespace_err(namespace)
723        }
724    }
725
726    async fn namespace_exists(&self, namespace: &NamespaceIdent) -> Result<bool> {
727        let namespace_str = namespace.join(".");
728
729        let table_namespaces = self
730            .fetch_rows(
731                &format!(
732                    "SELECT 1 FROM {CATALOG_TABLE_NAME}
733                     WHERE {CATALOG_FIELD_CATALOG_NAME} = ?
734                      AND {CATALOG_FIELD_TABLE_NAMESPACE} = ?
735                     LIMIT 1"
736                ),
737                vec![Some(&self.name), Some(&namespace_str)],
738            )
739            .await?;
740
741        if !table_namespaces.is_empty() {
742            Ok(true)
743        } else {
744            let namespaces = self
745                .fetch_rows(
746                    &format!(
747                        "SELECT 1 FROM {NAMESPACE_TABLE_NAME}
748                         WHERE {CATALOG_FIELD_CATALOG_NAME} = ?
749                          AND {NAMESPACE_FIELD_NAME} = ?
750                         LIMIT 1"
751                    ),
752                    vec![Some(&self.name), Some(&namespace_str)],
753                )
754                .await?;
755            if !namespaces.is_empty() {
756                Ok(true)
757            } else {
758                Ok(false)
759            }
760        }
761    }
762
763    async fn update_namespace(
764        &self,
765        namespace: &NamespaceIdent,
766        properties: HashMap<String, String>,
767    ) -> Result<()> {
768        let exists = self.namespace_exists(namespace).await?;
769        if exists {
770            let existing_properties = self.get_namespace(namespace).await?.properties().clone();
771            let namespace_str = namespace.join(".");
772
773            let mut updates = vec![];
774            let mut inserts = vec![];
775
776            for (key, value) in properties.iter() {
777                if existing_properties.contains_key(key) {
778                    if existing_properties.get(key) != Some(value) {
779                        updates.push((key, value));
780                    }
781                } else {
782                    inserts.push((key, value));
783                }
784            }
785
786            let mut tx = self.connection.begin().await.map_err(from_sqlx_error)?;
787            let update_stmt = format!(
788                "UPDATE {NAMESPACE_TABLE_NAME} SET {NAMESPACE_FIELD_PROPERTY_VALUE} = ?
789                 WHERE {CATALOG_FIELD_CATALOG_NAME} = ?
790                 AND {NAMESPACE_FIELD_NAME} = ?
791                 AND {NAMESPACE_FIELD_PROPERTY_KEY} = ?"
792            );
793
794            let insert_stmt = format!(
795                "INSERT INTO {NAMESPACE_TABLE_NAME} ({CATALOG_FIELD_CATALOG_NAME}, {NAMESPACE_FIELD_NAME}, {NAMESPACE_FIELD_PROPERTY_KEY}, {NAMESPACE_FIELD_PROPERTY_VALUE})
796                 VALUES (?, ?, ?, ?)"
797            );
798
799            for (key, value) in updates {
800                self.execute(
801                    &update_stmt,
802                    vec![
803                        Some(value),
804                        Some(&self.name),
805                        Some(&namespace_str),
806                        Some(key),
807                    ],
808                    Some(&mut tx),
809                )
810                .await?;
811            }
812
813            for (key, value) in inserts {
814                self.execute(
815                    &insert_stmt,
816                    vec![
817                        Some(&self.name),
818                        Some(&namespace_str),
819                        Some(key),
820                        Some(value),
821                    ],
822                    Some(&mut tx),
823                )
824                .await?;
825            }
826
827            let _ = tx.commit().await.map_err(from_sqlx_error)?;
828
829            Ok(())
830        } else {
831            no_such_namespace_err(namespace)
832        }
833    }
834
835    async fn drop_namespace(&self, namespace: &NamespaceIdent) -> Result<()> {
836        let exists = self.namespace_exists(namespace).await?;
837        if exists {
838            // if there are tables in the namespace, don't allow drop.
839            let tables = self.list_tables(namespace).await?;
840            if !tables.is_empty() {
841                return Err(Error::new(
842                    ErrorKind::Unexpected,
843                    format!(
844                        "Namespace {:?} is not empty. {} tables exist.",
845                        namespace,
846                        tables.len()
847                    ),
848                ));
849            }
850
851            self.execute(
852                &format!(
853                    "DELETE FROM {NAMESPACE_TABLE_NAME}
854                     WHERE {NAMESPACE_FIELD_NAME} = ?
855                      AND {CATALOG_FIELD_CATALOG_NAME} = ?"
856                ),
857                vec![Some(&namespace.join(".")), Some(&self.name)],
858                None,
859            )
860            .await?;
861
862            Ok(())
863        } else {
864            no_such_namespace_err(namespace)
865        }
866    }
867
868    async fn list_tables(&self, namespace: &NamespaceIdent) -> Result<Vec<TableIdent>> {
869        let exists = self.namespace_exists(namespace).await?;
870        if exists {
871            let rows = self
872                .fetch_rows(
873                    &format!(
874                        "SELECT {CATALOG_FIELD_TABLE_NAME},
875                                {CATALOG_FIELD_TABLE_NAMESPACE}
876                         FROM {CATALOG_TABLE_NAME}
877                         WHERE {CATALOG_FIELD_TABLE_NAMESPACE} = ?
878                          AND {CATALOG_FIELD_CATALOG_NAME} = ?
879                          {}",
880                        self.schema_version.record_type_filter()
881                    ),
882                    vec![Some(&namespace.join(".")), Some(&self.name)],
883                )
884                .await?;
885
886            let mut tables = HashSet::<TableIdent>::with_capacity(rows.len());
887
888            for row in rows.iter() {
889                let tbl = row
890                    .try_get::<String, _>(CATALOG_FIELD_TABLE_NAME)
891                    .map_err(from_sqlx_error)?;
892                let ns_strs = row
893                    .try_get::<String, _>(CATALOG_FIELD_TABLE_NAMESPACE)
894                    .map_err(from_sqlx_error)?;
895                let ns = NamespaceIdent::from_strs(ns_strs.split("."))?;
896                tables.insert(TableIdent::new(ns, tbl));
897            }
898
899            Ok(tables.into_iter().collect::<Vec<TableIdent>>())
900        } else {
901            no_such_namespace_err(namespace)
902        }
903    }
904
905    async fn table_exists(&self, identifier: &TableIdent) -> Result<bool> {
906        let namespace = identifier.namespace().join(".");
907        let table_name = identifier.name();
908        let table_counts = self
909            .fetch_rows(
910                &format!(
911                    "SELECT 1
912                     FROM {CATALOG_TABLE_NAME}
913                     WHERE {CATALOG_FIELD_TABLE_NAMESPACE} = ?
914                      AND {CATALOG_FIELD_CATALOG_NAME} = ?
915                      AND {CATALOG_FIELD_TABLE_NAME} = ?
916                      {}",
917                    self.schema_version.record_type_filter()
918                ),
919                vec![Some(&namespace), Some(&self.name), Some(table_name)],
920            )
921            .await?;
922
923        if !table_counts.is_empty() {
924            Ok(true)
925        } else {
926            Ok(false)
927        }
928    }
929
930    async fn drop_table(&self, identifier: &TableIdent) -> Result<()> {
931        if !self.table_exists(identifier).await? {
932            return no_such_table_err(identifier);
933        }
934
935        self.execute(
936            &format!(
937                "DELETE FROM {CATALOG_TABLE_NAME}
938                 WHERE {CATALOG_FIELD_CATALOG_NAME} = ?
939                  AND {CATALOG_FIELD_TABLE_NAME} = ?
940                  AND {CATALOG_FIELD_TABLE_NAMESPACE} = ?
941                  {}",
942                self.schema_version.record_type_filter()
943            ),
944            vec![
945                Some(&self.name),
946                Some(identifier.name()),
947                Some(&identifier.namespace().join(".")),
948            ],
949            None,
950        )
951        .await?;
952
953        Ok(())
954    }
955
956    async fn purge_table(&self, table: &TableIdent) -> Result<()> {
957        let table_info = self.load_table(table).await?;
958        self.drop_table(table).await?;
959        iceberg::drop_table_data(&table_info).await
960    }
961
962    async fn load_table(&self, identifier: &TableIdent) -> Result<Table> {
963        if !self.table_exists(identifier).await? {
964            return no_such_table_err(identifier);
965        }
966
967        let rows = self
968            .fetch_rows(
969                &format!(
970                    "SELECT {CATALOG_FIELD_METADATA_LOCATION_PROP}
971                     FROM {CATALOG_TABLE_NAME}
972                     WHERE {CATALOG_FIELD_CATALOG_NAME} = ?
973                      AND {CATALOG_FIELD_TABLE_NAME} = ?
974                      AND {CATALOG_FIELD_TABLE_NAMESPACE} = ?
975                      {}",
976                    self.schema_version.record_type_filter()
977                ),
978                vec![
979                    Some(&self.name),
980                    Some(identifier.name()),
981                    Some(&identifier.namespace().join(".")),
982                ],
983            )
984            .await?;
985
986        if rows.is_empty() {
987            return no_such_table_err(identifier);
988        }
989
990        let row = &rows[0];
991        let tbl_metadata_location = row
992            .try_get::<String, _>(CATALOG_FIELD_METADATA_LOCATION_PROP)
993            .map_err(from_sqlx_error)?;
994
995        let metadata = TableMetadata::read_from(&self.fileio, &tbl_metadata_location).await?;
996
997        let mut builder = Table::builder()
998            .file_io(self.fileio.clone())
999            .identifier(identifier.clone())
1000            .metadata_location(tbl_metadata_location)
1001            .metadata(metadata)
1002            .runtime(self.runtime.clone());
1003        if let Some(kms_client) = self.kms_client.clone() {
1004            builder = builder.kms_client(kms_client);
1005        }
1006        Ok(builder.build()?)
1007    }
1008
1009    async fn create_table(
1010        &self,
1011        namespace: &NamespaceIdent,
1012        creation: TableCreation,
1013    ) -> Result<Table> {
1014        if self.schema_version != SchemaVersion::V1 {
1015            return Err(Error::new(
1016                ErrorKind::FeatureUnsupported,
1017                format!(
1018                    "Table creation is not supported for SQL catalog schema version {}",
1019                    self.schema_version
1020                ),
1021            ));
1022        }
1023
1024        if !self.namespace_exists(namespace).await? {
1025            return no_such_namespace_err(namespace);
1026        }
1027
1028        let tbl_name = creation.name.clone();
1029        let tbl_ident = TableIdent::new(namespace.clone(), tbl_name.clone());
1030
1031        if self.table_exists(&tbl_ident).await? {
1032            return table_already_exists_err(&tbl_ident);
1033        }
1034
1035        let tbl_creation = if creation.location.is_some() {
1036            creation
1037        } else {
1038            // fall back to namespace-specific location
1039            // and then to warehouse location
1040            let nsp_properties = self.get_namespace(namespace).await?.properties().clone();
1041            let nsp_location = match nsp_properties.get(NAMESPACE_LOCATION_PROPERTY_KEY) {
1042                Some(location) => location.clone(),
1043                None => {
1044                    format!(
1045                        "{}/{}",
1046                        &self.properties.warehouse_location,
1047                        namespace.join("/")
1048                    )
1049                }
1050            };
1051
1052            let tbl_location = format!("{}/{}", nsp_location, tbl_ident.name());
1053
1054            TableCreation {
1055                location: Some(tbl_location),
1056                ..creation
1057            }
1058        };
1059
1060        let tbl_metadata = TableMetadataBuilder::from_table_creation(tbl_creation)?
1061            .build()?
1062            .metadata;
1063        let tbl_metadata_location = MetadataLocation::try_new_with_metadata(&tbl_metadata)?;
1064
1065        tbl_metadata
1066            .write_to(&self.fileio, &tbl_metadata_location)
1067            .await?;
1068
1069        let tbl_metadata_location_str = tbl_metadata_location.to_string();
1070        self.execute(&format!(
1071            "INSERT INTO {CATALOG_TABLE_NAME}
1072             ({CATALOG_FIELD_CATALOG_NAME}, {CATALOG_FIELD_TABLE_NAMESPACE}, {CATALOG_FIELD_TABLE_NAME}, {CATALOG_FIELD_METADATA_LOCATION_PROP}, {CATALOG_FIELD_RECORD_TYPE})
1073             VALUES (?, ?, ?, ?, ?)
1074            "), vec![Some(&self.name), Some(&namespace.join(".")), Some(&tbl_name.clone()), Some(&tbl_metadata_location_str), Some(CATALOG_FIELD_TABLE_RECORD_TYPE)], None).await?;
1075
1076        let mut builder = Table::builder()
1077            .file_io(self.fileio.clone())
1078            .metadata_location(tbl_metadata_location_str)
1079            .identifier(tbl_ident)
1080            .metadata(tbl_metadata)
1081            .runtime(self.runtime.clone());
1082        if let Some(kms_client) = self.kms_client.clone() {
1083            builder = builder.kms_client(kms_client);
1084        }
1085        Ok(builder.build()?)
1086    }
1087
1088    async fn rename_table(&self, src: &TableIdent, dest: &TableIdent) -> Result<()> {
1089        if src == dest {
1090            return Ok(());
1091        }
1092
1093        if !self.table_exists(src).await? {
1094            return no_such_table_err(src);
1095        }
1096
1097        if !self.namespace_exists(dest.namespace()).await? {
1098            return no_such_namespace_err(dest.namespace());
1099        }
1100
1101        if self.table_exists(dest).await? {
1102            return table_already_exists_err(dest);
1103        }
1104
1105        self.execute(
1106            &format!(
1107                "UPDATE {CATALOG_TABLE_NAME}
1108                 SET {CATALOG_FIELD_TABLE_NAME} = ?, {CATALOG_FIELD_TABLE_NAMESPACE} = ?
1109                 WHERE {CATALOG_FIELD_CATALOG_NAME} = ?
1110                  AND {CATALOG_FIELD_TABLE_NAME} = ?
1111                  AND {CATALOG_FIELD_TABLE_NAMESPACE} = ?
1112                  {}",
1113                self.schema_version.record_type_filter()
1114            ),
1115            vec![
1116                Some(dest.name()),
1117                Some(&dest.namespace().join(".")),
1118                Some(&self.name),
1119                Some(src.name()),
1120                Some(&src.namespace().join(".")),
1121            ],
1122            None,
1123        )
1124        .await?;
1125
1126        Ok(())
1127    }
1128
1129    async fn register_table(
1130        &self,
1131        table_ident: &TableIdent,
1132        metadata_location: String,
1133    ) -> Result<Table> {
1134        if self.schema_version != SchemaVersion::V1 {
1135            return Err(Error::new(
1136                ErrorKind::FeatureUnsupported,
1137                format!(
1138                    "Table registration is not supported for SQL catalog schema version {}",
1139                    self.schema_version
1140                ),
1141            ));
1142        }
1143
1144        if self.table_exists(table_ident).await? {
1145            return table_already_exists_err(table_ident);
1146        }
1147
1148        let metadata = TableMetadata::read_from(&self.fileio, &metadata_location).await?;
1149
1150        let namespace = table_ident.namespace();
1151        let tbl_name = table_ident.name().to_string();
1152
1153        self.execute(&format!(
1154            "INSERT INTO {CATALOG_TABLE_NAME}
1155             ({CATALOG_FIELD_CATALOG_NAME}, {CATALOG_FIELD_TABLE_NAMESPACE}, {CATALOG_FIELD_TABLE_NAME}, {CATALOG_FIELD_METADATA_LOCATION_PROP}, {CATALOG_FIELD_RECORD_TYPE})
1156             VALUES (?, ?, ?, ?, ?)
1157            "), vec![Some(&self.name), Some(&namespace.join(".")), Some(&tbl_name), Some(&metadata_location), Some(CATALOG_FIELD_TABLE_RECORD_TYPE)], None).await?;
1158
1159        let mut builder = Table::builder()
1160            .identifier(table_ident.clone())
1161            .metadata_location(metadata_location)
1162            .metadata(metadata)
1163            .file_io(self.fileio.clone())
1164            .runtime(self.runtime.clone());
1165        if let Some(kms_client) = self.kms_client.clone() {
1166            builder = builder.kms_client(kms_client);
1167        }
1168        Ok(builder.build()?)
1169    }
1170
1171    /// Updates an existing table within the SQL catalog.
1172    async fn update_table(&self, commit: TableCommit) -> Result<Table> {
1173        let table_ident = commit.identifier().clone();
1174        let current_table = self.load_table(&table_ident).await?;
1175        let current_metadata_location = current_table.metadata_location_result()?.to_string();
1176        let base_metadata = current_table.metadata_ref();
1177
1178        let staged_table = commit.apply(current_table)?;
1179        let staged_metadata_location_str = staged_table.metadata_location_result()?;
1180        let staged_metadata_location = MetadataLocation::from_str(staged_metadata_location_str)?;
1181
1182        staged_table
1183            .metadata()
1184            .write_to(staged_table.file_io(), &staged_metadata_location)
1185            .await?;
1186
1187        let staged_metadata_location_str = staged_metadata_location.to_string();
1188        let update_result = self
1189            .execute(
1190                &format!(
1191                    "UPDATE {CATALOG_TABLE_NAME}
1192                     SET {CATALOG_FIELD_METADATA_LOCATION_PROP} = ?, {CATALOG_FIELD_PREVIOUS_METADATA_LOCATION_PROP} = ?
1193                     WHERE {CATALOG_FIELD_CATALOG_NAME} = ?
1194                      AND {CATALOG_FIELD_TABLE_NAME} = ?
1195                      AND {CATALOG_FIELD_TABLE_NAMESPACE} = ?
1196                      {}
1197                      AND {CATALOG_FIELD_METADATA_LOCATION_PROP} = ?",
1198                    self.schema_version.record_type_filter()
1199                ),
1200                vec![
1201                    Some(&staged_metadata_location_str),
1202                    Some(current_metadata_location.as_str()),
1203                    Some(&self.name),
1204                    Some(table_ident.name()),
1205                    Some(&table_ident.namespace().join(".")),
1206                    Some(current_metadata_location.as_str()),
1207                ],
1208                None,
1209            )
1210            .await?;
1211
1212        if update_result.rows_affected() == 0 {
1213            return Err(Error::new(
1214                ErrorKind::CatalogCommitConflicts,
1215                format!("Commit conflicted for table: {table_ident}"),
1216            )
1217            .with_retryable(true));
1218        }
1219
1220        delete_removed_metadata_files(
1221            staged_table.file_io(),
1222            &base_metadata,
1223            staged_table.metadata(),
1224        )
1225        .await;
1226
1227        Ok(staged_table)
1228    }
1229}
1230
1231#[cfg(test)]
1232mod tests {
1233    use std::collections::{HashMap, HashSet};
1234    use std::hash::Hash;
1235    use std::sync::Arc;
1236
1237    use iceberg::io::LocalFsStorageFactory;
1238    use iceberg::spec::{NestedField, PartitionSpec, PrimitiveType, Schema, SortOrder, Type};
1239    use iceberg::table::Table;
1240    use iceberg::{
1241        Catalog, CatalogBuilder, ErrorKind, Namespace, NamespaceIdent, TableCreation, TableIdent,
1242    };
1243    use itertools::Itertools;
1244    use regex::Regex;
1245    use sqlx::any::install_default_drivers;
1246    use sqlx::migrate::MigrateDatabase;
1247    use sqlx::{Column, Executor};
1248    use tempfile::TempDir;
1249
1250    use crate::catalog::{
1251        CATALOG_FIELD_RECORD_TYPE, CATALOG_TABLE_NAME, IDLE_TIMEOUT, MAX_CONNECTIONS,
1252        NAMESPACE_LOCATION_PROPERTY_KEY, NAMESPACE_TABLE_NAME, SQL_CATALOG_PROP_BIND_STYLE,
1253        SQL_CATALOG_PROP_BIND_STYLE_LEGACY, SQL_CATALOG_PROP_SCHEMA_VERSION, SQL_CATALOG_PROP_URI,
1254        SQL_CATALOG_PROP_WAREHOUSE, SqlCatalogProperties, TEST_BEFORE_ACQUIRE,
1255    };
1256    use crate::{SchemaVersion, SqlBindStyle, SqlCatalog, SqlCatalogBuilder};
1257
1258    const UUID_REGEX_STR: &str = "[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}";
1259
1260    fn temp_path() -> String {
1261        let temp_dir = TempDir::new().unwrap();
1262        temp_dir.path().to_str().unwrap().to_string()
1263    }
1264
1265    #[test]
1266    fn test_catalog_properties() {
1267        let properties = SqlCatalogProperties::from_properties(&HashMap::from([
1268            (
1269                SQL_CATALOG_PROP_URI.to_string(),
1270                "sqlite://catalog".to_string(),
1271            ),
1272            (
1273                SQL_CATALOG_PROP_WAREHOUSE.to_string(),
1274                "/warehouse".to_string(),
1275            ),
1276            (
1277                SQL_CATALOG_PROP_BIND_STYLE_LEGACY.to_string(),
1278                SqlBindStyle::QMark.to_string(),
1279            ),
1280            (
1281                SQL_CATALOG_PROP_SCHEMA_VERSION.to_string(),
1282                "V1".to_string(),
1283            ),
1284            ("pool.max-connections".to_string(), "5".to_string()),
1285            ("pool.idle-timeout".to_string(), "20".to_string()),
1286            ("pool.test-before-acquire".to_string(), "false".to_string()),
1287        ]))
1288        .unwrap();
1289
1290        assert_eq!(properties.uri, "sqlite://catalog");
1291        assert_eq!(properties.warehouse_location, "/warehouse");
1292        assert_eq!(properties.sql_bind_style, SqlBindStyle::QMark);
1293        assert_eq!(properties.schema_version, Some(SchemaVersion::V1));
1294        assert_eq!(properties.max_connections, 5);
1295        assert_eq!(properties.idle_timeout, 20);
1296        assert!(!properties.test_before_acquire);
1297    }
1298
1299    #[test]
1300    fn test_catalog_properties_defaults() {
1301        let properties = SqlCatalogProperties::from_properties(&HashMap::new()).unwrap();
1302
1303        assert_eq!(properties.uri, "");
1304        assert_eq!(properties.warehouse_location, "");
1305        assert_eq!(properties.sql_bind_style, SqlBindStyle::DollarNumeric);
1306        assert_eq!(properties.schema_version, None);
1307        assert_eq!(properties.max_connections, MAX_CONNECTIONS);
1308        assert_eq!(properties.idle_timeout, IDLE_TIMEOUT);
1309        assert_eq!(properties.test_before_acquire, TEST_BEFORE_ACQUIRE);
1310    }
1311
1312    #[test]
1313    fn test_preferred_bind_style_key_takes_precedence_over_legacy_key() {
1314        let properties = SqlCatalogProperties::from_properties(&HashMap::from([
1315            (
1316                SQL_CATALOG_PROP_BIND_STYLE.to_string(),
1317                SqlBindStyle::QMark.to_string(),
1318            ),
1319            (
1320                SQL_CATALOG_PROP_BIND_STYLE_LEGACY.to_string(),
1321                SqlBindStyle::DollarNumeric.to_string(),
1322            ),
1323        ]))
1324        .unwrap();
1325
1326        assert_eq!(properties.sql_bind_style, SqlBindStyle::QMark);
1327    }
1328
1329    #[test]
1330    fn test_invalid_legacy_bind_style_error_names_configured_key() {
1331        let error = SqlCatalogProperties::from_properties(&HashMap::from([(
1332            SQL_CATALOG_PROP_BIND_STYLE_LEGACY.to_string(),
1333            "invalid".to_string(),
1334        )]))
1335        .unwrap_err();
1336
1337        assert_eq!(error.kind(), ErrorKind::DataInvalid);
1338        assert_eq!(
1339            error.message(),
1340            "`sql_bind_style` values are valid only if they're `DollarNumeric` or `QMark`"
1341        );
1342    }
1343
1344    fn to_set<T: Eq + Hash>(vec: Vec<T>) -> HashSet<T> {
1345        HashSet::from_iter(vec)
1346    }
1347
1348    fn default_properties() -> HashMap<String, String> {
1349        HashMap::from([("exists".to_string(), "true".to_string())])
1350    }
1351
1352    /// Create a new SQLite catalog for testing. If name is not specified it defaults to "iceberg".
1353    async fn new_sql_catalog(
1354        warehouse_location: String,
1355        name: Option<impl ToString>,
1356    ) -> impl Catalog {
1357        let name = if let Some(name) = name {
1358            name.to_string()
1359        } else {
1360            "iceberg".to_string()
1361        };
1362        let sql_lite_uri = format!("sqlite:{}", temp_path());
1363        sqlx::Sqlite::create_database(&sql_lite_uri).await.unwrap();
1364
1365        let props = HashMap::from_iter([
1366            (SQL_CATALOG_PROP_URI.to_string(), sql_lite_uri.to_string()),
1367            (SQL_CATALOG_PROP_WAREHOUSE.to_string(), warehouse_location),
1368            (
1369                SQL_CATALOG_PROP_BIND_STYLE.to_string(),
1370                SqlBindStyle::DollarNumeric.to_string(),
1371            ),
1372        ]);
1373        SqlCatalogBuilder::default()
1374            .with_storage_factory(Arc::new(LocalFsStorageFactory))
1375            .load(&name, props)
1376            .await
1377            .unwrap()
1378    }
1379
1380    async fn create_namespace<C: Catalog>(catalog: &C, namespace_ident: &NamespaceIdent) {
1381        let _ = catalog
1382            .create_namespace(namespace_ident, HashMap::new())
1383            .await
1384            .unwrap();
1385    }
1386
1387    async fn create_namespaces<C: Catalog>(catalog: &C, namespace_idents: &Vec<&NamespaceIdent>) {
1388        for namespace_ident in namespace_idents {
1389            let _ = create_namespace(catalog, namespace_ident).await;
1390        }
1391    }
1392
1393    fn simple_table_schema() -> Schema {
1394        Schema::builder()
1395            .with_fields(vec![
1396                NestedField::required(1, "foo", Type::Primitive(PrimitiveType::Int)).into(),
1397            ])
1398            .build()
1399            .unwrap()
1400    }
1401
1402    async fn create_table<C: Catalog>(catalog: &C, table_ident: &TableIdent) {
1403        let _ = catalog
1404            .create_table(
1405                &table_ident.namespace,
1406                TableCreation::builder()
1407                    .name(table_ident.name().into())
1408                    .schema(simple_table_schema())
1409                    .location(temp_path())
1410                    .build(),
1411            )
1412            .await
1413            .unwrap();
1414    }
1415
1416    async fn create_tables<C: Catalog>(catalog: &C, table_idents: Vec<&TableIdent>) {
1417        for table_ident in table_idents {
1418            create_table(catalog, table_ident).await;
1419        }
1420    }
1421
1422    fn assert_table_eq(table: &Table, expected_table_ident: &TableIdent, expected_schema: &Schema) {
1423        assert_eq!(table.identifier(), expected_table_ident);
1424
1425        let metadata = table.metadata();
1426
1427        assert_eq!(metadata.current_schema().as_ref(), expected_schema);
1428
1429        let expected_partition_spec = PartitionSpec::builder(expected_schema.clone())
1430            .with_spec_id(0)
1431            .build()
1432            .unwrap();
1433
1434        assert_eq!(
1435            metadata
1436                .partition_specs_iter()
1437                .map(|p| p.as_ref())
1438                .collect_vec(),
1439            vec![&expected_partition_spec]
1440        );
1441
1442        let expected_sorted_order = SortOrder::builder()
1443            .with_order_id(0)
1444            .with_fields(vec![])
1445            .build(expected_schema)
1446            .unwrap();
1447
1448        assert_eq!(
1449            metadata
1450                .sort_orders_iter()
1451                .map(|s| s.as_ref())
1452                .collect_vec(),
1453            vec![&expected_sorted_order]
1454        );
1455
1456        assert_eq!(metadata.properties(), &HashMap::new());
1457
1458        assert!(!table.readonly());
1459    }
1460
1461    fn assert_table_metadata_location_matches(table: &Table, regex_str: &str) {
1462        let actual = table.metadata_location().unwrap().to_string();
1463        let regex = Regex::new(regex_str).unwrap();
1464        assert!(regex.is_match(&actual))
1465    }
1466
1467    #[tokio::test]
1468    async fn test_initialized() {
1469        let warehouse_loc = temp_path();
1470        new_sql_catalog(warehouse_loc.clone(), Some("iceberg")).await;
1471        // catalog instantiation should not fail even if tables exist
1472        new_sql_catalog(warehouse_loc.clone(), Some("iceberg")).await;
1473        new_sql_catalog(warehouse_loc.clone(), Some("iceberg")).await;
1474    }
1475
1476    async fn new_commit_error_catalog() -> SqlCatalog {
1477        let sql_lite_uri = format!("sqlite:{}", temp_path());
1478        sqlx::Sqlite::create_database(&sql_lite_uri).await.unwrap();
1479        let catalog = SqlCatalogBuilder::default()
1480            .with_storage_factory(Arc::new(LocalFsStorageFactory))
1481            .prop("pool.max-connections", "1")
1482            .load(
1483                "iceberg",
1484                HashMap::from_iter([
1485                    (SQL_CATALOG_PROP_URI.to_string(), sql_lite_uri),
1486                    (SQL_CATALOG_PROP_WAREHOUSE.to_string(), temp_path()),
1487                ]),
1488            )
1489            .await
1490            .unwrap();
1491
1492        catalog
1493            .connection
1494            .execute("PRAGMA foreign_keys = ON")
1495            .await
1496            .unwrap();
1497        // This deferred constraint lets an INSERT succeed while COMMIT fails.
1498        catalog
1499            .connection
1500            .execute("CREATE TABLE parent(id INTEGER PRIMARY KEY)")
1501            .await
1502            .unwrap();
1503        catalog
1504            .connection
1505            .execute(
1506                "CREATE TABLE child(parent_id INTEGER REFERENCES parent(id) \
1507                 DEFERRABLE INITIALLY DEFERRED)",
1508            )
1509            .await
1510            .unwrap();
1511
1512        catalog
1513    }
1514
1515    #[tokio::test]
1516    async fn test_execute_returns_commit_error() {
1517        let catalog = new_commit_error_catalog().await;
1518
1519        // Make the public namespace operation insert a child row whose deferred
1520        // foreign-key constraint succeeds during execution but fails at commit.
1521        let trigger = format!(
1522            "CREATE TRIGGER fail_namespace_commit
1523             AFTER INSERT ON {NAMESPACE_TABLE_NAME}
1524             BEGIN INSERT INTO child VALUES (1); END"
1525        );
1526        catalog.connection.execute(trigger.as_str()).await.unwrap();
1527
1528        let failed_namespace = NamespaceIdent::new("failed".into());
1529        let error = catalog
1530            .create_namespace(&failed_namespace, HashMap::new())
1531            .await
1532            .unwrap_err();
1533        assert_eq!(error.kind(), ErrorKind::Unexpected);
1534        assert!(!catalog.namespace_exists(&failed_namespace).await.unwrap());
1535
1536        // A valid relationship confirms that successful transactions still commit.
1537        catalog
1538            .connection
1539            .execute("INSERT INTO parent VALUES (1)")
1540            .await
1541            .unwrap();
1542        let committed_namespace = NamespaceIdent::new("committed".into());
1543        catalog
1544            .create_namespace(&committed_namespace, HashMap::new())
1545            .await
1546            .unwrap();
1547        assert!(
1548            catalog
1549                .namespace_exists(&committed_namespace)
1550                .await
1551                .unwrap()
1552        );
1553    }
1554
1555    // Regression test: storage-backend props set on the catalog must reach
1556    // the FileIO; otherwise authenticated backends fail with 401s on writes.
1557    #[tokio::test]
1558    async fn test_storage_props_propagate_to_file_io() {
1559        let sql_lite_uri = format!("sqlite:{}", temp_path());
1560        sqlx::Sqlite::create_database(&sql_lite_uri).await.unwrap();
1561        let warehouse_location = temp_path();
1562
1563        let catalog = SqlCatalogBuilder::default()
1564            .with_storage_factory(Arc::new(LocalFsStorageFactory))
1565            .load(
1566                "iceberg",
1567                HashMap::from_iter([
1568                    (SQL_CATALOG_PROP_URI.to_string(), sql_lite_uri),
1569                    (
1570                        SQL_CATALOG_PROP_WAREHOUSE.to_string(),
1571                        warehouse_location.clone(),
1572                    ),
1573                    ("s3.region".to_string(), "us-east-1".to_string()),
1574                    ("hf.token".to_string(), "hf_test_token".to_string()),
1575                ]),
1576            )
1577            .await
1578            .unwrap();
1579
1580        let props = catalog.fileio.config().props();
1581        assert_eq!(props.get(SQL_CATALOG_PROP_URI), None);
1582        assert_eq!(
1583            props.get(SQL_CATALOG_PROP_WAREHOUSE),
1584            Some(&warehouse_location)
1585        );
1586        assert_eq!(props.get("s3.region"), Some(&"us-east-1".to_string()));
1587        assert_eq!(props.get("hf.token"), Some(&"hf_test_token".to_string()));
1588    }
1589
1590    #[tokio::test]
1591    async fn test_builder_method() {
1592        let sql_lite_uri = format!("sqlite:{}", temp_path());
1593        sqlx::Sqlite::create_database(&sql_lite_uri).await.unwrap();
1594        let warehouse_location = temp_path();
1595
1596        let catalog = SqlCatalogBuilder::default()
1597            .with_storage_factory(Arc::new(LocalFsStorageFactory))
1598            .uri(sql_lite_uri.to_string())
1599            .warehouse_location(warehouse_location.clone())
1600            .sql_bind_style(SqlBindStyle::QMark)
1601            .load("iceberg", HashMap::default())
1602            .await;
1603        assert!(catalog.is_ok());
1604
1605        let catalog = catalog.unwrap();
1606        assert!(catalog.properties.warehouse_location == warehouse_location);
1607        assert!(catalog.properties.sql_bind_style == SqlBindStyle::QMark);
1608    }
1609
1610    /// Overwriting an sqlite database with a non-existent path causes
1611    /// catalog generation to fail
1612    #[tokio::test]
1613    async fn test_builder_props_non_existent_path_fails() {
1614        let sql_lite_uri = format!("sqlite:{}", temp_path());
1615        let sql_lite_uri2 = format!("sqlite:{}", temp_path());
1616        sqlx::Sqlite::create_database(&sql_lite_uri).await.unwrap();
1617        let warehouse_location = temp_path();
1618
1619        let catalog = SqlCatalogBuilder::default()
1620            .with_storage_factory(Arc::new(LocalFsStorageFactory))
1621            .uri(sql_lite_uri)
1622            .warehouse_location(warehouse_location)
1623            .load(
1624                "iceberg",
1625                HashMap::from_iter([(SQL_CATALOG_PROP_URI.to_string(), sql_lite_uri2)]),
1626            )
1627            .await;
1628        assert!(catalog.is_err());
1629    }
1630
1631    /// Even when an invalid URI is specified in a builder method,
1632    /// it can be successfully overridden with a valid URI in props
1633    /// for catalog generation to succeed.
1634    #[tokio::test]
1635    async fn test_builder_props_set_valid_uri() {
1636        let sql_lite_uri = format!("sqlite:{}", temp_path());
1637        let sql_lite_uri2 = format!("sqlite:{}", temp_path());
1638        sqlx::Sqlite::create_database(&sql_lite_uri).await.unwrap();
1639        let warehouse_location = temp_path();
1640
1641        let catalog = SqlCatalogBuilder::default()
1642            .with_storage_factory(Arc::new(LocalFsStorageFactory))
1643            .uri(sql_lite_uri2)
1644            .warehouse_location(warehouse_location)
1645            .load(
1646                "iceberg",
1647                HashMap::from_iter([(SQL_CATALOG_PROP_URI.to_string(), sql_lite_uri.clone())]),
1648            )
1649            .await;
1650        assert!(catalog.is_ok());
1651    }
1652
1653    /// values assigned via props take precedence
1654    #[tokio::test]
1655    async fn test_builder_props_take_precedence() {
1656        let sql_lite_uri = format!("sqlite:{}", temp_path());
1657        sqlx::Sqlite::create_database(&sql_lite_uri).await.unwrap();
1658        let warehouse_location = temp_path();
1659        let warehouse_location2 = temp_path();
1660
1661        let catalog = SqlCatalogBuilder::default()
1662            .with_storage_factory(Arc::new(LocalFsStorageFactory))
1663            .warehouse_location(warehouse_location2)
1664            .sql_bind_style(SqlBindStyle::DollarNumeric)
1665            .load(
1666                "iceberg",
1667                HashMap::from_iter([
1668                    (SQL_CATALOG_PROP_URI.to_string(), sql_lite_uri),
1669                    (
1670                        SQL_CATALOG_PROP_WAREHOUSE.to_string(),
1671                        warehouse_location.clone(),
1672                    ),
1673                    (
1674                        SQL_CATALOG_PROP_BIND_STYLE.to_string(),
1675                        SqlBindStyle::QMark.to_string(),
1676                    ),
1677                ]),
1678            )
1679            .await;
1680
1681        assert!(catalog.is_ok());
1682
1683        let catalog = catalog.unwrap();
1684        assert!(catalog.properties.warehouse_location == warehouse_location);
1685        assert!(catalog.properties.sql_bind_style == SqlBindStyle::QMark);
1686    }
1687
1688    #[tokio::test]
1689    async fn test_load_legacy_bind_style_overrides_builder_method() {
1690        let sql_lite_uri = format!("sqlite:{}", temp_path());
1691        sqlx::Sqlite::create_database(&sql_lite_uri).await.unwrap();
1692
1693        let catalog = SqlCatalogBuilder::default()
1694            .with_storage_factory(Arc::new(LocalFsStorageFactory))
1695            .sql_bind_style(SqlBindStyle::DollarNumeric)
1696            .load(
1697                "iceberg",
1698                HashMap::from([
1699                    (SQL_CATALOG_PROP_URI.to_string(), sql_lite_uri),
1700                    (SQL_CATALOG_PROP_WAREHOUSE.to_string(), temp_path()),
1701                    (
1702                        SQL_CATALOG_PROP_BIND_STYLE_LEGACY.to_string(),
1703                        SqlBindStyle::QMark.to_string(),
1704                    ),
1705                ]),
1706            )
1707            .await
1708            .unwrap();
1709
1710        assert_eq!(catalog.properties.sql_bind_style, SqlBindStyle::QMark);
1711    }
1712
1713    /// values assigned via props take precedence
1714    #[tokio::test]
1715    async fn test_builder_props_take_precedence_props() {
1716        let sql_lite_uri = format!("sqlite:{}", temp_path());
1717        let sql_lite_uri2 = format!("sqlite:{}", temp_path());
1718        sqlx::Sqlite::create_database(&sql_lite_uri).await.unwrap();
1719        let warehouse_location = temp_path();
1720        let warehouse_location2 = temp_path();
1721
1722        let props = HashMap::from_iter([
1723            (SQL_CATALOG_PROP_URI.to_string(), sql_lite_uri.clone()),
1724            (
1725                SQL_CATALOG_PROP_WAREHOUSE.to_string(),
1726                warehouse_location.clone(),
1727            ),
1728            (
1729                SQL_CATALOG_PROP_BIND_STYLE.to_string(),
1730                SqlBindStyle::QMark.to_string(),
1731            ),
1732        ]);
1733        let props2 = HashMap::from_iter([
1734            (SQL_CATALOG_PROP_URI.to_string(), sql_lite_uri2.clone()),
1735            (
1736                SQL_CATALOG_PROP_WAREHOUSE.to_string(),
1737                warehouse_location2.clone(),
1738            ),
1739            (
1740                SQL_CATALOG_PROP_BIND_STYLE.to_string(),
1741                SqlBindStyle::DollarNumeric.to_string(),
1742            ),
1743        ]);
1744
1745        let catalog = SqlCatalogBuilder::default()
1746            .with_storage_factory(Arc::new(LocalFsStorageFactory))
1747            .props(props2)
1748            .load("iceberg", props)
1749            .await;
1750
1751        assert!(catalog.is_ok());
1752
1753        let catalog = catalog.unwrap();
1754        assert!(catalog.properties.warehouse_location == warehouse_location);
1755        assert!(catalog.properties.sql_bind_style == SqlBindStyle::QMark);
1756    }
1757
1758    /// values assigned via props take precedence
1759    #[tokio::test]
1760    async fn test_builder_props_take_precedence_prop() {
1761        let sql_lite_uri = format!("sqlite:{}", temp_path());
1762        let sql_lite_uri2 = format!("sqlite:{}", temp_path());
1763        sqlx::Sqlite::create_database(&sql_lite_uri).await.unwrap();
1764        let warehouse_location = temp_path();
1765        let warehouse_location2 = temp_path();
1766
1767        let props = HashMap::from_iter([
1768            (SQL_CATALOG_PROP_URI.to_string(), sql_lite_uri.clone()),
1769            (
1770                SQL_CATALOG_PROP_WAREHOUSE.to_string(),
1771                warehouse_location.clone(),
1772            ),
1773            (
1774                SQL_CATALOG_PROP_BIND_STYLE.to_string(),
1775                SqlBindStyle::QMark.to_string(),
1776            ),
1777        ]);
1778
1779        let catalog = SqlCatalogBuilder::default()
1780            .with_storage_factory(Arc::new(LocalFsStorageFactory))
1781            .prop(SQL_CATALOG_PROP_URI.to_string(), sql_lite_uri2)
1782            .prop(SQL_CATALOG_PROP_WAREHOUSE.to_string(), warehouse_location2)
1783            .prop(
1784                SQL_CATALOG_PROP_BIND_STYLE.to_string(),
1785                SqlBindStyle::DollarNumeric.to_string(),
1786            )
1787            .load("iceberg", props)
1788            .await;
1789
1790        assert!(catalog.is_ok());
1791
1792        let catalog = catalog.unwrap();
1793        assert!(catalog.properties.warehouse_location == warehouse_location);
1794        assert!(catalog.properties.sql_bind_style == SqlBindStyle::QMark);
1795    }
1796
1797    /// invalid value for `SqlBindStyle` causes catalog creation to fail
1798    #[tokio::test]
1799    async fn test_builder_props_invalid_bind_style_fails() {
1800        let sql_lite_uri = format!("sqlite:{}", temp_path());
1801        sqlx::Sqlite::create_database(&sql_lite_uri).await.unwrap();
1802        let warehouse_location = temp_path();
1803
1804        let catalog = SqlCatalogBuilder::default()
1805            .with_storage_factory(Arc::new(LocalFsStorageFactory))
1806            .load(
1807                "iceberg",
1808                HashMap::from_iter([
1809                    (SQL_CATALOG_PROP_URI.to_string(), sql_lite_uri),
1810                    (SQL_CATALOG_PROP_WAREHOUSE.to_string(), warehouse_location),
1811                    (SQL_CATALOG_PROP_BIND_STYLE.to_string(), "AAA".to_string()),
1812                ]),
1813            )
1814            .await;
1815
1816        assert!(catalog.is_err());
1817    }
1818
1819    #[tokio::test]
1820    async fn test_builder_props_invalid_pool_property_fails() {
1821        for property in [
1822            "pool.max-connections",
1823            "pool.idle-timeout",
1824            "pool.test-before-acquire",
1825        ] {
1826            let error = SqlCatalogBuilder::default()
1827                .with_storage_factory(Arc::new(LocalFsStorageFactory))
1828                .prop(property, "invalid")
1829                .load("iceberg", HashMap::new())
1830                .await
1831                .unwrap_err();
1832
1833            assert_eq!(error.kind(), ErrorKind::DataInvalid);
1834            assert!(error.to_string().contains(property));
1835            assert!(error.to_string().contains("invalid"));
1836        }
1837    }
1838
1839    #[tokio::test]
1840    async fn test_list_namespaces_returns_empty_vector() {
1841        let warehouse_loc = temp_path();
1842        let catalog = new_sql_catalog(warehouse_loc, Some("iceberg")).await;
1843
1844        assert_eq!(catalog.list_namespaces(None).await.unwrap(), vec![]);
1845    }
1846
1847    #[tokio::test]
1848    async fn test_list_namespaces_returns_empty_different_name() {
1849        let warehouse_loc = temp_path();
1850        let catalog = new_sql_catalog(warehouse_loc.clone(), Some("iceberg")).await;
1851        let namespace_ident_1 = NamespaceIdent::new("a".into());
1852        let namespace_ident_2 = NamespaceIdent::new("b".into());
1853        create_namespaces(&catalog, &vec![&namespace_ident_1, &namespace_ident_2]).await;
1854        assert_eq!(
1855            to_set(catalog.list_namespaces(None).await.unwrap()),
1856            to_set(vec![namespace_ident_1, namespace_ident_2])
1857        );
1858
1859        let catalog2 = new_sql_catalog(warehouse_loc, Some("test")).await;
1860        assert_eq!(catalog2.list_namespaces(None).await.unwrap(), vec![]);
1861    }
1862
1863    #[tokio::test]
1864    async fn test_list_namespaces_returns_only_top_level_namespaces() {
1865        let warehouse_loc = temp_path();
1866        let catalog = new_sql_catalog(warehouse_loc, Some("iceberg")).await;
1867        let namespace_ident_1 = NamespaceIdent::new("a".into());
1868        let namespace_ident_2 = NamespaceIdent::from_strs(vec!["a", "b"]).unwrap();
1869        let namespace_ident_3 = NamespaceIdent::new("b".into());
1870        create_namespaces(&catalog, &vec![
1871            &namespace_ident_1,
1872            &namespace_ident_2,
1873            &namespace_ident_3,
1874        ])
1875        .await;
1876
1877        assert_eq!(
1878            to_set(catalog.list_namespaces(None).await.unwrap()),
1879            to_set(vec![namespace_ident_1, namespace_ident_3])
1880        );
1881    }
1882
1883    #[tokio::test]
1884    async fn test_list_namespaces_returns_no_namespaces_under_parent() {
1885        let warehouse_loc = temp_path();
1886        let catalog = new_sql_catalog(warehouse_loc, Some("iceberg")).await;
1887        let namespace_ident_1 = NamespaceIdent::new("a".into());
1888        let namespace_ident_2 = NamespaceIdent::new("b".into());
1889        create_namespaces(&catalog, &vec![&namespace_ident_1, &namespace_ident_2]).await;
1890
1891        assert_eq!(
1892            catalog
1893                .list_namespaces(Some(&namespace_ident_1))
1894                .await
1895                .unwrap(),
1896            vec![]
1897        );
1898    }
1899
1900    #[tokio::test]
1901    async fn test_list_namespaces_returns_namespace_under_parent() {
1902        let warehouse_loc = temp_path();
1903        let catalog = new_sql_catalog(warehouse_loc, Some("iceberg")).await;
1904        let namespace_ident_1 = NamespaceIdent::new("a".into());
1905        let namespace_ident_2 = NamespaceIdent::from_strs(vec!["a", "b"]).unwrap();
1906        let namespace_ident_3 = NamespaceIdent::new("c".into());
1907        create_namespaces(&catalog, &vec![
1908            &namespace_ident_1,
1909            &namespace_ident_2,
1910            &namespace_ident_3,
1911        ])
1912        .await;
1913
1914        assert_eq!(
1915            to_set(catalog.list_namespaces(None).await.unwrap()),
1916            to_set(vec![namespace_ident_1.clone(), namespace_ident_3])
1917        );
1918
1919        assert_eq!(
1920            catalog
1921                .list_namespaces(Some(&namespace_ident_1))
1922                .await
1923                .unwrap(),
1924            vec![NamespaceIdent::from_strs(vec!["a", "b"]).unwrap()]
1925        );
1926    }
1927
1928    #[tokio::test]
1929    async fn test_list_namespaces_returns_multiple_namespaces_under_parent() {
1930        let warehouse_loc = temp_path();
1931        let catalog = new_sql_catalog(warehouse_loc, Some("iceberg")).await;
1932        let namespace_ident_1 = NamespaceIdent::new("a".to_string());
1933        let namespace_ident_2 = NamespaceIdent::from_strs(vec!["a", "a"]).unwrap();
1934        let namespace_ident_3 = NamespaceIdent::from_strs(vec!["a", "b"]).unwrap();
1935        let namespace_ident_4 = NamespaceIdent::from_strs(vec!["a", "c"]).unwrap();
1936        let namespace_ident_5 = NamespaceIdent::new("b".into());
1937        create_namespaces(&catalog, &vec![
1938            &namespace_ident_1,
1939            &namespace_ident_2,
1940            &namespace_ident_3,
1941            &namespace_ident_4,
1942            &namespace_ident_5,
1943        ])
1944        .await;
1945
1946        assert_eq!(
1947            to_set(
1948                catalog
1949                    .list_namespaces(Some(&namespace_ident_1))
1950                    .await
1951                    .unwrap()
1952            ),
1953            to_set(vec![
1954                NamespaceIdent::from_strs(vec!["a", "a"]).unwrap(),
1955                NamespaceIdent::from_strs(vec!["a", "b"]).unwrap(),
1956                NamespaceIdent::from_strs(vec!["a", "c"]).unwrap(),
1957            ])
1958        );
1959    }
1960
1961    #[tokio::test]
1962    async fn test_namespace_exists_returns_false() {
1963        let warehouse_loc = temp_path();
1964        let catalog = new_sql_catalog(warehouse_loc, Some("iceberg")).await;
1965        let namespace_ident = NamespaceIdent::new("a".into());
1966        create_namespace(&catalog, &namespace_ident).await;
1967
1968        assert!(
1969            !catalog
1970                .namespace_exists(&NamespaceIdent::new("b".into()))
1971                .await
1972                .unwrap()
1973        );
1974    }
1975
1976    #[tokio::test]
1977    async fn test_namespace_exists_returns_true() {
1978        let warehouse_loc = temp_path();
1979        let catalog = new_sql_catalog(warehouse_loc, Some("iceberg")).await;
1980        let namespace_ident = NamespaceIdent::new("a".into());
1981        create_namespace(&catalog, &namespace_ident).await;
1982
1983        assert!(catalog.namespace_exists(&namespace_ident).await.unwrap());
1984    }
1985
1986    #[tokio::test]
1987    async fn test_create_namespace_with_properties() {
1988        let warehouse_loc = temp_path();
1989        let catalog = new_sql_catalog(warehouse_loc, Some("iceberg")).await;
1990        let namespace_ident = NamespaceIdent::new("abc".into());
1991
1992        let mut properties = default_properties();
1993        properties.insert("k".into(), "v".into());
1994
1995        assert_eq!(
1996            catalog
1997                .create_namespace(&namespace_ident, properties.clone())
1998                .await
1999                .unwrap(),
2000            Namespace::with_properties(namespace_ident.clone(), properties.clone())
2001        );
2002
2003        assert_eq!(
2004            catalog.get_namespace(&namespace_ident).await.unwrap(),
2005            Namespace::with_properties(namespace_ident, properties)
2006        );
2007    }
2008
2009    #[tokio::test]
2010    async fn test_create_nested_namespace() {
2011        let warehouse_loc = temp_path();
2012        let catalog = new_sql_catalog(warehouse_loc, Some("iceberg")).await;
2013        let parent_namespace_ident = NamespaceIdent::new("a".into());
2014        create_namespace(&catalog, &parent_namespace_ident).await;
2015
2016        let child_namespace_ident = NamespaceIdent::from_strs(vec!["a", "b"]).unwrap();
2017
2018        assert_eq!(
2019            catalog
2020                .create_namespace(&child_namespace_ident, HashMap::new())
2021                .await
2022                .unwrap(),
2023            Namespace::new(child_namespace_ident.clone())
2024        );
2025
2026        assert_eq!(
2027            catalog.get_namespace(&child_namespace_ident).await.unwrap(),
2028            Namespace::with_properties(child_namespace_ident, default_properties())
2029        );
2030    }
2031
2032    #[tokio::test]
2033    async fn test_create_deeply_nested_namespace() {
2034        let warehouse_loc = temp_path();
2035        let catalog = new_sql_catalog(warehouse_loc, Some("iceberg")).await;
2036        let namespace_ident_a = NamespaceIdent::new("a".into());
2037        let namespace_ident_a_b = NamespaceIdent::from_strs(vec!["a", "b"]).unwrap();
2038        create_namespaces(&catalog, &vec![&namespace_ident_a, &namespace_ident_a_b]).await;
2039
2040        let namespace_ident_a_b_c = NamespaceIdent::from_strs(vec!["a", "b", "c"]).unwrap();
2041
2042        assert_eq!(
2043            catalog
2044                .create_namespace(&namespace_ident_a_b_c, HashMap::new())
2045                .await
2046                .unwrap(),
2047            Namespace::new(namespace_ident_a_b_c.clone())
2048        );
2049
2050        assert_eq!(
2051            catalog.get_namespace(&namespace_ident_a_b_c).await.unwrap(),
2052            Namespace::with_properties(namespace_ident_a_b_c, default_properties())
2053        );
2054    }
2055
2056    #[tokio::test]
2057    async fn test_update_namespace_noop() {
2058        let warehouse_loc = temp_path();
2059        let catalog = new_sql_catalog(warehouse_loc, Some("iceberg")).await;
2060        let namespace_ident = NamespaceIdent::new("a".into());
2061        create_namespace(&catalog, &namespace_ident).await;
2062
2063        catalog
2064            .update_namespace(&namespace_ident, HashMap::new())
2065            .await
2066            .unwrap();
2067
2068        assert_eq!(
2069            *catalog
2070                .get_namespace(&namespace_ident)
2071                .await
2072                .unwrap()
2073                .properties(),
2074            HashMap::from_iter([("exists".to_string(), "true".to_string())])
2075        )
2076    }
2077
2078    #[tokio::test]
2079    async fn test_update_nested_namespace() {
2080        let warehouse_loc = temp_path();
2081        let catalog = new_sql_catalog(warehouse_loc, Some("iceberg")).await;
2082        let namespace_ident = NamespaceIdent::from_strs(["a", "b"]).unwrap();
2083        create_namespace(&catalog, &namespace_ident).await;
2084
2085        let mut props = HashMap::from_iter([
2086            ("prop1".to_string(), "val1".to_string()),
2087            ("prop2".into(), "val2".into()),
2088        ]);
2089
2090        catalog
2091            .update_namespace(&namespace_ident, props.clone())
2092            .await
2093            .unwrap();
2094
2095        props.insert("exists".into(), "true".into());
2096
2097        assert_eq!(
2098            *catalog
2099                .get_namespace(&namespace_ident)
2100                .await
2101                .unwrap()
2102                .properties(),
2103            props
2104        )
2105    }
2106
2107    #[tokio::test]
2108    async fn test_update_namespace_errors_if_nested_namespace_doesnt_exist() {
2109        let warehouse_loc = temp_path();
2110        let catalog = new_sql_catalog(warehouse_loc, Some("iceberg")).await;
2111        let namespace_ident = NamespaceIdent::from_strs(["a", "b"]).unwrap();
2112
2113        let props = HashMap::from_iter([
2114            ("prop1".to_string(), "val1".to_string()),
2115            ("prop2".into(), "val2".into()),
2116        ]);
2117
2118        let err = catalog
2119            .update_namespace(&namespace_ident, props)
2120            .await
2121            .unwrap_err();
2122
2123        assert_eq!(
2124            err.message(),
2125            format!("No such namespace: {namespace_ident:?}")
2126        );
2127    }
2128
2129    #[tokio::test]
2130    async fn test_drop_nested_namespace() {
2131        let warehouse_loc = temp_path();
2132        let catalog = new_sql_catalog(warehouse_loc, Some("iceberg")).await;
2133        let namespace_ident_a = NamespaceIdent::new("a".into());
2134        let namespace_ident_a_b = NamespaceIdent::from_strs(vec!["a", "b"]).unwrap();
2135        create_namespaces(&catalog, &vec![&namespace_ident_a, &namespace_ident_a_b]).await;
2136
2137        catalog.drop_namespace(&namespace_ident_a_b).await.unwrap();
2138
2139        assert!(
2140            !catalog
2141                .namespace_exists(&namespace_ident_a_b)
2142                .await
2143                .unwrap()
2144        );
2145
2146        assert!(catalog.namespace_exists(&namespace_ident_a).await.unwrap());
2147    }
2148
2149    #[tokio::test]
2150    async fn test_drop_deeply_nested_namespace() {
2151        let warehouse_loc = temp_path();
2152        let catalog = new_sql_catalog(warehouse_loc, Some("iceberg")).await;
2153        let namespace_ident_a = NamespaceIdent::new("a".into());
2154        let namespace_ident_a_b = NamespaceIdent::from_strs(vec!["a", "b"]).unwrap();
2155        let namespace_ident_a_b_c = NamespaceIdent::from_strs(vec!["a", "b", "c"]).unwrap();
2156        create_namespaces(&catalog, &vec![
2157            &namespace_ident_a,
2158            &namespace_ident_a_b,
2159            &namespace_ident_a_b_c,
2160        ])
2161        .await;
2162
2163        catalog
2164            .drop_namespace(&namespace_ident_a_b_c)
2165            .await
2166            .unwrap();
2167
2168        assert!(
2169            !catalog
2170                .namespace_exists(&namespace_ident_a_b_c)
2171                .await
2172                .unwrap()
2173        );
2174
2175        assert!(
2176            catalog
2177                .namespace_exists(&namespace_ident_a_b)
2178                .await
2179                .unwrap()
2180        );
2181
2182        assert!(catalog.namespace_exists(&namespace_ident_a).await.unwrap());
2183    }
2184
2185    #[tokio::test]
2186    async fn test_drop_namespace_throws_error_if_nested_namespace_doesnt_exist() {
2187        let warehouse_loc = temp_path();
2188        let catalog = new_sql_catalog(warehouse_loc, Some("iceberg")).await;
2189        create_namespace(&catalog, &NamespaceIdent::new("a".into())).await;
2190
2191        let non_existent_namespace_ident =
2192            NamespaceIdent::from_vec(vec!["a".into(), "b".into()]).unwrap();
2193        assert_eq!(
2194            catalog
2195                .drop_namespace(&non_existent_namespace_ident)
2196                .await
2197                .unwrap_err()
2198                .to_string(),
2199            format!("NamespaceNotFound => No such namespace: {non_existent_namespace_ident:?}")
2200        )
2201    }
2202
2203    #[tokio::test]
2204    async fn test_dropping_a_namespace_does_not_drop_namespaces_nested_under_that_one() {
2205        let warehouse_loc = temp_path();
2206        let catalog = new_sql_catalog(warehouse_loc, Some("iceberg")).await;
2207        let namespace_ident_a = NamespaceIdent::new("a".into());
2208        let namespace_ident_a_b = NamespaceIdent::from_strs(vec!["a", "b"]).unwrap();
2209        create_namespaces(&catalog, &vec![&namespace_ident_a, &namespace_ident_a_b]).await;
2210
2211        catalog.drop_namespace(&namespace_ident_a).await.unwrap();
2212
2213        assert!(!catalog.namespace_exists(&namespace_ident_a).await.unwrap());
2214
2215        assert!(
2216            catalog
2217                .namespace_exists(&namespace_ident_a_b)
2218                .await
2219                .unwrap()
2220        );
2221    }
2222
2223    #[tokio::test]
2224    async fn test_create_table_falls_back_to_namespace_location_if_table_location_is_missing() {
2225        let warehouse_loc = temp_path();
2226        let catalog = new_sql_catalog(warehouse_loc, Some("iceberg")).await;
2227
2228        let namespace_ident = NamespaceIdent::new("a".into());
2229        let mut namespace_properties = HashMap::new();
2230        let namespace_location = temp_path();
2231        namespace_properties.insert(
2232            NAMESPACE_LOCATION_PROPERTY_KEY.to_string(),
2233            namespace_location.to_string(),
2234        );
2235        catalog
2236            .create_namespace(&namespace_ident, namespace_properties)
2237            .await
2238            .unwrap();
2239
2240        let table_name = "tbl1";
2241        let expected_table_ident = TableIdent::new(namespace_ident.clone(), table_name.into());
2242        let expected_table_metadata_location_regex =
2243            format!("^{namespace_location}/tbl1/metadata/00000-{UUID_REGEX_STR}.metadata.json$",);
2244
2245        let table = catalog
2246            .create_table(
2247                &namespace_ident,
2248                TableCreation::builder()
2249                    .name(table_name.into())
2250                    .schema(simple_table_schema())
2251                    // no location specified for table
2252                    .build(),
2253            )
2254            .await
2255            .unwrap();
2256        assert_table_eq(&table, &expected_table_ident, &simple_table_schema());
2257        assert_table_metadata_location_matches(&table, &expected_table_metadata_location_regex);
2258
2259        let table = catalog.load_table(&expected_table_ident).await.unwrap();
2260        assert_table_eq(&table, &expected_table_ident, &simple_table_schema());
2261        assert_table_metadata_location_matches(&table, &expected_table_metadata_location_regex);
2262    }
2263
2264    #[tokio::test]
2265    async fn test_create_table_in_nested_namespace_falls_back_to_nested_namespace_location_if_table_location_is_missing()
2266     {
2267        let warehouse_loc = temp_path();
2268        let catalog = new_sql_catalog(warehouse_loc, Some("iceberg")).await;
2269
2270        let namespace_ident = NamespaceIdent::new("a".into());
2271        let mut namespace_properties = HashMap::new();
2272        let namespace_location = temp_path();
2273        namespace_properties.insert(
2274            NAMESPACE_LOCATION_PROPERTY_KEY.to_string(),
2275            namespace_location.to_string(),
2276        );
2277        catalog
2278            .create_namespace(&namespace_ident, namespace_properties)
2279            .await
2280            .unwrap();
2281
2282        let nested_namespace_ident = NamespaceIdent::from_strs(vec!["a", "b"]).unwrap();
2283        let mut nested_namespace_properties = HashMap::new();
2284        let nested_namespace_location = temp_path();
2285        nested_namespace_properties.insert(
2286            NAMESPACE_LOCATION_PROPERTY_KEY.to_string(),
2287            nested_namespace_location.to_string(),
2288        );
2289        catalog
2290            .create_namespace(&nested_namespace_ident, nested_namespace_properties)
2291            .await
2292            .unwrap();
2293
2294        let table_name = "tbl1";
2295        let expected_table_ident =
2296            TableIdent::new(nested_namespace_ident.clone(), table_name.into());
2297        let expected_table_metadata_location_regex = format!(
2298            "^{nested_namespace_location}/tbl1/metadata/00000-{UUID_REGEX_STR}.metadata.json$",
2299        );
2300
2301        let table = catalog
2302            .create_table(
2303                &nested_namespace_ident,
2304                TableCreation::builder()
2305                    .name(table_name.into())
2306                    .schema(simple_table_schema())
2307                    // no location specified for table
2308                    .build(),
2309            )
2310            .await
2311            .unwrap();
2312        assert_table_eq(&table, &expected_table_ident, &simple_table_schema());
2313        assert_table_metadata_location_matches(&table, &expected_table_metadata_location_regex);
2314
2315        let table = catalog.load_table(&expected_table_ident).await.unwrap();
2316        assert_table_eq(&table, &expected_table_ident, &simple_table_schema());
2317        assert_table_metadata_location_matches(&table, &expected_table_metadata_location_regex);
2318    }
2319
2320    #[tokio::test]
2321    async fn test_create_table_falls_back_to_warehouse_location_if_both_table_location_and_namespace_location_are_missing()
2322     {
2323        let warehouse_loc = temp_path();
2324        let catalog = new_sql_catalog(warehouse_loc.clone(), Some("iceberg")).await;
2325
2326        let namespace_ident = NamespaceIdent::new("a".into());
2327        // note: no location specified in namespace_properties
2328        let namespace_properties = HashMap::new();
2329        catalog
2330            .create_namespace(&namespace_ident, namespace_properties)
2331            .await
2332            .unwrap();
2333
2334        let table_name = "tbl1";
2335        let expected_table_ident = TableIdent::new(namespace_ident.clone(), table_name.into());
2336        let expected_table_metadata_location_regex =
2337            format!("^{warehouse_loc}/a/tbl1/metadata/00000-{UUID_REGEX_STR}.metadata.json$");
2338
2339        let table = catalog
2340            .create_table(
2341                &namespace_ident,
2342                TableCreation::builder()
2343                    .name(table_name.into())
2344                    .schema(simple_table_schema())
2345                    // no location specified for table
2346                    .build(),
2347            )
2348            .await
2349            .unwrap();
2350        assert_table_eq(&table, &expected_table_ident, &simple_table_schema());
2351        assert_table_metadata_location_matches(&table, &expected_table_metadata_location_regex);
2352
2353        let table = catalog.load_table(&expected_table_ident).await.unwrap();
2354        assert_table_eq(&table, &expected_table_ident, &simple_table_schema());
2355        assert_table_metadata_location_matches(&table, &expected_table_metadata_location_regex);
2356    }
2357
2358    #[tokio::test]
2359    async fn test_create_table_in_nested_namespace_falls_back_to_warehouse_location_if_both_table_location_and_namespace_location_are_missing()
2360     {
2361        let warehouse_loc = temp_path();
2362        let catalog = new_sql_catalog(warehouse_loc.clone(), Some("iceberg")).await;
2363
2364        let namespace_ident = NamespaceIdent::new("a".into());
2365        create_namespace(&catalog, &namespace_ident).await;
2366
2367        let nested_namespace_ident = NamespaceIdent::from_strs(vec!["a", "b"]).unwrap();
2368        create_namespace(&catalog, &nested_namespace_ident).await;
2369
2370        let table_name = "tbl1";
2371        let expected_table_ident =
2372            TableIdent::new(nested_namespace_ident.clone(), table_name.into());
2373        let expected_table_metadata_location_regex =
2374            format!("^{warehouse_loc}/a/b/tbl1/metadata/00000-{UUID_REGEX_STR}.metadata.json$");
2375
2376        let table = catalog
2377            .create_table(
2378                &nested_namespace_ident,
2379                TableCreation::builder()
2380                    .name(table_name.into())
2381                    .schema(simple_table_schema())
2382                    // no location specified for table
2383                    .build(),
2384            )
2385            .await
2386            .unwrap();
2387        assert_table_eq(&table, &expected_table_ident, &simple_table_schema());
2388        assert_table_metadata_location_matches(&table, &expected_table_metadata_location_regex);
2389
2390        let table = catalog.load_table(&expected_table_ident).await.unwrap();
2391        assert_table_eq(&table, &expected_table_ident, &simple_table_schema());
2392        assert_table_metadata_location_matches(&table, &expected_table_metadata_location_regex);
2393    }
2394
2395    #[tokio::test]
2396    async fn test_create_table_throws_error_if_table_with_same_name_already_exists() {
2397        let warehouse_loc = temp_path();
2398        let catalog = new_sql_catalog(warehouse_loc.clone(), Some("iceberg")).await;
2399        let namespace_ident = NamespaceIdent::new("a".into());
2400        create_namespace(&catalog, &namespace_ident).await;
2401        let table_name = "tbl1";
2402        let table_ident = TableIdent::new(namespace_ident.clone(), table_name.into());
2403        create_table(&catalog, &table_ident).await;
2404
2405        let tmp_dir = TempDir::new().unwrap();
2406        let location = tmp_dir.path().to_str().unwrap().to_string();
2407
2408        assert_eq!(
2409            catalog
2410                .create_table(
2411                    &namespace_ident,
2412                    TableCreation::builder()
2413                        .name(table_name.into())
2414                        .schema(simple_table_schema())
2415                        .location(location)
2416                        .build()
2417                )
2418                .await
2419                .unwrap_err()
2420                .to_string(),
2421            format!(
2422                "TableAlreadyExists => Table {:?} already exists.",
2423                &table_ident
2424            )
2425        );
2426    }
2427
2428    #[tokio::test]
2429    async fn test_rename_table_src_table_is_same_as_dst_table() {
2430        let warehouse_loc = temp_path();
2431        let catalog = new_sql_catalog(warehouse_loc, Some("iceberg")).await;
2432        let namespace_ident = NamespaceIdent::new("n1".into());
2433        create_namespace(&catalog, &namespace_ident).await;
2434        let table_ident = TableIdent::new(namespace_ident.clone(), "tbl".into());
2435        create_table(&catalog, &table_ident).await;
2436
2437        catalog
2438            .rename_table(&table_ident, &table_ident)
2439            .await
2440            .unwrap();
2441
2442        assert_eq!(catalog.list_tables(&namespace_ident).await.unwrap(), vec![
2443            table_ident
2444        ],);
2445    }
2446
2447    #[tokio::test]
2448    async fn test_rename_table_across_nested_namespaces() {
2449        let warehouse_loc = temp_path();
2450        let catalog = new_sql_catalog(warehouse_loc, Some("iceberg")).await;
2451        let namespace_ident_a = NamespaceIdent::new("a".into());
2452        let namespace_ident_a_b = NamespaceIdent::from_strs(vec!["a", "b"]).unwrap();
2453        let namespace_ident_a_b_c = NamespaceIdent::from_strs(vec!["a", "b", "c"]).unwrap();
2454        create_namespaces(&catalog, &vec![
2455            &namespace_ident_a,
2456            &namespace_ident_a_b,
2457            &namespace_ident_a_b_c,
2458        ])
2459        .await;
2460
2461        let src_table_ident = TableIdent::new(namespace_ident_a_b_c.clone(), "tbl1".into());
2462        create_tables(&catalog, vec![&src_table_ident]).await;
2463
2464        let dst_table_ident = TableIdent::new(namespace_ident_a_b.clone(), "tbl1".into());
2465        catalog
2466            .rename_table(&src_table_ident, &dst_table_ident)
2467            .await
2468            .unwrap();
2469
2470        assert!(!catalog.table_exists(&src_table_ident).await.unwrap());
2471
2472        assert!(catalog.table_exists(&dst_table_ident).await.unwrap());
2473    }
2474
2475    #[tokio::test]
2476    async fn test_rename_table_throws_error_if_dst_namespace_doesnt_exist() {
2477        let warehouse_loc = temp_path();
2478        let catalog = new_sql_catalog(warehouse_loc, Some("iceberg")).await;
2479        let src_namespace_ident = NamespaceIdent::new("n1".into());
2480        let src_table_ident = TableIdent::new(src_namespace_ident.clone(), "tbl1".into());
2481        create_namespace(&catalog, &src_namespace_ident).await;
2482        create_table(&catalog, &src_table_ident).await;
2483
2484        let non_existent_dst_namespace_ident = NamespaceIdent::new("n2".into());
2485        let dst_table_ident =
2486            TableIdent::new(non_existent_dst_namespace_ident.clone(), "tbl1".into());
2487        assert_eq!(
2488            catalog
2489                .rename_table(&src_table_ident, &dst_table_ident)
2490                .await
2491                .unwrap_err()
2492                .to_string(),
2493            format!("NamespaceNotFound => No such namespace: {non_existent_dst_namespace_ident:?}"),
2494        );
2495    }
2496
2497    /// Creates a V0 SQLite database (no `iceberg_type` column) with one pre-inserted table row.
2498    /// Returns the SQLite URI and the temp dir that owns the database file.
2499    async fn create_v0_sqlite_db() -> (String, TempDir) {
2500        let temp_dir = TempDir::new().unwrap();
2501        let uri = format!(
2502            "sqlite:{}",
2503            temp_dir.path().join("catalog.db").to_str().unwrap()
2504        );
2505        sqlx::Sqlite::create_database(&uri).await.unwrap();
2506        let pool = sqlx::AnyPool::connect(&uri).await.unwrap();
2507        sqlx::query(
2508            "CREATE TABLE iceberg_tables (
2509                catalog_name VARCHAR(255) NOT NULL,
2510                table_namespace VARCHAR(255) NOT NULL,
2511                table_name VARCHAR(255) NOT NULL,
2512                metadata_location VARCHAR(1000),
2513                previous_metadata_location VARCHAR(1000),
2514                PRIMARY KEY (catalog_name, table_namespace, table_name)
2515            )",
2516        )
2517        .execute(&pool)
2518        .await
2519        .unwrap();
2520        sqlx::query(
2521            "INSERT INTO iceberg_tables
2522             (catalog_name, table_namespace, table_name, metadata_location)
2523             VALUES ('iceberg', 'test_namespace', 'existing_test_table', '/tmp/fake-location')",
2524        )
2525        .execute(&pool)
2526        .await
2527        .unwrap();
2528        pool.close().await;
2529        (uri, temp_dir)
2530    }
2531
2532    /// Creates a V1 SQLite database (with an `iceberg_type` column) with one pre-inserted table row.
2533    /// Returns the SQLite URI and the temp dir that owns the database file.
2534    async fn create_v1_sqlite_db() -> (String, TempDir) {
2535        let temp_dir = TempDir::new().unwrap();
2536        let uri = format!(
2537            "sqlite:{}",
2538            temp_dir.path().join("catalog.db").to_str().unwrap()
2539        );
2540        sqlx::Sqlite::create_database(&uri).await.unwrap();
2541        let pool = sqlx::AnyPool::connect(&uri).await.unwrap();
2542        sqlx::query(
2543            "CREATE TABLE iceberg_tables (
2544                catalog_name VARCHAR(255) NOT NULL,
2545                table_namespace VARCHAR(255) NOT NULL,
2546                table_name VARCHAR(255) NOT NULL,
2547                metadata_location VARCHAR(1000),
2548                previous_metadata_location VARCHAR(1000),
2549                iceberg_type VARCHAR(5),
2550                PRIMARY KEY (catalog_name, table_namespace, table_name)
2551            )",
2552        )
2553        .execute(&pool)
2554        .await
2555        .unwrap();
2556        sqlx::query(
2557            "INSERT INTO iceberg_tables
2558             (catalog_name, table_namespace, table_name, metadata_location, iceberg_type)
2559             VALUES ('iceberg', 'test_namespace', 'existing_test_table', '/tmp/fake-location', 'TABLE')",
2560        )
2561        .execute(&pool)
2562        .await
2563        .unwrap();
2564        pool.close().await;
2565        (uri, temp_dir)
2566    }
2567
2568    /// Catalog properties for opening `uri` with `warehouse` as the warehouse location,
2569    /// optionally requesting a specific `sql.schema-version`.
2570    fn catalog_props(
2571        uri: &str,
2572        warehouse: &TempDir,
2573        schema_version: Option<SchemaVersion>,
2574    ) -> HashMap<String, String> {
2575        let mut props = HashMap::from_iter([
2576            (SQL_CATALOG_PROP_URI.to_string(), uri.to_string()),
2577            (
2578                SQL_CATALOG_PROP_WAREHOUSE.to_string(),
2579                warehouse.path().to_str().unwrap().to_string(),
2580            ),
2581            (
2582                SQL_CATALOG_PROP_BIND_STYLE.to_string(),
2583                SqlBindStyle::QMark.to_string(),
2584            ),
2585        ]);
2586        if let Some(schema_version) = schema_version {
2587            props.insert(
2588                SQL_CATALOG_PROP_SCHEMA_VERSION.to_string(),
2589                schema_version.to_string(),
2590            );
2591        }
2592        props
2593    }
2594
2595    /// Whether the catalog table in the database at `uri` has the `iceberg_type` column.
2596    ///
2597    /// Connects independently of any catalog under test, so it reports what is actually on disk.
2598    async fn record_type_column_exists(uri: &str) -> bool {
2599        let probe_pool = sqlx::AnyPool::connect(uri).await.unwrap();
2600        let column_exists = probe_pool
2601            .describe(&format!("SELECT * FROM {CATALOG_TABLE_NAME}"))
2602            .await
2603            .expect("connection and query should succeed")
2604            .columns()
2605            .iter()
2606            .any(|column| {
2607                column
2608                    .name()
2609                    .eq_ignore_ascii_case(CATALOG_FIELD_RECORD_TYPE)
2610            });
2611        probe_pool.close().await;
2612        column_exists
2613    }
2614
2615    #[tokio::test]
2616    async fn test_detect_schema_version() {
2617        install_default_drivers();
2618
2619        let detected_schema = {
2620            let (uri, _temp_dir) = create_v0_sqlite_db().await;
2621            let pool = sqlx::AnyPool::connect(&uri).await.unwrap();
2622            let detected_schema = SchemaVersion::detect(&pool).await.unwrap();
2623            pool.close().await;
2624            detected_schema
2625        };
2626        assert_eq!(
2627            detected_schema,
2628            SchemaVersion::V0,
2629            "a catalog table without an iceberg_type column should be V0",
2630        );
2631
2632        let detected_schema = {
2633            let (uri, _temp_dir) = create_v1_sqlite_db().await;
2634            let pool = sqlx::AnyPool::connect(&uri).await.unwrap();
2635            let detected_schema = SchemaVersion::detect(&pool).await.unwrap();
2636            pool.close().await;
2637            detected_schema
2638        };
2639        assert_eq!(
2640            detected_schema,
2641            SchemaVersion::V1,
2642            "a catalog table with an iceberg_type column should be V1",
2643        );
2644    }
2645
2646    #[tokio::test]
2647    async fn test_detect_schema_version_surfaces_errors() {
2648        install_default_drivers();
2649
2650        let temp_dir = TempDir::new().unwrap();
2651        let uri = format!(
2652            "sqlite:{}",
2653            temp_dir.path().join("catalog.db").to_str().unwrap()
2654        );
2655        sqlx::Sqlite::create_database(&uri).await.unwrap();
2656        let pool = sqlx::AnyPool::connect(&uri).await.unwrap();
2657
2658        // No `iceberg_tables` table at all, so the schema version is unknowable.
2659        let err = SchemaVersion::detect(&pool)
2660            .await
2661            .expect_err("detection should fail rather than report V0");
2662        pool.close().await;
2663
2664        assert!(
2665            err.to_string().contains("iceberg_tables"),
2666            "error should name the table it failed to introspect, got: {err}"
2667        );
2668    }
2669
2670    #[tokio::test]
2671    async fn test_v0_schema_migration() {
2672        install_default_drivers();
2673
2674        let (uri, temp_dir) = create_v0_sqlite_db().await;
2675
2676        // Opening the catalog with sql.schema-version=V1 should migrate the V0 schema.
2677        let props = HashMap::from_iter([
2678            (SQL_CATALOG_PROP_URI.to_string(), uri.clone()),
2679            (
2680                SQL_CATALOG_PROP_WAREHOUSE.to_string(),
2681                temp_dir.path().to_str().unwrap().to_string(),
2682            ),
2683            (
2684                SQL_CATALOG_PROP_BIND_STYLE.to_string(),
2685                SqlBindStyle::QMark.to_string(),
2686            ),
2687            (
2688                SQL_CATALOG_PROP_SCHEMA_VERSION.to_string(),
2689                SchemaVersion::V1.to_string(),
2690            ),
2691        ]);
2692        let catalog = SqlCatalogBuilder::default()
2693            .with_storage_factory(Arc::new(LocalFsStorageFactory))
2694            .load("iceberg", props)
2695            .await
2696            .expect("should open V0 catalog and migrate schema when sql.schema-version=V1");
2697
2698        // The V0 row (no "iceberg_type" column) should be treated as a TABLE after migration.
2699        let namespace = NamespaceIdent::from_strs(["test_namespace"]).unwrap();
2700        let tables = catalog.list_tables(&namespace).await.unwrap();
2701        assert_eq!(tables.len(), 1);
2702        assert_eq!(tables[0].name(), "existing_test_table");
2703
2704        assert!(
2705            record_type_column_exists(&uri).await,
2706            "iceberg_type column should exist when sql.schema-version=V1 was set",
2707        );
2708    }
2709
2710    #[tokio::test]
2711    async fn test_v0_schema_no_migration_without_property() {
2712        install_default_drivers();
2713
2714        let (uri, temp_dir) = create_v0_sqlite_db().await;
2715
2716        // Opening without sql.schema-version=V1 should NOT migrate — but should still work.
2717        let props = HashMap::from_iter([
2718            (SQL_CATALOG_PROP_URI.to_string(), uri.clone()),
2719            (
2720                SQL_CATALOG_PROP_WAREHOUSE.to_string(),
2721                temp_dir.path().to_str().unwrap().to_string(),
2722            ),
2723            (
2724                SQL_CATALOG_PROP_BIND_STYLE.to_string(),
2725                SqlBindStyle::QMark.to_string(),
2726            ),
2727        ]);
2728        let catalog = SqlCatalogBuilder::default()
2729            .with_storage_factory(Arc::new(LocalFsStorageFactory))
2730            .load("iceberg", props)
2731            .await
2732            .expect("should open V0 catalog without migrating");
2733
2734        assert_eq!(catalog.schema_version, SchemaVersion::V0);
2735
2736        // The table should still be visible via V0 queries (no iceberg_type filter).
2737        let namespace = NamespaceIdent::from_strs(["test_namespace"]).unwrap();
2738        let tables = catalog.list_tables(&namespace).await.unwrap();
2739        assert_eq!(tables.len(), 1);
2740        assert_eq!(tables[0].name(), "existing_test_table");
2741
2742        assert!(
2743            !record_type_column_exists(&uri).await,
2744            "iceberg_type column should not exist when sql.schema-version=V1 was not set"
2745        );
2746    }
2747
2748    #[tokio::test]
2749    async fn test_explicit_v0_schema_version_is_honored() {
2750        install_default_drivers();
2751
2752        let (uri, temp_dir) = create_v0_sqlite_db().await;
2753
2754        let catalog = SqlCatalogBuilder::default()
2755            .with_storage_factory(Arc::new(LocalFsStorageFactory))
2756            .load(
2757                "iceberg",
2758                catalog_props(&uri, &temp_dir, Some(SchemaVersion::V0)),
2759            )
2760            .await
2761            .expect("requesting V0 against a V0 catalog table should succeed");
2762
2763        assert_eq!(catalog.schema_version, SchemaVersion::V0);
2764
2765        let namespace = NamespaceIdent::from_strs(["test_namespace"]).unwrap();
2766        let tables = catalog.list_tables(&namespace).await.unwrap();
2767        assert_eq!(tables.len(), 1);
2768        assert_eq!(tables[0].name(), "existing_test_table");
2769
2770        assert!(
2771            !record_type_column_exists(&uri).await,
2772            "iceberg_type column should not be added when V0 was requested"
2773        );
2774    }
2775
2776    #[tokio::test]
2777    async fn test_v0_schema_version_is_ignored_on_v1_table() {
2778        install_default_drivers();
2779
2780        let (uri, temp_dir) = create_v1_sqlite_db().await;
2781
2782        let catalog = SqlCatalogBuilder::default()
2783            .with_storage_factory(Arc::new(LocalFsStorageFactory))
2784            .load(
2785                "iceberg",
2786                catalog_props(&uri, &temp_dir, Some(SchemaVersion::V0)),
2787            )
2788            .await
2789            .expect("requesting V0 against a V1 catalog table should succeed");
2790
2791        assert_eq!(catalog.schema_version, SchemaVersion::V1);
2792
2793        let namespace = NamespaceIdent::from_strs(["test_namespace"]).unwrap();
2794        let tables = catalog.list_tables(&namespace).await.unwrap();
2795        assert_eq!(tables.len(), 1);
2796        assert_eq!(tables[0].name(), "existing_test_table");
2797    }
2798
2799    #[tokio::test]
2800    async fn test_v0_schema_version_on_empty_database_yields_v1() {
2801        install_default_drivers();
2802
2803        let temp_dir = TempDir::new().unwrap();
2804        let uri = format!(
2805            "sqlite:{}",
2806            temp_dir.path().join("catalog.db").to_str().unwrap()
2807        );
2808        sqlx::Sqlite::create_database(&uri).await.unwrap();
2809
2810        let catalog = SqlCatalogBuilder::default()
2811            .with_storage_factory(Arc::new(LocalFsStorageFactory))
2812            .load(
2813                "iceberg",
2814                catalog_props(&uri, &temp_dir, Some(SchemaVersion::V0)),
2815            )
2816            .await
2817            .expect("requesting V0 against an empty database should succeed");
2818
2819        assert_eq!(catalog.schema_version, SchemaVersion::V1);
2820    }
2821
2822    #[tokio::test]
2823    async fn test_create_table_unsupported_on_v0_schema() {
2824        install_default_drivers();
2825
2826        let (uri, temp_dir) = create_v0_sqlite_db().await;
2827
2828        let catalog = SqlCatalogBuilder::default()
2829            .with_storage_factory(Arc::new(LocalFsStorageFactory))
2830            .load("iceberg", catalog_props(&uri, &temp_dir, None))
2831            .await
2832            .expect("should open V0 catalog without migrating");
2833        assert_eq!(catalog.schema_version, SchemaVersion::V0);
2834
2835        // Inserts always populate `iceberg_type`, which a V0 catalog table does not have, so
2836        // creation is refused up front rather than failing in the database.
2837        let namespace = NamespaceIdent::from_strs(["test_namespace"]).unwrap();
2838        let err = catalog
2839            .create_table(
2840                &namespace,
2841                TableCreation::builder()
2842                    .name("created_test_table".to_string())
2843                    .schema(simple_table_schema())
2844                    .location(temp_path())
2845                    .build(),
2846            )
2847            .await
2848            .expect_err("table creation should be rejected on a V0 catalog table");
2849
2850        assert_eq!(err.kind(), ErrorKind::FeatureUnsupported);
2851        assert!(
2852            err.to_string().contains("Table creation is not supported"),
2853            "error should explain that table creation is unsupported, got: {err}"
2854        );
2855
2856        let tables = catalog.list_tables(&namespace).await.unwrap();
2857        assert_eq!(
2858            tables.len(),
2859            1,
2860            "only the pre-existing row should be present, got: {tables:?}"
2861        );
2862    }
2863
2864    #[tokio::test]
2865    async fn test_register_table_unsupported_on_v0_schema() {
2866        install_default_drivers();
2867
2868        let (uri, temp_dir) = create_v0_sqlite_db().await;
2869
2870        let catalog = SqlCatalogBuilder::default()
2871            .with_storage_factory(Arc::new(LocalFsStorageFactory))
2872            .load("iceberg", catalog_props(&uri, &temp_dir, None))
2873            .await
2874            .expect("should open V0 catalog without migrating");
2875        assert_eq!(catalog.schema_version, SchemaVersion::V0);
2876
2877        let namespace = NamespaceIdent::from_strs(["test_namespace"]).unwrap();
2878        let table_ident = TableIdent::new(namespace.clone(), "registered_test_table".to_string());
2879        // Register with non-existent table to ensure test is verifying that schema version is checked first.
2880        let err = catalog
2881            .register_table(
2882                &table_ident,
2883                "/tmp/does-not-exist/metadata.json".to_string(),
2884            )
2885            .await
2886            .expect_err("table registration should be rejected on a V0 catalog table");
2887
2888        assert_eq!(err.kind(), ErrorKind::FeatureUnsupported);
2889        assert!(
2890            err.to_string()
2891                .contains("Table registration is not supported"),
2892            "error should explain that table registration is unsupported, got: {err}"
2893        );
2894
2895        let tables = catalog.list_tables(&namespace).await.unwrap();
2896        assert_eq!(
2897            tables.len(),
2898            1,
2899            "only the pre-existing row should be present, got: {tables:?}"
2900        );
2901    }
2902
2903    #[tokio::test]
2904    async fn test_create_table_supported_after_v0_migration() {
2905        install_default_drivers();
2906
2907        let (uri, temp_dir) = create_v0_sqlite_db().await;
2908
2909        let catalog = SqlCatalogBuilder::default()
2910            .with_storage_factory(Arc::new(LocalFsStorageFactory))
2911            .load(
2912                "iceberg",
2913                catalog_props(&uri, &temp_dir, Some(SchemaVersion::V1)),
2914            )
2915            .await
2916            .expect("should open V0 catalog and migrate schema when sql.schema-version=V1");
2917        assert_eq!(catalog.schema_version, SchemaVersion::V1);
2918
2919        let namespace = NamespaceIdent::from_strs(["test_namespace"]).unwrap();
2920        catalog
2921            .create_table(
2922                &namespace,
2923                TableCreation::builder()
2924                    .name("created_test_table".to_string())
2925                    .schema(simple_table_schema())
2926                    .location(temp_path())
2927                    .build(),
2928            )
2929            .await
2930            .expect("table creation should succeed once the schema is migrated to V1");
2931
2932        let table_names = catalog
2933            .list_tables(&namespace)
2934            .await
2935            .unwrap()
2936            .iter()
2937            .map(|table_ident| table_ident.name().to_string())
2938            .sorted()
2939            .collect_vec();
2940        assert_eq!(
2941            table_names,
2942            vec!["created_test_table", "existing_test_table"],
2943            "the migrated pre-existing row and the new table should both be listed"
2944        );
2945    }
2946
2947    #[tokio::test]
2948    async fn test_invalid_schema_version_is_rejected() {
2949        install_default_drivers();
2950
2951        let (uri, temp_dir) = create_v0_sqlite_db().await;
2952
2953        // An unrecognized sql.schema-version value must fail fast rather than silently
2954        // falling back to V0.
2955        let props = HashMap::from_iter([
2956            (SQL_CATALOG_PROP_URI.to_string(), uri),
2957            (
2958                SQL_CATALOG_PROP_WAREHOUSE.to_string(),
2959                temp_dir.path().to_str().unwrap().to_string(),
2960            ),
2961            (
2962                SQL_CATALOG_PROP_BIND_STYLE.to_string(),
2963                SqlBindStyle::QMark.to_string(),
2964            ),
2965            (
2966                SQL_CATALOG_PROP_SCHEMA_VERSION.to_string(),
2967                "v2".to_string(),
2968            ),
2969        ]);
2970        let result = SqlCatalogBuilder::default()
2971            .with_storage_factory(Arc::new(LocalFsStorageFactory))
2972            .load("iceberg", props)
2973            .await;
2974
2975        let err = result.expect_err("an invalid sql.schema-version should be rejected");
2976        assert_eq!(err.kind(), ErrorKind::DataInvalid);
2977    }
2978
2979    #[tokio::test]
2980    async fn test_legacy_bind_style_key_is_accepted() {
2981        install_default_drivers();
2982
2983        let (uri, temp_dir) = create_v0_sqlite_db().await;
2984
2985        // The legacy `sql_bind_style` key must keep working alongside the new `sql.bind-style`.
2986        let props = HashMap::from_iter([
2987            (SQL_CATALOG_PROP_URI.to_string(), uri),
2988            (
2989                SQL_CATALOG_PROP_WAREHOUSE.to_string(),
2990                temp_dir.path().to_str().unwrap().to_string(),
2991            ),
2992            (
2993                SQL_CATALOG_PROP_BIND_STYLE_LEGACY.to_string(),
2994                SqlBindStyle::QMark.to_string(),
2995            ),
2996        ]);
2997        let catalog = SqlCatalogBuilder::default()
2998            .with_storage_factory(Arc::new(LocalFsStorageFactory))
2999            .load("iceberg", props)
3000            .await
3001            .expect("legacy sql_bind_style key should still be accepted");
3002
3003        assert_eq!(catalog.properties.sql_bind_style, SqlBindStyle::QMark);
3004    }
3005}