1use std::collections::{HashMap, HashSet};
19use std::str::FromStr;
20use std::sync::Arc;
21use std::time::Duration;
22
23use async_trait::async_trait;
24use iceberg::encryption::kms::{KeyManagementClient, KmsClientFactory};
25use iceberg::io::{FileIO, FileIOBuilder, StorageFactory};
26use iceberg::spec::{TableMetadata, TableMetadataBuilder};
27use iceberg::table::Table;
28use iceberg::{
29 Catalog, CatalogBuilder, Error, ErrorKind, MetadataLocation, Namespace, NamespaceIdent, Result,
30 Runtime, TableCommit, TableCreation, TableIdent, delete_removed_metadata_files,
31};
32use iceberg_property_macro::Properties;
33use sqlx::any::{AnyPoolOptions, AnyQueryResult, AnyRow, install_default_drivers};
34use sqlx::{Any, AnyPool, Column, Executor, Row, Transaction};
35
36use crate::error::{
37 from_sqlx_error, no_such_namespace_err, no_such_table_err, table_already_exists_err,
38};
39
40pub const SQL_CATALOG_PROP_URI: &str = "uri";
42pub const SQL_CATALOG_PROP_WAREHOUSE: &str = "warehouse";
44pub const SQL_CATALOG_PROP_BIND_STYLE: &str = "sql.bind-style";
46const SQL_CATALOG_PROP_BIND_STYLE_LEGACY: &str = "sql_bind_style";
49pub const SQL_CATALOG_PROP_SCHEMA_VERSION: &str = "sql.schema-version";
60
61static CATALOG_TABLE_NAME: &str = "iceberg_tables";
62static CATALOG_FIELD_CATALOG_NAME: &str = "catalog_name";
63static CATALOG_FIELD_TABLE_NAME: &str = "table_name";
64static CATALOG_FIELD_TABLE_NAMESPACE: &str = "table_namespace";
65static CATALOG_FIELD_METADATA_LOCATION_PROP: &str = "metadata_location";
66static CATALOG_FIELD_PREVIOUS_METADATA_LOCATION_PROP: &str = "previous_metadata_location";
67static CATALOG_FIELD_RECORD_TYPE: &str = "iceberg_type";
68static CATALOG_FIELD_TABLE_RECORD_TYPE: &str = "TABLE";
69
70static NAMESPACE_TABLE_NAME: &str = "iceberg_namespace_properties";
71static NAMESPACE_FIELD_NAME: &str = "namespace";
72static NAMESPACE_FIELD_PROPERTY_KEY: &str = "property_key";
73static NAMESPACE_FIELD_PROPERTY_VALUE: &str = "property_value";
74
75static NAMESPACE_LOCATION_PROPERTY_KEY: &str = "location";
76
77static MAX_CONNECTIONS: u32 = 10; static IDLE_TIMEOUT: u64 = 10; static TEST_BEFORE_ACQUIRE: bool = true; fn parse_pool_property<T>(value: &str) -> Result<T>
83where
84 T: FromStr,
85 T::Err: std::error::Error + Send + Sync + 'static,
86{
87 value.parse().map_err(|error| {
88 Error::new(
89 ErrorKind::DataInvalid,
90 "Failed to parse SQL catalog pool property",
91 )
92 .with_context("value", value)
93 .with_source(error)
94 })
95}
96
97#[derive(Debug, Default)]
99pub struct SqlCatalogBuilder {
100 props: HashMap<String, String>,
101 storage_factory: Option<Arc<dyn StorageFactory>>,
102 kms_client_factory: Option<Arc<dyn KmsClientFactory>>,
103 runtime: Option<Runtime>,
104}
105
106impl SqlCatalogBuilder {
107 pub fn uri(mut self, uri: impl Into<String>) -> Self {
112 self.props
113 .insert(SQL_CATALOG_PROP_URI.to_string(), uri.into());
114 self
115 }
116
117 pub fn warehouse_location(mut self, location: impl Into<String>) -> Self {
122 self.props
123 .insert(SQL_CATALOG_PROP_WAREHOUSE.to_string(), location.into());
124 self
125 }
126
127 pub fn sql_bind_style(mut self, sql_bind_style: SqlBindStyle) -> Self {
133 self.props.insert(
134 SQL_CATALOG_PROP_BIND_STYLE.to_string(),
135 sql_bind_style.to_string(),
136 );
137 self
138 }
139
140 pub fn props(mut self, props: HashMap<String, String>) -> Self {
145 for (k, v) in props {
146 self.props.insert(k, v);
147 }
148 self
149 }
150
151 pub fn prop(mut self, key: impl Into<String>, value: impl Into<String>) -> Self {
158 self.props.insert(key.into(), value.into());
159 self
160 }
161}
162
163impl CatalogBuilder for SqlCatalogBuilder {
164 type C = SqlCatalog;
165
166 fn with_storage_factory(mut self, storage_factory: Arc<dyn StorageFactory>) -> Self {
167 self.storage_factory = Some(storage_factory);
168 self
169 }
170
171 fn with_kms_client_factory(mut self, kms_client_factory: Arc<dyn KmsClientFactory>) -> Self {
172 self.kms_client_factory = Some(kms_client_factory);
173 self
174 }
175
176 fn with_runtime(mut self, runtime: Runtime) -> Self {
177 self.runtime = Some(runtime);
178 self
179 }
180
181 fn load(
182 self,
183 name: impl Into<String>,
184 props: HashMap<String, String>,
185 ) -> impl Future<Output = Result<Self::C>> + Send {
186 let name = name.into();
187
188 async move {
189 if name.trim().is_empty() {
190 return Err(Error::new(
191 ErrorKind::DataInvalid,
192 "Catalog name cannot be empty",
193 ));
194 }
195
196 let load_overrides_bind_style = props.contains_key(SQL_CATALOG_PROP_BIND_STYLE)
197 || props.contains_key(SQL_CATALOG_PROP_BIND_STYLE_LEGACY);
198 let mut merged_props = self.props;
199 if load_overrides_bind_style {
200 merged_props.remove(SQL_CATALOG_PROP_BIND_STYLE);
203 merged_props.remove(SQL_CATALOG_PROP_BIND_STYLE_LEGACY);
204 }
205 merged_props.extend(props);
206 let catalog_properties = SqlCatalogProperties::from_properties(&merged_props)?;
207
208 merged_props.remove(SQL_CATALOG_PROP_URI);
211
212 let runtime = match self.runtime {
213 Some(rt) => rt,
214 None => Runtime::try_current()?,
215 };
216 let kms_client = match self.kms_client_factory {
217 Some(factory) => Some(factory.create_kms_client(&merged_props).await?),
218 None => None,
219 };
220 SqlCatalog::new(
221 name,
222 catalog_properties,
223 merged_props,
224 self.storage_factory,
225 runtime,
226 kms_client,
227 )
228 .await
229 }
230 }
231}
232
233fn parse_sql_bind_style(
234 properties: &HashMap<String, String>,
235 key: &str,
236 additional_keys: &[&str],
237 default: SqlBindStyle,
238) -> Result<SqlBindStyle> {
239 let configured_value = properties.get(key).map(|value| (key, value)).or_else(|| {
240 additional_keys
241 .iter()
242 .find_map(|alt_key| properties.get(*alt_key).map(|value| (*alt_key, value)))
243 });
244
245 configured_value.map_or(Ok(default), |(configured_key, value)| {
246 SqlBindStyle::from_str(value).map_err(|_| {
247 Error::new(
248 ErrorKind::DataInvalid,
249 format!(
250 "`{}` values are valid only if they're `{}` or `{}`",
251 configured_key,
252 SqlBindStyle::DollarNumeric,
253 SqlBindStyle::QMark
254 ),
255 )
256 })
257 })
258}
259
260fn parse_schema_version(value: &str) -> Result<SchemaVersion> {
261 SchemaVersion::from_str(value).map_err(|_| {
262 Error::new(
263 ErrorKind::DataInvalid,
264 format!(
265 "`{}` values are valid only if they're `{}` or `{}`",
266 SQL_CATALOG_PROP_SCHEMA_VERSION,
267 SchemaVersion::V0,
268 SchemaVersion::V1
269 ),
270 )
271 })
272}
273
274#[derive(Debug, Properties)]
275pub(crate) struct SqlCatalogProperties {
276 #[property(key = SQL_CATALOG_PROP_URI, default = "")]
277 uri: String,
278 #[property(key = SQL_CATALOG_PROP_WAREHOUSE, default = "")]
279 warehouse_location: String,
280 #[property(
281 key = SQL_CATALOG_PROP_BIND_STYLE,
282 additional_keys = [SQL_CATALOG_PROP_BIND_STYLE_LEGACY],
283 default = SqlBindStyle::DollarNumeric,
284 parse_properties_with = parse_sql_bind_style
285 )]
286 sql_bind_style: SqlBindStyle,
287 #[property(
288 key = SQL_CATALOG_PROP_SCHEMA_VERSION,
289 default = None,
290 parse_with = parse_schema_version
291 )]
292 schema_version: Option<SchemaVersion>,
293 #[property(
294 key = "pool.max-connections",
295 default = MAX_CONNECTIONS,
296 parse_with = parse_pool_property
297 )]
298 max_connections: u32,
299 #[property(
300 key = "pool.idle-timeout",
301 default = IDLE_TIMEOUT,
302 parse_with = parse_pool_property
303 )]
304 idle_timeout: u64,
305 #[property(
306 key = "pool.test-before-acquire",
307 default = TEST_BEFORE_ACQUIRE,
308 parse_with = parse_pool_property
309 )]
310 test_before_acquire: bool,
311}
312
313#[derive(Debug)]
314pub struct SqlCatalog {
319 name: String,
320 properties: SqlCatalogProperties,
321 connection: AnyPool,
322 fileio: FileIO,
323 runtime: Runtime,
324 kms_client: Option<Arc<dyn KeyManagementClient>>,
325 schema_version: SchemaVersion,
326}
327
328#[derive(Debug, Clone, Copy, PartialEq, strum::EnumString, strum::Display)]
329#[strum(ascii_case_insensitive)]
330pub enum SchemaVersion {
332 V0,
334 V1,
336}
337
338impl SchemaVersion {
339 async fn detect(pool: &AnyPool) -> Result<Self> {
341 let catalog_table_description = pool
342 .describe(&format!("SELECT * FROM {CATALOG_TABLE_NAME}"))
343 .await
344 .map_err(from_sqlx_error)?;
345
346 let has_type_column = catalog_table_description.columns().iter().any(|column| {
347 column
348 .name()
349 .eq_ignore_ascii_case(CATALOG_FIELD_RECORD_TYPE)
350 });
351
352 Ok(if has_type_column {
353 SchemaVersion::V1
354 } else {
355 SchemaVersion::V0
356 })
357 }
358
359 fn record_type_filter(self) -> String {
365 match self {
366 SchemaVersion::V1 => format!(
367 "AND ({CATALOG_FIELD_RECORD_TYPE} = '{CATALOG_FIELD_TABLE_RECORD_TYPE}' \
368 OR {CATALOG_FIELD_RECORD_TYPE} IS NULL)"
369 ),
370 SchemaVersion::V0 => String::new(),
371 }
372 }
373
374 fn migration_sql(self) -> Option<String> {
378 match self {
379 SchemaVersion::V1 => Some(format!(
380 "ALTER TABLE {CATALOG_TABLE_NAME} ADD COLUMN {CATALOG_FIELD_RECORD_TYPE} VARCHAR(5)"
381 )),
382 SchemaVersion::V0 => None,
383 }
384 }
385}
386
387#[derive(Debug, PartialEq, strum::EnumString, strum::Display)]
388pub enum SqlBindStyle {
390 DollarNumeric,
392 QMark,
394}
395
396impl SqlCatalog {
397 async fn new(
399 name: String,
400 properties: SqlCatalogProperties,
401 props: HashMap<String, String>,
402 storage_factory: Option<Arc<dyn StorageFactory>>,
403 runtime: Runtime,
404 kms_client: Option<Arc<dyn KeyManagementClient>>,
405 ) -> Result<Self> {
406 let factory = storage_factory.ok_or_else(|| {
407 Error::new(
408 ErrorKind::Unexpected,
409 "StorageFactory must be provided for SqlCatalog. Use `with_storage_factory` to configure it.",
410 )
411 })?;
412 install_default_drivers();
413 let fileio = FileIOBuilder::new(factory).with_props(props).build();
416
417 let pool = AnyPoolOptions::new()
418 .max_connections(properties.max_connections)
419 .idle_timeout(Duration::from_secs(properties.idle_timeout))
420 .test_before_acquire(properties.test_before_acquire)
421 .connect(&properties.uri)
422 .await
423 .map_err(from_sqlx_error)?;
424
425 sqlx::query(&format!(
426 "CREATE TABLE IF NOT EXISTS {CATALOG_TABLE_NAME} (
427 {CATALOG_FIELD_CATALOG_NAME} VARCHAR(255) NOT NULL,
428 {CATALOG_FIELD_TABLE_NAMESPACE} VARCHAR(255) NOT NULL,
429 {CATALOG_FIELD_TABLE_NAME} VARCHAR(255) NOT NULL,
430 {CATALOG_FIELD_METADATA_LOCATION_PROP} VARCHAR(1000),
431 {CATALOG_FIELD_PREVIOUS_METADATA_LOCATION_PROP} VARCHAR(1000),
432 {CATALOG_FIELD_RECORD_TYPE} VARCHAR(5),
433 PRIMARY KEY ({CATALOG_FIELD_CATALOG_NAME}, {CATALOG_FIELD_TABLE_NAMESPACE}, {CATALOG_FIELD_TABLE_NAME}))"
434 ))
435 .execute(&pool)
436 .await
437 .map_err(from_sqlx_error)?;
438
439 sqlx::query(&format!(
440 "CREATE TABLE IF NOT EXISTS {NAMESPACE_TABLE_NAME} (
441 {CATALOG_FIELD_CATALOG_NAME} VARCHAR(255) NOT NULL,
442 {NAMESPACE_FIELD_NAME} VARCHAR(255) NOT NULL,
443 {NAMESPACE_FIELD_PROPERTY_KEY} VARCHAR(255),
444 {NAMESPACE_FIELD_PROPERTY_VALUE} VARCHAR(1000),
445 PRIMARY KEY ({CATALOG_FIELD_CATALOG_NAME}, {NAMESPACE_FIELD_NAME}, {NAMESPACE_FIELD_PROPERTY_KEY}))"
446 ))
447 .execute(&pool)
448 .await
449 .map_err(from_sqlx_error)?;
450
451 let detected_schema_version = SchemaVersion::detect(&pool).await?;
452 let expected_schema_version = properties.schema_version;
453
454 let schema_version = match (detected_schema_version, expected_schema_version) {
456 (SchemaVersion::V1, Some(SchemaVersion::V1) | None) => {
457 tracing::debug!(
458 "detected {CATALOG_TABLE_NAME} schema {} which already supports views",
459 detected_schema_version,
460 );
461 SchemaVersion::V1
462 }
463 (SchemaVersion::V0, Some(expected_schema_version @ SchemaVersion::V1)) => {
464 tracing::warn!(
465 "table {CATALOG_TABLE_NAME} has inferred schema {} but expected schema {}, performing migration",
466 detected_schema_version,
467 expected_schema_version,
468 );
469 if let Some(migration_sql) = SchemaVersion::V1.migration_sql() {
470 sqlx::query(&migration_sql)
471 .execute(&pool)
472 .await
473 .map_err(from_sqlx_error)?;
474 }
475 SchemaVersion::V1
476 }
477 (SchemaVersion::V0, Some(SchemaVersion::V0) | None) => {
478 tracing::warn!(
479 "table {CATALOG_TABLE_NAME} has inferred schema {}; SQL catalog is initialized without view support, table creation, and table registration. \
480 To auto-migrate the database schema, set {}=V1",
481 detected_schema_version,
482 SQL_CATALOG_PROP_SCHEMA_VERSION,
483 );
484 SchemaVersion::V0
485 }
486 (SchemaVersion::V1, Some(expected_schema_version @ SchemaVersion::V0)) => {
487 tracing::warn!(
488 "ignoring expected schema {} for table {CATALOG_TABLE_NAME}: the table is \
489 already at schema {}, and downgrade migration is not supported",
490 expected_schema_version,
491 detected_schema_version,
492 );
493 SchemaVersion::V1
494 }
495 };
496
497 Ok(SqlCatalog {
498 name,
499 properties,
500 connection: pool,
501 fileio,
502 runtime,
503 kms_client,
504 schema_version,
505 })
506 }
507
508 fn replace_placeholders(&self, query: &str) -> String {
510 match self.properties.sql_bind_style {
511 SqlBindStyle::DollarNumeric => {
512 let mut count = 1;
513 query
514 .chars()
515 .fold(String::with_capacity(query.len()), |mut acc, c| {
516 if c == '?' {
517 acc.push('$');
518 acc.push_str(&count.to_string());
519 count += 1;
520 } else {
521 acc.push(c);
522 }
523 acc
524 })
525 }
526 _ => query.to_owned(),
527 }
528 }
529
530 async fn fetch_rows(&self, query: &str, args: Vec<Option<&str>>) -> Result<Vec<AnyRow>> {
532 let query_with_placeholders = self.replace_placeholders(query);
533
534 let mut sqlx_query = sqlx::query(&query_with_placeholders);
535 for arg in args {
536 sqlx_query = sqlx_query.bind(arg);
537 }
538
539 sqlx_query
540 .fetch_all(&self.connection)
541 .await
542 .map_err(from_sqlx_error)
543 }
544
545 async fn execute(
547 &self,
548 query: &str,
549 args: Vec<Option<&str>>,
550 transaction: Option<&mut Transaction<'_, Any>>,
551 ) -> Result<AnyQueryResult> {
552 let query_with_placeholders = self.replace_placeholders(query);
553
554 let mut sqlx_query = sqlx::query(&query_with_placeholders);
555 for arg in args {
556 sqlx_query = sqlx_query.bind(arg);
557 }
558
559 match transaction {
560 Some(t) => sqlx_query.execute(&mut **t).await.map_err(from_sqlx_error),
561 None => {
562 let mut tx = self.connection.begin().await.map_err(from_sqlx_error)?;
563 let result = sqlx_query
564 .execute(&mut *tx)
565 .await
566 .map_err(from_sqlx_error)?;
567 tx.commit().await.map_err(from_sqlx_error)?;
568 Ok(result)
569 }
570 }
571 }
572}
573
574#[async_trait]
575impl Catalog for SqlCatalog {
576 async fn list_namespaces(
577 &self,
578 parent: Option<&NamespaceIdent>,
579 ) -> Result<Vec<NamespaceIdent>> {
580 let all_namespaces_stmt = format!(
582 "SELECT {CATALOG_FIELD_TABLE_NAMESPACE}
583 FROM {CATALOG_TABLE_NAME}
584 WHERE {CATALOG_FIELD_CATALOG_NAME} = ?
585 UNION
586 SELECT {NAMESPACE_FIELD_NAME}
587 FROM {NAMESPACE_TABLE_NAME}
588 WHERE {CATALOG_FIELD_CATALOG_NAME} = ?"
589 );
590
591 let namespace_rows = self
592 .fetch_rows(&all_namespaces_stmt, vec![
593 Some(&self.name),
594 Some(&self.name),
595 ])
596 .await?;
597
598 let mut namespaces = HashSet::<NamespaceIdent>::with_capacity(namespace_rows.len());
599
600 if let Some(parent) = parent {
601 if self.namespace_exists(parent).await? {
602 let parent_str = parent.join(".");
603
604 for row in namespace_rows.iter() {
605 let nsp = row.try_get::<String, _>(0).map_err(from_sqlx_error)?;
606 if nsp != parent_str && nsp.starts_with(&parent_str) {
608 namespaces.insert(NamespaceIdent::from_strs(nsp.split("."))?);
609 }
610 }
611
612 Ok(namespaces.into_iter().collect::<Vec<NamespaceIdent>>())
613 } else {
614 no_such_namespace_err(parent)
615 }
616 } else {
617 for row in namespace_rows.iter() {
618 let nsp = row.try_get::<String, _>(0).map_err(from_sqlx_error)?;
619 let mut levels = nsp.split(".").collect::<Vec<&str>>();
620 if !levels.is_empty() {
621 let first_level = levels.drain(..1).collect::<Vec<&str>>();
622 namespaces.insert(NamespaceIdent::from_strs(first_level)?);
623 }
624 }
625
626 Ok(namespaces.into_iter().collect::<Vec<NamespaceIdent>>())
627 }
628 }
629
630 async fn create_namespace(
631 &self,
632 namespace: &NamespaceIdent,
633 properties: HashMap<String, String>,
634 ) -> Result<Namespace> {
635 let exists = self.namespace_exists(namespace).await?;
636
637 if exists {
638 return Err(Error::new(
639 ErrorKind::NamespaceAlreadyExists,
640 format!("Namespace {namespace:?} already exists"),
641 ));
642 }
643
644 let namespace_str = namespace.join(".");
645 let insert = format!(
646 "INSERT INTO {NAMESPACE_TABLE_NAME} ({CATALOG_FIELD_CATALOG_NAME}, {NAMESPACE_FIELD_NAME}, {NAMESPACE_FIELD_PROPERTY_KEY}, {NAMESPACE_FIELD_PROPERTY_VALUE})
647 VALUES (?, ?, ?, ?)");
648 if !properties.is_empty() {
649 let mut insert_properties = properties.clone();
650 insert_properties.insert("exists".to_string(), "true".to_string());
651
652 let mut query_args = Vec::with_capacity(insert_properties.len() * 4);
653 let mut insert_stmt = insert.clone();
654 for (index, (key, value)) in insert_properties.iter().enumerate() {
655 query_args.extend_from_slice(&[
656 Some(self.name.as_str()),
657 Some(namespace_str.as_str()),
658 Some(key.as_str()),
659 Some(value.as_str()),
660 ]);
661 if index > 0 {
662 insert_stmt.push_str(", (?, ?, ?, ?)");
663 }
664 }
665
666 self.execute(&insert_stmt, query_args, None).await?;
667
668 Ok(Namespace::with_properties(
669 namespace.clone(),
670 insert_properties,
671 ))
672 } else {
673 self.execute(
675 &insert,
676 vec![
677 Some(&self.name),
678 Some(&namespace_str),
679 Some("exists"),
680 Some("true"),
681 ],
682 None,
683 )
684 .await?;
685 Ok(Namespace::with_properties(namespace.clone(), properties))
686 }
687 }
688
689 async fn get_namespace(&self, namespace: &NamespaceIdent) -> Result<Namespace> {
690 let exists = self.namespace_exists(namespace).await?;
691 if exists {
692 let namespace_props = self
693 .fetch_rows(
694 &format!(
695 "SELECT
696 {NAMESPACE_FIELD_NAME},
697 {NAMESPACE_FIELD_PROPERTY_KEY},
698 {NAMESPACE_FIELD_PROPERTY_VALUE}
699 FROM {NAMESPACE_TABLE_NAME}
700 WHERE {CATALOG_FIELD_CATALOG_NAME} = ?
701 AND {NAMESPACE_FIELD_NAME} = ?"
702 ),
703 vec![Some(&self.name), Some(&namespace.join("."))],
704 )
705 .await?;
706
707 let mut properties = HashMap::with_capacity(namespace_props.len());
708
709 for row in namespace_props {
710 let key = row
711 .try_get::<String, _>(NAMESPACE_FIELD_PROPERTY_KEY)
712 .map_err(from_sqlx_error)?;
713 let value = row
714 .try_get::<String, _>(NAMESPACE_FIELD_PROPERTY_VALUE)
715 .map_err(from_sqlx_error)?;
716
717 properties.insert(key, value);
718 }
719
720 Ok(Namespace::with_properties(namespace.clone(), properties))
721 } else {
722 no_such_namespace_err(namespace)
723 }
724 }
725
726 async fn namespace_exists(&self, namespace: &NamespaceIdent) -> Result<bool> {
727 let namespace_str = namespace.join(".");
728
729 let table_namespaces = self
730 .fetch_rows(
731 &format!(
732 "SELECT 1 FROM {CATALOG_TABLE_NAME}
733 WHERE {CATALOG_FIELD_CATALOG_NAME} = ?
734 AND {CATALOG_FIELD_TABLE_NAMESPACE} = ?
735 LIMIT 1"
736 ),
737 vec![Some(&self.name), Some(&namespace_str)],
738 )
739 .await?;
740
741 if !table_namespaces.is_empty() {
742 Ok(true)
743 } else {
744 let namespaces = self
745 .fetch_rows(
746 &format!(
747 "SELECT 1 FROM {NAMESPACE_TABLE_NAME}
748 WHERE {CATALOG_FIELD_CATALOG_NAME} = ?
749 AND {NAMESPACE_FIELD_NAME} = ?
750 LIMIT 1"
751 ),
752 vec![Some(&self.name), Some(&namespace_str)],
753 )
754 .await?;
755 if !namespaces.is_empty() {
756 Ok(true)
757 } else {
758 Ok(false)
759 }
760 }
761 }
762
763 async fn update_namespace(
764 &self,
765 namespace: &NamespaceIdent,
766 properties: HashMap<String, String>,
767 ) -> Result<()> {
768 let exists = self.namespace_exists(namespace).await?;
769 if exists {
770 let existing_properties = self.get_namespace(namespace).await?.properties().clone();
771 let namespace_str = namespace.join(".");
772
773 let mut updates = vec![];
774 let mut inserts = vec![];
775
776 for (key, value) in properties.iter() {
777 if existing_properties.contains_key(key) {
778 if existing_properties.get(key) != Some(value) {
779 updates.push((key, value));
780 }
781 } else {
782 inserts.push((key, value));
783 }
784 }
785
786 let mut tx = self.connection.begin().await.map_err(from_sqlx_error)?;
787 let update_stmt = format!(
788 "UPDATE {NAMESPACE_TABLE_NAME} SET {NAMESPACE_FIELD_PROPERTY_VALUE} = ?
789 WHERE {CATALOG_FIELD_CATALOG_NAME} = ?
790 AND {NAMESPACE_FIELD_NAME} = ?
791 AND {NAMESPACE_FIELD_PROPERTY_KEY} = ?"
792 );
793
794 let insert_stmt = format!(
795 "INSERT INTO {NAMESPACE_TABLE_NAME} ({CATALOG_FIELD_CATALOG_NAME}, {NAMESPACE_FIELD_NAME}, {NAMESPACE_FIELD_PROPERTY_KEY}, {NAMESPACE_FIELD_PROPERTY_VALUE})
796 VALUES (?, ?, ?, ?)"
797 );
798
799 for (key, value) in updates {
800 self.execute(
801 &update_stmt,
802 vec![
803 Some(value),
804 Some(&self.name),
805 Some(&namespace_str),
806 Some(key),
807 ],
808 Some(&mut tx),
809 )
810 .await?;
811 }
812
813 for (key, value) in inserts {
814 self.execute(
815 &insert_stmt,
816 vec![
817 Some(&self.name),
818 Some(&namespace_str),
819 Some(key),
820 Some(value),
821 ],
822 Some(&mut tx),
823 )
824 .await?;
825 }
826
827 let _ = tx.commit().await.map_err(from_sqlx_error)?;
828
829 Ok(())
830 } else {
831 no_such_namespace_err(namespace)
832 }
833 }
834
835 async fn drop_namespace(&self, namespace: &NamespaceIdent) -> Result<()> {
836 let exists = self.namespace_exists(namespace).await?;
837 if exists {
838 let tables = self.list_tables(namespace).await?;
840 if !tables.is_empty() {
841 return Err(Error::new(
842 ErrorKind::Unexpected,
843 format!(
844 "Namespace {:?} is not empty. {} tables exist.",
845 namespace,
846 tables.len()
847 ),
848 ));
849 }
850
851 self.execute(
852 &format!(
853 "DELETE FROM {NAMESPACE_TABLE_NAME}
854 WHERE {NAMESPACE_FIELD_NAME} = ?
855 AND {CATALOG_FIELD_CATALOG_NAME} = ?"
856 ),
857 vec![Some(&namespace.join(".")), Some(&self.name)],
858 None,
859 )
860 .await?;
861
862 Ok(())
863 } else {
864 no_such_namespace_err(namespace)
865 }
866 }
867
868 async fn list_tables(&self, namespace: &NamespaceIdent) -> Result<Vec<TableIdent>> {
869 let exists = self.namespace_exists(namespace).await?;
870 if exists {
871 let rows = self
872 .fetch_rows(
873 &format!(
874 "SELECT {CATALOG_FIELD_TABLE_NAME},
875 {CATALOG_FIELD_TABLE_NAMESPACE}
876 FROM {CATALOG_TABLE_NAME}
877 WHERE {CATALOG_FIELD_TABLE_NAMESPACE} = ?
878 AND {CATALOG_FIELD_CATALOG_NAME} = ?
879 {}",
880 self.schema_version.record_type_filter()
881 ),
882 vec![Some(&namespace.join(".")), Some(&self.name)],
883 )
884 .await?;
885
886 let mut tables = HashSet::<TableIdent>::with_capacity(rows.len());
887
888 for row in rows.iter() {
889 let tbl = row
890 .try_get::<String, _>(CATALOG_FIELD_TABLE_NAME)
891 .map_err(from_sqlx_error)?;
892 let ns_strs = row
893 .try_get::<String, _>(CATALOG_FIELD_TABLE_NAMESPACE)
894 .map_err(from_sqlx_error)?;
895 let ns = NamespaceIdent::from_strs(ns_strs.split("."))?;
896 tables.insert(TableIdent::new(ns, tbl));
897 }
898
899 Ok(tables.into_iter().collect::<Vec<TableIdent>>())
900 } else {
901 no_such_namespace_err(namespace)
902 }
903 }
904
905 async fn table_exists(&self, identifier: &TableIdent) -> Result<bool> {
906 let namespace = identifier.namespace().join(".");
907 let table_name = identifier.name();
908 let table_counts = self
909 .fetch_rows(
910 &format!(
911 "SELECT 1
912 FROM {CATALOG_TABLE_NAME}
913 WHERE {CATALOG_FIELD_TABLE_NAMESPACE} = ?
914 AND {CATALOG_FIELD_CATALOG_NAME} = ?
915 AND {CATALOG_FIELD_TABLE_NAME} = ?
916 {}",
917 self.schema_version.record_type_filter()
918 ),
919 vec![Some(&namespace), Some(&self.name), Some(table_name)],
920 )
921 .await?;
922
923 if !table_counts.is_empty() {
924 Ok(true)
925 } else {
926 Ok(false)
927 }
928 }
929
930 async fn drop_table(&self, identifier: &TableIdent) -> Result<()> {
931 if !self.table_exists(identifier).await? {
932 return no_such_table_err(identifier);
933 }
934
935 self.execute(
936 &format!(
937 "DELETE FROM {CATALOG_TABLE_NAME}
938 WHERE {CATALOG_FIELD_CATALOG_NAME} = ?
939 AND {CATALOG_FIELD_TABLE_NAME} = ?
940 AND {CATALOG_FIELD_TABLE_NAMESPACE} = ?
941 {}",
942 self.schema_version.record_type_filter()
943 ),
944 vec![
945 Some(&self.name),
946 Some(identifier.name()),
947 Some(&identifier.namespace().join(".")),
948 ],
949 None,
950 )
951 .await?;
952
953 Ok(())
954 }
955
956 async fn purge_table(&self, table: &TableIdent) -> Result<()> {
957 let table_info = self.load_table(table).await?;
958 self.drop_table(table).await?;
959 iceberg::drop_table_data(&table_info).await
960 }
961
962 async fn load_table(&self, identifier: &TableIdent) -> Result<Table> {
963 if !self.table_exists(identifier).await? {
964 return no_such_table_err(identifier);
965 }
966
967 let rows = self
968 .fetch_rows(
969 &format!(
970 "SELECT {CATALOG_FIELD_METADATA_LOCATION_PROP}
971 FROM {CATALOG_TABLE_NAME}
972 WHERE {CATALOG_FIELD_CATALOG_NAME} = ?
973 AND {CATALOG_FIELD_TABLE_NAME} = ?
974 AND {CATALOG_FIELD_TABLE_NAMESPACE} = ?
975 {}",
976 self.schema_version.record_type_filter()
977 ),
978 vec![
979 Some(&self.name),
980 Some(identifier.name()),
981 Some(&identifier.namespace().join(".")),
982 ],
983 )
984 .await?;
985
986 if rows.is_empty() {
987 return no_such_table_err(identifier);
988 }
989
990 let row = &rows[0];
991 let tbl_metadata_location = row
992 .try_get::<String, _>(CATALOG_FIELD_METADATA_LOCATION_PROP)
993 .map_err(from_sqlx_error)?;
994
995 let metadata = TableMetadata::read_from(&self.fileio, &tbl_metadata_location).await?;
996
997 let mut builder = Table::builder()
998 .file_io(self.fileio.clone())
999 .identifier(identifier.clone())
1000 .metadata_location(tbl_metadata_location)
1001 .metadata(metadata)
1002 .runtime(self.runtime.clone());
1003 if let Some(kms_client) = self.kms_client.clone() {
1004 builder = builder.kms_client(kms_client);
1005 }
1006 Ok(builder.build()?)
1007 }
1008
1009 async fn create_table(
1010 &self,
1011 namespace: &NamespaceIdent,
1012 creation: TableCreation,
1013 ) -> Result<Table> {
1014 if self.schema_version != SchemaVersion::V1 {
1015 return Err(Error::new(
1016 ErrorKind::FeatureUnsupported,
1017 format!(
1018 "Table creation is not supported for SQL catalog schema version {}",
1019 self.schema_version
1020 ),
1021 ));
1022 }
1023
1024 if !self.namespace_exists(namespace).await? {
1025 return no_such_namespace_err(namespace);
1026 }
1027
1028 let tbl_name = creation.name.clone();
1029 let tbl_ident = TableIdent::new(namespace.clone(), tbl_name.clone());
1030
1031 if self.table_exists(&tbl_ident).await? {
1032 return table_already_exists_err(&tbl_ident);
1033 }
1034
1035 let tbl_creation = if creation.location.is_some() {
1036 creation
1037 } else {
1038 let nsp_properties = self.get_namespace(namespace).await?.properties().clone();
1041 let nsp_location = match nsp_properties.get(NAMESPACE_LOCATION_PROPERTY_KEY) {
1042 Some(location) => location.clone(),
1043 None => {
1044 format!(
1045 "{}/{}",
1046 &self.properties.warehouse_location,
1047 namespace.join("/")
1048 )
1049 }
1050 };
1051
1052 let tbl_location = format!("{}/{}", nsp_location, tbl_ident.name());
1053
1054 TableCreation {
1055 location: Some(tbl_location),
1056 ..creation
1057 }
1058 };
1059
1060 let tbl_metadata = TableMetadataBuilder::from_table_creation(tbl_creation)?
1061 .build()?
1062 .metadata;
1063 let tbl_metadata_location = MetadataLocation::try_new_with_metadata(&tbl_metadata)?;
1064
1065 tbl_metadata
1066 .write_to(&self.fileio, &tbl_metadata_location)
1067 .await?;
1068
1069 let tbl_metadata_location_str = tbl_metadata_location.to_string();
1070 self.execute(&format!(
1071 "INSERT INTO {CATALOG_TABLE_NAME}
1072 ({CATALOG_FIELD_CATALOG_NAME}, {CATALOG_FIELD_TABLE_NAMESPACE}, {CATALOG_FIELD_TABLE_NAME}, {CATALOG_FIELD_METADATA_LOCATION_PROP}, {CATALOG_FIELD_RECORD_TYPE})
1073 VALUES (?, ?, ?, ?, ?)
1074 "), vec![Some(&self.name), Some(&namespace.join(".")), Some(&tbl_name.clone()), Some(&tbl_metadata_location_str), Some(CATALOG_FIELD_TABLE_RECORD_TYPE)], None).await?;
1075
1076 let mut builder = Table::builder()
1077 .file_io(self.fileio.clone())
1078 .metadata_location(tbl_metadata_location_str)
1079 .identifier(tbl_ident)
1080 .metadata(tbl_metadata)
1081 .runtime(self.runtime.clone());
1082 if let Some(kms_client) = self.kms_client.clone() {
1083 builder = builder.kms_client(kms_client);
1084 }
1085 Ok(builder.build()?)
1086 }
1087
1088 async fn rename_table(&self, src: &TableIdent, dest: &TableIdent) -> Result<()> {
1089 if src == dest {
1090 return Ok(());
1091 }
1092
1093 if !self.table_exists(src).await? {
1094 return no_such_table_err(src);
1095 }
1096
1097 if !self.namespace_exists(dest.namespace()).await? {
1098 return no_such_namespace_err(dest.namespace());
1099 }
1100
1101 if self.table_exists(dest).await? {
1102 return table_already_exists_err(dest);
1103 }
1104
1105 self.execute(
1106 &format!(
1107 "UPDATE {CATALOG_TABLE_NAME}
1108 SET {CATALOG_FIELD_TABLE_NAME} = ?, {CATALOG_FIELD_TABLE_NAMESPACE} = ?
1109 WHERE {CATALOG_FIELD_CATALOG_NAME} = ?
1110 AND {CATALOG_FIELD_TABLE_NAME} = ?
1111 AND {CATALOG_FIELD_TABLE_NAMESPACE} = ?
1112 {}",
1113 self.schema_version.record_type_filter()
1114 ),
1115 vec![
1116 Some(dest.name()),
1117 Some(&dest.namespace().join(".")),
1118 Some(&self.name),
1119 Some(src.name()),
1120 Some(&src.namespace().join(".")),
1121 ],
1122 None,
1123 )
1124 .await?;
1125
1126 Ok(())
1127 }
1128
1129 async fn register_table(
1130 &self,
1131 table_ident: &TableIdent,
1132 metadata_location: String,
1133 ) -> Result<Table> {
1134 if self.schema_version != SchemaVersion::V1 {
1135 return Err(Error::new(
1136 ErrorKind::FeatureUnsupported,
1137 format!(
1138 "Table registration is not supported for SQL catalog schema version {}",
1139 self.schema_version
1140 ),
1141 ));
1142 }
1143
1144 if self.table_exists(table_ident).await? {
1145 return table_already_exists_err(table_ident);
1146 }
1147
1148 let metadata = TableMetadata::read_from(&self.fileio, &metadata_location).await?;
1149
1150 let namespace = table_ident.namespace();
1151 let tbl_name = table_ident.name().to_string();
1152
1153 self.execute(&format!(
1154 "INSERT INTO {CATALOG_TABLE_NAME}
1155 ({CATALOG_FIELD_CATALOG_NAME}, {CATALOG_FIELD_TABLE_NAMESPACE}, {CATALOG_FIELD_TABLE_NAME}, {CATALOG_FIELD_METADATA_LOCATION_PROP}, {CATALOG_FIELD_RECORD_TYPE})
1156 VALUES (?, ?, ?, ?, ?)
1157 "), vec![Some(&self.name), Some(&namespace.join(".")), Some(&tbl_name), Some(&metadata_location), Some(CATALOG_FIELD_TABLE_RECORD_TYPE)], None).await?;
1158
1159 let mut builder = Table::builder()
1160 .identifier(table_ident.clone())
1161 .metadata_location(metadata_location)
1162 .metadata(metadata)
1163 .file_io(self.fileio.clone())
1164 .runtime(self.runtime.clone());
1165 if let Some(kms_client) = self.kms_client.clone() {
1166 builder = builder.kms_client(kms_client);
1167 }
1168 Ok(builder.build()?)
1169 }
1170
1171 async fn update_table(&self, commit: TableCommit) -> Result<Table> {
1173 let table_ident = commit.identifier().clone();
1174 let current_table = self.load_table(&table_ident).await?;
1175 let current_metadata_location = current_table.metadata_location_result()?.to_string();
1176 let base_metadata = current_table.metadata_ref();
1177
1178 let staged_table = commit.apply(current_table)?;
1179 let staged_metadata_location_str = staged_table.metadata_location_result()?;
1180 let staged_metadata_location = MetadataLocation::from_str(staged_metadata_location_str)?;
1181
1182 staged_table
1183 .metadata()
1184 .write_to(staged_table.file_io(), &staged_metadata_location)
1185 .await?;
1186
1187 let staged_metadata_location_str = staged_metadata_location.to_string();
1188 let update_result = self
1189 .execute(
1190 &format!(
1191 "UPDATE {CATALOG_TABLE_NAME}
1192 SET {CATALOG_FIELD_METADATA_LOCATION_PROP} = ?, {CATALOG_FIELD_PREVIOUS_METADATA_LOCATION_PROP} = ?
1193 WHERE {CATALOG_FIELD_CATALOG_NAME} = ?
1194 AND {CATALOG_FIELD_TABLE_NAME} = ?
1195 AND {CATALOG_FIELD_TABLE_NAMESPACE} = ?
1196 {}
1197 AND {CATALOG_FIELD_METADATA_LOCATION_PROP} = ?",
1198 self.schema_version.record_type_filter()
1199 ),
1200 vec![
1201 Some(&staged_metadata_location_str),
1202 Some(current_metadata_location.as_str()),
1203 Some(&self.name),
1204 Some(table_ident.name()),
1205 Some(&table_ident.namespace().join(".")),
1206 Some(current_metadata_location.as_str()),
1207 ],
1208 None,
1209 )
1210 .await?;
1211
1212 if update_result.rows_affected() == 0 {
1213 return Err(Error::new(
1214 ErrorKind::CatalogCommitConflicts,
1215 format!("Commit conflicted for table: {table_ident}"),
1216 )
1217 .with_retryable(true));
1218 }
1219
1220 delete_removed_metadata_files(
1221 staged_table.file_io(),
1222 &base_metadata,
1223 staged_table.metadata(),
1224 )
1225 .await;
1226
1227 Ok(staged_table)
1228 }
1229}
1230
1231#[cfg(test)]
1232mod tests {
1233 use std::collections::{HashMap, HashSet};
1234 use std::hash::Hash;
1235 use std::sync::Arc;
1236
1237 use iceberg::io::LocalFsStorageFactory;
1238 use iceberg::spec::{NestedField, PartitionSpec, PrimitiveType, Schema, SortOrder, Type};
1239 use iceberg::table::Table;
1240 use iceberg::{
1241 Catalog, CatalogBuilder, ErrorKind, Namespace, NamespaceIdent, TableCreation, TableIdent,
1242 };
1243 use itertools::Itertools;
1244 use regex::Regex;
1245 use sqlx::any::install_default_drivers;
1246 use sqlx::migrate::MigrateDatabase;
1247 use sqlx::{Column, Executor};
1248 use tempfile::TempDir;
1249
1250 use crate::catalog::{
1251 CATALOG_FIELD_RECORD_TYPE, CATALOG_TABLE_NAME, IDLE_TIMEOUT, MAX_CONNECTIONS,
1252 NAMESPACE_LOCATION_PROPERTY_KEY, NAMESPACE_TABLE_NAME, SQL_CATALOG_PROP_BIND_STYLE,
1253 SQL_CATALOG_PROP_BIND_STYLE_LEGACY, SQL_CATALOG_PROP_SCHEMA_VERSION, SQL_CATALOG_PROP_URI,
1254 SQL_CATALOG_PROP_WAREHOUSE, SqlCatalogProperties, TEST_BEFORE_ACQUIRE,
1255 };
1256 use crate::{SchemaVersion, SqlBindStyle, SqlCatalog, SqlCatalogBuilder};
1257
1258 const UUID_REGEX_STR: &str = "[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}";
1259
1260 fn temp_path() -> String {
1261 let temp_dir = TempDir::new().unwrap();
1262 temp_dir.path().to_str().unwrap().to_string()
1263 }
1264
1265 #[test]
1266 fn test_catalog_properties() {
1267 let properties = SqlCatalogProperties::from_properties(&HashMap::from([
1268 (
1269 SQL_CATALOG_PROP_URI.to_string(),
1270 "sqlite://catalog".to_string(),
1271 ),
1272 (
1273 SQL_CATALOG_PROP_WAREHOUSE.to_string(),
1274 "/warehouse".to_string(),
1275 ),
1276 (
1277 SQL_CATALOG_PROP_BIND_STYLE_LEGACY.to_string(),
1278 SqlBindStyle::QMark.to_string(),
1279 ),
1280 (
1281 SQL_CATALOG_PROP_SCHEMA_VERSION.to_string(),
1282 "V1".to_string(),
1283 ),
1284 ("pool.max-connections".to_string(), "5".to_string()),
1285 ("pool.idle-timeout".to_string(), "20".to_string()),
1286 ("pool.test-before-acquire".to_string(), "false".to_string()),
1287 ]))
1288 .unwrap();
1289
1290 assert_eq!(properties.uri, "sqlite://catalog");
1291 assert_eq!(properties.warehouse_location, "/warehouse");
1292 assert_eq!(properties.sql_bind_style, SqlBindStyle::QMark);
1293 assert_eq!(properties.schema_version, Some(SchemaVersion::V1));
1294 assert_eq!(properties.max_connections, 5);
1295 assert_eq!(properties.idle_timeout, 20);
1296 assert!(!properties.test_before_acquire);
1297 }
1298
1299 #[test]
1300 fn test_catalog_properties_defaults() {
1301 let properties = SqlCatalogProperties::from_properties(&HashMap::new()).unwrap();
1302
1303 assert_eq!(properties.uri, "");
1304 assert_eq!(properties.warehouse_location, "");
1305 assert_eq!(properties.sql_bind_style, SqlBindStyle::DollarNumeric);
1306 assert_eq!(properties.schema_version, None);
1307 assert_eq!(properties.max_connections, MAX_CONNECTIONS);
1308 assert_eq!(properties.idle_timeout, IDLE_TIMEOUT);
1309 assert_eq!(properties.test_before_acquire, TEST_BEFORE_ACQUIRE);
1310 }
1311
1312 #[test]
1313 fn test_preferred_bind_style_key_takes_precedence_over_legacy_key() {
1314 let properties = SqlCatalogProperties::from_properties(&HashMap::from([
1315 (
1316 SQL_CATALOG_PROP_BIND_STYLE.to_string(),
1317 SqlBindStyle::QMark.to_string(),
1318 ),
1319 (
1320 SQL_CATALOG_PROP_BIND_STYLE_LEGACY.to_string(),
1321 SqlBindStyle::DollarNumeric.to_string(),
1322 ),
1323 ]))
1324 .unwrap();
1325
1326 assert_eq!(properties.sql_bind_style, SqlBindStyle::QMark);
1327 }
1328
1329 #[test]
1330 fn test_invalid_legacy_bind_style_error_names_configured_key() {
1331 let error = SqlCatalogProperties::from_properties(&HashMap::from([(
1332 SQL_CATALOG_PROP_BIND_STYLE_LEGACY.to_string(),
1333 "invalid".to_string(),
1334 )]))
1335 .unwrap_err();
1336
1337 assert_eq!(error.kind(), ErrorKind::DataInvalid);
1338 assert_eq!(
1339 error.message(),
1340 "`sql_bind_style` values are valid only if they're `DollarNumeric` or `QMark`"
1341 );
1342 }
1343
1344 fn to_set<T: Eq + Hash>(vec: Vec<T>) -> HashSet<T> {
1345 HashSet::from_iter(vec)
1346 }
1347
1348 fn default_properties() -> HashMap<String, String> {
1349 HashMap::from([("exists".to_string(), "true".to_string())])
1350 }
1351
1352 async fn new_sql_catalog(
1354 warehouse_location: String,
1355 name: Option<impl ToString>,
1356 ) -> impl Catalog {
1357 let name = if let Some(name) = name {
1358 name.to_string()
1359 } else {
1360 "iceberg".to_string()
1361 };
1362 let sql_lite_uri = format!("sqlite:{}", temp_path());
1363 sqlx::Sqlite::create_database(&sql_lite_uri).await.unwrap();
1364
1365 let props = HashMap::from_iter([
1366 (SQL_CATALOG_PROP_URI.to_string(), sql_lite_uri.to_string()),
1367 (SQL_CATALOG_PROP_WAREHOUSE.to_string(), warehouse_location),
1368 (
1369 SQL_CATALOG_PROP_BIND_STYLE.to_string(),
1370 SqlBindStyle::DollarNumeric.to_string(),
1371 ),
1372 ]);
1373 SqlCatalogBuilder::default()
1374 .with_storage_factory(Arc::new(LocalFsStorageFactory))
1375 .load(&name, props)
1376 .await
1377 .unwrap()
1378 }
1379
1380 async fn create_namespace<C: Catalog>(catalog: &C, namespace_ident: &NamespaceIdent) {
1381 let _ = catalog
1382 .create_namespace(namespace_ident, HashMap::new())
1383 .await
1384 .unwrap();
1385 }
1386
1387 async fn create_namespaces<C: Catalog>(catalog: &C, namespace_idents: &Vec<&NamespaceIdent>) {
1388 for namespace_ident in namespace_idents {
1389 let _ = create_namespace(catalog, namespace_ident).await;
1390 }
1391 }
1392
1393 fn simple_table_schema() -> Schema {
1394 Schema::builder()
1395 .with_fields(vec![
1396 NestedField::required(1, "foo", Type::Primitive(PrimitiveType::Int)).into(),
1397 ])
1398 .build()
1399 .unwrap()
1400 }
1401
1402 async fn create_table<C: Catalog>(catalog: &C, table_ident: &TableIdent) {
1403 let _ = catalog
1404 .create_table(
1405 &table_ident.namespace,
1406 TableCreation::builder()
1407 .name(table_ident.name().into())
1408 .schema(simple_table_schema())
1409 .location(temp_path())
1410 .build(),
1411 )
1412 .await
1413 .unwrap();
1414 }
1415
1416 async fn create_tables<C: Catalog>(catalog: &C, table_idents: Vec<&TableIdent>) {
1417 for table_ident in table_idents {
1418 create_table(catalog, table_ident).await;
1419 }
1420 }
1421
1422 fn assert_table_eq(table: &Table, expected_table_ident: &TableIdent, expected_schema: &Schema) {
1423 assert_eq!(table.identifier(), expected_table_ident);
1424
1425 let metadata = table.metadata();
1426
1427 assert_eq!(metadata.current_schema().as_ref(), expected_schema);
1428
1429 let expected_partition_spec = PartitionSpec::builder(expected_schema.clone())
1430 .with_spec_id(0)
1431 .build()
1432 .unwrap();
1433
1434 assert_eq!(
1435 metadata
1436 .partition_specs_iter()
1437 .map(|p| p.as_ref())
1438 .collect_vec(),
1439 vec![&expected_partition_spec]
1440 );
1441
1442 let expected_sorted_order = SortOrder::builder()
1443 .with_order_id(0)
1444 .with_fields(vec![])
1445 .build(expected_schema)
1446 .unwrap();
1447
1448 assert_eq!(
1449 metadata
1450 .sort_orders_iter()
1451 .map(|s| s.as_ref())
1452 .collect_vec(),
1453 vec![&expected_sorted_order]
1454 );
1455
1456 assert_eq!(metadata.properties(), &HashMap::new());
1457
1458 assert!(!table.readonly());
1459 }
1460
1461 fn assert_table_metadata_location_matches(table: &Table, regex_str: &str) {
1462 let actual = table.metadata_location().unwrap().to_string();
1463 let regex = Regex::new(regex_str).unwrap();
1464 assert!(regex.is_match(&actual))
1465 }
1466
1467 #[tokio::test]
1468 async fn test_initialized() {
1469 let warehouse_loc = temp_path();
1470 new_sql_catalog(warehouse_loc.clone(), Some("iceberg")).await;
1471 new_sql_catalog(warehouse_loc.clone(), Some("iceberg")).await;
1473 new_sql_catalog(warehouse_loc.clone(), Some("iceberg")).await;
1474 }
1475
1476 async fn new_commit_error_catalog() -> SqlCatalog {
1477 let sql_lite_uri = format!("sqlite:{}", temp_path());
1478 sqlx::Sqlite::create_database(&sql_lite_uri).await.unwrap();
1479 let catalog = SqlCatalogBuilder::default()
1480 .with_storage_factory(Arc::new(LocalFsStorageFactory))
1481 .prop("pool.max-connections", "1")
1482 .load(
1483 "iceberg",
1484 HashMap::from_iter([
1485 (SQL_CATALOG_PROP_URI.to_string(), sql_lite_uri),
1486 (SQL_CATALOG_PROP_WAREHOUSE.to_string(), temp_path()),
1487 ]),
1488 )
1489 .await
1490 .unwrap();
1491
1492 catalog
1493 .connection
1494 .execute("PRAGMA foreign_keys = ON")
1495 .await
1496 .unwrap();
1497 catalog
1499 .connection
1500 .execute("CREATE TABLE parent(id INTEGER PRIMARY KEY)")
1501 .await
1502 .unwrap();
1503 catalog
1504 .connection
1505 .execute(
1506 "CREATE TABLE child(parent_id INTEGER REFERENCES parent(id) \
1507 DEFERRABLE INITIALLY DEFERRED)",
1508 )
1509 .await
1510 .unwrap();
1511
1512 catalog
1513 }
1514
1515 #[tokio::test]
1516 async fn test_execute_returns_commit_error() {
1517 let catalog = new_commit_error_catalog().await;
1518
1519 let trigger = format!(
1522 "CREATE TRIGGER fail_namespace_commit
1523 AFTER INSERT ON {NAMESPACE_TABLE_NAME}
1524 BEGIN INSERT INTO child VALUES (1); END"
1525 );
1526 catalog.connection.execute(trigger.as_str()).await.unwrap();
1527
1528 let failed_namespace = NamespaceIdent::new("failed".into());
1529 let error = catalog
1530 .create_namespace(&failed_namespace, HashMap::new())
1531 .await
1532 .unwrap_err();
1533 assert_eq!(error.kind(), ErrorKind::Unexpected);
1534 assert!(!catalog.namespace_exists(&failed_namespace).await.unwrap());
1535
1536 catalog
1538 .connection
1539 .execute("INSERT INTO parent VALUES (1)")
1540 .await
1541 .unwrap();
1542 let committed_namespace = NamespaceIdent::new("committed".into());
1543 catalog
1544 .create_namespace(&committed_namespace, HashMap::new())
1545 .await
1546 .unwrap();
1547 assert!(
1548 catalog
1549 .namespace_exists(&committed_namespace)
1550 .await
1551 .unwrap()
1552 );
1553 }
1554
1555 #[tokio::test]
1558 async fn test_storage_props_propagate_to_file_io() {
1559 let sql_lite_uri = format!("sqlite:{}", temp_path());
1560 sqlx::Sqlite::create_database(&sql_lite_uri).await.unwrap();
1561 let warehouse_location = temp_path();
1562
1563 let catalog = SqlCatalogBuilder::default()
1564 .with_storage_factory(Arc::new(LocalFsStorageFactory))
1565 .load(
1566 "iceberg",
1567 HashMap::from_iter([
1568 (SQL_CATALOG_PROP_URI.to_string(), sql_lite_uri),
1569 (
1570 SQL_CATALOG_PROP_WAREHOUSE.to_string(),
1571 warehouse_location.clone(),
1572 ),
1573 ("s3.region".to_string(), "us-east-1".to_string()),
1574 ("hf.token".to_string(), "hf_test_token".to_string()),
1575 ]),
1576 )
1577 .await
1578 .unwrap();
1579
1580 let props = catalog.fileio.config().props();
1581 assert_eq!(props.get(SQL_CATALOG_PROP_URI), None);
1582 assert_eq!(
1583 props.get(SQL_CATALOG_PROP_WAREHOUSE),
1584 Some(&warehouse_location)
1585 );
1586 assert_eq!(props.get("s3.region"), Some(&"us-east-1".to_string()));
1587 assert_eq!(props.get("hf.token"), Some(&"hf_test_token".to_string()));
1588 }
1589
1590 #[tokio::test]
1591 async fn test_builder_method() {
1592 let sql_lite_uri = format!("sqlite:{}", temp_path());
1593 sqlx::Sqlite::create_database(&sql_lite_uri).await.unwrap();
1594 let warehouse_location = temp_path();
1595
1596 let catalog = SqlCatalogBuilder::default()
1597 .with_storage_factory(Arc::new(LocalFsStorageFactory))
1598 .uri(sql_lite_uri.to_string())
1599 .warehouse_location(warehouse_location.clone())
1600 .sql_bind_style(SqlBindStyle::QMark)
1601 .load("iceberg", HashMap::default())
1602 .await;
1603 assert!(catalog.is_ok());
1604
1605 let catalog = catalog.unwrap();
1606 assert!(catalog.properties.warehouse_location == warehouse_location);
1607 assert!(catalog.properties.sql_bind_style == SqlBindStyle::QMark);
1608 }
1609
1610 #[tokio::test]
1613 async fn test_builder_props_non_existent_path_fails() {
1614 let sql_lite_uri = format!("sqlite:{}", temp_path());
1615 let sql_lite_uri2 = format!("sqlite:{}", temp_path());
1616 sqlx::Sqlite::create_database(&sql_lite_uri).await.unwrap();
1617 let warehouse_location = temp_path();
1618
1619 let catalog = SqlCatalogBuilder::default()
1620 .with_storage_factory(Arc::new(LocalFsStorageFactory))
1621 .uri(sql_lite_uri)
1622 .warehouse_location(warehouse_location)
1623 .load(
1624 "iceberg",
1625 HashMap::from_iter([(SQL_CATALOG_PROP_URI.to_string(), sql_lite_uri2)]),
1626 )
1627 .await;
1628 assert!(catalog.is_err());
1629 }
1630
1631 #[tokio::test]
1635 async fn test_builder_props_set_valid_uri() {
1636 let sql_lite_uri = format!("sqlite:{}", temp_path());
1637 let sql_lite_uri2 = format!("sqlite:{}", temp_path());
1638 sqlx::Sqlite::create_database(&sql_lite_uri).await.unwrap();
1639 let warehouse_location = temp_path();
1640
1641 let catalog = SqlCatalogBuilder::default()
1642 .with_storage_factory(Arc::new(LocalFsStorageFactory))
1643 .uri(sql_lite_uri2)
1644 .warehouse_location(warehouse_location)
1645 .load(
1646 "iceberg",
1647 HashMap::from_iter([(SQL_CATALOG_PROP_URI.to_string(), sql_lite_uri.clone())]),
1648 )
1649 .await;
1650 assert!(catalog.is_ok());
1651 }
1652
1653 #[tokio::test]
1655 async fn test_builder_props_take_precedence() {
1656 let sql_lite_uri = format!("sqlite:{}", temp_path());
1657 sqlx::Sqlite::create_database(&sql_lite_uri).await.unwrap();
1658 let warehouse_location = temp_path();
1659 let warehouse_location2 = temp_path();
1660
1661 let catalog = SqlCatalogBuilder::default()
1662 .with_storage_factory(Arc::new(LocalFsStorageFactory))
1663 .warehouse_location(warehouse_location2)
1664 .sql_bind_style(SqlBindStyle::DollarNumeric)
1665 .load(
1666 "iceberg",
1667 HashMap::from_iter([
1668 (SQL_CATALOG_PROP_URI.to_string(), sql_lite_uri),
1669 (
1670 SQL_CATALOG_PROP_WAREHOUSE.to_string(),
1671 warehouse_location.clone(),
1672 ),
1673 (
1674 SQL_CATALOG_PROP_BIND_STYLE.to_string(),
1675 SqlBindStyle::QMark.to_string(),
1676 ),
1677 ]),
1678 )
1679 .await;
1680
1681 assert!(catalog.is_ok());
1682
1683 let catalog = catalog.unwrap();
1684 assert!(catalog.properties.warehouse_location == warehouse_location);
1685 assert!(catalog.properties.sql_bind_style == SqlBindStyle::QMark);
1686 }
1687
1688 #[tokio::test]
1689 async fn test_load_legacy_bind_style_overrides_builder_method() {
1690 let sql_lite_uri = format!("sqlite:{}", temp_path());
1691 sqlx::Sqlite::create_database(&sql_lite_uri).await.unwrap();
1692
1693 let catalog = SqlCatalogBuilder::default()
1694 .with_storage_factory(Arc::new(LocalFsStorageFactory))
1695 .sql_bind_style(SqlBindStyle::DollarNumeric)
1696 .load(
1697 "iceberg",
1698 HashMap::from([
1699 (SQL_CATALOG_PROP_URI.to_string(), sql_lite_uri),
1700 (SQL_CATALOG_PROP_WAREHOUSE.to_string(), temp_path()),
1701 (
1702 SQL_CATALOG_PROP_BIND_STYLE_LEGACY.to_string(),
1703 SqlBindStyle::QMark.to_string(),
1704 ),
1705 ]),
1706 )
1707 .await
1708 .unwrap();
1709
1710 assert_eq!(catalog.properties.sql_bind_style, SqlBindStyle::QMark);
1711 }
1712
1713 #[tokio::test]
1715 async fn test_builder_props_take_precedence_props() {
1716 let sql_lite_uri = format!("sqlite:{}", temp_path());
1717 let sql_lite_uri2 = format!("sqlite:{}", temp_path());
1718 sqlx::Sqlite::create_database(&sql_lite_uri).await.unwrap();
1719 let warehouse_location = temp_path();
1720 let warehouse_location2 = temp_path();
1721
1722 let props = HashMap::from_iter([
1723 (SQL_CATALOG_PROP_URI.to_string(), sql_lite_uri.clone()),
1724 (
1725 SQL_CATALOG_PROP_WAREHOUSE.to_string(),
1726 warehouse_location.clone(),
1727 ),
1728 (
1729 SQL_CATALOG_PROP_BIND_STYLE.to_string(),
1730 SqlBindStyle::QMark.to_string(),
1731 ),
1732 ]);
1733 let props2 = HashMap::from_iter([
1734 (SQL_CATALOG_PROP_URI.to_string(), sql_lite_uri2.clone()),
1735 (
1736 SQL_CATALOG_PROP_WAREHOUSE.to_string(),
1737 warehouse_location2.clone(),
1738 ),
1739 (
1740 SQL_CATALOG_PROP_BIND_STYLE.to_string(),
1741 SqlBindStyle::DollarNumeric.to_string(),
1742 ),
1743 ]);
1744
1745 let catalog = SqlCatalogBuilder::default()
1746 .with_storage_factory(Arc::new(LocalFsStorageFactory))
1747 .props(props2)
1748 .load("iceberg", props)
1749 .await;
1750
1751 assert!(catalog.is_ok());
1752
1753 let catalog = catalog.unwrap();
1754 assert!(catalog.properties.warehouse_location == warehouse_location);
1755 assert!(catalog.properties.sql_bind_style == SqlBindStyle::QMark);
1756 }
1757
1758 #[tokio::test]
1760 async fn test_builder_props_take_precedence_prop() {
1761 let sql_lite_uri = format!("sqlite:{}", temp_path());
1762 let sql_lite_uri2 = format!("sqlite:{}", temp_path());
1763 sqlx::Sqlite::create_database(&sql_lite_uri).await.unwrap();
1764 let warehouse_location = temp_path();
1765 let warehouse_location2 = temp_path();
1766
1767 let props = HashMap::from_iter([
1768 (SQL_CATALOG_PROP_URI.to_string(), sql_lite_uri.clone()),
1769 (
1770 SQL_CATALOG_PROP_WAREHOUSE.to_string(),
1771 warehouse_location.clone(),
1772 ),
1773 (
1774 SQL_CATALOG_PROP_BIND_STYLE.to_string(),
1775 SqlBindStyle::QMark.to_string(),
1776 ),
1777 ]);
1778
1779 let catalog = SqlCatalogBuilder::default()
1780 .with_storage_factory(Arc::new(LocalFsStorageFactory))
1781 .prop(SQL_CATALOG_PROP_URI.to_string(), sql_lite_uri2)
1782 .prop(SQL_CATALOG_PROP_WAREHOUSE.to_string(), warehouse_location2)
1783 .prop(
1784 SQL_CATALOG_PROP_BIND_STYLE.to_string(),
1785 SqlBindStyle::DollarNumeric.to_string(),
1786 )
1787 .load("iceberg", props)
1788 .await;
1789
1790 assert!(catalog.is_ok());
1791
1792 let catalog = catalog.unwrap();
1793 assert!(catalog.properties.warehouse_location == warehouse_location);
1794 assert!(catalog.properties.sql_bind_style == SqlBindStyle::QMark);
1795 }
1796
1797 #[tokio::test]
1799 async fn test_builder_props_invalid_bind_style_fails() {
1800 let sql_lite_uri = format!("sqlite:{}", temp_path());
1801 sqlx::Sqlite::create_database(&sql_lite_uri).await.unwrap();
1802 let warehouse_location = temp_path();
1803
1804 let catalog = SqlCatalogBuilder::default()
1805 .with_storage_factory(Arc::new(LocalFsStorageFactory))
1806 .load(
1807 "iceberg",
1808 HashMap::from_iter([
1809 (SQL_CATALOG_PROP_URI.to_string(), sql_lite_uri),
1810 (SQL_CATALOG_PROP_WAREHOUSE.to_string(), warehouse_location),
1811 (SQL_CATALOG_PROP_BIND_STYLE.to_string(), "AAA".to_string()),
1812 ]),
1813 )
1814 .await;
1815
1816 assert!(catalog.is_err());
1817 }
1818
1819 #[tokio::test]
1820 async fn test_builder_props_invalid_pool_property_fails() {
1821 for property in [
1822 "pool.max-connections",
1823 "pool.idle-timeout",
1824 "pool.test-before-acquire",
1825 ] {
1826 let error = SqlCatalogBuilder::default()
1827 .with_storage_factory(Arc::new(LocalFsStorageFactory))
1828 .prop(property, "invalid")
1829 .load("iceberg", HashMap::new())
1830 .await
1831 .unwrap_err();
1832
1833 assert_eq!(error.kind(), ErrorKind::DataInvalid);
1834 assert!(error.to_string().contains(property));
1835 assert!(error.to_string().contains("invalid"));
1836 }
1837 }
1838
1839 #[tokio::test]
1840 async fn test_list_namespaces_returns_empty_vector() {
1841 let warehouse_loc = temp_path();
1842 let catalog = new_sql_catalog(warehouse_loc, Some("iceberg")).await;
1843
1844 assert_eq!(catalog.list_namespaces(None).await.unwrap(), vec![]);
1845 }
1846
1847 #[tokio::test]
1848 async fn test_list_namespaces_returns_empty_different_name() {
1849 let warehouse_loc = temp_path();
1850 let catalog = new_sql_catalog(warehouse_loc.clone(), Some("iceberg")).await;
1851 let namespace_ident_1 = NamespaceIdent::new("a".into());
1852 let namespace_ident_2 = NamespaceIdent::new("b".into());
1853 create_namespaces(&catalog, &vec![&namespace_ident_1, &namespace_ident_2]).await;
1854 assert_eq!(
1855 to_set(catalog.list_namespaces(None).await.unwrap()),
1856 to_set(vec![namespace_ident_1, namespace_ident_2])
1857 );
1858
1859 let catalog2 = new_sql_catalog(warehouse_loc, Some("test")).await;
1860 assert_eq!(catalog2.list_namespaces(None).await.unwrap(), vec![]);
1861 }
1862
1863 #[tokio::test]
1864 async fn test_list_namespaces_returns_only_top_level_namespaces() {
1865 let warehouse_loc = temp_path();
1866 let catalog = new_sql_catalog(warehouse_loc, Some("iceberg")).await;
1867 let namespace_ident_1 = NamespaceIdent::new("a".into());
1868 let namespace_ident_2 = NamespaceIdent::from_strs(vec!["a", "b"]).unwrap();
1869 let namespace_ident_3 = NamespaceIdent::new("b".into());
1870 create_namespaces(&catalog, &vec![
1871 &namespace_ident_1,
1872 &namespace_ident_2,
1873 &namespace_ident_3,
1874 ])
1875 .await;
1876
1877 assert_eq!(
1878 to_set(catalog.list_namespaces(None).await.unwrap()),
1879 to_set(vec![namespace_ident_1, namespace_ident_3])
1880 );
1881 }
1882
1883 #[tokio::test]
1884 async fn test_list_namespaces_returns_no_namespaces_under_parent() {
1885 let warehouse_loc = temp_path();
1886 let catalog = new_sql_catalog(warehouse_loc, Some("iceberg")).await;
1887 let namespace_ident_1 = NamespaceIdent::new("a".into());
1888 let namespace_ident_2 = NamespaceIdent::new("b".into());
1889 create_namespaces(&catalog, &vec![&namespace_ident_1, &namespace_ident_2]).await;
1890
1891 assert_eq!(
1892 catalog
1893 .list_namespaces(Some(&namespace_ident_1))
1894 .await
1895 .unwrap(),
1896 vec![]
1897 );
1898 }
1899
1900 #[tokio::test]
1901 async fn test_list_namespaces_returns_namespace_under_parent() {
1902 let warehouse_loc = temp_path();
1903 let catalog = new_sql_catalog(warehouse_loc, Some("iceberg")).await;
1904 let namespace_ident_1 = NamespaceIdent::new("a".into());
1905 let namespace_ident_2 = NamespaceIdent::from_strs(vec!["a", "b"]).unwrap();
1906 let namespace_ident_3 = NamespaceIdent::new("c".into());
1907 create_namespaces(&catalog, &vec![
1908 &namespace_ident_1,
1909 &namespace_ident_2,
1910 &namespace_ident_3,
1911 ])
1912 .await;
1913
1914 assert_eq!(
1915 to_set(catalog.list_namespaces(None).await.unwrap()),
1916 to_set(vec![namespace_ident_1.clone(), namespace_ident_3])
1917 );
1918
1919 assert_eq!(
1920 catalog
1921 .list_namespaces(Some(&namespace_ident_1))
1922 .await
1923 .unwrap(),
1924 vec![NamespaceIdent::from_strs(vec!["a", "b"]).unwrap()]
1925 );
1926 }
1927
1928 #[tokio::test]
1929 async fn test_list_namespaces_returns_multiple_namespaces_under_parent() {
1930 let warehouse_loc = temp_path();
1931 let catalog = new_sql_catalog(warehouse_loc, Some("iceberg")).await;
1932 let namespace_ident_1 = NamespaceIdent::new("a".to_string());
1933 let namespace_ident_2 = NamespaceIdent::from_strs(vec!["a", "a"]).unwrap();
1934 let namespace_ident_3 = NamespaceIdent::from_strs(vec!["a", "b"]).unwrap();
1935 let namespace_ident_4 = NamespaceIdent::from_strs(vec!["a", "c"]).unwrap();
1936 let namespace_ident_5 = NamespaceIdent::new("b".into());
1937 create_namespaces(&catalog, &vec![
1938 &namespace_ident_1,
1939 &namespace_ident_2,
1940 &namespace_ident_3,
1941 &namespace_ident_4,
1942 &namespace_ident_5,
1943 ])
1944 .await;
1945
1946 assert_eq!(
1947 to_set(
1948 catalog
1949 .list_namespaces(Some(&namespace_ident_1))
1950 .await
1951 .unwrap()
1952 ),
1953 to_set(vec![
1954 NamespaceIdent::from_strs(vec!["a", "a"]).unwrap(),
1955 NamespaceIdent::from_strs(vec!["a", "b"]).unwrap(),
1956 NamespaceIdent::from_strs(vec!["a", "c"]).unwrap(),
1957 ])
1958 );
1959 }
1960
1961 #[tokio::test]
1962 async fn test_namespace_exists_returns_false() {
1963 let warehouse_loc = temp_path();
1964 let catalog = new_sql_catalog(warehouse_loc, Some("iceberg")).await;
1965 let namespace_ident = NamespaceIdent::new("a".into());
1966 create_namespace(&catalog, &namespace_ident).await;
1967
1968 assert!(
1969 !catalog
1970 .namespace_exists(&NamespaceIdent::new("b".into()))
1971 .await
1972 .unwrap()
1973 );
1974 }
1975
1976 #[tokio::test]
1977 async fn test_namespace_exists_returns_true() {
1978 let warehouse_loc = temp_path();
1979 let catalog = new_sql_catalog(warehouse_loc, Some("iceberg")).await;
1980 let namespace_ident = NamespaceIdent::new("a".into());
1981 create_namespace(&catalog, &namespace_ident).await;
1982
1983 assert!(catalog.namespace_exists(&namespace_ident).await.unwrap());
1984 }
1985
1986 #[tokio::test]
1987 async fn test_create_namespace_with_properties() {
1988 let warehouse_loc = temp_path();
1989 let catalog = new_sql_catalog(warehouse_loc, Some("iceberg")).await;
1990 let namespace_ident = NamespaceIdent::new("abc".into());
1991
1992 let mut properties = default_properties();
1993 properties.insert("k".into(), "v".into());
1994
1995 assert_eq!(
1996 catalog
1997 .create_namespace(&namespace_ident, properties.clone())
1998 .await
1999 .unwrap(),
2000 Namespace::with_properties(namespace_ident.clone(), properties.clone())
2001 );
2002
2003 assert_eq!(
2004 catalog.get_namespace(&namespace_ident).await.unwrap(),
2005 Namespace::with_properties(namespace_ident, properties)
2006 );
2007 }
2008
2009 #[tokio::test]
2010 async fn test_create_nested_namespace() {
2011 let warehouse_loc = temp_path();
2012 let catalog = new_sql_catalog(warehouse_loc, Some("iceberg")).await;
2013 let parent_namespace_ident = NamespaceIdent::new("a".into());
2014 create_namespace(&catalog, &parent_namespace_ident).await;
2015
2016 let child_namespace_ident = NamespaceIdent::from_strs(vec!["a", "b"]).unwrap();
2017
2018 assert_eq!(
2019 catalog
2020 .create_namespace(&child_namespace_ident, HashMap::new())
2021 .await
2022 .unwrap(),
2023 Namespace::new(child_namespace_ident.clone())
2024 );
2025
2026 assert_eq!(
2027 catalog.get_namespace(&child_namespace_ident).await.unwrap(),
2028 Namespace::with_properties(child_namespace_ident, default_properties())
2029 );
2030 }
2031
2032 #[tokio::test]
2033 async fn test_create_deeply_nested_namespace() {
2034 let warehouse_loc = temp_path();
2035 let catalog = new_sql_catalog(warehouse_loc, Some("iceberg")).await;
2036 let namespace_ident_a = NamespaceIdent::new("a".into());
2037 let namespace_ident_a_b = NamespaceIdent::from_strs(vec!["a", "b"]).unwrap();
2038 create_namespaces(&catalog, &vec![&namespace_ident_a, &namespace_ident_a_b]).await;
2039
2040 let namespace_ident_a_b_c = NamespaceIdent::from_strs(vec!["a", "b", "c"]).unwrap();
2041
2042 assert_eq!(
2043 catalog
2044 .create_namespace(&namespace_ident_a_b_c, HashMap::new())
2045 .await
2046 .unwrap(),
2047 Namespace::new(namespace_ident_a_b_c.clone())
2048 );
2049
2050 assert_eq!(
2051 catalog.get_namespace(&namespace_ident_a_b_c).await.unwrap(),
2052 Namespace::with_properties(namespace_ident_a_b_c, default_properties())
2053 );
2054 }
2055
2056 #[tokio::test]
2057 async fn test_update_namespace_noop() {
2058 let warehouse_loc = temp_path();
2059 let catalog = new_sql_catalog(warehouse_loc, Some("iceberg")).await;
2060 let namespace_ident = NamespaceIdent::new("a".into());
2061 create_namespace(&catalog, &namespace_ident).await;
2062
2063 catalog
2064 .update_namespace(&namespace_ident, HashMap::new())
2065 .await
2066 .unwrap();
2067
2068 assert_eq!(
2069 *catalog
2070 .get_namespace(&namespace_ident)
2071 .await
2072 .unwrap()
2073 .properties(),
2074 HashMap::from_iter([("exists".to_string(), "true".to_string())])
2075 )
2076 }
2077
2078 #[tokio::test]
2079 async fn test_update_nested_namespace() {
2080 let warehouse_loc = temp_path();
2081 let catalog = new_sql_catalog(warehouse_loc, Some("iceberg")).await;
2082 let namespace_ident = NamespaceIdent::from_strs(["a", "b"]).unwrap();
2083 create_namespace(&catalog, &namespace_ident).await;
2084
2085 let mut props = HashMap::from_iter([
2086 ("prop1".to_string(), "val1".to_string()),
2087 ("prop2".into(), "val2".into()),
2088 ]);
2089
2090 catalog
2091 .update_namespace(&namespace_ident, props.clone())
2092 .await
2093 .unwrap();
2094
2095 props.insert("exists".into(), "true".into());
2096
2097 assert_eq!(
2098 *catalog
2099 .get_namespace(&namespace_ident)
2100 .await
2101 .unwrap()
2102 .properties(),
2103 props
2104 )
2105 }
2106
2107 #[tokio::test]
2108 async fn test_update_namespace_errors_if_nested_namespace_doesnt_exist() {
2109 let warehouse_loc = temp_path();
2110 let catalog = new_sql_catalog(warehouse_loc, Some("iceberg")).await;
2111 let namespace_ident = NamespaceIdent::from_strs(["a", "b"]).unwrap();
2112
2113 let props = HashMap::from_iter([
2114 ("prop1".to_string(), "val1".to_string()),
2115 ("prop2".into(), "val2".into()),
2116 ]);
2117
2118 let err = catalog
2119 .update_namespace(&namespace_ident, props)
2120 .await
2121 .unwrap_err();
2122
2123 assert_eq!(
2124 err.message(),
2125 format!("No such namespace: {namespace_ident:?}")
2126 );
2127 }
2128
2129 #[tokio::test]
2130 async fn test_drop_nested_namespace() {
2131 let warehouse_loc = temp_path();
2132 let catalog = new_sql_catalog(warehouse_loc, Some("iceberg")).await;
2133 let namespace_ident_a = NamespaceIdent::new("a".into());
2134 let namespace_ident_a_b = NamespaceIdent::from_strs(vec!["a", "b"]).unwrap();
2135 create_namespaces(&catalog, &vec![&namespace_ident_a, &namespace_ident_a_b]).await;
2136
2137 catalog.drop_namespace(&namespace_ident_a_b).await.unwrap();
2138
2139 assert!(
2140 !catalog
2141 .namespace_exists(&namespace_ident_a_b)
2142 .await
2143 .unwrap()
2144 );
2145
2146 assert!(catalog.namespace_exists(&namespace_ident_a).await.unwrap());
2147 }
2148
2149 #[tokio::test]
2150 async fn test_drop_deeply_nested_namespace() {
2151 let warehouse_loc = temp_path();
2152 let catalog = new_sql_catalog(warehouse_loc, Some("iceberg")).await;
2153 let namespace_ident_a = NamespaceIdent::new("a".into());
2154 let namespace_ident_a_b = NamespaceIdent::from_strs(vec!["a", "b"]).unwrap();
2155 let namespace_ident_a_b_c = NamespaceIdent::from_strs(vec!["a", "b", "c"]).unwrap();
2156 create_namespaces(&catalog, &vec![
2157 &namespace_ident_a,
2158 &namespace_ident_a_b,
2159 &namespace_ident_a_b_c,
2160 ])
2161 .await;
2162
2163 catalog
2164 .drop_namespace(&namespace_ident_a_b_c)
2165 .await
2166 .unwrap();
2167
2168 assert!(
2169 !catalog
2170 .namespace_exists(&namespace_ident_a_b_c)
2171 .await
2172 .unwrap()
2173 );
2174
2175 assert!(
2176 catalog
2177 .namespace_exists(&namespace_ident_a_b)
2178 .await
2179 .unwrap()
2180 );
2181
2182 assert!(catalog.namespace_exists(&namespace_ident_a).await.unwrap());
2183 }
2184
2185 #[tokio::test]
2186 async fn test_drop_namespace_throws_error_if_nested_namespace_doesnt_exist() {
2187 let warehouse_loc = temp_path();
2188 let catalog = new_sql_catalog(warehouse_loc, Some("iceberg")).await;
2189 create_namespace(&catalog, &NamespaceIdent::new("a".into())).await;
2190
2191 let non_existent_namespace_ident =
2192 NamespaceIdent::from_vec(vec!["a".into(), "b".into()]).unwrap();
2193 assert_eq!(
2194 catalog
2195 .drop_namespace(&non_existent_namespace_ident)
2196 .await
2197 .unwrap_err()
2198 .to_string(),
2199 format!("NamespaceNotFound => No such namespace: {non_existent_namespace_ident:?}")
2200 )
2201 }
2202
2203 #[tokio::test]
2204 async fn test_dropping_a_namespace_does_not_drop_namespaces_nested_under_that_one() {
2205 let warehouse_loc = temp_path();
2206 let catalog = new_sql_catalog(warehouse_loc, Some("iceberg")).await;
2207 let namespace_ident_a = NamespaceIdent::new("a".into());
2208 let namespace_ident_a_b = NamespaceIdent::from_strs(vec!["a", "b"]).unwrap();
2209 create_namespaces(&catalog, &vec![&namespace_ident_a, &namespace_ident_a_b]).await;
2210
2211 catalog.drop_namespace(&namespace_ident_a).await.unwrap();
2212
2213 assert!(!catalog.namespace_exists(&namespace_ident_a).await.unwrap());
2214
2215 assert!(
2216 catalog
2217 .namespace_exists(&namespace_ident_a_b)
2218 .await
2219 .unwrap()
2220 );
2221 }
2222
2223 #[tokio::test]
2224 async fn test_create_table_falls_back_to_namespace_location_if_table_location_is_missing() {
2225 let warehouse_loc = temp_path();
2226 let catalog = new_sql_catalog(warehouse_loc, Some("iceberg")).await;
2227
2228 let namespace_ident = NamespaceIdent::new("a".into());
2229 let mut namespace_properties = HashMap::new();
2230 let namespace_location = temp_path();
2231 namespace_properties.insert(
2232 NAMESPACE_LOCATION_PROPERTY_KEY.to_string(),
2233 namespace_location.to_string(),
2234 );
2235 catalog
2236 .create_namespace(&namespace_ident, namespace_properties)
2237 .await
2238 .unwrap();
2239
2240 let table_name = "tbl1";
2241 let expected_table_ident = TableIdent::new(namespace_ident.clone(), table_name.into());
2242 let expected_table_metadata_location_regex =
2243 format!("^{namespace_location}/tbl1/metadata/00000-{UUID_REGEX_STR}.metadata.json$",);
2244
2245 let table = catalog
2246 .create_table(
2247 &namespace_ident,
2248 TableCreation::builder()
2249 .name(table_name.into())
2250 .schema(simple_table_schema())
2251 .build(),
2253 )
2254 .await
2255 .unwrap();
2256 assert_table_eq(&table, &expected_table_ident, &simple_table_schema());
2257 assert_table_metadata_location_matches(&table, &expected_table_metadata_location_regex);
2258
2259 let table = catalog.load_table(&expected_table_ident).await.unwrap();
2260 assert_table_eq(&table, &expected_table_ident, &simple_table_schema());
2261 assert_table_metadata_location_matches(&table, &expected_table_metadata_location_regex);
2262 }
2263
2264 #[tokio::test]
2265 async fn test_create_table_in_nested_namespace_falls_back_to_nested_namespace_location_if_table_location_is_missing()
2266 {
2267 let warehouse_loc = temp_path();
2268 let catalog = new_sql_catalog(warehouse_loc, Some("iceberg")).await;
2269
2270 let namespace_ident = NamespaceIdent::new("a".into());
2271 let mut namespace_properties = HashMap::new();
2272 let namespace_location = temp_path();
2273 namespace_properties.insert(
2274 NAMESPACE_LOCATION_PROPERTY_KEY.to_string(),
2275 namespace_location.to_string(),
2276 );
2277 catalog
2278 .create_namespace(&namespace_ident, namespace_properties)
2279 .await
2280 .unwrap();
2281
2282 let nested_namespace_ident = NamespaceIdent::from_strs(vec!["a", "b"]).unwrap();
2283 let mut nested_namespace_properties = HashMap::new();
2284 let nested_namespace_location = temp_path();
2285 nested_namespace_properties.insert(
2286 NAMESPACE_LOCATION_PROPERTY_KEY.to_string(),
2287 nested_namespace_location.to_string(),
2288 );
2289 catalog
2290 .create_namespace(&nested_namespace_ident, nested_namespace_properties)
2291 .await
2292 .unwrap();
2293
2294 let table_name = "tbl1";
2295 let expected_table_ident =
2296 TableIdent::new(nested_namespace_ident.clone(), table_name.into());
2297 let expected_table_metadata_location_regex = format!(
2298 "^{nested_namespace_location}/tbl1/metadata/00000-{UUID_REGEX_STR}.metadata.json$",
2299 );
2300
2301 let table = catalog
2302 .create_table(
2303 &nested_namespace_ident,
2304 TableCreation::builder()
2305 .name(table_name.into())
2306 .schema(simple_table_schema())
2307 .build(),
2309 )
2310 .await
2311 .unwrap();
2312 assert_table_eq(&table, &expected_table_ident, &simple_table_schema());
2313 assert_table_metadata_location_matches(&table, &expected_table_metadata_location_regex);
2314
2315 let table = catalog.load_table(&expected_table_ident).await.unwrap();
2316 assert_table_eq(&table, &expected_table_ident, &simple_table_schema());
2317 assert_table_metadata_location_matches(&table, &expected_table_metadata_location_regex);
2318 }
2319
2320 #[tokio::test]
2321 async fn test_create_table_falls_back_to_warehouse_location_if_both_table_location_and_namespace_location_are_missing()
2322 {
2323 let warehouse_loc = temp_path();
2324 let catalog = new_sql_catalog(warehouse_loc.clone(), Some("iceberg")).await;
2325
2326 let namespace_ident = NamespaceIdent::new("a".into());
2327 let namespace_properties = HashMap::new();
2329 catalog
2330 .create_namespace(&namespace_ident, namespace_properties)
2331 .await
2332 .unwrap();
2333
2334 let table_name = "tbl1";
2335 let expected_table_ident = TableIdent::new(namespace_ident.clone(), table_name.into());
2336 let expected_table_metadata_location_regex =
2337 format!("^{warehouse_loc}/a/tbl1/metadata/00000-{UUID_REGEX_STR}.metadata.json$");
2338
2339 let table = catalog
2340 .create_table(
2341 &namespace_ident,
2342 TableCreation::builder()
2343 .name(table_name.into())
2344 .schema(simple_table_schema())
2345 .build(),
2347 )
2348 .await
2349 .unwrap();
2350 assert_table_eq(&table, &expected_table_ident, &simple_table_schema());
2351 assert_table_metadata_location_matches(&table, &expected_table_metadata_location_regex);
2352
2353 let table = catalog.load_table(&expected_table_ident).await.unwrap();
2354 assert_table_eq(&table, &expected_table_ident, &simple_table_schema());
2355 assert_table_metadata_location_matches(&table, &expected_table_metadata_location_regex);
2356 }
2357
2358 #[tokio::test]
2359 async fn test_create_table_in_nested_namespace_falls_back_to_warehouse_location_if_both_table_location_and_namespace_location_are_missing()
2360 {
2361 let warehouse_loc = temp_path();
2362 let catalog = new_sql_catalog(warehouse_loc.clone(), Some("iceberg")).await;
2363
2364 let namespace_ident = NamespaceIdent::new("a".into());
2365 create_namespace(&catalog, &namespace_ident).await;
2366
2367 let nested_namespace_ident = NamespaceIdent::from_strs(vec!["a", "b"]).unwrap();
2368 create_namespace(&catalog, &nested_namespace_ident).await;
2369
2370 let table_name = "tbl1";
2371 let expected_table_ident =
2372 TableIdent::new(nested_namespace_ident.clone(), table_name.into());
2373 let expected_table_metadata_location_regex =
2374 format!("^{warehouse_loc}/a/b/tbl1/metadata/00000-{UUID_REGEX_STR}.metadata.json$");
2375
2376 let table = catalog
2377 .create_table(
2378 &nested_namespace_ident,
2379 TableCreation::builder()
2380 .name(table_name.into())
2381 .schema(simple_table_schema())
2382 .build(),
2384 )
2385 .await
2386 .unwrap();
2387 assert_table_eq(&table, &expected_table_ident, &simple_table_schema());
2388 assert_table_metadata_location_matches(&table, &expected_table_metadata_location_regex);
2389
2390 let table = catalog.load_table(&expected_table_ident).await.unwrap();
2391 assert_table_eq(&table, &expected_table_ident, &simple_table_schema());
2392 assert_table_metadata_location_matches(&table, &expected_table_metadata_location_regex);
2393 }
2394
2395 #[tokio::test]
2396 async fn test_create_table_throws_error_if_table_with_same_name_already_exists() {
2397 let warehouse_loc = temp_path();
2398 let catalog = new_sql_catalog(warehouse_loc.clone(), Some("iceberg")).await;
2399 let namespace_ident = NamespaceIdent::new("a".into());
2400 create_namespace(&catalog, &namespace_ident).await;
2401 let table_name = "tbl1";
2402 let table_ident = TableIdent::new(namespace_ident.clone(), table_name.into());
2403 create_table(&catalog, &table_ident).await;
2404
2405 let tmp_dir = TempDir::new().unwrap();
2406 let location = tmp_dir.path().to_str().unwrap().to_string();
2407
2408 assert_eq!(
2409 catalog
2410 .create_table(
2411 &namespace_ident,
2412 TableCreation::builder()
2413 .name(table_name.into())
2414 .schema(simple_table_schema())
2415 .location(location)
2416 .build()
2417 )
2418 .await
2419 .unwrap_err()
2420 .to_string(),
2421 format!(
2422 "TableAlreadyExists => Table {:?} already exists.",
2423 &table_ident
2424 )
2425 );
2426 }
2427
2428 #[tokio::test]
2429 async fn test_rename_table_src_table_is_same_as_dst_table() {
2430 let warehouse_loc = temp_path();
2431 let catalog = new_sql_catalog(warehouse_loc, Some("iceberg")).await;
2432 let namespace_ident = NamespaceIdent::new("n1".into());
2433 create_namespace(&catalog, &namespace_ident).await;
2434 let table_ident = TableIdent::new(namespace_ident.clone(), "tbl".into());
2435 create_table(&catalog, &table_ident).await;
2436
2437 catalog
2438 .rename_table(&table_ident, &table_ident)
2439 .await
2440 .unwrap();
2441
2442 assert_eq!(catalog.list_tables(&namespace_ident).await.unwrap(), vec![
2443 table_ident
2444 ],);
2445 }
2446
2447 #[tokio::test]
2448 async fn test_rename_table_across_nested_namespaces() {
2449 let warehouse_loc = temp_path();
2450 let catalog = new_sql_catalog(warehouse_loc, Some("iceberg")).await;
2451 let namespace_ident_a = NamespaceIdent::new("a".into());
2452 let namespace_ident_a_b = NamespaceIdent::from_strs(vec!["a", "b"]).unwrap();
2453 let namespace_ident_a_b_c = NamespaceIdent::from_strs(vec!["a", "b", "c"]).unwrap();
2454 create_namespaces(&catalog, &vec![
2455 &namespace_ident_a,
2456 &namespace_ident_a_b,
2457 &namespace_ident_a_b_c,
2458 ])
2459 .await;
2460
2461 let src_table_ident = TableIdent::new(namespace_ident_a_b_c.clone(), "tbl1".into());
2462 create_tables(&catalog, vec![&src_table_ident]).await;
2463
2464 let dst_table_ident = TableIdent::new(namespace_ident_a_b.clone(), "tbl1".into());
2465 catalog
2466 .rename_table(&src_table_ident, &dst_table_ident)
2467 .await
2468 .unwrap();
2469
2470 assert!(!catalog.table_exists(&src_table_ident).await.unwrap());
2471
2472 assert!(catalog.table_exists(&dst_table_ident).await.unwrap());
2473 }
2474
2475 #[tokio::test]
2476 async fn test_rename_table_throws_error_if_dst_namespace_doesnt_exist() {
2477 let warehouse_loc = temp_path();
2478 let catalog = new_sql_catalog(warehouse_loc, Some("iceberg")).await;
2479 let src_namespace_ident = NamespaceIdent::new("n1".into());
2480 let src_table_ident = TableIdent::new(src_namespace_ident.clone(), "tbl1".into());
2481 create_namespace(&catalog, &src_namespace_ident).await;
2482 create_table(&catalog, &src_table_ident).await;
2483
2484 let non_existent_dst_namespace_ident = NamespaceIdent::new("n2".into());
2485 let dst_table_ident =
2486 TableIdent::new(non_existent_dst_namespace_ident.clone(), "tbl1".into());
2487 assert_eq!(
2488 catalog
2489 .rename_table(&src_table_ident, &dst_table_ident)
2490 .await
2491 .unwrap_err()
2492 .to_string(),
2493 format!("NamespaceNotFound => No such namespace: {non_existent_dst_namespace_ident:?}"),
2494 );
2495 }
2496
2497 async fn create_v0_sqlite_db() -> (String, TempDir) {
2500 let temp_dir = TempDir::new().unwrap();
2501 let uri = format!(
2502 "sqlite:{}",
2503 temp_dir.path().join("catalog.db").to_str().unwrap()
2504 );
2505 sqlx::Sqlite::create_database(&uri).await.unwrap();
2506 let pool = sqlx::AnyPool::connect(&uri).await.unwrap();
2507 sqlx::query(
2508 "CREATE TABLE iceberg_tables (
2509 catalog_name VARCHAR(255) NOT NULL,
2510 table_namespace VARCHAR(255) NOT NULL,
2511 table_name VARCHAR(255) NOT NULL,
2512 metadata_location VARCHAR(1000),
2513 previous_metadata_location VARCHAR(1000),
2514 PRIMARY KEY (catalog_name, table_namespace, table_name)
2515 )",
2516 )
2517 .execute(&pool)
2518 .await
2519 .unwrap();
2520 sqlx::query(
2521 "INSERT INTO iceberg_tables
2522 (catalog_name, table_namespace, table_name, metadata_location)
2523 VALUES ('iceberg', 'test_namespace', 'existing_test_table', '/tmp/fake-location')",
2524 )
2525 .execute(&pool)
2526 .await
2527 .unwrap();
2528 pool.close().await;
2529 (uri, temp_dir)
2530 }
2531
2532 async fn create_v1_sqlite_db() -> (String, TempDir) {
2535 let temp_dir = TempDir::new().unwrap();
2536 let uri = format!(
2537 "sqlite:{}",
2538 temp_dir.path().join("catalog.db").to_str().unwrap()
2539 );
2540 sqlx::Sqlite::create_database(&uri).await.unwrap();
2541 let pool = sqlx::AnyPool::connect(&uri).await.unwrap();
2542 sqlx::query(
2543 "CREATE TABLE iceberg_tables (
2544 catalog_name VARCHAR(255) NOT NULL,
2545 table_namespace VARCHAR(255) NOT NULL,
2546 table_name VARCHAR(255) NOT NULL,
2547 metadata_location VARCHAR(1000),
2548 previous_metadata_location VARCHAR(1000),
2549 iceberg_type VARCHAR(5),
2550 PRIMARY KEY (catalog_name, table_namespace, table_name)
2551 )",
2552 )
2553 .execute(&pool)
2554 .await
2555 .unwrap();
2556 sqlx::query(
2557 "INSERT INTO iceberg_tables
2558 (catalog_name, table_namespace, table_name, metadata_location, iceberg_type)
2559 VALUES ('iceberg', 'test_namespace', 'existing_test_table', '/tmp/fake-location', 'TABLE')",
2560 )
2561 .execute(&pool)
2562 .await
2563 .unwrap();
2564 pool.close().await;
2565 (uri, temp_dir)
2566 }
2567
2568 fn catalog_props(
2571 uri: &str,
2572 warehouse: &TempDir,
2573 schema_version: Option<SchemaVersion>,
2574 ) -> HashMap<String, String> {
2575 let mut props = HashMap::from_iter([
2576 (SQL_CATALOG_PROP_URI.to_string(), uri.to_string()),
2577 (
2578 SQL_CATALOG_PROP_WAREHOUSE.to_string(),
2579 warehouse.path().to_str().unwrap().to_string(),
2580 ),
2581 (
2582 SQL_CATALOG_PROP_BIND_STYLE.to_string(),
2583 SqlBindStyle::QMark.to_string(),
2584 ),
2585 ]);
2586 if let Some(schema_version) = schema_version {
2587 props.insert(
2588 SQL_CATALOG_PROP_SCHEMA_VERSION.to_string(),
2589 schema_version.to_string(),
2590 );
2591 }
2592 props
2593 }
2594
2595 async fn record_type_column_exists(uri: &str) -> bool {
2599 let probe_pool = sqlx::AnyPool::connect(uri).await.unwrap();
2600 let column_exists = probe_pool
2601 .describe(&format!("SELECT * FROM {CATALOG_TABLE_NAME}"))
2602 .await
2603 .expect("connection and query should succeed")
2604 .columns()
2605 .iter()
2606 .any(|column| {
2607 column
2608 .name()
2609 .eq_ignore_ascii_case(CATALOG_FIELD_RECORD_TYPE)
2610 });
2611 probe_pool.close().await;
2612 column_exists
2613 }
2614
2615 #[tokio::test]
2616 async fn test_detect_schema_version() {
2617 install_default_drivers();
2618
2619 let detected_schema = {
2620 let (uri, _temp_dir) = create_v0_sqlite_db().await;
2621 let pool = sqlx::AnyPool::connect(&uri).await.unwrap();
2622 let detected_schema = SchemaVersion::detect(&pool).await.unwrap();
2623 pool.close().await;
2624 detected_schema
2625 };
2626 assert_eq!(
2627 detected_schema,
2628 SchemaVersion::V0,
2629 "a catalog table without an iceberg_type column should be V0",
2630 );
2631
2632 let detected_schema = {
2633 let (uri, _temp_dir) = create_v1_sqlite_db().await;
2634 let pool = sqlx::AnyPool::connect(&uri).await.unwrap();
2635 let detected_schema = SchemaVersion::detect(&pool).await.unwrap();
2636 pool.close().await;
2637 detected_schema
2638 };
2639 assert_eq!(
2640 detected_schema,
2641 SchemaVersion::V1,
2642 "a catalog table with an iceberg_type column should be V1",
2643 );
2644 }
2645
2646 #[tokio::test]
2647 async fn test_detect_schema_version_surfaces_errors() {
2648 install_default_drivers();
2649
2650 let temp_dir = TempDir::new().unwrap();
2651 let uri = format!(
2652 "sqlite:{}",
2653 temp_dir.path().join("catalog.db").to_str().unwrap()
2654 );
2655 sqlx::Sqlite::create_database(&uri).await.unwrap();
2656 let pool = sqlx::AnyPool::connect(&uri).await.unwrap();
2657
2658 let err = SchemaVersion::detect(&pool)
2660 .await
2661 .expect_err("detection should fail rather than report V0");
2662 pool.close().await;
2663
2664 assert!(
2665 err.to_string().contains("iceberg_tables"),
2666 "error should name the table it failed to introspect, got: {err}"
2667 );
2668 }
2669
2670 #[tokio::test]
2671 async fn test_v0_schema_migration() {
2672 install_default_drivers();
2673
2674 let (uri, temp_dir) = create_v0_sqlite_db().await;
2675
2676 let props = HashMap::from_iter([
2678 (SQL_CATALOG_PROP_URI.to_string(), uri.clone()),
2679 (
2680 SQL_CATALOG_PROP_WAREHOUSE.to_string(),
2681 temp_dir.path().to_str().unwrap().to_string(),
2682 ),
2683 (
2684 SQL_CATALOG_PROP_BIND_STYLE.to_string(),
2685 SqlBindStyle::QMark.to_string(),
2686 ),
2687 (
2688 SQL_CATALOG_PROP_SCHEMA_VERSION.to_string(),
2689 SchemaVersion::V1.to_string(),
2690 ),
2691 ]);
2692 let catalog = SqlCatalogBuilder::default()
2693 .with_storage_factory(Arc::new(LocalFsStorageFactory))
2694 .load("iceberg", props)
2695 .await
2696 .expect("should open V0 catalog and migrate schema when sql.schema-version=V1");
2697
2698 let namespace = NamespaceIdent::from_strs(["test_namespace"]).unwrap();
2700 let tables = catalog.list_tables(&namespace).await.unwrap();
2701 assert_eq!(tables.len(), 1);
2702 assert_eq!(tables[0].name(), "existing_test_table");
2703
2704 assert!(
2705 record_type_column_exists(&uri).await,
2706 "iceberg_type column should exist when sql.schema-version=V1 was set",
2707 );
2708 }
2709
2710 #[tokio::test]
2711 async fn test_v0_schema_no_migration_without_property() {
2712 install_default_drivers();
2713
2714 let (uri, temp_dir) = create_v0_sqlite_db().await;
2715
2716 let props = HashMap::from_iter([
2718 (SQL_CATALOG_PROP_URI.to_string(), uri.clone()),
2719 (
2720 SQL_CATALOG_PROP_WAREHOUSE.to_string(),
2721 temp_dir.path().to_str().unwrap().to_string(),
2722 ),
2723 (
2724 SQL_CATALOG_PROP_BIND_STYLE.to_string(),
2725 SqlBindStyle::QMark.to_string(),
2726 ),
2727 ]);
2728 let catalog = SqlCatalogBuilder::default()
2729 .with_storage_factory(Arc::new(LocalFsStorageFactory))
2730 .load("iceberg", props)
2731 .await
2732 .expect("should open V0 catalog without migrating");
2733
2734 assert_eq!(catalog.schema_version, SchemaVersion::V0);
2735
2736 let namespace = NamespaceIdent::from_strs(["test_namespace"]).unwrap();
2738 let tables = catalog.list_tables(&namespace).await.unwrap();
2739 assert_eq!(tables.len(), 1);
2740 assert_eq!(tables[0].name(), "existing_test_table");
2741
2742 assert!(
2743 !record_type_column_exists(&uri).await,
2744 "iceberg_type column should not exist when sql.schema-version=V1 was not set"
2745 );
2746 }
2747
2748 #[tokio::test]
2749 async fn test_explicit_v0_schema_version_is_honored() {
2750 install_default_drivers();
2751
2752 let (uri, temp_dir) = create_v0_sqlite_db().await;
2753
2754 let catalog = SqlCatalogBuilder::default()
2755 .with_storage_factory(Arc::new(LocalFsStorageFactory))
2756 .load(
2757 "iceberg",
2758 catalog_props(&uri, &temp_dir, Some(SchemaVersion::V0)),
2759 )
2760 .await
2761 .expect("requesting V0 against a V0 catalog table should succeed");
2762
2763 assert_eq!(catalog.schema_version, SchemaVersion::V0);
2764
2765 let namespace = NamespaceIdent::from_strs(["test_namespace"]).unwrap();
2766 let tables = catalog.list_tables(&namespace).await.unwrap();
2767 assert_eq!(tables.len(), 1);
2768 assert_eq!(tables[0].name(), "existing_test_table");
2769
2770 assert!(
2771 !record_type_column_exists(&uri).await,
2772 "iceberg_type column should not be added when V0 was requested"
2773 );
2774 }
2775
2776 #[tokio::test]
2777 async fn test_v0_schema_version_is_ignored_on_v1_table() {
2778 install_default_drivers();
2779
2780 let (uri, temp_dir) = create_v1_sqlite_db().await;
2781
2782 let catalog = SqlCatalogBuilder::default()
2783 .with_storage_factory(Arc::new(LocalFsStorageFactory))
2784 .load(
2785 "iceberg",
2786 catalog_props(&uri, &temp_dir, Some(SchemaVersion::V0)),
2787 )
2788 .await
2789 .expect("requesting V0 against a V1 catalog table should succeed");
2790
2791 assert_eq!(catalog.schema_version, SchemaVersion::V1);
2792
2793 let namespace = NamespaceIdent::from_strs(["test_namespace"]).unwrap();
2794 let tables = catalog.list_tables(&namespace).await.unwrap();
2795 assert_eq!(tables.len(), 1);
2796 assert_eq!(tables[0].name(), "existing_test_table");
2797 }
2798
2799 #[tokio::test]
2800 async fn test_v0_schema_version_on_empty_database_yields_v1() {
2801 install_default_drivers();
2802
2803 let temp_dir = TempDir::new().unwrap();
2804 let uri = format!(
2805 "sqlite:{}",
2806 temp_dir.path().join("catalog.db").to_str().unwrap()
2807 );
2808 sqlx::Sqlite::create_database(&uri).await.unwrap();
2809
2810 let catalog = SqlCatalogBuilder::default()
2811 .with_storage_factory(Arc::new(LocalFsStorageFactory))
2812 .load(
2813 "iceberg",
2814 catalog_props(&uri, &temp_dir, Some(SchemaVersion::V0)),
2815 )
2816 .await
2817 .expect("requesting V0 against an empty database should succeed");
2818
2819 assert_eq!(catalog.schema_version, SchemaVersion::V1);
2820 }
2821
2822 #[tokio::test]
2823 async fn test_create_table_unsupported_on_v0_schema() {
2824 install_default_drivers();
2825
2826 let (uri, temp_dir) = create_v0_sqlite_db().await;
2827
2828 let catalog = SqlCatalogBuilder::default()
2829 .with_storage_factory(Arc::new(LocalFsStorageFactory))
2830 .load("iceberg", catalog_props(&uri, &temp_dir, None))
2831 .await
2832 .expect("should open V0 catalog without migrating");
2833 assert_eq!(catalog.schema_version, SchemaVersion::V0);
2834
2835 let namespace = NamespaceIdent::from_strs(["test_namespace"]).unwrap();
2838 let err = catalog
2839 .create_table(
2840 &namespace,
2841 TableCreation::builder()
2842 .name("created_test_table".to_string())
2843 .schema(simple_table_schema())
2844 .location(temp_path())
2845 .build(),
2846 )
2847 .await
2848 .expect_err("table creation should be rejected on a V0 catalog table");
2849
2850 assert_eq!(err.kind(), ErrorKind::FeatureUnsupported);
2851 assert!(
2852 err.to_string().contains("Table creation is not supported"),
2853 "error should explain that table creation is unsupported, got: {err}"
2854 );
2855
2856 let tables = catalog.list_tables(&namespace).await.unwrap();
2857 assert_eq!(
2858 tables.len(),
2859 1,
2860 "only the pre-existing row should be present, got: {tables:?}"
2861 );
2862 }
2863
2864 #[tokio::test]
2865 async fn test_register_table_unsupported_on_v0_schema() {
2866 install_default_drivers();
2867
2868 let (uri, temp_dir) = create_v0_sqlite_db().await;
2869
2870 let catalog = SqlCatalogBuilder::default()
2871 .with_storage_factory(Arc::new(LocalFsStorageFactory))
2872 .load("iceberg", catalog_props(&uri, &temp_dir, None))
2873 .await
2874 .expect("should open V0 catalog without migrating");
2875 assert_eq!(catalog.schema_version, SchemaVersion::V0);
2876
2877 let namespace = NamespaceIdent::from_strs(["test_namespace"]).unwrap();
2878 let table_ident = TableIdent::new(namespace.clone(), "registered_test_table".to_string());
2879 let err = catalog
2881 .register_table(
2882 &table_ident,
2883 "/tmp/does-not-exist/metadata.json".to_string(),
2884 )
2885 .await
2886 .expect_err("table registration should be rejected on a V0 catalog table");
2887
2888 assert_eq!(err.kind(), ErrorKind::FeatureUnsupported);
2889 assert!(
2890 err.to_string()
2891 .contains("Table registration is not supported"),
2892 "error should explain that table registration is unsupported, got: {err}"
2893 );
2894
2895 let tables = catalog.list_tables(&namespace).await.unwrap();
2896 assert_eq!(
2897 tables.len(),
2898 1,
2899 "only the pre-existing row should be present, got: {tables:?}"
2900 );
2901 }
2902
2903 #[tokio::test]
2904 async fn test_create_table_supported_after_v0_migration() {
2905 install_default_drivers();
2906
2907 let (uri, temp_dir) = create_v0_sqlite_db().await;
2908
2909 let catalog = SqlCatalogBuilder::default()
2910 .with_storage_factory(Arc::new(LocalFsStorageFactory))
2911 .load(
2912 "iceberg",
2913 catalog_props(&uri, &temp_dir, Some(SchemaVersion::V1)),
2914 )
2915 .await
2916 .expect("should open V0 catalog and migrate schema when sql.schema-version=V1");
2917 assert_eq!(catalog.schema_version, SchemaVersion::V1);
2918
2919 let namespace = NamespaceIdent::from_strs(["test_namespace"]).unwrap();
2920 catalog
2921 .create_table(
2922 &namespace,
2923 TableCreation::builder()
2924 .name("created_test_table".to_string())
2925 .schema(simple_table_schema())
2926 .location(temp_path())
2927 .build(),
2928 )
2929 .await
2930 .expect("table creation should succeed once the schema is migrated to V1");
2931
2932 let table_names = catalog
2933 .list_tables(&namespace)
2934 .await
2935 .unwrap()
2936 .iter()
2937 .map(|table_ident| table_ident.name().to_string())
2938 .sorted()
2939 .collect_vec();
2940 assert_eq!(
2941 table_names,
2942 vec!["created_test_table", "existing_test_table"],
2943 "the migrated pre-existing row and the new table should both be listed"
2944 );
2945 }
2946
2947 #[tokio::test]
2948 async fn test_invalid_schema_version_is_rejected() {
2949 install_default_drivers();
2950
2951 let (uri, temp_dir) = create_v0_sqlite_db().await;
2952
2953 let props = HashMap::from_iter([
2956 (SQL_CATALOG_PROP_URI.to_string(), uri),
2957 (
2958 SQL_CATALOG_PROP_WAREHOUSE.to_string(),
2959 temp_dir.path().to_str().unwrap().to_string(),
2960 ),
2961 (
2962 SQL_CATALOG_PROP_BIND_STYLE.to_string(),
2963 SqlBindStyle::QMark.to_string(),
2964 ),
2965 (
2966 SQL_CATALOG_PROP_SCHEMA_VERSION.to_string(),
2967 "v2".to_string(),
2968 ),
2969 ]);
2970 let result = SqlCatalogBuilder::default()
2971 .with_storage_factory(Arc::new(LocalFsStorageFactory))
2972 .load("iceberg", props)
2973 .await;
2974
2975 let err = result.expect_err("an invalid sql.schema-version should be rejected");
2976 assert_eq!(err.kind(), ErrorKind::DataInvalid);
2977 }
2978
2979 #[tokio::test]
2980 async fn test_legacy_bind_style_key_is_accepted() {
2981 install_default_drivers();
2982
2983 let (uri, temp_dir) = create_v0_sqlite_db().await;
2984
2985 let props = HashMap::from_iter([
2987 (SQL_CATALOG_PROP_URI.to_string(), uri),
2988 (
2989 SQL_CATALOG_PROP_WAREHOUSE.to_string(),
2990 temp_dir.path().to_str().unwrap().to_string(),
2991 ),
2992 (
2993 SQL_CATALOG_PROP_BIND_STYLE_LEGACY.to_string(),
2994 SqlBindStyle::QMark.to_string(),
2995 ),
2996 ]);
2997 let catalog = SqlCatalogBuilder::default()
2998 .with_storage_factory(Arc::new(LocalFsStorageFactory))
2999 .load("iceberg", props)
3000 .await
3001 .expect("legacy sql_bind_style key should still be accepted");
3002
3003 assert_eq!(catalog.properties.sql_bind_style, SqlBindStyle::QMark);
3004 }
3005}