1use std::sync::Arc;
21
22use crate::arrow::ArrowReaderBuilder;
23use crate::encryption::EncryptionManager;
24use crate::encryption::kms::KeyManagementClient;
25use crate::error::invalid_data;
26use crate::inspect::MetadataTable;
27use crate::io::FileIO;
28use crate::io::object_cache::ObjectCache;
29use crate::runtime::Runtime;
30use crate::scan::TableScanBuilder;
31use crate::spec::{
32 ManifestListReader, ManifestReader, SchemaRef, SnapshotRef, TableMetadata, TableMetadataRef,
33};
34use crate::{Result, TableIdent};
35
36pub struct TableBuilder {
38 file_io: Option<FileIO>,
39 metadata_location: Option<String>,
40 metadata: Option<TableMetadataRef>,
41 identifier: Option<TableIdent>,
42 kms_client: Option<Arc<dyn KeyManagementClient>>,
43 readonly: bool,
44 disable_cache: bool,
45 cache_size_bytes: Option<u64>,
46 runtime: Option<Runtime>,
47}
48
49impl TableBuilder {
50 pub(crate) fn new() -> Self {
51 Self {
52 file_io: None,
53 metadata_location: None,
54 metadata: None,
55 identifier: None,
56 kms_client: None,
57 readonly: false,
58 disable_cache: false,
59 cache_size_bytes: None,
60 runtime: None,
61 }
62 }
63
64 pub fn file_io(mut self, file_io: FileIO) -> Self {
66 self.file_io = Some(file_io);
67 self
68 }
69
70 pub fn metadata_location<T: Into<String>>(mut self, metadata_location: T) -> Self {
72 self.metadata_location = Some(metadata_location.into());
73 self
74 }
75
76 pub fn metadata<T: Into<TableMetadataRef>>(mut self, metadata: T) -> Self {
78 self.metadata = Some(metadata.into());
79 self
80 }
81
82 pub fn identifier(mut self, identifier: TableIdent) -> Self {
84 self.identifier = Some(identifier);
85 self
86 }
87
88 pub fn readonly(mut self, readonly: bool) -> Self {
90 self.readonly = readonly;
91 self
92 }
93
94 pub fn disable_cache(mut self) -> Self {
98 self.disable_cache = true;
99 self
100 }
101
102 pub fn cache_size_bytes(mut self, cache_size_bytes: u64) -> Self {
104 self.cache_size_bytes = Some(cache_size_bytes);
105 self
106 }
107
108 pub fn runtime(mut self, runtime: Runtime) -> Self {
110 self.runtime = Some(runtime);
111 self
112 }
113
114 pub fn kms_client(mut self, kms_client: Arc<dyn KeyManagementClient>) -> Self {
120 self.kms_client = Some(kms_client);
121 self
122 }
123
124 pub fn build(self) -> Result<Table> {
126 let Self {
127 file_io,
128 metadata_location,
129 metadata,
130 identifier,
131 kms_client,
132 readonly,
133 disable_cache,
134 cache_size_bytes,
135 runtime,
136 } = self;
137
138 let Some(file_io) = file_io else {
139 return Err(invalid_data!(
140 "FileIO must be provided with TableBuilder.file_io()"
141 ));
142 };
143
144 let Some(metadata) = metadata else {
145 return Err(invalid_data!(
146 "TableMetadataRef must be provided with TableBuilder.metadata()"
147 ));
148 };
149
150 let Some(identifier) = identifier else {
151 return Err(invalid_data!(
152 "TableIdent must be provided with TableBuilder.identifier()"
153 ));
154 };
155
156 let Some(runtime) = runtime else {
157 return Err(invalid_data!(
158 "Runtime must be provided with TableBuilder.runtime()"
159 ));
160 };
161
162 let encryption_manager =
163 EncryptionManager::from_table_metadata(kms_client.as_ref(), &metadata)?;
164
165 let object_cache = if disable_cache {
166 Arc::new(ObjectCache::with_disabled_cache(
167 file_io.clone(),
168 encryption_manager.clone(),
169 runtime.clone(),
170 ))
171 } else if let Some(cache_size_bytes) = cache_size_bytes {
172 Arc::new(ObjectCache::new_with_capacity(
173 file_io.clone(),
174 cache_size_bytes,
175 encryption_manager.clone(),
176 runtime.clone(),
177 ))
178 } else {
179 Arc::new(ObjectCache::new(
180 file_io.clone(),
181 encryption_manager.clone(),
182 runtime.clone(),
183 ))
184 };
185
186 Ok(Table {
187 file_io,
188 metadata_location,
189 metadata,
190 identifier,
191 readonly,
192 object_cache,
193 runtime,
194 encryption_manager,
195 })
196 }
197}
198
199#[derive(Debug, Clone)]
201pub struct Table {
202 file_io: FileIO,
203 metadata_location: Option<String>,
204 metadata: TableMetadataRef,
205 identifier: TableIdent,
206 readonly: bool,
207 object_cache: Arc<ObjectCache>,
208 runtime: Runtime,
209 encryption_manager: Option<Arc<EncryptionManager>>,
210}
211
212impl Table {
213 pub(crate) fn with_metadata(mut self, metadata: TableMetadataRef) -> Self {
215 self.metadata = metadata;
216 self
217 }
218
219 pub(crate) fn with_metadata_location(mut self, metadata_location: String) -> Self {
221 self.metadata_location = Some(metadata_location);
222 self
223 }
224
225 pub fn with_runtime(mut self, runtime: Runtime) -> Self {
230 self.object_cache = Arc::new(self.object_cache.with_runtime(runtime.clone()));
231 self.runtime = runtime;
232 self
233 }
234
235 pub fn builder() -> TableBuilder {
237 TableBuilder::new()
238 }
239
240 pub fn identifier(&self) -> &TableIdent {
242 &self.identifier
243 }
244 pub fn metadata(&self) -> &TableMetadata {
246 &self.metadata
247 }
248
249 pub fn metadata_ref(&self) -> TableMetadataRef {
251 self.metadata.clone()
252 }
253
254 pub fn metadata_location(&self) -> Option<&str> {
256 self.metadata_location.as_deref()
257 }
258
259 pub fn metadata_location_result(&self) -> Result<&str> {
261 self.metadata_location.as_deref().ok_or(invalid_data!(
262 "Metadata location does not exist for table: {}",
263 self.identifier
264 ))
265 }
266
267 pub fn file_io(&self) -> &FileIO {
269 &self.file_io
270 }
271
272 pub(crate) fn object_cache(&self) -> Arc<ObjectCache> {
274 self.object_cache.clone()
275 }
276
277 pub fn encryption_manager(&self) -> Option<&Arc<EncryptionManager>> {
284 self.encryption_manager.as_ref()
285 }
286
287 pub fn scan(&self) -> TableScanBuilder<'_> {
289 TableScanBuilder::new(self)
290 }
291
292 pub fn inspect(&self) -> MetadataTable<'_> {
295 MetadataTable::new(self)
296 }
297
298 pub(crate) fn runtime(&self) -> &Runtime {
300 &self.runtime
301 }
302
303 pub fn readonly(&self) -> bool {
305 self.readonly
306 }
307
308 pub fn current_schema_ref(&self) -> SchemaRef {
310 self.metadata.current_schema().clone()
311 }
312
313 pub fn manifest_list_reader(&self, snapshot: &SnapshotRef) -> ManifestListReader {
315 ManifestListReader::new(
316 snapshot.clone(),
317 self.file_io.clone(),
318 self.metadata.clone(),
319 self.encryption_manager.clone(),
320 )
321 }
322
323 pub fn manifest_reader(&self) -> ManifestReader {
330 ManifestReader::new(self.file_io.clone(), self.runtime.clone())
331 }
332
333 pub fn reader_builder(&self) -> ArrowReaderBuilder {
335 ArrowReaderBuilder::new(self.file_io.clone(), self.runtime().clone())
336 }
337}
338
339#[derive(Debug, Clone)]
363pub struct StaticTable(Table);
364
365impl StaticTable {
366 pub async fn from_metadata(
368 metadata: TableMetadata,
369 table_ident: TableIdent,
370 file_io: FileIO,
371 ) -> Result<Self> {
372 let table = Table::builder()
373 .metadata(metadata)
374 .identifier(table_ident)
375 .file_io(file_io.clone())
376 .runtime(Runtime::try_current()?)
377 .readonly(true)
378 .build();
379
380 Ok(Self(table?))
381 }
382 pub async fn from_metadata_file(
384 metadata_location: &str,
385 table_ident: TableIdent,
386 file_io: FileIO,
387 ) -> Result<Self> {
388 let metadata = TableMetadata::read_from(&file_io, metadata_location).await?;
389
390 let table = Table::builder()
391 .metadata(metadata)
392 .metadata_location(metadata_location)
393 .identifier(table_ident)
394 .file_io(file_io.clone())
395 .runtime(Runtime::try_current()?)
396 .readonly(true)
397 .build();
398
399 Ok(Self(table?))
400 }
401
402 pub fn scan(&self) -> TableScanBuilder<'_> {
404 self.0.scan()
405 }
406
407 pub fn metadata(&self) -> TableMetadataRef {
409 self.0.metadata_ref()
410 }
411
412 pub fn into_table(self) -> Table {
416 self.0
417 }
418
419 pub fn reader_builder(&self) -> ArrowReaderBuilder {
421 self.0.reader_builder()
422 }
423}
424
425#[cfg(test)]
426mod tests {
427 use std::fs;
428
429 use super::*;
430 use crate::ErrorKind;
431 use crate::encryption::SensitiveBytes;
432 use crate::encryption::kms::MemoryKeyManagementClient;
433 use crate::spec::TableProperties;
434 use crate::test_utils::scan::TableTestFixture;
435
436 fn load_test_metadata(filename: &str) -> TableMetadata {
437 let path = format!(
438 "{}/testdata/table_metadata/{}",
439 env!("CARGO_MANIFEST_DIR"),
440 filename
441 );
442 let json = fs::read_to_string(path).unwrap();
443 serde_json::from_str(&json).unwrap()
444 }
445
446 #[tokio::test]
447 async fn test_static_table_from_file() {
448 let metadata_file_name = "TableMetadataV2Valid.json";
449 let metadata_file_path = format!(
450 "{}/testdata/table_metadata/{}",
451 env!("CARGO_MANIFEST_DIR"),
452 metadata_file_name
453 );
454 let file_io = FileIO::new_with_fs();
455 let static_identifier = TableIdent::from_strs(["static_ns", "static_table"]).unwrap();
456 let static_table =
457 StaticTable::from_metadata_file(&metadata_file_path, static_identifier, file_io)
458 .await
459 .unwrap();
460 let snapshot_id = static_table
461 .metadata()
462 .current_snapshot()
463 .unwrap()
464 .snapshot_id();
465 assert_eq!(
466 snapshot_id, 3055729675574597004,
467 "snapshot id from metadata don't match"
468 );
469 }
470
471 #[tokio::test]
472 async fn test_static_into_table() {
473 let metadata_file_name = "TableMetadataV2Valid.json";
474 let metadata_file_path = format!(
475 "{}/testdata/table_metadata/{}",
476 env!("CARGO_MANIFEST_DIR"),
477 metadata_file_name
478 );
479 let file_io = FileIO::new_with_fs();
480 let static_identifier = TableIdent::from_strs(["static_ns", "static_table"]).unwrap();
481 let static_table =
482 StaticTable::from_metadata_file(&metadata_file_path, static_identifier, file_io)
483 .await
484 .unwrap();
485 let table = static_table.into_table();
486 assert!(table.readonly());
487 assert_eq!(table.identifier.name(), "static_table");
488 assert_eq!(
489 table.metadata_location(),
490 Some(metadata_file_path).as_deref()
491 );
492 }
493
494 #[tokio::test]
495 async fn test_table_readonly_flag() {
496 let metadata_file_name = "TableMetadataV2Valid.json";
497 let metadata_file_path = format!(
498 "{}/testdata/table_metadata/{}",
499 env!("CARGO_MANIFEST_DIR"),
500 metadata_file_name
501 );
502 let file_io = FileIO::new_with_fs();
503 let metadata_file = file_io.new_input(metadata_file_path).unwrap();
504 let metadata_file_content = metadata_file.read().await.unwrap();
505 let table_metadata =
506 serde_json::from_slice::<TableMetadata>(&metadata_file_content).unwrap();
507 let static_identifier = TableIdent::from_strs(["ns", "table"]).unwrap();
508 let table = Table::builder()
509 .metadata(table_metadata)
510 .identifier(static_identifier)
511 .file_io(file_io.clone())
512 .runtime(Runtime::try_current().unwrap())
513 .build()
514 .unwrap();
515 assert!(!table.readonly());
516 assert_eq!(table.identifier.name(), "table");
517 }
518
519 #[tokio::test]
523 async fn test_table_with_runtime_moves_work_and_keeps_object_cache() {
524 let old = tokio::runtime::Builder::new_multi_thread().build().unwrap();
525 let mut fixture = TableTestFixture::new();
526 fixture.setup_manifest_files().await;
527 let table = Table::builder()
530 .metadata(fixture.table.metadata().clone())
531 .identifier(fixture.table.identifier().clone())
532 .file_io(fixture.table.file_io().clone())
533 .runtime(Runtime::new(&old))
534 .build()
535 .unwrap();
536
537 let snapshot = table.metadata().current_snapshot().unwrap().clone();
539 let manifest_list = table
540 .object_cache()
541 .get_manifest_list(&snapshot, &table.metadata_ref())
542 .await
543 .unwrap();
544 old.shutdown_background();
547
548 let rebound = table.clone().with_runtime(Runtime::try_current().unwrap());
549 assert_eq!(
550 rebound.runtime().io().spawn(async { 42 }).await.unwrap(),
551 42
552 );
553
554 let manifest_file = manifest_list.entries()[0].clone();
556 rebound
557 .object_cache()
558 .get_manifest(&manifest_file)
559 .await
560 .unwrap();
561
562 fs::remove_file(snapshot.manifest_list()).unwrap();
565 let from_cache = rebound
566 .object_cache()
567 .get_manifest_list(&snapshot, &rebound.metadata_ref())
568 .await
569 .unwrap();
570 assert!(Arc::ptr_eq(&manifest_list, &from_cache));
571 }
572
573 fn make_kms() -> Arc<dyn KeyManagementClient> {
574 let kms = MemoryKeyManagementClient::new();
575 kms.add_master_key("master-1").unwrap();
576 Arc::new(kms)
577 }
578
579 #[tokio::test]
580 async fn table_decrypts_manifest_list_via_object_cache() {
581 let mut metadata: TableMetadata = load_test_metadata("TableMetadataV3ValidEncryption.json");
584
585 let manifest_list_path = format!(
587 "{}/testdata/manifests_lists/manifest-list-v3-encrypted.avro",
588 env!("CARGO_MANIFEST_DIR"),
589 );
590 let snapshot = metadata.snapshots.get_mut(&1).unwrap();
591 let mut patched = snapshot.as_ref().clone();
592 patched.manifest_list = manifest_list_path;
593 *snapshot = Arc::new(patched);
594
595 let kms: Arc<dyn KeyManagementClient> = {
597 let k = MemoryKeyManagementClient::new();
598 k.add_master_key_bytes(
599 "master-1",
600 SensitiveBytes::new([
601 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x09, 0x0a, 0x0b, 0x0c,
602 0x0d, 0x0e, 0x0f,
603 ]),
604 )
605 .unwrap();
606 Arc::new(k)
607 };
608
609 let table = Table::builder()
610 .file_io(FileIO::new_with_fs())
611 .metadata(metadata)
612 .identifier(TableIdent::from_strs(["ns", "enc"]).unwrap())
613 .kms_client(kms)
614 .runtime(Runtime::try_current().unwrap())
615 .build()
616 .unwrap();
617
618 let snapshot_ref = table.metadata().current_snapshot().unwrap();
619 let manifest_list = table
620 .object_cache()
621 .get_manifest_list(snapshot_ref, &table.metadata_ref())
622 .await
623 .unwrap();
624 assert_eq!(manifest_list.entries().len(), 0);
625 }
626
627 #[tokio::test]
628 async fn table_builder_errors_when_encryption_key_id_set_but_no_kms() {
629 let metadata: TableMetadata = load_test_metadata("TableMetadataV3ValidEncryption.json");
630
631 let err = Table::builder()
632 .file_io(FileIO::new_with_memory())
633 .metadata(metadata)
634 .identifier(TableIdent::from_strs(["ns", "enc"]).unwrap())
635 .runtime(Runtime::try_current().unwrap())
636 .build()
637 .unwrap_err();
638 assert_eq!(err.kind(), ErrorKind::PreconditionFailed);
639 }
640
641 #[tokio::test]
642 async fn table_builder_skips_encryption_on_pre_v3_table() {
643 let mut metadata: TableMetadata = load_test_metadata("TableMetadataV2ValidMinimal.json");
646 metadata.properties.insert(
647 TableProperties::PROPERTY_ENCRYPTION_KEY_ID.to_string(),
648 "master-1".to_string(),
649 );
650
651 let table = Table::builder()
652 .file_io(FileIO::new_with_memory())
653 .metadata(metadata)
654 .identifier(TableIdent::from_strs(["ns", "enc"]).unwrap())
655 .kms_client(make_kms())
656 .runtime(Runtime::try_current().unwrap())
657 .build()
658 .unwrap();
659 assert!(table.encryption_manager().is_none());
660 }
661
662 #[tokio::test]
663 async fn table_builder_skips_encryption_when_property_absent() {
664 let metadata: TableMetadata = load_test_metadata("TableMetadataV3ValidMinimal.json");
665 let table = Table::builder()
666 .file_io(FileIO::new_with_memory())
667 .metadata(metadata)
668 .identifier(TableIdent::from_strs(["ns", "plain"]).unwrap())
669 .kms_client(make_kms())
670 .runtime(Runtime::try_current().unwrap())
671 .build()
672 .unwrap();
673 assert!(table.encryption_manager().is_none());
674 }
675}