Skip to main content

iceberg/
table.rs

1// Licensed to the Apache Software Foundation (ASF) under one
2// or more contributor license agreements.  See the NOTICE file
3// distributed with this work for additional information
4// regarding copyright ownership.  The ASF licenses this file
5// to you under the Apache License, Version 2.0 (the
6// "License"); you may not use this file except in compliance
7// with the License.  You may obtain a copy of the License at
8//
9//   http://www.apache.org/licenses/LICENSE-2.0
10//
11// Unless required by applicable law or agreed to in writing,
12// software distributed under the License is distributed on an
13// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14// KIND, either express or implied.  See the License for the
15// specific language governing permissions and limitations
16// under the License.
17
18//! Table API for Apache Iceberg
19
20use std::sync::Arc;
21
22use crate::arrow::ArrowReaderBuilder;
23use crate::encryption::EncryptionManager;
24use crate::encryption::kms::KeyManagementClient;
25use crate::error::invalid_data;
26use crate::inspect::MetadataTable;
27use crate::io::FileIO;
28use crate::io::object_cache::ObjectCache;
29use crate::runtime::Runtime;
30use crate::scan::TableScanBuilder;
31use crate::spec::{
32    ManifestListReader, ManifestReader, SchemaRef, SnapshotRef, TableMetadata, TableMetadataRef,
33};
34use crate::{Result, TableIdent};
35
36/// Builder to create table scan.
37pub struct TableBuilder {
38    file_io: Option<FileIO>,
39    metadata_location: Option<String>,
40    metadata: Option<TableMetadataRef>,
41    identifier: Option<TableIdent>,
42    kms_client: Option<Arc<dyn KeyManagementClient>>,
43    readonly: bool,
44    disable_cache: bool,
45    cache_size_bytes: Option<u64>,
46    runtime: Option<Runtime>,
47}
48
49impl TableBuilder {
50    pub(crate) fn new() -> Self {
51        Self {
52            file_io: None,
53            metadata_location: None,
54            metadata: None,
55            identifier: None,
56            kms_client: None,
57            readonly: false,
58            disable_cache: false,
59            cache_size_bytes: None,
60            runtime: None,
61        }
62    }
63
64    /// required - sets the necessary FileIO to use for the table
65    pub fn file_io(mut self, file_io: FileIO) -> Self {
66        self.file_io = Some(file_io);
67        self
68    }
69
70    /// optional - sets the tables metadata location
71    pub fn metadata_location<T: Into<String>>(mut self, metadata_location: T) -> Self {
72        self.metadata_location = Some(metadata_location.into());
73        self
74    }
75
76    /// required - passes in the TableMetadata to use for the Table
77    pub fn metadata<T: Into<TableMetadataRef>>(mut self, metadata: T) -> Self {
78        self.metadata = Some(metadata.into());
79        self
80    }
81
82    /// required - passes in the TableIdent to use for the Table
83    pub fn identifier(mut self, identifier: TableIdent) -> Self {
84        self.identifier = Some(identifier);
85        self
86    }
87
88    /// specifies if the Table is readonly or not (default not)
89    pub fn readonly(mut self, readonly: bool) -> Self {
90        self.readonly = readonly;
91        self
92    }
93
94    /// specifies if the Table's metadata cache will be disabled,
95    /// so that reads of Manifests and ManifestLists will never
96    /// get cached.
97    pub fn disable_cache(mut self) -> Self {
98        self.disable_cache = true;
99        self
100    }
101
102    /// optionally set a non-default metadata cache size
103    pub fn cache_size_bytes(mut self, cache_size_bytes: u64) -> Self {
104        self.cache_size_bytes = Some(cache_size_bytes);
105        self
106    }
107
108    /// Set the Runtime for this table to use when spawning tasks.
109    pub fn runtime(mut self, runtime: Runtime) -> Self {
110        self.runtime = Some(runtime);
111        self
112    }
113
114    /// optional - sets the KMS client used to unwrap keys for table encryption.
115    ///
116    /// If the table metadata has the `encryption.key-id` property set, a
117    /// [`KeyManagementClient`] must be provided here so the table can build
118    /// an [`EncryptionManager`]; otherwise [`Self::build`] will return an error.
119    pub fn kms_client(mut self, kms_client: Arc<dyn KeyManagementClient>) -> Self {
120        self.kms_client = Some(kms_client);
121        self
122    }
123
124    /// build the Table
125    pub fn build(self) -> Result<Table> {
126        let Self {
127            file_io,
128            metadata_location,
129            metadata,
130            identifier,
131            kms_client,
132            readonly,
133            disable_cache,
134            cache_size_bytes,
135            runtime,
136        } = self;
137
138        let Some(file_io) = file_io else {
139            return Err(invalid_data!(
140                "FileIO must be provided with TableBuilder.file_io()"
141            ));
142        };
143
144        let Some(metadata) = metadata else {
145            return Err(invalid_data!(
146                "TableMetadataRef must be provided with TableBuilder.metadata()"
147            ));
148        };
149
150        let Some(identifier) = identifier else {
151            return Err(invalid_data!(
152                "TableIdent must be provided with TableBuilder.identifier()"
153            ));
154        };
155
156        let Some(runtime) = runtime else {
157            return Err(invalid_data!(
158                "Runtime must be provided with TableBuilder.runtime()"
159            ));
160        };
161
162        let encryption_manager =
163            EncryptionManager::from_table_metadata(kms_client.as_ref(), &metadata)?;
164
165        let object_cache = if disable_cache {
166            Arc::new(ObjectCache::with_disabled_cache(
167                file_io.clone(),
168                encryption_manager.clone(),
169                runtime.clone(),
170            ))
171        } else if let Some(cache_size_bytes) = cache_size_bytes {
172            Arc::new(ObjectCache::new_with_capacity(
173                file_io.clone(),
174                cache_size_bytes,
175                encryption_manager.clone(),
176                runtime.clone(),
177            ))
178        } else {
179            Arc::new(ObjectCache::new(
180                file_io.clone(),
181                encryption_manager.clone(),
182                runtime.clone(),
183            ))
184        };
185
186        Ok(Table {
187            file_io,
188            metadata_location,
189            metadata,
190            identifier,
191            readonly,
192            object_cache,
193            runtime,
194            encryption_manager,
195        })
196    }
197}
198
199/// Table represents a table in the catalog.
200#[derive(Debug, Clone)]
201pub struct Table {
202    file_io: FileIO,
203    metadata_location: Option<String>,
204    metadata: TableMetadataRef,
205    identifier: TableIdent,
206    readonly: bool,
207    object_cache: Arc<ObjectCache>,
208    runtime: Runtime,
209    encryption_manager: Option<Arc<EncryptionManager>>,
210}
211
212impl Table {
213    /// Sets the [`Table`] metadata and returns an updated instance with the new metadata applied.
214    pub(crate) fn with_metadata(mut self, metadata: TableMetadataRef) -> Self {
215        self.metadata = metadata;
216        self
217    }
218
219    /// Sets the [`Table`] metadata location and returns an updated instance.
220    pub(crate) fn with_metadata_location(mut self, metadata_location: String) -> Self {
221        self.metadata_location = Some(metadata_location);
222        self
223    }
224
225    /// Returns this table bound to `runtime`, sharing its manifest cache.
226    ///
227    /// Tasks such as scan planning and reads are spawned on `runtime`. A table rebuilt
228    /// through [`Table::builder`] starts with an empty cache.
229    pub fn with_runtime(mut self, runtime: Runtime) -> Self {
230        self.object_cache = Arc::new(self.object_cache.with_runtime(runtime.clone()));
231        self.runtime = runtime;
232        self
233    }
234
235    /// Returns a TableBuilder to build a table
236    pub fn builder() -> TableBuilder {
237        TableBuilder::new()
238    }
239
240    /// Returns table identifier.
241    pub fn identifier(&self) -> &TableIdent {
242        &self.identifier
243    }
244    /// Returns current metadata.
245    pub fn metadata(&self) -> &TableMetadata {
246        &self.metadata
247    }
248
249    /// Returns current metadata ref.
250    pub fn metadata_ref(&self) -> TableMetadataRef {
251        self.metadata.clone()
252    }
253
254    /// Returns current metadata location.
255    pub fn metadata_location(&self) -> Option<&str> {
256        self.metadata_location.as_deref()
257    }
258
259    /// Returns current metadata location in a result.
260    pub fn metadata_location_result(&self) -> Result<&str> {
261        self.metadata_location.as_deref().ok_or(invalid_data!(
262            "Metadata location does not exist for table: {}",
263            self.identifier
264        ))
265    }
266
267    /// Returns file io used in this table.
268    pub fn file_io(&self) -> &FileIO {
269        &self.file_io
270    }
271
272    /// Returns this table's object cache
273    pub(crate) fn object_cache(&self) -> Arc<ObjectCache> {
274        self.object_cache.clone()
275    }
276
277    /// Returns the [`EncryptionManager`] for this table, if encryption is
278    /// configured.
279    ///
280    /// A manager is present iff the table metadata has the
281    /// `encryption.key-id` property set and a [`KeyManagementClient`] was
282    /// supplied to the [`TableBuilder`].
283    pub fn encryption_manager(&self) -> Option<&Arc<EncryptionManager>> {
284        self.encryption_manager.as_ref()
285    }
286
287    /// Creates a table scan.
288    pub fn scan(&self) -> TableScanBuilder<'_> {
289        TableScanBuilder::new(self)
290    }
291
292    /// Creates a metadata table which provides table-like APIs for inspecting metadata.
293    /// See [`MetadataTable`] for more details.
294    pub fn inspect(&self) -> MetadataTable<'_> {
295        MetadataTable::new(self)
296    }
297
298    /// Returns the [`Runtime`] for this table.
299    pub(crate) fn runtime(&self) -> &Runtime {
300        &self.runtime
301    }
302
303    /// Returns the flag indicating whether the `Table` is readonly or not
304    pub fn readonly(&self) -> bool {
305        self.readonly
306    }
307
308    /// Returns the current schema as a shared reference.
309    pub fn current_schema_ref(&self) -> SchemaRef {
310        self.metadata.current_schema().clone()
311    }
312
313    /// Creates a [`ManifestListReader`] for the given snapshot.
314    pub fn manifest_list_reader(&self, snapshot: &SnapshotRef) -> ManifestListReader {
315        ManifestListReader::new(
316            snapshot.clone(),
317            self.file_io.clone(),
318            self.metadata.clone(),
319            self.encryption_manager.clone(),
320        )
321    }
322
323    /// Creates a [`ManifestReader`] for loading manifests referenced by this table.
324    ///
325    /// The reader parses manifests on this table's [`Runtime::cpu`] handle, so
326    /// [`ManifestReader::read`] returns an
327    /// [`ErrorKind::Unexpected`](crate::ErrorKind::Unexpected) error once that
328    /// runtime has shut down.
329    pub fn manifest_reader(&self) -> ManifestReader {
330        ManifestReader::new(self.file_io.clone(), self.runtime.clone())
331    }
332
333    /// Create a reader for the table.
334    pub fn reader_builder(&self) -> ArrowReaderBuilder {
335        ArrowReaderBuilder::new(self.file_io.clone(), self.runtime().clone())
336    }
337}
338
339/// `StaticTable` is a read-only table struct that can be created from a metadata file or from `TableMetaData` without a catalog.
340/// It can only be used to read metadata and for table scan.
341/// # Examples
342///
343/// ```rust, no_run
344/// # use iceberg::io::FileIO;
345/// # use iceberg::table::StaticTable;
346/// # use iceberg::TableIdent;
347/// # async fn example() {
348/// let metadata_file_location = "s3://bucket_name/path/to/metadata.json";
349/// let file_io = FileIO::new_with_fs();
350/// let static_identifier = TableIdent::from_strs(["static_ns", "static_table"]).unwrap();
351/// let static_table =
352///     StaticTable::from_metadata_file(&metadata_file_location, static_identifier, file_io)
353///         .await
354///         .unwrap();
355/// let snapshot_id = static_table
356///     .metadata()
357///     .current_snapshot()
358///     .unwrap()
359///     .snapshot_id();
360/// # }
361/// ```
362#[derive(Debug, Clone)]
363pub struct StaticTable(Table);
364
365impl StaticTable {
366    /// Creates a static table from a given `TableMetadata` and `FileIO`
367    pub async fn from_metadata(
368        metadata: TableMetadata,
369        table_ident: TableIdent,
370        file_io: FileIO,
371    ) -> Result<Self> {
372        let table = Table::builder()
373            .metadata(metadata)
374            .identifier(table_ident)
375            .file_io(file_io.clone())
376            .runtime(Runtime::try_current()?)
377            .readonly(true)
378            .build();
379
380        Ok(Self(table?))
381    }
382    /// Creates a static table directly from metadata file and `FileIO`
383    pub async fn from_metadata_file(
384        metadata_location: &str,
385        table_ident: TableIdent,
386        file_io: FileIO,
387    ) -> Result<Self> {
388        let metadata = TableMetadata::read_from(&file_io, metadata_location).await?;
389
390        let table = Table::builder()
391            .metadata(metadata)
392            .metadata_location(metadata_location)
393            .identifier(table_ident)
394            .file_io(file_io.clone())
395            .runtime(Runtime::try_current()?)
396            .readonly(true)
397            .build();
398
399        Ok(Self(table?))
400    }
401
402    /// Create a TableScanBuilder for the static table.
403    pub fn scan(&self) -> TableScanBuilder<'_> {
404        self.0.scan()
405    }
406
407    /// Get TableMetadataRef for the static table
408    pub fn metadata(&self) -> TableMetadataRef {
409        self.0.metadata_ref()
410    }
411
412    /// Consumes the `StaticTable` and return it as a `Table`
413    /// Please use this method carefully as the Table it returns remains detached from a catalog
414    /// and can't be used to perform modifications on the table.
415    pub fn into_table(self) -> Table {
416        self.0
417    }
418
419    /// Create a reader for the table.
420    pub fn reader_builder(&self) -> ArrowReaderBuilder {
421        self.0.reader_builder()
422    }
423}
424
425#[cfg(test)]
426mod tests {
427    use std::fs;
428
429    use super::*;
430    use crate::ErrorKind;
431    use crate::encryption::SensitiveBytes;
432    use crate::encryption::kms::MemoryKeyManagementClient;
433    use crate::spec::TableProperties;
434    use crate::test_utils::scan::TableTestFixture;
435
436    fn load_test_metadata(filename: &str) -> TableMetadata {
437        let path = format!(
438            "{}/testdata/table_metadata/{}",
439            env!("CARGO_MANIFEST_DIR"),
440            filename
441        );
442        let json = fs::read_to_string(path).unwrap();
443        serde_json::from_str(&json).unwrap()
444    }
445
446    #[tokio::test]
447    async fn test_static_table_from_file() {
448        let metadata_file_name = "TableMetadataV2Valid.json";
449        let metadata_file_path = format!(
450            "{}/testdata/table_metadata/{}",
451            env!("CARGO_MANIFEST_DIR"),
452            metadata_file_name
453        );
454        let file_io = FileIO::new_with_fs();
455        let static_identifier = TableIdent::from_strs(["static_ns", "static_table"]).unwrap();
456        let static_table =
457            StaticTable::from_metadata_file(&metadata_file_path, static_identifier, file_io)
458                .await
459                .unwrap();
460        let snapshot_id = static_table
461            .metadata()
462            .current_snapshot()
463            .unwrap()
464            .snapshot_id();
465        assert_eq!(
466            snapshot_id, 3055729675574597004,
467            "snapshot id from metadata don't match"
468        );
469    }
470
471    #[tokio::test]
472    async fn test_static_into_table() {
473        let metadata_file_name = "TableMetadataV2Valid.json";
474        let metadata_file_path = format!(
475            "{}/testdata/table_metadata/{}",
476            env!("CARGO_MANIFEST_DIR"),
477            metadata_file_name
478        );
479        let file_io = FileIO::new_with_fs();
480        let static_identifier = TableIdent::from_strs(["static_ns", "static_table"]).unwrap();
481        let static_table =
482            StaticTable::from_metadata_file(&metadata_file_path, static_identifier, file_io)
483                .await
484                .unwrap();
485        let table = static_table.into_table();
486        assert!(table.readonly());
487        assert_eq!(table.identifier.name(), "static_table");
488        assert_eq!(
489            table.metadata_location(),
490            Some(metadata_file_path).as_deref()
491        );
492    }
493
494    #[tokio::test]
495    async fn test_table_readonly_flag() {
496        let metadata_file_name = "TableMetadataV2Valid.json";
497        let metadata_file_path = format!(
498            "{}/testdata/table_metadata/{}",
499            env!("CARGO_MANIFEST_DIR"),
500            metadata_file_name
501        );
502        let file_io = FileIO::new_with_fs();
503        let metadata_file = file_io.new_input(metadata_file_path).unwrap();
504        let metadata_file_content = metadata_file.read().await.unwrap();
505        let table_metadata =
506            serde_json::from_slice::<TableMetadata>(&metadata_file_content).unwrap();
507        let static_identifier = TableIdent::from_strs(["ns", "table"]).unwrap();
508        let table = Table::builder()
509            .metadata(table_metadata)
510            .identifier(static_identifier)
511            .file_io(file_io.clone())
512            .runtime(Runtime::try_current().unwrap())
513            .build()
514            .unwrap();
515        assert!(!table.readonly());
516        assert_eq!(table.identifier.name(), "table");
517    }
518
519    /// `with_runtime` must move both the table's own spawns and the object
520    /// cache's manifest parses onto the new runtime, while the rebound cache
521    /// keeps serving what the original already loaded.
522    #[tokio::test]
523    async fn test_table_with_runtime_moves_work_and_keeps_object_cache() {
524        let old = tokio::runtime::Builder::new_multi_thread().build().unwrap();
525        let mut fixture = TableTestFixture::new();
526        fixture.setup_manifest_files().await;
527        // Built on `old` from the start, so its object cache is bound to `old`
528        // too; `with_runtime` on a table built elsewhere would not show that.
529        let table = Table::builder()
530            .metadata(fixture.table.metadata().clone())
531            .identifier(fixture.table.identifier().clone())
532            .file_io(fixture.table.file_io().clone())
533            .runtime(Runtime::new(&old))
534            .build()
535            .unwrap();
536
537        // Load the manifest list through the original table so it is cached.
538        let snapshot = table.metadata().current_snapshot().unwrap().clone();
539        let manifest_list = table
540            .object_cache()
541            .get_manifest_list(&snapshot, &table.metadata_ref())
542            .await
543            .unwrap();
544        // With the old runtime shut down, every spawn below succeeds only if
545        // `rebound` uses the new one.
546        old.shutdown_background();
547
548        let rebound = table.clone().with_runtime(Runtime::try_current().unwrap());
549        assert_eq!(
550            rebound.runtime().io().spawn(async { 42 }).await.unwrap(),
551            42
552        );
553
554        // The rebound cache parses a manifest it has not seen on the new runtime.
555        let manifest_file = manifest_list.entries()[0].clone();
556        rebound
557            .object_cache()
558            .get_manifest(&manifest_file)
559            .await
560            .unwrap();
561
562        // And it still holds the manifest list the original table loaded: with
563        // the file gone, only a cache hit can answer.
564        fs::remove_file(snapshot.manifest_list()).unwrap();
565        let from_cache = rebound
566            .object_cache()
567            .get_manifest_list(&snapshot, &rebound.metadata_ref())
568            .await
569            .unwrap();
570        assert!(Arc::ptr_eq(&manifest_list, &from_cache));
571    }
572
573    fn make_kms() -> Arc<dyn KeyManagementClient> {
574        let kms = MemoryKeyManagementClient::new();
575        kms.add_master_key("master-1").unwrap();
576        Arc::new(kms)
577    }
578
579    #[tokio::test]
580    async fn table_decrypts_manifest_list_via_object_cache() {
581        // The fixture contains a snapshot with key-id, encryption-keys (KEK + wrapped DEK),
582        // all generated with the master key bytes below.
583        let mut metadata: TableMetadata = load_test_metadata("TableMetadataV3ValidEncryption.json");
584
585        // Point the snapshot's manifest-list at the testdata file on disk.
586        let manifest_list_path = format!(
587            "{}/testdata/manifests_lists/manifest-list-v3-encrypted.avro",
588            env!("CARGO_MANIFEST_DIR"),
589        );
590        let snapshot = metadata.snapshots.get_mut(&1).unwrap();
591        let mut patched = snapshot.as_ref().clone();
592        patched.manifest_list = manifest_list_path;
593        *snapshot = Arc::new(patched);
594
595        // Seed the KMS with the same master key bytes used to generate the fixture.
596        let kms: Arc<dyn KeyManagementClient> = {
597            let k = MemoryKeyManagementClient::new();
598            k.add_master_key_bytes(
599                "master-1",
600                SensitiveBytes::new([
601                    0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x09, 0x0a, 0x0b, 0x0c,
602                    0x0d, 0x0e, 0x0f,
603                ]),
604            )
605            .unwrap();
606            Arc::new(k)
607        };
608
609        let table = Table::builder()
610            .file_io(FileIO::new_with_fs())
611            .metadata(metadata)
612            .identifier(TableIdent::from_strs(["ns", "enc"]).unwrap())
613            .kms_client(kms)
614            .runtime(Runtime::try_current().unwrap())
615            .build()
616            .unwrap();
617
618        let snapshot_ref = table.metadata().current_snapshot().unwrap();
619        let manifest_list = table
620            .object_cache()
621            .get_manifest_list(snapshot_ref, &table.metadata_ref())
622            .await
623            .unwrap();
624        assert_eq!(manifest_list.entries().len(), 0);
625    }
626
627    #[tokio::test]
628    async fn table_builder_errors_when_encryption_key_id_set_but_no_kms() {
629        let metadata: TableMetadata = load_test_metadata("TableMetadataV3ValidEncryption.json");
630
631        let err = Table::builder()
632            .file_io(FileIO::new_with_memory())
633            .metadata(metadata)
634            .identifier(TableIdent::from_strs(["ns", "enc"]).unwrap())
635            .runtime(Runtime::try_current().unwrap())
636            .build()
637            .unwrap_err();
638        assert_eq!(err.kind(), ErrorKind::PreconditionFailed);
639    }
640
641    #[tokio::test]
642    async fn table_builder_skips_encryption_on_pre_v3_table() {
643        // Encryption is a v3 spec feature; pre-v3 tables silently skip
644        // encryption even if encryption.key-id is set.
645        let mut metadata: TableMetadata = load_test_metadata("TableMetadataV2ValidMinimal.json");
646        metadata.properties.insert(
647            TableProperties::PROPERTY_ENCRYPTION_KEY_ID.to_string(),
648            "master-1".to_string(),
649        );
650
651        let table = Table::builder()
652            .file_io(FileIO::new_with_memory())
653            .metadata(metadata)
654            .identifier(TableIdent::from_strs(["ns", "enc"]).unwrap())
655            .kms_client(make_kms())
656            .runtime(Runtime::try_current().unwrap())
657            .build()
658            .unwrap();
659        assert!(table.encryption_manager().is_none());
660    }
661
662    #[tokio::test]
663    async fn table_builder_skips_encryption_when_property_absent() {
664        let metadata: TableMetadata = load_test_metadata("TableMetadataV3ValidMinimal.json");
665        let table = Table::builder()
666            .file_io(FileIO::new_with_memory())
667            .metadata(metadata)
668            .identifier(TableIdent::from_strs(["ns", "plain"]).unwrap())
669            .kms_client(make_kms())
670            .runtime(Runtime::try_current().unwrap())
671            .build()
672            .unwrap();
673        assert!(table.encryption_manager().is_none());
674    }
675}