Trait Storage

pub trait Storage:
    Debug
    + Send
    + Sync
    + Serialize
    + Deserialize {
    // Required methods
    fn exists<'life0, 'life1, 'async_trait>(
        &'life0 self,
        path: &'life1 str,
    ) -> Pin<Box<dyn Future<Output = Result<bool>> + Send + 'async_trait>>
       where Self: 'async_trait,
             'life0: 'async_trait,
             'life1: 'async_trait;
    fn metadata<'life0, 'life1, 'async_trait>(
        &'life0 self,
        path: &'life1 str,
    ) -> Pin<Box<dyn Future<Output = Result<FileMetadata>> + Send + 'async_trait>>
       where Self: 'async_trait,
             'life0: 'async_trait,
             'life1: 'async_trait;
    fn read<'life0, 'life1, 'async_trait>(
        &'life0 self,
        path: &'life1 str,
    ) -> Pin<Box<dyn Future<Output = Result<Bytes>> + Send + 'async_trait>>
       where Self: 'async_trait,
             'life0: 'async_trait,
             'life1: 'async_trait;
    fn reader<'life0, 'life1, 'async_trait>(
        &'life0 self,
        path: &'life1 str,
    ) -> Pin<Box<dyn Future<Output = Result<Box<dyn FileRead>>> + Send + 'async_trait>>
       where Self: 'async_trait,
             'life0: 'async_trait,
             'life1: 'async_trait;
    fn write<'life0, 'life1, 'async_trait>(
        &'life0 self,
        path: &'life1 str,
        bs: Bytes,
    ) -> Pin<Box<dyn Future<Output = Result<()>> + Send + 'async_trait>>
       where Self: 'async_trait,
             'life0: 'async_trait,
             'life1: 'async_trait;
    fn writer<'life0, 'life1, 'async_trait>(
        &'life0 self,
        path: &'life1 str,
    ) -> Pin<Box<dyn Future<Output = Result<Box<dyn FileWrite>>> + Send + 'async_trait>>
       where Self: 'async_trait,
             'life0: 'async_trait,
             'life1: 'async_trait;
    fn delete<'life0, 'life1, 'async_trait>(
        &'life0 self,
        path: &'life1 str,
    ) -> Pin<Box<dyn Future<Output = Result<()>> + Send + 'async_trait>>
       where Self: 'async_trait,
             'life0: 'async_trait,
             'life1: 'async_trait;
    fn delete_prefix<'life0, 'life1, 'async_trait>(
        &'life0 self,
        path: &'life1 str,
    ) -> Pin<Box<dyn Future<Output = Result<()>> + Send + 'async_trait>>
       where Self: 'async_trait,
             'life0: 'async_trait,
             'life1: 'async_trait;
    fn new_input(&self, path: &str) -> Result<InputFile>;
    fn new_output(&self, path: &str) -> Result<OutputFile>;
}

Trait for storage operations in Iceberg.

The trait supports serialization via typetag, allowing storage instances to be serialized and deserialized across process boundaries.

Third-party implementations can implement this trait to provide custom storage backends.

Implementing Custom Storage

To implement a custom storage backend:

1. Create a struct that implements this trait
2. Add #[typetag::serde] attribute for serialization support
3. Implement all required methods

Example

#[derive(Debug, Clone, serde::Serialize, serde::Deserialize)]
struct MyStorage {
    // custom fields
}

#[async_trait]
#[typetag::serde]
impl Storage for MyStorage {
    async fn exists(&self, path: &str) -> Result<bool> {
        // implementation
        todo!()
    }
    // ... implement other methods
}

TODO remove below when the trait is integrated with FileIO and Catalog
This trait is under heavy development and is not used anywhere as of now
Please DO NOT implement it

Required Methods

fn exists<'life0, 'life1, 'async_trait>( &'life0 self, path: &'life1 str, ) -> Pin<Box<dyn Future<Output = Result<bool>> + Send + 'async_trait>>

where Self: 'async_trait, 'life0: 'async_trait, 'life1: 'async_trait,

Check if a file exists at the given path

fn metadata<'life0, 'life1, 'async_trait>( &'life0 self, path: &'life1 str, ) -> Pin<Box<dyn Future<Output = Result<FileMetadata>> + Send + 'async_trait>>

where Self: 'async_trait, 'life0: 'async_trait, 'life1: 'async_trait,

Get metadata from an input path

fn read<'life0, 'life1, 'async_trait>( &'life0 self, path: &'life1 str, ) -> Pin<Box<dyn Future<Output = Result<Bytes>> + Send + 'async_trait>>

where Self: 'async_trait, 'life0: 'async_trait, 'life1: 'async_trait,

Read bytes from a path

fn reader<'life0, 'life1, 'async_trait>( &'life0 self, path: &'life1 str, ) -> Pin<Box<dyn Future<Output = Result<Box<dyn FileRead>>> + Send + 'async_trait>>

where Self: 'async_trait, 'life0: 'async_trait, 'life1: 'async_trait,

Get FileRead from a path

fn write<'life0, 'life1, 'async_trait>( &'life0 self, path: &'life1 str, bs: Bytes, ) -> Pin<Box<dyn Future<Output = Result<()>> + Send + 'async_trait>>

where Self: 'async_trait, 'life0: 'async_trait, 'life1: 'async_trait,

Write bytes to an output path

fn writer<'life0, 'life1, 'async_trait>( &'life0 self, path: &'life1 str, ) -> Pin<Box<dyn Future<Output = Result<Box<dyn FileWrite>>> + Send + 'async_trait>>

where Self: 'async_trait, 'life0: 'async_trait, 'life1: 'async_trait,

Get FileWrite from a path

fn delete<'life0, 'life1, 'async_trait>( &'life0 self, path: &'life1 str, ) -> Pin<Box<dyn Future<Output = Result<()>> + Send + 'async_trait>>

where Self: 'async_trait, 'life0: 'async_trait, 'life1: 'async_trait,

Delete a file at the given path

fn delete_prefix<'life0, 'life1, 'async_trait>( &'life0 self, path: &'life1 str, ) -> Pin<Box<dyn Future<Output = Result<()>> + Send + 'async_trait>>

where Self: 'async_trait, 'life0: 'async_trait, 'life1: 'async_trait,

Delete all files with the given prefix

fn new_input(&self, path: &str) -> Result<InputFile>

Create a new input file for reading

fn new_output(&self, path: &str) -> Result<OutputFile>

Create a new output file for writing

Trait Implementations

impl<'typetag> Serialize for dyn Storage + 'typetag

fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error>

where S: Serializer,

Serialize this value into the given Serde serializer.

impl<'typetag> Serialize for dyn Storage + Send + 'typetag

fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error>

where S: Serializer,

Serialize this value into the given Serde serializer.

impl<'typetag> Serialize for dyn Storage + Send + Sync + 'typetag

fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error>

where S: Serializer,

Serialize this value into the given Serde serializer.

impl<'typetag> Serialize for dyn Storage + Sync + 'typetag

fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error>

where S: Serializer,

Serialize this value into the given Serde serializer.

Implementors