iceberg/io/
storage.rs

1// Licensed to the Apache Software Foundation (ASF) under one
2// or more contributor license agreements.  See the NOTICE file
3// distributed with this work for additional information
4// regarding copyright ownership.  The ASF licenses this file
5// to you under the Apache License, Version 2.0 (the
6// "License"); you may not use this file except in compliance
7// with the License.  You may obtain a copy of the License at
8//
9//   http://www.apache.org/licenses/LICENSE-2.0
10//
11// Unless required by applicable law or agreed to in writing,
12// software distributed under the License is distributed on an
13// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14// KIND, either express or implied.  See the License for the
15// specific language governing permissions and limitations
16// under the License.
17
18//! Storage interfaces for Iceberg.
19
20use std::fmt::Debug;
21use std::sync::Arc;
22
23use async_trait::async_trait;
24use bytes::Bytes;
25
26use super::{FileMetadata, FileRead, FileWrite, InputFile, OutputFile};
27use crate::Result;
28pub use crate::io::config::StorageConfig;
29
30/// Trait for storage operations in Iceberg.
31///
32/// The trait supports serialization via `typetag`, allowing storage instances to be
33/// serialized and deserialized across process boundaries.
34///
35/// Third-party implementations can implement this trait to provide custom storage backends.
36///
37/// # Implementing Custom Storage
38///
39/// To implement a custom storage backend:
40///
41/// 1. Create a struct that implements this trait
42/// 2. Add `#[typetag::serde]` attribute for serialization support
43/// 3. Implement all required methods
44///
45/// # Example
46///
47/// ```rust,ignore
48/// #[derive(Debug, Clone, serde::Serialize, serde::Deserialize)]
49/// struct MyStorage {
50///     // custom fields
51/// }
52///
53/// #[async_trait]
54/// #[typetag::serde]
55/// impl Storage for MyStorage {
56///     async fn exists(&self, path: &str) -> Result<bool> {
57///         // implementation
58///         todo!()
59///     }
60///     // ... implement other methods
61/// }
62///
63/// TODO remove below when the trait is integrated with FileIO and Catalog
64/// # NOTE
65/// This trait is under heavy development and is not used anywhere as of now
66/// Please DO NOT implement it
67/// ```
68#[async_trait]
69#[typetag::serde(tag = "type")]
70pub trait Storage: Debug + Send + Sync {
71    /// Check if a file exists at the given path
72    async fn exists(&self, path: &str) -> Result<bool>;
73
74    /// Get metadata from an input path
75    async fn metadata(&self, path: &str) -> Result<FileMetadata>;
76
77    /// Read bytes from a path
78    async fn read(&self, path: &str) -> Result<Bytes>;
79
80    /// Get FileRead from a path
81    async fn reader(&self, path: &str) -> Result<Box<dyn FileRead>>;
82
83    /// Write bytes to an output path
84    async fn write(&self, path: &str, bs: Bytes) -> Result<()>;
85
86    /// Get FileWrite from a path
87    async fn writer(&self, path: &str) -> Result<Box<dyn FileWrite>>;
88
89    /// Delete a file at the given path
90    async fn delete(&self, path: &str) -> Result<()>;
91
92    /// Delete all files with the given prefix
93    async fn delete_prefix(&self, path: &str) -> Result<()>;
94
95    /// Create a new input file for reading
96    fn new_input(&self, path: &str) -> Result<InputFile>;
97
98    /// Create a new output file for writing
99    fn new_output(&self, path: &str) -> Result<OutputFile>;
100}
101
102/// Factory for creating Storage instances from configuration.
103///
104/// Implement this trait to provide custom storage backends. The factory pattern
105/// allows for lazy initialization of storage instances and enables users to
106/// inject custom storage implementations into catalogs.
107///
108/// # Example
109///
110/// ```rust,ignore
111/// #[derive(Debug, Clone, serde::Serialize, serde::Deserialize)]
112/// struct MyCustomStorageFactory {
113///     // custom configuration
114/// }
115///
116/// #[typetag::serde]
117/// impl StorageFactory for MyCustomStorageFactory {
118///     fn build(&self, config: &StorageConfig) -> Result<Arc<dyn Storage>> {
119///         // Create and return custom storage implementation
120///         todo!()
121///     }
122/// }
123///
124/// TODO remove below when the trait is integrated with FileIO and Catalog
125/// # NOTE
126/// This trait is under heavy development and is not used anywhere as of now
127/// Please DO NOT implement it
128/// ```
129#[typetag::serde(tag = "type")]
130pub trait StorageFactory: Debug + Send + Sync {
131    /// Build a new Storage instance from the given configuration.
132    ///
133    /// # Arguments
134    ///
135    /// * `config` - The storage configuration containing scheme and properties
136    ///
137    /// # Returns
138    ///
139    /// A `Result` containing an `Arc<dyn Storage>` on success, or an error
140    /// if the storage could not be created.
141    fn build(&self, config: &StorageConfig) -> Result<Arc<dyn Storage>>;
142}
iceberg/io/storage.rs

iceberg/io/
storage.rs
