Enum OpenDalStorage

pub enum OpenDalStorage {
    Memory(Operator),
    LocalFs,
    S3 {
        configured_scheme: String,
        config: Arc<S3Config>,
        customized_credential_load: Option<CustomAwsCredentialLoader>,
    },
    Gcs {
        config: Arc<GcsConfig>,
    },
    Oss {
        config: Arc<OssConfig>,
    },
    Azdls {
        configured_scheme: AzureStorageScheme,
        config: Arc<AzdlsConfig>,
    },
}

OpenDAL-based storage implementation.

Variants

Memory(Operator)

Memory storage variant.

LocalFs

Local filesystem storage variant.

S3

S3 storage variant.

Fields

configured_scheme: String

s3 storage could have s3:// and s3a://. Storing the scheme string here to return the correct path.

config: Arc<S3Config>

S3 configuration.

customized_credential_load: Option<CustomAwsCredentialLoader>

Custom AWS credential loader.

Gcs

GCS storage variant.

Fields

config: Arc<GcsConfig>

GCS configuration.

Oss

OSS storage variant.

Fields

config: Arc<OssConfig>

OSS configuration.

Azdls

Azure Data Lake Storage variant. Expects paths of the form abfs[s]://<filesystem>@<account>.dfs.<endpoint-suffix>/<path> or wasb[s]://<container>@<account>.blob.<endpoint-suffix>/<path>.

Fields

configured_scheme: AzureStorageScheme

The configured Azure storage scheme. Because Azdls accepts multiple possible schemes, we store the full passed scheme here to later validate schemes passed via paths.

config: Arc<AzdlsConfig>

Azure DLS configuration.

Trait Implementations

impl Clone for OpenDalStorage

fn clone(&self) -> OpenDalStorage

Returns a duplicate of the value.

const fn clone_from(&mut self, source: &Self)

Performs copy-assignment from source.

impl Debug for OpenDalStorage

fn fmt(&self, f: &mut Formatter<'_>) -> Result

Formats the value using the given formatter.

impl<'de> Deserialize<'de> for OpenDalStorage

fn deserialize<__D>(__deserializer: __D) -> Result<Self, __D::Error>

where __D: Deserializer<'de>,

Deserialize this value from the given Serde deserializer.

impl Serialize for OpenDalStorage

fn serialize<__S>(&self, __serializer: __S) -> Result<__S::Ok, __S::Error>

where __S: Serializer,

Serialize this value into the given Serde serializer.

impl Storage for OpenDalStorage

fn exists<'life0, 'life1, 'async_trait>( &'life0 self, path: &'life1 str, ) -> Pin<Box<dyn Future<Output = Result<bool>> + Send + 'async_trait>>

where Self: 'async_trait, 'life0: 'async_trait, 'life1: 'async_trait,

Check if a file exists at the given path

fn metadata<'life0, 'life1, 'async_trait>( &'life0 self, path: &'life1 str, ) -> Pin<Box<dyn Future<Output = Result<FileMetadata>> + Send + 'async_trait>>

where Self: 'async_trait, 'life0: 'async_trait, 'life1: 'async_trait,

Get metadata from an input path

fn read<'life0, 'life1, 'async_trait>( &'life0 self, path: &'life1 str, ) -> Pin<Box<dyn Future<Output = Result<Bytes>> + Send + 'async_trait>>

where Self: 'async_trait, 'life0: 'async_trait, 'life1: 'async_trait,

Read bytes from a path

fn reader<'life0, 'life1, 'async_trait>( &'life0 self, path: &'life1 str, ) -> Pin<Box<dyn Future<Output = Result<Box<dyn FileRead>>> + Send + 'async_trait>>

where Self: 'async_trait, 'life0: 'async_trait, 'life1: 'async_trait,

Get FileRead from a path

fn write<'life0, 'life1, 'async_trait>( &'life0 self, path: &'life1 str, bs: Bytes, ) -> Pin<Box<dyn Future<Output = Result<()>> + Send + 'async_trait>>

where Self: 'async_trait, 'life0: 'async_trait, 'life1: 'async_trait,

Write bytes to an output path

fn writer<'life0, 'life1, 'async_trait>( &'life0 self, path: &'life1 str, ) -> Pin<Box<dyn Future<Output = Result<Box<dyn FileWrite>>> + Send + 'async_trait>>

where Self: 'async_trait, 'life0: 'async_trait, 'life1: 'async_trait,

Get FileWrite from a path

fn delete<'life0, 'life1, 'async_trait>( &'life0 self, path: &'life1 str, ) -> Pin<Box<dyn Future<Output = Result<()>> + Send + 'async_trait>>

where Self: 'async_trait, 'life0: 'async_trait, 'life1: 'async_trait,

Delete a file at the given path

fn delete_prefix<'life0, 'life1, 'async_trait>( &'life0 self, path: &'life1 str, ) -> Pin<Box<dyn Future<Output = Result<()>> + Send + 'async_trait>>

where Self: 'async_trait, 'life0: 'async_trait, 'life1: 'async_trait,

Delete all files with the given prefix

fn new_input(&self, path: &str) -> Result<InputFile>

Create a new input file for reading

fn new_output(&self, path: &str) -> Result<OutputFile>

Create a new output file for writing