Enum OpenDalStorage 

pub enum OpenDalStorage {
    Memory {
        operator: Operator,
        client_config: OpenDalClientConfig,
    },
    LocalFs {
        client_config: OpenDalClientConfig,
    },
    S3 {
        config: Arc<S3Config>,
        customized_credential_load: Option<CustomAwsCredentialLoader>,
        client_config: OpenDalClientConfig,
    },
    Gcs {
        config: Arc<GcsConfig>,
        client_config: OpenDalClientConfig,
    },
    Oss {
        config: Arc<OssConfig>,
        client_config: OpenDalClientConfig,
    },
    Azdls {
        config: Arc<AzdlsConfig>,
        client_config: OpenDalClientConfig,
    },
    Hf {
        config: Arc<HfConfig>,
        client_config: OpenDalClientConfig,
    },
}

OpenDAL-based storage implementation.

The serialized representation is not a stable format and may change between crate versions.

Variants

Memory

Memory storage variant.

Fields

operator: Operator

Pre-built memory operator.

client_config: OpenDalClientConfig

Backend-independent client settings.

LocalFs

Local filesystem storage variant.

Fields

client_config: OpenDalClientConfig

Backend-independent client settings.

S3

S3 storage variant.

Accepts any S3-family URL (s3://, s3a://, s3n://); the scheme is derived from the path at call time.

Fields

config: Arc<S3Config>

S3 configuration.

customized_credential_load: Option<CustomAwsCredentialLoader>

Custom AWS credential loader.

client_config: OpenDalClientConfig

Backend-independent client settings.

Gcs

GCS storage variant.

Fields

config: Arc<GcsConfig>

GCS configuration.

client_config: OpenDalClientConfig

Backend-independent client settings.

Oss

OSS storage variant.

Fields

config: Arc<OssConfig>

OSS configuration.

client_config: OpenDalClientConfig

Backend-independent client settings.

Azdls

Azure Data Lake Storage variant.

Accepts paths of the form abfs[s]://<filesystem>@<account>.dfs.<endpoint-suffix>/<path> or wasb[s]://<container>@<account>.blob.<endpoint-suffix>/<path>. The scheme is derived from the path at call time.

Fields

config: Arc<AzdlsConfig>

Azure DLS configuration.

client_config: OpenDalClientConfig

Backend-independent client settings.

Hf

HuggingFace Hub storage variant.

Accepts paths of the form hf://<repo_type>/<owner>/<repo>[@<revision>]/<path_in_repo>, where <repo_type> must be one of models, datasets, spaces, or buckets.

Fields

config: Arc<HfConfig>

HuggingFace Hub configuration (token + endpoint).

client_config: OpenDalClientConfig

Backend-independent client settings.

Trait Implementations

impl Clone for OpenDalStorage

fn clone(&self) -> OpenDalStorage

Returns a duplicate of the value.

fn clone_from(&mut self, source: &Self)

Performs copy-assignment from source.

impl Debug for OpenDalStorage

fn fmt(&self, f: &mut Formatter<'_>) -> Result

Formats the value using the given formatter.

impl<'de> Deserialize<'de> for OpenDalStorage

fn deserialize<__D>(__deserializer: __D) -> Result<Self, __D::Error>

where __D: Deserializer<'de>,

Deserialize this value from the given Serde deserializer.

impl Serialize for OpenDalStorage

fn serialize<__S>(&self, __serializer: __S) -> Result<__S::Ok, __S::Error>

where __S: Serializer,

Serialize this value into the given Serde serializer.

impl Storage for OpenDalStorage

fn exists<'life0, 'life1, 'async_trait>( &'life0 self, path: &'life1 str, ) -> Pin<Box<dyn Future<Output = Result<bool>> + Send + 'async_trait>>

where Self: 'async_trait, 'life0: 'async_trait, 'life1: 'async_trait,

Check if a file exists at the given path

fn metadata<'life0, 'life1, 'async_trait>( &'life0 self, path: &'life1 str, ) -> Pin<Box<dyn Future<Output = Result<FileMetadata>> + Send + 'async_trait>>

where Self: 'async_trait, 'life0: 'async_trait, 'life1: 'async_trait,

Get metadata from an input path

fn read<'life0, 'life1, 'async_trait>( &'life0 self, path: &'life1 str, ) -> Pin<Box<dyn Future<Output = Result<Bytes>> + Send + 'async_trait>>

where Self: 'async_trait, 'life0: 'async_trait, 'life1: 'async_trait,

Read bytes from a path

fn reader<'life0, 'life1, 'async_trait>( &'life0 self, path: &'life1 str, ) -> Pin<Box<dyn Future<Output = Result<Box<dyn FileRead>>> + Send + 'async_trait>>

where Self: 'async_trait, 'life0: 'async_trait, 'life1: 'async_trait,

Get FileRead from a path

fn write<'life0, 'life1, 'async_trait>( &'life0 self, path: &'life1 str, bs: Bytes, ) -> Pin<Box<dyn Future<Output = Result<()>> + Send + 'async_trait>>

where Self: 'async_trait, 'life0: 'async_trait, 'life1: 'async_trait,

Write bytes to an output path

fn writer<'life0, 'life1, 'async_trait>( &'life0 self, path: &'life1 str, ) -> Pin<Box<dyn Future<Output = Result<Box<dyn FileWrite>>> + Send + 'async_trait>>

where Self: 'async_trait, 'life0: 'async_trait, 'life1: 'async_trait,

Get FileWrite from a path

fn delete<'life0, 'life1, 'async_trait>( &'life0 self, path: &'life1 str, ) -> Pin<Box<dyn Future<Output = Result<()>> + Send + 'async_trait>>

where Self: 'async_trait, 'life0: 'async_trait, 'life1: 'async_trait,

Delete a file at the given path

fn delete_prefix<'life0, 'life1, 'async_trait>( &'life0 self, path: &'life1 str, ) -> Pin<Box<dyn Future<Output = Result<()>> + Send + 'async_trait>>

where Self: 'async_trait, 'life0: 'async_trait, 'life1: 'async_trait,

Delete all files with the given prefix

fn delete_stream<'life0, 'async_trait>( &'life0 self, paths: BoxStream<'static, String>, ) -> Pin<Box<dyn Future<Output = Result<()>> + Send + 'async_trait>>

where Self: 'async_trait, 'life0: 'async_trait,

Delete multiple files from a stream of paths.

fn new_input(&self, path: &str) -> Result<InputFile>

Create a new input file for reading

fn new_output(&self, path: &str) -> Result<OutputFile>

Create a new output file for writing