Skip to main content

iceberg_storage_opendal/
resolving.rs

1// Licensed to the Apache Software Foundation (ASF) under one
2// or more contributor license agreements.  See the NOTICE file
3// distributed with this work for additional information
4// regarding copyright ownership.  The ASF licenses this file
5// to you under the Apache License, Version 2.0 (the
6// "License"); you may not use this file except in compliance
7// with the License.  You may obtain a copy of the License at
8//
9//   http://www.apache.org/licenses/LICENSE-2.0
10//
11// Unless required by applicable law or agreed to in writing,
12// software distributed under the License is distributed on an
13// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14// KIND, either express or implied.  See the License for the
15// specific language governing permissions and limitations
16// under the License.
17
18//! Resolving storage that auto-detects the scheme from a path and delegates
19//! to the appropriate [`OpenDalStorage`] variant.
20
21use std::collections::HashMap;
22use std::sync::{Arc, RwLock};
23
24use async_trait::async_trait;
25use bytes::Bytes;
26use futures::StreamExt;
27use futures::stream::BoxStream;
28use iceberg::io::{
29    FileMetadata, FileRead, FileWrite, InputFile, OutputFile, Storage, StorageConfig,
30    StorageFactory,
31};
32use iceberg::{Error, ErrorKind, Result};
33use serde::{Deserialize, Serialize};
34use url::Url;
35
36#[cfg(feature = "opendal-s3")]
37use crate::s3::CustomAwsCredentialLoader;
38use crate::{OpenDalClientConfig, OpenDalStorage};
39
40/// Schemes supported by OpenDalResolvingStorage
41pub const SCHEME_MEMORY: &str = "memory";
42pub const SCHEME_FILE: &str = "file";
43pub const SCHEME_S3: &str = "s3";
44pub const SCHEME_S3A: &str = "s3a";
45pub const SCHEME_S3N: &str = "s3n";
46pub const SCHEME_GS: &str = "gs";
47pub const SCHEME_GCS: &str = "gcs";
48pub const SCHEME_OSS: &str = "oss";
49pub const SCHEME_ABFSS: &str = "abfss";
50pub const SCHEME_ABFS: &str = "abfs";
51pub const SCHEME_WASBS: &str = "wasbs";
52pub const SCHEME_WASB: &str = "wasb";
53pub const SCHEME_HF: &str = "hf";
54pub const SCHEME_HDFS: &str = "hdfs";
55
56/// Parse a URL scheme string.
57fn parse_scheme(scheme: &str) -> Result<&'static str> {
58    match scheme {
59        SCHEME_MEMORY => Ok("memory"),
60        SCHEME_FILE | "" => Ok("file"),
61        SCHEME_S3 | SCHEME_S3A | SCHEME_S3N => Ok("s3"),
62        SCHEME_GS | SCHEME_GCS => Ok("gcs"),
63        SCHEME_OSS => Ok("oss"),
64        SCHEME_ABFSS | SCHEME_ABFS | SCHEME_WASBS | SCHEME_WASB => Ok("azdls"),
65        SCHEME_HF => Ok("hf"),
66        SCHEME_HDFS => Ok("hdfs"),
67        s => Err(Error::new(
68            ErrorKind::FeatureUnsupported,
69            format!("Unsupported storage scheme: {s}"),
70        )),
71    }
72}
73
74/// Extract the scheme from a path URL.
75fn extract_scheme(path: &str) -> Result<&'static str> {
76    let url = Url::parse(path).map_err(|e| {
77        Error::new(
78            ErrorKind::DataInvalid,
79            format!("Invalid path: {path}, failed to parse URL: {e}"),
80        )
81    })?;
82    parse_scheme(url.scheme())
83}
84
85/// Build an [`OpenDalStorage`] variant for the given scheme and config properties.
86fn build_storage_for_scheme(
87    scheme: &'static str,
88    props: &HashMap<String, String>,
89    #[cfg(feature = "opendal-s3")] customized_credential_load: &Option<CustomAwsCredentialLoader>,
90) -> Result<OpenDalStorage> {
91    let client_config = OpenDalClientConfig::from_properties(props)?;
92    match scheme {
93        #[cfg(feature = "opendal-s3")]
94        "s3" => {
95            let config = crate::s3::s3_config_parse(props.clone())?;
96            Ok(OpenDalStorage::S3 {
97                config: Arc::new(config),
98                multipart_part_size: crate::s3::s3_multipart_part_size_parse(props)?,
99                customized_credential_load: customized_credential_load.clone(),
100                client_config: crate::s3::s3_client_config(props, client_config)?,
101            })
102        }
103        #[cfg(feature = "opendal-gcs")]
104        "gcs" => {
105            let config = crate::gcs::gcs_config_parse(props.clone())?;
106            Ok(OpenDalStorage::Gcs {
107                config: Arc::new(config),
108                client_config,
109            })
110        }
111        #[cfg(feature = "opendal-oss")]
112        "oss" => {
113            let config = crate::oss::oss_config_parse(props.clone())?;
114            Ok(OpenDalStorage::Oss {
115                config: Arc::new(config),
116                client_config,
117            })
118        }
119        #[cfg(feature = "opendal-azdls")]
120        "azdls" => {
121            let config = crate::azdls::azdls_config_parse(props.clone())?;
122            Ok(OpenDalStorage::Azdls {
123                config: Arc::new(config),
124                client_config,
125            })
126        }
127        #[cfg(feature = "opendal-fs")]
128        "file" => Ok(OpenDalStorage::LocalFs { client_config }),
129        #[cfg(feature = "opendal-memory")]
130        "memory" => Ok(OpenDalStorage::Memory {
131            operator: crate::memory::memory_config_build()?,
132            client_config,
133        }),
134        #[cfg(feature = "opendal-hf")]
135        "hf" => {
136            let config = crate::hf::hf_config_parse(props.clone())?;
137            Ok(OpenDalStorage::Hf {
138                config: Arc::new(config),
139                client_config,
140            })
141        }
142        #[cfg(feature = "opendal-hdfs-native")]
143        "hdfs" => {
144            let config = crate::hdfs_native::hdfs_native_config_parse(props.clone())?;
145            Ok(OpenDalStorage::HdfsNative(crate::HdfsNativeStorage::new(
146                config,
147                client_config,
148            )))
149        }
150        unsupported => Err(Error::new(
151            ErrorKind::FeatureUnsupported,
152            format!("Unsupported storage scheme: {unsupported}"),
153        )),
154    }
155}
156
157/// A resolving storage factory that creates [`OpenDalResolvingStorage`] instances.
158///
159/// This factory accepts paths from any supported storage system and dynamically
160/// delegates operations to the appropriate [`OpenDalStorage`] variant based on
161/// the path scheme.
162///
163/// # Serialization
164///
165/// Serialization fails when a custom S3 credential loader is configured because the loader holds
166/// process-local state that cannot be reconstructed in another process. Construct the factory
167/// without a custom loader before serializing it.
168///
169/// # Example
170///
171/// ```rust,ignore
172/// use std::sync::Arc;
173/// use iceberg::io::FileIOBuilder;
174/// use iceberg_storage_opendal::OpenDalResolvingStorageFactory;
175///
176/// let factory = OpenDalResolvingStorageFactory::new();
177/// let file_io = FileIOBuilder::new(Arc::new(factory))
178///     .with_prop("s3.region", "us-east-1")
179///     .build();
180/// ```
181#[derive(Clone, Debug, Serialize, Deserialize)]
182pub struct OpenDalResolvingStorageFactory {
183    /// Custom AWS credential loader for S3 storage.
184    #[cfg(feature = "opendal-s3")]
185    #[serde(
186        skip_deserializing,
187        skip_serializing_if = "Option::is_none",
188        serialize_with = "crate::serialize_custom_credential_loader"
189    )]
190    customized_credential_load: Option<CustomAwsCredentialLoader>,
191}
192
193impl Default for OpenDalResolvingStorageFactory {
194    fn default() -> Self {
195        Self::new()
196    }
197}
198
199impl OpenDalResolvingStorageFactory {
200    /// Create a new resolving storage factory.
201    pub fn new() -> Self {
202        Self {
203            #[cfg(feature = "opendal-s3")]
204            customized_credential_load: None,
205        }
206    }
207
208    /// Set a custom AWS credential loader for S3 storage.
209    #[cfg(feature = "opendal-s3")]
210    pub fn with_s3_credential_loader(mut self, loader: CustomAwsCredentialLoader) -> Self {
211        self.customized_credential_load = Some(loader);
212        self
213    }
214}
215
216#[typetag::serde]
217impl StorageFactory for OpenDalResolvingStorageFactory {
218    fn build(&self, config: &StorageConfig) -> Result<Arc<dyn Storage>> {
219        Ok(Arc::new(OpenDalResolvingStorage {
220            props: config.props().clone(),
221            storages: RwLock::new(HashMap::new()),
222            #[cfg(feature = "opendal-s3")]
223            customized_credential_load: self.customized_credential_load.clone(),
224        }))
225    }
226}
227
228/// A resolving storage that auto-detects the scheme from a path and delegates
229/// to the appropriate [`OpenDalStorage`] variant.
230///
231/// Sub-storages are lazily created on first use for each scheme and cached
232/// for subsequent operations. Scheme aliases like `s3`/`s3a`/`s3n` map to
233/// the same canonical scheme, so they share a storage instance.
234#[derive(Debug, Serialize, Deserialize)]
235pub struct OpenDalResolvingStorage {
236    /// Configuration properties shared across all backends.
237    props: HashMap<String, String>,
238    /// Cache of canonical scheme to storage mappings.
239    #[serde(skip, default)]
240    storages: RwLock<HashMap<&'static str, Arc<OpenDalStorage>>>,
241    /// Custom AWS credential loader for S3 storage.
242    #[cfg(feature = "opendal-s3")]
243    #[serde(skip)]
244    customized_credential_load: Option<CustomAwsCredentialLoader>,
245}
246
247impl OpenDalResolvingStorage {
248    /// Resolve the storage for the given path by extracting the canonical scheme and
249    /// returning the cached or newly-created [`OpenDalStorage`].
250    fn resolve(&self, path: &str) -> Result<Arc<OpenDalStorage>> {
251        let scheme = extract_scheme(path)?;
252
253        // Fast path: check read lock first.
254        {
255            let cache = self
256                .storages
257                .read()
258                .map_err(|_| Error::new(ErrorKind::Unexpected, "Storage cache lock poisoned"))?;
259            if let Some(storage) = cache.get(&scheme) {
260                return Ok(storage.clone());
261            }
262        }
263
264        // Slow path: build and insert under write lock.
265        let mut cache = self
266            .storages
267            .write()
268            .map_err(|_| Error::new(ErrorKind::Unexpected, "Storage cache lock poisoned"))?;
269
270        // Double-check after acquiring write lock.
271        if let Some(storage) = cache.get(&scheme) {
272            return Ok(storage.clone());
273        }
274
275        let storage = build_storage_for_scheme(
276            scheme,
277            &self.props,
278            #[cfg(feature = "opendal-s3")]
279            &self.customized_credential_load,
280        )?;
281        let storage = Arc::new(storage);
282        cache.insert(scheme, storage.clone());
283        Ok(storage)
284    }
285}
286
287#[async_trait]
288#[typetag::serde]
289impl Storage for OpenDalResolvingStorage {
290    async fn exists(&self, path: &str) -> Result<bool> {
291        self.resolve(path)?.exists(path).await
292    }
293
294    async fn metadata(&self, path: &str) -> Result<FileMetadata> {
295        self.resolve(path)?.metadata(path).await
296    }
297
298    async fn read(&self, path: &str) -> Result<Bytes> {
299        self.resolve(path)?.read(path).await
300    }
301
302    async fn reader(&self, path: &str) -> Result<Box<dyn FileRead>> {
303        self.resolve(path)?.reader(path).await
304    }
305
306    async fn write(&self, path: &str, bs: Bytes) -> Result<()> {
307        self.resolve(path)?.write(path, bs).await
308    }
309
310    async fn writer(&self, path: &str) -> Result<Box<dyn FileWrite>> {
311        self.resolve(path)?.writer(path).await
312    }
313
314    async fn delete(&self, path: &str) -> Result<()> {
315        self.resolve(path)?.delete(path).await
316    }
317
318    async fn delete_prefix(&self, path: &str) -> Result<()> {
319        self.resolve(path)?.delete_prefix(path).await
320    }
321
322    async fn delete_stream(&self, mut paths: BoxStream<'static, String>) -> Result<()> {
323        // Group paths by canonical scheme so each resolved storage receives a batch,
324        // avoiding repeated operator creation per path.
325        let mut grouped: HashMap<&'static str, Vec<String>> = HashMap::new();
326        while let Some(path) = paths.next().await {
327            let scheme = extract_scheme(&path)?;
328            grouped.entry(scheme).or_default().push(path);
329        }
330
331        for (_, paths) in grouped {
332            let storage = self.resolve(&paths[0])?;
333            storage
334                .delete_stream(futures::stream::iter(paths).boxed())
335                .await?;
336        }
337        Ok(())
338    }
339
340    fn new_input(&self, path: &str) -> Result<InputFile> {
341        Ok(InputFile::new(
342            Arc::new(self.resolve(path)?.as_ref().clone()),
343            path.to_string(),
344        ))
345    }
346
347    fn new_output(&self, path: &str) -> Result<OutputFile> {
348        Ok(OutputFile::new(
349            Arc::new(self.resolve(path)?.as_ref().clone()),
350            path.to_string(),
351        ))
352    }
353}
354
355#[cfg(test)]
356mod tests {
357    use std::time::Duration;
358
359    use super::*;
360    use crate::OPENDAL_IO_TIMEOUT_MS;
361
362    #[cfg(feature = "opendal-s3")]
363    #[derive(Debug)]
364    struct EmptyCredentialLoader;
365
366    #[cfg(feature = "opendal-s3")]
367    impl crate::s3::ProvideCredential for EmptyCredentialLoader {
368        type Credential = crate::s3::AwsCredential;
369
370        async fn provide_credential(
371            &self,
372            _ctx: &reqsign_core::Context,
373        ) -> reqsign_core::Result<Option<Self::Credential>> {
374            Ok(None)
375        }
376    }
377
378    #[cfg(feature = "opendal-s3")]
379    #[test]
380    fn test_custom_credential_loader_serialization_fails() {
381        let factory = OpenDalResolvingStorageFactory::new()
382            .with_s3_credential_loader(CustomAwsCredentialLoader::new(EmptyCredentialLoader));
383        let file_io = iceberg::io::FileIOBuilder::new(Arc::new(factory)).build();
384
385        let err = file_io.serialize_all().unwrap_err();
386        assert!(
387            err.to_string()
388                .contains("custom AWS credential loaders cannot be serialized")
389        );
390    }
391
392    /// Builds a resolving storage with empty props, suitable for `resolve()`
393    /// calls that don't actually hit any backend.
394    fn empty_resolving_storage() -> OpenDalResolvingStorage {
395        OpenDalResolvingStorage {
396            props: HashMap::new(),
397            storages: RwLock::new(HashMap::new()),
398            #[cfg(feature = "opendal-s3")]
399            customized_credential_load: None,
400        }
401    }
402
403    #[test]
404    fn test_resolve_propagates_io_timeout() {
405        let mut storage = empty_resolving_storage();
406        storage
407            .props
408            .insert(OPENDAL_IO_TIMEOUT_MS.to_string(), "45000".to_string());
409
410        let paths: &[&str] = &[
411            #[cfg(feature = "opendal-memory")]
412            "memory:/key",
413            #[cfg(feature = "opendal-fs")]
414            "file:/key",
415            #[cfg(feature = "opendal-s3")]
416            "s3://bucket/key",
417            #[cfg(feature = "opendal-gcs")]
418            "gs://bucket/key",
419            #[cfg(feature = "opendal-oss")]
420            "oss://bucket/key",
421            #[cfg(feature = "opendal-azdls")]
422            "abfss://myfs@myaccount.dfs.core.windows.net/key",
423            #[cfg(feature = "opendal-hf")]
424            "hf://datasets/user/repo/key",
425        ];
426        for path in paths {
427            let resolved = storage.resolve(path).unwrap();
428            assert_eq!(
429                resolved.client_config().io_timeout(),
430                Duration::from_secs(45),
431                "{path}"
432            );
433        }
434    }
435
436    #[cfg(feature = "opendal-s3")]
437    #[test]
438    fn test_resolve_s3_aliases_share_instance() {
439        let storage = empty_resolving_storage();
440
441        // All three S3-family schemes must collapse to a single cached
442        // `Arc<OpenDalStorage>` so that catalogs handing the resolver a mix
443        // of `s3://`, `s3a://`, `s3n://` paths don't rebuild operators.
444        let a = storage.resolve("s3://bucket/key").unwrap();
445        let b = storage.resolve("s3a://bucket/key").unwrap();
446        let c = storage.resolve("s3n://bucket/key").unwrap();
447
448        assert!(Arc::ptr_eq(&a, &b), "s3 and s3a should share one instance");
449        assert!(Arc::ptr_eq(&a, &c), "s3 and s3n should share one instance");
450    }
451
452    #[cfg(feature = "opendal-hdfs-native")]
453    #[test]
454    fn test_resolve_hdfs_native_returns_hdfs_variant() {
455        let storage = empty_resolving_storage();
456
457        let resolved = storage.resolve("hdfs://nameservice1/a/b").unwrap();
458
459        assert!(
460            matches!(&*resolved, OpenDalStorage::HdfsNative(_)),
461            "expected HdfsNative variant, got {resolved:?}"
462        );
463    }
464
465    #[cfg(feature = "opendal-hdfs-native")]
466    #[test]
467    fn test_resolve_hdfs_native_distinct_authorities_share_instance() {
468        let storage = empty_resolving_storage();
469
470        let a = storage.resolve("hdfs://ns1/a").unwrap();
471        let b = storage.resolve("hdfs://ns2/b").unwrap();
472
473        assert!(
474            Arc::ptr_eq(&a, &b),
475            "different authorities should share the OpenDalStorage::HdfsNative instance (operator cache is internal)"
476        );
477    }
478
479    #[cfg(feature = "opendal-azdls")]
480    #[test]
481    fn test_resolve_azdls_aliases_share_instance() {
482        let storage = empty_resolving_storage();
483
484        let path_for = |scheme: &str| {
485            format!("{scheme}://myfs@myaccount.dfs.core.windows.net/path/to/file.parquet")
486        };
487
488        // All Azure schemes collapse onto one cached instance.
489        let abfss = storage.resolve(&path_for("abfss")).unwrap();
490        let abfs = storage.resolve(&path_for("abfs")).unwrap();
491
492        assert!(
493            Arc::ptr_eq(&abfss, &abfs),
494            "abfss and abfs should share one instance"
495        );
496    }
497}