Skip to main content

iceberg_storage_opendal/
lib.rs

1// Licensed to the Apache Software Foundation (ASF) under one
2// or more contributor license agreements.  See the NOTICE file
3// distributed with this work for additional information
4// regarding copyright ownership.  The ASF licenses this file
5// to you under the Apache License, Version 2.0 (the
6// "License"); you may not use this file except in compliance
7// with the License.  You may obtain a copy of the License at
8//
9//   http://www.apache.org/licenses/LICENSE-2.0
10//
11// Unless required by applicable law or agreed to in writing,
12// software distributed under the License is distributed on an
13// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14// KIND, either express or implied.  See the License for the
15// specific language governing permissions and limitations
16// under the License.
17
18//! OpenDAL-based storage implementation for Apache Iceberg.
19//!
20//! This crate provides [`OpenDalStorage`] and [`OpenDalStorageFactory`],
21//! which implement the [`Storage`] and
22//! [`StorageFactory`] traits from the `iceberg` crate
23//! using [OpenDAL](https://opendal.apache.org/) as the backend.
24
25mod utils;
26
27use std::collections::HashMap;
28use std::collections::hash_map::Entry;
29use std::num::NonZeroU64;
30use std::sync::Arc;
31use std::time::Duration;
32
33use async_trait::async_trait;
34use bytes::Bytes;
35use cfg_if::cfg_if;
36use futures::StreamExt;
37use futures::stream::BoxStream;
38use iceberg::io::{
39    FileMetadata, FileRead, FileWrite, InputFile, OutputFile, Storage, StorageConfig,
40    StorageFactory,
41};
42use iceberg::{Error, ErrorKind, Result};
43use iceberg_property_macro::Properties;
44use opendal::Operator;
45use opendal::layers::{RetryLayer, TimeoutLayer};
46use opendal::options::WriteOptions;
47use serde::{Deserialize, Serialize};
48use utils::from_opendal_error;
49
50cfg_if! {
51    if #[cfg(feature = "opendal-azdls")] {
52        mod azdls;
53        use azdls::*;
54        use opendal::services::AzdlsConfig;
55    }
56}
57
58cfg_if! {
59    if #[cfg(feature = "opendal-hf")] {
60        mod hf;
61        use hf::*;
62        use opendal::services::HfConfig;
63    }
64}
65
66cfg_if! {
67    if #[cfg(feature = "opendal-fs")] {
68        mod fs;
69        use fs::*;
70    }
71}
72
73cfg_if! {
74    if #[cfg(feature = "opendal-gcs")] {
75        mod gcs;
76        use gcs::*;
77        use opendal::services::GcsConfig;
78    }
79}
80
81cfg_if! {
82    if #[cfg(feature = "opendal-hdfs-native")] {
83        mod hdfs_native;
84        use hdfs_native::*;
85        pub use hdfs_native::HdfsNativeStorage;
86    }
87}
88
89cfg_if! {
90    if #[cfg(feature = "opendal-memory")] {
91        mod memory;
92        use memory::*;
93    }
94}
95
96cfg_if! {
97    if #[cfg(feature = "opendal-oss")] {
98        mod oss;
99        use opendal::services::OssConfig;
100        use oss::*;
101    }
102}
103
104cfg_if! {
105    if #[cfg(feature = "opendal-s3")] {
106        mod s3;
107        use opendal::services::S3Config;
108        pub use s3::*;
109    }
110}
111
112mod resolving;
113pub use resolving::{OpenDalResolvingStorage, OpenDalResolvingStorageFactory};
114
115/// Timeout in milliseconds for each IO call on a reader, writer, lister or deleter, applied per
116/// retry attempt via OpenDAL's `TimeoutLayer::with_io_timeout`. Defaults to 10 seconds.
117///
118/// OpenDAL's separate `timeout`, which bounds whole control operations such as `stat`, is not
119/// affected and stays at its 60-second default.
120pub const OPENDAL_IO_TIMEOUT_MS: &str = "opendal.io-timeout-ms";
121
122/// Number of times to retry a failed operation on any backend, `0` to disable retrying. Unset keeps
123/// OpenDAL's default. For S3, [`S3_RETRY_NUM_RETRIES`](iceberg::io::S3_RETRY_NUM_RETRIES) takes precedence.
124pub const OPENDAL_RETRY_NUM_RETRIES: &str = "opendal.retry.num-retries";
125
126/// Minimum wait in milliseconds before retrying a failed operation on any backend. Unset keeps
127/// OpenDAL's default. For S3, [`S3_RETRY_MIN_WAIT_MS`](iceberg::io::S3_RETRY_MIN_WAIT_MS) takes precedence.
128pub const OPENDAL_RETRY_MIN_WAIT_MS: &str = "opendal.retry.min-wait-ms";
129
130/// Maximum wait in milliseconds between two retries on any backend. Unset keeps OpenDAL's default.
131/// For S3, [`S3_RETRY_MAX_WAIT_MS`](iceberg::io::S3_RETRY_MAX_WAIT_MS) takes precedence.
132pub const OPENDAL_RETRY_MAX_WAIT_MS: &str = "opendal.retry.max-wait-ms";
133
134/// Matches OpenDAL's `TimeoutLayer` default.
135const DEFAULT_IO_TIMEOUT_MS: NonZeroU64 = NonZeroU64::new(10_000).unwrap();
136
137/// Backend-independent client settings shared by every [`OpenDalStorage`] variant.
138#[derive(Clone, Debug, Properties, Serialize, Deserialize)]
139#[serde(default)]
140pub struct OpenDalClientConfig {
141    /// IO timeout in milliseconds.
142    #[property(
143        key = OPENDAL_IO_TIMEOUT_MS,
144        default = DEFAULT_IO_TIMEOUT_MS,
145        parse_with = parse_positive_ms
146    )]
147    io_timeout_ms: NonZeroU64,
148    /// Retries on any backend.
149    #[property(key = OPENDAL_RETRY_NUM_RETRIES, default = None)]
150    retry_num_retries: Option<usize>,
151    /// Delay before the first retry on any backend in milliseconds.
152    #[property(
153        key = OPENDAL_RETRY_MIN_WAIT_MS,
154        default = None,
155        parse_with = parse_positive_ms
156    )]
157    retry_min_wait_ms: Option<NonZeroU64>,
158    /// Upper bound of the delay between two retries on any backend in milliseconds.
159    #[property(
160        key = OPENDAL_RETRY_MAX_WAIT_MS,
161        default = None,
162        parse_with = parse_positive_ms
163    )]
164    retry_max_wait_ms: Option<NonZeroU64>,
165}
166
167impl Default for OpenDalClientConfig {
168    fn default() -> Self {
169        Self {
170            io_timeout_ms: DEFAULT_IO_TIMEOUT_MS,
171            retry_num_retries: None,
172            retry_min_wait_ms: None,
173            retry_max_wait_ms: None,
174        }
175    }
176}
177
178impl OpenDalClientConfig {
179    pub(crate) fn io_timeout(&self) -> Duration {
180        Duration::from_millis(self.io_timeout_ms.get())
181    }
182
183    /// The retry policy, with jitter; OpenDAL's default when no key is set.
184    pub(crate) fn retry_layer(&self) -> RetryLayer {
185        let mut layer = RetryLayer::new();
186        if self.retry_num_retries.is_none()
187            && self.retry_min_wait_ms.is_none()
188            && self.retry_max_wait_ms.is_none()
189        {
190            return layer;
191        }
192        if let Some(times) = self.retry_num_retries {
193            layer = layer.with_max_times(times);
194        }
195        if let Some(ms) = self.retry_min_wait_ms {
196            layer = layer.with_min_delay(Duration::from_millis(ms.get()));
197        }
198        if let Some(ms) = self.retry_max_wait_ms {
199            layer = layer.with_max_delay(Duration::from_millis(ms.get()));
200        }
201        layer.with_jitter()
202    }
203}
204
205fn parse_positive_ms(value: &str) -> Result<NonZeroU64> {
206    value.parse().map_err(|error| {
207        Error::new(
208            ErrorKind::DataInvalid,
209            "Expected a positive integer number of milliseconds",
210        )
211        .with_context("value", format!("{value:?}"))
212        .with_source(error)
213    })
214}
215
216/// OpenDAL-based storage factory.
217///
218/// Maps scheme to the corresponding OpenDalStorage storage variant.
219/// Use this factory with `FileIOBuilder::new(factory)` to create FileIO instances.
220///
221/// # Serialization
222///
223/// The receiving binary must enable the feature corresponding to the serialized backend variant.
224/// For example, deserializing `OpenDalStorageFactory::S3` requires the `opendal-s3` feature.
225///
226/// Serialization fails when the `OpenDalStorageFactory::S3` variant contains a custom AWS
227/// credential loader because the loader holds process-local state that cannot be reconstructed in
228/// another process. Construct the factory without a custom loader before serializing it.
229#[derive(Clone, Debug, Serialize, Deserialize)]
230pub enum OpenDalStorageFactory {
231    /// Memory storage factory.
232    #[cfg(feature = "opendal-memory")]
233    Memory,
234    /// Local filesystem storage factory.
235    #[cfg(feature = "opendal-fs")]
236    Fs,
237    /// S3 storage factory.
238    #[cfg(feature = "opendal-s3")]
239    S3 {
240        /// Custom AWS credential loader.
241        #[serde(
242            skip_deserializing,
243            skip_serializing_if = "Option::is_none",
244            serialize_with = "serialize_custom_credential_loader"
245        )]
246        customized_credential_load: Option<CustomAwsCredentialLoader>,
247    },
248    /// GCS storage factory.
249    #[cfg(feature = "opendal-gcs")]
250    Gcs,
251    /// HDFS storage factory.
252    #[cfg(feature = "opendal-hdfs-native")]
253    HdfsNative,
254    /// OSS storage factory.
255    #[cfg(feature = "opendal-oss")]
256    Oss,
257    /// Azure Data Lake Storage factory.
258    #[cfg(feature = "opendal-azdls")]
259    Azdls,
260    /// HuggingFace Hub storage factory.
261    #[cfg(feature = "opendal-hf")]
262    Hf,
263}
264
265#[cfg(feature = "opendal-s3")]
266pub(crate) fn serialize_custom_credential_loader<S>(
267    _loader: &Option<CustomAwsCredentialLoader>,
268    _serializer: S,
269) -> std::result::Result<S::Ok, S::Error>
270where
271    S: serde::Serializer,
272{
273    Err(serde::ser::Error::custom(
274        "custom AWS credential loaders cannot be serialized",
275    ))
276}
277
278#[typetag::serde(name = "OpenDalStorageFactory")]
279impl StorageFactory for OpenDalStorageFactory {
280    #[allow(unused_variables)]
281    fn build(&self, config: &StorageConfig) -> Result<Arc<dyn Storage>> {
282        let client_config = OpenDalClientConfig::from_properties(config.props())?;
283        match self {
284            #[cfg(feature = "opendal-memory")]
285            OpenDalStorageFactory::Memory => Ok(Arc::new(OpenDalStorage::Memory {
286                operator: memory_config_build()?,
287                client_config,
288            })),
289            #[cfg(feature = "opendal-fs")]
290            OpenDalStorageFactory::Fs => Ok(Arc::new(OpenDalStorage::LocalFs { client_config })),
291            #[cfg(feature = "opendal-s3")]
292            OpenDalStorageFactory::S3 {
293                customized_credential_load,
294            } => Ok(Arc::new(OpenDalStorage::S3 {
295                config: s3_config_parse(config.props().clone())?.into(),
296                multipart_part_size: s3_multipart_part_size_parse(config.props())?,
297                customized_credential_load: customized_credential_load.clone(),
298                client_config: s3_client_config(config.props(), client_config)?,
299            })),
300            #[cfg(feature = "opendal-gcs")]
301            OpenDalStorageFactory::Gcs => Ok(Arc::new(OpenDalStorage::Gcs {
302                config: gcs_config_parse(config.props().clone())?.into(),
303                client_config,
304            })),
305            #[cfg(feature = "opendal-hdfs-native")]
306            OpenDalStorageFactory::HdfsNative => Ok(Arc::new(OpenDalStorage::HdfsNative(
307                HdfsNativeStorage::new(
308                    hdfs_native_config_parse(config.props().clone())?,
309                    client_config,
310                ),
311            ))),
312            #[cfg(feature = "opendal-oss")]
313            OpenDalStorageFactory::Oss => Ok(Arc::new(OpenDalStorage::Oss {
314                config: oss_config_parse(config.props().clone())?.into(),
315                client_config,
316            })),
317            #[cfg(feature = "opendal-azdls")]
318            OpenDalStorageFactory::Azdls => Ok(Arc::new(OpenDalStorage::Azdls {
319                config: azdls_config_parse(config.props().clone())?.into(),
320                client_config,
321            })),
322            #[cfg(feature = "opendal-hf")]
323            OpenDalStorageFactory::Hf => Ok(Arc::new(OpenDalStorage::Hf {
324                config: hf_config_parse(config.props().clone())?.into(),
325                client_config,
326            })),
327            #[cfg(all(
328                not(feature = "opendal-memory"),
329                not(feature = "opendal-fs"),
330                not(feature = "opendal-s3"),
331                not(feature = "opendal-gcs"),
332                not(feature = "opendal-oss"),
333                not(feature = "opendal-azdls"),
334                not(feature = "opendal-hf"),
335                not(feature = "opendal-hdfs-native"),
336            ))]
337            _ => Err(Error::new(
338                ErrorKind::FeatureUnsupported,
339                "No storage service has been enabled",
340            )),
341        }
342    }
343}
344
345/// Default memory operator for serde deserialization.
346#[cfg(feature = "opendal-memory")]
347fn default_memory_operator() -> Operator {
348    memory_config_build().expect("Failed to create default memory operator")
349}
350
351/// OpenDAL-based storage implementation.
352///
353/// The serialized representation is not a stable format and may change between crate versions.
354#[derive(Clone, Debug, Serialize, Deserialize)]
355pub enum OpenDalStorage {
356    /// Memory storage variant.
357    #[cfg(feature = "opendal-memory")]
358    Memory {
359        /// Pre-built memory operator.
360        #[serde(skip, default = "self::default_memory_operator")]
361        operator: Operator,
362        /// Backend-independent client settings.
363        #[serde(default)]
364        client_config: OpenDalClientConfig,
365    },
366    /// Local filesystem storage variant.
367    #[cfg(feature = "opendal-fs")]
368    LocalFs {
369        /// Backend-independent client settings.
370        #[serde(default)]
371        client_config: OpenDalClientConfig,
372    },
373    /// S3 storage variant.
374    ///
375    /// Accepts any S3-family URL (`s3://`, `s3a://`, `s3n://`); the scheme is
376    /// derived from the path at call time.
377    #[cfg(feature = "opendal-s3")]
378    #[non_exhaustive]
379    S3 {
380        /// S3 configuration.
381        config: Arc<S3Config>,
382        /// Bytes carried by one multipart upload request.
383        #[serde(default = "default_multipart_part_size")]
384        multipart_part_size: u64,
385        /// Custom AWS credential loader.
386        #[serde(skip)]
387        customized_credential_load: Option<CustomAwsCredentialLoader>,
388        /// Backend-independent client settings.
389        #[serde(default)]
390        client_config: OpenDalClientConfig,
391    },
392    /// GCS storage variant.
393    #[cfg(feature = "opendal-gcs")]
394    Gcs {
395        /// GCS configuration.
396        config: Arc<GcsConfig>,
397        /// Backend-independent client settings.
398        #[serde(default)]
399        client_config: OpenDalClientConfig,
400    },
401    /// HDFS storage variant; see [`HdfsNativeStorage`].
402    #[cfg(feature = "opendal-hdfs-native")]
403    HdfsNative(HdfsNativeStorage),
404    /// OSS storage variant.
405    #[cfg(feature = "opendal-oss")]
406    Oss {
407        /// OSS configuration.
408        config: Arc<OssConfig>,
409        /// Backend-independent client settings.
410        #[serde(default)]
411        client_config: OpenDalClientConfig,
412    },
413    /// Azure Data Lake Storage variant.
414    ///
415    /// Accepts paths of the form
416    /// `abfs[s]://<filesystem>@<account>.dfs.<endpoint-suffix>/<path>` or
417    /// `wasb[s]://<container>@<account>.blob.<endpoint-suffix>/<path>`.
418    /// The scheme is derived from the path at call time.
419    #[cfg(feature = "opendal-azdls")]
420    Azdls {
421        /// Azure DLS configuration.
422        config: Arc<AzdlsConfig>,
423        /// Backend-independent client settings.
424        #[serde(default)]
425        client_config: OpenDalClientConfig,
426    },
427    /// HuggingFace Hub storage variant.
428    ///
429    /// Accepts paths of the form
430    /// `hf://<repo_type>/<owner>/<repo>[@<revision>]/<path_in_repo>`,
431    /// where `<repo_type>` must be one of `models`, `datasets`, `spaces`, or `buckets`.
432    #[cfg(feature = "opendal-hf")]
433    Hf {
434        /// HuggingFace Hub configuration (token + endpoint).
435        config: Arc<HfConfig>,
436        /// Backend-independent client settings.
437        #[serde(default)]
438        client_config: OpenDalClientConfig,
439    },
440}
441
442impl OpenDalStorage {
443    /// Creates operator from path.
444    ///
445    /// # Arguments
446    ///
447    /// * path: It should be *absolute* path starting with scheme string used to construct [`FileIO`](iceberg::io::FileIO).
448    ///
449    /// # Returns
450    ///
451    /// The return value consists of two parts:
452    ///
453    /// * An [`opendal::Operator`] instance used to operate on file.
454    /// * Relative path to the root uri of [`opendal::Operator`].
455    #[allow(unreachable_code, unused_variables)]
456    pub(crate) fn create_operator<'a>(
457        &self,
458        path: &'a impl AsRef<str>,
459    ) -> Result<(Operator, &'a str)> {
460        let path = path.as_ref();
461        let (operator, relative_path): (Operator, &str) = match self {
462            #[cfg(feature = "opendal-memory")]
463            OpenDalStorage::Memory { operator: op, .. } => {
464                if let Some(stripped) = path.strip_prefix("memory:/") {
465                    (op.clone(), stripped)
466                } else {
467                    (op.clone(), &path[1..])
468                }
469            }
470            #[cfg(feature = "opendal-fs")]
471            OpenDalStorage::LocalFs { .. } => {
472                let op = fs_config_build()?;
473                if let Some(stripped) = path.strip_prefix("file:/") {
474                    (op, stripped)
475                } else {
476                    (op, &path[1..])
477                }
478            }
479            #[cfg(feature = "opendal-s3")]
480            OpenDalStorage::S3 {
481                config,
482                customized_credential_load,
483                ..
484            } => {
485                let op = s3_config_build(config, customized_credential_load, path)?;
486                let op_info = op.info();
487
488                // Use the URL scheme in the path for prefix matching. This enables
489                // use of S3-compatible storage backends using custom schemes (e.g., `minio://`, `r2://`).
490                let url = url::Url::parse(path).map_err(|e| {
491                    Error::new(
492                        ErrorKind::DataInvalid,
493                        format!("Invalid s3 url: {path}: {e}"),
494                    )
495                })?;
496                let prefix = format!("{}://{}/", url.scheme(), op_info.name());
497                if path.starts_with(&prefix) {
498                    (op, &path[prefix.len()..])
499                } else {
500                    return Err(Error::new(
501                        ErrorKind::DataInvalid,
502                        format!("Invalid s3 url: {path}, should start with {prefix}"),
503                    ));
504                }
505            }
506            #[cfg(feature = "opendal-gcs")]
507            OpenDalStorage::Gcs { config, .. } => {
508                let operator = gcs_config_build(config, path)?;
509                let prefix = format!("gs://{}/", operator.info().name());
510                if path.starts_with(&prefix) {
511                    (operator, &path[prefix.len()..])
512                } else {
513                    return Err(Error::new(
514                        ErrorKind::DataInvalid,
515                        format!("Invalid gcs url: {path}, should start with {prefix}"),
516                    ));
517                }
518            }
519            #[cfg(feature = "opendal-hdfs-native")]
520            OpenDalStorage::HdfsNative(storage) => {
521                hdfs_native_create_operator(path, &storage.config, &storage.operators)?
522            }
523            #[cfg(feature = "opendal-oss")]
524            OpenDalStorage::Oss { config, .. } => {
525                let op = oss_config_build(config, path)?;
526                let prefix = format!("oss://{}/", op.info().name());
527                if path.starts_with(&prefix) {
528                    (op, &path[prefix.len()..])
529                } else {
530                    return Err(Error::new(
531                        ErrorKind::DataInvalid,
532                        format!("Invalid oss url: {path}, should start with {prefix}"),
533                    ));
534                }
535            }
536            #[cfg(feature = "opendal-azdls")]
537            OpenDalStorage::Azdls { config, .. } => azdls_create_operator(path, config)?,
538            #[cfg(feature = "opendal-hf")]
539            OpenDalStorage::Hf { config, .. } => hf_config_build(config, path)?,
540            #[cfg(all(
541                not(feature = "opendal-s3"),
542                not(feature = "opendal-fs"),
543                not(feature = "opendal-gcs"),
544                not(feature = "opendal-oss"),
545                not(feature = "opendal-azdls"),
546                not(feature = "opendal-hf"),
547                not(feature = "opendal-hdfs-native"),
548            ))]
549            _ => {
550                return Err(Error::new(
551                    ErrorKind::FeatureUnsupported,
552                    "No storage service has been enabled",
553                ));
554            }
555        };
556
557        // Apply observability/resilience layers. TimeoutLayer must be
558        // inside RetryLayer so each retry attempt is independently
559        // bounded — without a per-attempt timeout, a future parked on a
560        // silently dropped TCP connection never produces an `Err` and
561        // RetryLayer cannot retry, leaving the caller hung indefinitely.
562        // See: https://opendal.apache.org/docs/rust/opendal/layers/struct.TimeoutLayer.html
563        //
564        // Transient errors are common for object stores; we retry temporary
565        // failures with exponential backoff. The retry behavior also
566        // benefits non-object-store backends.
567        let operator = operator
568            .layer(TimeoutLayer::new().with_io_timeout(self.client_config().io_timeout()))
569            .layer(self.client_config().retry_layer());
570        Ok((operator, relative_path))
571    }
572
573    pub(crate) fn client_config(&self) -> &OpenDalClientConfig {
574        match self {
575            #[cfg(feature = "opendal-memory")]
576            OpenDalStorage::Memory { client_config, .. } => client_config,
577            #[cfg(feature = "opendal-fs")]
578            OpenDalStorage::LocalFs { client_config } => client_config,
579            #[cfg(feature = "opendal-s3")]
580            OpenDalStorage::S3 { client_config, .. } => client_config,
581            #[cfg(feature = "opendal-gcs")]
582            OpenDalStorage::Gcs { client_config, .. } => client_config,
583            #[cfg(feature = "opendal-oss")]
584            OpenDalStorage::Oss { client_config, .. } => client_config,
585            #[cfg(feature = "opendal-azdls")]
586            OpenDalStorage::Azdls { client_config, .. } => client_config,
587            #[cfg(feature = "opendal-hf")]
588            OpenDalStorage::Hf { client_config, .. } => client_config,
589            #[cfg(feature = "opendal-hdfs-native")]
590            OpenDalStorage::HdfsNative(storage) => &storage.client_config,
591            #[cfg(all(
592                not(feature = "opendal-memory"),
593                not(feature = "opendal-s3"),
594                not(feature = "opendal-fs"),
595                not(feature = "opendal-gcs"),
596                not(feature = "opendal-oss"),
597                not(feature = "opendal-azdls"),
598                not(feature = "opendal-hf"),
599                not(feature = "opendal-hdfs-native"),
600            ))]
601            _ => unreachable!(),
602        }
603    }
604
605    /// Bounds the S3 request size. Without a chunk size OpenDAL turns each caller
606    /// buffer into one request, and `ParquetWriter` hands over a whole row group
607    /// at a time. Other backends keep OpenDAL's defaults.
608    #[allow(unreachable_patterns)]
609    fn write_options(&self) -> WriteOptions {
610        match self {
611            #[cfg(feature = "opendal-s3")]
612            OpenDalStorage::S3 {
613                multipart_part_size,
614                ..
615            } => WriteOptions {
616                chunk: Some(usize::try_from(*multipart_part_size).unwrap_or(usize::MAX)),
617                ..WriteOptions::default()
618            },
619            _ => WriteOptions::default(),
620        }
621    }
622
623    /// Returns a cache key used by `delete_stream` to group paths by storage operator.
624    ///
625    /// For most backends the URL host (bucket name) is sufficient. For HF the host
626    /// encodes the repo type, not the repo identity, so a more specific key is used.
627    fn batch_key_for_path(&self, path: &str) -> String {
628        match self {
629            #[cfg(feature = "opendal-hf")]
630            OpenDalStorage::Hf { .. } => hf_batch_key(path),
631            // The URL host alone would merge distinct NameNodes that differ
632            // only by port; key by the effective NameNode instead.
633            #[cfg(feature = "opendal-hdfs-native")]
634            OpenDalStorage::HdfsNative(storage) => hdfs_native_batch_key(&storage.config, path),
635            _ => url::Url::parse(path)
636                .ok()
637                .and_then(|u| u.host_str().map(|s| s.to_string()))
638                .unwrap_or_default(),
639        }
640    }
641
642    /// Extracts the relative path from an absolute path without building an operator.
643    ///
644    /// This is a lightweight alternative to [`create_operator`](Self::create_operator) for cases
645    /// where only the relative path is needed (e.g. bulk deletes where the operator is already
646    /// available).
647    #[allow(unreachable_code, unused_variables)]
648    pub(crate) fn relativize_path<'a>(&self, path: &'a str) -> Result<&'a str> {
649        match self {
650            #[cfg(feature = "opendal-memory")]
651            OpenDalStorage::Memory { .. } => {
652                Ok(path.strip_prefix("memory:/").unwrap_or(&path[1..]))
653            }
654            #[cfg(feature = "opendal-fs")]
655            OpenDalStorage::LocalFs { .. } => Ok(path.strip_prefix("file:/").unwrap_or(&path[1..])),
656            #[cfg(feature = "opendal-s3")]
657            OpenDalStorage::S3 { .. } => {
658                let url = url::Url::parse(path)?;
659                let bucket = url.host_str().ok_or_else(|| {
660                    Error::new(
661                        ErrorKind::DataInvalid,
662                        format!("Invalid s3 url: {path}, missing bucket"),
663                    )
664                })?;
665                let prefix = format!("{}://{}/", url.scheme(), bucket);
666                if path.starts_with(&prefix) {
667                    Ok(&path[prefix.len()..])
668                } else {
669                    Err(Error::new(
670                        ErrorKind::DataInvalid,
671                        format!("Invalid s3 url: {path}, should start with {prefix}"),
672                    ))
673                }
674            }
675            #[cfg(feature = "opendal-gcs")]
676            OpenDalStorage::Gcs { .. } => {
677                let url = url::Url::parse(path)?;
678                let bucket = url.host_str().ok_or_else(|| {
679                    Error::new(
680                        ErrorKind::DataInvalid,
681                        format!("Invalid gcs url: {path}, missing bucket"),
682                    )
683                })?;
684                let prefix = format!("gs://{}/", bucket);
685                if path.starts_with(&prefix) {
686                    Ok(&path[prefix.len()..])
687                } else {
688                    Err(Error::new(
689                        ErrorKind::DataInvalid,
690                        format!("Invalid gcs url: {path}, should start with {prefix}"),
691                    ))
692                }
693            }
694            #[cfg(feature = "opendal-hdfs-native")]
695            OpenDalStorage::HdfsNative(storage) => {
696                let (_, relative_path) = hdfs_native_effective_name_node(&storage.config, path)?;
697                Ok(relative_path)
698            }
699            #[cfg(feature = "opendal-oss")]
700            OpenDalStorage::Oss { .. } => {
701                let url = url::Url::parse(path)?;
702                let bucket = url.host_str().ok_or_else(|| {
703                    Error::new(
704                        ErrorKind::DataInvalid,
705                        format!("Invalid oss url: {path}, missing bucket"),
706                    )
707                })?;
708                let prefix = format!("oss://{}/", bucket);
709                if path.starts_with(&prefix) {
710                    Ok(&path[prefix.len()..])
711                } else {
712                    Err(Error::new(
713                        ErrorKind::DataInvalid,
714                        format!("Invalid oss url: {path}, should start with {prefix}"),
715                    ))
716                }
717            }
718            #[cfg(feature = "opendal-azdls")]
719            OpenDalStorage::Azdls { config, .. } => {
720                let azure_path = path.parse::<AzureStoragePath>()?;
721                match_path_with_config(&azure_path, config)?;
722                Ok(azure_path.relative_path(path))
723            }
724            #[cfg(feature = "opendal-hf")]
725            OpenDalStorage::Hf { .. } => {
726                let parsed = HfUri::parse(path).ok_or_else(|| {
727                    Error::new(ErrorKind::DataInvalid, format!("Invalid hf url: {path}"))
728                })?;
729                Ok(&path[path.len() - parsed.path.len()..])
730            }
731            #[cfg(all(
732                not(feature = "opendal-s3"),
733                not(feature = "opendal-fs"),
734                not(feature = "opendal-gcs"),
735                not(feature = "opendal-oss"),
736                not(feature = "opendal-azdls"),
737                not(feature = "opendal-hf"),
738                not(feature = "opendal-hdfs-native"),
739            ))]
740            _ => Err(Error::new(
741                ErrorKind::FeatureUnsupported,
742                "No storage service has been enabled",
743            )),
744        }
745    }
746}
747
748#[typetag::serde(name = "OpenDalStorage")]
749#[async_trait]
750impl Storage for OpenDalStorage {
751    async fn exists(&self, path: &str) -> Result<bool> {
752        let (op, relative_path) = self.create_operator(&path)?;
753        Ok(op.exists(relative_path).await.map_err(from_opendal_error)?)
754    }
755
756    async fn metadata(&self, path: &str) -> Result<FileMetadata> {
757        let (op, relative_path) = self.create_operator(&path)?;
758        let meta = op.stat(relative_path).await.map_err(from_opendal_error)?;
759        Ok(FileMetadata {
760            size: meta.content_length(),
761        })
762    }
763
764    async fn read(&self, path: &str) -> Result<Bytes> {
765        let (op, relative_path) = self.create_operator(&path)?;
766        Ok(op
767            .read(relative_path)
768            .await
769            .map_err(from_opendal_error)?
770            .to_bytes())
771    }
772
773    async fn reader(&self, path: &str) -> Result<Box<dyn FileRead>> {
774        let (op, relative_path) = self.create_operator(&path)?;
775        Ok(Box::new(OpenDalReader(
776            op.reader(relative_path).await.map_err(from_opendal_error)?,
777        )))
778    }
779
780    async fn write(&self, path: &str, bs: Bytes) -> Result<()> {
781        let (op, relative_path) = self.create_operator(&path)?;
782        op.write_options(relative_path, bs, self.write_options())
783            .await
784            .map_err(from_opendal_error)?;
785        Ok(())
786    }
787
788    async fn writer(&self, path: &str) -> Result<Box<dyn FileWrite>> {
789        let (op, relative_path) = self.create_operator(&path)?;
790        Ok(Box::new(OpenDalWriter::new(
791            op.writer_options(relative_path, self.write_options())
792                .await
793                .map_err(from_opendal_error)?,
794        )))
795    }
796
797    async fn delete(&self, path: &str) -> Result<()> {
798        let (op, relative_path) = self.create_operator(&path)?;
799        Ok(op.delete(relative_path).await.map_err(from_opendal_error)?)
800    }
801
802    async fn delete_prefix(&self, path: &str) -> Result<()> {
803        let (op, relative_path) = self.create_operator(&path)?;
804        let path = if relative_path.ends_with('/') {
805            relative_path.to_string()
806        } else {
807            format!("{relative_path}/")
808        };
809        Ok(op
810            .delete_with(&path)
811            .recursive(true)
812            .await
813            .map_err(from_opendal_error)?)
814    }
815
816    async fn delete_stream(&self, mut paths: BoxStream<'static, String>) -> Result<()> {
817        let mut deleters: HashMap<String, opendal::Deleter> = HashMap::new();
818
819        while let Some(path) = paths.next().await {
820            let bucket = self.batch_key_for_path(&path);
821
822            let (relative_path, deleter) = match deleters.entry(bucket) {
823                Entry::Occupied(entry) => {
824                    (self.relativize_path(&path)?.to_string(), entry.into_mut())
825                }
826                Entry::Vacant(entry) => {
827                    let (op, rel) = self.create_operator(&path)?;
828                    let rel = rel.to_string();
829                    let deleter = op.deleter().await.map_err(from_opendal_error)?;
830                    (rel, entry.insert(deleter))
831                }
832            };
833
834            deleter
835                .delete(relative_path)
836                .await
837                .map_err(from_opendal_error)?;
838        }
839
840        for (_, mut deleter) in deleters {
841            deleter.close().await.map_err(from_opendal_error)?;
842        }
843
844        Ok(())
845    }
846
847    #[allow(unreachable_code, unused_variables)]
848    fn new_input(&self, path: &str) -> Result<InputFile> {
849        Ok(InputFile::new(Arc::new(self.clone()), path.to_string()))
850    }
851
852    #[allow(unreachable_code, unused_variables)]
853    fn new_output(&self, path: &str) -> Result<OutputFile> {
854        Ok(OutputFile::new(Arc::new(self.clone()), path.to_string()))
855    }
856}
857
858// Newtype wrappers for opendal types to satisfy orphan rules.
859// We can't implement iceberg's FileRead/FileWrite traits directly on opendal's
860// Reader/Writer since neither trait nor type is defined in this crate.
861
862/// Wrapper around `opendal::Reader` that implements `FileRead`.
863pub(crate) struct OpenDalReader(pub(crate) opendal::Reader);
864
865#[async_trait]
866impl FileRead for OpenDalReader {
867    async fn read(&self, range: std::ops::Range<u64>) -> Result<Bytes> {
868        Ok(opendal::Reader::read(&self.0, range)
869            .await
870            .map_err(from_opendal_error)?
871            .to_bytes())
872    }
873}
874
875/// Wrapper around `opendal::Writer` that implements `FileWrite`.
876pub(crate) struct OpenDalWriter {
877    inner: opendal::Writer,
878    bytes_written: u64,
879}
880
881impl OpenDalWriter {
882    pub(crate) fn new(inner: opendal::Writer) -> Self {
883        Self {
884            inner,
885            bytes_written: 0,
886        }
887    }
888}
889
890#[async_trait]
891impl FileWrite for OpenDalWriter {
892    async fn write(&mut self, bs: Bytes) -> Result<()> {
893        let len = bs.len() as u64;
894        opendal::Writer::write(&mut self.inner, bs)
895            .await
896            .map_err(from_opendal_error)?;
897        self.bytes_written += len;
898        Ok(())
899    }
900
901    async fn close(&mut self) -> Result<FileMetadata> {
902        let metadata = opendal::Writer::close(&mut self.inner)
903            .await
904            .map_err(from_opendal_error)?;
905
906        // Object stores may omit the size (reported as 0); validate only a reported nonzero size.
907        let reported_size = metadata.content_length();
908        if reported_size != 0 && reported_size != self.bytes_written {
909            return Err(Error::new(
910                ErrorKind::Unexpected,
911                format!(
912                    "Wrote {} bytes but storage reports {reported_size}",
913                    self.bytes_written
914                ),
915            ));
916        }
917
918        Ok(FileMetadata {
919            size: self.bytes_written,
920        })
921    }
922}
923
924#[cfg(test)]
925mod tests {
926    use super::*;
927
928    fn client_config(value: &str) -> Result<OpenDalClientConfig> {
929        OpenDalClientConfig::from_properties(&HashMap::from([(
930            OPENDAL_IO_TIMEOUT_MS.to_string(),
931            value.to_string(),
932        )]))
933    }
934
935    #[test]
936    fn test_io_timeout_parsing() {
937        let unset = OpenDalClientConfig::from_properties(&HashMap::new()).unwrap();
938        assert_eq!(
939            unset.io_timeout(),
940            Duration::from_millis(DEFAULT_IO_TIMEOUT_MS.get())
941        );
942
943        let max = u64::MAX.to_string();
944        for (valid, ms) in [("45000", 45_000), ("1", 1), (max.as_str(), u64::MAX)] {
945            assert_eq!(
946                client_config(valid).unwrap().io_timeout(),
947                Duration::from_millis(ms),
948                "{valid}"
949            );
950        }
951
952        for invalid in ["0", "-1", "12.5", "abc", "", "18446744073709551616"] {
953            let err = client_config(invalid).unwrap_err().to_string();
954            assert!(err.contains(OPENDAL_IO_TIMEOUT_MS), "{invalid}");
955            assert!(err.contains(&format!("value: {invalid:?}")), "{err}");
956            let reason = invalid.parse::<NonZeroU64>().unwrap_err().to_string();
957            assert!(err.contains(&reason), "{err}");
958        }
959    }
960
961    #[test]
962    fn test_default_timeouts_match_opendal() {
963        // `TimeoutLayer` has no getters, so compare through `Debug`.
964        let opendal_default = format!("{:?}", TimeoutLayer::new());
965        let layer = |timeout, io_timeout| {
966            format!(
967                "{:?}",
968                TimeoutLayer::new()
969                    .with_timeout(timeout)
970                    .with_io_timeout(io_timeout)
971            )
972        };
973        let control = Duration::from_secs(60);
974        let io = Duration::from_millis(DEFAULT_IO_TIMEOUT_MS.get());
975        assert_eq!(opendal_default, layer(control, io));
976        assert_ne!(opendal_default, layer(control + Duration::from_secs(1), io));
977        assert_ne!(
978            opendal_default,
979            layer(control, io + Duration::from_millis(1))
980        );
981    }
982
983    fn retry_config(props: &[(&str, &str)]) -> Result<OpenDalClientConfig> {
984        OpenDalClientConfig::from_properties(
985            &props
986                .iter()
987                .map(|(k, v)| (k.to_string(), v.to_string()))
988                .collect::<HashMap<_, _>>(),
989        )
990    }
991
992    fn retry_layer_debug(props: &[(&str, &str)]) -> Result<String> {
993        Ok(format!("{:?}", retry_config(props)?.retry_layer()))
994    }
995
996    #[test]
997    fn test_retry_defaults_match_opendal() {
998        // `RetryLayer` has no getters, so compare through `Debug`.
999        assert_eq!(
1000            retry_layer_debug(&[]).unwrap(),
1001            format!("{:?}", RetryLayer::new())
1002        );
1003        assert_ne!(
1004            format!("{:?}", RetryLayer::new()),
1005            format!("{:?}", RetryLayer::new().with_jitter())
1006        );
1007    }
1008
1009    #[test]
1010    fn test_retry_properties_reach_retry_layer() {
1011        let expected = RetryLayer::new()
1012            .with_max_times(5)
1013            .with_min_delay(Duration::from_millis(100))
1014            .with_max_delay(Duration::from_millis(200))
1015            .with_jitter();
1016        assert_eq!(
1017            retry_layer_debug(&[
1018                (OPENDAL_RETRY_NUM_RETRIES, "5"),
1019                (OPENDAL_RETRY_MIN_WAIT_MS, "100"),
1020                (OPENDAL_RETRY_MAX_WAIT_MS, "200"),
1021            ])
1022            .unwrap(),
1023            format!("{expected:?}")
1024        );
1025
1026        // any one property switches jitter on and leaves the others at OpenDAL's defaults
1027        assert_eq!(
1028            retry_layer_debug(&[(OPENDAL_RETRY_NUM_RETRIES, "0")]).unwrap(),
1029            format!("{:?}", RetryLayer::new().with_max_times(0).with_jitter())
1030        );
1031    }
1032
1033    #[test]
1034    fn test_retry_property_parsing_rejects_invalid_values() {
1035        for (key, invalid) in [
1036            (OPENDAL_RETRY_NUM_RETRIES, "-1"),
1037            (OPENDAL_RETRY_NUM_RETRIES, "abc"),
1038            (OPENDAL_RETRY_NUM_RETRIES, ""),
1039            (OPENDAL_RETRY_MIN_WAIT_MS, "0"),
1040            (OPENDAL_RETRY_MIN_WAIT_MS, "abc"),
1041            (OPENDAL_RETRY_MAX_WAIT_MS, "0"),
1042            (OPENDAL_RETRY_MAX_WAIT_MS, "12.5"),
1043        ] {
1044            let err = retry_config(&[(key, invalid)]).unwrap_err().to_string();
1045            assert!(err.contains(key), "{key}={invalid}: {err}");
1046        }
1047    }
1048
1049    #[cfg(feature = "opendal-s3")]
1050    #[test]
1051    fn test_client_config_serde_round_trip() {
1052        let storage = OpenDalStorage::S3 {
1053            config: Arc::new(S3Config::default()),
1054            multipart_part_size: default_multipart_part_size(),
1055            customized_credential_load: None,
1056            client_config: client_config("45000").unwrap(),
1057        };
1058
1059        let mut value = serde_json::to_value(&storage).unwrap();
1060        let restored: OpenDalStorage = serde_json::from_value(value.clone()).unwrap();
1061        assert_eq!(
1062            restored.client_config().io_timeout(),
1063            Duration::from_secs(45)
1064        );
1065
1066        let mut zero = value.clone();
1067        zero["S3"]["client_config"]["io_timeout_ms"] = 0.into();
1068        assert!(serde_json::from_value::<OpenDalStorage>(zero).is_err());
1069
1070        value["S3"].as_object_mut().unwrap().remove("client_config");
1071        let restored: OpenDalStorage = serde_json::from_value(value).unwrap();
1072        assert_eq!(
1073            restored.client_config().io_timeout(),
1074            Duration::from_millis(DEFAULT_IO_TIMEOUT_MS.get())
1075        );
1076    }
1077
1078    #[cfg(all(feature = "opendal-fs", feature = "opendal-memory"))]
1079    #[test]
1080    fn test_old_unit_variant_forms_are_rejected() {
1081        for old in [r#""LocalFs""#, r#""Memory""#, r#"{"Memory":null}"#] {
1082            assert!(
1083                serde_json::from_str::<OpenDalStorage>(old).is_err(),
1084                "{old}"
1085            );
1086        }
1087    }
1088
1089    #[cfg(feature = "opendal-memory")]
1090    #[test]
1091    fn test_factory_rejects_invalid_io_timeout() {
1092        let config = StorageConfig::new().with_prop(OPENDAL_IO_TIMEOUT_MS, "nope");
1093
1094        let err = OpenDalStorageFactory::Memory.build(&config).unwrap_err();
1095        assert!(err.to_string().contains(OPENDAL_IO_TIMEOUT_MS));
1096    }
1097
1098    #[test]
1099    fn test_factory_propagates_io_timeout() {
1100        let config = StorageConfig::new().with_prop(OPENDAL_IO_TIMEOUT_MS, "45000");
1101        let factories = [
1102            #[cfg(feature = "opendal-memory")]
1103            OpenDalStorageFactory::Memory,
1104            #[cfg(feature = "opendal-fs")]
1105            OpenDalStorageFactory::Fs,
1106            #[cfg(feature = "opendal-s3")]
1107            OpenDalStorageFactory::S3 {
1108                customized_credential_load: None,
1109            },
1110            #[cfg(feature = "opendal-gcs")]
1111            OpenDalStorageFactory::Gcs,
1112            #[cfg(feature = "opendal-oss")]
1113            OpenDalStorageFactory::Oss,
1114            #[cfg(feature = "opendal-azdls")]
1115            OpenDalStorageFactory::Azdls,
1116            #[cfg(feature = "opendal-hf")]
1117            OpenDalStorageFactory::Hf,
1118            #[cfg(feature = "opendal-hdfs-native")]
1119            OpenDalStorageFactory::HdfsNative,
1120        ];
1121        for factory in factories {
1122            // `build` returns `dyn Storage`, so read the config back from its serialized form.
1123            let storage = factory.build(&config).unwrap();
1124            let json = serde_json::to_value(&*storage).unwrap();
1125            let client_config = json
1126                .as_object()
1127                .unwrap()
1128                .values()
1129                .find_map(|variant| variant.get("client_config"))
1130                .unwrap_or_else(|| panic!("{json}"));
1131            assert_eq!(client_config["io_timeout_ms"], 45_000, "{factory:?}");
1132        }
1133    }
1134
1135    #[cfg(feature = "opendal-memory")]
1136    #[tokio::test(start_paused = true)]
1137    async fn test_io_timeout_reaches_timeout_layer() {
1138        use opendal::layers::ConcurrentLimitLayer;
1139
1140        // A zero concurrency limit stalls every IO call; paused time skips the waits.
1141        let storage = OpenDalStorage::Memory {
1142            operator: default_memory_operator().layer(ConcurrentLimitLayer::new(0)),
1143            client_config: client_config("45000").unwrap(),
1144        };
1145
1146        let err = storage
1147            .read("memory:/stalled")
1148            .await
1149            .unwrap_err()
1150            .to_string();
1151        assert!(err.contains("io operation timeout reached"), "{err}");
1152        assert!(err.contains("{ timeout: 45 }"), "{err}");
1153    }
1154
1155    #[cfg(feature = "opendal-s3")]
1156    #[derive(Debug)]
1157    struct EmptyCredentialLoader;
1158
1159    #[cfg(feature = "opendal-s3")]
1160    impl ProvideCredential for EmptyCredentialLoader {
1161        type Credential = AwsCredential;
1162
1163        async fn provide_credential(
1164            &self,
1165            _ctx: &reqsign_core::Context,
1166        ) -> reqsign_core::Result<Option<AwsCredential>> {
1167            Ok(None)
1168        }
1169    }
1170
1171    #[cfg(feature = "opendal-s3")]
1172    #[test]
1173    fn test_s3_factory_custom_credential_loader_serialization_fails() {
1174        let file_io = iceberg::io::FileIOBuilder::new(Arc::new(OpenDalStorageFactory::S3 {
1175            customized_credential_load: Some(CustomAwsCredentialLoader::new(EmptyCredentialLoader)),
1176        }))
1177        .build();
1178
1179        let err = file_io.serialize_all().unwrap_err();
1180        assert!(
1181            err.to_string()
1182                .contains("custom AWS credential loaders cannot be serialized")
1183        );
1184    }
1185
1186    #[cfg(feature = "opendal-memory")]
1187    #[test]
1188    fn test_default_memory_operator() {
1189        let op = default_memory_operator();
1190        assert_eq!(op.info().scheme().to_string(), "memory");
1191    }
1192
1193    #[cfg(feature = "opendal-memory")]
1194    #[tokio::test]
1195    async fn test_writer_close_returns_stored_size() {
1196        use iceberg::encryption::{EncryptedOutputFile, StandardKeyMetadata};
1197
1198        // Note: the memory service does report a content length, so this only pins the happy
1199        // path. The counter in `OpenDalWriter` is what covers services that don't, such as S3.
1200        let storage = Arc::new(OpenDalStorage::Memory {
1201            operator: default_memory_operator(),
1202            client_config: OpenDalClientConfig::default(),
1203        });
1204        let path = "memory:///stored-size";
1205        for plaintext in [
1206            Bytes::new(),
1207            Bytes::from_static(b"test data"),
1208            Bytes::from(vec![7; 3 * 1024]),
1209        ] {
1210            let mut writer = storage.writer(path).await.unwrap();
1211            for chunk in plaintext.chunks(1024) {
1212                writer.write(Bytes::copy_from_slice(chunk)).await.unwrap();
1213            }
1214            let metadata = writer.close().await.unwrap();
1215            assert_eq!(metadata.size, plaintext.len() as u64);
1216            assert_eq!(metadata.size, storage.metadata(path).await.unwrap().size);
1217
1218            let output = EncryptedOutputFile::new(
1219                OutputFile::new(storage.clone(), path.to_string()),
1220                StandardKeyMetadata::try_new(b"0123456789abcdef").unwrap(),
1221            );
1222            let metadata = output.write(plaintext.clone()).await.unwrap();
1223            assert!(metadata.size > plaintext.len() as u64);
1224            assert_eq!(metadata.size, storage.metadata(path).await.unwrap().size);
1225        }
1226    }
1227
1228    #[cfg(feature = "opendal-memory")]
1229    #[test]
1230    fn test_relativize_path_memory() {
1231        let storage = OpenDalStorage::Memory {
1232            operator: default_memory_operator(),
1233            client_config: OpenDalClientConfig::default(),
1234        };
1235
1236        assert_eq!(
1237            storage.relativize_path("memory:/path/to/file").unwrap(),
1238            "path/to/file"
1239        );
1240        // Without the scheme prefix, falls back to stripping the leading slash
1241        assert_eq!(
1242            storage.relativize_path("/path/to/file").unwrap(),
1243            "path/to/file"
1244        );
1245    }
1246
1247    #[cfg(feature = "opendal-fs")]
1248    #[test]
1249    fn test_relativize_path_fs() {
1250        let storage = OpenDalStorage::LocalFs {
1251            client_config: OpenDalClientConfig::default(),
1252        };
1253
1254        assert_eq!(
1255            storage
1256                .relativize_path("file:/tmp/data/file.parquet")
1257                .unwrap(),
1258            "tmp/data/file.parquet"
1259        );
1260        assert_eq!(
1261            storage.relativize_path("/tmp/data/file.parquet").unwrap(),
1262            "tmp/data/file.parquet"
1263        );
1264    }
1265
1266    #[cfg(feature = "opendal-s3")]
1267    #[test]
1268    fn test_relativize_path_s3() {
1269        let storage = OpenDalStorage::S3 {
1270            config: Arc::new(S3Config::default()),
1271            multipart_part_size: default_multipart_part_size(),
1272            customized_credential_load: None,
1273            client_config: OpenDalClientConfig::default(),
1274        };
1275
1276        // All S3-family schemes are accepted by the same storage instance.
1277        // Custom schemes for S3-compatible stores (e.g., `minio://`) are also
1278        // accepted because the path's scheme is used as-is for prefix matching.
1279        for scheme in ["s3", "s3a", "s3n", "minio"] {
1280            assert_eq!(
1281                storage
1282                    .relativize_path(&format!("{scheme}://my-bucket/path/to/file.parquet"))
1283                    .unwrap(),
1284                "path/to/file.parquet"
1285            );
1286        }
1287    }
1288
1289    #[cfg(feature = "opendal-gcs")]
1290    #[test]
1291    fn test_relativize_path_gcs() {
1292        let storage = OpenDalStorage::Gcs {
1293            config: Arc::new(GcsConfig::default()),
1294            client_config: OpenDalClientConfig::default(),
1295        };
1296
1297        assert_eq!(
1298            storage
1299                .relativize_path("gs://my-bucket/path/to/file.parquet")
1300                .unwrap(),
1301            "path/to/file.parquet"
1302        );
1303    }
1304
1305    #[cfg(feature = "opendal-gcs")]
1306    #[test]
1307    fn test_relativize_path_gcs_invalid_scheme() {
1308        let storage = OpenDalStorage::Gcs {
1309            config: Arc::new(GcsConfig::default()),
1310            client_config: OpenDalClientConfig::default(),
1311        };
1312
1313        assert!(
1314            storage
1315                .relativize_path("s3://my-bucket/path/to/file.parquet")
1316                .is_err()
1317        );
1318    }
1319
1320    #[cfg(feature = "opendal-oss")]
1321    #[test]
1322    fn test_relativize_path_oss() {
1323        let storage = OpenDalStorage::Oss {
1324            config: Arc::new(OssConfig::default()),
1325            client_config: OpenDalClientConfig::default(),
1326        };
1327
1328        assert_eq!(
1329            storage
1330                .relativize_path("oss://my-bucket/path/to/file.parquet")
1331                .unwrap(),
1332            "path/to/file.parquet"
1333        );
1334    }
1335
1336    #[cfg(feature = "opendal-oss")]
1337    #[test]
1338    fn test_relativize_path_oss_invalid_scheme() {
1339        let storage = OpenDalStorage::Oss {
1340            config: Arc::new(OssConfig::default()),
1341            client_config: OpenDalClientConfig::default(),
1342        };
1343
1344        assert!(
1345            storage
1346                .relativize_path("s3://my-bucket/path/to/file.parquet")
1347                .is_err()
1348        );
1349    }
1350
1351    #[cfg(feature = "opendal-azdls")]
1352    #[test]
1353    fn test_relativize_path_azdls() {
1354        let storage = OpenDalStorage::Azdls {
1355            config: Arc::new(AzdlsConfig {
1356                account_name: Some("myaccount".to_string()),
1357                endpoint: Some("https://myaccount.dfs.core.windows.net".to_string()),
1358                ..Default::default()
1359            }),
1360            client_config: OpenDalClientConfig::default(),
1361        };
1362
1363        assert_eq!(
1364            storage
1365                .relativize_path("abfss://myfs@myaccount.dfs.core.windows.net/path/to/file.parquet")
1366                .unwrap(),
1367            "path/to/file.parquet"
1368        );
1369    }
1370}