1mod utils;
26
27use std::collections::HashMap;
28use std::collections::hash_map::Entry;
29use std::num::NonZeroU64;
30use std::sync::Arc;
31use std::time::Duration;
32
33use async_trait::async_trait;
34use bytes::Bytes;
35use cfg_if::cfg_if;
36use futures::StreamExt;
37use futures::stream::BoxStream;
38use iceberg::io::{
39 FileMetadata, FileRead, FileWrite, InputFile, OutputFile, Storage, StorageConfig,
40 StorageFactory,
41};
42use iceberg::{Error, ErrorKind, Result};
43use iceberg_property_macro::Properties;
44use opendal::Operator;
45use opendal::layers::{RetryLayer, TimeoutLayer};
46use opendal::options::WriteOptions;
47use serde::{Deserialize, Serialize};
48use utils::from_opendal_error;
49
50cfg_if! {
51 if #[cfg(feature = "opendal-azdls")] {
52 mod azdls;
53 use azdls::*;
54 use opendal::services::AzdlsConfig;
55 }
56}
57
58cfg_if! {
59 if #[cfg(feature = "opendal-hf")] {
60 mod hf;
61 use hf::*;
62 use opendal::services::HfConfig;
63 }
64}
65
66cfg_if! {
67 if #[cfg(feature = "opendal-fs")] {
68 mod fs;
69 use fs::*;
70 }
71}
72
73cfg_if! {
74 if #[cfg(feature = "opendal-gcs")] {
75 mod gcs;
76 use gcs::*;
77 use opendal::services::GcsConfig;
78 }
79}
80
81cfg_if! {
82 if #[cfg(feature = "opendal-hdfs-native")] {
83 mod hdfs_native;
84 use hdfs_native::*;
85 pub use hdfs_native::HdfsNativeStorage;
86 }
87}
88
89cfg_if! {
90 if #[cfg(feature = "opendal-memory")] {
91 mod memory;
92 use memory::*;
93 }
94}
95
96cfg_if! {
97 if #[cfg(feature = "opendal-oss")] {
98 mod oss;
99 use opendal::services::OssConfig;
100 use oss::*;
101 }
102}
103
104cfg_if! {
105 if #[cfg(feature = "opendal-s3")] {
106 mod s3;
107 use opendal::services::S3Config;
108 pub use s3::*;
109 }
110}
111
112mod resolving;
113pub use resolving::{OpenDalResolvingStorage, OpenDalResolvingStorageFactory};
114
115pub const OPENDAL_IO_TIMEOUT_MS: &str = "opendal.io-timeout-ms";
121
122pub const OPENDAL_RETRY_NUM_RETRIES: &str = "opendal.retry.num-retries";
125
126pub const OPENDAL_RETRY_MIN_WAIT_MS: &str = "opendal.retry.min-wait-ms";
129
130pub const OPENDAL_RETRY_MAX_WAIT_MS: &str = "opendal.retry.max-wait-ms";
133
134const DEFAULT_IO_TIMEOUT_MS: NonZeroU64 = NonZeroU64::new(10_000).unwrap();
136
137#[derive(Clone, Debug, Properties, Serialize, Deserialize)]
139#[serde(default)]
140pub struct OpenDalClientConfig {
141 #[property(
143 key = OPENDAL_IO_TIMEOUT_MS,
144 default = DEFAULT_IO_TIMEOUT_MS,
145 parse_with = parse_positive_ms
146 )]
147 io_timeout_ms: NonZeroU64,
148 #[property(key = OPENDAL_RETRY_NUM_RETRIES, default = None)]
150 retry_num_retries: Option<usize>,
151 #[property(
153 key = OPENDAL_RETRY_MIN_WAIT_MS,
154 default = None,
155 parse_with = parse_positive_ms
156 )]
157 retry_min_wait_ms: Option<NonZeroU64>,
158 #[property(
160 key = OPENDAL_RETRY_MAX_WAIT_MS,
161 default = None,
162 parse_with = parse_positive_ms
163 )]
164 retry_max_wait_ms: Option<NonZeroU64>,
165}
166
167impl Default for OpenDalClientConfig {
168 fn default() -> Self {
169 Self {
170 io_timeout_ms: DEFAULT_IO_TIMEOUT_MS,
171 retry_num_retries: None,
172 retry_min_wait_ms: None,
173 retry_max_wait_ms: None,
174 }
175 }
176}
177
178impl OpenDalClientConfig {
179 pub(crate) fn io_timeout(&self) -> Duration {
180 Duration::from_millis(self.io_timeout_ms.get())
181 }
182
183 pub(crate) fn retry_layer(&self) -> RetryLayer {
185 let mut layer = RetryLayer::new();
186 if self.retry_num_retries.is_none()
187 && self.retry_min_wait_ms.is_none()
188 && self.retry_max_wait_ms.is_none()
189 {
190 return layer;
191 }
192 if let Some(times) = self.retry_num_retries {
193 layer = layer.with_max_times(times);
194 }
195 if let Some(ms) = self.retry_min_wait_ms {
196 layer = layer.with_min_delay(Duration::from_millis(ms.get()));
197 }
198 if let Some(ms) = self.retry_max_wait_ms {
199 layer = layer.with_max_delay(Duration::from_millis(ms.get()));
200 }
201 layer.with_jitter()
202 }
203}
204
205fn parse_positive_ms(value: &str) -> Result<NonZeroU64> {
206 value.parse().map_err(|error| {
207 Error::new(
208 ErrorKind::DataInvalid,
209 "Expected a positive integer number of milliseconds",
210 )
211 .with_context("value", format!("{value:?}"))
212 .with_source(error)
213 })
214}
215
216#[derive(Clone, Debug, Serialize, Deserialize)]
230pub enum OpenDalStorageFactory {
231 #[cfg(feature = "opendal-memory")]
233 Memory,
234 #[cfg(feature = "opendal-fs")]
236 Fs,
237 #[cfg(feature = "opendal-s3")]
239 S3 {
240 #[serde(
242 skip_deserializing,
243 skip_serializing_if = "Option::is_none",
244 serialize_with = "serialize_custom_credential_loader"
245 )]
246 customized_credential_load: Option<CustomAwsCredentialLoader>,
247 },
248 #[cfg(feature = "opendal-gcs")]
250 Gcs,
251 #[cfg(feature = "opendal-hdfs-native")]
253 HdfsNative,
254 #[cfg(feature = "opendal-oss")]
256 Oss,
257 #[cfg(feature = "opendal-azdls")]
259 Azdls,
260 #[cfg(feature = "opendal-hf")]
262 Hf,
263}
264
265#[cfg(feature = "opendal-s3")]
266pub(crate) fn serialize_custom_credential_loader<S>(
267 _loader: &Option<CustomAwsCredentialLoader>,
268 _serializer: S,
269) -> std::result::Result<S::Ok, S::Error>
270where
271 S: serde::Serializer,
272{
273 Err(serde::ser::Error::custom(
274 "custom AWS credential loaders cannot be serialized",
275 ))
276}
277
278#[typetag::serde(name = "OpenDalStorageFactory")]
279impl StorageFactory for OpenDalStorageFactory {
280 #[allow(unused_variables)]
281 fn build(&self, config: &StorageConfig) -> Result<Arc<dyn Storage>> {
282 let client_config = OpenDalClientConfig::from_properties(config.props())?;
283 match self {
284 #[cfg(feature = "opendal-memory")]
285 OpenDalStorageFactory::Memory => Ok(Arc::new(OpenDalStorage::Memory {
286 operator: memory_config_build()?,
287 client_config,
288 })),
289 #[cfg(feature = "opendal-fs")]
290 OpenDalStorageFactory::Fs => Ok(Arc::new(OpenDalStorage::LocalFs { client_config })),
291 #[cfg(feature = "opendal-s3")]
292 OpenDalStorageFactory::S3 {
293 customized_credential_load,
294 } => Ok(Arc::new(OpenDalStorage::S3 {
295 config: s3_config_parse(config.props().clone())?.into(),
296 multipart_part_size: s3_multipart_part_size_parse(config.props())?,
297 customized_credential_load: customized_credential_load.clone(),
298 client_config: s3_client_config(config.props(), client_config)?,
299 })),
300 #[cfg(feature = "opendal-gcs")]
301 OpenDalStorageFactory::Gcs => Ok(Arc::new(OpenDalStorage::Gcs {
302 config: gcs_config_parse(config.props().clone())?.into(),
303 client_config,
304 })),
305 #[cfg(feature = "opendal-hdfs-native")]
306 OpenDalStorageFactory::HdfsNative => Ok(Arc::new(OpenDalStorage::HdfsNative(
307 HdfsNativeStorage::new(
308 hdfs_native_config_parse(config.props().clone())?,
309 client_config,
310 ),
311 ))),
312 #[cfg(feature = "opendal-oss")]
313 OpenDalStorageFactory::Oss => Ok(Arc::new(OpenDalStorage::Oss {
314 config: oss_config_parse(config.props().clone())?.into(),
315 client_config,
316 })),
317 #[cfg(feature = "opendal-azdls")]
318 OpenDalStorageFactory::Azdls => Ok(Arc::new(OpenDalStorage::Azdls {
319 config: azdls_config_parse(config.props().clone())?.into(),
320 client_config,
321 })),
322 #[cfg(feature = "opendal-hf")]
323 OpenDalStorageFactory::Hf => Ok(Arc::new(OpenDalStorage::Hf {
324 config: hf_config_parse(config.props().clone())?.into(),
325 client_config,
326 })),
327 #[cfg(all(
328 not(feature = "opendal-memory"),
329 not(feature = "opendal-fs"),
330 not(feature = "opendal-s3"),
331 not(feature = "opendal-gcs"),
332 not(feature = "opendal-oss"),
333 not(feature = "opendal-azdls"),
334 not(feature = "opendal-hf"),
335 not(feature = "opendal-hdfs-native"),
336 ))]
337 _ => Err(Error::new(
338 ErrorKind::FeatureUnsupported,
339 "No storage service has been enabled",
340 )),
341 }
342 }
343}
344
345#[cfg(feature = "opendal-memory")]
347fn default_memory_operator() -> Operator {
348 memory_config_build().expect("Failed to create default memory operator")
349}
350
351#[derive(Clone, Debug, Serialize, Deserialize)]
355pub enum OpenDalStorage {
356 #[cfg(feature = "opendal-memory")]
358 Memory {
359 #[serde(skip, default = "self::default_memory_operator")]
361 operator: Operator,
362 #[serde(default)]
364 client_config: OpenDalClientConfig,
365 },
366 #[cfg(feature = "opendal-fs")]
368 LocalFs {
369 #[serde(default)]
371 client_config: OpenDalClientConfig,
372 },
373 #[cfg(feature = "opendal-s3")]
378 #[non_exhaustive]
379 S3 {
380 config: Arc<S3Config>,
382 #[serde(default = "default_multipart_part_size")]
384 multipart_part_size: u64,
385 #[serde(skip)]
387 customized_credential_load: Option<CustomAwsCredentialLoader>,
388 #[serde(default)]
390 client_config: OpenDalClientConfig,
391 },
392 #[cfg(feature = "opendal-gcs")]
394 Gcs {
395 config: Arc<GcsConfig>,
397 #[serde(default)]
399 client_config: OpenDalClientConfig,
400 },
401 #[cfg(feature = "opendal-hdfs-native")]
403 HdfsNative(HdfsNativeStorage),
404 #[cfg(feature = "opendal-oss")]
406 Oss {
407 config: Arc<OssConfig>,
409 #[serde(default)]
411 client_config: OpenDalClientConfig,
412 },
413 #[cfg(feature = "opendal-azdls")]
420 Azdls {
421 config: Arc<AzdlsConfig>,
423 #[serde(default)]
425 client_config: OpenDalClientConfig,
426 },
427 #[cfg(feature = "opendal-hf")]
433 Hf {
434 config: Arc<HfConfig>,
436 #[serde(default)]
438 client_config: OpenDalClientConfig,
439 },
440}
441
442impl OpenDalStorage {
443 #[allow(unreachable_code, unused_variables)]
456 pub(crate) fn create_operator<'a>(
457 &self,
458 path: &'a impl AsRef<str>,
459 ) -> Result<(Operator, &'a str)> {
460 let path = path.as_ref();
461 let (operator, relative_path): (Operator, &str) = match self {
462 #[cfg(feature = "opendal-memory")]
463 OpenDalStorage::Memory { operator: op, .. } => {
464 if let Some(stripped) = path.strip_prefix("memory:/") {
465 (op.clone(), stripped)
466 } else {
467 (op.clone(), &path[1..])
468 }
469 }
470 #[cfg(feature = "opendal-fs")]
471 OpenDalStorage::LocalFs { .. } => {
472 let op = fs_config_build()?;
473 if let Some(stripped) = path.strip_prefix("file:/") {
474 (op, stripped)
475 } else {
476 (op, &path[1..])
477 }
478 }
479 #[cfg(feature = "opendal-s3")]
480 OpenDalStorage::S3 {
481 config,
482 customized_credential_load,
483 ..
484 } => {
485 let op = s3_config_build(config, customized_credential_load, path)?;
486 let op_info = op.info();
487
488 let url = url::Url::parse(path).map_err(|e| {
491 Error::new(
492 ErrorKind::DataInvalid,
493 format!("Invalid s3 url: {path}: {e}"),
494 )
495 })?;
496 let prefix = format!("{}://{}/", url.scheme(), op_info.name());
497 if path.starts_with(&prefix) {
498 (op, &path[prefix.len()..])
499 } else {
500 return Err(Error::new(
501 ErrorKind::DataInvalid,
502 format!("Invalid s3 url: {path}, should start with {prefix}"),
503 ));
504 }
505 }
506 #[cfg(feature = "opendal-gcs")]
507 OpenDalStorage::Gcs { config, .. } => {
508 let operator = gcs_config_build(config, path)?;
509 let prefix = format!("gs://{}/", operator.info().name());
510 if path.starts_with(&prefix) {
511 (operator, &path[prefix.len()..])
512 } else {
513 return Err(Error::new(
514 ErrorKind::DataInvalid,
515 format!("Invalid gcs url: {path}, should start with {prefix}"),
516 ));
517 }
518 }
519 #[cfg(feature = "opendal-hdfs-native")]
520 OpenDalStorage::HdfsNative(storage) => {
521 hdfs_native_create_operator(path, &storage.config, &storage.operators)?
522 }
523 #[cfg(feature = "opendal-oss")]
524 OpenDalStorage::Oss { config, .. } => {
525 let op = oss_config_build(config, path)?;
526 let prefix = format!("oss://{}/", op.info().name());
527 if path.starts_with(&prefix) {
528 (op, &path[prefix.len()..])
529 } else {
530 return Err(Error::new(
531 ErrorKind::DataInvalid,
532 format!("Invalid oss url: {path}, should start with {prefix}"),
533 ));
534 }
535 }
536 #[cfg(feature = "opendal-azdls")]
537 OpenDalStorage::Azdls { config, .. } => azdls_create_operator(path, config)?,
538 #[cfg(feature = "opendal-hf")]
539 OpenDalStorage::Hf { config, .. } => hf_config_build(config, path)?,
540 #[cfg(all(
541 not(feature = "opendal-s3"),
542 not(feature = "opendal-fs"),
543 not(feature = "opendal-gcs"),
544 not(feature = "opendal-oss"),
545 not(feature = "opendal-azdls"),
546 not(feature = "opendal-hf"),
547 not(feature = "opendal-hdfs-native"),
548 ))]
549 _ => {
550 return Err(Error::new(
551 ErrorKind::FeatureUnsupported,
552 "No storage service has been enabled",
553 ));
554 }
555 };
556
557 let operator = operator
568 .layer(TimeoutLayer::new().with_io_timeout(self.client_config().io_timeout()))
569 .layer(self.client_config().retry_layer());
570 Ok((operator, relative_path))
571 }
572
573 pub(crate) fn client_config(&self) -> &OpenDalClientConfig {
574 match self {
575 #[cfg(feature = "opendal-memory")]
576 OpenDalStorage::Memory { client_config, .. } => client_config,
577 #[cfg(feature = "opendal-fs")]
578 OpenDalStorage::LocalFs { client_config } => client_config,
579 #[cfg(feature = "opendal-s3")]
580 OpenDalStorage::S3 { client_config, .. } => client_config,
581 #[cfg(feature = "opendal-gcs")]
582 OpenDalStorage::Gcs { client_config, .. } => client_config,
583 #[cfg(feature = "opendal-oss")]
584 OpenDalStorage::Oss { client_config, .. } => client_config,
585 #[cfg(feature = "opendal-azdls")]
586 OpenDalStorage::Azdls { client_config, .. } => client_config,
587 #[cfg(feature = "opendal-hf")]
588 OpenDalStorage::Hf { client_config, .. } => client_config,
589 #[cfg(feature = "opendal-hdfs-native")]
590 OpenDalStorage::HdfsNative(storage) => &storage.client_config,
591 #[cfg(all(
592 not(feature = "opendal-memory"),
593 not(feature = "opendal-s3"),
594 not(feature = "opendal-fs"),
595 not(feature = "opendal-gcs"),
596 not(feature = "opendal-oss"),
597 not(feature = "opendal-azdls"),
598 not(feature = "opendal-hf"),
599 not(feature = "opendal-hdfs-native"),
600 ))]
601 _ => unreachable!(),
602 }
603 }
604
605 #[allow(unreachable_patterns)]
609 fn write_options(&self) -> WriteOptions {
610 match self {
611 #[cfg(feature = "opendal-s3")]
612 OpenDalStorage::S3 {
613 multipart_part_size,
614 ..
615 } => WriteOptions {
616 chunk: Some(usize::try_from(*multipart_part_size).unwrap_or(usize::MAX)),
617 ..WriteOptions::default()
618 },
619 _ => WriteOptions::default(),
620 }
621 }
622
623 fn batch_key_for_path(&self, path: &str) -> String {
628 match self {
629 #[cfg(feature = "opendal-hf")]
630 OpenDalStorage::Hf { .. } => hf_batch_key(path),
631 #[cfg(feature = "opendal-hdfs-native")]
634 OpenDalStorage::HdfsNative(storage) => hdfs_native_batch_key(&storage.config, path),
635 _ => url::Url::parse(path)
636 .ok()
637 .and_then(|u| u.host_str().map(|s| s.to_string()))
638 .unwrap_or_default(),
639 }
640 }
641
642 #[allow(unreachable_code, unused_variables)]
648 pub(crate) fn relativize_path<'a>(&self, path: &'a str) -> Result<&'a str> {
649 match self {
650 #[cfg(feature = "opendal-memory")]
651 OpenDalStorage::Memory { .. } => {
652 Ok(path.strip_prefix("memory:/").unwrap_or(&path[1..]))
653 }
654 #[cfg(feature = "opendal-fs")]
655 OpenDalStorage::LocalFs { .. } => Ok(path.strip_prefix("file:/").unwrap_or(&path[1..])),
656 #[cfg(feature = "opendal-s3")]
657 OpenDalStorage::S3 { .. } => {
658 let url = url::Url::parse(path)?;
659 let bucket = url.host_str().ok_or_else(|| {
660 Error::new(
661 ErrorKind::DataInvalid,
662 format!("Invalid s3 url: {path}, missing bucket"),
663 )
664 })?;
665 let prefix = format!("{}://{}/", url.scheme(), bucket);
666 if path.starts_with(&prefix) {
667 Ok(&path[prefix.len()..])
668 } else {
669 Err(Error::new(
670 ErrorKind::DataInvalid,
671 format!("Invalid s3 url: {path}, should start with {prefix}"),
672 ))
673 }
674 }
675 #[cfg(feature = "opendal-gcs")]
676 OpenDalStorage::Gcs { .. } => {
677 let url = url::Url::parse(path)?;
678 let bucket = url.host_str().ok_or_else(|| {
679 Error::new(
680 ErrorKind::DataInvalid,
681 format!("Invalid gcs url: {path}, missing bucket"),
682 )
683 })?;
684 let prefix = format!("gs://{}/", bucket);
685 if path.starts_with(&prefix) {
686 Ok(&path[prefix.len()..])
687 } else {
688 Err(Error::new(
689 ErrorKind::DataInvalid,
690 format!("Invalid gcs url: {path}, should start with {prefix}"),
691 ))
692 }
693 }
694 #[cfg(feature = "opendal-hdfs-native")]
695 OpenDalStorage::HdfsNative(storage) => {
696 let (_, relative_path) = hdfs_native_effective_name_node(&storage.config, path)?;
697 Ok(relative_path)
698 }
699 #[cfg(feature = "opendal-oss")]
700 OpenDalStorage::Oss { .. } => {
701 let url = url::Url::parse(path)?;
702 let bucket = url.host_str().ok_or_else(|| {
703 Error::new(
704 ErrorKind::DataInvalid,
705 format!("Invalid oss url: {path}, missing bucket"),
706 )
707 })?;
708 let prefix = format!("oss://{}/", bucket);
709 if path.starts_with(&prefix) {
710 Ok(&path[prefix.len()..])
711 } else {
712 Err(Error::new(
713 ErrorKind::DataInvalid,
714 format!("Invalid oss url: {path}, should start with {prefix}"),
715 ))
716 }
717 }
718 #[cfg(feature = "opendal-azdls")]
719 OpenDalStorage::Azdls { config, .. } => {
720 let azure_path = path.parse::<AzureStoragePath>()?;
721 match_path_with_config(&azure_path, config)?;
722 Ok(azure_path.relative_path(path))
723 }
724 #[cfg(feature = "opendal-hf")]
725 OpenDalStorage::Hf { .. } => {
726 let parsed = HfUri::parse(path).ok_or_else(|| {
727 Error::new(ErrorKind::DataInvalid, format!("Invalid hf url: {path}"))
728 })?;
729 Ok(&path[path.len() - parsed.path.len()..])
730 }
731 #[cfg(all(
732 not(feature = "opendal-s3"),
733 not(feature = "opendal-fs"),
734 not(feature = "opendal-gcs"),
735 not(feature = "opendal-oss"),
736 not(feature = "opendal-azdls"),
737 not(feature = "opendal-hf"),
738 not(feature = "opendal-hdfs-native"),
739 ))]
740 _ => Err(Error::new(
741 ErrorKind::FeatureUnsupported,
742 "No storage service has been enabled",
743 )),
744 }
745 }
746}
747
748#[typetag::serde(name = "OpenDalStorage")]
749#[async_trait]
750impl Storage for OpenDalStorage {
751 async fn exists(&self, path: &str) -> Result<bool> {
752 let (op, relative_path) = self.create_operator(&path)?;
753 Ok(op.exists(relative_path).await.map_err(from_opendal_error)?)
754 }
755
756 async fn metadata(&self, path: &str) -> Result<FileMetadata> {
757 let (op, relative_path) = self.create_operator(&path)?;
758 let meta = op.stat(relative_path).await.map_err(from_opendal_error)?;
759 Ok(FileMetadata {
760 size: meta.content_length(),
761 })
762 }
763
764 async fn read(&self, path: &str) -> Result<Bytes> {
765 let (op, relative_path) = self.create_operator(&path)?;
766 Ok(op
767 .read(relative_path)
768 .await
769 .map_err(from_opendal_error)?
770 .to_bytes())
771 }
772
773 async fn reader(&self, path: &str) -> Result<Box<dyn FileRead>> {
774 let (op, relative_path) = self.create_operator(&path)?;
775 Ok(Box::new(OpenDalReader(
776 op.reader(relative_path).await.map_err(from_opendal_error)?,
777 )))
778 }
779
780 async fn write(&self, path: &str, bs: Bytes) -> Result<()> {
781 let (op, relative_path) = self.create_operator(&path)?;
782 op.write_options(relative_path, bs, self.write_options())
783 .await
784 .map_err(from_opendal_error)?;
785 Ok(())
786 }
787
788 async fn writer(&self, path: &str) -> Result<Box<dyn FileWrite>> {
789 let (op, relative_path) = self.create_operator(&path)?;
790 Ok(Box::new(OpenDalWriter::new(
791 op.writer_options(relative_path, self.write_options())
792 .await
793 .map_err(from_opendal_error)?,
794 )))
795 }
796
797 async fn delete(&self, path: &str) -> Result<()> {
798 let (op, relative_path) = self.create_operator(&path)?;
799 Ok(op.delete(relative_path).await.map_err(from_opendal_error)?)
800 }
801
802 async fn delete_prefix(&self, path: &str) -> Result<()> {
803 let (op, relative_path) = self.create_operator(&path)?;
804 let path = if relative_path.ends_with('/') {
805 relative_path.to_string()
806 } else {
807 format!("{relative_path}/")
808 };
809 Ok(op
810 .delete_with(&path)
811 .recursive(true)
812 .await
813 .map_err(from_opendal_error)?)
814 }
815
816 async fn delete_stream(&self, mut paths: BoxStream<'static, String>) -> Result<()> {
817 let mut deleters: HashMap<String, opendal::Deleter> = HashMap::new();
818
819 while let Some(path) = paths.next().await {
820 let bucket = self.batch_key_for_path(&path);
821
822 let (relative_path, deleter) = match deleters.entry(bucket) {
823 Entry::Occupied(entry) => {
824 (self.relativize_path(&path)?.to_string(), entry.into_mut())
825 }
826 Entry::Vacant(entry) => {
827 let (op, rel) = self.create_operator(&path)?;
828 let rel = rel.to_string();
829 let deleter = op.deleter().await.map_err(from_opendal_error)?;
830 (rel, entry.insert(deleter))
831 }
832 };
833
834 deleter
835 .delete(relative_path)
836 .await
837 .map_err(from_opendal_error)?;
838 }
839
840 for (_, mut deleter) in deleters {
841 deleter.close().await.map_err(from_opendal_error)?;
842 }
843
844 Ok(())
845 }
846
847 #[allow(unreachable_code, unused_variables)]
848 fn new_input(&self, path: &str) -> Result<InputFile> {
849 Ok(InputFile::new(Arc::new(self.clone()), path.to_string()))
850 }
851
852 #[allow(unreachable_code, unused_variables)]
853 fn new_output(&self, path: &str) -> Result<OutputFile> {
854 Ok(OutputFile::new(Arc::new(self.clone()), path.to_string()))
855 }
856}
857
858pub(crate) struct OpenDalReader(pub(crate) opendal::Reader);
864
865#[async_trait]
866impl FileRead for OpenDalReader {
867 async fn read(&self, range: std::ops::Range<u64>) -> Result<Bytes> {
868 Ok(opendal::Reader::read(&self.0, range)
869 .await
870 .map_err(from_opendal_error)?
871 .to_bytes())
872 }
873}
874
875pub(crate) struct OpenDalWriter {
877 inner: opendal::Writer,
878 bytes_written: u64,
879}
880
881impl OpenDalWriter {
882 pub(crate) fn new(inner: opendal::Writer) -> Self {
883 Self {
884 inner,
885 bytes_written: 0,
886 }
887 }
888}
889
890#[async_trait]
891impl FileWrite for OpenDalWriter {
892 async fn write(&mut self, bs: Bytes) -> Result<()> {
893 let len = bs.len() as u64;
894 opendal::Writer::write(&mut self.inner, bs)
895 .await
896 .map_err(from_opendal_error)?;
897 self.bytes_written += len;
898 Ok(())
899 }
900
901 async fn close(&mut self) -> Result<FileMetadata> {
902 let metadata = opendal::Writer::close(&mut self.inner)
903 .await
904 .map_err(from_opendal_error)?;
905
906 let reported_size = metadata.content_length();
908 if reported_size != 0 && reported_size != self.bytes_written {
909 return Err(Error::new(
910 ErrorKind::Unexpected,
911 format!(
912 "Wrote {} bytes but storage reports {reported_size}",
913 self.bytes_written
914 ),
915 ));
916 }
917
918 Ok(FileMetadata {
919 size: self.bytes_written,
920 })
921 }
922}
923
924#[cfg(test)]
925mod tests {
926 use super::*;
927
928 fn client_config(value: &str) -> Result<OpenDalClientConfig> {
929 OpenDalClientConfig::from_properties(&HashMap::from([(
930 OPENDAL_IO_TIMEOUT_MS.to_string(),
931 value.to_string(),
932 )]))
933 }
934
935 #[test]
936 fn test_io_timeout_parsing() {
937 let unset = OpenDalClientConfig::from_properties(&HashMap::new()).unwrap();
938 assert_eq!(
939 unset.io_timeout(),
940 Duration::from_millis(DEFAULT_IO_TIMEOUT_MS.get())
941 );
942
943 let max = u64::MAX.to_string();
944 for (valid, ms) in [("45000", 45_000), ("1", 1), (max.as_str(), u64::MAX)] {
945 assert_eq!(
946 client_config(valid).unwrap().io_timeout(),
947 Duration::from_millis(ms),
948 "{valid}"
949 );
950 }
951
952 for invalid in ["0", "-1", "12.5", "abc", "", "18446744073709551616"] {
953 let err = client_config(invalid).unwrap_err().to_string();
954 assert!(err.contains(OPENDAL_IO_TIMEOUT_MS), "{invalid}");
955 assert!(err.contains(&format!("value: {invalid:?}")), "{err}");
956 let reason = invalid.parse::<NonZeroU64>().unwrap_err().to_string();
957 assert!(err.contains(&reason), "{err}");
958 }
959 }
960
961 #[test]
962 fn test_default_timeouts_match_opendal() {
963 let opendal_default = format!("{:?}", TimeoutLayer::new());
965 let layer = |timeout, io_timeout| {
966 format!(
967 "{:?}",
968 TimeoutLayer::new()
969 .with_timeout(timeout)
970 .with_io_timeout(io_timeout)
971 )
972 };
973 let control = Duration::from_secs(60);
974 let io = Duration::from_millis(DEFAULT_IO_TIMEOUT_MS.get());
975 assert_eq!(opendal_default, layer(control, io));
976 assert_ne!(opendal_default, layer(control + Duration::from_secs(1), io));
977 assert_ne!(
978 opendal_default,
979 layer(control, io + Duration::from_millis(1))
980 );
981 }
982
983 fn retry_config(props: &[(&str, &str)]) -> Result<OpenDalClientConfig> {
984 OpenDalClientConfig::from_properties(
985 &props
986 .iter()
987 .map(|(k, v)| (k.to_string(), v.to_string()))
988 .collect::<HashMap<_, _>>(),
989 )
990 }
991
992 fn retry_layer_debug(props: &[(&str, &str)]) -> Result<String> {
993 Ok(format!("{:?}", retry_config(props)?.retry_layer()))
994 }
995
996 #[test]
997 fn test_retry_defaults_match_opendal() {
998 assert_eq!(
1000 retry_layer_debug(&[]).unwrap(),
1001 format!("{:?}", RetryLayer::new())
1002 );
1003 assert_ne!(
1004 format!("{:?}", RetryLayer::new()),
1005 format!("{:?}", RetryLayer::new().with_jitter())
1006 );
1007 }
1008
1009 #[test]
1010 fn test_retry_properties_reach_retry_layer() {
1011 let expected = RetryLayer::new()
1012 .with_max_times(5)
1013 .with_min_delay(Duration::from_millis(100))
1014 .with_max_delay(Duration::from_millis(200))
1015 .with_jitter();
1016 assert_eq!(
1017 retry_layer_debug(&[
1018 (OPENDAL_RETRY_NUM_RETRIES, "5"),
1019 (OPENDAL_RETRY_MIN_WAIT_MS, "100"),
1020 (OPENDAL_RETRY_MAX_WAIT_MS, "200"),
1021 ])
1022 .unwrap(),
1023 format!("{expected:?}")
1024 );
1025
1026 assert_eq!(
1028 retry_layer_debug(&[(OPENDAL_RETRY_NUM_RETRIES, "0")]).unwrap(),
1029 format!("{:?}", RetryLayer::new().with_max_times(0).with_jitter())
1030 );
1031 }
1032
1033 #[test]
1034 fn test_retry_property_parsing_rejects_invalid_values() {
1035 for (key, invalid) in [
1036 (OPENDAL_RETRY_NUM_RETRIES, "-1"),
1037 (OPENDAL_RETRY_NUM_RETRIES, "abc"),
1038 (OPENDAL_RETRY_NUM_RETRIES, ""),
1039 (OPENDAL_RETRY_MIN_WAIT_MS, "0"),
1040 (OPENDAL_RETRY_MIN_WAIT_MS, "abc"),
1041 (OPENDAL_RETRY_MAX_WAIT_MS, "0"),
1042 (OPENDAL_RETRY_MAX_WAIT_MS, "12.5"),
1043 ] {
1044 let err = retry_config(&[(key, invalid)]).unwrap_err().to_string();
1045 assert!(err.contains(key), "{key}={invalid}: {err}");
1046 }
1047 }
1048
1049 #[cfg(feature = "opendal-s3")]
1050 #[test]
1051 fn test_client_config_serde_round_trip() {
1052 let storage = OpenDalStorage::S3 {
1053 config: Arc::new(S3Config::default()),
1054 multipart_part_size: default_multipart_part_size(),
1055 customized_credential_load: None,
1056 client_config: client_config("45000").unwrap(),
1057 };
1058
1059 let mut value = serde_json::to_value(&storage).unwrap();
1060 let restored: OpenDalStorage = serde_json::from_value(value.clone()).unwrap();
1061 assert_eq!(
1062 restored.client_config().io_timeout(),
1063 Duration::from_secs(45)
1064 );
1065
1066 let mut zero = value.clone();
1067 zero["S3"]["client_config"]["io_timeout_ms"] = 0.into();
1068 assert!(serde_json::from_value::<OpenDalStorage>(zero).is_err());
1069
1070 value["S3"].as_object_mut().unwrap().remove("client_config");
1071 let restored: OpenDalStorage = serde_json::from_value(value).unwrap();
1072 assert_eq!(
1073 restored.client_config().io_timeout(),
1074 Duration::from_millis(DEFAULT_IO_TIMEOUT_MS.get())
1075 );
1076 }
1077
1078 #[cfg(all(feature = "opendal-fs", feature = "opendal-memory"))]
1079 #[test]
1080 fn test_old_unit_variant_forms_are_rejected() {
1081 for old in [r#""LocalFs""#, r#""Memory""#, r#"{"Memory":null}"#] {
1082 assert!(
1083 serde_json::from_str::<OpenDalStorage>(old).is_err(),
1084 "{old}"
1085 );
1086 }
1087 }
1088
1089 #[cfg(feature = "opendal-memory")]
1090 #[test]
1091 fn test_factory_rejects_invalid_io_timeout() {
1092 let config = StorageConfig::new().with_prop(OPENDAL_IO_TIMEOUT_MS, "nope");
1093
1094 let err = OpenDalStorageFactory::Memory.build(&config).unwrap_err();
1095 assert!(err.to_string().contains(OPENDAL_IO_TIMEOUT_MS));
1096 }
1097
1098 #[test]
1099 fn test_factory_propagates_io_timeout() {
1100 let config = StorageConfig::new().with_prop(OPENDAL_IO_TIMEOUT_MS, "45000");
1101 let factories = [
1102 #[cfg(feature = "opendal-memory")]
1103 OpenDalStorageFactory::Memory,
1104 #[cfg(feature = "opendal-fs")]
1105 OpenDalStorageFactory::Fs,
1106 #[cfg(feature = "opendal-s3")]
1107 OpenDalStorageFactory::S3 {
1108 customized_credential_load: None,
1109 },
1110 #[cfg(feature = "opendal-gcs")]
1111 OpenDalStorageFactory::Gcs,
1112 #[cfg(feature = "opendal-oss")]
1113 OpenDalStorageFactory::Oss,
1114 #[cfg(feature = "opendal-azdls")]
1115 OpenDalStorageFactory::Azdls,
1116 #[cfg(feature = "opendal-hf")]
1117 OpenDalStorageFactory::Hf,
1118 #[cfg(feature = "opendal-hdfs-native")]
1119 OpenDalStorageFactory::HdfsNative,
1120 ];
1121 for factory in factories {
1122 let storage = factory.build(&config).unwrap();
1124 let json = serde_json::to_value(&*storage).unwrap();
1125 let client_config = json
1126 .as_object()
1127 .unwrap()
1128 .values()
1129 .find_map(|variant| variant.get("client_config"))
1130 .unwrap_or_else(|| panic!("{json}"));
1131 assert_eq!(client_config["io_timeout_ms"], 45_000, "{factory:?}");
1132 }
1133 }
1134
1135 #[cfg(feature = "opendal-memory")]
1136 #[tokio::test(start_paused = true)]
1137 async fn test_io_timeout_reaches_timeout_layer() {
1138 use opendal::layers::ConcurrentLimitLayer;
1139
1140 let storage = OpenDalStorage::Memory {
1142 operator: default_memory_operator().layer(ConcurrentLimitLayer::new(0)),
1143 client_config: client_config("45000").unwrap(),
1144 };
1145
1146 let err = storage
1147 .read("memory:/stalled")
1148 .await
1149 .unwrap_err()
1150 .to_string();
1151 assert!(err.contains("io operation timeout reached"), "{err}");
1152 assert!(err.contains("{ timeout: 45 }"), "{err}");
1153 }
1154
1155 #[cfg(feature = "opendal-s3")]
1156 #[derive(Debug)]
1157 struct EmptyCredentialLoader;
1158
1159 #[cfg(feature = "opendal-s3")]
1160 impl ProvideCredential for EmptyCredentialLoader {
1161 type Credential = AwsCredential;
1162
1163 async fn provide_credential(
1164 &self,
1165 _ctx: &reqsign_core::Context,
1166 ) -> reqsign_core::Result<Option<AwsCredential>> {
1167 Ok(None)
1168 }
1169 }
1170
1171 #[cfg(feature = "opendal-s3")]
1172 #[test]
1173 fn test_s3_factory_custom_credential_loader_serialization_fails() {
1174 let file_io = iceberg::io::FileIOBuilder::new(Arc::new(OpenDalStorageFactory::S3 {
1175 customized_credential_load: Some(CustomAwsCredentialLoader::new(EmptyCredentialLoader)),
1176 }))
1177 .build();
1178
1179 let err = file_io.serialize_all().unwrap_err();
1180 assert!(
1181 err.to_string()
1182 .contains("custom AWS credential loaders cannot be serialized")
1183 );
1184 }
1185
1186 #[cfg(feature = "opendal-memory")]
1187 #[test]
1188 fn test_default_memory_operator() {
1189 let op = default_memory_operator();
1190 assert_eq!(op.info().scheme().to_string(), "memory");
1191 }
1192
1193 #[cfg(feature = "opendal-memory")]
1194 #[tokio::test]
1195 async fn test_writer_close_returns_stored_size() {
1196 use iceberg::encryption::{EncryptedOutputFile, StandardKeyMetadata};
1197
1198 let storage = Arc::new(OpenDalStorage::Memory {
1201 operator: default_memory_operator(),
1202 client_config: OpenDalClientConfig::default(),
1203 });
1204 let path = "memory:///stored-size";
1205 for plaintext in [
1206 Bytes::new(),
1207 Bytes::from_static(b"test data"),
1208 Bytes::from(vec![7; 3 * 1024]),
1209 ] {
1210 let mut writer = storage.writer(path).await.unwrap();
1211 for chunk in plaintext.chunks(1024) {
1212 writer.write(Bytes::copy_from_slice(chunk)).await.unwrap();
1213 }
1214 let metadata = writer.close().await.unwrap();
1215 assert_eq!(metadata.size, plaintext.len() as u64);
1216 assert_eq!(metadata.size, storage.metadata(path).await.unwrap().size);
1217
1218 let output = EncryptedOutputFile::new(
1219 OutputFile::new(storage.clone(), path.to_string()),
1220 StandardKeyMetadata::try_new(b"0123456789abcdef").unwrap(),
1221 );
1222 let metadata = output.write(plaintext.clone()).await.unwrap();
1223 assert!(metadata.size > plaintext.len() as u64);
1224 assert_eq!(metadata.size, storage.metadata(path).await.unwrap().size);
1225 }
1226 }
1227
1228 #[cfg(feature = "opendal-memory")]
1229 #[test]
1230 fn test_relativize_path_memory() {
1231 let storage = OpenDalStorage::Memory {
1232 operator: default_memory_operator(),
1233 client_config: OpenDalClientConfig::default(),
1234 };
1235
1236 assert_eq!(
1237 storage.relativize_path("memory:/path/to/file").unwrap(),
1238 "path/to/file"
1239 );
1240 assert_eq!(
1242 storage.relativize_path("/path/to/file").unwrap(),
1243 "path/to/file"
1244 );
1245 }
1246
1247 #[cfg(feature = "opendal-fs")]
1248 #[test]
1249 fn test_relativize_path_fs() {
1250 let storage = OpenDalStorage::LocalFs {
1251 client_config: OpenDalClientConfig::default(),
1252 };
1253
1254 assert_eq!(
1255 storage
1256 .relativize_path("file:/tmp/data/file.parquet")
1257 .unwrap(),
1258 "tmp/data/file.parquet"
1259 );
1260 assert_eq!(
1261 storage.relativize_path("/tmp/data/file.parquet").unwrap(),
1262 "tmp/data/file.parquet"
1263 );
1264 }
1265
1266 #[cfg(feature = "opendal-s3")]
1267 #[test]
1268 fn test_relativize_path_s3() {
1269 let storage = OpenDalStorage::S3 {
1270 config: Arc::new(S3Config::default()),
1271 multipart_part_size: default_multipart_part_size(),
1272 customized_credential_load: None,
1273 client_config: OpenDalClientConfig::default(),
1274 };
1275
1276 for scheme in ["s3", "s3a", "s3n", "minio"] {
1280 assert_eq!(
1281 storage
1282 .relativize_path(&format!("{scheme}://my-bucket/path/to/file.parquet"))
1283 .unwrap(),
1284 "path/to/file.parquet"
1285 );
1286 }
1287 }
1288
1289 #[cfg(feature = "opendal-gcs")]
1290 #[test]
1291 fn test_relativize_path_gcs() {
1292 let storage = OpenDalStorage::Gcs {
1293 config: Arc::new(GcsConfig::default()),
1294 client_config: OpenDalClientConfig::default(),
1295 };
1296
1297 assert_eq!(
1298 storage
1299 .relativize_path("gs://my-bucket/path/to/file.parquet")
1300 .unwrap(),
1301 "path/to/file.parquet"
1302 );
1303 }
1304
1305 #[cfg(feature = "opendal-gcs")]
1306 #[test]
1307 fn test_relativize_path_gcs_invalid_scheme() {
1308 let storage = OpenDalStorage::Gcs {
1309 config: Arc::new(GcsConfig::default()),
1310 client_config: OpenDalClientConfig::default(),
1311 };
1312
1313 assert!(
1314 storage
1315 .relativize_path("s3://my-bucket/path/to/file.parquet")
1316 .is_err()
1317 );
1318 }
1319
1320 #[cfg(feature = "opendal-oss")]
1321 #[test]
1322 fn test_relativize_path_oss() {
1323 let storage = OpenDalStorage::Oss {
1324 config: Arc::new(OssConfig::default()),
1325 client_config: OpenDalClientConfig::default(),
1326 };
1327
1328 assert_eq!(
1329 storage
1330 .relativize_path("oss://my-bucket/path/to/file.parquet")
1331 .unwrap(),
1332 "path/to/file.parquet"
1333 );
1334 }
1335
1336 #[cfg(feature = "opendal-oss")]
1337 #[test]
1338 fn test_relativize_path_oss_invalid_scheme() {
1339 let storage = OpenDalStorage::Oss {
1340 config: Arc::new(OssConfig::default()),
1341 client_config: OpenDalClientConfig::default(),
1342 };
1343
1344 assert!(
1345 storage
1346 .relativize_path("s3://my-bucket/path/to/file.parquet")
1347 .is_err()
1348 );
1349 }
1350
1351 #[cfg(feature = "opendal-azdls")]
1352 #[test]
1353 fn test_relativize_path_azdls() {
1354 let storage = OpenDalStorage::Azdls {
1355 config: Arc::new(AzdlsConfig {
1356 account_name: Some("myaccount".to_string()),
1357 endpoint: Some("https://myaccount.dfs.core.windows.net".to_string()),
1358 ..Default::default()
1359 }),
1360 client_config: OpenDalClientConfig::default(),
1361 };
1362
1363 assert_eq!(
1364 storage
1365 .relativize_path("abfss://myfs@myaccount.dfs.core.windows.net/path/to/file.parquet")
1366 .unwrap(),
1367 "path/to/file.parquet"
1368 );
1369 }
1370}