1use std::collections::HashMap;
22use std::sync::{Arc, RwLock};
23
24use async_trait::async_trait;
25use bytes::Bytes;
26use futures::StreamExt;
27use futures::stream::BoxStream;
28use iceberg::io::{
29 FileMetadata, FileRead, FileWrite, InputFile, OutputFile, Storage, StorageConfig,
30 StorageFactory,
31};
32use iceberg::{Error, ErrorKind, Result};
33use serde::{Deserialize, Serialize};
34use url::Url;
35
36#[cfg(feature = "opendal-s3")]
37use crate::s3::CustomAwsCredentialLoader;
38use crate::{OpenDalClientConfig, OpenDalStorage};
39
40pub const SCHEME_MEMORY: &str = "memory";
42pub const SCHEME_FILE: &str = "file";
43pub const SCHEME_S3: &str = "s3";
44pub const SCHEME_S3A: &str = "s3a";
45pub const SCHEME_S3N: &str = "s3n";
46pub const SCHEME_GS: &str = "gs";
47pub const SCHEME_GCS: &str = "gcs";
48pub const SCHEME_OSS: &str = "oss";
49pub const SCHEME_ABFSS: &str = "abfss";
50pub const SCHEME_ABFS: &str = "abfs";
51pub const SCHEME_WASBS: &str = "wasbs";
52pub const SCHEME_WASB: &str = "wasb";
53pub const SCHEME_HF: &str = "hf";
54pub const SCHEME_HDFS: &str = "hdfs";
55
56fn parse_scheme(scheme: &str) -> Result<&'static str> {
58 match scheme {
59 SCHEME_MEMORY => Ok("memory"),
60 SCHEME_FILE | "" => Ok("file"),
61 SCHEME_S3 | SCHEME_S3A | SCHEME_S3N => Ok("s3"),
62 SCHEME_GS | SCHEME_GCS => Ok("gcs"),
63 SCHEME_OSS => Ok("oss"),
64 SCHEME_ABFSS | SCHEME_ABFS | SCHEME_WASBS | SCHEME_WASB => Ok("azdls"),
65 SCHEME_HF => Ok("hf"),
66 SCHEME_HDFS => Ok("hdfs"),
67 s => Err(Error::new(
68 ErrorKind::FeatureUnsupported,
69 format!("Unsupported storage scheme: {s}"),
70 )),
71 }
72}
73
74fn extract_scheme(path: &str) -> Result<&'static str> {
76 let url = Url::parse(path).map_err(|e| {
77 Error::new(
78 ErrorKind::DataInvalid,
79 format!("Invalid path: {path}, failed to parse URL: {e}"),
80 )
81 })?;
82 parse_scheme(url.scheme())
83}
84
85fn build_storage_for_scheme(
87 scheme: &'static str,
88 props: &HashMap<String, String>,
89 #[cfg(feature = "opendal-s3")] customized_credential_load: &Option<CustomAwsCredentialLoader>,
90) -> Result<OpenDalStorage> {
91 let client_config = OpenDalClientConfig::from_properties(props)?;
92 match scheme {
93 #[cfg(feature = "opendal-s3")]
94 "s3" => {
95 let config = crate::s3::s3_config_parse(props.clone())?;
96 Ok(OpenDalStorage::S3 {
97 config: Arc::new(config),
98 multipart_part_size: crate::s3::s3_multipart_part_size_parse(props)?,
99 customized_credential_load: customized_credential_load.clone(),
100 client_config: crate::s3::s3_client_config(props, client_config)?,
101 })
102 }
103 #[cfg(feature = "opendal-gcs")]
104 "gcs" => {
105 let config = crate::gcs::gcs_config_parse(props.clone())?;
106 Ok(OpenDalStorage::Gcs {
107 config: Arc::new(config),
108 client_config,
109 })
110 }
111 #[cfg(feature = "opendal-oss")]
112 "oss" => {
113 let config = crate::oss::oss_config_parse(props.clone())?;
114 Ok(OpenDalStorage::Oss {
115 config: Arc::new(config),
116 client_config,
117 })
118 }
119 #[cfg(feature = "opendal-azdls")]
120 "azdls" => {
121 let config = crate::azdls::azdls_config_parse(props.clone())?;
122 Ok(OpenDalStorage::Azdls {
123 config: Arc::new(config),
124 client_config,
125 })
126 }
127 #[cfg(feature = "opendal-fs")]
128 "file" => Ok(OpenDalStorage::LocalFs { client_config }),
129 #[cfg(feature = "opendal-memory")]
130 "memory" => Ok(OpenDalStorage::Memory {
131 operator: crate::memory::memory_config_build()?,
132 client_config,
133 }),
134 #[cfg(feature = "opendal-hf")]
135 "hf" => {
136 let config = crate::hf::hf_config_parse(props.clone())?;
137 Ok(OpenDalStorage::Hf {
138 config: Arc::new(config),
139 client_config,
140 })
141 }
142 #[cfg(feature = "opendal-hdfs-native")]
143 "hdfs" => {
144 let config = crate::hdfs_native::hdfs_native_config_parse(props.clone())?;
145 Ok(OpenDalStorage::HdfsNative(crate::HdfsNativeStorage::new(
146 config,
147 client_config,
148 )))
149 }
150 unsupported => Err(Error::new(
151 ErrorKind::FeatureUnsupported,
152 format!("Unsupported storage scheme: {unsupported}"),
153 )),
154 }
155}
156
157#[derive(Clone, Debug, Serialize, Deserialize)]
182pub struct OpenDalResolvingStorageFactory {
183 #[cfg(feature = "opendal-s3")]
185 #[serde(
186 skip_deserializing,
187 skip_serializing_if = "Option::is_none",
188 serialize_with = "crate::serialize_custom_credential_loader"
189 )]
190 customized_credential_load: Option<CustomAwsCredentialLoader>,
191}
192
193impl Default for OpenDalResolvingStorageFactory {
194 fn default() -> Self {
195 Self::new()
196 }
197}
198
199impl OpenDalResolvingStorageFactory {
200 pub fn new() -> Self {
202 Self {
203 #[cfg(feature = "opendal-s3")]
204 customized_credential_load: None,
205 }
206 }
207
208 #[cfg(feature = "opendal-s3")]
210 pub fn with_s3_credential_loader(mut self, loader: CustomAwsCredentialLoader) -> Self {
211 self.customized_credential_load = Some(loader);
212 self
213 }
214}
215
216#[typetag::serde]
217impl StorageFactory for OpenDalResolvingStorageFactory {
218 fn build(&self, config: &StorageConfig) -> Result<Arc<dyn Storage>> {
219 Ok(Arc::new(OpenDalResolvingStorage {
220 props: config.props().clone(),
221 storages: RwLock::new(HashMap::new()),
222 #[cfg(feature = "opendal-s3")]
223 customized_credential_load: self.customized_credential_load.clone(),
224 }))
225 }
226}
227
228#[derive(Debug, Serialize, Deserialize)]
235pub struct OpenDalResolvingStorage {
236 props: HashMap<String, String>,
238 #[serde(skip, default)]
240 storages: RwLock<HashMap<&'static str, Arc<OpenDalStorage>>>,
241 #[cfg(feature = "opendal-s3")]
243 #[serde(skip)]
244 customized_credential_load: Option<CustomAwsCredentialLoader>,
245}
246
247impl OpenDalResolvingStorage {
248 fn resolve(&self, path: &str) -> Result<Arc<OpenDalStorage>> {
251 let scheme = extract_scheme(path)?;
252
253 {
255 let cache = self
256 .storages
257 .read()
258 .map_err(|_| Error::new(ErrorKind::Unexpected, "Storage cache lock poisoned"))?;
259 if let Some(storage) = cache.get(&scheme) {
260 return Ok(storage.clone());
261 }
262 }
263
264 let mut cache = self
266 .storages
267 .write()
268 .map_err(|_| Error::new(ErrorKind::Unexpected, "Storage cache lock poisoned"))?;
269
270 if let Some(storage) = cache.get(&scheme) {
272 return Ok(storage.clone());
273 }
274
275 let storage = build_storage_for_scheme(
276 scheme,
277 &self.props,
278 #[cfg(feature = "opendal-s3")]
279 &self.customized_credential_load,
280 )?;
281 let storage = Arc::new(storage);
282 cache.insert(scheme, storage.clone());
283 Ok(storage)
284 }
285}
286
287#[async_trait]
288#[typetag::serde]
289impl Storage for OpenDalResolvingStorage {
290 async fn exists(&self, path: &str) -> Result<bool> {
291 self.resolve(path)?.exists(path).await
292 }
293
294 async fn metadata(&self, path: &str) -> Result<FileMetadata> {
295 self.resolve(path)?.metadata(path).await
296 }
297
298 async fn read(&self, path: &str) -> Result<Bytes> {
299 self.resolve(path)?.read(path).await
300 }
301
302 async fn reader(&self, path: &str) -> Result<Box<dyn FileRead>> {
303 self.resolve(path)?.reader(path).await
304 }
305
306 async fn write(&self, path: &str, bs: Bytes) -> Result<()> {
307 self.resolve(path)?.write(path, bs).await
308 }
309
310 async fn writer(&self, path: &str) -> Result<Box<dyn FileWrite>> {
311 self.resolve(path)?.writer(path).await
312 }
313
314 async fn delete(&self, path: &str) -> Result<()> {
315 self.resolve(path)?.delete(path).await
316 }
317
318 async fn delete_prefix(&self, path: &str) -> Result<()> {
319 self.resolve(path)?.delete_prefix(path).await
320 }
321
322 async fn delete_stream(&self, mut paths: BoxStream<'static, String>) -> Result<()> {
323 let mut grouped: HashMap<&'static str, Vec<String>> = HashMap::new();
326 while let Some(path) = paths.next().await {
327 let scheme = extract_scheme(&path)?;
328 grouped.entry(scheme).or_default().push(path);
329 }
330
331 for (_, paths) in grouped {
332 let storage = self.resolve(&paths[0])?;
333 storage
334 .delete_stream(futures::stream::iter(paths).boxed())
335 .await?;
336 }
337 Ok(())
338 }
339
340 fn new_input(&self, path: &str) -> Result<InputFile> {
341 Ok(InputFile::new(
342 Arc::new(self.resolve(path)?.as_ref().clone()),
343 path.to_string(),
344 ))
345 }
346
347 fn new_output(&self, path: &str) -> Result<OutputFile> {
348 Ok(OutputFile::new(
349 Arc::new(self.resolve(path)?.as_ref().clone()),
350 path.to_string(),
351 ))
352 }
353}
354
355#[cfg(test)]
356mod tests {
357 use std::time::Duration;
358
359 use super::*;
360 use crate::OPENDAL_IO_TIMEOUT_MS;
361
362 #[cfg(feature = "opendal-s3")]
363 #[derive(Debug)]
364 struct EmptyCredentialLoader;
365
366 #[cfg(feature = "opendal-s3")]
367 impl crate::s3::ProvideCredential for EmptyCredentialLoader {
368 type Credential = crate::s3::AwsCredential;
369
370 async fn provide_credential(
371 &self,
372 _ctx: &reqsign_core::Context,
373 ) -> reqsign_core::Result<Option<Self::Credential>> {
374 Ok(None)
375 }
376 }
377
378 #[cfg(feature = "opendal-s3")]
379 #[test]
380 fn test_custom_credential_loader_serialization_fails() {
381 let factory = OpenDalResolvingStorageFactory::new()
382 .with_s3_credential_loader(CustomAwsCredentialLoader::new(EmptyCredentialLoader));
383 let file_io = iceberg::io::FileIOBuilder::new(Arc::new(factory)).build();
384
385 let err = file_io.serialize_all().unwrap_err();
386 assert!(
387 err.to_string()
388 .contains("custom AWS credential loaders cannot be serialized")
389 );
390 }
391
392 fn empty_resolving_storage() -> OpenDalResolvingStorage {
395 OpenDalResolvingStorage {
396 props: HashMap::new(),
397 storages: RwLock::new(HashMap::new()),
398 #[cfg(feature = "opendal-s3")]
399 customized_credential_load: None,
400 }
401 }
402
403 #[test]
404 fn test_resolve_propagates_io_timeout() {
405 let mut storage = empty_resolving_storage();
406 storage
407 .props
408 .insert(OPENDAL_IO_TIMEOUT_MS.to_string(), "45000".to_string());
409
410 let paths: &[&str] = &[
411 #[cfg(feature = "opendal-memory")]
412 "memory:/key",
413 #[cfg(feature = "opendal-fs")]
414 "file:/key",
415 #[cfg(feature = "opendal-s3")]
416 "s3://bucket/key",
417 #[cfg(feature = "opendal-gcs")]
418 "gs://bucket/key",
419 #[cfg(feature = "opendal-oss")]
420 "oss://bucket/key",
421 #[cfg(feature = "opendal-azdls")]
422 "abfss://myfs@myaccount.dfs.core.windows.net/key",
423 #[cfg(feature = "opendal-hf")]
424 "hf://datasets/user/repo/key",
425 ];
426 for path in paths {
427 let resolved = storage.resolve(path).unwrap();
428 assert_eq!(
429 resolved.client_config().io_timeout(),
430 Duration::from_secs(45),
431 "{path}"
432 );
433 }
434 }
435
436 #[cfg(feature = "opendal-s3")]
437 #[test]
438 fn test_resolve_s3_aliases_share_instance() {
439 let storage = empty_resolving_storage();
440
441 let a = storage.resolve("s3://bucket/key").unwrap();
445 let b = storage.resolve("s3a://bucket/key").unwrap();
446 let c = storage.resolve("s3n://bucket/key").unwrap();
447
448 assert!(Arc::ptr_eq(&a, &b), "s3 and s3a should share one instance");
449 assert!(Arc::ptr_eq(&a, &c), "s3 and s3n should share one instance");
450 }
451
452 #[cfg(feature = "opendal-hdfs-native")]
453 #[test]
454 fn test_resolve_hdfs_native_returns_hdfs_variant() {
455 let storage = empty_resolving_storage();
456
457 let resolved = storage.resolve("hdfs://nameservice1/a/b").unwrap();
458
459 assert!(
460 matches!(&*resolved, OpenDalStorage::HdfsNative(_)),
461 "expected HdfsNative variant, got {resolved:?}"
462 );
463 }
464
465 #[cfg(feature = "opendal-hdfs-native")]
466 #[test]
467 fn test_resolve_hdfs_native_distinct_authorities_share_instance() {
468 let storage = empty_resolving_storage();
469
470 let a = storage.resolve("hdfs://ns1/a").unwrap();
471 let b = storage.resolve("hdfs://ns2/b").unwrap();
472
473 assert!(
474 Arc::ptr_eq(&a, &b),
475 "different authorities should share the OpenDalStorage::HdfsNative instance (operator cache is internal)"
476 );
477 }
478
479 #[cfg(feature = "opendal-azdls")]
480 #[test]
481 fn test_resolve_azdls_aliases_share_instance() {
482 let storage = empty_resolving_storage();
483
484 let path_for = |scheme: &str| {
485 format!("{scheme}://myfs@myaccount.dfs.core.windows.net/path/to/file.parquet")
486 };
487
488 let abfss = storage.resolve(&path_for("abfss")).unwrap();
490 let abfs = storage.resolve(&path_for("abfs")).unwrap();
491
492 assert!(
493 Arc::ptr_eq(&abfss, &abfs),
494 "abfss and abfs should share one instance"
495 );
496 }
497}