iceberg/io/storage/config/
mod.rs

1// Licensed to the Apache Software Foundation (ASF) under one
2// or more contributor license agreements.  See the NOTICE file
3// distributed with this work for additional information
4// regarding copyright ownership.  The ASF licenses this file
5// to you under the Apache License, Version 2.0 (the
6// "License"); you may not use this file except in compliance
7// with the License.  You may obtain a copy of the License at
8//
9//   http://www.apache.org/licenses/LICENSE-2.0
10//
11// Unless required by applicable law or agreed to in writing,
12// software distributed under the License is distributed on an
13// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14// KIND, either express or implied.  See the License for the
15// specific language governing permissions and limitations
16// under the License.
17
18// TODO Add specific configs
19//! Storage configuration for storage backends.
20//!
21//! This module provides configuration types for various storage backends.
22//! The configuration types are designed to be used with the `StorageFactory`
23//! trait to create storage instances.
24//!
25//! # Available Configurations
26//!
27//! - [`StorageConfig`]: Base configuration containing properties for storage backends
28//! - [`S3Config`]: Amazon S3 specific configuration
29//! - [`GcsConfig`]: Google Cloud Storage specific configuration
30//! - [`OssConfig`]: Alibaba Cloud OSS specific configuration
31//! - [`AzdlsConfig`]: Azure Data Lake Storage specific configuration
32
33mod azdls;
34mod gcs;
35mod hf;
36mod oss;
37mod s3;
38
39use std::collections::HashMap;
40
41pub use azdls::*;
42pub use gcs::*;
43pub use hf::*;
44pub use oss::*;
45pub use s3::*;
46use serde::{Deserialize, Serialize};
47
48/// Configuration properties for storage backends.
49///
50/// This struct contains only configuration properties without specifying
51/// which storage backend to use. The storage type is determined by the
52/// explicit factory selection.
53#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize, Default)]
54pub struct StorageConfig {
55    /// Configuration properties for the storage backend
56    props: HashMap<String, String>,
57}
58
59impl StorageConfig {
60    /// Create a new empty StorageConfig.
61    pub fn new() -> Self {
62        Self {
63            props: HashMap::new(),
64        }
65    }
66
67    /// Create a StorageConfig from existing properties.
68    ///
69    /// # Arguments
70    ///
71    /// * `props` - Configuration properties for the storage backend
72    pub fn from_props(props: HashMap<String, String>) -> Self {
73        Self { props }
74    }
75
76    /// Get all configuration properties.
77    pub fn props(&self) -> &HashMap<String, String> {
78        &self.props
79    }
80
81    /// Get a specific configuration property by key.
82    ///
83    /// # Arguments
84    ///
85    /// * `key` - The property key to look up
86    ///
87    /// # Returns
88    ///
89    /// An `Option` containing a reference to the property value if it exists.
90    pub fn get(&self, key: &str) -> Option<&String> {
91        self.props.get(key)
92    }
93
94    /// Add a configuration property.
95    ///
96    /// This is a builder-style method that returns `self` for chaining.
97    ///
98    /// # Arguments
99    ///
100    /// * `key` - The property key
101    /// * `value` - The property value
102    pub fn with_prop(mut self, key: impl Into<String>, value: impl Into<String>) -> Self {
103        self.props.insert(key.into(), value.into());
104        self
105    }
106
107    /// Add multiple configuration properties.
108    ///
109    /// This is a builder-style method that returns `self` for chaining.
110    ///
111    /// # Arguments
112    ///
113    /// * `props` - An iterator of key-value pairs to add
114    pub fn with_props(
115        mut self,
116        props: impl IntoIterator<Item = (impl Into<String>, impl Into<String>)>,
117    ) -> Self {
118        self.props
119            .extend(props.into_iter().map(|(k, v)| (k.into(), v.into())));
120        self
121    }
122}
123
124#[cfg(test)]
125mod tests {
126    use super::*;
127
128    #[test]
129    fn test_storage_config_new() {
130        let config = StorageConfig::new();
131
132        assert!(config.props().is_empty());
133    }
134
135    #[test]
136    fn test_storage_config_from_props() {
137        let props = HashMap::from([
138            ("region".to_string(), "us-east-1".to_string()),
139            ("bucket".to_string(), "my-bucket".to_string()),
140        ]);
141        let config = StorageConfig::from_props(props.clone());
142
143        assert_eq!(config.props(), &props);
144    }
145
146    #[test]
147    fn test_storage_config_default() {
148        let config = StorageConfig::default();
149
150        assert!(config.props().is_empty());
151    }
152
153    #[test]
154    fn test_storage_config_get() {
155        let config = StorageConfig::new().with_prop("region", "us-east-1");
156
157        assert_eq!(config.get("region"), Some(&"us-east-1".to_string()));
158        assert_eq!(config.get("nonexistent"), None);
159    }
160
161    #[test]
162    fn test_storage_config_with_prop() {
163        let config = StorageConfig::new()
164            .with_prop("region", "us-east-1")
165            .with_prop("bucket", "my-bucket");
166
167        assert_eq!(config.get("region"), Some(&"us-east-1".to_string()));
168        assert_eq!(config.get("bucket"), Some(&"my-bucket".to_string()));
169    }
170
171    #[test]
172    fn test_storage_config_with_props() {
173        let additional_props = vec![("key1", "value1"), ("key2", "value2")];
174        let config = StorageConfig::new().with_props(additional_props);
175
176        assert_eq!(config.get("key1"), Some(&"value1".to_string()));
177        assert_eq!(config.get("key2"), Some(&"value2".to_string()));
178    }
179
180    #[test]
181    fn test_storage_config_clone() {
182        let config = StorageConfig::new().with_prop("region", "us-east-1");
183        let cloned = config.clone();
184
185        assert_eq!(config, cloned);
186        assert_eq!(cloned.get("region"), Some(&"us-east-1".to_string()));
187    }
188
189    #[test]
190    fn test_storage_config_serialization_roundtrip() {
191        let config = StorageConfig::new()
192            .with_prop("region", "us-east-1")
193            .with_prop("bucket", "my-bucket");
194
195        let serialized = serde_json::to_string(&config).unwrap();
196        let deserialized: StorageConfig = serde_json::from_str(&serialized).unwrap();
197
198        assert_eq!(config, deserialized);
199    }
200
201    #[test]
202    fn test_storage_config_clone_independence() {
203        let original = StorageConfig::new().with_prop("region", "us-east-1");
204        let mut cloned = original.clone();
205
206        // Modify the clone
207        cloned = cloned.with_prop("region", "eu-west-1");
208        cloned = cloned.with_prop("new_key", "new_value");
209
210        // Original should be unchanged
211        assert_eq!(original.get("region"), Some(&"us-east-1".to_string()));
212        assert_eq!(original.get("new_key"), None);
213
214        // Clone should have the new values
215        assert_eq!(cloned.get("region"), Some(&"eu-west-1".to_string()));
216        assert_eq!(cloned.get("new_key"), Some(&"new_value".to_string()));
217    }
218
219    #[test]
220    fn test_storage_config_from_props_empty() {
221        let config = StorageConfig::from_props(HashMap::new());
222
223        assert!(config.props().is_empty());
224    }
225
226    #[test]
227    fn test_storage_config_serialization_empty() {
228        let config = StorageConfig::new();
229
230        let serialized = serde_json::to_string(&config).unwrap();
231        let deserialized: StorageConfig = serde_json::from_str(&serialized).unwrap();
232
233        assert_eq!(config, deserialized);
234        assert!(deserialized.props().is_empty());
235    }
236}
iceberg/io/storage/config/mod.rs

iceberg/io/storage/config/
mod.rs
