iceberg/io/storage/config/
mod.rs

1// Licensed to the Apache Software Foundation (ASF) under one
2// or more contributor license agreements.  See the NOTICE file
3// distributed with this work for additional information
4// regarding copyright ownership.  The ASF licenses this file
5// to you under the Apache License, Version 2.0 (the
6// "License"); you may not use this file except in compliance
7// with the License.  You may obtain a copy of the License at
8//
9//   http://www.apache.org/licenses/LICENSE-2.0
10//
11// Unless required by applicable law or agreed to in writing,
12// software distributed under the License is distributed on an
13// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14// KIND, either express or implied.  See the License for the
15// specific language governing permissions and limitations
16// under the License.
17
18// TODO Add specific configs
19//! Storage configuration for storage backends.
20//!
21//! This module provides configuration types for various storage backends.
22//! The configuration types are designed to be used with the `StorageFactory`
23//! trait to create storage instances.
24//!
25//! # Available Configurations
26//!
27//! - [`StorageConfig`]: Base configuration containing properties for storage backends
28//! - [`S3Config`]: Amazon S3 specific configuration
29//! - [`GcsConfig`]: Google Cloud Storage specific configuration
30//! - [`OssConfig`]: Alibaba Cloud OSS specific configuration
31//! - [`AzdlsConfig`]: Azure Data Lake Storage specific configuration
32
33mod azdls;
34mod gcs;
35mod hdfs_native;
36mod hf;
37mod oss;
38mod s3;
39
40use std::collections::HashMap;
41
42pub use azdls::*;
43pub use gcs::*;
44pub use hdfs_native::*;
45pub use hf::*;
46pub use oss::*;
47pub use s3::*;
48use serde::{Deserialize, Serialize};
49
50/// Configuration properties for storage backends.
51///
52/// This struct contains only configuration properties without specifying
53/// which storage backend to use. The storage type is determined by the
54/// explicit factory selection.
55#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize, Default)]
56pub struct StorageConfig {
57    /// Configuration properties for the storage backend
58    props: HashMap<String, String>,
59}
60
61impl StorageConfig {
62    /// Create a new empty StorageConfig.
63    pub fn new() -> Self {
64        Self {
65            props: HashMap::new(),
66        }
67    }
68
69    /// Create a StorageConfig from existing properties.
70    ///
71    /// # Arguments
72    ///
73    /// * `props` - Configuration properties for the storage backend
74    pub fn from_props(props: HashMap<String, String>) -> Self {
75        Self { props }
76    }
77
78    /// Get all configuration properties.
79    pub fn props(&self) -> &HashMap<String, String> {
80        &self.props
81    }
82
83    /// Get a specific configuration property by key.
84    ///
85    /// # Arguments
86    ///
87    /// * `key` - The property key to look up
88    ///
89    /// # Returns
90    ///
91    /// An `Option` containing a reference to the property value if it exists.
92    pub fn get(&self, key: &str) -> Option<&String> {
93        self.props.get(key)
94    }
95
96    /// Add a configuration property.
97    ///
98    /// This is a builder-style method that returns `self` for chaining.
99    ///
100    /// # Arguments
101    ///
102    /// * `key` - The property key
103    /// * `value` - The property value
104    pub fn with_prop(mut self, key: impl Into<String>, value: impl Into<String>) -> Self {
105        self.props.insert(key.into(), value.into());
106        self
107    }
108
109    /// Add multiple configuration properties.
110    ///
111    /// This is a builder-style method that returns `self` for chaining.
112    ///
113    /// # Arguments
114    ///
115    /// * `props` - An iterator of key-value pairs to add
116    pub fn with_props(
117        mut self,
118        props: impl IntoIterator<Item = (impl Into<String>, impl Into<String>)>,
119    ) -> Self {
120        self.props
121            .extend(props.into_iter().map(|(k, v)| (k.into(), v.into())));
122        self
123    }
124}
125
126#[cfg(test)]
127mod tests {
128    use super::*;
129
130    #[test]
131    fn test_storage_config_new() {
132        let config = StorageConfig::new();
133
134        assert!(config.props().is_empty());
135    }
136
137    #[test]
138    fn test_storage_config_from_props() {
139        let props = HashMap::from([
140            ("region".to_string(), "us-east-1".to_string()),
141            ("bucket".to_string(), "my-bucket".to_string()),
142        ]);
143        let config = StorageConfig::from_props(props.clone());
144
145        assert_eq!(config.props(), &props);
146    }
147
148    #[test]
149    fn test_storage_config_default() {
150        let config = StorageConfig::default();
151
152        assert!(config.props().is_empty());
153    }
154
155    #[test]
156    fn test_storage_config_get() {
157        let config = StorageConfig::new().with_prop("region", "us-east-1");
158
159        assert_eq!(config.get("region"), Some(&"us-east-1".to_string()));
160        assert_eq!(config.get("nonexistent"), None);
161    }
162
163    #[test]
164    fn test_storage_config_with_prop() {
165        let config = StorageConfig::new()
166            .with_prop("region", "us-east-1")
167            .with_prop("bucket", "my-bucket");
168
169        assert_eq!(config.get("region"), Some(&"us-east-1".to_string()));
170        assert_eq!(config.get("bucket"), Some(&"my-bucket".to_string()));
171    }
172
173    #[test]
174    fn test_storage_config_with_props() {
175        let additional_props = vec![("key1", "value1"), ("key2", "value2")];
176        let config = StorageConfig::new().with_props(additional_props);
177
178        assert_eq!(config.get("key1"), Some(&"value1".to_string()));
179        assert_eq!(config.get("key2"), Some(&"value2".to_string()));
180    }
181
182    #[test]
183    fn test_storage_config_clone() {
184        let config = StorageConfig::new().with_prop("region", "us-east-1");
185        let cloned = config.clone();
186
187        assert_eq!(config, cloned);
188        assert_eq!(cloned.get("region"), Some(&"us-east-1".to_string()));
189    }
190
191    #[test]
192    fn test_storage_config_serialization_roundtrip() {
193        let config = StorageConfig::new()
194            .with_prop("region", "us-east-1")
195            .with_prop("bucket", "my-bucket");
196
197        let serialized = serde_json::to_string(&config).unwrap();
198        let deserialized: StorageConfig = serde_json::from_str(&serialized).unwrap();
199
200        assert_eq!(config, deserialized);
201    }
202
203    #[test]
204    fn test_storage_config_clone_independence() {
205        let original = StorageConfig::new().with_prop("region", "us-east-1");
206        let mut cloned = original.clone();
207
208        // Modify the clone
209        cloned = cloned.with_prop("region", "eu-west-1");
210        cloned = cloned.with_prop("new_key", "new_value");
211
212        // Original should be unchanged
213        assert_eq!(original.get("region"), Some(&"us-east-1".to_string()));
214        assert_eq!(original.get("new_key"), None);
215
216        // Clone should have the new values
217        assert_eq!(cloned.get("region"), Some(&"eu-west-1".to_string()));
218        assert_eq!(cloned.get("new_key"), Some(&"new_value".to_string()));
219    }
220
221    #[test]
222    fn test_storage_config_from_props_empty() {
223        let config = StorageConfig::from_props(HashMap::new());
224
225        assert!(config.props().is_empty());
226    }
227
228    #[test]
229    fn test_storage_config_serialization_empty() {
230        let config = StorageConfig::new();
231
232        let serialized = serde_json::to_string(&config).unwrap();
233        let deserialized: StorageConfig = serde_json::from_str(&serialized).unwrap();
234
235        assert_eq!(config, deserialized);
236        assert!(deserialized.props().is_empty());
237    }
238}
iceberg/io/storage/config/mod.rs

iceberg/io/storage/config/
mod.rs
