iceberg/io/storage/config/
mod.rs

1// Licensed to the Apache Software Foundation (ASF) under one
2// or more contributor license agreements.  See the NOTICE file
3// distributed with this work for additional information
4// regarding copyright ownership.  The ASF licenses this file
5// to you under the Apache License, Version 2.0 (the
6// "License"); you may not use this file except in compliance
7// with the License.  You may obtain a copy of the License at
8//
9//   http://www.apache.org/licenses/LICENSE-2.0
10//
11// Unless required by applicable law or agreed to in writing,
12// software distributed under the License is distributed on an
13// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14// KIND, either express or implied.  See the License for the
15// specific language governing permissions and limitations
16// under the License.
17
18// TODO Add specific configs
19//! Storage configuration for storage backends.
20//!
21//! This module provides configuration types for various storage backends.
22//! The configuration types are designed to be used with the `StorageFactory`
23//! trait to create storage instances.
24//!
25//! # Available Configurations
26//!
27//! - [`StorageConfig`]: Base configuration containing properties for storage backends
28//! - [`S3Config`]: Amazon S3 specific configuration
29//! - [`GcsConfig`]: Google Cloud Storage specific configuration
30//! - [`OssConfig`]: Alibaba Cloud OSS specific configuration
31//! - [`AzdlsConfig`]: Azure Data Lake Storage specific configuration
32
33mod azdls;
34mod gcs;
35mod hf;
36mod oss;
37mod s3;
38
39use std::collections::HashMap;
40
41pub use azdls::*;
42pub use gcs::*;
43pub use hf::*;
44pub use oss::*;
45pub use s3::*;
46use serde::{Deserialize, Serialize};
47
48/// Configuration properties for storage backends.
49///
50/// This struct contains only configuration properties without specifying
51/// which storage backend to use. The storage type is determined by the
52/// explicit factory selection.
53/// ```
54#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize, Default)]
55pub struct StorageConfig {
56    /// Configuration properties for the storage backend
57    props: HashMap<String, String>,
58}
59
60impl StorageConfig {
61    /// Create a new empty StorageConfig.
62    pub fn new() -> Self {
63        Self {
64            props: HashMap::new(),
65        }
66    }
67
68    /// Create a StorageConfig from existing properties.
69    ///
70    /// # Arguments
71    ///
72    /// * `props` - Configuration properties for the storage backend
73    pub fn from_props(props: HashMap<String, String>) -> Self {
74        Self { props }
75    }
76
77    /// Get all configuration properties.
78    pub fn props(&self) -> &HashMap<String, String> {
79        &self.props
80    }
81
82    /// Get a specific configuration property by key.
83    ///
84    /// # Arguments
85    ///
86    /// * `key` - The property key to look up
87    ///
88    /// # Returns
89    ///
90    /// An `Option` containing a reference to the property value if it exists.
91    pub fn get(&self, key: &str) -> Option<&String> {
92        self.props.get(key)
93    }
94
95    /// Add a configuration property.
96    ///
97    /// This is a builder-style method that returns `self` for chaining.
98    ///
99    /// # Arguments
100    ///
101    /// * `key` - The property key
102    /// * `value` - The property value
103    pub fn with_prop(mut self, key: impl Into<String>, value: impl Into<String>) -> Self {
104        self.props.insert(key.into(), value.into());
105        self
106    }
107
108    /// Add multiple configuration properties.
109    ///
110    /// This is a builder-style method that returns `self` for chaining.
111    ///
112    /// # Arguments
113    ///
114    /// * `props` - An iterator of key-value pairs to add
115    pub fn with_props(
116        mut self,
117        props: impl IntoIterator<Item = (impl Into<String>, impl Into<String>)>,
118    ) -> Self {
119        self.props
120            .extend(props.into_iter().map(|(k, v)| (k.into(), v.into())));
121        self
122    }
123}
124
125#[cfg(test)]
126mod tests {
127    use super::*;
128
129    #[test]
130    fn test_storage_config_new() {
131        let config = StorageConfig::new();
132
133        assert!(config.props().is_empty());
134    }
135
136    #[test]
137    fn test_storage_config_from_props() {
138        let props = HashMap::from([
139            ("region".to_string(), "us-east-1".to_string()),
140            ("bucket".to_string(), "my-bucket".to_string()),
141        ]);
142        let config = StorageConfig::from_props(props.clone());
143
144        assert_eq!(config.props(), &props);
145    }
146
147    #[test]
148    fn test_storage_config_default() {
149        let config = StorageConfig::default();
150
151        assert!(config.props().is_empty());
152    }
153
154    #[test]
155    fn test_storage_config_get() {
156        let config = StorageConfig::new().with_prop("region", "us-east-1");
157
158        assert_eq!(config.get("region"), Some(&"us-east-1".to_string()));
159        assert_eq!(config.get("nonexistent"), None);
160    }
161
162    #[test]
163    fn test_storage_config_with_prop() {
164        let config = StorageConfig::new()
165            .with_prop("region", "us-east-1")
166            .with_prop("bucket", "my-bucket");
167
168        assert_eq!(config.get("region"), Some(&"us-east-1".to_string()));
169        assert_eq!(config.get("bucket"), Some(&"my-bucket".to_string()));
170    }
171
172    #[test]
173    fn test_storage_config_with_props() {
174        let additional_props = vec![("key1", "value1"), ("key2", "value2")];
175        let config = StorageConfig::new().with_props(additional_props);
176
177        assert_eq!(config.get("key1"), Some(&"value1".to_string()));
178        assert_eq!(config.get("key2"), Some(&"value2".to_string()));
179    }
180
181    #[test]
182    fn test_storage_config_clone() {
183        let config = StorageConfig::new().with_prop("region", "us-east-1");
184        let cloned = config.clone();
185
186        assert_eq!(config, cloned);
187        assert_eq!(cloned.get("region"), Some(&"us-east-1".to_string()));
188    }
189
190    #[test]
191    fn test_storage_config_serialization_roundtrip() {
192        let config = StorageConfig::new()
193            .with_prop("region", "us-east-1")
194            .with_prop("bucket", "my-bucket");
195
196        let serialized = serde_json::to_string(&config).unwrap();
197        let deserialized: StorageConfig = serde_json::from_str(&serialized).unwrap();
198
199        assert_eq!(config, deserialized);
200    }
201
202    #[test]
203    fn test_storage_config_clone_independence() {
204        let original = StorageConfig::new().with_prop("region", "us-east-1");
205        let mut cloned = original.clone();
206
207        // Modify the clone
208        cloned = cloned.with_prop("region", "eu-west-1");
209        cloned = cloned.with_prop("new_key", "new_value");
210
211        // Original should be unchanged
212        assert_eq!(original.get("region"), Some(&"us-east-1".to_string()));
213        assert_eq!(original.get("new_key"), None);
214
215        // Clone should have the new values
216        assert_eq!(cloned.get("region"), Some(&"eu-west-1".to_string()));
217        assert_eq!(cloned.get("new_key"), Some(&"new_value".to_string()));
218    }
219
220    #[test]
221    fn test_storage_config_from_props_empty() {
222        let config = StorageConfig::from_props(HashMap::new());
223
224        assert!(config.props().is_empty());
225    }
226
227    #[test]
228    fn test_storage_config_serialization_empty() {
229        let config = StorageConfig::new();
230
231        let serialized = serde_json::to_string(&config).unwrap();
232        let deserialized: StorageConfig = serde_json::from_str(&serialized).unwrap();
233
234        assert_eq!(config, deserialized);
235        assert!(deserialized.props().is_empty());
236    }
237}
iceberg/io/storage/config/mod.rs

iceberg/io/storage/config/
mod.rs
