iceberg/io/storage/config/
s3.rs1use serde::{Deserialize, Serialize};
24use typed_builder::TypedBuilder;
25
26use super::StorageConfig;
27use crate::error::invalid_data;
28use crate::io::is_truthy;
29use crate::{Error, Result};
30
31pub const S3_ENDPOINT: &str = "s3.endpoint";
33pub const S3_ACCESS_KEY_ID: &str = "s3.access-key-id";
35pub const S3_SECRET_ACCESS_KEY: &str = "s3.secret-access-key";
37pub const S3_SESSION_TOKEN: &str = "s3.session-token";
39pub const S3_REGION: &str = "s3.region";
41pub const CLIENT_REGION: &str = "client.region";
43pub const S3_PATH_STYLE_ACCESS: &str = "s3.path-style-access";
45pub const S3_SSE_TYPE: &str = "s3.sse.type";
47pub const S3_SSE_KEY: &str = "s3.sse.key";
52pub const S3_SSE_MD5: &str = "s3.sse.md5";
54pub const S3_ASSUME_ROLE_ARN: &str = "client.assume-role.arn";
57pub const S3_ASSUME_ROLE_EXTERNAL_ID: &str = "client.assume-role.external-id";
59pub const S3_ASSUME_ROLE_SESSION_NAME: &str = "client.assume-role.session-name";
61pub const S3_ALLOW_ANONYMOUS: &str = "s3.allow-anonymous";
63pub const S3_DISABLE_EC2_METADATA: &str = "s3.disable-ec2-metadata";
66pub const S3_DISABLE_CONFIG_LOAD: &str = "s3.disable-config-load";
68pub const S3_MULTIPART_PART_SIZE_BYTES: &str = "s3.multipart.part-size-bytes";
78pub const S3_RETRY_NUM_RETRIES: &str = "s3.retry.num-retries";
81pub const S3_RETRY_MIN_WAIT_MS: &str = "s3.retry.min-wait-ms";
83pub const S3_RETRY_MAX_WAIT_MS: &str = "s3.retry.max-wait-ms";
85
86#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize, TypedBuilder)]
98pub struct S3Config {
99 #[builder(default, setter(strip_option, into))]
101 pub endpoint: Option<String>,
102 #[builder(default, setter(strip_option, into))]
104 pub access_key_id: Option<String>,
105 #[builder(default, setter(strip_option, into))]
107 pub secret_access_key: Option<String>,
108 #[builder(default, setter(strip_option, into))]
110 pub session_token: Option<String>,
111 #[builder(default, setter(strip_option, into))]
113 pub region: Option<String>,
114 #[builder(default = true)]
118 pub enable_virtual_host_style: bool,
119 #[builder(default, setter(strip_option, into))]
121 pub server_side_encryption: Option<String>,
122 #[builder(default, setter(strip_option, into))]
124 pub server_side_encryption_aws_kms_key_id: Option<String>,
125 #[builder(default, setter(strip_option, into))]
127 pub server_side_encryption_customer_algorithm: Option<String>,
128 #[builder(default, setter(strip_option, into))]
130 pub server_side_encryption_customer_key: Option<String>,
131 #[builder(default, setter(strip_option, into))]
133 pub server_side_encryption_customer_key_md5: Option<String>,
134 #[builder(default, setter(strip_option, into))]
136 pub role_arn: Option<String>,
137 #[builder(default, setter(strip_option, into))]
139 pub external_id: Option<String>,
140 #[builder(default, setter(strip_option, into))]
142 pub role_session_name: Option<String>,
143 #[builder(default)]
145 pub allow_anonymous: bool,
146 #[builder(default)]
148 pub disable_ec2_metadata: bool,
149 #[builder(default)]
151 pub disable_config_load: bool,
152}
153
154impl Default for S3Config {
155 fn default() -> Self {
156 Self::builder().build()
157 }
158}
159
160impl TryFrom<&StorageConfig> for S3Config {
161 type Error = Error;
162
163 fn try_from(config: &StorageConfig) -> Result<Self> {
164 let props = config.props();
165
166 let mut cfg = S3Config::default();
167
168 if let Some(endpoint) = props.get(S3_ENDPOINT) {
169 cfg.endpoint = Some(endpoint.clone());
170 }
171 if let Some(access_key_id) = props.get(S3_ACCESS_KEY_ID) {
172 cfg.access_key_id = Some(access_key_id.clone());
173 }
174 if let Some(secret_access_key) = props.get(S3_SECRET_ACCESS_KEY) {
175 cfg.secret_access_key = Some(secret_access_key.clone());
176 }
177 if let Some(session_token) = props.get(S3_SESSION_TOKEN) {
178 cfg.session_token = Some(session_token.clone());
179 }
180 if let Some(region) = props.get(S3_REGION) {
181 cfg.region = Some(region.clone());
182 }
183 if let Some(region) = props.get(CLIENT_REGION) {
185 cfg.region = Some(region.clone());
186 }
187 if let Some(path_style_access) = props.get(S3_PATH_STYLE_ACCESS) {
188 cfg.enable_virtual_host_style = !is_truthy(path_style_access.to_lowercase().as_str());
189 }
190 if let Some(arn) = props.get(S3_ASSUME_ROLE_ARN) {
191 cfg.role_arn = Some(arn.clone());
192 }
193 if let Some(external_id) = props.get(S3_ASSUME_ROLE_EXTERNAL_ID) {
194 cfg.external_id = Some(external_id.clone());
195 }
196 if let Some(session_name) = props.get(S3_ASSUME_ROLE_SESSION_NAME) {
197 cfg.role_session_name = Some(session_name.clone());
198 }
199
200 let s3_sse_key = props.get(S3_SSE_KEY).cloned();
202 if let Some(sse_type) = props.get(S3_SSE_TYPE) {
203 match sse_type.to_lowercase().as_str() {
204 "none" => {}
206 "s3" => {
208 cfg.server_side_encryption = Some("AES256".to_string());
209 }
210 "kms" => {
212 cfg.server_side_encryption = Some("aws:kms".to_string());
213 cfg.server_side_encryption_aws_kms_key_id = s3_sse_key;
214 }
215 "custom" => {
217 cfg.server_side_encryption_customer_algorithm = Some("AES256".to_string());
218 cfg.server_side_encryption_customer_key = s3_sse_key;
219 cfg.server_side_encryption_customer_key_md5 = props.get(S3_SSE_MD5).cloned();
220 }
221 _ => {
222 return Err(invalid_data!(
223 "Invalid {S3_SSE_TYPE}: {sse_type}. Expected one of (custom, kms, s3, none)"
224 ));
225 }
226 }
227 }
228
229 if let Some(allow_anonymous) = props.get(S3_ALLOW_ANONYMOUS)
230 && is_truthy(allow_anonymous.to_lowercase().as_str())
231 {
232 cfg.allow_anonymous = true;
233 }
234 if let Some(disable_ec2_metadata) = props.get(S3_DISABLE_EC2_METADATA)
235 && is_truthy(disable_ec2_metadata.to_lowercase().as_str())
236 {
237 cfg.disable_ec2_metadata = true;
238 }
239 if let Some(disable_config_load) = props.get(S3_DISABLE_CONFIG_LOAD)
240 && is_truthy(disable_config_load.to_lowercase().as_str())
241 {
242 cfg.disable_config_load = true;
243 }
244
245 Ok(cfg)
246 }
247}
248
249#[cfg(test)]
250mod tests {
251 use super::*;
252
253 #[test]
254 fn test_s3_config_builder() {
255 let config = S3Config::builder()
256 .region("us-east-1")
257 .access_key_id("my-access-key")
258 .secret_access_key("my-secret-key")
259 .endpoint("http://localhost:9000")
260 .build();
261
262 assert_eq!(config.region.as_deref(), Some("us-east-1"));
263 assert_eq!(config.access_key_id.as_deref(), Some("my-access-key"));
264 assert_eq!(config.secret_access_key.as_deref(), Some("my-secret-key"));
265 assert_eq!(config.endpoint.as_deref(), Some("http://localhost:9000"));
266 }
267
268 #[test]
269 fn test_s3_config_from_storage_config() {
270 let storage_config = StorageConfig::new()
271 .with_prop(S3_REGION, "us-east-1")
272 .with_prop(S3_ACCESS_KEY_ID, "my-access-key")
273 .with_prop(S3_SECRET_ACCESS_KEY, "my-secret-key")
274 .with_prop(S3_ENDPOINT, "http://localhost:9000");
275
276 let s3_config = S3Config::try_from(&storage_config).unwrap();
277
278 assert_eq!(s3_config.region.as_deref(), Some("us-east-1"));
279 assert_eq!(s3_config.access_key_id.as_deref(), Some("my-access-key"));
280 assert_eq!(
281 s3_config.secret_access_key.as_deref(),
282 Some("my-secret-key")
283 );
284 assert_eq!(s3_config.endpoint.as_deref(), Some("http://localhost:9000"));
285 }
286
287 #[test]
288 fn test_s3_config_client_region_precedence() {
289 let storage_config = StorageConfig::new()
290 .with_prop(S3_REGION, "us-east-1")
291 .with_prop(CLIENT_REGION, "eu-west-1");
292
293 let s3_config = S3Config::try_from(&storage_config).unwrap();
294
295 assert_eq!(s3_config.region.as_deref(), Some("eu-west-1"));
297 }
298
299 #[test]
300 fn test_s3_config_default_is_virtual_host_style() {
301 assert!(S3Config::default().enable_virtual_host_style);
303 assert!(
304 S3Config::try_from(&StorageConfig::new())
305 .unwrap()
306 .enable_virtual_host_style
307 );
308 }
309
310 #[test]
311 fn test_s3_config_path_style_access() {
312 let storage_config = StorageConfig::new().with_prop(S3_PATH_STYLE_ACCESS, "true");
313
314 let s3_config = S3Config::try_from(&storage_config).unwrap();
315
316 assert!(!s3_config.enable_virtual_host_style);
318 }
319
320 #[test]
321 fn test_s3_config_sse_kms() {
322 let storage_config = StorageConfig::new()
323 .with_prop(S3_SSE_TYPE, "kms")
324 .with_prop(S3_SSE_KEY, "my-kms-key-id");
325
326 let s3_config = S3Config::try_from(&storage_config).unwrap();
327
328 assert_eq!(s3_config.server_side_encryption.as_deref(), Some("aws:kms"));
329 assert_eq!(
330 s3_config.server_side_encryption_aws_kms_key_id.as_deref(),
331 Some("my-kms-key-id")
332 );
333 }
334
335 #[test]
336 fn test_s3_config_allow_anonymous() {
337 let storage_config = StorageConfig::new().with_prop(S3_ALLOW_ANONYMOUS, "true");
338
339 let s3_config = S3Config::try_from(&storage_config).unwrap();
340
341 assert!(s3_config.allow_anonymous);
342 }
343}