1use std::collections::HashMap;
19use std::num::NonZeroU64;
20use std::sync::Arc;
21
22use iceberg::io::{
23 CLIENT_REGION, S3_ACCESS_KEY_ID, S3_ALLOW_ANONYMOUS, S3_ASSUME_ROLE_ARN,
24 S3_ASSUME_ROLE_EXTERNAL_ID, S3_ASSUME_ROLE_SESSION_NAME, S3_DISABLE_CONFIG_LOAD,
25 S3_DISABLE_EC2_METADATA, S3_ENDPOINT, S3_MULTIPART_PART_SIZE_BYTES, S3_PATH_STYLE_ACCESS,
26 S3_REGION, S3_RETRY_MAX_WAIT_MS, S3_RETRY_MIN_WAIT_MS, S3_RETRY_NUM_RETRIES,
27 S3_SECRET_ACCESS_KEY, S3_SESSION_TOKEN, S3_SSE_KEY, S3_SSE_MD5, S3_SSE_TYPE,
28};
29use iceberg::{Error, ErrorKind, Result};
30use iceberg_property_macro::Properties;
31use opendal::services::S3Config;
32use opendal::{Configurator, Operator};
33pub use reqsign_aws_v4::Credential as AwsCredential;
35pub use reqsign_core::ProvideCredential;
37use reqsign_core::{ProvideCredentialChain, ProvideCredentialDyn};
38use url::Url;
39
40use crate::utils::{from_opendal_error, is_truthy};
41use crate::{OpenDalClientConfig, parse_positive_ms};
42
43#[derive(Properties)]
44struct S3RetryConfig {
45 #[property(key = S3_RETRY_NUM_RETRIES, default = None)]
46 num_retries: Option<usize>,
47 #[property(key = S3_RETRY_MIN_WAIT_MS, default = None, parse_with = parse_positive_ms)]
48 min_wait_ms: Option<NonZeroU64>,
49 #[property(key = S3_RETRY_MAX_WAIT_MS, default = None, parse_with = parse_positive_ms)]
50 max_wait_ms: Option<NonZeroU64>,
51}
52
53pub(crate) fn s3_client_config(
55 props: &HashMap<String, String>,
56 mut client_config: OpenDalClientConfig,
57) -> Result<OpenDalClientConfig> {
58 let retry = S3RetryConfig::from_properties(props)?;
59 client_config.retry_num_retries = retry.num_retries.or(client_config.retry_num_retries);
60 client_config.retry_min_wait_ms = retry.min_wait_ms.or(client_config.retry_min_wait_ms);
61 client_config.retry_max_wait_ms = retry.max_wait_ms.or(client_config.retry_max_wait_ms);
62 Ok(client_config)
63}
64
65const MULTIPART_PART_SIZE_MIN: u64 = 5 * 1024 * 1024;
67
68const MULTIPART_PART_SIZE_MAX: u64 = i32::MAX as u64;
70
71const MULTIPART_PART_SIZE_DEFAULT: u64 = 32 * 1024 * 1024;
73
74pub(crate) fn default_multipart_part_size() -> u64 {
75 MULTIPART_PART_SIZE_DEFAULT
76}
77
78pub(crate) fn s3_multipart_part_size_parse(m: &HashMap<String, String>) -> Result<u64> {
80 let Some(value) = m.get(S3_MULTIPART_PART_SIZE_BYTES) else {
81 return Ok(default_multipart_part_size());
82 };
83 let part_size = value.parse::<u64>().map_err(|e| {
84 Error::new(
85 ErrorKind::DataInvalid,
86 format!("Invalid {S3_MULTIPART_PART_SIZE_BYTES}: {value}: {e}"),
87 )
88 })?;
89 if part_size < MULTIPART_PART_SIZE_MIN {
90 return Err(Error::new(
91 ErrorKind::DataInvalid,
92 format!(
93 "Invalid {S3_MULTIPART_PART_SIZE_BYTES}: {part_size} bytes is below the S3 minimum part size of {} MiB",
94 MULTIPART_PART_SIZE_MIN / (1024 * 1024)
95 ),
96 ));
97 }
98 if part_size > MULTIPART_PART_SIZE_MAX {
99 return Err(Error::new(
100 ErrorKind::DataInvalid,
101 format!(
102 "Invalid {S3_MULTIPART_PART_SIZE_BYTES}: {part_size} bytes is above {MULTIPART_PART_SIZE_MAX} bytes, the largest value Java accepts"
103 ),
104 ));
105 }
106 Ok(part_size)
107}
108
109pub(crate) fn s3_config_parse(mut m: HashMap<String, String>) -> Result<S3Config> {
111 let mut cfg = S3Config::default();
112 cfg.enable_virtual_host_style = true;
118 if let Some(endpoint) = m.remove(S3_ENDPOINT) {
119 cfg.endpoint = Some(endpoint);
120 };
121 if let Some(access_key_id) = m.remove(S3_ACCESS_KEY_ID) {
122 cfg.access_key_id = Some(access_key_id);
123 };
124 if let Some(secret_access_key) = m.remove(S3_SECRET_ACCESS_KEY) {
125 cfg.secret_access_key = Some(secret_access_key);
126 };
127 if let Some(session_token) = m.remove(S3_SESSION_TOKEN) {
128 cfg.session_token = Some(session_token);
129 };
130 if let Some(region) = m.remove(S3_REGION) {
131 cfg.region = Some(region);
132 };
133 if let Some(region) = m.remove(CLIENT_REGION) {
134 cfg.region = Some(region);
135 };
136 if let Some(path_style_access) = m.remove(S3_PATH_STYLE_ACCESS) {
137 cfg.enable_virtual_host_style = !is_truthy(path_style_access.to_lowercase().as_str());
138 };
139 if let Some(arn) = m.remove(S3_ASSUME_ROLE_ARN) {
140 cfg.role_arn = Some(arn);
141 }
142 if let Some(external_id) = m.remove(S3_ASSUME_ROLE_EXTERNAL_ID) {
143 cfg.external_id = Some(external_id);
144 };
145 if let Some(session_name) = m.remove(S3_ASSUME_ROLE_SESSION_NAME) {
146 cfg.role_session_name = Some(session_name);
147 };
148 let s3_sse_key = m.remove(S3_SSE_KEY);
149 if let Some(sse_type) = m.remove(S3_SSE_TYPE) {
150 match sse_type.to_lowercase().as_str() {
151 "none" => {}
153 "s3" => {
155 cfg.server_side_encryption = Some("AES256".to_string());
156 }
157 "kms" => {
159 cfg.server_side_encryption = Some("aws:kms".to_string());
160 cfg.server_side_encryption_aws_kms_key_id = s3_sse_key;
161 }
162 "custom" => {
164 cfg.server_side_encryption_customer_algorithm = Some("AES256".to_string());
165 cfg.server_side_encryption_customer_key = s3_sse_key;
166 cfg.server_side_encryption_customer_key_md5 = m.remove(S3_SSE_MD5);
167 }
168 _ => {
169 return Err(Error::new(
170 ErrorKind::DataInvalid,
171 format!(
172 "Invalid {S3_SSE_TYPE}: {sse_type}. Expected one of (custom, kms, s3, none)"
173 ),
174 ));
175 }
176 }
177 };
178
179 if let Some(allow_anonymous) = m.remove(S3_ALLOW_ANONYMOUS)
180 && is_truthy(allow_anonymous.to_lowercase().as_str())
181 {
182 cfg.skip_signature = true;
183 }
184 if let Some(disable_ec2_metadata) = m.remove(S3_DISABLE_EC2_METADATA)
185 && is_truthy(disable_ec2_metadata.to_lowercase().as_str())
186 {
187 cfg.disable_ec2_metadata = true;
188 };
189 if let Some(disable_config_load) = m.remove(S3_DISABLE_CONFIG_LOAD)
190 && is_truthy(disable_config_load.to_lowercase().as_str())
191 {
192 cfg.disable_config_load = true;
193 };
194
195 Ok(cfg)
196}
197
198pub(crate) fn s3_config_build(
200 cfg: &S3Config,
201 customized_credential_load: &Option<CustomAwsCredentialLoader>,
202 path: &str,
203) -> Result<Operator> {
204 let url = Url::parse(path)?;
205 let bucket = url.host_str().ok_or_else(|| {
206 Error::new(
207 ErrorKind::DataInvalid,
208 format!("Invalid s3 url: {path}, missing bucket"),
209 )
210 })?;
211
212 let mut builder = cfg
213 .clone()
214 .into_builder()
215 .bucket(bucket);
217
218 if let Some(loader) = customized_credential_load {
219 let chain = ProvideCredentialChain::new().push(Arc::clone(&loader.0));
220 builder = builder.credential_provider_chain(chain);
221 }
222
223 Operator::new(builder).map_err(from_opendal_error)
224}
225
226pub struct CustomAwsCredentialLoader(Arc<dyn ProvideCredentialDyn<Credential = AwsCredential>>);
232
233impl Clone for CustomAwsCredentialLoader {
234 fn clone(&self) -> Self {
235 Self(Arc::clone(&self.0))
236 }
237}
238
239impl std::fmt::Debug for CustomAwsCredentialLoader {
240 fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
241 f.debug_struct("CustomAwsCredentialLoader")
242 .finish_non_exhaustive()
243 }
244}
245
246impl CustomAwsCredentialLoader {
247 pub fn new(provider: impl ProvideCredential<Credential = AwsCredential> + 'static) -> Self {
249 Self(Arc::new(provider) as Arc<dyn ProvideCredentialDyn<Credential = AwsCredential>>)
250 }
251}
252
253#[cfg(test)]
254mod tests {
255 use std::collections::HashMap;
256 use std::time::Duration;
257
258 use iceberg::Result;
259 use iceberg::io::{
260 S3_MULTIPART_PART_SIZE_BYTES, S3_PATH_STYLE_ACCESS, S3_RETRY_MAX_WAIT_MS,
261 S3_RETRY_MIN_WAIT_MS, S3_RETRY_NUM_RETRIES,
262 };
263 use opendal::layers::RetryLayer;
264
265 use super::{
266 MULTIPART_PART_SIZE_DEFAULT, MULTIPART_PART_SIZE_MAX, MULTIPART_PART_SIZE_MIN,
267 s3_client_config, s3_config_parse, s3_multipart_part_size_parse,
268 };
269 use crate::{
270 OPENDAL_RETRY_MAX_WAIT_MS, OPENDAL_RETRY_MIN_WAIT_MS, OPENDAL_RETRY_NUM_RETRIES,
271 OpenDalClientConfig,
272 };
273
274 fn parse_with(prop: Option<&str>) -> bool {
275 let mut props = HashMap::new();
276 if let Some(v) = prop {
277 props.insert(S3_PATH_STYLE_ACCESS.to_string(), v.to_string());
278 }
279 s3_config_parse(props).unwrap().enable_virtual_host_style
280 }
281
282 #[test]
283 fn s3_config_parse_path_style_access() {
284 assert!(parse_with(None));
286 assert!(parse_with(Some("false")));
287 assert!(!parse_with(Some("true")));
288 }
289
290 fn parse_part_size(prop: Option<&str>) -> Result<u64> {
291 let mut props = HashMap::new();
292 if let Some(v) = prop {
293 props.insert(S3_MULTIPART_PART_SIZE_BYTES.to_string(), v.to_string());
294 }
295 s3_multipart_part_size_parse(&props)
296 }
297
298 #[test]
299 fn s3_multipart_part_size_parse_defaults_and_overrides() {
300 assert_eq!(MULTIPART_PART_SIZE_DEFAULT, 32 * 1024 * 1024);
302 assert_eq!(parse_part_size(None).unwrap(), MULTIPART_PART_SIZE_DEFAULT);
303 assert_eq!(parse_part_size(Some("8388608")).unwrap(), 8 * 1024 * 1024);
304 }
305
306 #[test]
307 fn s3_multipart_part_size_parse_rejects_invalid() {
308 assert_eq!(MULTIPART_PART_SIZE_MIN, 5 * 1024 * 1024);
310 assert!(parse_part_size(Some(&MULTIPART_PART_SIZE_MIN.to_string())).is_ok());
311 assert!(parse_part_size(Some(&(MULTIPART_PART_SIZE_MIN - 1).to_string())).is_err());
312 assert!(parse_part_size(Some(&MULTIPART_PART_SIZE_MAX.to_string())).is_ok());
313 assert!(parse_part_size(Some(&(MULTIPART_PART_SIZE_MAX + 1).to_string())).is_err());
314 assert!(parse_part_size(Some("not-a-number")).is_err());
315 }
316
317 fn s3_retry_layer(props: &[(&str, &str)]) -> Result<String> {
318 let props = props
319 .iter()
320 .map(|(k, v)| (k.to_string(), v.to_string()))
321 .collect::<HashMap<_, _>>();
322 let client_config =
323 s3_client_config(&props, OpenDalClientConfig::from_properties(&props)?)?;
324 Ok(format!("{:?}", client_config.retry_layer()))
325 }
326
327 #[test]
328 fn test_s3_retry_properties_reach_retry_layer() {
329 let expected = RetryLayer::new()
330 .with_max_times(10)
331 .with_min_delay(Duration::from_secs(2))
332 .with_max_delay(Duration::from_secs(20))
333 .with_jitter();
334 assert_eq!(
335 s3_retry_layer(&[
336 (S3_RETRY_NUM_RETRIES, "10"),
337 (S3_RETRY_MIN_WAIT_MS, "2000"),
338 (S3_RETRY_MAX_WAIT_MS, "20000"),
339 ])
340 .unwrap(),
341 format!("{expected:?}")
342 );
343 assert_eq!(
344 s3_retry_layer(&[(S3_RETRY_NUM_RETRIES, "0")]).unwrap(),
345 format!("{:?}", RetryLayer::new().with_max_times(0).with_jitter())
346 );
347 assert_eq!(
348 s3_retry_layer(&[]).unwrap(),
349 format!("{:?}", RetryLayer::new())
350 );
351 }
352
353 #[test]
354 fn test_s3_retry_properties_win_over_the_generic_ones_per_key() {
355 let expected = RetryLayer::new()
356 .with_max_times(10)
357 .with_min_delay(Duration::from_secs(2))
358 .with_max_delay(Duration::from_secs(20))
359 .with_jitter();
360 assert_eq!(
361 s3_retry_layer(&[
362 (OPENDAL_RETRY_NUM_RETRIES, "5"),
363 (OPENDAL_RETRY_MIN_WAIT_MS, "100"),
364 (OPENDAL_RETRY_MAX_WAIT_MS, "200"),
365 (S3_RETRY_NUM_RETRIES, "10"),
366 (S3_RETRY_MIN_WAIT_MS, "2000"),
367 (S3_RETRY_MAX_WAIT_MS, "20000"),
368 ])
369 .unwrap(),
370 format!("{expected:?}")
371 );
372
373 let mixed = RetryLayer::new()
375 .with_max_times(10)
376 .with_min_delay(Duration::from_millis(100))
377 .with_max_delay(Duration::from_secs(20))
378 .with_jitter();
379 assert_eq!(
380 s3_retry_layer(&[
381 (S3_RETRY_NUM_RETRIES, "10"),
382 (OPENDAL_RETRY_MIN_WAIT_MS, "100"),
383 (OPENDAL_RETRY_MAX_WAIT_MS, "200"),
384 (S3_RETRY_MAX_WAIT_MS, "20000"),
385 ])
386 .unwrap(),
387 format!("{mixed:?}")
388 );
389 }
390
391 #[test]
392 fn test_s3_retry_property_parsing_rejects_invalid_values() {
393 for (key, invalid) in [
394 (S3_RETRY_NUM_RETRIES, "-1"),
395 (S3_RETRY_NUM_RETRIES, "abc"),
396 (S3_RETRY_NUM_RETRIES, ""),
397 (S3_RETRY_MIN_WAIT_MS, "0"),
398 (S3_RETRY_MIN_WAIT_MS, "abc"),
399 (S3_RETRY_MAX_WAIT_MS, "0"),
400 (S3_RETRY_MAX_WAIT_MS, "12.5"),
401 ] {
402 let err = s3_retry_layer(&[(key, invalid)]).unwrap_err().to_string();
403 assert!(err.contains(key), "{key}={invalid}: {err}");
404 }
405 }
406
407 #[cfg(feature = "opendal-memory")]
408 #[test]
409 fn test_other_backends_ignore_the_s3_retry_properties() {
410 use iceberg::io::{StorageConfig, StorageFactory};
411
412 use crate::OpenDalStorageFactory;
413
414 let config = StorageConfig::new().with_props([
415 (S3_RETRY_NUM_RETRIES.to_string(), "abc".to_string()),
416 (OPENDAL_RETRY_NUM_RETRIES.to_string(), "5".to_string()),
417 ]);
418 let storage = OpenDalStorageFactory::Memory.build(&config).unwrap();
419 let json = serde_json::to_string(&storage).unwrap();
420 assert!(json.contains(r#""retry_num_retries":5"#), "{json}");
421 }
422}