Skip to main content

iceberg_storage_opendal/
s3.rs

1// Licensed to the Apache Software Foundation (ASF) under one
2// or more contributor license agreements.  See the NOTICE file
3// distributed with this work for additional information
4// regarding copyright ownership.  The ASF licenses this file
5// to you under the Apache License, Version 2.0 (the
6// "License"); you may not use this file except in compliance
7// with the License.  You may obtain a copy of the License at
8//
9//   http://www.apache.org/licenses/LICENSE-2.0
10//
11// Unless required by applicable law or agreed to in writing,
12// software distributed under the License is distributed on an
13// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14// KIND, either express or implied.  See the License for the
15// specific language governing permissions and limitations
16// under the License.
17
18use std::collections::HashMap;
19use std::num::NonZeroU64;
20use std::sync::Arc;
21
22use iceberg::io::{
23    CLIENT_REGION, S3_ACCESS_KEY_ID, S3_ALLOW_ANONYMOUS, S3_ASSUME_ROLE_ARN,
24    S3_ASSUME_ROLE_EXTERNAL_ID, S3_ASSUME_ROLE_SESSION_NAME, S3_DISABLE_CONFIG_LOAD,
25    S3_DISABLE_EC2_METADATA, S3_ENDPOINT, S3_MULTIPART_PART_SIZE_BYTES, S3_PATH_STYLE_ACCESS,
26    S3_REGION, S3_RETRY_MAX_WAIT_MS, S3_RETRY_MIN_WAIT_MS, S3_RETRY_NUM_RETRIES,
27    S3_SECRET_ACCESS_KEY, S3_SESSION_TOKEN, S3_SSE_KEY, S3_SSE_MD5, S3_SSE_TYPE,
28};
29use iceberg::{Error, ErrorKind, Result};
30use iceberg_property_macro::Properties;
31use opendal::services::S3Config;
32use opendal::{Configurator, Operator};
33/// AWS credentials: access key ID, secret access key, and optional session token.
34pub use reqsign_aws_v4::Credential as AwsCredential;
35/// Trait for types that can asynchronously supply [`AwsCredential`] to a [`CustomAwsCredentialLoader`].
36pub use reqsign_core::ProvideCredential;
37use reqsign_core::{ProvideCredentialChain, ProvideCredentialDyn};
38use url::Url;
39
40use crate::utils::{from_opendal_error, is_truthy};
41use crate::{OpenDalClientConfig, parse_positive_ms};
42
43#[derive(Properties)]
44struct S3RetryConfig {
45    #[property(key = S3_RETRY_NUM_RETRIES, default = None)]
46    num_retries: Option<usize>,
47    #[property(key = S3_RETRY_MIN_WAIT_MS, default = None, parse_with = parse_positive_ms)]
48    min_wait_ms: Option<NonZeroU64>,
49    #[property(key = S3_RETRY_MAX_WAIT_MS, default = None, parse_with = parse_positive_ms)]
50    max_wait_ms: Option<NonZeroU64>,
51}
52
53/// Overlays the `s3.retry.*` properties onto the `opendal.retry.*` settings of `client_config`, per key.
54pub(crate) fn s3_client_config(
55    props: &HashMap<String, String>,
56    mut client_config: OpenDalClientConfig,
57) -> Result<OpenDalClientConfig> {
58    let retry = S3RetryConfig::from_properties(props)?;
59    client_config.retry_num_retries = retry.num_retries.or(client_config.retry_num_retries);
60    client_config.retry_min_wait_ms = retry.min_wait_ms.or(client_config.retry_min_wait_ms);
61    client_config.retry_max_wait_ms = retry.max_wait_ms.or(client_config.retry_max_wait_ms);
62    Ok(client_config)
63}
64
65/// S3 rejects a non-final part smaller than this.
66const MULTIPART_PART_SIZE_MIN: u64 = 5 * 1024 * 1024;
67
68/// Java reads this property as an `int`, so a larger value fails there.
69const MULTIPART_PART_SIZE_MAX: u64 = i32::MAX as u64;
70
71/// Matches Java `S3FileIOProperties.MULTIPART_SIZE_DEFAULT`.
72const MULTIPART_PART_SIZE_DEFAULT: u64 = 32 * 1024 * 1024;
73
74pub(crate) fn default_multipart_part_size() -> u64 {
75    MULTIPART_PART_SIZE_DEFAULT
76}
77
78/// Parse iceberg props to s3 multipart upload part size.
79pub(crate) fn s3_multipart_part_size_parse(m: &HashMap<String, String>) -> Result<u64> {
80    let Some(value) = m.get(S3_MULTIPART_PART_SIZE_BYTES) else {
81        return Ok(default_multipart_part_size());
82    };
83    let part_size = value.parse::<u64>().map_err(|e| {
84        Error::new(
85            ErrorKind::DataInvalid,
86            format!("Invalid {S3_MULTIPART_PART_SIZE_BYTES}: {value}: {e}"),
87        )
88    })?;
89    if part_size < MULTIPART_PART_SIZE_MIN {
90        return Err(Error::new(
91            ErrorKind::DataInvalid,
92            format!(
93                "Invalid {S3_MULTIPART_PART_SIZE_BYTES}: {part_size} bytes is below the S3 minimum part size of {} MiB",
94                MULTIPART_PART_SIZE_MIN / (1024 * 1024)
95            ),
96        ));
97    }
98    if part_size > MULTIPART_PART_SIZE_MAX {
99        return Err(Error::new(
100            ErrorKind::DataInvalid,
101            format!(
102                "Invalid {S3_MULTIPART_PART_SIZE_BYTES}: {part_size} bytes is above {MULTIPART_PART_SIZE_MAX} bytes, the largest value Java accepts"
103            ),
104        ));
105    }
106    Ok(part_size)
107}
108
109/// Parse iceberg props to s3 config.
110pub(crate) fn s3_config_parse(mut m: HashMap<String, String>) -> Result<S3Config> {
111    let mut cfg = S3Config::default();
112    // Match Iceberg `S3FileIOProperties.PATH_STYLE_ACCESS_DEFAULT = false`:
113    // virtual-host-style addressing is the spec default. opendal's own
114    // default is path-style, which disagrees with the Java SDK and breaks
115    // S3-compatible stores that only accept virtual-hosted-style URLs.
116    // Any explicit `s3.path-style-access` property below overrides this.
117    cfg.enable_virtual_host_style = true;
118    if let Some(endpoint) = m.remove(S3_ENDPOINT) {
119        cfg.endpoint = Some(endpoint);
120    };
121    if let Some(access_key_id) = m.remove(S3_ACCESS_KEY_ID) {
122        cfg.access_key_id = Some(access_key_id);
123    };
124    if let Some(secret_access_key) = m.remove(S3_SECRET_ACCESS_KEY) {
125        cfg.secret_access_key = Some(secret_access_key);
126    };
127    if let Some(session_token) = m.remove(S3_SESSION_TOKEN) {
128        cfg.session_token = Some(session_token);
129    };
130    if let Some(region) = m.remove(S3_REGION) {
131        cfg.region = Some(region);
132    };
133    if let Some(region) = m.remove(CLIENT_REGION) {
134        cfg.region = Some(region);
135    };
136    if let Some(path_style_access) = m.remove(S3_PATH_STYLE_ACCESS) {
137        cfg.enable_virtual_host_style = !is_truthy(path_style_access.to_lowercase().as_str());
138    };
139    if let Some(arn) = m.remove(S3_ASSUME_ROLE_ARN) {
140        cfg.role_arn = Some(arn);
141    }
142    if let Some(external_id) = m.remove(S3_ASSUME_ROLE_EXTERNAL_ID) {
143        cfg.external_id = Some(external_id);
144    };
145    if let Some(session_name) = m.remove(S3_ASSUME_ROLE_SESSION_NAME) {
146        cfg.role_session_name = Some(session_name);
147    };
148    let s3_sse_key = m.remove(S3_SSE_KEY);
149    if let Some(sse_type) = m.remove(S3_SSE_TYPE) {
150        match sse_type.to_lowercase().as_str() {
151            // No Server Side Encryption
152            "none" => {}
153            // S3 SSE-S3 encryption (S3 managed keys). https://docs.aws.amazon.com/AmazonS3/latest/dev/UsingServerSideEncryption.html
154            "s3" => {
155                cfg.server_side_encryption = Some("AES256".to_string());
156            }
157            // S3 SSE KMS, either using default or custom KMS key. https://docs.aws.amazon.com/AmazonS3/latest/dev/UsingKMSEncryption.html
158            "kms" => {
159                cfg.server_side_encryption = Some("aws:kms".to_string());
160                cfg.server_side_encryption_aws_kms_key_id = s3_sse_key;
161            }
162            // S3 SSE-C, using customer managed keys. https://docs.aws.amazon.com/AmazonS3/latest/dev/ServerSideEncryptionCustomerKeys.html
163            "custom" => {
164                cfg.server_side_encryption_customer_algorithm = Some("AES256".to_string());
165                cfg.server_side_encryption_customer_key = s3_sse_key;
166                cfg.server_side_encryption_customer_key_md5 = m.remove(S3_SSE_MD5);
167            }
168            _ => {
169                return Err(Error::new(
170                    ErrorKind::DataInvalid,
171                    format!(
172                        "Invalid {S3_SSE_TYPE}: {sse_type}. Expected one of (custom, kms, s3, none)"
173                    ),
174                ));
175            }
176        }
177    };
178
179    if let Some(allow_anonymous) = m.remove(S3_ALLOW_ANONYMOUS)
180        && is_truthy(allow_anonymous.to_lowercase().as_str())
181    {
182        cfg.skip_signature = true;
183    }
184    if let Some(disable_ec2_metadata) = m.remove(S3_DISABLE_EC2_METADATA)
185        && is_truthy(disable_ec2_metadata.to_lowercase().as_str())
186    {
187        cfg.disable_ec2_metadata = true;
188    };
189    if let Some(disable_config_load) = m.remove(S3_DISABLE_CONFIG_LOAD)
190        && is_truthy(disable_config_load.to_lowercase().as_str())
191    {
192        cfg.disable_config_load = true;
193    };
194
195    Ok(cfg)
196}
197
198/// Build new opendal operator from give path.
199pub(crate) fn s3_config_build(
200    cfg: &S3Config,
201    customized_credential_load: &Option<CustomAwsCredentialLoader>,
202    path: &str,
203) -> Result<Operator> {
204    let url = Url::parse(path)?;
205    let bucket = url.host_str().ok_or_else(|| {
206        Error::new(
207            ErrorKind::DataInvalid,
208            format!("Invalid s3 url: {path}, missing bucket"),
209        )
210    })?;
211
212    let mut builder = cfg
213        .clone()
214        .into_builder()
215        // Set bucket name.
216        .bucket(bucket);
217
218    if let Some(loader) = customized_credential_load {
219        let chain = ProvideCredentialChain::new().push(Arc::clone(&loader.0));
220        builder = builder.credential_provider_chain(chain);
221    }
222
223    Operator::new(builder).map_err(from_opendal_error)
224}
225
226/// Custom AWS credential loader.
227///
228/// Wraps any [`ProvideCredential`] implementation for use with the S3 storage backend.
229/// Use [`CustomAwsCredentialLoader::new`] to create one, then pass it to
230/// [`OpenDalStorageFactory::S3`](crate::OpenDalStorageFactory).
231pub struct CustomAwsCredentialLoader(Arc<dyn ProvideCredentialDyn<Credential = AwsCredential>>);
232
233impl Clone for CustomAwsCredentialLoader {
234    fn clone(&self) -> Self {
235        Self(Arc::clone(&self.0))
236    }
237}
238
239impl std::fmt::Debug for CustomAwsCredentialLoader {
240    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
241        f.debug_struct("CustomAwsCredentialLoader")
242            .finish_non_exhaustive()
243    }
244}
245
246impl CustomAwsCredentialLoader {
247    /// Create a new custom AWS credential loader from any [`ProvideCredential`] implementation.
248    pub fn new(provider: impl ProvideCredential<Credential = AwsCredential> + 'static) -> Self {
249        Self(Arc::new(provider) as Arc<dyn ProvideCredentialDyn<Credential = AwsCredential>>)
250    }
251}
252
253#[cfg(test)]
254mod tests {
255    use std::collections::HashMap;
256    use std::time::Duration;
257
258    use iceberg::Result;
259    use iceberg::io::{
260        S3_MULTIPART_PART_SIZE_BYTES, S3_PATH_STYLE_ACCESS, S3_RETRY_MAX_WAIT_MS,
261        S3_RETRY_MIN_WAIT_MS, S3_RETRY_NUM_RETRIES,
262    };
263    use opendal::layers::RetryLayer;
264
265    use super::{
266        MULTIPART_PART_SIZE_DEFAULT, MULTIPART_PART_SIZE_MAX, MULTIPART_PART_SIZE_MIN,
267        s3_client_config, s3_config_parse, s3_multipart_part_size_parse,
268    };
269    use crate::{
270        OPENDAL_RETRY_MAX_WAIT_MS, OPENDAL_RETRY_MIN_WAIT_MS, OPENDAL_RETRY_NUM_RETRIES,
271        OpenDalClientConfig,
272    };
273
274    fn parse_with(prop: Option<&str>) -> bool {
275        let mut props = HashMap::new();
276        if let Some(v) = prop {
277            props.insert(S3_PATH_STYLE_ACCESS.to_string(), v.to_string());
278        }
279        s3_config_parse(props).unwrap().enable_virtual_host_style
280    }
281
282    #[test]
283    fn s3_config_parse_path_style_access() {
284        // Match Iceberg S3FileIOProperties.PATH_STYLE_ACCESS_DEFAULT = false.
285        assert!(parse_with(None));
286        assert!(parse_with(Some("false")));
287        assert!(!parse_with(Some("true")));
288    }
289
290    fn parse_part_size(prop: Option<&str>) -> Result<u64> {
291        let mut props = HashMap::new();
292        if let Some(v) = prop {
293            props.insert(S3_MULTIPART_PART_SIZE_BYTES.to_string(), v.to_string());
294        }
295        s3_multipart_part_size_parse(&props)
296    }
297
298    #[test]
299    fn s3_multipart_part_size_parse_defaults_and_overrides() {
300        // Match Iceberg S3FileIOProperties.MULTIPART_SIZE_DEFAULT = 32 MiB.
301        assert_eq!(MULTIPART_PART_SIZE_DEFAULT, 32 * 1024 * 1024);
302        assert_eq!(parse_part_size(None).unwrap(), MULTIPART_PART_SIZE_DEFAULT);
303        assert_eq!(parse_part_size(Some("8388608")).unwrap(), 8 * 1024 * 1024);
304    }
305
306    #[test]
307    fn s3_multipart_part_size_parse_rejects_invalid() {
308        // Match Iceberg S3FileIOProperties.MULTIPART_SIZE_MIN = 5 MiB.
309        assert_eq!(MULTIPART_PART_SIZE_MIN, 5 * 1024 * 1024);
310        assert!(parse_part_size(Some(&MULTIPART_PART_SIZE_MIN.to_string())).is_ok());
311        assert!(parse_part_size(Some(&(MULTIPART_PART_SIZE_MIN - 1).to_string())).is_err());
312        assert!(parse_part_size(Some(&MULTIPART_PART_SIZE_MAX.to_string())).is_ok());
313        assert!(parse_part_size(Some(&(MULTIPART_PART_SIZE_MAX + 1).to_string())).is_err());
314        assert!(parse_part_size(Some("not-a-number")).is_err());
315    }
316
317    fn s3_retry_layer(props: &[(&str, &str)]) -> Result<String> {
318        let props = props
319            .iter()
320            .map(|(k, v)| (k.to_string(), v.to_string()))
321            .collect::<HashMap<_, _>>();
322        let client_config =
323            s3_client_config(&props, OpenDalClientConfig::from_properties(&props)?)?;
324        Ok(format!("{:?}", client_config.retry_layer()))
325    }
326
327    #[test]
328    fn test_s3_retry_properties_reach_retry_layer() {
329        let expected = RetryLayer::new()
330            .with_max_times(10)
331            .with_min_delay(Duration::from_secs(2))
332            .with_max_delay(Duration::from_secs(20))
333            .with_jitter();
334        assert_eq!(
335            s3_retry_layer(&[
336                (S3_RETRY_NUM_RETRIES, "10"),
337                (S3_RETRY_MIN_WAIT_MS, "2000"),
338                (S3_RETRY_MAX_WAIT_MS, "20000"),
339            ])
340            .unwrap(),
341            format!("{expected:?}")
342        );
343        assert_eq!(
344            s3_retry_layer(&[(S3_RETRY_NUM_RETRIES, "0")]).unwrap(),
345            format!("{:?}", RetryLayer::new().with_max_times(0).with_jitter())
346        );
347        assert_eq!(
348            s3_retry_layer(&[]).unwrap(),
349            format!("{:?}", RetryLayer::new())
350        );
351    }
352
353    #[test]
354    fn test_s3_retry_properties_win_over_the_generic_ones_per_key() {
355        let expected = RetryLayer::new()
356            .with_max_times(10)
357            .with_min_delay(Duration::from_secs(2))
358            .with_max_delay(Duration::from_secs(20))
359            .with_jitter();
360        assert_eq!(
361            s3_retry_layer(&[
362                (OPENDAL_RETRY_NUM_RETRIES, "5"),
363                (OPENDAL_RETRY_MIN_WAIT_MS, "100"),
364                (OPENDAL_RETRY_MAX_WAIT_MS, "200"),
365                (S3_RETRY_NUM_RETRIES, "10"),
366                (S3_RETRY_MIN_WAIT_MS, "2000"),
367                (S3_RETRY_MAX_WAIT_MS, "20000"),
368            ])
369            .unwrap(),
370            format!("{expected:?}")
371        );
372
373        // a key set at one layer only is taken from that layer
374        let mixed = RetryLayer::new()
375            .with_max_times(10)
376            .with_min_delay(Duration::from_millis(100))
377            .with_max_delay(Duration::from_secs(20))
378            .with_jitter();
379        assert_eq!(
380            s3_retry_layer(&[
381                (S3_RETRY_NUM_RETRIES, "10"),
382                (OPENDAL_RETRY_MIN_WAIT_MS, "100"),
383                (OPENDAL_RETRY_MAX_WAIT_MS, "200"),
384                (S3_RETRY_MAX_WAIT_MS, "20000"),
385            ])
386            .unwrap(),
387            format!("{mixed:?}")
388        );
389    }
390
391    #[test]
392    fn test_s3_retry_property_parsing_rejects_invalid_values() {
393        for (key, invalid) in [
394            (S3_RETRY_NUM_RETRIES, "-1"),
395            (S3_RETRY_NUM_RETRIES, "abc"),
396            (S3_RETRY_NUM_RETRIES, ""),
397            (S3_RETRY_MIN_WAIT_MS, "0"),
398            (S3_RETRY_MIN_WAIT_MS, "abc"),
399            (S3_RETRY_MAX_WAIT_MS, "0"),
400            (S3_RETRY_MAX_WAIT_MS, "12.5"),
401        ] {
402            let err = s3_retry_layer(&[(key, invalid)]).unwrap_err().to_string();
403            assert!(err.contains(key), "{key}={invalid}: {err}");
404        }
405    }
406
407    #[cfg(feature = "opendal-memory")]
408    #[test]
409    fn test_other_backends_ignore_the_s3_retry_properties() {
410        use iceberg::io::{StorageConfig, StorageFactory};
411
412        use crate::OpenDalStorageFactory;
413
414        let config = StorageConfig::new().with_props([
415            (S3_RETRY_NUM_RETRIES.to_string(), "abc".to_string()),
416            (OPENDAL_RETRY_NUM_RETRIES.to_string(), "5".to_string()),
417        ]);
418        let storage = OpenDalStorageFactory::Memory.build(&config).unwrap();
419        let json = serde_json::to_string(&storage).unwrap();
420        assert!(json.contains(r#""retry_num_retries":5"#), "{json}");
421    }
422}