Skip to main content

iceberg/io/storage/config/
s3.rs

1// Licensed to the Apache Software Foundation (ASF) under one
2// or more contributor license agreements.  See the NOTICE file
3// distributed with this work for additional information
4// regarding copyright ownership.  The ASF licenses this file
5// to you under the Apache License, Version 2.0 (the
6// "License"); you may not use this file except in compliance
7// with the License.  You may obtain a copy of the License at
8//
9//   http://www.apache.org/licenses/LICENSE-2.0
10//
11// Unless required by applicable law or agreed to in writing,
12// software distributed under the License is distributed on an
13// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14// KIND, either express or implied.  See the License for the
15// specific language governing permissions and limitations
16// under the License.
17
18//! Amazon S3 storage configuration.
19//!
20//! This module provides configuration constants and types for Amazon S3 storage.
21//! These are based on the [Iceberg S3 FileIO configuration](https://py.iceberg.apache.org/configuration/#s3).
22
23use serde::{Deserialize, Serialize};
24use typed_builder::TypedBuilder;
25
26use super::StorageConfig;
27use crate::error::invalid_data;
28use crate::io::is_truthy;
29use crate::{Error, Result};
30
31/// S3 endpoint URL.
32pub const S3_ENDPOINT: &str = "s3.endpoint";
33/// S3 access key ID.
34pub const S3_ACCESS_KEY_ID: &str = "s3.access-key-id";
35/// S3 secret access key.
36pub const S3_SECRET_ACCESS_KEY: &str = "s3.secret-access-key";
37/// S3 session token (required when using temporary credentials).
38pub const S3_SESSION_TOKEN: &str = "s3.session-token";
39/// S3 region.
40pub const S3_REGION: &str = "s3.region";
41/// Region to use for the S3 client (takes precedence over [`S3_REGION`]).
42pub const CLIENT_REGION: &str = "client.region";
43/// S3 Path Style Access.
44pub const S3_PATH_STYLE_ACCESS: &str = "s3.path-style-access";
45/// S3 Server Side Encryption Type.
46pub const S3_SSE_TYPE: &str = "s3.sse.type";
47/// S3 Server Side Encryption Key.
48/// If S3 encryption type is kms, input is a KMS Key ID.
49/// In case this property is not set, default key "aws/s3" is used.
50/// If encryption type is custom, input is a custom base-64 AES256 symmetric key.
51pub const S3_SSE_KEY: &str = "s3.sse.key";
52/// S3 Server Side Encryption MD5.
53pub const S3_SSE_MD5: &str = "s3.sse.md5";
54/// If set, all AWS clients will assume a role of the given ARN, instead of using the default
55/// credential chain.
56pub const S3_ASSUME_ROLE_ARN: &str = "client.assume-role.arn";
57/// Optional external ID used to assume an IAM role.
58pub const S3_ASSUME_ROLE_EXTERNAL_ID: &str = "client.assume-role.external-id";
59/// Optional session name used to assume an IAM role.
60pub const S3_ASSUME_ROLE_SESSION_NAME: &str = "client.assume-role.session-name";
61/// Option to skip signing requests (e.g. for public buckets/folders).
62pub const S3_ALLOW_ANONYMOUS: &str = "s3.allow-anonymous";
63/// Option to skip loading the credential from EC2 metadata (typically used in conjunction with
64/// `S3_ALLOW_ANONYMOUS`).
65pub const S3_DISABLE_EC2_METADATA: &str = "s3.disable-ec2-metadata";
66/// Option to skip loading configuration from config file and the env.
67pub const S3_DISABLE_CONFIG_LOAD: &str = "s3.disable-config-load";
68/// Size in bytes of each part of a multipart upload. Defaults to 32 MiB, matching Java
69/// `S3FileIOProperties.MULTIPART_SIZE_DEFAULT`. Must be at least 5 MiB, the S3 minimum,
70/// and at most `i32::MAX`, since Java reads the property as an `int`.
71///
72/// Each part travels as one request, so the part size bounds how much data has to
73/// transfer within the per-operation IO timeout. Lower it on a slow connection; if a
74/// 5 MiB part still cannot finish in time, raise `opendal.io-timeout-ms` instead.
75///
76/// Only `iceberg-storage-opendal` reads this property, so [`S3Config`] has no matching field.
77pub const S3_MULTIPART_PART_SIZE_BYTES: &str = "s3.multipart.part-size-bytes";
78/// Number of times to retry an S3 operation, `0` to disable retrying. Retries back off exponentially,
79/// with jitter, between [`S3_RETRY_MIN_WAIT_MS`] and [`S3_RETRY_MAX_WAIT_MS`].
80pub const S3_RETRY_NUM_RETRIES: &str = "s3.retry.num-retries";
81/// Minimum wait in milliseconds before retrying an S3 operation.
82pub const S3_RETRY_MIN_WAIT_MS: &str = "s3.retry.min-wait-ms";
83/// Maximum wait in milliseconds between two retries of an S3 operation.
84pub const S3_RETRY_MAX_WAIT_MS: &str = "s3.retry.max-wait-ms";
85
86/// Amazon S3 storage configuration.
87///
88/// This struct contains all the configuration options for connecting to Amazon S3.
89/// Use the builder pattern via `S3Config::builder()` to construct instances.
90///
91/// Defaults follow the Iceberg `S3FileIOProperties` spec (see
92/// [`PATH_STYLE_ACCESS_DEFAULT = false`](https://github.com/apache/iceberg/blob/main/aws/src/main/java/org/apache/iceberg/aws/s3/S3FileIOProperties.java)),
93/// i.e. virtual-host-style addressing is enabled unless
94/// `s3.path-style-access=true` is explicitly set. This matches what
95/// Java clients do out of the box and is required for a number of
96/// S3-compatible stores that do not support path-style URLs.
97#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize, TypedBuilder)]
98pub struct S3Config {
99    /// S3 endpoint URL.
100    #[builder(default, setter(strip_option, into))]
101    pub endpoint: Option<String>,
102    /// S3 access key ID.
103    #[builder(default, setter(strip_option, into))]
104    pub access_key_id: Option<String>,
105    /// S3 secret access key.
106    #[builder(default, setter(strip_option, into))]
107    pub secret_access_key: Option<String>,
108    /// S3 session token.
109    #[builder(default, setter(strip_option, into))]
110    pub session_token: Option<String>,
111    /// S3 region.
112    #[builder(default, setter(strip_option, into))]
113    pub region: Option<String>,
114    /// Enable virtual host style (opposite of path style access).
115    ///
116    /// Defaults to `true` to match Iceberg `S3FileIOProperties.PATH_STYLE_ACCESS_DEFAULT = false`.
117    #[builder(default = true)]
118    pub enable_virtual_host_style: bool,
119    /// Server side encryption type.
120    #[builder(default, setter(strip_option, into))]
121    pub server_side_encryption: Option<String>,
122    /// Server side encryption AWS KMS key ID.
123    #[builder(default, setter(strip_option, into))]
124    pub server_side_encryption_aws_kms_key_id: Option<String>,
125    /// Server side encryption customer algorithm.
126    #[builder(default, setter(strip_option, into))]
127    pub server_side_encryption_customer_algorithm: Option<String>,
128    /// Server side encryption customer key.
129    #[builder(default, setter(strip_option, into))]
130    pub server_side_encryption_customer_key: Option<String>,
131    /// Server side encryption customer key MD5.
132    #[builder(default, setter(strip_option, into))]
133    pub server_side_encryption_customer_key_md5: Option<String>,
134    /// Role ARN for assuming a role.
135    #[builder(default, setter(strip_option, into))]
136    pub role_arn: Option<String>,
137    /// External ID for assuming a role.
138    #[builder(default, setter(strip_option, into))]
139    pub external_id: Option<String>,
140    /// Session name for assuming a role.
141    #[builder(default, setter(strip_option, into))]
142    pub role_session_name: Option<String>,
143    /// Allow anonymous access.
144    #[builder(default)]
145    pub allow_anonymous: bool,
146    /// Disable EC2 metadata.
147    #[builder(default)]
148    pub disable_ec2_metadata: bool,
149    /// Disable config load.
150    #[builder(default)]
151    pub disable_config_load: bool,
152}
153
154impl Default for S3Config {
155    fn default() -> Self {
156        Self::builder().build()
157    }
158}
159
160impl TryFrom<&StorageConfig> for S3Config {
161    type Error = Error;
162
163    fn try_from(config: &StorageConfig) -> Result<Self> {
164        let props = config.props();
165
166        let mut cfg = S3Config::default();
167
168        if let Some(endpoint) = props.get(S3_ENDPOINT) {
169            cfg.endpoint = Some(endpoint.clone());
170        }
171        if let Some(access_key_id) = props.get(S3_ACCESS_KEY_ID) {
172            cfg.access_key_id = Some(access_key_id.clone());
173        }
174        if let Some(secret_access_key) = props.get(S3_SECRET_ACCESS_KEY) {
175            cfg.secret_access_key = Some(secret_access_key.clone());
176        }
177        if let Some(session_token) = props.get(S3_SESSION_TOKEN) {
178            cfg.session_token = Some(session_token.clone());
179        }
180        if let Some(region) = props.get(S3_REGION) {
181            cfg.region = Some(region.clone());
182        }
183        // CLIENT_REGION takes precedence over S3_REGION
184        if let Some(region) = props.get(CLIENT_REGION) {
185            cfg.region = Some(region.clone());
186        }
187        if let Some(path_style_access) = props.get(S3_PATH_STYLE_ACCESS) {
188            cfg.enable_virtual_host_style = !is_truthy(path_style_access.to_lowercase().as_str());
189        }
190        if let Some(arn) = props.get(S3_ASSUME_ROLE_ARN) {
191            cfg.role_arn = Some(arn.clone());
192        }
193        if let Some(external_id) = props.get(S3_ASSUME_ROLE_EXTERNAL_ID) {
194            cfg.external_id = Some(external_id.clone());
195        }
196        if let Some(session_name) = props.get(S3_ASSUME_ROLE_SESSION_NAME) {
197            cfg.role_session_name = Some(session_name.clone());
198        }
199
200        // Handle SSE configuration
201        let s3_sse_key = props.get(S3_SSE_KEY).cloned();
202        if let Some(sse_type) = props.get(S3_SSE_TYPE) {
203            match sse_type.to_lowercase().as_str() {
204                // No Server Side Encryption
205                "none" => {}
206                // S3 SSE-S3 encryption (S3 managed keys). https://docs.aws.amazon.com/AmazonS3/latest/dev/UsingServerSideEncryption.html
207                "s3" => {
208                    cfg.server_side_encryption = Some("AES256".to_string());
209                }
210                // S3 SSE KMS, either using default or custom KMS key. https://docs.aws.amazon.com/AmazonS3/latest/dev/UsingKMSEncryption.html
211                "kms" => {
212                    cfg.server_side_encryption = Some("aws:kms".to_string());
213                    cfg.server_side_encryption_aws_kms_key_id = s3_sse_key;
214                }
215                // S3 SSE-C, using customer managed keys. https://docs.aws.amazon.com/AmazonS3/latest/dev/ServerSideEncryptionCustomerKeys.html
216                "custom" => {
217                    cfg.server_side_encryption_customer_algorithm = Some("AES256".to_string());
218                    cfg.server_side_encryption_customer_key = s3_sse_key;
219                    cfg.server_side_encryption_customer_key_md5 = props.get(S3_SSE_MD5).cloned();
220                }
221                _ => {
222                    return Err(invalid_data!(
223                        "Invalid {S3_SSE_TYPE}: {sse_type}. Expected one of (custom, kms, s3, none)"
224                    ));
225                }
226            }
227        }
228
229        if let Some(allow_anonymous) = props.get(S3_ALLOW_ANONYMOUS)
230            && is_truthy(allow_anonymous.to_lowercase().as_str())
231        {
232            cfg.allow_anonymous = true;
233        }
234        if let Some(disable_ec2_metadata) = props.get(S3_DISABLE_EC2_METADATA)
235            && is_truthy(disable_ec2_metadata.to_lowercase().as_str())
236        {
237            cfg.disable_ec2_metadata = true;
238        }
239        if let Some(disable_config_load) = props.get(S3_DISABLE_CONFIG_LOAD)
240            && is_truthy(disable_config_load.to_lowercase().as_str())
241        {
242            cfg.disable_config_load = true;
243        }
244
245        Ok(cfg)
246    }
247}
248
249#[cfg(test)]
250mod tests {
251    use super::*;
252
253    #[test]
254    fn test_s3_config_builder() {
255        let config = S3Config::builder()
256            .region("us-east-1")
257            .access_key_id("my-access-key")
258            .secret_access_key("my-secret-key")
259            .endpoint("http://localhost:9000")
260            .build();
261
262        assert_eq!(config.region.as_deref(), Some("us-east-1"));
263        assert_eq!(config.access_key_id.as_deref(), Some("my-access-key"));
264        assert_eq!(config.secret_access_key.as_deref(), Some("my-secret-key"));
265        assert_eq!(config.endpoint.as_deref(), Some("http://localhost:9000"));
266    }
267
268    #[test]
269    fn test_s3_config_from_storage_config() {
270        let storage_config = StorageConfig::new()
271            .with_prop(S3_REGION, "us-east-1")
272            .with_prop(S3_ACCESS_KEY_ID, "my-access-key")
273            .with_prop(S3_SECRET_ACCESS_KEY, "my-secret-key")
274            .with_prop(S3_ENDPOINT, "http://localhost:9000");
275
276        let s3_config = S3Config::try_from(&storage_config).unwrap();
277
278        assert_eq!(s3_config.region.as_deref(), Some("us-east-1"));
279        assert_eq!(s3_config.access_key_id.as_deref(), Some("my-access-key"));
280        assert_eq!(
281            s3_config.secret_access_key.as_deref(),
282            Some("my-secret-key")
283        );
284        assert_eq!(s3_config.endpoint.as_deref(), Some("http://localhost:9000"));
285    }
286
287    #[test]
288    fn test_s3_config_client_region_precedence() {
289        let storage_config = StorageConfig::new()
290            .with_prop(S3_REGION, "us-east-1")
291            .with_prop(CLIENT_REGION, "eu-west-1");
292
293        let s3_config = S3Config::try_from(&storage_config).unwrap();
294
295        // CLIENT_REGION should take precedence
296        assert_eq!(s3_config.region.as_deref(), Some("eu-west-1"));
297    }
298
299    #[test]
300    fn test_s3_config_default_is_virtual_host_style() {
301        // Matches Iceberg S3FileIOProperties.PATH_STYLE_ACCESS_DEFAULT = false.
302        assert!(S3Config::default().enable_virtual_host_style);
303        assert!(
304            S3Config::try_from(&StorageConfig::new())
305                .unwrap()
306                .enable_virtual_host_style
307        );
308    }
309
310    #[test]
311    fn test_s3_config_path_style_access() {
312        let storage_config = StorageConfig::new().with_prop(S3_PATH_STYLE_ACCESS, "true");
313
314        let s3_config = S3Config::try_from(&storage_config).unwrap();
315
316        // path style access = true means virtual host style = false
317        assert!(!s3_config.enable_virtual_host_style);
318    }
319
320    #[test]
321    fn test_s3_config_sse_kms() {
322        let storage_config = StorageConfig::new()
323            .with_prop(S3_SSE_TYPE, "kms")
324            .with_prop(S3_SSE_KEY, "my-kms-key-id");
325
326        let s3_config = S3Config::try_from(&storage_config).unwrap();
327
328        assert_eq!(s3_config.server_side_encryption.as_deref(), Some("aws:kms"));
329        assert_eq!(
330            s3_config.server_side_encryption_aws_kms_key_id.as_deref(),
331            Some("my-kms-key-id")
332        );
333    }
334
335    #[test]
336    fn test_s3_config_allow_anonymous() {
337        let storage_config = StorageConfig::new().with_prop(S3_ALLOW_ANONYMOUS, "true");
338
339        let s3_config = S3Config::try_from(&storage_config).unwrap();
340
341        assert!(s3_config.allow_anonymous);
342    }
343}