iceberg/io/config/
gcs.rs

1// Licensed to the Apache Software Foundation (ASF) under one
2// or more contributor license agreements.  See the NOTICE file
3// distributed with this work for additional information
4// regarding copyright ownership.  The ASF licenses this file
5// to you under the Apache License, Version 2.0 (the
6// "License"); you may not use this file except in compliance
7// with the License.  You may obtain a copy of the License at
8//
9//   http://www.apache.org/licenses/LICENSE-2.0
10//
11// Unless required by applicable law or agreed to in writing,
12// software distributed under the License is distributed on an
13// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14// KIND, either express or implied.  See the License for the
15// specific language governing permissions and limitations
16// under the License.
17
18//! Google Cloud Storage configuration.
19//!
20//! This module provides configuration constants and types for Google Cloud Storage.
21//! Reference: https://github.com/apache/iceberg/blob/main/gcp/src/main/java/org/apache/iceberg/gcp/GCPProperties.java
22
23use serde::{Deserialize, Serialize};
24use typed_builder::TypedBuilder;
25
26use super::StorageConfig;
27use crate::Result;
28use crate::io::is_truthy;
29
30/// Google Cloud Project ID.
31pub const GCS_PROJECT_ID: &str = "gcs.project-id";
32/// Google Cloud Storage endpoint.
33pub const GCS_SERVICE_PATH: &str = "gcs.service.path";
34/// Google Cloud user project.
35pub const GCS_USER_PROJECT: &str = "gcs.user-project";
36/// Allow unauthenticated requests.
37pub const GCS_NO_AUTH: &str = "gcs.no-auth";
38/// Google Cloud Storage credentials JSON string, base64 encoded.
39///
40/// E.g. base64::prelude::BASE64_STANDARD.encode(serde_json::to_string(credential).as_bytes())
41pub const GCS_CREDENTIALS_JSON: &str = "gcs.credentials-json";
42/// Google Cloud Storage token.
43pub const GCS_TOKEN: &str = "gcs.oauth2.token";
44/// Option to skip signing requests (e.g. for public buckets/folders).
45pub const GCS_ALLOW_ANONYMOUS: &str = "gcs.allow-anonymous";
46/// Option to skip loading the credential from GCE metadata server.
47pub const GCS_DISABLE_VM_METADATA: &str = "gcs.disable-vm-metadata";
48/// Option to skip loading configuration from config file and the env.
49pub const GCS_DISABLE_CONFIG_LOAD: &str = "gcs.disable-config-load";
50
51/// Google Cloud Storage configuration.
52///
53/// This struct contains all the configuration options for connecting to Google Cloud Storage.
54/// Use the builder pattern via `GcsConfig::builder()` to construct instances.
55/// ```
56#[derive(Clone, Debug, Default, PartialEq, Eq, Serialize, Deserialize, TypedBuilder)]
57pub struct GcsConfig {
58    /// Google Cloud Project ID.
59    #[builder(default, setter(strip_option, into))]
60    pub project_id: Option<String>,
61    /// GCS service endpoint.
62    #[builder(default, setter(strip_option, into))]
63    pub endpoint: Option<String>,
64    /// User project for requester pays buckets.
65    #[builder(default, setter(strip_option, into))]
66    pub user_project: Option<String>,
67    /// Credentials JSON (base64 encoded).
68    #[builder(default, setter(strip_option, into))]
69    pub credential: Option<String>,
70    /// OAuth2 token.
71    #[builder(default, setter(strip_option, into))]
72    pub token: Option<String>,
73    /// Allow anonymous access.
74    #[builder(default)]
75    pub allow_anonymous: bool,
76    /// Disable VM metadata.
77    #[builder(default)]
78    pub disable_vm_metadata: bool,
79    /// Disable config load.
80    #[builder(default)]
81    pub disable_config_load: bool,
82}
83
84impl TryFrom<&StorageConfig> for GcsConfig {
85    type Error = crate::Error;
86
87    fn try_from(config: &StorageConfig) -> Result<Self> {
88        let props = config.props();
89
90        let mut cfg = GcsConfig::default();
91
92        if let Some(project_id) = props.get(GCS_PROJECT_ID) {
93            cfg.project_id = Some(project_id.clone());
94        }
95        if let Some(endpoint) = props.get(GCS_SERVICE_PATH) {
96            cfg.endpoint = Some(endpoint.clone());
97        }
98        if let Some(user_project) = props.get(GCS_USER_PROJECT) {
99            cfg.user_project = Some(user_project.clone());
100        }
101        if let Some(credential) = props.get(GCS_CREDENTIALS_JSON) {
102            cfg.credential = Some(credential.clone());
103        }
104        if let Some(token) = props.get(GCS_TOKEN) {
105            cfg.token = Some(token.clone());
106        }
107
108        // GCS_NO_AUTH enables all anonymous/no-auth options
109        if props.get(GCS_NO_AUTH).is_some() {
110            cfg.allow_anonymous = true;
111            cfg.disable_vm_metadata = true;
112            cfg.disable_config_load = true;
113        }
114
115        if let Some(allow_anonymous) = props.get(GCS_ALLOW_ANONYMOUS)
116            && is_truthy(allow_anonymous.to_lowercase().as_str())
117        {
118            cfg.allow_anonymous = true;
119        }
120        if let Some(disable_vm_metadata) = props.get(GCS_DISABLE_VM_METADATA)
121            && is_truthy(disable_vm_metadata.to_lowercase().as_str())
122        {
123            cfg.disable_vm_metadata = true;
124        }
125        if let Some(disable_config_load) = props.get(GCS_DISABLE_CONFIG_LOAD)
126            && is_truthy(disable_config_load.to_lowercase().as_str())
127        {
128            cfg.disable_config_load = true;
129        }
130
131        Ok(cfg)
132    }
133}
134
135#[cfg(test)]
136mod tests {
137    use super::*;
138
139    #[test]
140    fn test_gcs_config_builder() {
141        let config = GcsConfig::builder()
142            .project_id("my-project")
143            .credential("base64-creds")
144            .endpoint("http://localhost:4443")
145            .build();
146
147        assert_eq!(config.project_id.as_deref(), Some("my-project"));
148        assert_eq!(config.credential.as_deref(), Some("base64-creds"));
149        assert_eq!(config.endpoint.as_deref(), Some("http://localhost:4443"));
150    }
151
152    #[test]
153    fn test_gcs_config_from_storage_config() {
154        let storage_config = StorageConfig::new()
155            .with_prop(GCS_PROJECT_ID, "my-project")
156            .with_prop(GCS_CREDENTIALS_JSON, "base64-creds")
157            .with_prop(GCS_SERVICE_PATH, "http://localhost:4443");
158
159        let gcs_config = GcsConfig::try_from(&storage_config).unwrap();
160
161        assert_eq!(gcs_config.project_id.as_deref(), Some("my-project"));
162        assert_eq!(gcs_config.credential.as_deref(), Some("base64-creds"));
163        assert_eq!(
164            gcs_config.endpoint.as_deref(),
165            Some("http://localhost:4443")
166        );
167    }
168
169    #[test]
170    fn test_gcs_config_no_auth() {
171        let storage_config = StorageConfig::new().with_prop(GCS_NO_AUTH, "true");
172
173        let gcs_config = GcsConfig::try_from(&storage_config).unwrap();
174
175        assert!(gcs_config.allow_anonymous);
176        assert!(gcs_config.disable_vm_metadata);
177        assert!(gcs_config.disable_config_load);
178    }
179
180    #[test]
181    fn test_gcs_config_allow_anonymous() {
182        let storage_config = StorageConfig::new().with_prop(GCS_ALLOW_ANONYMOUS, "true");
183
184        let gcs_config = GcsConfig::try_from(&storage_config).unwrap();
185
186        assert!(gcs_config.allow_anonymous);
187        assert!(!gcs_config.disable_vm_metadata);
188    }
189}
iceberg/io/config/gcs.rs

iceberg/io/config/
gcs.rs
