iceberg/io/storage/config/
gcs.rs

1// Licensed to the Apache Software Foundation (ASF) under one
2// or more contributor license agreements.  See the NOTICE file
3// distributed with this work for additional information
4// regarding copyright ownership.  The ASF licenses this file
5// to you under the Apache License, Version 2.0 (the
6// "License"); you may not use this file except in compliance
7// with the License.  You may obtain a copy of the License at
8//
9//   http://www.apache.org/licenses/LICENSE-2.0
10//
11// Unless required by applicable law or agreed to in writing,
12// software distributed under the License is distributed on an
13// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14// KIND, either express or implied.  See the License for the
15// specific language governing permissions and limitations
16// under the License.
17
18//! Google Cloud Storage configuration.
19//!
20//! This module provides configuration constants and types for Google Cloud Storage.
21//! Reference: <https://github.com/apache/iceberg/blob/main/gcp/src/main/java/org/apache/iceberg/gcp/GCPProperties.java>
22
23use serde::{Deserialize, Serialize};
24use typed_builder::TypedBuilder;
25
26use super::StorageConfig;
27use crate::Result;
28use crate::io::is_truthy;
29
30/// Google Cloud Project ID.
31pub const GCS_PROJECT_ID: &str = "gcs.project-id";
32/// Google Cloud Storage endpoint.
33pub const GCS_SERVICE_HOST: &str = "gcs.service.host";
34/// Google Cloud user project.
35pub const GCS_USER_PROJECT: &str = "gcs.user-project";
36/// Allow unauthenticated requests.
37pub const GCS_NO_AUTH: &str = "gcs.no-auth";
38/// Google Cloud Storage credentials JSON string, base64 encoded.
39///
40/// E.g. base64::prelude::BASE64_STANDARD.encode(serde_json::to_string(credential).as_bytes())
41pub const GCS_CREDENTIALS_JSON: &str = "gcs.credentials-json";
42/// Google Cloud Storage token.
43pub const GCS_TOKEN: &str = "gcs.oauth2.token";
44/// Option to skip signing requests (e.g. for public buckets/folders).
45pub const GCS_ALLOW_ANONYMOUS: &str = "gcs.allow-anonymous";
46/// Option to skip loading the credential from GCE metadata server.
47pub const GCS_DISABLE_VM_METADATA: &str = "gcs.disable-vm-metadata";
48/// Option to skip loading configuration from config file and the env.
49pub const GCS_DISABLE_CONFIG_LOAD: &str = "gcs.disable-config-load";
50
51/// Google Cloud Storage configuration.
52///
53/// This struct contains all the configuration options for connecting to Google Cloud Storage.
54/// Use the builder pattern via `GcsConfig::builder()` to construct instances.
55#[derive(Clone, Debug, Default, PartialEq, Eq, Serialize, Deserialize, TypedBuilder)]
56pub struct GcsConfig {
57    /// Google Cloud Project ID.
58    #[builder(default, setter(strip_option, into))]
59    pub project_id: Option<String>,
60    /// GCS service endpoint.
61    #[builder(default, setter(strip_option, into))]
62    pub endpoint: Option<String>,
63    /// User project for requester pays buckets.
64    #[builder(default, setter(strip_option, into))]
65    pub user_project: Option<String>,
66    /// Credentials JSON (base64 encoded).
67    #[builder(default, setter(strip_option, into))]
68    pub credential: Option<String>,
69    /// OAuth2 token.
70    #[builder(default, setter(strip_option, into))]
71    pub token: Option<String>,
72    /// Allow anonymous access.
73    #[builder(default)]
74    pub allow_anonymous: bool,
75    /// Disable VM metadata.
76    #[builder(default)]
77    pub disable_vm_metadata: bool,
78    /// Disable config load.
79    #[builder(default)]
80    pub disable_config_load: bool,
81}
82
83impl TryFrom<&StorageConfig> for GcsConfig {
84    type Error = crate::Error;
85
86    fn try_from(config: &StorageConfig) -> Result<Self> {
87        let props = config.props();
88
89        let mut cfg = GcsConfig::default();
90
91        if let Some(project_id) = props.get(GCS_PROJECT_ID) {
92            cfg.project_id = Some(project_id.clone());
93        }
94        if let Some(endpoint) = props.get(GCS_SERVICE_HOST) {
95            cfg.endpoint = Some(endpoint.clone());
96        }
97        if let Some(user_project) = props.get(GCS_USER_PROJECT) {
98            cfg.user_project = Some(user_project.clone());
99        }
100        if let Some(credential) = props.get(GCS_CREDENTIALS_JSON) {
101            cfg.credential = Some(credential.clone());
102        }
103        if let Some(token) = props.get(GCS_TOKEN) {
104            cfg.token = Some(token.clone());
105        }
106
107        // GCS_NO_AUTH enables all anonymous/no-auth options
108        if props.get(GCS_NO_AUTH).is_some() {
109            cfg.allow_anonymous = true;
110            cfg.disable_vm_metadata = true;
111            cfg.disable_config_load = true;
112        }
113
114        if let Some(allow_anonymous) = props.get(GCS_ALLOW_ANONYMOUS)
115            && is_truthy(allow_anonymous.to_lowercase().as_str())
116        {
117            cfg.allow_anonymous = true;
118        }
119        if let Some(disable_vm_metadata) = props.get(GCS_DISABLE_VM_METADATA)
120            && is_truthy(disable_vm_metadata.to_lowercase().as_str())
121        {
122            cfg.disable_vm_metadata = true;
123        }
124        if let Some(disable_config_load) = props.get(GCS_DISABLE_CONFIG_LOAD)
125            && is_truthy(disable_config_load.to_lowercase().as_str())
126        {
127            cfg.disable_config_load = true;
128        }
129
130        Ok(cfg)
131    }
132}
133
134#[cfg(test)]
135mod tests {
136    use super::*;
137
138    #[test]
139    fn test_gcs_config_builder() {
140        let config = GcsConfig::builder()
141            .project_id("my-project")
142            .credential("base64-creds")
143            .endpoint("http://localhost:4443")
144            .build();
145
146        assert_eq!(config.project_id.as_deref(), Some("my-project"));
147        assert_eq!(config.credential.as_deref(), Some("base64-creds"));
148        assert_eq!(config.endpoint.as_deref(), Some("http://localhost:4443"));
149    }
150
151    #[test]
152    fn test_gcs_config_from_storage_config() {
153        let storage_config = StorageConfig::new()
154            .with_prop(GCS_PROJECT_ID, "my-project")
155            .with_prop(GCS_CREDENTIALS_JSON, "base64-creds")
156            .with_prop(GCS_SERVICE_HOST, "http://localhost:4443");
157
158        let gcs_config = GcsConfig::try_from(&storage_config).unwrap();
159
160        assert_eq!(gcs_config.project_id.as_deref(), Some("my-project"));
161        assert_eq!(gcs_config.credential.as_deref(), Some("base64-creds"));
162        assert_eq!(
163            gcs_config.endpoint.as_deref(),
164            Some("http://localhost:4443")
165        );
166    }
167
168    #[test]
169    fn test_gcs_config_no_auth() {
170        let storage_config = StorageConfig::new().with_prop(GCS_NO_AUTH, "true");
171
172        let gcs_config = GcsConfig::try_from(&storage_config).unwrap();
173
174        assert!(gcs_config.allow_anonymous);
175        assert!(gcs_config.disable_vm_metadata);
176        assert!(gcs_config.disable_config_load);
177    }
178
179    #[test]
180    fn test_gcs_config_allow_anonymous() {
181        let storage_config = StorageConfig::new().with_prop(GCS_ALLOW_ANONYMOUS, "true");
182
183        let gcs_config = GcsConfig::try_from(&storage_config).unwrap();
184
185        assert!(gcs_config.allow_anonymous);
186        assert!(!gcs_config.disable_vm_metadata);
187    }
188}
iceberg/io/storage/config/gcs.rs

iceberg/io/storage/config/
gcs.rs
