iceberg/spec/
table_properties.rs

1// Licensed to the Apache Software Foundation (ASF) under one
2// or more contributor license agreements.  See the NOTICE file
3// distributed with this work for additional information
4// regarding copyright ownership.  The ASF licenses this file
5// to you under the Apache License, Version 2.0 (the
6// "License"); you may not use this file except in compliance
7// with the License.  You may obtain a copy of the License at
8//
9//   http://www.apache.org/licenses/LICENSE-2.0
10//
11// Unless required by applicable law or agreed to in writing,
12// software distributed under the License is distributed on an
13// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14// KIND, either express or implied.  See the License for the
15// specific language governing permissions and limitations
16// under the License.
17
18use std::collections::HashMap;
19
20// Helper function to parse a property from a HashMap
21// If the property is not found, use the default value
22fn parse_property<T: std::str::FromStr>(
23    properties: &HashMap<String, String>,
24    key: &str,
25    default: T,
26) -> Result<T, anyhow::Error>
27where
28    <T as std::str::FromStr>::Err: std::fmt::Display,
29{
30    properties.get(key).map_or(Ok(default), |value| {
31        value
32            .parse::<T>()
33            .map_err(|e| anyhow::anyhow!("Invalid value for {key}: {e}"))
34    })
35}
36
37/// TableProperties that contains the properties of a table.
38#[derive(Debug)]
39pub struct TableProperties {
40    /// The number of times to retry a commit.
41    pub commit_num_retries: usize,
42    /// The minimum wait time between retries.
43    pub commit_min_retry_wait_ms: u64,
44    /// The maximum wait time between retries.
45    pub commit_max_retry_wait_ms: u64,
46    /// The total timeout for commit retries.
47    pub commit_total_retry_timeout_ms: u64,
48    /// The default format for files.
49    pub write_format_default: String,
50    /// The target file size for files.
51    pub write_target_file_size_bytes: usize,
52    /// Whether to use `FanoutWriter` for partitioned tables.
53    pub write_datafusion_fanout_enabled: bool,
54}
55
56impl TableProperties {
57    /// Reserved table property for table format version.
58    ///
59    /// Iceberg will default a new table's format version to the latest stable and recommended
60    /// version. This reserved property keyword allows users to override the Iceberg format version of
61    /// the table metadata.
62    ///
63    /// If this table property exists when creating a table, the table will use the specified format
64    /// version. If a table updates this property, it will try to upgrade to the specified format
65    /// version.
66    pub const PROPERTY_FORMAT_VERSION: &str = "format-version";
67    /// Reserved table property for table UUID.
68    pub const PROPERTY_UUID: &str = "uuid";
69    /// Reserved table property for the total number of snapshots.
70    pub const PROPERTY_SNAPSHOT_COUNT: &str = "snapshot-count";
71    /// Reserved table property for current snapshot summary.
72    pub const PROPERTY_CURRENT_SNAPSHOT_SUMMARY: &str = "current-snapshot-summary";
73    /// Reserved table property for current snapshot id.
74    pub const PROPERTY_CURRENT_SNAPSHOT_ID: &str = "current-snapshot-id";
75    /// Reserved table property for current snapshot timestamp.
76    pub const PROPERTY_CURRENT_SNAPSHOT_TIMESTAMP: &str = "current-snapshot-timestamp-ms";
77    /// Reserved table property for the JSON representation of current schema.
78    pub const PROPERTY_CURRENT_SCHEMA: &str = "current-schema";
79    /// Reserved table property for the JSON representation of current(default) partition spec.
80    pub const PROPERTY_DEFAULT_PARTITION_SPEC: &str = "default-partition-spec";
81    /// Reserved table property for the JSON representation of current(default) sort order.
82    pub const PROPERTY_DEFAULT_SORT_ORDER: &str = "default-sort-order";
83
84    /// Property key for max number of previous versions to keep.
85    pub const PROPERTY_METADATA_PREVIOUS_VERSIONS_MAX: &str =
86        "write.metadata.previous-versions-max";
87    /// Default value for max number of previous versions to keep.
88    pub const PROPERTY_METADATA_PREVIOUS_VERSIONS_MAX_DEFAULT: usize = 100;
89
90    /// Property key for max number of partitions to keep summary stats for.
91    pub const PROPERTY_WRITE_PARTITION_SUMMARY_LIMIT: &str = "write.summary.partition-limit";
92    /// Default value for the max number of partitions to keep summary stats for.
93    pub const PROPERTY_WRITE_PARTITION_SUMMARY_LIMIT_DEFAULT: u64 = 0;
94
95    /// Reserved Iceberg table properties list.
96    ///
97    /// Reserved table properties are only used to control behaviors when creating or updating a
98    /// table. The value of these properties are not persisted as a part of the table metadata.
99    pub const RESERVED_PROPERTIES: [&str; 9] = [
100        Self::PROPERTY_FORMAT_VERSION,
101        Self::PROPERTY_UUID,
102        Self::PROPERTY_SNAPSHOT_COUNT,
103        Self::PROPERTY_CURRENT_SNAPSHOT_ID,
104        Self::PROPERTY_CURRENT_SNAPSHOT_SUMMARY,
105        Self::PROPERTY_CURRENT_SNAPSHOT_TIMESTAMP,
106        Self::PROPERTY_CURRENT_SCHEMA,
107        Self::PROPERTY_DEFAULT_PARTITION_SPEC,
108        Self::PROPERTY_DEFAULT_SORT_ORDER,
109    ];
110
111    /// Property key for number of commit retries.
112    pub const PROPERTY_COMMIT_NUM_RETRIES: &str = "commit.retry.num-retries";
113    /// Default value for number of commit retries.
114    pub const PROPERTY_COMMIT_NUM_RETRIES_DEFAULT: usize = 4;
115
116    /// Property key for minimum wait time (ms) between retries.
117    pub const PROPERTY_COMMIT_MIN_RETRY_WAIT_MS: &str = "commit.retry.min-wait-ms";
118    /// Default value for minimum wait time (ms) between retries.
119    pub const PROPERTY_COMMIT_MIN_RETRY_WAIT_MS_DEFAULT: u64 = 100;
120
121    /// Property key for maximum wait time (ms) between retries.
122    pub const PROPERTY_COMMIT_MAX_RETRY_WAIT_MS: &str = "commit.retry.max-wait-ms";
123    /// Default value for maximum wait time (ms) between retries.
124    pub const PROPERTY_COMMIT_MAX_RETRY_WAIT_MS_DEFAULT: u64 = 60 * 1000; // 1 minute
125
126    /// Property key for total maximum retry time (ms).
127    pub const PROPERTY_COMMIT_TOTAL_RETRY_TIME_MS: &str = "commit.retry.total-timeout-ms";
128    /// Default value for total maximum retry time (ms).
129    pub const PROPERTY_COMMIT_TOTAL_RETRY_TIME_MS_DEFAULT: u64 = 30 * 60 * 1000; // 30 minutes
130
131    /// Default file format for data files
132    pub const PROPERTY_DEFAULT_FILE_FORMAT: &str = "write.format.default";
133    /// Default file format for delete files
134    pub const PROPERTY_DELETE_DEFAULT_FILE_FORMAT: &str = "write.delete.format.default";
135    /// Default value for data file format
136    pub const PROPERTY_DEFAULT_FILE_FORMAT_DEFAULT: &str = "parquet";
137
138    /// Target file size for newly written files.
139    pub const PROPERTY_WRITE_TARGET_FILE_SIZE_BYTES: &str = "write.target-file-size-bytes";
140    /// Default target file size
141    pub const PROPERTY_WRITE_TARGET_FILE_SIZE_BYTES_DEFAULT: usize = 512 * 1024 * 1024; // 512 MB
142    /// Whether to use `FanoutWriter` for partitioned tables (handles unsorted data).
143    /// If false, uses `ClusteredWriter` (requires sorted data, more memory efficient).
144    pub const PROPERTY_DATAFUSION_WRITE_FANOUT_ENABLED: &str = "write.datafusion.fanout.enabled";
145    /// Default value for fanout writer enabled
146    pub const PROPERTY_DATAFUSION_WRITE_FANOUT_ENABLED_DEFAULT: bool = true;
147}
148
149impl TryFrom<&HashMap<String, String>> for TableProperties {
150    // parse by entry key or use default value
151    type Error = anyhow::Error;
152
153    fn try_from(props: &HashMap<String, String>) -> Result<Self, Self::Error> {
154        Ok(TableProperties {
155            commit_num_retries: parse_property(
156                props,
157                TableProperties::PROPERTY_COMMIT_NUM_RETRIES,
158                TableProperties::PROPERTY_COMMIT_NUM_RETRIES_DEFAULT,
159            )?,
160            commit_min_retry_wait_ms: parse_property(
161                props,
162                TableProperties::PROPERTY_COMMIT_MIN_RETRY_WAIT_MS,
163                TableProperties::PROPERTY_COMMIT_MIN_RETRY_WAIT_MS_DEFAULT,
164            )?,
165            commit_max_retry_wait_ms: parse_property(
166                props,
167                TableProperties::PROPERTY_COMMIT_MAX_RETRY_WAIT_MS,
168                TableProperties::PROPERTY_COMMIT_MAX_RETRY_WAIT_MS_DEFAULT,
169            )?,
170            commit_total_retry_timeout_ms: parse_property(
171                props,
172                TableProperties::PROPERTY_COMMIT_TOTAL_RETRY_TIME_MS,
173                TableProperties::PROPERTY_COMMIT_TOTAL_RETRY_TIME_MS_DEFAULT,
174            )?,
175            write_format_default: parse_property(
176                props,
177                TableProperties::PROPERTY_DEFAULT_FILE_FORMAT,
178                TableProperties::PROPERTY_DEFAULT_FILE_FORMAT_DEFAULT.to_string(),
179            )?,
180            write_target_file_size_bytes: parse_property(
181                props,
182                TableProperties::PROPERTY_WRITE_TARGET_FILE_SIZE_BYTES,
183                TableProperties::PROPERTY_WRITE_TARGET_FILE_SIZE_BYTES_DEFAULT,
184            )?,
185            write_datafusion_fanout_enabled: parse_property(
186                props,
187                TableProperties::PROPERTY_DATAFUSION_WRITE_FANOUT_ENABLED,
188                TableProperties::PROPERTY_DATAFUSION_WRITE_FANOUT_ENABLED_DEFAULT,
189            )?,
190        })
191    }
192}
193
194#[cfg(test)]
195mod tests {
196    use super::*;
197
198    #[test]
199    fn test_table_properties_default() {
200        let props = HashMap::new();
201        let table_properties = TableProperties::try_from(&props).unwrap();
202        assert_eq!(
203            table_properties.commit_num_retries,
204            TableProperties::PROPERTY_COMMIT_NUM_RETRIES_DEFAULT
205        );
206        assert_eq!(
207            table_properties.commit_min_retry_wait_ms,
208            TableProperties::PROPERTY_COMMIT_MIN_RETRY_WAIT_MS_DEFAULT
209        );
210        assert_eq!(
211            table_properties.commit_max_retry_wait_ms,
212            TableProperties::PROPERTY_COMMIT_MAX_RETRY_WAIT_MS_DEFAULT
213        );
214        assert_eq!(
215            table_properties.write_format_default,
216            TableProperties::PROPERTY_DEFAULT_FILE_FORMAT_DEFAULT.to_string()
217        );
218        assert_eq!(
219            table_properties.write_target_file_size_bytes,
220            TableProperties::PROPERTY_WRITE_TARGET_FILE_SIZE_BYTES_DEFAULT
221        );
222    }
223
224    #[test]
225    fn test_table_properties_valid() {
226        let props = HashMap::from([
227            (
228                TableProperties::PROPERTY_COMMIT_NUM_RETRIES.to_string(),
229                "10".to_string(),
230            ),
231            (
232                TableProperties::PROPERTY_COMMIT_MAX_RETRY_WAIT_MS.to_string(),
233                "20".to_string(),
234            ),
235            (
236                TableProperties::PROPERTY_DEFAULT_FILE_FORMAT.to_string(),
237                "avro".to_string(),
238            ),
239            (
240                TableProperties::PROPERTY_WRITE_TARGET_FILE_SIZE_BYTES.to_string(),
241                "512".to_string(),
242            ),
243        ]);
244        let table_properties = TableProperties::try_from(&props).unwrap();
245        assert_eq!(table_properties.commit_num_retries, 10);
246        assert_eq!(table_properties.commit_max_retry_wait_ms, 20);
247        assert_eq!(table_properties.write_format_default, "avro".to_string());
248        assert_eq!(table_properties.write_target_file_size_bytes, 512);
249    }
250
251    #[test]
252    fn test_table_properties_invalid() {
253        let invalid_retries = HashMap::from([(
254            TableProperties::PROPERTY_COMMIT_NUM_RETRIES.to_string(),
255            "abc".to_string(),
256        )]);
257
258        let table_properties = TableProperties::try_from(&invalid_retries).unwrap_err();
259        assert!(
260            table_properties.to_string().contains(
261                "Invalid value for commit.retry.num-retries: invalid digit found in string"
262            )
263        );
264
265        let invalid_min_wait = HashMap::from([(
266            TableProperties::PROPERTY_COMMIT_MIN_RETRY_WAIT_MS.to_string(),
267            "abc".to_string(),
268        )]);
269        let table_properties = TableProperties::try_from(&invalid_min_wait).unwrap_err();
270        assert!(
271            table_properties.to_string().contains(
272                "Invalid value for commit.retry.min-wait-ms: invalid digit found in string"
273            )
274        );
275
276        let invalid_max_wait = HashMap::from([(
277            TableProperties::PROPERTY_COMMIT_MAX_RETRY_WAIT_MS.to_string(),
278            "abc".to_string(),
279        )]);
280        let table_properties = TableProperties::try_from(&invalid_max_wait).unwrap_err();
281        assert!(
282            table_properties.to_string().contains(
283                "Invalid value for commit.retry.max-wait-ms: invalid digit found in string"
284            )
285        );
286
287        let invalid_target_size = HashMap::from([(
288            TableProperties::PROPERTY_WRITE_TARGET_FILE_SIZE_BYTES.to_string(),
289            "abc".to_string(),
290        )]);
291        let table_properties = TableProperties::try_from(&invalid_target_size).unwrap_err();
292        assert!(table_properties.to_string().contains(
293            "Invalid value for write.target-file-size-bytes: invalid digit found in string"
294        ));
295    }
296}
iceberg/spec/table_properties.rs

iceberg/spec/
table_properties.rs
