iceberg/spec/
table_properties.rs

1// Licensed to the Apache Software Foundation (ASF) under one
2// or more contributor license agreements.  See the NOTICE file
3// distributed with this work for additional information
4// regarding copyright ownership.  The ASF licenses this file
5// to you under the Apache License, Version 2.0 (the
6// "License"); you may not use this file except in compliance
7// with the License.  You may obtain a copy of the License at
8//
9//   http://www.apache.org/licenses/LICENSE-2.0
10//
11// Unless required by applicable law or agreed to in writing,
12// software distributed under the License is distributed on an
13// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14// KIND, either express or implied.  See the License for the
15// specific language governing permissions and limitations
16// under the License.
17
18use std::collections::HashMap;
19
20use iceberg_property_macro::properties_view;
21
22use crate::compression::CompressionCodec;
23use crate::encryption::AesKeySize;
24use crate::error::{Error, ErrorKind, Result};
25use crate::spec::NameMapping;
26use crate::util::location::strip_trailing_slash;
27
28fn parse_location_property(path: &str) -> Result<String> {
29    if path.is_empty() {
30        return Err(Error::new(ErrorKind::DataInvalid, "path must not be empty"));
31    }
32
33    Ok(strip_trailing_slash(path).to_string())
34}
35
36fn parse_metadata_compression(value: &str) -> Result<CompressionCodec> {
37    // Handle empty string as None
38    if value.is_empty() {
39        return Ok(CompressionCodec::None);
40    }
41
42    // Lowercase the value for case-insensitive parsing
43    let lowercase_value = value.to_lowercase();
44
45    // Use serde to parse the codec (which has rename_all = "lowercase")
46    let codec: CompressionCodec = serde_json::from_value(serde_json::Value::String(
47        lowercase_value,
48    ))
49    .map_err(|_| {
50        Error::new(
51            ErrorKind::DataInvalid,
52            format!(
53                "Invalid metadata compression codec: {value}. Only '{}' and '{}' are supported.",
54                CompressionCodec::None.name(),
55                CompressionCodec::gzip_default().name()
56            ),
57        )
58    })?;
59
60    // Validate that only None and Gzip are used for metadata
61    match codec {
62        CompressionCodec::None | CompressionCodec::Gzip(_) => Ok(codec),
63        _ => Err(Error::new(
64            ErrorKind::DataInvalid,
65            format!(
66                "Invalid metadata compression codec: {value}. Only '{}' and '{}' are supported for metadata files.",
67                CompressionCodec::None.name(),
68                CompressionCodec::gzip_default().name()
69            ),
70        )),
71    }
72}
73
74/// Parse the Parquet data-file compression codec (`write.parquet.compression-codec`)
75/// and fold in the compression level (`write.parquet.compression-level`) for the
76/// codecs that accept one (`zstd`, `gzip`, `brotli`).
77fn parse_parquet_compression(
78    properties: &HashMap<String, String>,
79    codec_key: &str,
80    additional_keys: &[&str],
81    default: CompressionCodec,
82) -> Result<CompressionCodec> {
83    let level_key = additional_keys[0];
84    let codec = properties
85        .get(codec_key)
86        .map(|value| {
87            serde_json::from_value(serde_json::Value::String(value.to_lowercase())).map_err(|_| {
88                Error::new(
89                    ErrorKind::DataInvalid,
90                    format!(
91                        "Invalid Parquet compression codec: {value}. Supported codecs: \
92                         uncompressed, snappy, gzip, lzo, brotli, lz4, lz4_raw, zstd"
93                    ),
94                )
95            })
96        })
97        .transpose()?
98        .unwrap_or(default);
99
100    let level = properties
101        .get(level_key)
102        .map(|value| {
103            value.parse::<u8>().map_err(|error| {
104                Error::new(
105                    ErrorKind::DataInvalid,
106                    format!("Invalid value for {level_key}: {error}"),
107                )
108            })
109        })
110        .transpose()?;
111
112    Ok(match (codec, level) {
113        (CompressionCodec::Zstd(_), Some(level)) => CompressionCodec::Zstd(level),
114        (CompressionCodec::Gzip(_), Some(level)) => CompressionCodec::Gzip(level),
115        (CompressionCodec::Brotli(_), Some(level)) => CompressionCodec::Brotli(level),
116        (codec, _) => codec,
117    })
118}
119
120properties_view! {
121/// Typed view over the properties of a table.
122#[derive(Debug)]
123pub struct TableProperties {
124    /// The number of times to retry a commit.
125    #[property(
126        key = Self::PROPERTY_COMMIT_NUM_RETRIES,
127        default = Self::PROPERTY_COMMIT_NUM_RETRIES_DEFAULT,
128        getter
129    )]
130    commit_num_retries: usize,
131    /// The minimum wait time between retries.
132    #[property(
133        key = Self::PROPERTY_COMMIT_MIN_RETRY_WAIT_MS,
134        default = Self::PROPERTY_COMMIT_MIN_RETRY_WAIT_MS_DEFAULT,
135        getter
136    )]
137    commit_min_retry_wait_ms: u64,
138    /// The maximum wait time between retries.
139    #[property(
140        key = Self::PROPERTY_COMMIT_MAX_RETRY_WAIT_MS,
141        default = Self::PROPERTY_COMMIT_MAX_RETRY_WAIT_MS_DEFAULT,
142        getter
143    )]
144    commit_max_retry_wait_ms: u64,
145    /// The total timeout for commit retries.
146    #[property(
147        key = Self::PROPERTY_COMMIT_TOTAL_RETRY_TIME_MS,
148        default = Self::PROPERTY_COMMIT_TOTAL_RETRY_TIME_MS_DEFAULT,
149        getter
150    )]
151    commit_total_retry_timeout_ms: u64,
152    /// The default format for files.
153    #[property(
154        key = Self::PROPERTY_DEFAULT_FILE_FORMAT,
155        default = Self::PROPERTY_DEFAULT_FILE_FORMAT_DEFAULT,
156        getter
157    )]
158    write_format_default: String,
159    /// The target file size for files.
160    #[property(
161        key = Self::PROPERTY_WRITE_TARGET_FILE_SIZE_BYTES,
162        default = Self::PROPERTY_WRITE_TARGET_FILE_SIZE_BYTES_DEFAULT,
163        getter
164    )]
165    write_target_file_size_bytes: usize,
166    /// Base directory for metadata files (manifests, manifest lists), with any
167    /// trailing slash trimmed. `None` if `write.metadata.path` is not set.
168    #[property(
169        key = Self::PROPERTY_WRITE_METADATA_PATH,
170        default = None,
171        parse_with = parse_location_property,
172        getter
173    )]
174    write_metadata_path: Option<String>,
175    /// Compression codec for metadata files (JSON)
176    #[property(
177        key = Self::PROPERTY_METADATA_COMPRESSION_CODEC,
178        default = CompressionCodec::None,
179        parse_with = parse_metadata_compression,
180        getter
181    )]
182    metadata_compression_codec: CompressionCodec,
183    /// Whether to use `FanoutWriter` for partitioned tables.
184    #[property(
185        key = Self::PROPERTY_DATAFUSION_WRITE_FANOUT_ENABLED,
186        default = Self::PROPERTY_DATAFUSION_WRITE_FANOUT_ENABLED_DEFAULT,
187        getter
188    )]
189    write_datafusion_fanout_enabled: bool,
190    /// Whether garbage collection is enabled on drop.
191    /// When `false`, data files will not be deleted when a table is dropped.
192    #[property(
193        key = Self::PROPERTY_GC_ENABLED,
194        default = Self::PROPERTY_GC_ENABLED_DEFAULT,
195        getter
196    )]
197    gc_enabled: bool,
198    /// Default maximum age of a snapshot to keep when expiring snapshots.
199    #[property(
200        key = Self::PROPERTY_MAX_SNAPSHOT_AGE_MS,
201        default = Self::PROPERTY_MAX_SNAPSHOT_AGE_MS_DEFAULT,
202        getter
203    )]
204    max_snapshot_age_ms: i64,
205    /// Default minimum number of snapshots to keep per branch when expiring snapshots.
206    #[property(
207        key = Self::PROPERTY_MIN_SNAPSHOTS_TO_KEEP,
208        default = Self::PROPERTY_MIN_SNAPSHOTS_TO_KEEP_DEFAULT,
209        getter
210    )]
211    min_snapshots_to_keep: usize,
212    /// Default maximum age of a snapshot reference to keep when expiring snapshots.
213    #[property(
214        key = Self::PROPERTY_MAX_REF_AGE_MS,
215        default = Self::PROPERTY_MAX_REF_AGE_MS_DEFAULT,
216        getter
217    )]
218    max_ref_age_ms: i64,
219    /// Whether content-defined chunking is enabled.
220    /// `true` only when `write.parquet.content-defined-chunking.enabled = "true"`.
221    #[property(
222        key = Self::PROPERTY_PARQUET_CDC_ENABLED,
223        default = Self::PROPERTY_PARQUET_CDC_ENABLED_DEFAULT,
224        getter
225    )]
226    cdc_enabled: bool,
227    /// Content-defined chunking minimum chunk size in bytes.
228    #[property(
229        key = Self::PROPERTY_PARQUET_CDC_MIN_CHUNK_SIZE,
230        default = Self::PROPERTY_PARQUET_CDC_MIN_CHUNK_SIZE_DEFAULT,
231        getter
232    )]
233    cdc_min_chunk_size: usize,
234    /// Content-defined chunking maximum chunk size in bytes.
235    #[property(
236        key = Self::PROPERTY_PARQUET_CDC_MAX_CHUNK_SIZE,
237        default = Self::PROPERTY_PARQUET_CDC_MAX_CHUNK_SIZE_DEFAULT,
238        getter
239    )]
240    cdc_max_chunk_size: usize,
241    /// Content-defined chunking normalization level (gearhash bit adjustment).
242    #[property(
243        key = Self::PROPERTY_PARQUET_CDC_NORM_LEVEL,
244        default = Self::PROPERTY_PARQUET_CDC_NORM_LEVEL_DEFAULT,
245        getter
246    )]
247    cdc_norm_level: i32,
248    /// Parquet compression codec for data files, with the resolved compression
249    /// level folded in (from `write.parquet.compression-level`, or the codec's
250    /// default when unset).
251    #[property(
252        key = Self::PROPERTY_PARQUET_COMPRESSION_CODEC,
253        additional_keys = [Self::PROPERTY_PARQUET_COMPRESSION_LEVEL],
254        default = CompressionCodec::zstd_default(),
255        parse_properties_with = parse_parquet_compression,
256        getter
257    )]
258    parquet_compression_codec: CompressionCodec,
259    /// Approximate maximum Parquet row group size in bytes.
260    #[property(
261        key = Self::PROPERTY_PARQUET_ROW_GROUP_SIZE_BYTES,
262        default = Self::PROPERTY_PARQUET_ROW_GROUP_SIZE_BYTES_DEFAULT,
263        getter
264    )]
265    parquet_row_group_size_bytes: usize,
266    /// Approximate maximum Parquet data page size in bytes.
267    #[property(
268        key = Self::PROPERTY_PARQUET_PAGE_SIZE_BYTES,
269        default = Self::PROPERTY_PARQUET_PAGE_SIZE_BYTES_DEFAULT,
270        getter
271    )]
272    parquet_page_size_bytes: usize,
273    /// Maximum number of rows per Parquet data page.
274    #[property(
275        key = Self::PROPERTY_PARQUET_PAGE_ROW_LIMIT,
276        default = Self::PROPERTY_PARQUET_PAGE_ROW_LIMIT_DEFAULT,
277        getter
278    )]
279    parquet_page_row_limit: usize,
280    /// Approximate maximum Parquet dictionary page size in bytes.
281    #[property(
282        key = Self::PROPERTY_PARQUET_DICT_SIZE_BYTES,
283        default = Self::PROPERTY_PARQUET_DICT_SIZE_BYTES_DEFAULT,
284        getter
285    )]
286    parquet_dict_size_bytes: usize,
287    /// The master key id used to encrypt this table's manifest list and data
288    /// files. `None` if `encryption.key-id` is not set.
289    #[property(
290        key = Self::PROPERTY_ENCRYPTION_KEY_ID,
291        default = None,
292        getter
293    )]
294    encryption_key_id: Option<String>,
295    /// The encryption data encryption key length in bytes.
296    #[property(
297        key = Self::PROPERTY_ENCRYPTION_DATA_KEY_LENGTH,
298        default = Self::PROPERTY_ENCRYPTION_DATA_KEY_LENGTH_DEFAULT,
299        getter
300    )]
301    encryption_data_key_length: usize,
302    /// Base directory for data files, with any trailing slash trimmed.
303    #[property(
304        key = Self::PROPERTY_WRITE_DATA_LOCATION,
305        default = None,
306        parse_with = parse_location_property,
307        getter
308    )]
309    write_data_location: Option<String>,
310    /// Deprecated table property for data file write location, with any trailing slash trimmed.
311    ///
312    /// Property will be removed at a later date.
313    /// Superseded by [`TableProperties::write_data_location`].
314    #[property(
315        key = Self::PROPERTY_WRITE_FOLDER_STORAGE_LOCATION,
316        default = None,
317        parse_with = parse_location_property,
318        getter
319    )]
320    write_folder_storage_location: Option<String>,
321    /// Deprecated table property for data file write location for object storage location generator,
322    /// with any trailing slash trimmed.
323    ///
324    /// Property will be removed at a later date.
325    /// Superseded by [`TableProperties::write_data_location`].
326    #[property(
327        key = Self::PROPERTY_WRITE_OBJECT_STORAGE_LOCATION,
328        default = None,
329        parse_with = parse_location_property,
330        getter
331    )]
332    write_object_storage_location: Option<String>,
333    /// Whether partition values are included in object storage paths.
334    #[property(
335        key = Self::PROPERTY_WRITE_OBJECT_STORAGE_PARTITIONED_PATHS,
336        default = Self::PROPERTY_WRITE_OBJECT_STORAGE_PARTITIONED_PATHS_DEFAULT,
337        getter
338    )]
339    write_object_storage_partitioned_paths: bool,
340    /// The table's default name mapping, used to assign field ids when reading data files
341    /// that carry no field id metadata. `None` if `schema.name-mapping.default` is not set.
342    #[property(
343        key = Self::PROPERTY_DEFAULT_NAME_MAPPING,
344        default = None,
345        getter
346    )]
347    default_name_mapping: Option<NameMapping>,
348}
349}
350
351impl TableProperties<'_> {
352    /// Reserved table property for table format version.
353    ///
354    /// Iceberg will default a new table's format version to the latest stable and recommended
355    /// version. This reserved property keyword allows users to override the Iceberg format version of
356    /// the table metadata.
357    ///
358    /// If this table property exists when creating a table, the table will use the specified format
359    /// version. If a table updates this property, it will try to upgrade to the specified format
360    /// version.
361    pub const PROPERTY_FORMAT_VERSION: &'static str = "format-version";
362    /// Reserved table property for table UUID.
363    pub const PROPERTY_UUID: &'static str = "uuid";
364    /// Reserved table property for the total number of snapshots.
365    pub const PROPERTY_SNAPSHOT_COUNT: &'static str = "snapshot-count";
366    /// Reserved table property for current snapshot summary.
367    pub const PROPERTY_CURRENT_SNAPSHOT_SUMMARY: &'static str = "current-snapshot-summary";
368    /// Reserved table property for current snapshot id.
369    pub const PROPERTY_CURRENT_SNAPSHOT_ID: &'static str = "current-snapshot-id";
370    /// Reserved table property for current snapshot timestamp.
371    pub const PROPERTY_CURRENT_SNAPSHOT_TIMESTAMP: &'static str = "current-snapshot-timestamp-ms";
372    /// Reserved table property for the JSON representation of current schema.
373    pub const PROPERTY_CURRENT_SCHEMA: &'static str = "current-schema";
374    /// Reserved table property for the JSON representation of current(default) partition spec.
375    pub const PROPERTY_DEFAULT_PARTITION_SPEC: &'static str = "default-partition-spec";
376    /// Reserved table property for the JSON representation of current(default) sort order.
377    pub const PROPERTY_DEFAULT_SORT_ORDER: &'static str = "default-sort-order";
378
379    /// Property key for max number of previous versions to keep.
380    pub const PROPERTY_METADATA_PREVIOUS_VERSIONS_MAX: &'static str =
381        "write.metadata.previous-versions-max";
382    /// Default value for max number of previous versions to keep.
383    pub const PROPERTY_METADATA_PREVIOUS_VERSIONS_MAX_DEFAULT: usize = 100;
384
385    /// Property key for max number of partitions to keep summary stats for.
386    pub const PROPERTY_WRITE_PARTITION_SUMMARY_LIMIT: &'static str =
387        "write.summary.partition-limit";
388    /// Default value for the max number of partitions to keep summary stats for.
389    pub const PROPERTY_WRITE_PARTITION_SUMMARY_LIMIT_DEFAULT: u64 = 0;
390
391    /// Reserved Iceberg table properties list.
392    ///
393    /// Reserved table properties are only used to control behaviors when creating or updating a
394    /// table. The value of these properties are not persisted as a part of the table metadata.
395    pub const RESERVED_PROPERTIES: [&'static str; 9] = [
396        Self::PROPERTY_FORMAT_VERSION,
397        Self::PROPERTY_UUID,
398        Self::PROPERTY_SNAPSHOT_COUNT,
399        Self::PROPERTY_CURRENT_SNAPSHOT_ID,
400        Self::PROPERTY_CURRENT_SNAPSHOT_SUMMARY,
401        Self::PROPERTY_CURRENT_SNAPSHOT_TIMESTAMP,
402        Self::PROPERTY_CURRENT_SCHEMA,
403        Self::PROPERTY_DEFAULT_PARTITION_SPEC,
404        Self::PROPERTY_DEFAULT_SORT_ORDER,
405    ];
406
407    /// Property key for number of commit retries.
408    pub const PROPERTY_COMMIT_NUM_RETRIES: &'static str = "commit.retry.num-retries";
409    /// Default value for number of commit retries.
410    pub const PROPERTY_COMMIT_NUM_RETRIES_DEFAULT: usize = 4;
411
412    /// Property key for minimum wait time (ms) between retries.
413    pub const PROPERTY_COMMIT_MIN_RETRY_WAIT_MS: &'static str = "commit.retry.min-wait-ms";
414    /// Default value for minimum wait time (ms) between retries.
415    pub const PROPERTY_COMMIT_MIN_RETRY_WAIT_MS_DEFAULT: u64 = 100;
416
417    /// Property key for maximum wait time (ms) between retries.
418    pub const PROPERTY_COMMIT_MAX_RETRY_WAIT_MS: &'static str = "commit.retry.max-wait-ms";
419    /// Default value for maximum wait time (ms) between retries.
420    pub const PROPERTY_COMMIT_MAX_RETRY_WAIT_MS_DEFAULT: u64 = 60 * 1000; // 1 minute
421
422    /// Property key for total maximum retry time (ms).
423    pub const PROPERTY_COMMIT_TOTAL_RETRY_TIME_MS: &'static str = "commit.retry.total-timeout-ms";
424    /// Default value for total maximum retry time (ms).
425    pub const PROPERTY_COMMIT_TOTAL_RETRY_TIME_MS_DEFAULT: u64 = 30 * 60 * 1000; // 30 minutes
426
427    /// Default file format for data files
428    pub const PROPERTY_DEFAULT_FILE_FORMAT: &'static str = "write.format.default";
429    /// Default file format for delete files
430    pub const PROPERTY_DELETE_DEFAULT_FILE_FORMAT: &'static str = "write.delete.format.default";
431    /// Default value for data file format
432    pub const PROPERTY_DEFAULT_FILE_FORMAT_DEFAULT: &'static str = "parquet";
433
434    /// Target file size for newly written files.
435    pub const PROPERTY_WRITE_TARGET_FILE_SIZE_BYTES: &'static str = "write.target-file-size-bytes";
436    /// Default target file size
437    pub const PROPERTY_WRITE_TARGET_FILE_SIZE_BYTES_DEFAULT: usize = 512 * 1024 * 1024; // 512 MB
438
439    /// Base location for metadata files (manifests, manifest lists, table metadata).
440    /// When unset, metadata files default to the `metadata` directory under the table
441    /// location.
442    pub const PROPERTY_WRITE_METADATA_PATH: &'static str = "write.metadata.path";
443
444    /// Property key for the table's default name mapping, stored as a JSON
445    /// [`NameMapping`] document.
446    pub const PROPERTY_DEFAULT_NAME_MAPPING: &'static str = "schema.name-mapping.default";
447
448    /// Compression codec for metadata files (JSON)
449    pub const PROPERTY_METADATA_COMPRESSION_CODEC: &'static str =
450        "write.metadata.compression-codec";
451    /// Default metadata compression codec - uncompressed
452    pub const PROPERTY_METADATA_COMPRESSION_CODEC_DEFAULT: &'static str = "none";
453    /// Whether to use `FanoutWriter` for partitioned tables (handles unsorted data).
454    /// If false, uses `ClusteredWriter` (requires sorted data, more memory efficient).
455    pub const PROPERTY_DATAFUSION_WRITE_FANOUT_ENABLED: &'static str =
456        "write.datafusion.fanout.enabled";
457    /// Default value for fanout writer enabled
458    pub const PROPERTY_DATAFUSION_WRITE_FANOUT_ENABLED_DEFAULT: bool = true;
459
460    /// Property key for enabling garbage collection on drop.
461    /// When set to `false`, data files will not be deleted when a table is dropped.
462    /// Defaults to `true`.
463    pub const PROPERTY_GC_ENABLED: &'static str = "gc.enabled";
464    /// Default value for gc.enabled
465    pub const PROPERTY_GC_ENABLED_DEFAULT: bool = true;
466
467    /// Property key for the default maximum age of a snapshot to keep when expiring snapshots.
468    pub const PROPERTY_MAX_SNAPSHOT_AGE_MS: &'static str = "history.expire.max-snapshot-age-ms";
469    /// Default value for history.expire.max-snapshot-age-ms (5 days).
470    pub const PROPERTY_MAX_SNAPSHOT_AGE_MS_DEFAULT: i64 = 5 * 24 * 60 * 60 * 1000;
471    /// Property key for the default minimum number of snapshots to keep when expiring snapshots.
472    pub const PROPERTY_MIN_SNAPSHOTS_TO_KEEP: &'static str = "history.expire.min-snapshots-to-keep";
473    /// Default value for history.expire.min-snapshots-to-keep.
474    pub const PROPERTY_MIN_SNAPSHOTS_TO_KEEP_DEFAULT: usize = 1;
475    /// Property key for the default maximum age of a snapshot reference to keep when expiring.
476    pub const PROPERTY_MAX_REF_AGE_MS: &'static str = "history.expire.max-ref-age-ms";
477    /// Default value for history.expire.max-ref-age-ms (effectively never expire refs).
478    pub const PROPERTY_MAX_REF_AGE_MS_DEFAULT: i64 = i64::MAX;
479
480    /// Enable content-defined chunking with parquet defaults (or per-property overrides).
481    pub const PROPERTY_PARQUET_CDC_ENABLED: &'static str =
482        "write.parquet.content-defined-chunking.enabled";
483    /// Default value for content-defined chunking enabled.
484    pub const PROPERTY_PARQUET_CDC_ENABLED_DEFAULT: bool = false;
485    /// Minimum chunk size in bytes for content-defined chunking.
486    pub const PROPERTY_PARQUET_CDC_MIN_CHUNK_SIZE: &'static str =
487        "write.parquet.content-defined-chunking.min-chunk-size";
488    /// Default matches `parquet::file::properties::DEFAULT_CDC_MIN_CHUNK_SIZE`.
489    pub const PROPERTY_PARQUET_CDC_MIN_CHUNK_SIZE_DEFAULT: usize = 256 * 1024;
490    /// Maximum chunk size in bytes for content-defined chunking.
491    pub const PROPERTY_PARQUET_CDC_MAX_CHUNK_SIZE: &'static str =
492        "write.parquet.content-defined-chunking.max-chunk-size";
493    /// Default matches `parquet::file::properties::DEFAULT_CDC_MAX_CHUNK_SIZE`.
494    pub const PROPERTY_PARQUET_CDC_MAX_CHUNK_SIZE_DEFAULT: usize = 1024 * 1024;
495    /// Normalization level (gearhash bit adjustment) for content-defined chunking.
496    pub const PROPERTY_PARQUET_CDC_NORM_LEVEL: &'static str =
497        "write.parquet.content-defined-chunking.norm-level";
498    /// Default matches `parquet::file::properties::DEFAULT_CDC_NORM_LEVEL`.
499    pub const PROPERTY_PARQUET_CDC_NORM_LEVEL_DEFAULT: i32 = 0;
500
501    /// Compression codec for Parquet data files (e.g. `zstd`, `gzip`, `snappy`,
502    /// `lz4`, `lz4_raw`, `brotli`, `lzo`, `uncompressed`). The codec name is
503    /// parsed into a [`CompressionCodec`] when properties are parsed; the level's
504    /// range is validated when the writer is built.
505    pub const PROPERTY_PARQUET_COMPRESSION_CODEC: &'static str = "write.parquet.compression-codec";
506    /// Default Parquet compression codec.
507    pub const PROPERTY_PARQUET_COMPRESSION_CODEC_DEFAULT: &'static str = "zstd";
508    /// Compression level for Parquet data files, for codecs that take one
509    /// (`gzip`, `zstd`, `brotli`). When unset, the codec's default level is used.
510    pub const PROPERTY_PARQUET_COMPRESSION_LEVEL: &'static str = "write.parquet.compression-level";
511
512    /// Approximate maximum size of a Parquet row group in bytes.
513    pub const PROPERTY_PARQUET_ROW_GROUP_SIZE_BYTES: &'static str =
514        "write.parquet.row-group-size-bytes";
515    /// Default Parquet row group size in bytes.
516    pub const PROPERTY_PARQUET_ROW_GROUP_SIZE_BYTES_DEFAULT: usize = 128 * 1024 * 1024;
517
518    /// Approximate maximum size of a Parquet data page in bytes.
519    pub const PROPERTY_PARQUET_PAGE_SIZE_BYTES: &'static str = "write.parquet.page-size-bytes";
520    /// Default Parquet page size in bytes.
521    pub const PROPERTY_PARQUET_PAGE_SIZE_BYTES_DEFAULT: usize = 1024 * 1024;
522
523    /// Maximum number of rows per Parquet data page.
524    pub const PROPERTY_PARQUET_PAGE_ROW_LIMIT: &'static str = "write.parquet.page-row-limit";
525    /// Default Parquet page row limit.
526    pub const PROPERTY_PARQUET_PAGE_ROW_LIMIT_DEFAULT: usize = 20000;
527
528    /// Approximate maximum size of the Parquet dictionary page in bytes.
529    pub const PROPERTY_PARQUET_DICT_SIZE_BYTES: &'static str = "write.parquet.dict-size-bytes";
530    /// Default Parquet dictionary page size in bytes.
531    pub const PROPERTY_PARQUET_DICT_SIZE_BYTES_DEFAULT: usize = 2 * 1024 * 1024;
532
533    /// Property key for the master key id used to encrypt the table's manifest
534    /// list and data files as defined in <https://iceberg.apache.org/docs/nightly/encryption/>.
535    pub const PROPERTY_ENCRYPTION_KEY_ID: &'static str = "encryption.key-id";
536
537    /// Property key for the encryption data encryption key (DEK) length in bytes.
538    pub const PROPERTY_ENCRYPTION_DATA_KEY_LENGTH: &'static str = "encryption.data-key-length";
539    /// Default value for the encryption DEK length (16 bytes = AES-128).
540    pub const PROPERTY_ENCRYPTION_DATA_KEY_LENGTH_DEFAULT: usize = 16;
541    /// Property key for the base directory for data files
542    pub const PROPERTY_WRITE_DATA_LOCATION: &'static str = "write.data.path";
543    /// Property key for deprecated [`TableProperties::write_folder_storage_location`]
544    pub const PROPERTY_WRITE_FOLDER_STORAGE_LOCATION: &'static str = "write.folder-storage.path";
545    /// Property key for deprecated object storage path, kept as a fallback for compatibility.
546    pub const PROPERTY_WRITE_OBJECT_STORAGE_LOCATION: &'static str = "write.object-storage.path";
547    /// Property key for controlling whether partition values are included in object storage paths.
548    pub const PROPERTY_WRITE_OBJECT_STORAGE_PARTITIONED_PATHS: &'static str =
549        "write.object-storage.partitioned-paths";
550    /// Default value for [`TableProperties::PROPERTY_WRITE_OBJECT_STORAGE_PARTITIONED_PATHS`]
551    pub const PROPERTY_WRITE_OBJECT_STORAGE_PARTITIONED_PATHS_DEFAULT: bool = true;
552
553    /// The AES key size to use when generating data encryption keys, derived
554    /// from `encryption.data-key-length`.
555    pub fn data_encryption_key_size(&self) -> Result<AesKeySize> {
556        AesKeySize::from_key_length(self.encryption_data_key_length()?)
557    }
558}
559
560#[cfg(test)]
561mod tests {
562    use super::*;
563    use crate::compression::CompressionCodec;
564
565    #[test]
566    fn test_table_properties_default() {
567        let props = HashMap::new();
568        let table_properties = TableProperties::new(&props);
569        assert_eq!(
570            table_properties.commit_num_retries().unwrap(),
571            TableProperties::PROPERTY_COMMIT_NUM_RETRIES_DEFAULT
572        );
573        assert_eq!(
574            table_properties.commit_min_retry_wait_ms().unwrap(),
575            TableProperties::PROPERTY_COMMIT_MIN_RETRY_WAIT_MS_DEFAULT
576        );
577        assert_eq!(
578            table_properties.commit_max_retry_wait_ms().unwrap(),
579            TableProperties::PROPERTY_COMMIT_MAX_RETRY_WAIT_MS_DEFAULT
580        );
581        assert_eq!(
582            table_properties.write_format_default().unwrap(),
583            TableProperties::PROPERTY_DEFAULT_FILE_FORMAT_DEFAULT.to_string()
584        );
585        assert_eq!(
586            table_properties.write_target_file_size_bytes().unwrap(),
587            TableProperties::PROPERTY_WRITE_TARGET_FILE_SIZE_BYTES_DEFAULT
588        );
589        // Test compression defaults (none means CompressionCodec::None)
590        assert_eq!(
591            table_properties.metadata_compression_codec().unwrap(),
592            CompressionCodec::None
593        );
594        assert_eq!(
595            table_properties.gc_enabled().unwrap(),
596            TableProperties::PROPERTY_GC_ENABLED_DEFAULT
597        );
598        assert_eq!(
599            table_properties.max_snapshot_age_ms().unwrap(),
600            TableProperties::PROPERTY_MAX_SNAPSHOT_AGE_MS_DEFAULT
601        );
602        assert_eq!(
603            table_properties.min_snapshots_to_keep().unwrap(),
604            TableProperties::PROPERTY_MIN_SNAPSHOTS_TO_KEEP_DEFAULT
605        );
606        assert_eq!(
607            table_properties.max_ref_age_ms().unwrap(),
608            TableProperties::PROPERTY_MAX_REF_AGE_MS_DEFAULT
609        );
610    }
611
612    #[test]
613    fn test_table_properties_history_expire_overrides() {
614        let props = HashMap::from([
615            (
616                TableProperties::PROPERTY_MAX_SNAPSHOT_AGE_MS.to_string(),
617                "1234".to_string(),
618            ),
619            (
620                TableProperties::PROPERTY_MIN_SNAPSHOTS_TO_KEEP.to_string(),
621                "7".to_string(),
622            ),
623            (
624                TableProperties::PROPERTY_MAX_REF_AGE_MS.to_string(),
625                "5678".to_string(),
626            ),
627        ]);
628        let table_properties = TableProperties::new(&props);
629        assert_eq!(table_properties.max_snapshot_age_ms().unwrap(), 1234);
630        assert_eq!(table_properties.min_snapshots_to_keep().unwrap(), 7);
631        assert_eq!(table_properties.max_ref_age_ms().unwrap(), 5678);
632    }
633
634    #[test]
635    fn test_table_properties_location_paths() {
636        // Test unset.
637        let raw_properties = HashMap::new();
638        let table_properties = TableProperties::new(&raw_properties);
639        assert_eq!(table_properties.write_metadata_path().unwrap(), None);
640        assert_eq!(table_properties.write_data_location().unwrap(), None);
641        assert_eq!(
642            table_properties.write_folder_storage_location().unwrap(),
643            None
644        );
645        assert_eq!(
646            table_properties.write_object_storage_location().unwrap(),
647            None
648        );
649
650        for key in [
651            TableProperties::PROPERTY_WRITE_METADATA_PATH,
652            TableProperties::PROPERTY_WRITE_DATA_LOCATION,
653            TableProperties::PROPERTY_WRITE_FOLDER_STORAGE_LOCATION,
654            TableProperties::PROPERTY_WRITE_OBJECT_STORAGE_LOCATION,
655        ] {
656            // Test empty paths are invalid and retain the property key as error context.
657            let raw_properties = HashMap::from([(key.to_string(), String::new())]);
658            let table_properties = TableProperties::new(&raw_properties);
659            let error = match key {
660                TableProperties::PROPERTY_WRITE_METADATA_PATH => {
661                    table_properties.write_metadata_path().unwrap_err()
662                }
663                TableProperties::PROPERTY_WRITE_DATA_LOCATION => {
664                    table_properties.write_data_location().unwrap_err()
665                }
666                TableProperties::PROPERTY_WRITE_FOLDER_STORAGE_LOCATION => table_properties
667                    .write_folder_storage_location()
668                    .unwrap_err(),
669                TableProperties::PROPERTY_WRITE_OBJECT_STORAGE_LOCATION => table_properties
670                    .write_object_storage_location()
671                    .unwrap_err(),
672                _ => unreachable!(),
673            };
674            assert_eq!(error.kind(), ErrorKind::DataInvalid);
675            assert!(format!("{error}").contains(key));
676
677            // Test all supported location properties share trailing-slash normalization.
678            let raw_properties = HashMap::from([(
679                key.to_string(),
680                "s3://other-bucket/custom-path/".to_string(),
681            )]);
682            let table_properties = TableProperties::new(&raw_properties);
683            let parsed = match key {
684                TableProperties::PROPERTY_WRITE_METADATA_PATH => {
685                    table_properties.write_metadata_path().unwrap()
686                }
687                TableProperties::PROPERTY_WRITE_DATA_LOCATION => {
688                    table_properties.write_data_location().unwrap()
689                }
690                TableProperties::PROPERTY_WRITE_FOLDER_STORAGE_LOCATION => {
691                    table_properties.write_folder_storage_location().unwrap()
692                }
693                TableProperties::PROPERTY_WRITE_OBJECT_STORAGE_LOCATION => {
694                    table_properties.write_object_storage_location().unwrap()
695                }
696                _ => unreachable!(),
697            };
698            assert_eq!(parsed.as_deref(), Some("s3://other-bucket/custom-path"));
699        }
700    }
701
702    #[test]
703    fn test_table_properties_compression() {
704        let props = HashMap::from([(
705            TableProperties::PROPERTY_METADATA_COMPRESSION_CODEC.to_string(),
706            "gzip".to_string(),
707        )]);
708        let table_properties = TableProperties::new(&props);
709        assert_eq!(
710            table_properties.metadata_compression_codec().unwrap(),
711            CompressionCodec::gzip_default()
712        );
713    }
714
715    #[test]
716    fn test_table_properties_compression_none() {
717        let props = HashMap::from([(
718            TableProperties::PROPERTY_METADATA_COMPRESSION_CODEC.to_string(),
719            "none".to_string(),
720        )]);
721        let table_properties = TableProperties::new(&props);
722        assert_eq!(
723            table_properties.metadata_compression_codec().unwrap(),
724            CompressionCodec::None
725        );
726    }
727
728    #[test]
729    fn test_table_properties_compression_case_insensitive() {
730        // Test uppercase
731        let props_upper = HashMap::from([(
732            TableProperties::PROPERTY_METADATA_COMPRESSION_CODEC.to_string(),
733            "GZIP".to_string(),
734        )]);
735        let table_properties = TableProperties::new(&props_upper);
736        assert_eq!(
737            table_properties.metadata_compression_codec().unwrap(),
738            CompressionCodec::gzip_default()
739        );
740
741        // Test mixed case
742        let props_mixed = HashMap::from([(
743            TableProperties::PROPERTY_METADATA_COMPRESSION_CODEC.to_string(),
744            "GzIp".to_string(),
745        )]);
746        let table_properties = TableProperties::new(&props_mixed);
747        assert_eq!(
748            table_properties.metadata_compression_codec().unwrap(),
749            CompressionCodec::gzip_default()
750        );
751
752        // Test "NONE" should also be case-insensitive
753        let props_none_upper = HashMap::from([(
754            TableProperties::PROPERTY_METADATA_COMPRESSION_CODEC.to_string(),
755            "NONE".to_string(),
756        )]);
757        let table_properties = TableProperties::new(&props_none_upper);
758        assert_eq!(
759            table_properties.metadata_compression_codec().unwrap(),
760            CompressionCodec::None
761        );
762    }
763
764    #[test]
765    fn test_table_properties_valid() {
766        let props = HashMap::from([
767            (
768                TableProperties::PROPERTY_COMMIT_NUM_RETRIES.to_string(),
769                "10".to_string(),
770            ),
771            (
772                TableProperties::PROPERTY_COMMIT_MAX_RETRY_WAIT_MS.to_string(),
773                "20".to_string(),
774            ),
775            (
776                TableProperties::PROPERTY_DEFAULT_FILE_FORMAT.to_string(),
777                "avro".to_string(),
778            ),
779            (
780                TableProperties::PROPERTY_WRITE_TARGET_FILE_SIZE_BYTES.to_string(),
781                "512".to_string(),
782            ),
783            (
784                TableProperties::PROPERTY_GC_ENABLED.to_string(),
785                "false".to_string(),
786            ),
787        ]);
788        let table_properties = TableProperties::new(&props);
789        assert_eq!(table_properties.commit_num_retries().unwrap(), 10);
790        assert_eq!(table_properties.commit_max_retry_wait_ms().unwrap(), 20);
791        assert_eq!(
792            table_properties.write_format_default().unwrap(),
793            "avro".to_string()
794        );
795        assert_eq!(
796            table_properties.write_target_file_size_bytes().unwrap(),
797            512
798        );
799        assert!(!table_properties.gc_enabled().unwrap());
800    }
801
802    #[test]
803    fn test_table_properties_invalid() {
804        let invalid_retries = HashMap::from([(
805            TableProperties::PROPERTY_COMMIT_NUM_RETRIES.to_string(),
806            "abc".to_string(),
807        )]);
808
809        let table_properties = TableProperties::new(&invalid_retries);
810        let error = table_properties.commit_num_retries().unwrap_err();
811        assert!(
812            error.to_string().contains(
813                "Invalid value for commit.retry.num-retries: invalid digit found in string"
814            )
815        );
816
817        let invalid_min_wait = HashMap::from([(
818            TableProperties::PROPERTY_COMMIT_MIN_RETRY_WAIT_MS.to_string(),
819            "abc".to_string(),
820        )]);
821        let table_properties = TableProperties::new(&invalid_min_wait);
822        let error = table_properties.commit_min_retry_wait_ms().unwrap_err();
823        assert!(
824            error.to_string().contains(
825                "Invalid value for commit.retry.min-wait-ms: invalid digit found in string"
826            )
827        );
828
829        let invalid_max_wait = HashMap::from([(
830            TableProperties::PROPERTY_COMMIT_MAX_RETRY_WAIT_MS.to_string(),
831            "abc".to_string(),
832        )]);
833        let table_properties = TableProperties::new(&invalid_max_wait);
834        let error = table_properties.commit_max_retry_wait_ms().unwrap_err();
835        assert!(
836            error.to_string().contains(
837                "Invalid value for commit.retry.max-wait-ms: invalid digit found in string"
838            )
839        );
840
841        let invalid_target_size = HashMap::from([(
842            TableProperties::PROPERTY_WRITE_TARGET_FILE_SIZE_BYTES.to_string(),
843            "abc".to_string(),
844        )]);
845        let table_properties = TableProperties::new(&invalid_target_size);
846        let error = table_properties.write_target_file_size_bytes().unwrap_err();
847        assert!(error.to_string().contains(
848            "Invalid value for write.target-file-size-bytes: invalid digit found in string"
849        ));
850
851        let invalid_gc_enabled = HashMap::from([(
852            TableProperties::PROPERTY_GC_ENABLED.to_string(),
853            "notabool".to_string(),
854        )]);
855        let table_properties = TableProperties::new(&invalid_gc_enabled);
856        let error = table_properties.gc_enabled().unwrap_err();
857        assert!(error.to_string().contains("Invalid value for gc.enabled"));
858    }
859
860    #[test]
861    fn test_table_properties_compression_invalid_rejected() {
862        let invalid_codecs = ["lz4", "zstd", "snappy"];
863
864        for codec in invalid_codecs {
865            let props = HashMap::from([(
866                TableProperties::PROPERTY_METADATA_COMPRESSION_CODEC.to_string(),
867                codec.to_string(),
868            )]);
869            let err = TableProperties::new(&props)
870                .metadata_compression_codec()
871                .unwrap_err();
872            let err_msg = err.to_string();
873            assert!(
874                err_msg.contains(&format!("Invalid metadata compression codec: {codec}")),
875                "Expected error message to contain codec '{codec}', got: {err_msg}"
876            );
877            assert!(
878                err_msg.contains("Only 'none' and 'gzip' are supported"),
879                "Expected error message to contain supported codecs, got: {err_msg}"
880            );
881        }
882    }
883
884    #[test]
885    fn test_cdc_disabled_by_default() {
886        let props = HashMap::new();
887        let tp = TableProperties::new(&props);
888        assert!(!tp.cdc_enabled().unwrap());
889    }
890
891    #[test]
892    fn test_cdc_enabled_via_flag() {
893        let props = HashMap::from([(
894            TableProperties::PROPERTY_PARQUET_CDC_ENABLED.to_string(),
895            "true".to_string(),
896        )]);
897        let tp = TableProperties::new(&props);
898        assert!(tp.cdc_enabled().unwrap());
899        assert_eq!(tp.cdc_min_chunk_size().unwrap(), 256 * 1024);
900        assert_eq!(tp.cdc_max_chunk_size().unwrap(), 1024 * 1024);
901        assert_eq!(tp.cdc_norm_level().unwrap(), 0);
902    }
903
904    #[test]
905    fn test_cdc_size_props_alone_do_not_enable() {
906        let props = HashMap::from([(
907            TableProperties::PROPERTY_PARQUET_CDC_MIN_CHUNK_SIZE.to_string(),
908            "262144".to_string(),
909        )]);
910        let tp = TableProperties::new(&props);
911        assert!(!tp.cdc_enabled().unwrap());
912    }
913
914    #[test]
915    fn test_cdc_custom_values() {
916        let props = HashMap::from([
917            (
918                TableProperties::PROPERTY_PARQUET_CDC_ENABLED.to_string(),
919                "true".to_string(),
920            ),
921            (
922                TableProperties::PROPERTY_PARQUET_CDC_MIN_CHUNK_SIZE.to_string(),
923                "200000".to_string(),
924            ),
925            (
926                TableProperties::PROPERTY_PARQUET_CDC_MAX_CHUNK_SIZE.to_string(),
927                "900000".to_string(),
928            ),
929            (
930                TableProperties::PROPERTY_PARQUET_CDC_NORM_LEVEL.to_string(),
931                "1".to_string(),
932            ),
933        ]);
934        let tp = TableProperties::new(&props);
935        assert!(tp.cdc_enabled().unwrap());
936        assert_eq!(tp.cdc_min_chunk_size().unwrap(), 200000);
937        assert_eq!(tp.cdc_max_chunk_size().unwrap(), 900000);
938        assert_eq!(tp.cdc_norm_level().unwrap(), 1);
939    }
940
941    #[test]
942    fn test_cdc_partial_override() {
943        let props = HashMap::from([
944            (
945                TableProperties::PROPERTY_PARQUET_CDC_ENABLED.to_string(),
946                "true".to_string(),
947            ),
948            (
949                TableProperties::PROPERTY_PARQUET_CDC_NORM_LEVEL.to_string(),
950                "2".to_string(),
951            ),
952        ]);
953        let tp = TableProperties::new(&props);
954        assert!(tp.cdc_enabled().unwrap());
955        assert_eq!(tp.cdc_min_chunk_size().unwrap(), 256 * 1024);
956        assert_eq!(tp.cdc_max_chunk_size().unwrap(), 1024 * 1024);
957        assert_eq!(tp.cdc_norm_level().unwrap(), 2);
958    }
959
960    #[test]
961    fn test_cdc_negative_norm_level() {
962        let props = HashMap::from([
963            (
964                TableProperties::PROPERTY_PARQUET_CDC_ENABLED.to_string(),
965                "true".to_string(),
966            ),
967            (
968                TableProperties::PROPERTY_PARQUET_CDC_NORM_LEVEL.to_string(),
969                "-2".to_string(),
970            ),
971        ]);
972        let tp = TableProperties::new(&props);
973        assert_eq!(tp.cdc_norm_level().unwrap(), -2);
974    }
975
976    #[test]
977    fn test_cdc_invalid_min_chunk_size() {
978        let props = HashMap::from([
979            (
980                TableProperties::PROPERTY_PARQUET_CDC_ENABLED.to_string(),
981                "true".to_string(),
982            ),
983            (
984                TableProperties::PROPERTY_PARQUET_CDC_MIN_CHUNK_SIZE.to_string(),
985                "not_a_number".to_string(),
986            ),
987        ]);
988        let err = TableProperties::new(&props)
989            .cdc_min_chunk_size()
990            .unwrap_err();
991        assert!(
992            err.to_string().contains(
993                "Invalid value for write.parquet.content-defined-chunking.min-chunk-size"
994            )
995        );
996    }
997
998    #[test]
999    fn test_cdc_invalid_norm_level() {
1000        let props = HashMap::from([
1001            (
1002                TableProperties::PROPERTY_PARQUET_CDC_ENABLED.to_string(),
1003                "true".to_string(),
1004            ),
1005            (
1006                TableProperties::PROPERTY_PARQUET_CDC_NORM_LEVEL.to_string(),
1007                "not_a_number".to_string(),
1008            ),
1009        ]);
1010        let err = TableProperties::new(&props).cdc_norm_level().unwrap_err();
1011        assert!(
1012            err.to_string()
1013                .contains("Invalid value for write.parquet.content-defined-chunking.norm-level")
1014        );
1015    }
1016
1017    #[test]
1018    fn test_cdc_no_properties() {
1019        let props = HashMap::from([("some.other.property".to_string(), "value".to_string())]);
1020        let tp = TableProperties::new(&props);
1021        assert!(!tp.cdc_enabled().unwrap());
1022    }
1023
1024    #[test]
1025    fn test_parquet_sizing_defaults() {
1026        let props = HashMap::new();
1027        let tp = TableProperties::new(&props);
1028        // Default codec is zstd at its default level.
1029        assert_eq!(
1030            tp.parquet_compression_codec().unwrap(),
1031            CompressionCodec::zstd_default()
1032        );
1033        assert_eq!(
1034            tp.parquet_row_group_size_bytes().unwrap(),
1035            TableProperties::PROPERTY_PARQUET_ROW_GROUP_SIZE_BYTES_DEFAULT
1036        );
1037        assert_eq!(
1038            tp.parquet_page_size_bytes().unwrap(),
1039            TableProperties::PROPERTY_PARQUET_PAGE_SIZE_BYTES_DEFAULT
1040        );
1041        assert_eq!(
1042            tp.parquet_page_row_limit().unwrap(),
1043            TableProperties::PROPERTY_PARQUET_PAGE_ROW_LIMIT_DEFAULT
1044        );
1045        assert_eq!(
1046            tp.parquet_dict_size_bytes().unwrap(),
1047            TableProperties::PROPERTY_PARQUET_DICT_SIZE_BYTES_DEFAULT
1048        );
1049    }
1050
1051    #[test]
1052    fn test_parquet_sizing_overrides() {
1053        let props = HashMap::from([
1054            (
1055                TableProperties::PROPERTY_PARQUET_COMPRESSION_CODEC.to_string(),
1056                "gzip".to_string(),
1057            ),
1058            (
1059                TableProperties::PROPERTY_PARQUET_COMPRESSION_LEVEL.to_string(),
1060                "4".to_string(),
1061            ),
1062            (
1063                TableProperties::PROPERTY_PARQUET_ROW_GROUP_SIZE_BYTES.to_string(),
1064                "1048576".to_string(),
1065            ),
1066            (
1067                TableProperties::PROPERTY_PARQUET_PAGE_SIZE_BYTES.to_string(),
1068                "65536".to_string(),
1069            ),
1070            (
1071                TableProperties::PROPERTY_PARQUET_PAGE_ROW_LIMIT.to_string(),
1072                "5000".to_string(),
1073            ),
1074            (
1075                TableProperties::PROPERTY_PARQUET_DICT_SIZE_BYTES.to_string(),
1076                "131072".to_string(),
1077            ),
1078        ]);
1079        let tp = TableProperties::new(&props);
1080        // Codec name and level are folded into a single CompressionCodec.
1081        assert_eq!(
1082            tp.parquet_compression_codec().unwrap(),
1083            CompressionCodec::Gzip(4)
1084        );
1085        assert_eq!(tp.parquet_row_group_size_bytes().unwrap(), 1048576);
1086        assert_eq!(tp.parquet_page_size_bytes().unwrap(), 65536);
1087        assert_eq!(tp.parquet_page_row_limit().unwrap(), 5000);
1088        assert_eq!(tp.parquet_dict_size_bytes().unwrap(), 131072);
1089    }
1090
1091    #[test]
1092    fn test_parquet_invalid_sizing_rejected() {
1093        let props = HashMap::from([(
1094            TableProperties::PROPERTY_PARQUET_ROW_GROUP_SIZE_BYTES.to_string(),
1095            "not_a_number".to_string(),
1096        )]);
1097        let err = TableProperties::new(&props)
1098            .parquet_row_group_size_bytes()
1099            .unwrap_err();
1100        assert_eq!(err.kind(), ErrorKind::DataInvalid);
1101        assert!(
1102            err.to_string()
1103                .contains(TableProperties::PROPERTY_PARQUET_ROW_GROUP_SIZE_BYTES)
1104        );
1105    }
1106
1107    #[test]
1108    fn test_parquet_all_codecs_parse() {
1109        // Every codec name parquet-java supports must parse (parity with Java's
1110        // `CompressionCodecName.valueOf`).
1111        for (name, expected) in [
1112            ("uncompressed", CompressionCodec::None),
1113            ("snappy", CompressionCodec::Snappy),
1114            ("gzip", CompressionCodec::gzip_default()),
1115            ("lzo", CompressionCodec::Lzo),
1116            ("brotli", CompressionCodec::brotli_default()),
1117            ("lz4", CompressionCodec::Lz4),
1118            ("lz4_raw", CompressionCodec::Lz4Raw),
1119            ("zstd", CompressionCodec::zstd_default()),
1120        ] {
1121            let props = HashMap::from([(
1122                TableProperties::PROPERTY_PARQUET_COMPRESSION_CODEC.to_string(),
1123                name.to_string(),
1124            )]);
1125            let tp = TableProperties::new(&props);
1126            assert_eq!(
1127                tp.parquet_compression_codec().unwrap(),
1128                expected,
1129                "codec {name}"
1130            );
1131        }
1132    }
1133
1134    #[test]
1135    fn test_parquet_compression_level_ignored_for_levelless_codec() {
1136        // A level set alongside a codec that carries none (e.g. snappy) is
1137        // ignored rather than rejected, matching parquet-java.
1138        let props = HashMap::from([
1139            (
1140                TableProperties::PROPERTY_PARQUET_COMPRESSION_CODEC.to_string(),
1141                "snappy".to_string(),
1142            ),
1143            (
1144                TableProperties::PROPERTY_PARQUET_COMPRESSION_LEVEL.to_string(),
1145                "5".to_string(),
1146            ),
1147        ]);
1148        let tp = TableProperties::new(&props);
1149        assert_eq!(
1150            tp.parquet_compression_codec().unwrap(),
1151            CompressionCodec::Snappy
1152        );
1153    }
1154
1155    #[test]
1156    fn test_parse_boolean_property_case_insensitive() {
1157        let false_variants = ["False", "FALSE"];
1158        let true_variants = ["True", "TRUE"];
1159
1160        for f in false_variants {
1161            let props = HashMap::from([(
1162                TableProperties::PROPERTY_WRITE_OBJECT_STORAGE_PARTITIONED_PATHS.to_string(),
1163                f.to_string(),
1164            )]);
1165            let tp = TableProperties::new(&props);
1166            assert!(!tp.write_object_storage_partitioned_paths().unwrap());
1167        }
1168
1169        for t in true_variants {
1170            let props = HashMap::from([(
1171                TableProperties::PROPERTY_WRITE_OBJECT_STORAGE_PARTITIONED_PATHS.to_string(),
1172                t.to_string(),
1173            )]);
1174            let tp = TableProperties::new(&props);
1175            assert!(tp.write_object_storage_partitioned_paths().unwrap());
1176        }
1177    }
1178
1179    #[test]
1180    fn test_table_properties_default_name_mapping() {
1181        // Test unset.
1182        let properties = HashMap::new();
1183        assert!(
1184            TableProperties::new(&properties)
1185                .default_name_mapping()
1186                .unwrap()
1187                .is_none()
1188        );
1189
1190        let properties = HashMap::from([(
1191            TableProperties::PROPERTY_DEFAULT_NAME_MAPPING.to_string(),
1192            r#"[{"field-id":1,"names":["id","record_id"]}]"#.to_string(),
1193        )]);
1194        let mapping = TableProperties::new(&properties)
1195            .default_name_mapping()
1196            .unwrap()
1197            .unwrap();
1198        assert_eq!(mapping.fields().len(), 1);
1199        assert_eq!(mapping.fields()[0].field_id(), Some(1));
1200        assert_eq!(mapping.fields()[0].names(), &[
1201            "id".to_string(),
1202            "record_id".to_string()
1203        ]);
1204    }
1205
1206    #[test]
1207    fn test_table_properties_malformed_name_mapping() {
1208        let properties = HashMap::from([(
1209            TableProperties::PROPERTY_DEFAULT_NAME_MAPPING.to_string(),
1210            "{ not valid json".to_string(),
1211        )]);
1212        let error = TableProperties::new(&properties)
1213            .default_name_mapping()
1214            .unwrap_err();
1215
1216        assert_eq!(error.kind(), ErrorKind::DataInvalid);
1217        // The property key must survive as error context.
1218        assert!(
1219            format!("{error}").contains(TableProperties::PROPERTY_DEFAULT_NAME_MAPPING),
1220            "{error}"
1221        );
1222    }
1223}
iceberg/spec/table_properties.rs

iceberg/spec/
table_properties.rs
