iceberg/spec/
table_properties.rs

1// Licensed to the Apache Software Foundation (ASF) under one
2// or more contributor license agreements.  See the NOTICE file
3// distributed with this work for additional information
4// regarding copyright ownership.  The ASF licenses this file
5// to you under the Apache License, Version 2.0 (the
6// "License"); you may not use this file except in compliance
7// with the License.  You may obtain a copy of the License at
8//
9//   http://www.apache.org/licenses/LICENSE-2.0
10//
11// Unless required by applicable law or agreed to in writing,
12// software distributed under the License is distributed on an
13// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14// KIND, either express or implied.  See the License for the
15// specific language governing permissions and limitations
16// under the License.
17
18use std::collections::HashMap;
19
20use iceberg_property_macro::properties_view;
21
22use crate::compression::CompressionCodec;
23use crate::encryption::AesKeySize;
24use crate::error::{Result, invalid_data};
25use crate::spec::NameMapping;
26use crate::util::location::strip_trailing_slash;
27
28fn parse_location_property(path: &str) -> Result<String> {
29    if path.is_empty() {
30        return Err(invalid_data!("path must not be empty"));
31    }
32
33    Ok(strip_trailing_slash(path).to_string())
34}
35
36fn parse_metadata_compression(value: &str) -> Result<CompressionCodec> {
37    // Handle empty string as None
38    if value.is_empty() {
39        return Ok(CompressionCodec::None);
40    }
41
42    // Lowercase the value for case-insensitive parsing
43    let lowercase_value = value.to_lowercase();
44
45    // Use serde to parse the codec (which has rename_all = "lowercase")
46    let codec: CompressionCodec =
47        serde_json::from_value(serde_json::Value::String(lowercase_value)).map_err(|_| {
48            invalid_data!(
49                "Invalid metadata compression codec: {value}. Only '{}' and '{}' are supported.",
50                CompressionCodec::None.name(),
51                CompressionCodec::gzip_default().name()
52            )
53        })?;
54
55    // Validate that only None and Gzip are used for metadata
56    match codec {
57        CompressionCodec::None | CompressionCodec::Gzip(_) => Ok(codec),
58        _ => Err(invalid_data!(
59            "Invalid metadata compression codec: {value}. Only '{}' and '{}' are supported for metadata files.",
60            CompressionCodec::None.name(),
61            CompressionCodec::gzip_default().name()
62        )),
63    }
64}
65
66/// Parse the Parquet data-file compression codec (`write.parquet.compression-codec`)
67/// and fold in the compression level (`write.parquet.compression-level`) for the
68/// codecs that accept one (`zstd`, `gzip`, `brotli`).
69fn parse_parquet_compression(
70    properties: &HashMap<String, String>,
71    codec_key: &str,
72    additional_keys: &[&str],
73    default: CompressionCodec,
74) -> Result<CompressionCodec> {
75    let level_key = additional_keys[0];
76    let codec = properties
77        .get(codec_key)
78        .map(|value| {
79            serde_json::from_value(serde_json::Value::String(value.to_lowercase())).map_err(|_| {
80                invalid_data!(
81                    "Invalid Parquet compression codec: {value}. Supported codecs: \
82                         uncompressed, snappy, gzip, lzo, brotli, lz4, lz4_raw, zstd"
83                )
84            })
85        })
86        .transpose()?
87        .unwrap_or(default);
88
89    let level = properties
90        .get(level_key)
91        .map(|value| {
92            value
93                .parse::<u8>()
94                .map_err(|error| invalid_data!("Invalid value for {level_key}: {error}"))
95        })
96        .transpose()?;
97
98    Ok(match (codec, level) {
99        (CompressionCodec::Zstd(_), Some(level)) => CompressionCodec::Zstd(level),
100        (CompressionCodec::Gzip(_), Some(level)) => CompressionCodec::Gzip(level),
101        (CompressionCodec::Brotli(_), Some(level)) => CompressionCodec::Brotli(level),
102        (codec, _) => codec,
103    })
104}
105
106properties_view! {
107/// Typed view over the properties of a table.
108#[derive(Debug)]
109pub struct TableProperties {
110    /// The number of times to retry a commit.
111    #[property(
112        key = Self::PROPERTY_COMMIT_NUM_RETRIES,
113        default = Self::PROPERTY_COMMIT_NUM_RETRIES_DEFAULT,
114        getter
115    )]
116    commit_num_retries: usize,
117    /// The minimum wait time between retries.
118    #[property(
119        key = Self::PROPERTY_COMMIT_MIN_RETRY_WAIT_MS,
120        default = Self::PROPERTY_COMMIT_MIN_RETRY_WAIT_MS_DEFAULT,
121        getter
122    )]
123    commit_min_retry_wait_ms: u64,
124    /// The maximum wait time between retries.
125    #[property(
126        key = Self::PROPERTY_COMMIT_MAX_RETRY_WAIT_MS,
127        default = Self::PROPERTY_COMMIT_MAX_RETRY_WAIT_MS_DEFAULT,
128        getter
129    )]
130    commit_max_retry_wait_ms: u64,
131    /// The total timeout for commit retries.
132    #[property(
133        key = Self::PROPERTY_COMMIT_TOTAL_RETRY_TIME_MS,
134        default = Self::PROPERTY_COMMIT_TOTAL_RETRY_TIME_MS_DEFAULT,
135        getter
136    )]
137    commit_total_retry_timeout_ms: u64,
138    /// The default format for files.
139    #[property(
140        key = Self::PROPERTY_DEFAULT_FILE_FORMAT,
141        default = Self::PROPERTY_DEFAULT_FILE_FORMAT_DEFAULT,
142        getter
143    )]
144    write_format_default: String,
145    /// The target file size for files.
146    #[property(
147        key = Self::PROPERTY_WRITE_TARGET_FILE_SIZE_BYTES,
148        default = Self::PROPERTY_WRITE_TARGET_FILE_SIZE_BYTES_DEFAULT,
149        getter
150    )]
151    write_target_file_size_bytes: usize,
152    /// Base directory for metadata files (manifests, manifest lists), with any
153    /// trailing slash trimmed. `None` if `write.metadata.path` is not set.
154    #[property(
155        key = Self::PROPERTY_WRITE_METADATA_PATH,
156        default = None,
157        parse_with = parse_location_property,
158        getter
159    )]
160    write_metadata_path: Option<String>,
161    /// Compression codec for metadata files (JSON)
162    #[property(
163        key = Self::PROPERTY_METADATA_COMPRESSION_CODEC,
164        default = CompressionCodec::None,
165        parse_with = parse_metadata_compression,
166        getter
167    )]
168    metadata_compression_codec: CompressionCodec,
169    /// Whether to use `FanoutWriter` for partitioned tables.
170    #[property(
171        key = Self::PROPERTY_DATAFUSION_WRITE_FANOUT_ENABLED,
172        default = Self::PROPERTY_DATAFUSION_WRITE_FANOUT_ENABLED_DEFAULT,
173        getter
174    )]
175    write_datafusion_fanout_enabled: bool,
176    /// Whether garbage collection is enabled on drop.
177    /// When `false`, data files will not be deleted when a table is dropped.
178    #[property(
179        key = Self::PROPERTY_GC_ENABLED,
180        default = Self::PROPERTY_GC_ENABLED_DEFAULT,
181        getter
182    )]
183    gc_enabled: bool,
184    /// Default maximum age of a snapshot to keep when expiring snapshots.
185    #[property(
186        key = Self::PROPERTY_MAX_SNAPSHOT_AGE_MS,
187        default = Self::PROPERTY_MAX_SNAPSHOT_AGE_MS_DEFAULT,
188        getter
189    )]
190    max_snapshot_age_ms: i64,
191    /// Default minimum number of snapshots to keep per branch when expiring snapshots.
192    #[property(
193        key = Self::PROPERTY_MIN_SNAPSHOTS_TO_KEEP,
194        default = Self::PROPERTY_MIN_SNAPSHOTS_TO_KEEP_DEFAULT,
195        getter
196    )]
197    min_snapshots_to_keep: usize,
198    /// Default maximum age of a snapshot reference to keep when expiring snapshots.
199    #[property(
200        key = Self::PROPERTY_MAX_REF_AGE_MS,
201        default = Self::PROPERTY_MAX_REF_AGE_MS_DEFAULT,
202        getter
203    )]
204    max_ref_age_ms: i64,
205    /// Whether content-defined chunking is enabled.
206    /// `true` only when `write.parquet.content-defined-chunking.enabled = "true"`.
207    #[property(
208        key = Self::PROPERTY_PARQUET_CDC_ENABLED,
209        default = Self::PROPERTY_PARQUET_CDC_ENABLED_DEFAULT,
210        getter
211    )]
212    cdc_enabled: bool,
213    /// Content-defined chunking minimum chunk size in bytes.
214    #[property(
215        key = Self::PROPERTY_PARQUET_CDC_MIN_CHUNK_SIZE,
216        default = Self::PROPERTY_PARQUET_CDC_MIN_CHUNK_SIZE_DEFAULT,
217        getter
218    )]
219    cdc_min_chunk_size: usize,
220    /// Content-defined chunking maximum chunk size in bytes.
221    #[property(
222        key = Self::PROPERTY_PARQUET_CDC_MAX_CHUNK_SIZE,
223        default = Self::PROPERTY_PARQUET_CDC_MAX_CHUNK_SIZE_DEFAULT,
224        getter
225    )]
226    cdc_max_chunk_size: usize,
227    /// Content-defined chunking normalization level (gearhash bit adjustment).
228    #[property(
229        key = Self::PROPERTY_PARQUET_CDC_NORM_LEVEL,
230        default = Self::PROPERTY_PARQUET_CDC_NORM_LEVEL_DEFAULT,
231        getter
232    )]
233    cdc_norm_level: i32,
234    /// Parquet compression codec for data files, with the resolved compression
235    /// level folded in (from `write.parquet.compression-level`, or the codec's
236    /// default when unset).
237    #[property(
238        key = Self::PROPERTY_PARQUET_COMPRESSION_CODEC,
239        additional_keys = [Self::PROPERTY_PARQUET_COMPRESSION_LEVEL],
240        default = CompressionCodec::zstd_default(),
241        parse_properties_with = parse_parquet_compression,
242        getter
243    )]
244    parquet_compression_codec: CompressionCodec,
245    /// Approximate maximum Parquet row group size in bytes.
246    #[property(
247        key = Self::PROPERTY_PARQUET_ROW_GROUP_SIZE_BYTES,
248        default = Self::PROPERTY_PARQUET_ROW_GROUP_SIZE_BYTES_DEFAULT,
249        getter
250    )]
251    parquet_row_group_size_bytes: usize,
252    /// Approximate maximum Parquet data page size in bytes.
253    #[property(
254        key = Self::PROPERTY_PARQUET_PAGE_SIZE_BYTES,
255        default = Self::PROPERTY_PARQUET_PAGE_SIZE_BYTES_DEFAULT,
256        getter
257    )]
258    parquet_page_size_bytes: usize,
259    /// Maximum number of rows per Parquet data page.
260    #[property(
261        key = Self::PROPERTY_PARQUET_PAGE_ROW_LIMIT,
262        default = Self::PROPERTY_PARQUET_PAGE_ROW_LIMIT_DEFAULT,
263        getter
264    )]
265    parquet_page_row_limit: usize,
266    /// Approximate maximum Parquet dictionary page size in bytes.
267    #[property(
268        key = Self::PROPERTY_PARQUET_DICT_SIZE_BYTES,
269        default = Self::PROPERTY_PARQUET_DICT_SIZE_BYTES_DEFAULT,
270        getter
271    )]
272    parquet_dict_size_bytes: usize,
273    /// The master key id used to encrypt this table's manifest list and data
274    /// files. `None` if `encryption.key-id` is not set.
275    #[property(
276        key = Self::PROPERTY_ENCRYPTION_KEY_ID,
277        default = None,
278        getter
279    )]
280    encryption_key_id: Option<String>,
281    /// The encryption data encryption key length in bytes.
282    #[property(
283        key = Self::PROPERTY_ENCRYPTION_DATA_KEY_LENGTH,
284        default = Self::PROPERTY_ENCRYPTION_DATA_KEY_LENGTH_DEFAULT,
285        getter
286    )]
287    encryption_data_key_length: usize,
288    /// Base directory for data files, with any trailing slash trimmed.
289    #[property(
290        key = Self::PROPERTY_WRITE_DATA_LOCATION,
291        default = None,
292        parse_with = parse_location_property,
293        getter
294    )]
295    write_data_location: Option<String>,
296    /// Deprecated table property for data file write location, with any trailing slash trimmed.
297    ///
298    /// Property will be removed at a later date.
299    /// Superseded by [`TableProperties::write_data_location`].
300    #[property(
301        key = Self::PROPERTY_WRITE_FOLDER_STORAGE_LOCATION,
302        default = None,
303        parse_with = parse_location_property,
304        getter
305    )]
306    write_folder_storage_location: Option<String>,
307    /// Deprecated table property for data file write location for object storage location generator,
308    /// with any trailing slash trimmed.
309    ///
310    /// Property will be removed at a later date.
311    /// Superseded by [`TableProperties::write_data_location`].
312    #[property(
313        key = Self::PROPERTY_WRITE_OBJECT_STORAGE_LOCATION,
314        default = None,
315        parse_with = parse_location_property,
316        getter
317    )]
318    write_object_storage_location: Option<String>,
319    /// Whether new data files use the object storage location layout, which
320    /// injects hash entropy into file paths to spread object-store prefixes.
321    #[property(
322        key = Self::PROPERTY_WRITE_OBJECT_STORAGE_ENABLED,
323        default = Self::PROPERTY_WRITE_OBJECT_STORAGE_ENABLED_DEFAULT,
324        getter
325    )]
326    write_object_storage_enabled: bool,
327    /// Whether partition values are included in object storage paths.
328    #[property(
329        key = Self::PROPERTY_WRITE_OBJECT_STORAGE_PARTITIONED_PATHS,
330        default = Self::PROPERTY_WRITE_OBJECT_STORAGE_PARTITIONED_PATHS_DEFAULT,
331        getter
332    )]
333    write_object_storage_partitioned_paths: bool,
334    /// The table's default name mapping, used to assign field ids when reading data files
335    /// that carry no field id metadata. `None` if `schema.name-mapping.default` is not set.
336    #[property(
337        key = Self::PROPERTY_DEFAULT_NAME_MAPPING,
338        default = None,
339        getter
340    )]
341    default_name_mapping: Option<NameMapping>,
342}
343}
344
345impl TableProperties<'_> {
346    /// Reserved table property for table format version.
347    ///
348    /// Iceberg will default a new table's format version to the latest stable and recommended
349    /// version. This reserved property keyword allows users to override the Iceberg format version of
350    /// the table metadata.
351    ///
352    /// If this table property exists when creating a table, the table will use the specified format
353    /// version. If a table updates this property, it will try to upgrade to the specified format
354    /// version.
355    pub const PROPERTY_FORMAT_VERSION: &'static str = "format-version";
356    /// Reserved table property for table UUID.
357    pub const PROPERTY_UUID: &'static str = "uuid";
358    /// Reserved table property for the total number of snapshots.
359    pub const PROPERTY_SNAPSHOT_COUNT: &'static str = "snapshot-count";
360    /// Reserved table property for current snapshot summary.
361    pub const PROPERTY_CURRENT_SNAPSHOT_SUMMARY: &'static str = "current-snapshot-summary";
362    /// Reserved table property for current snapshot id.
363    pub const PROPERTY_CURRENT_SNAPSHOT_ID: &'static str = "current-snapshot-id";
364    /// Reserved table property for current snapshot timestamp.
365    pub const PROPERTY_CURRENT_SNAPSHOT_TIMESTAMP: &'static str = "current-snapshot-timestamp-ms";
366    /// Reserved table property for the JSON representation of current schema.
367    pub const PROPERTY_CURRENT_SCHEMA: &'static str = "current-schema";
368    /// Reserved table property for the JSON representation of current(default) partition spec.
369    pub const PROPERTY_DEFAULT_PARTITION_SPEC: &'static str = "default-partition-spec";
370    /// Reserved table property for the JSON representation of current(default) sort order.
371    pub const PROPERTY_DEFAULT_SORT_ORDER: &'static str = "default-sort-order";
372
373    /// Property key for max number of previous versions to keep.
374    pub const PROPERTY_METADATA_PREVIOUS_VERSIONS_MAX: &'static str =
375        "write.metadata.previous-versions-max";
376    /// Default value for max number of previous versions to keep.
377    pub const PROPERTY_METADATA_PREVIOUS_VERSIONS_MAX_DEFAULT: usize = 100;
378
379    /// Property key for max number of partitions to keep summary stats for.
380    pub const PROPERTY_WRITE_PARTITION_SUMMARY_LIMIT: &'static str =
381        "write.summary.partition-limit";
382    /// Default value for the max number of partitions to keep summary stats for.
383    pub const PROPERTY_WRITE_PARTITION_SUMMARY_LIMIT_DEFAULT: u64 = 0;
384
385    /// Reserved Iceberg table properties list.
386    ///
387    /// Reserved table properties are only used to control behaviors when creating or updating a
388    /// table. The value of these properties are not persisted as a part of the table metadata.
389    pub const RESERVED_PROPERTIES: [&'static str; 9] = [
390        Self::PROPERTY_FORMAT_VERSION,
391        Self::PROPERTY_UUID,
392        Self::PROPERTY_SNAPSHOT_COUNT,
393        Self::PROPERTY_CURRENT_SNAPSHOT_ID,
394        Self::PROPERTY_CURRENT_SNAPSHOT_SUMMARY,
395        Self::PROPERTY_CURRENT_SNAPSHOT_TIMESTAMP,
396        Self::PROPERTY_CURRENT_SCHEMA,
397        Self::PROPERTY_DEFAULT_PARTITION_SPEC,
398        Self::PROPERTY_DEFAULT_SORT_ORDER,
399    ];
400
401    /// Property key for number of commit retries.
402    pub const PROPERTY_COMMIT_NUM_RETRIES: &'static str = "commit.retry.num-retries";
403    /// Default value for number of commit retries.
404    pub const PROPERTY_COMMIT_NUM_RETRIES_DEFAULT: usize = 4;
405
406    /// Property key for minimum wait time (ms) between retries.
407    pub const PROPERTY_COMMIT_MIN_RETRY_WAIT_MS: &'static str = "commit.retry.min-wait-ms";
408    /// Default value for minimum wait time (ms) between retries.
409    pub const PROPERTY_COMMIT_MIN_RETRY_WAIT_MS_DEFAULT: u64 = 100;
410
411    /// Property key for maximum wait time (ms) between retries.
412    pub const PROPERTY_COMMIT_MAX_RETRY_WAIT_MS: &'static str = "commit.retry.max-wait-ms";
413    /// Default value for maximum wait time (ms) between retries.
414    pub const PROPERTY_COMMIT_MAX_RETRY_WAIT_MS_DEFAULT: u64 = 60 * 1000; // 1 minute
415
416    /// Property key for total maximum retry time (ms).
417    pub const PROPERTY_COMMIT_TOTAL_RETRY_TIME_MS: &'static str = "commit.retry.total-timeout-ms";
418    /// Default value for total maximum retry time (ms).
419    pub const PROPERTY_COMMIT_TOTAL_RETRY_TIME_MS_DEFAULT: u64 = 30 * 60 * 1000; // 30 minutes
420
421    /// Default file format for data files
422    pub const PROPERTY_DEFAULT_FILE_FORMAT: &'static str = "write.format.default";
423    /// Default file format for delete files
424    pub const PROPERTY_DELETE_DEFAULT_FILE_FORMAT: &'static str = "write.delete.format.default";
425    /// Default value for data file format
426    pub const PROPERTY_DEFAULT_FILE_FORMAT_DEFAULT: &'static str = "parquet";
427
428    /// Target file size for newly written files.
429    pub const PROPERTY_WRITE_TARGET_FILE_SIZE_BYTES: &'static str = "write.target-file-size-bytes";
430    /// Default target file size
431    pub const PROPERTY_WRITE_TARGET_FILE_SIZE_BYTES_DEFAULT: usize = 512 * 1024 * 1024; // 512 MB
432
433    /// Base location for metadata files (manifests, manifest lists, table metadata).
434    /// When unset, metadata files default to the `metadata` directory under the table
435    /// location.
436    pub const PROPERTY_WRITE_METADATA_PATH: &'static str = "write.metadata.path";
437
438    /// Property key for the table's default name mapping, stored as a JSON
439    /// [`NameMapping`] document.
440    pub const PROPERTY_DEFAULT_NAME_MAPPING: &'static str = "schema.name-mapping.default";
441
442    /// Compression codec for metadata files (JSON)
443    pub const PROPERTY_METADATA_COMPRESSION_CODEC: &'static str =
444        "write.metadata.compression-codec";
445    /// Default metadata compression codec - uncompressed
446    pub const PROPERTY_METADATA_COMPRESSION_CODEC_DEFAULT: &'static str = "none";
447    /// Whether to use `FanoutWriter` for partitioned tables (handles unsorted data).
448    /// If false, uses `ClusteredWriter` (requires sorted data, more memory efficient).
449    pub const PROPERTY_DATAFUSION_WRITE_FANOUT_ENABLED: &'static str =
450        "write.datafusion.fanout.enabled";
451    /// Default value for fanout writer enabled
452    pub const PROPERTY_DATAFUSION_WRITE_FANOUT_ENABLED_DEFAULT: bool = true;
453
454    /// Property key for enabling garbage collection on drop.
455    /// When set to `false`, data files will not be deleted when a table is dropped.
456    /// Defaults to `true`.
457    pub const PROPERTY_GC_ENABLED: &'static str = "gc.enabled";
458    /// Default value for gc.enabled
459    pub const PROPERTY_GC_ENABLED_DEFAULT: bool = true;
460
461    /// Property key for the default maximum age of a snapshot to keep when expiring snapshots.
462    pub const PROPERTY_MAX_SNAPSHOT_AGE_MS: &'static str = "history.expire.max-snapshot-age-ms";
463    /// Default value for history.expire.max-snapshot-age-ms (5 days).
464    pub const PROPERTY_MAX_SNAPSHOT_AGE_MS_DEFAULT: i64 = 5 * 24 * 60 * 60 * 1000;
465    /// Property key for the default minimum number of snapshots to keep when expiring snapshots.
466    pub const PROPERTY_MIN_SNAPSHOTS_TO_KEEP: &'static str = "history.expire.min-snapshots-to-keep";
467    /// Default value for history.expire.min-snapshots-to-keep.
468    pub const PROPERTY_MIN_SNAPSHOTS_TO_KEEP_DEFAULT: usize = 1;
469    /// Property key for the default maximum age of a snapshot reference to keep when expiring.
470    pub const PROPERTY_MAX_REF_AGE_MS: &'static str = "history.expire.max-ref-age-ms";
471    /// Default value for history.expire.max-ref-age-ms (effectively never expire refs).
472    pub const PROPERTY_MAX_REF_AGE_MS_DEFAULT: i64 = i64::MAX;
473
474    /// Enable content-defined chunking with parquet defaults (or per-property overrides).
475    pub const PROPERTY_PARQUET_CDC_ENABLED: &'static str =
476        "write.parquet.content-defined-chunking.enabled";
477    /// Default value for content-defined chunking enabled.
478    pub const PROPERTY_PARQUET_CDC_ENABLED_DEFAULT: bool = false;
479    /// Minimum chunk size in bytes for content-defined chunking.
480    pub const PROPERTY_PARQUET_CDC_MIN_CHUNK_SIZE: &'static str =
481        "write.parquet.content-defined-chunking.min-chunk-size";
482    /// Default matches `parquet::file::properties::DEFAULT_CDC_MIN_CHUNK_SIZE`.
483    pub const PROPERTY_PARQUET_CDC_MIN_CHUNK_SIZE_DEFAULT: usize = 256 * 1024;
484    /// Maximum chunk size in bytes for content-defined chunking.
485    pub const PROPERTY_PARQUET_CDC_MAX_CHUNK_SIZE: &'static str =
486        "write.parquet.content-defined-chunking.max-chunk-size";
487    /// Default matches `parquet::file::properties::DEFAULT_CDC_MAX_CHUNK_SIZE`.
488    pub const PROPERTY_PARQUET_CDC_MAX_CHUNK_SIZE_DEFAULT: usize = 1024 * 1024;
489    /// Normalization level (gearhash bit adjustment) for content-defined chunking.
490    pub const PROPERTY_PARQUET_CDC_NORM_LEVEL: &'static str =
491        "write.parquet.content-defined-chunking.norm-level";
492    /// Default matches `parquet::file::properties::DEFAULT_CDC_NORM_LEVEL`.
493    pub const PROPERTY_PARQUET_CDC_NORM_LEVEL_DEFAULT: i32 = 0;
494
495    /// Compression codec for Parquet data files (e.g. `zstd`, `gzip`, `snappy`,
496    /// `lz4`, `lz4_raw`, `brotli`, `lzo`, `uncompressed`). The codec name is
497    /// parsed into a [`CompressionCodec`] when properties are parsed; the level's
498    /// range is validated when the writer is built.
499    pub const PROPERTY_PARQUET_COMPRESSION_CODEC: &'static str = "write.parquet.compression-codec";
500    /// Default Parquet compression codec.
501    pub const PROPERTY_PARQUET_COMPRESSION_CODEC_DEFAULT: &'static str = "zstd";
502    /// Compression level for Parquet data files, for codecs that take one
503    /// (`gzip`, `zstd`, `brotli`). When unset, the codec's default level is used.
504    pub const PROPERTY_PARQUET_COMPRESSION_LEVEL: &'static str = "write.parquet.compression-level";
505
506    /// Approximate maximum size of a Parquet row group in bytes.
507    pub const PROPERTY_PARQUET_ROW_GROUP_SIZE_BYTES: &'static str =
508        "write.parquet.row-group-size-bytes";
509    /// Default Parquet row group size in bytes.
510    pub const PROPERTY_PARQUET_ROW_GROUP_SIZE_BYTES_DEFAULT: usize = 128 * 1024 * 1024;
511
512    /// Approximate maximum size of a Parquet data page in bytes.
513    pub const PROPERTY_PARQUET_PAGE_SIZE_BYTES: &'static str = "write.parquet.page-size-bytes";
514    /// Default Parquet page size in bytes.
515    pub const PROPERTY_PARQUET_PAGE_SIZE_BYTES_DEFAULT: usize = 1024 * 1024;
516
517    /// Maximum number of rows per Parquet data page.
518    pub const PROPERTY_PARQUET_PAGE_ROW_LIMIT: &'static str = "write.parquet.page-row-limit";
519    /// Default Parquet page row limit.
520    pub const PROPERTY_PARQUET_PAGE_ROW_LIMIT_DEFAULT: usize = 20000;
521
522    /// Approximate maximum size of the Parquet dictionary page in bytes.
523    pub const PROPERTY_PARQUET_DICT_SIZE_BYTES: &'static str = "write.parquet.dict-size-bytes";
524    /// Default Parquet dictionary page size in bytes.
525    pub const PROPERTY_PARQUET_DICT_SIZE_BYTES_DEFAULT: usize = 2 * 1024 * 1024;
526
527    /// Property key for the master key id used to encrypt the table's manifest
528    /// list and data files as defined in <https://iceberg.apache.org/docs/nightly/encryption/>.
529    pub const PROPERTY_ENCRYPTION_KEY_ID: &'static str = "encryption.key-id";
530
531    /// Property key for the encryption data encryption key (DEK) length in bytes.
532    pub const PROPERTY_ENCRYPTION_DATA_KEY_LENGTH: &'static str = "encryption.data-key-length";
533    /// Default value for the encryption DEK length (16 bytes = AES-128).
534    pub const PROPERTY_ENCRYPTION_DATA_KEY_LENGTH_DEFAULT: usize = 16;
535    /// Property key for the base directory for data files
536    pub const PROPERTY_WRITE_DATA_LOCATION: &'static str = "write.data.path";
537    /// Property key for deprecated [`TableProperties::write_folder_storage_location`]
538    pub const PROPERTY_WRITE_FOLDER_STORAGE_LOCATION: &'static str = "write.folder-storage.path";
539    /// Property key for deprecated object storage path, kept as a fallback for compatibility.
540    pub const PROPERTY_WRITE_OBJECT_STORAGE_LOCATION: &'static str = "write.object-storage.path";
541    /// Property key for enabling the object storage location layout for new data files.
542    pub const PROPERTY_WRITE_OBJECT_STORAGE_ENABLED: &'static str = "write.object-storage.enabled";
543    /// Default value for [`TableProperties::PROPERTY_WRITE_OBJECT_STORAGE_ENABLED`]
544    pub const PROPERTY_WRITE_OBJECT_STORAGE_ENABLED_DEFAULT: bool = false;
545    /// Property key for controlling whether partition values are included in object storage paths.
546    pub const PROPERTY_WRITE_OBJECT_STORAGE_PARTITIONED_PATHS: &'static str =
547        "write.object-storage.partitioned-paths";
548    /// Default value for [`TableProperties::PROPERTY_WRITE_OBJECT_STORAGE_PARTITIONED_PATHS`]
549    pub const PROPERTY_WRITE_OBJECT_STORAGE_PARTITIONED_PATHS_DEFAULT: bool = true;
550
551    /// The AES key size to use when generating data encryption keys, derived
552    /// from `encryption.data-key-length`.
553    pub fn data_encryption_key_size(&self) -> Result<AesKeySize> {
554        AesKeySize::from_key_length(self.encryption_data_key_length()?)
555    }
556}
557
558#[cfg(test)]
559mod tests {
560    use super::*;
561    use crate::ErrorKind;
562    use crate::compression::CompressionCodec;
563
564    #[test]
565    fn test_table_properties_default() {
566        let props = HashMap::new();
567        let table_properties = TableProperties::new(&props);
568        assert_eq!(
569            table_properties.commit_num_retries().unwrap(),
570            TableProperties::PROPERTY_COMMIT_NUM_RETRIES_DEFAULT
571        );
572        assert_eq!(
573            table_properties.commit_min_retry_wait_ms().unwrap(),
574            TableProperties::PROPERTY_COMMIT_MIN_RETRY_WAIT_MS_DEFAULT
575        );
576        assert_eq!(
577            table_properties.commit_max_retry_wait_ms().unwrap(),
578            TableProperties::PROPERTY_COMMIT_MAX_RETRY_WAIT_MS_DEFAULT
579        );
580        assert_eq!(
581            table_properties.write_format_default().unwrap(),
582            TableProperties::PROPERTY_DEFAULT_FILE_FORMAT_DEFAULT.to_string()
583        );
584        assert_eq!(
585            table_properties.write_target_file_size_bytes().unwrap(),
586            TableProperties::PROPERTY_WRITE_TARGET_FILE_SIZE_BYTES_DEFAULT
587        );
588        // Test compression defaults (none means CompressionCodec::None)
589        assert_eq!(
590            table_properties.metadata_compression_codec().unwrap(),
591            CompressionCodec::None
592        );
593        assert_eq!(
594            table_properties.gc_enabled().unwrap(),
595            TableProperties::PROPERTY_GC_ENABLED_DEFAULT
596        );
597        assert_eq!(
598            table_properties.max_snapshot_age_ms().unwrap(),
599            TableProperties::PROPERTY_MAX_SNAPSHOT_AGE_MS_DEFAULT
600        );
601        assert_eq!(
602            table_properties.min_snapshots_to_keep().unwrap(),
603            TableProperties::PROPERTY_MIN_SNAPSHOTS_TO_KEEP_DEFAULT
604        );
605        assert_eq!(
606            table_properties.max_ref_age_ms().unwrap(),
607            TableProperties::PROPERTY_MAX_REF_AGE_MS_DEFAULT
608        );
609    }
610
611    #[test]
612    fn test_table_properties_history_expire_overrides() {
613        let props = HashMap::from([
614            (
615                TableProperties::PROPERTY_MAX_SNAPSHOT_AGE_MS.to_string(),
616                "1234".to_string(),
617            ),
618            (
619                TableProperties::PROPERTY_MIN_SNAPSHOTS_TO_KEEP.to_string(),
620                "7".to_string(),
621            ),
622            (
623                TableProperties::PROPERTY_MAX_REF_AGE_MS.to_string(),
624                "5678".to_string(),
625            ),
626        ]);
627        let table_properties = TableProperties::new(&props);
628        assert_eq!(table_properties.max_snapshot_age_ms().unwrap(), 1234);
629        assert_eq!(table_properties.min_snapshots_to_keep().unwrap(), 7);
630        assert_eq!(table_properties.max_ref_age_ms().unwrap(), 5678);
631    }
632
633    #[test]
634    fn test_table_properties_location_paths() {
635        // Test unset.
636        let raw_properties = HashMap::new();
637        let table_properties = TableProperties::new(&raw_properties);
638        assert_eq!(table_properties.write_metadata_path().unwrap(), None);
639        assert_eq!(table_properties.write_data_location().unwrap(), None);
640        assert_eq!(
641            table_properties.write_folder_storage_location().unwrap(),
642            None
643        );
644        assert_eq!(
645            table_properties.write_object_storage_location().unwrap(),
646            None
647        );
648
649        for key in [
650            TableProperties::PROPERTY_WRITE_METADATA_PATH,
651            TableProperties::PROPERTY_WRITE_DATA_LOCATION,
652            TableProperties::PROPERTY_WRITE_FOLDER_STORAGE_LOCATION,
653            TableProperties::PROPERTY_WRITE_OBJECT_STORAGE_LOCATION,
654        ] {
655            // Test empty paths are invalid and retain the property key as error context.
656            let raw_properties = HashMap::from([(key.to_string(), String::new())]);
657            let table_properties = TableProperties::new(&raw_properties);
658            let error = match key {
659                TableProperties::PROPERTY_WRITE_METADATA_PATH => {
660                    table_properties.write_metadata_path().unwrap_err()
661                }
662                TableProperties::PROPERTY_WRITE_DATA_LOCATION => {
663                    table_properties.write_data_location().unwrap_err()
664                }
665                TableProperties::PROPERTY_WRITE_FOLDER_STORAGE_LOCATION => table_properties
666                    .write_folder_storage_location()
667                    .unwrap_err(),
668                TableProperties::PROPERTY_WRITE_OBJECT_STORAGE_LOCATION => table_properties
669                    .write_object_storage_location()
670                    .unwrap_err(),
671                _ => unreachable!(),
672            };
673            assert_eq!(error.kind(), ErrorKind::DataInvalid);
674            assert!(format!("{error}").contains(key));
675
676            // Test all supported location properties share trailing-slash normalization.
677            let raw_properties = HashMap::from([(
678                key.to_string(),
679                "s3://other-bucket/custom-path/".to_string(),
680            )]);
681            let table_properties = TableProperties::new(&raw_properties);
682            let parsed = match key {
683                TableProperties::PROPERTY_WRITE_METADATA_PATH => {
684                    table_properties.write_metadata_path().unwrap()
685                }
686                TableProperties::PROPERTY_WRITE_DATA_LOCATION => {
687                    table_properties.write_data_location().unwrap()
688                }
689                TableProperties::PROPERTY_WRITE_FOLDER_STORAGE_LOCATION => {
690                    table_properties.write_folder_storage_location().unwrap()
691                }
692                TableProperties::PROPERTY_WRITE_OBJECT_STORAGE_LOCATION => {
693                    table_properties.write_object_storage_location().unwrap()
694                }
695                _ => unreachable!(),
696            };
697            assert_eq!(parsed.as_deref(), Some("s3://other-bucket/custom-path"));
698        }
699    }
700
701    #[test]
702    fn test_table_properties_compression() {
703        let props = HashMap::from([(
704            TableProperties::PROPERTY_METADATA_COMPRESSION_CODEC.to_string(),
705            "gzip".to_string(),
706        )]);
707        let table_properties = TableProperties::new(&props);
708        assert_eq!(
709            table_properties.metadata_compression_codec().unwrap(),
710            CompressionCodec::gzip_default()
711        );
712    }
713
714    #[test]
715    fn test_table_properties_compression_none() {
716        let props = HashMap::from([(
717            TableProperties::PROPERTY_METADATA_COMPRESSION_CODEC.to_string(),
718            "none".to_string(),
719        )]);
720        let table_properties = TableProperties::new(&props);
721        assert_eq!(
722            table_properties.metadata_compression_codec().unwrap(),
723            CompressionCodec::None
724        );
725    }
726
727    #[test]
728    fn test_table_properties_compression_case_insensitive() {
729        // Test uppercase
730        let props_upper = HashMap::from([(
731            TableProperties::PROPERTY_METADATA_COMPRESSION_CODEC.to_string(),
732            "GZIP".to_string(),
733        )]);
734        let table_properties = TableProperties::new(&props_upper);
735        assert_eq!(
736            table_properties.metadata_compression_codec().unwrap(),
737            CompressionCodec::gzip_default()
738        );
739
740        // Test mixed case
741        let props_mixed = HashMap::from([(
742            TableProperties::PROPERTY_METADATA_COMPRESSION_CODEC.to_string(),
743            "GzIp".to_string(),
744        )]);
745        let table_properties = TableProperties::new(&props_mixed);
746        assert_eq!(
747            table_properties.metadata_compression_codec().unwrap(),
748            CompressionCodec::gzip_default()
749        );
750
751        // Test "NONE" should also be case-insensitive
752        let props_none_upper = HashMap::from([(
753            TableProperties::PROPERTY_METADATA_COMPRESSION_CODEC.to_string(),
754            "NONE".to_string(),
755        )]);
756        let table_properties = TableProperties::new(&props_none_upper);
757        assert_eq!(
758            table_properties.metadata_compression_codec().unwrap(),
759            CompressionCodec::None
760        );
761    }
762
763    #[test]
764    fn test_table_properties_valid() {
765        let props = HashMap::from([
766            (
767                TableProperties::PROPERTY_COMMIT_NUM_RETRIES.to_string(),
768                "10".to_string(),
769            ),
770            (
771                TableProperties::PROPERTY_COMMIT_MAX_RETRY_WAIT_MS.to_string(),
772                "20".to_string(),
773            ),
774            (
775                TableProperties::PROPERTY_DEFAULT_FILE_FORMAT.to_string(),
776                "avro".to_string(),
777            ),
778            (
779                TableProperties::PROPERTY_WRITE_TARGET_FILE_SIZE_BYTES.to_string(),
780                "512".to_string(),
781            ),
782            (
783                TableProperties::PROPERTY_GC_ENABLED.to_string(),
784                "false".to_string(),
785            ),
786        ]);
787        let table_properties = TableProperties::new(&props);
788        assert_eq!(table_properties.commit_num_retries().unwrap(), 10);
789        assert_eq!(table_properties.commit_max_retry_wait_ms().unwrap(), 20);
790        assert_eq!(
791            table_properties.write_format_default().unwrap(),
792            "avro".to_string()
793        );
794        assert_eq!(
795            table_properties.write_target_file_size_bytes().unwrap(),
796            512
797        );
798        assert!(!table_properties.gc_enabled().unwrap());
799    }
800
801    #[test]
802    fn test_table_properties_invalid() {
803        let invalid_retries = HashMap::from([(
804            TableProperties::PROPERTY_COMMIT_NUM_RETRIES.to_string(),
805            "abc".to_string(),
806        )]);
807
808        let table_properties = TableProperties::new(&invalid_retries);
809        let error = table_properties.commit_num_retries().unwrap_err();
810        assert!(
811            error.to_string().contains(
812                "Invalid value for commit.retry.num-retries: invalid digit found in string"
813            )
814        );
815
816        let invalid_min_wait = HashMap::from([(
817            TableProperties::PROPERTY_COMMIT_MIN_RETRY_WAIT_MS.to_string(),
818            "abc".to_string(),
819        )]);
820        let table_properties = TableProperties::new(&invalid_min_wait);
821        let error = table_properties.commit_min_retry_wait_ms().unwrap_err();
822        assert!(
823            error.to_string().contains(
824                "Invalid value for commit.retry.min-wait-ms: invalid digit found in string"
825            )
826        );
827
828        let invalid_max_wait = HashMap::from([(
829            TableProperties::PROPERTY_COMMIT_MAX_RETRY_WAIT_MS.to_string(),
830            "abc".to_string(),
831        )]);
832        let table_properties = TableProperties::new(&invalid_max_wait);
833        let error = table_properties.commit_max_retry_wait_ms().unwrap_err();
834        assert!(
835            error.to_string().contains(
836                "Invalid value for commit.retry.max-wait-ms: invalid digit found in string"
837            )
838        );
839
840        let invalid_target_size = HashMap::from([(
841            TableProperties::PROPERTY_WRITE_TARGET_FILE_SIZE_BYTES.to_string(),
842            "abc".to_string(),
843        )]);
844        let table_properties = TableProperties::new(&invalid_target_size);
845        let error = table_properties.write_target_file_size_bytes().unwrap_err();
846        assert!(error.to_string().contains(
847            "Invalid value for write.target-file-size-bytes: invalid digit found in string"
848        ));
849
850        let invalid_gc_enabled = HashMap::from([(
851            TableProperties::PROPERTY_GC_ENABLED.to_string(),
852            "notabool".to_string(),
853        )]);
854        let table_properties = TableProperties::new(&invalid_gc_enabled);
855        let error = table_properties.gc_enabled().unwrap_err();
856        assert!(error.to_string().contains("Invalid value for gc.enabled"));
857    }
858
859    #[test]
860    fn test_table_properties_compression_invalid_rejected() {
861        let invalid_codecs = ["lz4", "zstd", "snappy"];
862
863        for codec in invalid_codecs {
864            let props = HashMap::from([(
865                TableProperties::PROPERTY_METADATA_COMPRESSION_CODEC.to_string(),
866                codec.to_string(),
867            )]);
868            let err = TableProperties::new(&props)
869                .metadata_compression_codec()
870                .unwrap_err();
871            let err_msg = err.to_string();
872            assert!(
873                err_msg.contains(&format!("Invalid metadata compression codec: {codec}")),
874                "Expected error message to contain codec '{codec}', got: {err_msg}"
875            );
876            assert!(
877                err_msg.contains("Only 'none' and 'gzip' are supported"),
878                "Expected error message to contain supported codecs, got: {err_msg}"
879            );
880        }
881    }
882
883    #[test]
884    fn test_cdc_disabled_by_default() {
885        let props = HashMap::new();
886        let tp = TableProperties::new(&props);
887        assert!(!tp.cdc_enabled().unwrap());
888    }
889
890    #[test]
891    fn test_cdc_enabled_via_flag() {
892        let props = HashMap::from([(
893            TableProperties::PROPERTY_PARQUET_CDC_ENABLED.to_string(),
894            "true".to_string(),
895        )]);
896        let tp = TableProperties::new(&props);
897        assert!(tp.cdc_enabled().unwrap());
898        assert_eq!(tp.cdc_min_chunk_size().unwrap(), 256 * 1024);
899        assert_eq!(tp.cdc_max_chunk_size().unwrap(), 1024 * 1024);
900        assert_eq!(tp.cdc_norm_level().unwrap(), 0);
901    }
902
903    #[test]
904    fn test_cdc_size_props_alone_do_not_enable() {
905        let props = HashMap::from([(
906            TableProperties::PROPERTY_PARQUET_CDC_MIN_CHUNK_SIZE.to_string(),
907            "262144".to_string(),
908        )]);
909        let tp = TableProperties::new(&props);
910        assert!(!tp.cdc_enabled().unwrap());
911    }
912
913    #[test]
914    fn test_cdc_custom_values() {
915        let props = HashMap::from([
916            (
917                TableProperties::PROPERTY_PARQUET_CDC_ENABLED.to_string(),
918                "true".to_string(),
919            ),
920            (
921                TableProperties::PROPERTY_PARQUET_CDC_MIN_CHUNK_SIZE.to_string(),
922                "200000".to_string(),
923            ),
924            (
925                TableProperties::PROPERTY_PARQUET_CDC_MAX_CHUNK_SIZE.to_string(),
926                "900000".to_string(),
927            ),
928            (
929                TableProperties::PROPERTY_PARQUET_CDC_NORM_LEVEL.to_string(),
930                "1".to_string(),
931            ),
932        ]);
933        let tp = TableProperties::new(&props);
934        assert!(tp.cdc_enabled().unwrap());
935        assert_eq!(tp.cdc_min_chunk_size().unwrap(), 200000);
936        assert_eq!(tp.cdc_max_chunk_size().unwrap(), 900000);
937        assert_eq!(tp.cdc_norm_level().unwrap(), 1);
938    }
939
940    #[test]
941    fn test_cdc_partial_override() {
942        let props = HashMap::from([
943            (
944                TableProperties::PROPERTY_PARQUET_CDC_ENABLED.to_string(),
945                "true".to_string(),
946            ),
947            (
948                TableProperties::PROPERTY_PARQUET_CDC_NORM_LEVEL.to_string(),
949                "2".to_string(),
950            ),
951        ]);
952        let tp = TableProperties::new(&props);
953        assert!(tp.cdc_enabled().unwrap());
954        assert_eq!(tp.cdc_min_chunk_size().unwrap(), 256 * 1024);
955        assert_eq!(tp.cdc_max_chunk_size().unwrap(), 1024 * 1024);
956        assert_eq!(tp.cdc_norm_level().unwrap(), 2);
957    }
958
959    #[test]
960    fn test_cdc_negative_norm_level() {
961        let props = HashMap::from([
962            (
963                TableProperties::PROPERTY_PARQUET_CDC_ENABLED.to_string(),
964                "true".to_string(),
965            ),
966            (
967                TableProperties::PROPERTY_PARQUET_CDC_NORM_LEVEL.to_string(),
968                "-2".to_string(),
969            ),
970        ]);
971        let tp = TableProperties::new(&props);
972        assert_eq!(tp.cdc_norm_level().unwrap(), -2);
973    }
974
975    #[test]
976    fn test_cdc_invalid_min_chunk_size() {
977        let props = HashMap::from([
978            (
979                TableProperties::PROPERTY_PARQUET_CDC_ENABLED.to_string(),
980                "true".to_string(),
981            ),
982            (
983                TableProperties::PROPERTY_PARQUET_CDC_MIN_CHUNK_SIZE.to_string(),
984                "not_a_number".to_string(),
985            ),
986        ]);
987        let err = TableProperties::new(&props)
988            .cdc_min_chunk_size()
989            .unwrap_err();
990        assert!(
991            err.to_string().contains(
992                "Invalid value for write.parquet.content-defined-chunking.min-chunk-size"
993            )
994        );
995    }
996
997    #[test]
998    fn test_cdc_invalid_norm_level() {
999        let props = HashMap::from([
1000            (
1001                TableProperties::PROPERTY_PARQUET_CDC_ENABLED.to_string(),
1002                "true".to_string(),
1003            ),
1004            (
1005                TableProperties::PROPERTY_PARQUET_CDC_NORM_LEVEL.to_string(),
1006                "not_a_number".to_string(),
1007            ),
1008        ]);
1009        let err = TableProperties::new(&props).cdc_norm_level().unwrap_err();
1010        assert!(
1011            err.to_string()
1012                .contains("Invalid value for write.parquet.content-defined-chunking.norm-level")
1013        );
1014    }
1015
1016    #[test]
1017    fn test_cdc_no_properties() {
1018        let props = HashMap::from([("some.other.property".to_string(), "value".to_string())]);
1019        let tp = TableProperties::new(&props);
1020        assert!(!tp.cdc_enabled().unwrap());
1021    }
1022
1023    #[test]
1024    fn test_parquet_sizing_defaults() {
1025        let props = HashMap::new();
1026        let tp = TableProperties::new(&props);
1027        // Default codec is zstd at its default level.
1028        assert_eq!(
1029            tp.parquet_compression_codec().unwrap(),
1030            CompressionCodec::zstd_default()
1031        );
1032        assert_eq!(
1033            tp.parquet_row_group_size_bytes().unwrap(),
1034            TableProperties::PROPERTY_PARQUET_ROW_GROUP_SIZE_BYTES_DEFAULT
1035        );
1036        assert_eq!(
1037            tp.parquet_page_size_bytes().unwrap(),
1038            TableProperties::PROPERTY_PARQUET_PAGE_SIZE_BYTES_DEFAULT
1039        );
1040        assert_eq!(
1041            tp.parquet_page_row_limit().unwrap(),
1042            TableProperties::PROPERTY_PARQUET_PAGE_ROW_LIMIT_DEFAULT
1043        );
1044        assert_eq!(
1045            tp.parquet_dict_size_bytes().unwrap(),
1046            TableProperties::PROPERTY_PARQUET_DICT_SIZE_BYTES_DEFAULT
1047        );
1048    }
1049
1050    #[test]
1051    fn test_parquet_sizing_overrides() {
1052        let props = HashMap::from([
1053            (
1054                TableProperties::PROPERTY_PARQUET_COMPRESSION_CODEC.to_string(),
1055                "gzip".to_string(),
1056            ),
1057            (
1058                TableProperties::PROPERTY_PARQUET_COMPRESSION_LEVEL.to_string(),
1059                "4".to_string(),
1060            ),
1061            (
1062                TableProperties::PROPERTY_PARQUET_ROW_GROUP_SIZE_BYTES.to_string(),
1063                "1048576".to_string(),
1064            ),
1065            (
1066                TableProperties::PROPERTY_PARQUET_PAGE_SIZE_BYTES.to_string(),
1067                "65536".to_string(),
1068            ),
1069            (
1070                TableProperties::PROPERTY_PARQUET_PAGE_ROW_LIMIT.to_string(),
1071                "5000".to_string(),
1072            ),
1073            (
1074                TableProperties::PROPERTY_PARQUET_DICT_SIZE_BYTES.to_string(),
1075                "131072".to_string(),
1076            ),
1077        ]);
1078        let tp = TableProperties::new(&props);
1079        // Codec name and level are folded into a single CompressionCodec.
1080        assert_eq!(
1081            tp.parquet_compression_codec().unwrap(),
1082            CompressionCodec::Gzip(4)
1083        );
1084        assert_eq!(tp.parquet_row_group_size_bytes().unwrap(), 1048576);
1085        assert_eq!(tp.parquet_page_size_bytes().unwrap(), 65536);
1086        assert_eq!(tp.parquet_page_row_limit().unwrap(), 5000);
1087        assert_eq!(tp.parquet_dict_size_bytes().unwrap(), 131072);
1088    }
1089
1090    #[test]
1091    fn test_parquet_invalid_sizing_rejected() {
1092        let props = HashMap::from([(
1093            TableProperties::PROPERTY_PARQUET_ROW_GROUP_SIZE_BYTES.to_string(),
1094            "not_a_number".to_string(),
1095        )]);
1096        let err = TableProperties::new(&props)
1097            .parquet_row_group_size_bytes()
1098            .unwrap_err();
1099        assert_eq!(err.kind(), ErrorKind::DataInvalid);
1100        assert!(
1101            err.to_string()
1102                .contains(TableProperties::PROPERTY_PARQUET_ROW_GROUP_SIZE_BYTES)
1103        );
1104    }
1105
1106    #[test]
1107    fn test_parquet_all_codecs_parse() {
1108        // Every codec name parquet-java supports must parse (parity with Java's
1109        // `CompressionCodecName.valueOf`).
1110        for (name, expected) in [
1111            ("uncompressed", CompressionCodec::None),
1112            ("snappy", CompressionCodec::Snappy),
1113            ("gzip", CompressionCodec::gzip_default()),
1114            ("lzo", CompressionCodec::Lzo),
1115            ("brotli", CompressionCodec::brotli_default()),
1116            ("lz4", CompressionCodec::Lz4),
1117            ("lz4_raw", CompressionCodec::Lz4Raw),
1118            ("zstd", CompressionCodec::zstd_default()),
1119        ] {
1120            let props = HashMap::from([(
1121                TableProperties::PROPERTY_PARQUET_COMPRESSION_CODEC.to_string(),
1122                name.to_string(),
1123            )]);
1124            let tp = TableProperties::new(&props);
1125            assert_eq!(
1126                tp.parquet_compression_codec().unwrap(),
1127                expected,
1128                "codec {name}"
1129            );
1130        }
1131    }
1132
1133    #[test]
1134    fn test_parquet_compression_level_ignored_for_levelless_codec() {
1135        // A level set alongside a codec that carries none (e.g. snappy) is
1136        // ignored rather than rejected, matching parquet-java.
1137        let props = HashMap::from([
1138            (
1139                TableProperties::PROPERTY_PARQUET_COMPRESSION_CODEC.to_string(),
1140                "snappy".to_string(),
1141            ),
1142            (
1143                TableProperties::PROPERTY_PARQUET_COMPRESSION_LEVEL.to_string(),
1144                "5".to_string(),
1145            ),
1146        ]);
1147        let tp = TableProperties::new(&props);
1148        assert_eq!(
1149            tp.parquet_compression_codec().unwrap(),
1150            CompressionCodec::Snappy
1151        );
1152    }
1153
1154    #[test]
1155    fn test_parse_boolean_property_case_insensitive() {
1156        let false_variants = ["False", "FALSE"];
1157        let true_variants = ["True", "TRUE"];
1158
1159        for f in false_variants {
1160            let props = HashMap::from([(
1161                TableProperties::PROPERTY_WRITE_OBJECT_STORAGE_PARTITIONED_PATHS.to_string(),
1162                f.to_string(),
1163            )]);
1164            let tp = TableProperties::new(&props);
1165            assert!(!tp.write_object_storage_partitioned_paths().unwrap());
1166        }
1167
1168        for t in true_variants {
1169            let props = HashMap::from([(
1170                TableProperties::PROPERTY_WRITE_OBJECT_STORAGE_PARTITIONED_PATHS.to_string(),
1171                t.to_string(),
1172            )]);
1173            let tp = TableProperties::new(&props);
1174            assert!(tp.write_object_storage_partitioned_paths().unwrap());
1175        }
1176    }
1177
1178    #[test]
1179    fn test_table_properties_default_name_mapping() {
1180        // Test unset.
1181        let properties = HashMap::new();
1182        assert!(
1183            TableProperties::new(&properties)
1184                .default_name_mapping()
1185                .unwrap()
1186                .is_none()
1187        );
1188
1189        let properties = HashMap::from([(
1190            TableProperties::PROPERTY_DEFAULT_NAME_MAPPING.to_string(),
1191            r#"[{"field-id":1,"names":["id","record_id"]}]"#.to_string(),
1192        )]);
1193        let mapping = TableProperties::new(&properties)
1194            .default_name_mapping()
1195            .unwrap()
1196            .unwrap();
1197        assert_eq!(mapping.fields().len(), 1);
1198        assert_eq!(mapping.fields()[0].field_id(), Some(1));
1199        assert_eq!(mapping.fields()[0].names(), &[
1200            "id".to_string(),
1201            "record_id".to_string()
1202        ]);
1203    }
1204
1205    #[test]
1206    fn test_table_properties_malformed_name_mapping() {
1207        let properties = HashMap::from([(
1208            TableProperties::PROPERTY_DEFAULT_NAME_MAPPING.to_string(),
1209            "{ not valid json".to_string(),
1210        )]);
1211        let error = TableProperties::new(&properties)
1212            .default_name_mapping()
1213            .unwrap_err();
1214
1215        assert_eq!(error.kind(), ErrorKind::DataInvalid);
1216        // The property key must survive as error context.
1217        assert!(
1218            format!("{error}").contains(TableProperties::PROPERTY_DEFAULT_NAME_MAPPING),
1219            "{error}"
1220        );
1221    }
1222}
iceberg/spec/table_properties.rs

iceberg/spec/
table_properties.rs
