1use std::collections::HashMap;
19
20use iceberg_property_macro::properties_view;
21
22use crate::compression::CompressionCodec;
23use crate::encryption::AesKeySize;
24use crate::error::{Result, invalid_data};
25use crate::spec::NameMapping;
26use crate::util::location::strip_trailing_slash;
27
28fn parse_location_property(path: &str) -> Result<String> {
29 if path.is_empty() {
30 return Err(invalid_data!("path must not be empty"));
31 }
32
33 Ok(strip_trailing_slash(path).to_string())
34}
35
36fn parse_metadata_compression(value: &str) -> Result<CompressionCodec> {
37 if value.is_empty() {
39 return Ok(CompressionCodec::None);
40 }
41
42 let lowercase_value = value.to_lowercase();
44
45 let codec: CompressionCodec =
47 serde_json::from_value(serde_json::Value::String(lowercase_value)).map_err(|_| {
48 invalid_data!(
49 "Invalid metadata compression codec: {value}. Only '{}' and '{}' are supported.",
50 CompressionCodec::None.name(),
51 CompressionCodec::gzip_default().name()
52 )
53 })?;
54
55 match codec {
57 CompressionCodec::None | CompressionCodec::Gzip(_) => Ok(codec),
58 _ => Err(invalid_data!(
59 "Invalid metadata compression codec: {value}. Only '{}' and '{}' are supported for metadata files.",
60 CompressionCodec::None.name(),
61 CompressionCodec::gzip_default().name()
62 )),
63 }
64}
65
66fn parse_parquet_compression(
70 properties: &HashMap<String, String>,
71 codec_key: &str,
72 additional_keys: &[&str],
73 default: CompressionCodec,
74) -> Result<CompressionCodec> {
75 let level_key = additional_keys[0];
76 let codec = properties
77 .get(codec_key)
78 .map(|value| {
79 serde_json::from_value(serde_json::Value::String(value.to_lowercase())).map_err(|_| {
80 invalid_data!(
81 "Invalid Parquet compression codec: {value}. Supported codecs: \
82 uncompressed, snappy, gzip, lzo, brotli, lz4, lz4_raw, zstd"
83 )
84 })
85 })
86 .transpose()?
87 .unwrap_or(default);
88
89 let level = properties
90 .get(level_key)
91 .map(|value| {
92 value
93 .parse::<u8>()
94 .map_err(|error| invalid_data!("Invalid value for {level_key}: {error}"))
95 })
96 .transpose()?;
97
98 Ok(match (codec, level) {
99 (CompressionCodec::Zstd(_), Some(level)) => CompressionCodec::Zstd(level),
100 (CompressionCodec::Gzip(_), Some(level)) => CompressionCodec::Gzip(level),
101 (CompressionCodec::Brotli(_), Some(level)) => CompressionCodec::Brotli(level),
102 (codec, _) => codec,
103 })
104}
105
106properties_view! {
107#[derive(Debug)]
109pub struct TableProperties {
110 #[property(
112 key = Self::PROPERTY_COMMIT_NUM_RETRIES,
113 default = Self::PROPERTY_COMMIT_NUM_RETRIES_DEFAULT,
114 getter
115 )]
116 commit_num_retries: usize,
117 #[property(
119 key = Self::PROPERTY_COMMIT_MIN_RETRY_WAIT_MS,
120 default = Self::PROPERTY_COMMIT_MIN_RETRY_WAIT_MS_DEFAULT,
121 getter
122 )]
123 commit_min_retry_wait_ms: u64,
124 #[property(
126 key = Self::PROPERTY_COMMIT_MAX_RETRY_WAIT_MS,
127 default = Self::PROPERTY_COMMIT_MAX_RETRY_WAIT_MS_DEFAULT,
128 getter
129 )]
130 commit_max_retry_wait_ms: u64,
131 #[property(
133 key = Self::PROPERTY_COMMIT_TOTAL_RETRY_TIME_MS,
134 default = Self::PROPERTY_COMMIT_TOTAL_RETRY_TIME_MS_DEFAULT,
135 getter
136 )]
137 commit_total_retry_timeout_ms: u64,
138 #[property(
140 key = Self::PROPERTY_DEFAULT_FILE_FORMAT,
141 default = Self::PROPERTY_DEFAULT_FILE_FORMAT_DEFAULT,
142 getter
143 )]
144 write_format_default: String,
145 #[property(
147 key = Self::PROPERTY_WRITE_TARGET_FILE_SIZE_BYTES,
148 default = Self::PROPERTY_WRITE_TARGET_FILE_SIZE_BYTES_DEFAULT,
149 getter
150 )]
151 write_target_file_size_bytes: usize,
152 #[property(
155 key = Self::PROPERTY_WRITE_METADATA_PATH,
156 default = None,
157 parse_with = parse_location_property,
158 getter
159 )]
160 write_metadata_path: Option<String>,
161 #[property(
163 key = Self::PROPERTY_METADATA_COMPRESSION_CODEC,
164 default = CompressionCodec::None,
165 parse_with = parse_metadata_compression,
166 getter
167 )]
168 metadata_compression_codec: CompressionCodec,
169 #[property(
171 key = Self::PROPERTY_DATAFUSION_WRITE_FANOUT_ENABLED,
172 default = Self::PROPERTY_DATAFUSION_WRITE_FANOUT_ENABLED_DEFAULT,
173 getter
174 )]
175 write_datafusion_fanout_enabled: bool,
176 #[property(
179 key = Self::PROPERTY_GC_ENABLED,
180 default = Self::PROPERTY_GC_ENABLED_DEFAULT,
181 getter
182 )]
183 gc_enabled: bool,
184 #[property(
186 key = Self::PROPERTY_MAX_SNAPSHOT_AGE_MS,
187 default = Self::PROPERTY_MAX_SNAPSHOT_AGE_MS_DEFAULT,
188 getter
189 )]
190 max_snapshot_age_ms: i64,
191 #[property(
193 key = Self::PROPERTY_MIN_SNAPSHOTS_TO_KEEP,
194 default = Self::PROPERTY_MIN_SNAPSHOTS_TO_KEEP_DEFAULT,
195 getter
196 )]
197 min_snapshots_to_keep: usize,
198 #[property(
200 key = Self::PROPERTY_MAX_REF_AGE_MS,
201 default = Self::PROPERTY_MAX_REF_AGE_MS_DEFAULT,
202 getter
203 )]
204 max_ref_age_ms: i64,
205 #[property(
208 key = Self::PROPERTY_PARQUET_CDC_ENABLED,
209 default = Self::PROPERTY_PARQUET_CDC_ENABLED_DEFAULT,
210 getter
211 )]
212 cdc_enabled: bool,
213 #[property(
215 key = Self::PROPERTY_PARQUET_CDC_MIN_CHUNK_SIZE,
216 default = Self::PROPERTY_PARQUET_CDC_MIN_CHUNK_SIZE_DEFAULT,
217 getter
218 )]
219 cdc_min_chunk_size: usize,
220 #[property(
222 key = Self::PROPERTY_PARQUET_CDC_MAX_CHUNK_SIZE,
223 default = Self::PROPERTY_PARQUET_CDC_MAX_CHUNK_SIZE_DEFAULT,
224 getter
225 )]
226 cdc_max_chunk_size: usize,
227 #[property(
229 key = Self::PROPERTY_PARQUET_CDC_NORM_LEVEL,
230 default = Self::PROPERTY_PARQUET_CDC_NORM_LEVEL_DEFAULT,
231 getter
232 )]
233 cdc_norm_level: i32,
234 #[property(
238 key = Self::PROPERTY_PARQUET_COMPRESSION_CODEC,
239 additional_keys = [Self::PROPERTY_PARQUET_COMPRESSION_LEVEL],
240 default = CompressionCodec::zstd_default(),
241 parse_properties_with = parse_parquet_compression,
242 getter
243 )]
244 parquet_compression_codec: CompressionCodec,
245 #[property(
247 key = Self::PROPERTY_PARQUET_ROW_GROUP_SIZE_BYTES,
248 default = Self::PROPERTY_PARQUET_ROW_GROUP_SIZE_BYTES_DEFAULT,
249 getter
250 )]
251 parquet_row_group_size_bytes: usize,
252 #[property(
254 key = Self::PROPERTY_PARQUET_PAGE_SIZE_BYTES,
255 default = Self::PROPERTY_PARQUET_PAGE_SIZE_BYTES_DEFAULT,
256 getter
257 )]
258 parquet_page_size_bytes: usize,
259 #[property(
261 key = Self::PROPERTY_PARQUET_PAGE_ROW_LIMIT,
262 default = Self::PROPERTY_PARQUET_PAGE_ROW_LIMIT_DEFAULT,
263 getter
264 )]
265 parquet_page_row_limit: usize,
266 #[property(
268 key = Self::PROPERTY_PARQUET_DICT_SIZE_BYTES,
269 default = Self::PROPERTY_PARQUET_DICT_SIZE_BYTES_DEFAULT,
270 getter
271 )]
272 parquet_dict_size_bytes: usize,
273 #[property(
276 key = Self::PROPERTY_ENCRYPTION_KEY_ID,
277 default = None,
278 getter
279 )]
280 encryption_key_id: Option<String>,
281 #[property(
283 key = Self::PROPERTY_ENCRYPTION_DATA_KEY_LENGTH,
284 default = Self::PROPERTY_ENCRYPTION_DATA_KEY_LENGTH_DEFAULT,
285 getter
286 )]
287 encryption_data_key_length: usize,
288 #[property(
290 key = Self::PROPERTY_WRITE_DATA_LOCATION,
291 default = None,
292 parse_with = parse_location_property,
293 getter
294 )]
295 write_data_location: Option<String>,
296 #[property(
301 key = Self::PROPERTY_WRITE_FOLDER_STORAGE_LOCATION,
302 default = None,
303 parse_with = parse_location_property,
304 getter
305 )]
306 write_folder_storage_location: Option<String>,
307 #[property(
313 key = Self::PROPERTY_WRITE_OBJECT_STORAGE_LOCATION,
314 default = None,
315 parse_with = parse_location_property,
316 getter
317 )]
318 write_object_storage_location: Option<String>,
319 #[property(
322 key = Self::PROPERTY_WRITE_OBJECT_STORAGE_ENABLED,
323 default = Self::PROPERTY_WRITE_OBJECT_STORAGE_ENABLED_DEFAULT,
324 getter
325 )]
326 write_object_storage_enabled: bool,
327 #[property(
329 key = Self::PROPERTY_WRITE_OBJECT_STORAGE_PARTITIONED_PATHS,
330 default = Self::PROPERTY_WRITE_OBJECT_STORAGE_PARTITIONED_PATHS_DEFAULT,
331 getter
332 )]
333 write_object_storage_partitioned_paths: bool,
334 #[property(
336 key = Self::PROPERTY_METADATA_DELETE_AFTER_COMMIT_ENABLED,
337 default = Self::PROPERTY_METADATA_DELETE_AFTER_COMMIT_ENABLED_DEFAULT,
338 getter
339 )]
340 metadata_delete_after_commit_enabled: bool,
341 #[property(
344 key = Self::PROPERTY_DEFAULT_NAME_MAPPING,
345 default = None,
346 getter
347 )]
348 default_name_mapping: Option<NameMapping>,
349}
350}
351
352impl TableProperties<'_> {
353 pub const PROPERTY_FORMAT_VERSION: &'static str = "format-version";
363 pub const PROPERTY_UUID: &'static str = "uuid";
365 pub const PROPERTY_SNAPSHOT_COUNT: &'static str = "snapshot-count";
367 pub const PROPERTY_CURRENT_SNAPSHOT_SUMMARY: &'static str = "current-snapshot-summary";
369 pub const PROPERTY_CURRENT_SNAPSHOT_ID: &'static str = "current-snapshot-id";
371 pub const PROPERTY_CURRENT_SNAPSHOT_TIMESTAMP: &'static str = "current-snapshot-timestamp-ms";
373 pub const PROPERTY_CURRENT_SCHEMA: &'static str = "current-schema";
375 pub const PROPERTY_DEFAULT_PARTITION_SPEC: &'static str = "default-partition-spec";
377 pub const PROPERTY_DEFAULT_SORT_ORDER: &'static str = "default-sort-order";
379
380 pub const PROPERTY_METADATA_PREVIOUS_VERSIONS_MAX: &'static str =
382 "write.metadata.previous-versions-max";
383 pub const PROPERTY_METADATA_PREVIOUS_VERSIONS_MAX_DEFAULT: usize = 100;
385
386 pub const PROPERTY_METADATA_DELETE_AFTER_COMMIT_ENABLED: &'static str =
389 "write.metadata.delete-after-commit.enabled";
390 pub const PROPERTY_METADATA_DELETE_AFTER_COMMIT_ENABLED_DEFAULT: bool = false;
392
393 pub const PROPERTY_WRITE_PARTITION_SUMMARY_LIMIT: &'static str =
395 "write.summary.partition-limit";
396 pub const PROPERTY_WRITE_PARTITION_SUMMARY_LIMIT_DEFAULT: u64 = 0;
398
399 pub const RESERVED_PROPERTIES: [&'static str; 9] = [
404 Self::PROPERTY_FORMAT_VERSION,
405 Self::PROPERTY_UUID,
406 Self::PROPERTY_SNAPSHOT_COUNT,
407 Self::PROPERTY_CURRENT_SNAPSHOT_ID,
408 Self::PROPERTY_CURRENT_SNAPSHOT_SUMMARY,
409 Self::PROPERTY_CURRENT_SNAPSHOT_TIMESTAMP,
410 Self::PROPERTY_CURRENT_SCHEMA,
411 Self::PROPERTY_DEFAULT_PARTITION_SPEC,
412 Self::PROPERTY_DEFAULT_SORT_ORDER,
413 ];
414
415 pub const PROPERTY_COMMIT_NUM_RETRIES: &'static str = "commit.retry.num-retries";
417 pub const PROPERTY_COMMIT_NUM_RETRIES_DEFAULT: usize = 4;
419
420 pub const PROPERTY_COMMIT_MIN_RETRY_WAIT_MS: &'static str = "commit.retry.min-wait-ms";
422 pub const PROPERTY_COMMIT_MIN_RETRY_WAIT_MS_DEFAULT: u64 = 100;
424
425 pub const PROPERTY_COMMIT_MAX_RETRY_WAIT_MS: &'static str = "commit.retry.max-wait-ms";
427 pub const PROPERTY_COMMIT_MAX_RETRY_WAIT_MS_DEFAULT: u64 = 60 * 1000; pub const PROPERTY_COMMIT_TOTAL_RETRY_TIME_MS: &'static str = "commit.retry.total-timeout-ms";
432 pub const PROPERTY_COMMIT_TOTAL_RETRY_TIME_MS_DEFAULT: u64 = 30 * 60 * 1000; pub const PROPERTY_DEFAULT_FILE_FORMAT: &'static str = "write.format.default";
437 pub const PROPERTY_DELETE_DEFAULT_FILE_FORMAT: &'static str = "write.delete.format.default";
439 pub const PROPERTY_DEFAULT_FILE_FORMAT_DEFAULT: &'static str = "parquet";
441
442 pub const PROPERTY_WRITE_TARGET_FILE_SIZE_BYTES: &'static str = "write.target-file-size-bytes";
444 pub const PROPERTY_WRITE_TARGET_FILE_SIZE_BYTES_DEFAULT: usize = 512 * 1024 * 1024; pub const PROPERTY_WRITE_METADATA_PATH: &'static str = "write.metadata.path";
451
452 pub const PROPERTY_DEFAULT_NAME_MAPPING: &'static str = "schema.name-mapping.default";
455
456 pub const PROPERTY_METADATA_COMPRESSION_CODEC: &'static str =
458 "write.metadata.compression-codec";
459 pub const PROPERTY_METADATA_COMPRESSION_CODEC_DEFAULT: &'static str = "none";
461 pub const PROPERTY_DATAFUSION_WRITE_FANOUT_ENABLED: &'static str =
464 "write.datafusion.fanout.enabled";
465 pub const PROPERTY_DATAFUSION_WRITE_FANOUT_ENABLED_DEFAULT: bool = true;
467
468 pub const PROPERTY_GC_ENABLED: &'static str = "gc.enabled";
472 pub const PROPERTY_GC_ENABLED_DEFAULT: bool = true;
474
475 pub const PROPERTY_MAX_SNAPSHOT_AGE_MS: &'static str = "history.expire.max-snapshot-age-ms";
477 pub const PROPERTY_MAX_SNAPSHOT_AGE_MS_DEFAULT: i64 = 5 * 24 * 60 * 60 * 1000;
479 pub const PROPERTY_MIN_SNAPSHOTS_TO_KEEP: &'static str = "history.expire.min-snapshots-to-keep";
481 pub const PROPERTY_MIN_SNAPSHOTS_TO_KEEP_DEFAULT: usize = 1;
483 pub const PROPERTY_MAX_REF_AGE_MS: &'static str = "history.expire.max-ref-age-ms";
485 pub const PROPERTY_MAX_REF_AGE_MS_DEFAULT: i64 = i64::MAX;
487
488 pub const PROPERTY_PARQUET_CDC_ENABLED: &'static str =
490 "write.parquet.content-defined-chunking.enabled";
491 pub const PROPERTY_PARQUET_CDC_ENABLED_DEFAULT: bool = false;
493 pub const PROPERTY_PARQUET_CDC_MIN_CHUNK_SIZE: &'static str =
495 "write.parquet.content-defined-chunking.min-chunk-size";
496 pub const PROPERTY_PARQUET_CDC_MIN_CHUNK_SIZE_DEFAULT: usize = 256 * 1024;
498 pub const PROPERTY_PARQUET_CDC_MAX_CHUNK_SIZE: &'static str =
500 "write.parquet.content-defined-chunking.max-chunk-size";
501 pub const PROPERTY_PARQUET_CDC_MAX_CHUNK_SIZE_DEFAULT: usize = 1024 * 1024;
503 pub const PROPERTY_PARQUET_CDC_NORM_LEVEL: &'static str =
505 "write.parquet.content-defined-chunking.norm-level";
506 pub const PROPERTY_PARQUET_CDC_NORM_LEVEL_DEFAULT: i32 = 0;
508
509 pub const PROPERTY_PARQUET_COMPRESSION_CODEC: &'static str = "write.parquet.compression-codec";
514 pub const PROPERTY_PARQUET_COMPRESSION_CODEC_DEFAULT: &'static str = "zstd";
516 pub const PROPERTY_PARQUET_COMPRESSION_LEVEL: &'static str = "write.parquet.compression-level";
519
520 pub const PROPERTY_PARQUET_ROW_GROUP_SIZE_BYTES: &'static str =
522 "write.parquet.row-group-size-bytes";
523 pub const PROPERTY_PARQUET_ROW_GROUP_SIZE_BYTES_DEFAULT: usize = 128 * 1024 * 1024;
525
526 pub const PROPERTY_PARQUET_PAGE_SIZE_BYTES: &'static str = "write.parquet.page-size-bytes";
528 pub const PROPERTY_PARQUET_PAGE_SIZE_BYTES_DEFAULT: usize = 1024 * 1024;
530
531 pub const PROPERTY_PARQUET_PAGE_ROW_LIMIT: &'static str = "write.parquet.page-row-limit";
533 pub const PROPERTY_PARQUET_PAGE_ROW_LIMIT_DEFAULT: usize = 20000;
535
536 pub const PROPERTY_PARQUET_DICT_SIZE_BYTES: &'static str = "write.parquet.dict-size-bytes";
538 pub const PROPERTY_PARQUET_DICT_SIZE_BYTES_DEFAULT: usize = 2 * 1024 * 1024;
540
541 pub const PROPERTY_ENCRYPTION_KEY_ID: &'static str = "encryption.key-id";
544
545 pub const PROPERTY_ENCRYPTION_DATA_KEY_LENGTH: &'static str = "encryption.data-key-length";
547 pub const PROPERTY_ENCRYPTION_DATA_KEY_LENGTH_DEFAULT: usize = 16;
549 pub const PROPERTY_WRITE_DATA_LOCATION: &'static str = "write.data.path";
551 pub const PROPERTY_WRITE_FOLDER_STORAGE_LOCATION: &'static str = "write.folder-storage.path";
553 pub const PROPERTY_WRITE_OBJECT_STORAGE_LOCATION: &'static str = "write.object-storage.path";
555 pub const PROPERTY_WRITE_OBJECT_STORAGE_ENABLED: &'static str = "write.object-storage.enabled";
557 pub const PROPERTY_WRITE_OBJECT_STORAGE_ENABLED_DEFAULT: bool = false;
559 pub const PROPERTY_WRITE_OBJECT_STORAGE_PARTITIONED_PATHS: &'static str =
561 "write.object-storage.partitioned-paths";
562 pub const PROPERTY_WRITE_OBJECT_STORAGE_PARTITIONED_PATHS_DEFAULT: bool = true;
564
565 pub fn data_encryption_key_size(&self) -> Result<AesKeySize> {
568 AesKeySize::from_key_length(self.encryption_data_key_length()?)
569 }
570}
571
572#[cfg(test)]
573mod tests {
574 use super::*;
575 use crate::ErrorKind;
576 use crate::compression::CompressionCodec;
577
578 #[test]
579 fn test_table_properties_default() {
580 let props = HashMap::new();
581 let table_properties = TableProperties::new(&props);
582 assert_eq!(
583 table_properties.commit_num_retries().unwrap(),
584 TableProperties::PROPERTY_COMMIT_NUM_RETRIES_DEFAULT
585 );
586 assert_eq!(
587 table_properties.commit_min_retry_wait_ms().unwrap(),
588 TableProperties::PROPERTY_COMMIT_MIN_RETRY_WAIT_MS_DEFAULT
589 );
590 assert_eq!(
591 table_properties.commit_max_retry_wait_ms().unwrap(),
592 TableProperties::PROPERTY_COMMIT_MAX_RETRY_WAIT_MS_DEFAULT
593 );
594 assert_eq!(
595 table_properties.write_format_default().unwrap(),
596 TableProperties::PROPERTY_DEFAULT_FILE_FORMAT_DEFAULT.to_string()
597 );
598 assert_eq!(
599 table_properties.write_target_file_size_bytes().unwrap(),
600 TableProperties::PROPERTY_WRITE_TARGET_FILE_SIZE_BYTES_DEFAULT
601 );
602 assert_eq!(
604 table_properties.metadata_compression_codec().unwrap(),
605 CompressionCodec::None
606 );
607 assert_eq!(
608 table_properties.gc_enabled().unwrap(),
609 TableProperties::PROPERTY_GC_ENABLED_DEFAULT
610 );
611 assert_eq!(
612 table_properties.max_snapshot_age_ms().unwrap(),
613 TableProperties::PROPERTY_MAX_SNAPSHOT_AGE_MS_DEFAULT
614 );
615 assert_eq!(
616 table_properties.min_snapshots_to_keep().unwrap(),
617 TableProperties::PROPERTY_MIN_SNAPSHOTS_TO_KEEP_DEFAULT
618 );
619 assert_eq!(
620 table_properties.max_ref_age_ms().unwrap(),
621 TableProperties::PROPERTY_MAX_REF_AGE_MS_DEFAULT
622 );
623 }
624
625 #[test]
626 fn test_table_properties_history_expire_overrides() {
627 let props = HashMap::from([
628 (
629 TableProperties::PROPERTY_MAX_SNAPSHOT_AGE_MS.to_string(),
630 "1234".to_string(),
631 ),
632 (
633 TableProperties::PROPERTY_MIN_SNAPSHOTS_TO_KEEP.to_string(),
634 "7".to_string(),
635 ),
636 (
637 TableProperties::PROPERTY_MAX_REF_AGE_MS.to_string(),
638 "5678".to_string(),
639 ),
640 ]);
641 let table_properties = TableProperties::new(&props);
642 assert_eq!(table_properties.max_snapshot_age_ms().unwrap(), 1234);
643 assert_eq!(table_properties.min_snapshots_to_keep().unwrap(), 7);
644 assert_eq!(table_properties.max_ref_age_ms().unwrap(), 5678);
645 }
646
647 #[test]
648 fn test_table_properties_location_paths() {
649 let raw_properties = HashMap::new();
651 let table_properties = TableProperties::new(&raw_properties);
652 assert_eq!(table_properties.write_metadata_path().unwrap(), None);
653 assert_eq!(table_properties.write_data_location().unwrap(), None);
654 assert_eq!(
655 table_properties.write_folder_storage_location().unwrap(),
656 None
657 );
658 assert_eq!(
659 table_properties.write_object_storage_location().unwrap(),
660 None
661 );
662
663 for key in [
664 TableProperties::PROPERTY_WRITE_METADATA_PATH,
665 TableProperties::PROPERTY_WRITE_DATA_LOCATION,
666 TableProperties::PROPERTY_WRITE_FOLDER_STORAGE_LOCATION,
667 TableProperties::PROPERTY_WRITE_OBJECT_STORAGE_LOCATION,
668 ] {
669 let raw_properties = HashMap::from([(key.to_string(), String::new())]);
671 let table_properties = TableProperties::new(&raw_properties);
672 let error = match key {
673 TableProperties::PROPERTY_WRITE_METADATA_PATH => {
674 table_properties.write_metadata_path().unwrap_err()
675 }
676 TableProperties::PROPERTY_WRITE_DATA_LOCATION => {
677 table_properties.write_data_location().unwrap_err()
678 }
679 TableProperties::PROPERTY_WRITE_FOLDER_STORAGE_LOCATION => table_properties
680 .write_folder_storage_location()
681 .unwrap_err(),
682 TableProperties::PROPERTY_WRITE_OBJECT_STORAGE_LOCATION => table_properties
683 .write_object_storage_location()
684 .unwrap_err(),
685 _ => unreachable!(),
686 };
687 assert_eq!(error.kind(), ErrorKind::DataInvalid);
688 assert!(format!("{error}").contains(key));
689
690 let raw_properties = HashMap::from([(
692 key.to_string(),
693 "s3://other-bucket/custom-path/".to_string(),
694 )]);
695 let table_properties = TableProperties::new(&raw_properties);
696 let parsed = match key {
697 TableProperties::PROPERTY_WRITE_METADATA_PATH => {
698 table_properties.write_metadata_path().unwrap()
699 }
700 TableProperties::PROPERTY_WRITE_DATA_LOCATION => {
701 table_properties.write_data_location().unwrap()
702 }
703 TableProperties::PROPERTY_WRITE_FOLDER_STORAGE_LOCATION => {
704 table_properties.write_folder_storage_location().unwrap()
705 }
706 TableProperties::PROPERTY_WRITE_OBJECT_STORAGE_LOCATION => {
707 table_properties.write_object_storage_location().unwrap()
708 }
709 _ => unreachable!(),
710 };
711 assert_eq!(parsed.as_deref(), Some("s3://other-bucket/custom-path"));
712 }
713 }
714
715 #[test]
716 fn test_table_properties_compression() {
717 let props = HashMap::from([(
718 TableProperties::PROPERTY_METADATA_COMPRESSION_CODEC.to_string(),
719 "gzip".to_string(),
720 )]);
721 let table_properties = TableProperties::new(&props);
722 assert_eq!(
723 table_properties.metadata_compression_codec().unwrap(),
724 CompressionCodec::gzip_default()
725 );
726 }
727
728 #[test]
729 fn test_table_properties_compression_none() {
730 let props = HashMap::from([(
731 TableProperties::PROPERTY_METADATA_COMPRESSION_CODEC.to_string(),
732 "none".to_string(),
733 )]);
734 let table_properties = TableProperties::new(&props);
735 assert_eq!(
736 table_properties.metadata_compression_codec().unwrap(),
737 CompressionCodec::None
738 );
739 }
740
741 #[test]
742 fn test_table_properties_compression_case_insensitive() {
743 let props_upper = HashMap::from([(
745 TableProperties::PROPERTY_METADATA_COMPRESSION_CODEC.to_string(),
746 "GZIP".to_string(),
747 )]);
748 let table_properties = TableProperties::new(&props_upper);
749 assert_eq!(
750 table_properties.metadata_compression_codec().unwrap(),
751 CompressionCodec::gzip_default()
752 );
753
754 let props_mixed = HashMap::from([(
756 TableProperties::PROPERTY_METADATA_COMPRESSION_CODEC.to_string(),
757 "GzIp".to_string(),
758 )]);
759 let table_properties = TableProperties::new(&props_mixed);
760 assert_eq!(
761 table_properties.metadata_compression_codec().unwrap(),
762 CompressionCodec::gzip_default()
763 );
764
765 let props_none_upper = HashMap::from([(
767 TableProperties::PROPERTY_METADATA_COMPRESSION_CODEC.to_string(),
768 "NONE".to_string(),
769 )]);
770 let table_properties = TableProperties::new(&props_none_upper);
771 assert_eq!(
772 table_properties.metadata_compression_codec().unwrap(),
773 CompressionCodec::None
774 );
775 }
776
777 #[test]
778 fn test_table_properties_valid() {
779 let props = HashMap::from([
780 (
781 TableProperties::PROPERTY_COMMIT_NUM_RETRIES.to_string(),
782 "10".to_string(),
783 ),
784 (
785 TableProperties::PROPERTY_COMMIT_MAX_RETRY_WAIT_MS.to_string(),
786 "20".to_string(),
787 ),
788 (
789 TableProperties::PROPERTY_DEFAULT_FILE_FORMAT.to_string(),
790 "avro".to_string(),
791 ),
792 (
793 TableProperties::PROPERTY_WRITE_TARGET_FILE_SIZE_BYTES.to_string(),
794 "512".to_string(),
795 ),
796 (
797 TableProperties::PROPERTY_GC_ENABLED.to_string(),
798 "false".to_string(),
799 ),
800 ]);
801 let table_properties = TableProperties::new(&props);
802 assert_eq!(table_properties.commit_num_retries().unwrap(), 10);
803 assert_eq!(table_properties.commit_max_retry_wait_ms().unwrap(), 20);
804 assert_eq!(
805 table_properties.write_format_default().unwrap(),
806 "avro".to_string()
807 );
808 assert_eq!(
809 table_properties.write_target_file_size_bytes().unwrap(),
810 512
811 );
812 assert!(!table_properties.gc_enabled().unwrap());
813 }
814
815 #[test]
816 fn test_table_properties_invalid() {
817 let invalid_retries = HashMap::from([(
818 TableProperties::PROPERTY_COMMIT_NUM_RETRIES.to_string(),
819 "abc".to_string(),
820 )]);
821
822 let table_properties = TableProperties::new(&invalid_retries);
823 let error = table_properties.commit_num_retries().unwrap_err();
824 assert!(
825 error.to_string().contains(
826 "Invalid value for commit.retry.num-retries: invalid digit found in string"
827 )
828 );
829
830 let invalid_min_wait = HashMap::from([(
831 TableProperties::PROPERTY_COMMIT_MIN_RETRY_WAIT_MS.to_string(),
832 "abc".to_string(),
833 )]);
834 let table_properties = TableProperties::new(&invalid_min_wait);
835 let error = table_properties.commit_min_retry_wait_ms().unwrap_err();
836 assert!(
837 error.to_string().contains(
838 "Invalid value for commit.retry.min-wait-ms: invalid digit found in string"
839 )
840 );
841
842 let invalid_max_wait = HashMap::from([(
843 TableProperties::PROPERTY_COMMIT_MAX_RETRY_WAIT_MS.to_string(),
844 "abc".to_string(),
845 )]);
846 let table_properties = TableProperties::new(&invalid_max_wait);
847 let error = table_properties.commit_max_retry_wait_ms().unwrap_err();
848 assert!(
849 error.to_string().contains(
850 "Invalid value for commit.retry.max-wait-ms: invalid digit found in string"
851 )
852 );
853
854 let invalid_target_size = HashMap::from([(
855 TableProperties::PROPERTY_WRITE_TARGET_FILE_SIZE_BYTES.to_string(),
856 "abc".to_string(),
857 )]);
858 let table_properties = TableProperties::new(&invalid_target_size);
859 let error = table_properties.write_target_file_size_bytes().unwrap_err();
860 assert!(error.to_string().contains(
861 "Invalid value for write.target-file-size-bytes: invalid digit found in string"
862 ));
863
864 let invalid_gc_enabled = HashMap::from([(
865 TableProperties::PROPERTY_GC_ENABLED.to_string(),
866 "notabool".to_string(),
867 )]);
868 let table_properties = TableProperties::new(&invalid_gc_enabled);
869 let error = table_properties.gc_enabled().unwrap_err();
870 assert!(error.to_string().contains("Invalid value for gc.enabled"));
871 }
872
873 #[test]
874 fn test_table_properties_compression_invalid_rejected() {
875 let invalid_codecs = ["lz4", "zstd", "snappy"];
876
877 for codec in invalid_codecs {
878 let props = HashMap::from([(
879 TableProperties::PROPERTY_METADATA_COMPRESSION_CODEC.to_string(),
880 codec.to_string(),
881 )]);
882 let err = TableProperties::new(&props)
883 .metadata_compression_codec()
884 .unwrap_err();
885 let err_msg = err.to_string();
886 assert!(
887 err_msg.contains(&format!("Invalid metadata compression codec: {codec}")),
888 "Expected error message to contain codec '{codec}', got: {err_msg}"
889 );
890 assert!(
891 err_msg.contains("Only 'none' and 'gzip' are supported"),
892 "Expected error message to contain supported codecs, got: {err_msg}"
893 );
894 }
895 }
896
897 #[test]
898 fn test_cdc_disabled_by_default() {
899 let props = HashMap::new();
900 let tp = TableProperties::new(&props);
901 assert!(!tp.cdc_enabled().unwrap());
902 }
903
904 #[test]
905 fn test_cdc_enabled_via_flag() {
906 let props = HashMap::from([(
907 TableProperties::PROPERTY_PARQUET_CDC_ENABLED.to_string(),
908 "true".to_string(),
909 )]);
910 let tp = TableProperties::new(&props);
911 assert!(tp.cdc_enabled().unwrap());
912 assert_eq!(tp.cdc_min_chunk_size().unwrap(), 256 * 1024);
913 assert_eq!(tp.cdc_max_chunk_size().unwrap(), 1024 * 1024);
914 assert_eq!(tp.cdc_norm_level().unwrap(), 0);
915 }
916
917 #[test]
918 fn test_cdc_size_props_alone_do_not_enable() {
919 let props = HashMap::from([(
920 TableProperties::PROPERTY_PARQUET_CDC_MIN_CHUNK_SIZE.to_string(),
921 "262144".to_string(),
922 )]);
923 let tp = TableProperties::new(&props);
924 assert!(!tp.cdc_enabled().unwrap());
925 }
926
927 #[test]
928 fn test_cdc_custom_values() {
929 let props = HashMap::from([
930 (
931 TableProperties::PROPERTY_PARQUET_CDC_ENABLED.to_string(),
932 "true".to_string(),
933 ),
934 (
935 TableProperties::PROPERTY_PARQUET_CDC_MIN_CHUNK_SIZE.to_string(),
936 "200000".to_string(),
937 ),
938 (
939 TableProperties::PROPERTY_PARQUET_CDC_MAX_CHUNK_SIZE.to_string(),
940 "900000".to_string(),
941 ),
942 (
943 TableProperties::PROPERTY_PARQUET_CDC_NORM_LEVEL.to_string(),
944 "1".to_string(),
945 ),
946 ]);
947 let tp = TableProperties::new(&props);
948 assert!(tp.cdc_enabled().unwrap());
949 assert_eq!(tp.cdc_min_chunk_size().unwrap(), 200000);
950 assert_eq!(tp.cdc_max_chunk_size().unwrap(), 900000);
951 assert_eq!(tp.cdc_norm_level().unwrap(), 1);
952 }
953
954 #[test]
955 fn test_cdc_partial_override() {
956 let props = HashMap::from([
957 (
958 TableProperties::PROPERTY_PARQUET_CDC_ENABLED.to_string(),
959 "true".to_string(),
960 ),
961 (
962 TableProperties::PROPERTY_PARQUET_CDC_NORM_LEVEL.to_string(),
963 "2".to_string(),
964 ),
965 ]);
966 let tp = TableProperties::new(&props);
967 assert!(tp.cdc_enabled().unwrap());
968 assert_eq!(tp.cdc_min_chunk_size().unwrap(), 256 * 1024);
969 assert_eq!(tp.cdc_max_chunk_size().unwrap(), 1024 * 1024);
970 assert_eq!(tp.cdc_norm_level().unwrap(), 2);
971 }
972
973 #[test]
974 fn test_cdc_negative_norm_level() {
975 let props = HashMap::from([
976 (
977 TableProperties::PROPERTY_PARQUET_CDC_ENABLED.to_string(),
978 "true".to_string(),
979 ),
980 (
981 TableProperties::PROPERTY_PARQUET_CDC_NORM_LEVEL.to_string(),
982 "-2".to_string(),
983 ),
984 ]);
985 let tp = TableProperties::new(&props);
986 assert_eq!(tp.cdc_norm_level().unwrap(), -2);
987 }
988
989 #[test]
990 fn test_cdc_invalid_min_chunk_size() {
991 let props = HashMap::from([
992 (
993 TableProperties::PROPERTY_PARQUET_CDC_ENABLED.to_string(),
994 "true".to_string(),
995 ),
996 (
997 TableProperties::PROPERTY_PARQUET_CDC_MIN_CHUNK_SIZE.to_string(),
998 "not_a_number".to_string(),
999 ),
1000 ]);
1001 let err = TableProperties::new(&props)
1002 .cdc_min_chunk_size()
1003 .unwrap_err();
1004 assert!(
1005 err.to_string().contains(
1006 "Invalid value for write.parquet.content-defined-chunking.min-chunk-size"
1007 )
1008 );
1009 }
1010
1011 #[test]
1012 fn test_cdc_invalid_norm_level() {
1013 let props = HashMap::from([
1014 (
1015 TableProperties::PROPERTY_PARQUET_CDC_ENABLED.to_string(),
1016 "true".to_string(),
1017 ),
1018 (
1019 TableProperties::PROPERTY_PARQUET_CDC_NORM_LEVEL.to_string(),
1020 "not_a_number".to_string(),
1021 ),
1022 ]);
1023 let err = TableProperties::new(&props).cdc_norm_level().unwrap_err();
1024 assert!(
1025 err.to_string()
1026 .contains("Invalid value for write.parquet.content-defined-chunking.norm-level")
1027 );
1028 }
1029
1030 #[test]
1031 fn test_cdc_no_properties() {
1032 let props = HashMap::from([("some.other.property".to_string(), "value".to_string())]);
1033 let tp = TableProperties::new(&props);
1034 assert!(!tp.cdc_enabled().unwrap());
1035 }
1036
1037 #[test]
1038 fn test_parquet_sizing_defaults() {
1039 let props = HashMap::new();
1040 let tp = TableProperties::new(&props);
1041 assert_eq!(
1043 tp.parquet_compression_codec().unwrap(),
1044 CompressionCodec::zstd_default()
1045 );
1046 assert_eq!(
1047 tp.parquet_row_group_size_bytes().unwrap(),
1048 TableProperties::PROPERTY_PARQUET_ROW_GROUP_SIZE_BYTES_DEFAULT
1049 );
1050 assert_eq!(
1051 tp.parquet_page_size_bytes().unwrap(),
1052 TableProperties::PROPERTY_PARQUET_PAGE_SIZE_BYTES_DEFAULT
1053 );
1054 assert_eq!(
1055 tp.parquet_page_row_limit().unwrap(),
1056 TableProperties::PROPERTY_PARQUET_PAGE_ROW_LIMIT_DEFAULT
1057 );
1058 assert_eq!(
1059 tp.parquet_dict_size_bytes().unwrap(),
1060 TableProperties::PROPERTY_PARQUET_DICT_SIZE_BYTES_DEFAULT
1061 );
1062 }
1063
1064 #[test]
1065 fn test_parquet_sizing_overrides() {
1066 let props = HashMap::from([
1067 (
1068 TableProperties::PROPERTY_PARQUET_COMPRESSION_CODEC.to_string(),
1069 "gzip".to_string(),
1070 ),
1071 (
1072 TableProperties::PROPERTY_PARQUET_COMPRESSION_LEVEL.to_string(),
1073 "4".to_string(),
1074 ),
1075 (
1076 TableProperties::PROPERTY_PARQUET_ROW_GROUP_SIZE_BYTES.to_string(),
1077 "1048576".to_string(),
1078 ),
1079 (
1080 TableProperties::PROPERTY_PARQUET_PAGE_SIZE_BYTES.to_string(),
1081 "65536".to_string(),
1082 ),
1083 (
1084 TableProperties::PROPERTY_PARQUET_PAGE_ROW_LIMIT.to_string(),
1085 "5000".to_string(),
1086 ),
1087 (
1088 TableProperties::PROPERTY_PARQUET_DICT_SIZE_BYTES.to_string(),
1089 "131072".to_string(),
1090 ),
1091 ]);
1092 let tp = TableProperties::new(&props);
1093 assert_eq!(
1095 tp.parquet_compression_codec().unwrap(),
1096 CompressionCodec::Gzip(4)
1097 );
1098 assert_eq!(tp.parquet_row_group_size_bytes().unwrap(), 1048576);
1099 assert_eq!(tp.parquet_page_size_bytes().unwrap(), 65536);
1100 assert_eq!(tp.parquet_page_row_limit().unwrap(), 5000);
1101 assert_eq!(tp.parquet_dict_size_bytes().unwrap(), 131072);
1102 }
1103
1104 #[test]
1105 fn test_parquet_invalid_sizing_rejected() {
1106 let props = HashMap::from([(
1107 TableProperties::PROPERTY_PARQUET_ROW_GROUP_SIZE_BYTES.to_string(),
1108 "not_a_number".to_string(),
1109 )]);
1110 let err = TableProperties::new(&props)
1111 .parquet_row_group_size_bytes()
1112 .unwrap_err();
1113 assert_eq!(err.kind(), ErrorKind::DataInvalid);
1114 assert!(
1115 err.to_string()
1116 .contains(TableProperties::PROPERTY_PARQUET_ROW_GROUP_SIZE_BYTES)
1117 );
1118 }
1119
1120 #[test]
1121 fn test_parquet_all_codecs_parse() {
1122 for (name, expected) in [
1125 ("uncompressed", CompressionCodec::None),
1126 ("snappy", CompressionCodec::Snappy),
1127 ("gzip", CompressionCodec::gzip_default()),
1128 ("lzo", CompressionCodec::Lzo),
1129 ("brotli", CompressionCodec::brotli_default()),
1130 ("lz4", CompressionCodec::Lz4),
1131 ("lz4_raw", CompressionCodec::Lz4Raw),
1132 ("zstd", CompressionCodec::zstd_default()),
1133 ] {
1134 let props = HashMap::from([(
1135 TableProperties::PROPERTY_PARQUET_COMPRESSION_CODEC.to_string(),
1136 name.to_string(),
1137 )]);
1138 let tp = TableProperties::new(&props);
1139 assert_eq!(
1140 tp.parquet_compression_codec().unwrap(),
1141 expected,
1142 "codec {name}"
1143 );
1144 }
1145 }
1146
1147 #[test]
1148 fn test_parquet_compression_level_ignored_for_levelless_codec() {
1149 let props = HashMap::from([
1152 (
1153 TableProperties::PROPERTY_PARQUET_COMPRESSION_CODEC.to_string(),
1154 "snappy".to_string(),
1155 ),
1156 (
1157 TableProperties::PROPERTY_PARQUET_COMPRESSION_LEVEL.to_string(),
1158 "5".to_string(),
1159 ),
1160 ]);
1161 let tp = TableProperties::new(&props);
1162 assert_eq!(
1163 tp.parquet_compression_codec().unwrap(),
1164 CompressionCodec::Snappy
1165 );
1166 }
1167
1168 #[test]
1169 fn test_parse_boolean_property_case_insensitive() {
1170 let false_variants = ["False", "FALSE"];
1171 let true_variants = ["True", "TRUE"];
1172
1173 for f in false_variants {
1174 let props = HashMap::from([(
1175 TableProperties::PROPERTY_WRITE_OBJECT_STORAGE_PARTITIONED_PATHS.to_string(),
1176 f.to_string(),
1177 )]);
1178 let tp = TableProperties::new(&props);
1179 assert!(!tp.write_object_storage_partitioned_paths().unwrap());
1180 }
1181
1182 for t in true_variants {
1183 let props = HashMap::from([(
1184 TableProperties::PROPERTY_WRITE_OBJECT_STORAGE_PARTITIONED_PATHS.to_string(),
1185 t.to_string(),
1186 )]);
1187 let tp = TableProperties::new(&props);
1188 assert!(tp.write_object_storage_partitioned_paths().unwrap());
1189 }
1190 }
1191
1192 #[test]
1193 fn test_table_properties_default_name_mapping() {
1194 let properties = HashMap::new();
1196 assert!(
1197 TableProperties::new(&properties)
1198 .default_name_mapping()
1199 .unwrap()
1200 .is_none()
1201 );
1202
1203 let properties = HashMap::from([(
1204 TableProperties::PROPERTY_DEFAULT_NAME_MAPPING.to_string(),
1205 r#"[{"field-id":1,"names":["id","record_id"]}]"#.to_string(),
1206 )]);
1207 let mapping = TableProperties::new(&properties)
1208 .default_name_mapping()
1209 .unwrap()
1210 .unwrap();
1211 assert_eq!(mapping.fields().len(), 1);
1212 assert_eq!(mapping.fields()[0].field_id(), Some(1));
1213 assert_eq!(mapping.fields()[0].names(), &[
1214 "id".to_string(),
1215 "record_id".to_string()
1216 ]);
1217 }
1218
1219 #[test]
1220 fn test_table_properties_malformed_name_mapping() {
1221 let properties = HashMap::from([(
1222 TableProperties::PROPERTY_DEFAULT_NAME_MAPPING.to_string(),
1223 "{ not valid json".to_string(),
1224 )]);
1225 let error = TableProperties::new(&properties)
1226 .default_name_mapping()
1227 .unwrap_err();
1228
1229 assert_eq!(error.kind(), ErrorKind::DataInvalid);
1230 assert!(
1232 format!("{error}").contains(TableProperties::PROPERTY_DEFAULT_NAME_MAPPING),
1233 "{error}"
1234 );
1235 }
1236}