Skip to main content

iceberg/writer/file_writer/
parquet_writer.rs

1// Licensed to the Apache Software Foundation (ASF) under one
2// or more contributor license agreements.  See the NOTICE file
3// distributed with this work for additional information
4// regarding copyright ownership.  The ASF licenses this file
5// to you under the Apache License, Version 2.0 (the
6// "License"); you may not use this file except in compliance
7// with the License.  You may obtain a copy of the License at
8//
9//   http://www.apache.org/licenses/LICENSE-2.0
10//
11// Unless required by applicable law or agreed to in writing,
12// software distributed under the License is distributed on an
13// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14// KIND, either express or implied.  See the License for the
15// specific language governing permissions and limitations
16// under the License.
17
18//! The module contains the file writer for parquet file format.
19
20use std::collections::HashMap;
21use std::sync::Arc;
22
23use arrow_schema::SchemaRef as ArrowSchemaRef;
24use bytes::Bytes;
25use futures::future::BoxFuture;
26use itertools::Itertools;
27use parquet::arrow::AsyncArrowWriter;
28use parquet::arrow::async_reader::AsyncFileReader;
29use parquet::arrow::async_writer::AsyncFileWriter as ArrowAsyncFileWriter;
30use parquet::basic::{BrotliLevel, Compression, GzipLevel, ZstdLevel};
31use parquet::encryption::encrypt::FileEncryptionProperties;
32use parquet::file::metadata::{ColumnChunkMetaData, ParquetMetaData};
33use parquet::file::properties::{CdcOptions, WriterProperties};
34use parquet::file::statistics::Statistics;
35use parquet::geospatial::bounding_box::BoundingBox;
36use parquet_geospatial::interval::{Interval, IntervalTrait, WraparoundInterval};
37
38use super::{FileWriter, FileWriterBuilder};
39use crate::arrow::{
40    ArrowFileReader, DEFAULT_MAP_FIELD_NAME, FieldMatchMode, NanValueCountVisitor,
41    get_parquet_stat_max_as_datum, get_parquet_stat_min_as_datum,
42};
43use crate::compression::CompressionCodec;
44use crate::encryption::{EncryptionManager, StandardKeyMetadata};
45use crate::error::invalid_data;
46use crate::io::{FileIO, FileWrite, OutputFile};
47use crate::spec::{
48    DataContentType, DataFileBuilder, DataFileFormat, Datum, ListType, Literal, MapType,
49    NestedFieldRef, PartitionSpec, PrimitiveLiteral, PrimitiveType, Schema, SchemaRef,
50    SchemaVisitor, Struct, StructType, TableMetadata, TableProperties, Type, VariantType,
51    visit_schema,
52};
53use crate::transform::create_transform_function;
54use crate::writer::{CurrentFileStatus, DataFile};
55use crate::{Error, ErrorKind, Result};
56
57/// ParquetWriterBuilder is used to builder a [`ParquetWriter`]
58#[derive(Clone, Debug)]
59pub struct ParquetWriterBuilder {
60    props: WriterProperties,
61    schema: SchemaRef,
62    match_mode: FieldMatchMode,
63    encryption_manager: Option<Arc<EncryptionManager>>,
64}
65
66impl ParquetWriterBuilder {
67    /// Create a new `ParquetWriterBuilder`
68    /// To construct the write result, the schema should contain the `PARQUET_FIELD_ID_META_KEY` metadata for each field.
69    ///
70    /// When writing into an existing Iceberg table, prefer
71    /// [`Self::from_table_properties`], which derives `WriterProperties` from
72    /// the table's `write.parquet.*` properties.
73    pub fn new(props: WriterProperties, schema: SchemaRef) -> Self {
74        Self::new_with_match_mode(props, schema, FieldMatchMode::Id)
75    }
76
77    /// Create a new `ParquetWriterBuilder` with custom match mode
78    pub fn new_with_match_mode(
79        props: WriterProperties,
80        schema: SchemaRef,
81        match_mode: FieldMatchMode,
82    ) -> Self {
83        Self {
84            props,
85            schema,
86            match_mode,
87            encryption_manager: None,
88        }
89    }
90
91    /// Build a `ParquetWriterBuilder` from Iceberg table properties and a
92    /// schema, translating `write.parquet.*` settings into `WriterProperties`
93    /// instead of using parquet-rs defaults.
94    pub fn from_table_properties(
95        table_props: &TableProperties<'_>,
96        schema: SchemaRef,
97    ) -> Result<Self> {
98        let cdc = if table_props.cdc_enabled()? {
99            Some(CdcOptions {
100                min_chunk_size: table_props.cdc_min_chunk_size()?,
101                max_chunk_size: table_props.cdc_max_chunk_size()?,
102                norm_level: table_props.cdc_norm_level()?,
103            })
104        } else {
105            None
106        };
107        let compression = parquet_compression(table_props.parquet_compression_codec()?)?;
108        let props = WriterProperties::builder()
109            .set_content_defined_chunking(cdc)
110            .set_compression(compression)
111            .set_max_row_group_bytes(Some(table_props.parquet_row_group_size_bytes()?))
112            .set_data_page_size_limit(table_props.parquet_page_size_bytes()?)
113            .set_data_page_row_count_limit(table_props.parquet_page_row_limit()?)
114            .set_dictionary_page_size_limit(table_props.parquet_dict_size_bytes()?)
115            .build();
116        Ok(Self::new_with_match_mode(props, schema, FieldMatchMode::Id))
117    }
118
119    /// Set the field match mode used to map Arrow fields to Iceberg fields.
120    ///
121    /// Defaults to [`FieldMatchMode::Id`]. Use [`FieldMatchMode::Name`] when the
122    /// incoming Arrow schema does not carry Iceberg field-id metadata.
123    pub fn with_match_mode(mut self, match_mode: FieldMatchMode) -> Self {
124        self.match_mode = match_mode;
125        self
126    }
127
128    /// Set the [`EncryptionManager`] used to encrypt written files.
129    pub fn with_encryption_manager(mut self, encryption_manager: Arc<EncryptionManager>) -> Self {
130        self.encryption_manager = Some(encryption_manager);
131        self
132    }
133}
134
135fn parquet_compression(codec: CompressionCodec) -> Result<Compression> {
136    let compression = match codec {
137        CompressionCodec::None => Compression::UNCOMPRESSED,
138        CompressionCodec::Snappy => Compression::SNAPPY,
139        CompressionCodec::Lzo => Compression::LZO,
140        CompressionCodec::Lz4 => Compression::LZ4,
141        CompressionCodec::Lz4Raw => Compression::LZ4_RAW,
142        CompressionCodec::Zstd(level) => {
143            let level =
144                ZstdLevel::try_new(level as i32).map_err(|e| invalid_level_error("zstd", e))?;
145            Compression::ZSTD(level)
146        }
147        CompressionCodec::Gzip(level) => {
148            let level =
149                GzipLevel::try_new(level as u32).map_err(|e| invalid_level_error("gzip", e))?;
150            Compression::GZIP(level)
151        }
152        CompressionCodec::Brotli(level) => {
153            let level =
154                BrotliLevel::try_new(level as u32).map_err(|e| invalid_level_error("brotli", e))?;
155            Compression::BROTLI(level)
156        }
157    };
158    Ok(compression)
159}
160
161fn invalid_level_error(codec: &str, source: impl Into<anyhow::Error>) -> Error {
162    invalid_data!("Invalid {codec} compression level").with_source(source)
163}
164
165impl FileWriterBuilder for ParquetWriterBuilder {
166    type R = ParquetWriter;
167
168    async fn build(&self, output_file: OutputFile) -> Result<Self::R> {
169        let key_metadata = self
170            .encryption_manager
171            .as_ref()
172            .map(|em| em.generate_key_metadata());
173        let writer_properties =
174            resolve_writer_properties(self.props.clone(), key_metadata.as_ref())?;
175        Ok(ParquetWriter {
176            schema: self.schema.clone(),
177            inner_writer: None,
178            writer_properties,
179            current_row_num: 0,
180            output_file,
181            nan_value_count_visitor: NanValueCountVisitor::new_with_match_mode(self.match_mode),
182            key_metadata,
183        })
184    }
185}
186
187/// A mapping from Parquet column path names to internal field id
188struct IndexByParquetPathName {
189    name_to_id: HashMap<String, i32>,
190
191    field_names: Vec<String>,
192
193    field_id: i32,
194}
195
196impl IndexByParquetPathName {
197    /// Creates a new, empty `IndexByParquetPathName`
198    pub fn new() -> Self {
199        Self {
200            name_to_id: HashMap::new(),
201            field_names: Vec::new(),
202            field_id: 0,
203        }
204    }
205
206    /// Retrieves the internal field ID
207    pub fn get(&self, name: &str) -> Option<&i32> {
208        self.name_to_id.get(name)
209    }
210}
211
212impl Default for IndexByParquetPathName {
213    fn default() -> Self {
214        Self::new()
215    }
216}
217
218impl SchemaVisitor for IndexByParquetPathName {
219    type T = ();
220
221    fn before_struct_field(&mut self, field: &NestedFieldRef) -> Result<()> {
222        self.field_names.push(field.name.to_string());
223        self.field_id = field.id;
224        Ok(())
225    }
226
227    fn after_struct_field(&mut self, _field: &NestedFieldRef) -> Result<()> {
228        self.field_names.pop();
229        Ok(())
230    }
231
232    fn before_list_element(&mut self, field: &NestedFieldRef) -> Result<()> {
233        self.field_names.push(format!("list.{}", field.name));
234        self.field_id = field.id;
235        Ok(())
236    }
237
238    fn after_list_element(&mut self, _field: &NestedFieldRef) -> Result<()> {
239        self.field_names.pop();
240        Ok(())
241    }
242
243    fn before_map_key(&mut self, field: &NestedFieldRef) -> Result<()> {
244        self.field_names
245            .push(format!("{DEFAULT_MAP_FIELD_NAME}.key"));
246        self.field_id = field.id;
247        Ok(())
248    }
249
250    fn after_map_key(&mut self, _field: &NestedFieldRef) -> Result<()> {
251        self.field_names.pop();
252        Ok(())
253    }
254
255    fn before_map_value(&mut self, field: &NestedFieldRef) -> Result<()> {
256        self.field_names
257            .push(format!("{DEFAULT_MAP_FIELD_NAME}.value"));
258        self.field_id = field.id;
259        Ok(())
260    }
261
262    fn after_map_value(&mut self, _field: &NestedFieldRef) -> Result<()> {
263        self.field_names.pop();
264        Ok(())
265    }
266
267    fn schema(&mut self, _schema: &Schema, _value: Self::T) -> Result<Self::T> {
268        Ok(())
269    }
270
271    fn field(&mut self, _field: &NestedFieldRef, _value: Self::T) -> Result<Self::T> {
272        Ok(())
273    }
274
275    fn r#struct(&mut self, _struct: &StructType, _results: Vec<Self::T>) -> Result<Self::T> {
276        Ok(())
277    }
278
279    fn list(&mut self, _list: &ListType, _value: Self::T) -> Result<Self::T> {
280        Ok(())
281    }
282
283    fn map(&mut self, _map: &MapType, _key_value: Self::T, _value: Self::T) -> Result<Self::T> {
284        Ok(())
285    }
286
287    fn primitive(&mut self, _p: &PrimitiveType) -> Result<Self::T> {
288        let full_name = self.field_names.iter().map(String::as_str).join(".");
289        let field_id = self.field_id;
290        if let Some(existing_field_id) = self.name_to_id.get(full_name.as_str()) {
291            return Err(invalid_data!(
292                "Invalid schema: multiple fields for name {full_name}: {field_id} and {existing_field_id}"
293            ));
294        } else {
295            self.name_to_id.insert(full_name, field_id);
296        }
297
298        Ok(())
299    }
300
301    fn variant(&mut self, _v: &VariantType) -> Result<Self::T> {
302        Err(Error::new(
303            ErrorKind::FeatureUnsupported,
304            "Writing variant columns to Parquet is not supported yet",
305        ))
306    }
307}
308
309/// `ParquetWriter`` is used to write arrow data into parquet file on storage.
310pub struct ParquetWriter {
311    schema: SchemaRef,
312    output_file: OutputFile,
313    inner_writer: Option<AsyncArrowWriter<AsyncFileWriter>>,
314    writer_properties: WriterProperties,
315    current_row_num: usize,
316    nan_value_count_visitor: NanValueCountVisitor,
317    key_metadata: Option<StandardKeyMetadata>,
318}
319
320/// Used to aggregate min and max value of each column.
321struct MinMaxColAggregator {
322    lower_bounds: HashMap<i32, Datum>,
323    upper_bounds: HashMap<i32, Datum>,
324    // None means at least one non-null row group has no bounding box: file bounds are unknown.
325    geo_bounds: HashMap<i32, (PrimitiveType, Option<GeospatialBounds>)>,
326    schema: SchemaRef,
327}
328
329struct GeospatialBounds {
330    x: WraparoundInterval,
331    y: Interval,
332    z: Interval,
333    m: Interval,
334}
335
336impl From<&BoundingBox> for GeospatialBounds {
337    fn from(bbox: &BoundingBox) -> Self {
338        Self {
339            x: WraparoundInterval::new(bbox.get_xmin(), bbox.get_xmax()),
340            y: Interval::new(bbox.get_ymin(), bbox.get_ymax()),
341            z: bbox
342                .get_zmin()
343                .zip(bbox.get_zmax())
344                .map(|(lo, hi)| Interval::new(lo, hi))
345                .unwrap_or_else(Interval::empty),
346            m: bbox
347                .get_mmin()
348                .zip(bbox.get_mmax())
349                .map(|(lo, hi)| Interval::new(lo, hi))
350                .unwrap_or_else(Interval::empty),
351        }
352    }
353}
354
355impl GeospatialBounds {
356    fn merge(&mut self, bbox: &BoundingBox) {
357        let other = Self::from(bbox);
358        self.x = self.x.merge_interval(&other.x);
359        self.y.update_interval(&other.y);
360        self.z.update_interval(&other.z);
361        self.m.update_interval(&other.m);
362    }
363
364    fn into_datums(self, ty: PrimitiveType) -> (Datum, Datum) {
365        let (xmin, xmax) = if matches!(ty, PrimitiveType::Geography(_)) {
366            // A full interval union uses infinities; Geography requires canonical longitudes.
367            (self.x.lo().max(-180.0), self.x.hi().min(180.0))
368        } else if self.x.is_wraparound() {
369            // Parquet permits wrapping geometry boxes; Iceberg only permits wrapping geography.
370            (f64::NEG_INFINITY, f64::INFINITY)
371        } else {
372            (self.x.lo(), self.x.hi())
373        };
374        let dimensions = if !self.m.is_empty() {
375            4
376        } else if !self.z.is_empty() {
377            3
378        } else {
379            2
380        };
381        let (zmin, zmax) = if self.z.is_empty() {
382            (f64::NAN, f64::NAN)
383        } else {
384            (self.z.lo(), self.z.hi())
385        };
386        // Iceberg bound serialization is XY[Z[M]] doubles, not WKB (spec Appendix D).
387        let encode = |point: [f64; 4]| {
388            PrimitiveLiteral::Binary(
389                point[..dimensions]
390                    .iter()
391                    .flat_map(|v| v.to_le_bytes())
392                    .collect(),
393            )
394        };
395        (
396            Datum::new(ty.clone(), encode([xmin, self.y.lo(), zmin, self.m.lo()])),
397            Datum::new(ty, encode([xmax, self.y.hi(), zmax, self.m.hi()])),
398        )
399    }
400}
401
402impl MinMaxColAggregator {
403    /// Creates new and empty `MinMaxColAggregator`
404    fn new(schema: SchemaRef) -> Self {
405        Self {
406            lower_bounds: HashMap::new(),
407            upper_bounds: HashMap::new(),
408            geo_bounds: HashMap::new(),
409            schema,
410        }
411    }
412
413    fn update_state_min(&mut self, field_id: i32, datum: Datum) {
414        self.lower_bounds
415            .entry(field_id)
416            .and_modify(|e| {
417                if *e > datum {
418                    *e = datum.clone()
419                }
420            })
421            .or_insert(datum);
422    }
423
424    fn update_state_max(&mut self, field_id: i32, datum: Datum) {
425        self.upper_bounds
426            .entry(field_id)
427            .and_modify(|e| {
428                if *e < datum {
429                    *e = datum.clone()
430                }
431            })
432            .or_insert(datum);
433    }
434
435    /// Update bounds from a column chunk.
436    fn update(&mut self, field_id: i32, column: &ColumnChunkMetaData) -> Result<()> {
437        let Some(ty) = self
438            .schema
439            .field_by_id(field_id)
440            .map(|f| f.field_type.as_ref())
441        else {
442            // Following java implementation: https://github.com/apache/iceberg/blob/29a2c456353a6120b8c882ed2ab544975b168d7b/parquet/src/main/java/org/apache/iceberg/parquet/ParquetUtil.java#L163
443            // Ignore the field if it is not in schema.
444            return Ok(());
445        };
446        let Type::Primitive(ty) = ty else {
447            return Err(Error::new(
448                ErrorKind::Unexpected,
449                format!("Composed type {ty} is not supported for min max aggregation."),
450            ));
451        };
452
453        match ty {
454            PrimitiveType::Geometry(_) | PrimitiveType::Geography(_) => {
455                // Null-only row groups do not contribute coordinates.
456                if column.num_values() == 0
457                    || column
458                        .statistics()
459                        .and_then(Statistics::null_count_opt)
460                        .is_some_and(|nulls| nulls == column.num_values() as u64)
461                {
462                    return Ok(());
463                }
464                // Parquet bounds already account for the logical type's edge interpolation.
465                let bbox = column
466                    .geo_statistics()
467                    .and_then(|stats| stats.bounding_box());
468                self.geo_bounds
469                    .entry(field_id)
470                    .and_modify(|(_, state)| {
471                        if let (Some(bounds), Some(bbox)) = (state.as_mut(), bbox) {
472                            bounds.merge(bbox);
473                        } else {
474                            *state = None;
475                        }
476                    })
477                    .or_insert_with(|| (ty.clone(), bbox.map(GeospatialBounds::from)));
478                return Ok(());
479            }
480            _ => {}
481        }
482        let Some(value) = column.statistics() else {
483            return Ok(());
484        };
485        let ty = ty.clone();
486
487        if value.min_is_exact() {
488            let Some(min_datum) = get_parquet_stat_min_as_datum(&ty, value)? else {
489                return Err(Error::new(
490                    ErrorKind::Unexpected,
491                    format!("Statistics {value} is not match with field type {ty}."),
492                ));
493            };
494
495            // Iceberg bounds must not be NaN.
496            if !min_datum.is_nan() {
497                self.update_state_min(field_id, min_datum);
498            }
499        }
500
501        if value.max_is_exact() {
502            let Some(max_datum) = get_parquet_stat_max_as_datum(&ty, value)? else {
503                return Err(Error::new(
504                    ErrorKind::Unexpected,
505                    format!("Statistics {value} is not match with field type {ty}."),
506                ));
507            };
508
509            if !max_datum.is_nan() {
510                self.update_state_max(field_id, max_datum);
511            }
512        }
513
514        Ok(())
515    }
516
517    /// Returns lower and upper bounds
518    fn produce(mut self) -> (HashMap<i32, Datum>, HashMap<i32, Datum>) {
519        for (field_id, (ty, bounds)) in self.geo_bounds {
520            if let Some(bounds) = bounds {
521                let (lower, upper) = bounds.into_datums(ty);
522                self.lower_bounds.insert(field_id, lower);
523                self.upper_bounds.insert(field_id, upper);
524            }
525        }
526        (self.lower_bounds, self.upper_bounds)
527    }
528}
529
530impl ParquetWriter {
531    /// Converts parquet files to data files
532    #[allow(dead_code)]
533    pub(crate) async fn parquet_files_to_data_files(
534        file_io: &FileIO,
535        file_paths: Vec<String>,
536        table_metadata: &TableMetadata,
537    ) -> Result<Vec<DataFile>> {
538        // TODO: support adding to partitioned table
539        let mut data_files: Vec<DataFile> = Vec::new();
540
541        for file_path in file_paths {
542            let input_file = file_io.new_input(&file_path)?;
543            let file_metadata = input_file.metadata().await?;
544            let file_size_in_bytes = file_metadata.size as usize;
545            let reader = input_file.reader().await?;
546
547            let mut parquet_reader = ArrowFileReader::new(file_metadata, reader);
548            let parquet_metadata = parquet_reader
549                .get_metadata(None)
550                .await
551                .map_err(|err| invalid_data!("Error reading Parquet metadata: {err}"))?;
552            let mut builder = ParquetWriter::parquet_to_data_file_builder(
553                table_metadata.current_schema().clone(),
554                parquet_metadata,
555                file_size_in_bytes,
556                file_path,
557                // TODO: Implement nan_value_counts here
558                HashMap::new(),
559                None,
560            )?;
561            builder.partition_spec_id(table_metadata.default_partition_spec_id());
562            let data_file = builder.build().unwrap();
563            data_files.push(data_file);
564        }
565
566        Ok(data_files)
567    }
568
569    /// `ParquetMetadata` to data file builder
570    pub(crate) fn parquet_to_data_file_builder(
571        schema: SchemaRef,
572        metadata: Arc<ParquetMetaData>,
573        written_size: usize,
574        file_path: String,
575        nan_value_counts: HashMap<i32, u64>,
576        key_metadata: Option<StandardKeyMetadata>,
577    ) -> Result<DataFileBuilder> {
578        let index_by_parquet_path = {
579            let mut visitor = IndexByParquetPathName::new();
580            visit_schema(&schema, &mut visitor)?;
581            visitor
582        };
583
584        let (column_sizes, value_counts, null_value_counts, (lower_bounds, upper_bounds)) = {
585            let mut per_col_size: HashMap<i32, u64> = HashMap::new();
586            let mut per_col_val_num: HashMap<i32, u64> = HashMap::new();
587            let mut per_col_null_val_num: HashMap<i32, u64> = HashMap::new();
588            let mut min_max_agg = MinMaxColAggregator::new(schema);
589
590            for row_group in metadata.row_groups() {
591                for column_chunk_metadata in row_group.columns() {
592                    let parquet_path = column_chunk_metadata.column_descr().path().string();
593
594                    let Some(&field_id) = index_by_parquet_path.get(&parquet_path) else {
595                        continue;
596                    };
597
598                    *per_col_size.entry(field_id).or_insert(0) +=
599                        column_chunk_metadata.compressed_size() as u64;
600                    *per_col_val_num.entry(field_id).or_insert(0) +=
601                        column_chunk_metadata.num_values() as u64;
602
603                    min_max_agg.update(field_id, column_chunk_metadata)?;
604
605                    if let Some(null_count) = column_chunk_metadata
606                        .statistics()
607                        .and_then(Statistics::null_count_opt)
608                    {
609                        *per_col_null_val_num.entry(field_id).or_insert(0) += null_count;
610                    }
611                }
612            }
613            (
614                per_col_size,
615                per_col_val_num,
616                per_col_null_val_num,
617                min_max_agg.produce(),
618            )
619        };
620
621        let key_metadata = match key_metadata {
622            Some(m) => Some(m.encode()?.into_vec()),
623            None => None,
624        };
625
626        let mut builder = DataFileBuilder::default();
627        builder
628            .content(DataContentType::Data)
629            .file_path(file_path)
630            .file_format(DataFileFormat::Parquet)
631            .partition(Struct::empty())
632            .record_count(metadata.file_metadata().num_rows() as u64)
633            .file_size_in_bytes(written_size as u64)
634            .column_sizes(column_sizes)
635            .value_counts(value_counts)
636            .null_value_counts(null_value_counts)
637            .nan_value_counts(nan_value_counts)
638            // # NOTE:
639            // - We can ignore implementing distinct_counts due to this: https://lists.apache.org/thread/j52tsojv0x4bopxyzsp7m7bqt23n5fnd
640            .lower_bounds(lower_bounds)
641            .upper_bounds(upper_bounds)
642            .key_metadata(key_metadata)
643            .split_offsets(Some(
644                metadata
645                    .row_groups()
646                    .iter()
647                    .filter_map(|group| group.file_offset())
648                    .collect(),
649            ));
650
651        Ok(builder)
652    }
653
654    #[allow(dead_code)]
655    fn partition_value_from_bounds(
656        table_spec: Arc<PartitionSpec>,
657        lower_bounds: &HashMap<i32, Datum>,
658        upper_bounds: &HashMap<i32, Datum>,
659    ) -> Result<Struct> {
660        let mut partition_literals: Vec<Option<Literal>> = Vec::new();
661
662        for field in table_spec.fields() {
663            if let (Some(lower), Some(upper)) = (
664                lower_bounds.get(&field.source_id),
665                upper_bounds.get(&field.source_id),
666            ) {
667                if !field.transform.preserves_order() {
668                    return Err(invalid_data!(
669                        "cannot infer partition value for non linear partition field (needs to preserve order): {} with transform {}",
670                        field.name,
671                        field.transform
672                    ));
673                }
674
675                if lower != upper {
676                    return Err(invalid_data!(
677                        "multiple partition values for field {}: lower: {:?}, upper: {:?}",
678                        field.name,
679                        lower,
680                        upper
681                    ));
682                }
683
684                let transform_fn = create_transform_function(&field.transform)?;
685                let transform_literal =
686                    Literal::from(transform_fn.transform_literal_result(lower)?);
687
688                partition_literals.push(Some(transform_literal));
689            } else {
690                partition_literals.push(None);
691            }
692        }
693
694        let partition_struct = Struct::from_iter(partition_literals);
695
696        Ok(partition_struct)
697    }
698}
699
700fn resolve_writer_properties(
701    writer_properties: WriterProperties,
702    key_metadata: Option<&StandardKeyMetadata>,
703) -> Result<WriterProperties> {
704    let Some(key_metadata) = key_metadata else {
705        return Ok(writer_properties);
706    };
707
708    if writer_properties.file_encryption_properties().is_some() {
709        return Err(Error::new(
710            ErrorKind::Unexpected,
711            "Parquet writer properties already have file encryption properties set",
712        ));
713    }
714
715    let mut builder =
716        FileEncryptionProperties::builder(key_metadata.encryption_key().as_bytes().to_vec());
717    if let Some(aad) = key_metadata.aad_prefix() {
718        builder = builder.with_aad_prefix(aad.to_vec());
719    }
720    let file_encryption_properties = builder.build().map_err(|e| {
721        Error::new(
722            ErrorKind::Unexpected,
723            "Failed to build parquet file encryption properties",
724        )
725        .with_source(e)
726    })?;
727
728    Ok(writer_properties
729        .into_builder()
730        .with_file_encryption_properties(file_encryption_properties)
731        .build())
732}
733
734impl FileWriter for ParquetWriter {
735    async fn write(&mut self, batch: &arrow_array::RecordBatch) -> Result<()> {
736        // Skip empty batch
737        if batch.num_rows() == 0 {
738            return Ok(());
739        }
740
741        self.current_row_num += batch.num_rows();
742
743        let batch_c = batch.clone();
744        self.nan_value_count_visitor
745            .compute(self.schema.clone(), batch_c)?;
746
747        // Lazy initialize the writer
748        let writer = if let Some(writer) = &mut self.inner_writer {
749            writer
750        } else {
751            let arrow_schema: ArrowSchemaRef = Arc::new(self.schema.as_ref().try_into()?);
752            let inner_writer = self.output_file.writer().await?;
753            let async_writer = AsyncFileWriter::new(inner_writer);
754            let writer = AsyncArrowWriter::try_new(
755                async_writer,
756                arrow_schema.clone(),
757                Some(self.writer_properties.clone()),
758            )
759            .map_err(|err| {
760                Error::new(ErrorKind::Unexpected, "Failed to build parquet writer.")
761                    .with_source(err)
762            })?;
763            self.inner_writer = Some(writer);
764            self.inner_writer.as_mut().unwrap()
765        };
766
767        writer.write(batch).await.map_err(|err| {
768            Error::new(
769                ErrorKind::Unexpected,
770                "Failed to write using parquet writer.",
771            )
772            .with_source(err)
773        })?;
774
775        Ok(())
776    }
777
778    async fn close(mut self) -> Result<Vec<DataFileBuilder>> {
779        let mut writer = match self.inner_writer.take() {
780            Some(writer) => writer,
781            None => return Ok(vec![]),
782        };
783
784        let metadata = writer.finish().await.map_err(|err| {
785            Error::new(ErrorKind::Unexpected, "Failed to finish parquet writer.").with_source(err)
786        })?;
787
788        let written_size = writer.bytes_written();
789
790        if self.current_row_num == 0 {
791            self.output_file.delete().await.map_err(|err| {
792                Error::new(
793                    ErrorKind::Unexpected,
794                    "Failed to delete empty parquet file.",
795                )
796                .with_source(err)
797            })?;
798            Ok(vec![])
799        } else {
800            let parquet_metadata = Arc::new(metadata);
801
802            Ok(vec![Self::parquet_to_data_file_builder(
803                self.schema,
804                parquet_metadata,
805                written_size,
806                self.output_file.location().to_string(),
807                self.nan_value_count_visitor.nan_value_counts,
808                self.key_metadata,
809            )?])
810        }
811    }
812}
813
814impl CurrentFileStatus for ParquetWriter {
815    fn current_file_path(&self) -> String {
816        self.output_file.location().to_string()
817    }
818
819    fn current_row_num(&self) -> usize {
820        self.current_row_num
821    }
822
823    fn current_written_size(&self) -> usize {
824        if let Some(inner) = self.inner_writer.as_ref() {
825            // inner/AsyncArrowWriter contains sync and async writers
826            // written size = bytes flushed to inner's async writer + bytes buffered in the inner's sync writer
827            inner.bytes_written() + inner.in_progress_size()
828        } else {
829            // inner writer is not initialized yet
830            0
831        }
832    }
833}
834
835/// AsyncFileWriter is a wrapper of FileWrite to make it compatible with tokio::io::AsyncWrite.
836///
837/// # NOTES
838///
839/// We keep this wrapper been used inside only.
840struct AsyncFileWriter(Box<dyn FileWrite>);
841
842impl AsyncFileWriter {
843    /// Create a new `AsyncFileWriter` with the given writer.
844    pub fn new(writer: Box<dyn FileWrite>) -> Self {
845        Self(writer)
846    }
847}
848
849impl ArrowAsyncFileWriter for AsyncFileWriter {
850    fn write(&mut self, bs: Bytes) -> BoxFuture<'_, parquet::errors::Result<()>> {
851        Box::pin(async {
852            self.0
853                .write(bs)
854                .await
855                .map_err(|err| parquet::errors::ParquetError::External(Box::new(err)))
856        })
857    }
858
859    fn complete(&mut self) -> BoxFuture<'_, parquet::errors::Result<()>> {
860        Box::pin(async {
861            // TODO(encryption): retain the stored file size in data-file key metadata.
862            self.0
863                .close()
864                .await
865                .map(|_| ())
866                .map_err(|err| parquet::errors::ParquetError::External(Box::new(err)))
867        })
868    }
869}
870
871#[cfg(test)]
872mod tests {
873    use std::collections::HashMap;
874    use std::sync::Arc;
875
876    use anyhow::Result;
877    use arrow_array::builder::{Float32Builder, Int32Builder, MapBuilder};
878    use arrow_array::types::{Float32Type, Int64Type};
879    use arrow_array::{
880        Array, ArrayRef, BooleanArray, Decimal128Array, Float32Array, Float64Array, Int32Array,
881        Int64Array, ListArray, MapArray, RecordBatch, StructArray,
882    };
883    use arrow_schema::{DataType, Field, Fields, SchemaRef as ArrowSchemaRef};
884    use arrow_select::concat::concat_batches;
885    use futures::TryStreamExt;
886    use parquet::arrow::PARQUET_FIELD_ID_META_KEY;
887    use parquet::arrow::arrow_reader::ParquetRecordBatchReaderBuilder;
888    use parquet::basic::{
889        BrotliLevel, Compression, EdgeInterpolationAlgorithm, GzipLevel, LogicalType, ZstdLevel,
890    };
891    use parquet::file::statistics::ValueStatistics;
892    use parquet::schema::types::ColumnPath;
893    use parquet_geospatial::testing::wkb_point_xy;
894    use tempfile::TempDir;
895    use uuid::Uuid;
896
897    use super::*;
898    use crate::Runtime;
899    use crate::arrow::{ArrowReaderBuilder, schema_to_arrow_schema};
900    use crate::io::FileIO;
901    use crate::scan::{FileScanTask, FileScanTaskStream};
902    use crate::spec::decimal_utils::{decimal_mantissa, decimal_new, decimal_scale};
903    use crate::spec::geospatial::EdgeInterpolationAlgorithm as IcebergEdgeInterpolationAlgorithm;
904    use crate::spec::{PrimitiveLiteral, Struct, *};
905    use crate::test_utils::make_encryption_manager;
906    use crate::writer::file_writer::location_generator::{
907        DefaultFileNameGenerator, DefaultLocationGenerator, FileNameGenerator, LocationGenerator,
908    };
909    use crate::writer::tests::check_parquet_data_file;
910
911    fn schema_for_all_type() -> Schema {
912        Schema::builder()
913            .with_schema_id(1)
914            .with_fields(vec![
915                NestedField::optional(0, "boolean", Type::Primitive(PrimitiveType::Boolean)).into(),
916                NestedField::optional(1, "int", Type::Primitive(PrimitiveType::Int)).into(),
917                NestedField::optional(2, "long", Type::Primitive(PrimitiveType::Long)).into(),
918                NestedField::optional(3, "float", Type::Primitive(PrimitiveType::Float)).into(),
919                NestedField::optional(4, "double", Type::Primitive(PrimitiveType::Double)).into(),
920                NestedField::optional(5, "string", Type::Primitive(PrimitiveType::String)).into(),
921                NestedField::optional(6, "binary", Type::Primitive(PrimitiveType::Binary)).into(),
922                NestedField::optional(7, "date", Type::Primitive(PrimitiveType::Date)).into(),
923                NestedField::optional(8, "time", Type::Primitive(PrimitiveType::Time)).into(),
924                NestedField::optional(9, "timestamp", Type::Primitive(PrimitiveType::Timestamp))
925                    .into(),
926                NestedField::optional(
927                    10,
928                    "timestamptz",
929                    Type::Primitive(PrimitiveType::Timestamptz),
930                )
931                .into(),
932                NestedField::optional(
933                    11,
934                    "timestamp_ns",
935                    Type::Primitive(PrimitiveType::TimestampNs),
936                )
937                .into(),
938                NestedField::optional(
939                    12,
940                    "timestamptz_ns",
941                    Type::Primitive(PrimitiveType::TimestamptzNs),
942                )
943                .into(),
944                NestedField::optional(
945                    13,
946                    "decimal",
947                    Type::Primitive(PrimitiveType::Decimal {
948                        precision: 10,
949                        scale: 5,
950                    }),
951                )
952                .into(),
953                NestedField::optional(14, "uuid", Type::Primitive(PrimitiveType::Uuid)).into(),
954                NestedField::optional(15, "fixed", Type::Primitive(PrimitiveType::Fixed(10)))
955                    .into(),
956                // Parquet Statistics will use different representation for Decimal with precision 38 and scale 5,
957                // so we need to add a new field for it.
958                NestedField::optional(
959                    16,
960                    "decimal_38",
961                    Type::Primitive(PrimitiveType::Decimal {
962                        precision: 38,
963                        scale: 5,
964                    }),
965                )
966                .into(),
967            ])
968            .build()
969            .unwrap()
970    }
971
972    fn nested_schema_for_test() -> Schema {
973        // Int, Struct(Int,Int), String, List(Int), Struct(Struct(Int)), Map(String, List(Int))
974        Schema::builder()
975            .with_schema_id(1)
976            .with_fields(vec![
977                NestedField::required(0, "col0", Type::Primitive(PrimitiveType::Long)).into(),
978                NestedField::required(
979                    1,
980                    "col1",
981                    Type::Struct(StructType::new(vec![
982                        NestedField::required(5, "col_1_5", Type::Primitive(PrimitiveType::Long))
983                            .into(),
984                        NestedField::required(6, "col_1_6", Type::Primitive(PrimitiveType::Long))
985                            .into(),
986                    ])),
987                )
988                .into(),
989                NestedField::required(2, "col2", Type::Primitive(PrimitiveType::String)).into(),
990                NestedField::required(
991                    3,
992                    "col3",
993                    Type::List(ListType::new(
994                        NestedField::required(7, "element", Type::Primitive(PrimitiveType::Long))
995                            .into(),
996                    )),
997                )
998                .into(),
999                NestedField::required(
1000                    4,
1001                    "col4",
1002                    Type::Struct(StructType::new(vec![
1003                        NestedField::required(
1004                            8,
1005                            "col_4_8",
1006                            Type::Struct(StructType::new(vec![
1007                                NestedField::required(
1008                                    9,
1009                                    "col_4_8_9",
1010                                    Type::Primitive(PrimitiveType::Long),
1011                                )
1012                                .into(),
1013                            ])),
1014                        )
1015                        .into(),
1016                    ])),
1017                )
1018                .into(),
1019                NestedField::required(
1020                    10,
1021                    "col5",
1022                    Type::Map(MapType::new(
1023                        NestedField::required(11, "key", Type::Primitive(PrimitiveType::String))
1024                            .into(),
1025                        NestedField::required(
1026                            12,
1027                            "value",
1028                            Type::List(ListType::new(
1029                                NestedField::required(
1030                                    13,
1031                                    "item",
1032                                    Type::Primitive(PrimitiveType::Long),
1033                                )
1034                                .into(),
1035                            )),
1036                        )
1037                        .into(),
1038                    )),
1039                )
1040                .into(),
1041            ])
1042            .build()
1043            .unwrap()
1044    }
1045
1046    #[tokio::test]
1047    async fn test_index_by_parquet_path() {
1048        let expect = HashMap::from([
1049            ("col0".to_string(), 0),
1050            ("col1.col_1_5".to_string(), 5),
1051            ("col1.col_1_6".to_string(), 6),
1052            ("col2".to_string(), 2),
1053            ("col3.list.element".to_string(), 7),
1054            ("col4.col_4_8.col_4_8_9".to_string(), 9),
1055            ("col5.key_value.key".to_string(), 11),
1056            ("col5.key_value.value.list.item".to_string(), 13),
1057        ]);
1058        let mut visitor = IndexByParquetPathName::new();
1059        visit_schema(&nested_schema_for_test(), &mut visitor).unwrap();
1060        assert_eq!(visitor.name_to_id, expect);
1061    }
1062
1063    #[test]
1064    fn test_index_by_parquet_path_variant_is_unsupported() {
1065        // Writing variant columns to Parquet is not supported yet; indexing a schema that
1066        // contains one must error rather than silently miss-map columns.
1067        let schema = Schema::builder()
1068            .with_fields(vec![
1069                NestedField::optional(1, "v", Type::Variant(VariantType)).into(),
1070            ])
1071            .build()
1072            .unwrap();
1073        let mut visitor = IndexByParquetPathName::new();
1074        let err = visit_schema(&schema, &mut visitor).unwrap_err();
1075        assert_eq!(err.kind(), ErrorKind::FeatureUnsupported, "{err}");
1076    }
1077
1078    #[tokio::test]
1079    async fn test_parquet_writer() -> Result<()> {
1080        let temp_dir = TempDir::new().unwrap();
1081        let file_io = FileIO::new_with_fs();
1082        let location_gen = DefaultLocationGenerator::with_data_location(
1083            temp_dir.path().to_str().unwrap().to_string(),
1084        );
1085        let file_name_gen =
1086            DefaultFileNameGenerator::new("test".to_string(), None, DataFileFormat::Parquet);
1087
1088        // prepare data
1089        let schema = {
1090            let fields =
1091                vec![
1092                    Field::new("col", DataType::Int64, true).with_metadata(HashMap::from([(
1093                        PARQUET_FIELD_ID_META_KEY.to_string(),
1094                        "0".to_string(),
1095                    )])),
1096                ];
1097            Arc::new(arrow_schema::Schema::new(fields))
1098        };
1099        let col = Arc::new(Int64Array::from_iter_values(0..1024)) as ArrayRef;
1100        let null_col = Arc::new(Int64Array::new_null(1024)) as ArrayRef;
1101        let to_write = RecordBatch::try_new(schema.clone(), vec![col]).unwrap();
1102        let to_write_null = RecordBatch::try_new(schema.clone(), vec![null_col]).unwrap();
1103
1104        let output_file = file_io.new_output(
1105            location_gen.generate_location(None, &file_name_gen.generate_file_name()),
1106        )?;
1107
1108        // write data
1109        let mut pw = ParquetWriterBuilder::new(
1110            WriterProperties::builder()
1111                .set_max_row_group_row_count(Some(128))
1112                .build(),
1113            Arc::new(to_write.schema().as_ref().try_into().unwrap()),
1114        )
1115        .build(output_file)
1116        .await?;
1117        pw.write(&to_write).await?;
1118        pw.write(&to_write_null).await?;
1119        let res = pw.close().await?;
1120        assert_eq!(res.len(), 1);
1121        let data_file = res
1122            .into_iter()
1123            .next()
1124            .unwrap()
1125            // Put dummy field for build successfully.
1126            .content(DataContentType::Data)
1127            .partition(Struct::empty())
1128            .partition_spec_id(0)
1129            .build()
1130            .unwrap();
1131
1132        // check data file
1133        assert_eq!(data_file.record_count(), 2048);
1134        assert_eq!(*data_file.value_counts(), HashMap::from([(0, 2048)]));
1135        assert_eq!(
1136            *data_file.lower_bounds(),
1137            HashMap::from([(0, Datum::long(0))])
1138        );
1139        assert_eq!(
1140            *data_file.upper_bounds(),
1141            HashMap::from([(0, Datum::long(1023))])
1142        );
1143        assert_eq!(*data_file.null_value_counts(), HashMap::from([(0, 1024)]));
1144
1145        // check the written file
1146        let expect_batch = concat_batches(&schema, vec![&to_write, &to_write_null]).unwrap();
1147        check_parquet_data_file(&file_io, &data_file, &expect_batch).await;
1148
1149        Ok(())
1150    }
1151
1152    #[tokio::test]
1153    async fn test_parquet_writer_encrypted_roundtrip() -> Result<()> {
1154        let temp_dir = TempDir::new().unwrap();
1155        let file_io = FileIO::new_with_fs();
1156        let location_gen = DefaultLocationGenerator::with_data_location(
1157            temp_dir.path().to_str().unwrap().to_string(),
1158        );
1159        let file_name_gen =
1160            DefaultFileNameGenerator::new("test".to_string(), None, DataFileFormat::Parquet);
1161
1162        let arrow_schema = Arc::new(arrow_schema::Schema::new(vec![
1163            Field::new("id", DataType::Int32, false).with_metadata(HashMap::from([(
1164                PARQUET_FIELD_ID_META_KEY.to_string(),
1165                "1".to_string(),
1166            )])),
1167        ]));
1168        let iceberg_schema: SchemaRef = Arc::new(arrow_schema.as_ref().try_into().unwrap());
1169        let batch = RecordBatch::try_new(arrow_schema, vec![Arc::new(Int32Array::from(vec![
1170            10, 20, 30,
1171        ])) as ArrayRef])
1172        .unwrap();
1173
1174        let output_file = file_io.new_output(
1175            location_gen.generate_location(None, &file_name_gen.generate_file_name()),
1176        )?;
1177        let raw_properties = HashMap::from([(
1178            TableProperties::PROPERTY_ENCRYPTION_KEY_ID.to_string(),
1179            "test-key".to_string(),
1180        )]);
1181        let table_properties = TableProperties::new(&raw_properties);
1182        let mut parquet_writer =
1183            ParquetWriterBuilder::from_table_properties(&table_properties, iceberg_schema.clone())?
1184                .with_encryption_manager(make_encryption_manager("test-key"))
1185                .build(output_file)
1186                .await?;
1187        parquet_writer.write(&batch).await?;
1188        let data_file = parquet_writer
1189            .close()
1190            .await?
1191            .into_iter()
1192            .next()
1193            .unwrap()
1194            .content(DataContentType::Data)
1195            .partition(Struct::empty())
1196            .partition_spec_id(0)
1197            .build()
1198            .unwrap();
1199
1200        let raw = file_io
1201            .new_input(data_file.file_path.clone())?
1202            .read()
1203            .await?;
1204        assert!(
1205            ParquetRecordBatchReaderBuilder::try_new(raw).is_err(),
1206            "an encrypted parquet file must not be readable without decryption"
1207        );
1208
1209        let reader = ArrowReaderBuilder::new(file_io, Runtime::current()).build();
1210        let task = FileScanTask::builder()
1211            .with_file_size_in_bytes(data_file.file_size_in_bytes())
1212            .with_start(0)
1213            .with_length(0)
1214            .with_data_file_path(data_file.file_path.clone())
1215            .with_data_file_format(DataFileFormat::Parquet)
1216            .with_schema(iceberg_schema)
1217            .with_project_field_ids(vec![1])
1218            .with_case_sensitive(false)
1219            .with_key_metadata(data_file.key_metadata().map(Box::from))
1220            .build()
1221            .unwrap();
1222        let tasks = Box::pin(futures::stream::iter(vec![Ok(task)])) as FileScanTaskStream;
1223        let batches: Vec<RecordBatch> = reader
1224            .read(tasks)
1225            .unwrap()
1226            .stream()
1227            .try_collect()
1228            .await
1229            .unwrap();
1230
1231        assert_eq!(batches.len(), 1);
1232        let ids = batches[0]
1233            .column(0)
1234            .as_any()
1235            .downcast_ref::<Int32Array>()
1236            .unwrap();
1237        assert_eq!(ids.values(), &[10, 20, 30]);
1238
1239        Ok(())
1240    }
1241
1242    #[tokio::test]
1243    async fn test_parquet_writer_with_complex_schema() -> Result<()> {
1244        let temp_dir = TempDir::new().unwrap();
1245        let file_io = FileIO::new_with_fs();
1246        let location_gen = DefaultLocationGenerator::with_data_location(
1247            temp_dir.path().to_str().unwrap().to_string(),
1248        );
1249        let file_name_gen =
1250            DefaultFileNameGenerator::new("test".to_string(), None, DataFileFormat::Parquet);
1251
1252        // prepare data
1253        let schema = nested_schema_for_test();
1254        let arrow_schema: ArrowSchemaRef = Arc::new((&schema).try_into().unwrap());
1255        let col0 = Arc::new(Int64Array::from_iter_values(0..1024)) as ArrayRef;
1256        let col1 = Arc::new(StructArray::new(
1257            {
1258                if let DataType::Struct(fields) = arrow_schema.field(1).data_type() {
1259                    fields.clone()
1260                } else {
1261                    unreachable!()
1262                }
1263            },
1264            vec![
1265                Arc::new(Int64Array::from_iter_values(0..1024)),
1266                Arc::new(Int64Array::from_iter_values(0..1024)),
1267            ],
1268            None,
1269        ));
1270        let col2 = Arc::new(arrow_array::StringArray::from_iter_values(
1271            (0..1024).map(|n| n.to_string()),
1272        )) as ArrayRef;
1273        let col3 = Arc::new({
1274            let list_parts = ListArray::from_iter_primitive::<Int64Type, _, _>(
1275                (0..1024).map(|n| Some(vec![Some(n)])),
1276            )
1277            .into_parts();
1278            ListArray::new(
1279                {
1280                    if let DataType::List(field) = arrow_schema.field(3).data_type() {
1281                        field.clone()
1282                    } else {
1283                        unreachable!()
1284                    }
1285                },
1286                list_parts.1,
1287                list_parts.2,
1288                list_parts.3,
1289            )
1290        }) as ArrayRef;
1291        let col4 = Arc::new(StructArray::new(
1292            {
1293                if let DataType::Struct(fields) = arrow_schema.field(4).data_type() {
1294                    fields.clone()
1295                } else {
1296                    unreachable!()
1297                }
1298            },
1299            vec![Arc::new(StructArray::new(
1300                {
1301                    if let DataType::Struct(fields) = arrow_schema.field(4).data_type() {
1302                        if let DataType::Struct(fields) = fields[0].data_type() {
1303                            fields.clone()
1304                        } else {
1305                            unreachable!()
1306                        }
1307                    } else {
1308                        unreachable!()
1309                    }
1310                },
1311                vec![Arc::new(Int64Array::from_iter_values(0..1024))],
1312                None,
1313            ))],
1314            None,
1315        ));
1316        let col5 = Arc::new({
1317            let mut map_array_builder = MapBuilder::new(
1318                None,
1319                arrow_array::builder::StringBuilder::new(),
1320                arrow_array::builder::ListBuilder::new(arrow_array::builder::PrimitiveBuilder::<
1321                    Int64Type,
1322                >::new()),
1323            );
1324            for i in 0..1024 {
1325                map_array_builder.keys().append_value(i.to_string());
1326                map_array_builder
1327                    .values()
1328                    .append_value(vec![Some(i as i64); i + 1]);
1329                map_array_builder.append(true)?;
1330            }
1331            let (_, offset_buffer, struct_array, null_buffer, ordered) =
1332                map_array_builder.finish().into_parts();
1333            let struct_array = {
1334                let (_, mut arrays, nulls) = struct_array.into_parts();
1335                let list_array = {
1336                    let list_array = arrays[1]
1337                        .as_any()
1338                        .downcast_ref::<ListArray>()
1339                        .unwrap()
1340                        .clone();
1341                    let (_, offsets, array, nulls) = list_array.into_parts();
1342                    let list_field = {
1343                        if let DataType::Map(map_field, _) = arrow_schema.field(5).data_type() {
1344                            if let DataType::Struct(fields) = map_field.data_type() {
1345                                if let DataType::List(list_field) = fields[1].data_type() {
1346                                    list_field.clone()
1347                                } else {
1348                                    unreachable!()
1349                                }
1350                            } else {
1351                                unreachable!()
1352                            }
1353                        } else {
1354                            unreachable!()
1355                        }
1356                    };
1357                    ListArray::new(list_field, offsets, array, nulls)
1358                };
1359                arrays[1] = Arc::new(list_array) as ArrayRef;
1360                StructArray::new(
1361                    {
1362                        if let DataType::Map(map_field, _) = arrow_schema.field(5).data_type() {
1363                            if let DataType::Struct(fields) = map_field.data_type() {
1364                                fields.clone()
1365                            } else {
1366                                unreachable!()
1367                            }
1368                        } else {
1369                            unreachable!()
1370                        }
1371                    },
1372                    arrays,
1373                    nulls,
1374                )
1375            };
1376            MapArray::new(
1377                {
1378                    if let DataType::Map(map_field, _) = arrow_schema.field(5).data_type() {
1379                        map_field.clone()
1380                    } else {
1381                        unreachable!()
1382                    }
1383                },
1384                offset_buffer,
1385                struct_array,
1386                null_buffer,
1387                ordered,
1388            )
1389        }) as ArrayRef;
1390        let to_write = RecordBatch::try_new(arrow_schema.clone(), vec![
1391            col0, col1, col2, col3, col4, col5,
1392        ])
1393        .unwrap();
1394        let output_file = file_io.new_output(
1395            location_gen.generate_location(None, &file_name_gen.generate_file_name()),
1396        )?;
1397
1398        // write data
1399        let mut pw =
1400            ParquetWriterBuilder::new(WriterProperties::builder().build(), Arc::new(schema))
1401                .build(output_file)
1402                .await?;
1403        pw.write(&to_write).await?;
1404        let res = pw.close().await?;
1405        assert_eq!(res.len(), 1);
1406        let data_file = res
1407            .into_iter()
1408            .next()
1409            .unwrap()
1410            // Put dummy field for build successfully.
1411            .content(DataContentType::Data)
1412            .partition(Struct::empty())
1413            .partition_spec_id(0)
1414            .build()
1415            .unwrap();
1416
1417        // check data file
1418        assert_eq!(data_file.record_count(), 1024);
1419        assert_eq!(
1420            *data_file.value_counts(),
1421            HashMap::from([
1422                (0, 1024),
1423                (5, 1024),
1424                (6, 1024),
1425                (2, 1024),
1426                (7, 1024),
1427                (9, 1024),
1428                (11, 1024),
1429                (13, (1..1025).sum()),
1430            ])
1431        );
1432        assert_eq!(
1433            *data_file.lower_bounds(),
1434            HashMap::from([
1435                (0, Datum::long(0)),
1436                (5, Datum::long(0)),
1437                (6, Datum::long(0)),
1438                (2, Datum::string("0")),
1439                (7, Datum::long(0)),
1440                (9, Datum::long(0)),
1441                (11, Datum::string("0")),
1442                (13, Datum::long(0))
1443            ])
1444        );
1445        assert_eq!(
1446            *data_file.upper_bounds(),
1447            HashMap::from([
1448                (0, Datum::long(1023)),
1449                (5, Datum::long(1023)),
1450                (6, Datum::long(1023)),
1451                (2, Datum::string("999")),
1452                (7, Datum::long(1023)),
1453                (9, Datum::long(1023)),
1454                (11, Datum::string("999")),
1455                (13, Datum::long(1023))
1456            ])
1457        );
1458
1459        // check the written file
1460        check_parquet_data_file(&file_io, &data_file, &to_write).await;
1461
1462        Ok(())
1463    }
1464
1465    #[tokio::test]
1466    async fn test_all_type_for_write() -> Result<()> {
1467        let temp_dir = TempDir::new().unwrap();
1468        let file_io = FileIO::new_with_fs();
1469        let location_gen = DefaultLocationGenerator::with_data_location(
1470            temp_dir.path().to_str().unwrap().to_string(),
1471        );
1472        let file_name_gen =
1473            DefaultFileNameGenerator::new("test".to_string(), None, DataFileFormat::Parquet);
1474
1475        // prepare data
1476        // generate iceberg schema for all type
1477        let schema = schema_for_all_type();
1478        let arrow_schema: ArrowSchemaRef = Arc::new((&schema).try_into().unwrap());
1479        let col0 = Arc::new(BooleanArray::from(vec![
1480            Some(true),
1481            Some(false),
1482            None,
1483            Some(true),
1484        ])) as ArrayRef;
1485        let col1 = Arc::new(Int32Array::from(vec![Some(1), Some(2), None, Some(4)])) as ArrayRef;
1486        let col2 = Arc::new(Int64Array::from(vec![Some(1), Some(2), None, Some(4)])) as ArrayRef;
1487        let col3 = Arc::new(Float32Array::from(vec![
1488            Some(0.5),
1489            Some(2.0),
1490            None,
1491            Some(3.5),
1492        ])) as ArrayRef;
1493        let col4 = Arc::new(Float64Array::from(vec![
1494            Some(0.5),
1495            Some(2.0),
1496            None,
1497            Some(3.5),
1498        ])) as ArrayRef;
1499        let col5 = Arc::new(arrow_array::StringArray::from(vec![
1500            Some("a"),
1501            Some("b"),
1502            None,
1503            Some("d"),
1504        ])) as ArrayRef;
1505        let col6 = Arc::new(arrow_array::BinaryArray::from_opt_vec(vec![
1506            Some(b"one"),
1507            None,
1508            Some(b""),
1509            Some(b"zzzz"),
1510        ])) as ArrayRef;
1511        let col7 = Arc::new(arrow_array::Date32Array::from(vec![
1512            Some(0),
1513            Some(1),
1514            None,
1515            Some(3),
1516        ])) as ArrayRef;
1517        let col8 = Arc::new(arrow_array::Time64MicrosecondArray::from(vec![
1518            Some(0),
1519            Some(1),
1520            None,
1521            Some(3),
1522        ])) as ArrayRef;
1523        let col9 = Arc::new(arrow_array::TimestampMicrosecondArray::from(vec![
1524            Some(0),
1525            Some(1),
1526            None,
1527            Some(3),
1528        ])) as ArrayRef;
1529        let col10 = Arc::new(
1530            arrow_array::TimestampMicrosecondArray::from(vec![Some(0), Some(1), None, Some(3)])
1531                .with_timezone_utc(),
1532        ) as ArrayRef;
1533        let col11 = Arc::new(arrow_array::TimestampNanosecondArray::from(vec![
1534            Some(0),
1535            Some(1),
1536            None,
1537            Some(3),
1538        ])) as ArrayRef;
1539        let col12 = Arc::new(
1540            arrow_array::TimestampNanosecondArray::from(vec![Some(0), Some(1), None, Some(3)])
1541                .with_timezone_utc(),
1542        ) as ArrayRef;
1543        let col13 = Arc::new(
1544            Decimal128Array::from(vec![Some(1), Some(2), None, Some(100)])
1545                .with_precision_and_scale(10, 5)
1546                .unwrap(),
1547        ) as ArrayRef;
1548        let col14 = Arc::new(
1549            arrow_array::FixedSizeBinaryArray::try_from_sparse_iter_with_size(
1550                vec![
1551                    Some(Uuid::from_u128(0).as_bytes().to_vec()),
1552                    Some(Uuid::from_u128(1).as_bytes().to_vec()),
1553                    None,
1554                    Some(Uuid::from_u128(3).as_bytes().to_vec()),
1555                ]
1556                .into_iter(),
1557                16,
1558            )
1559            .unwrap(),
1560        ) as ArrayRef;
1561        let col15 = Arc::new(
1562            arrow_array::FixedSizeBinaryArray::try_from_sparse_iter_with_size(
1563                vec![
1564                    Some(vec![1, 2, 3, 4, 5, 6, 7, 8, 9, 10]),
1565                    Some(vec![11, 12, 13, 14, 15, 16, 17, 18, 19, 20]),
1566                    None,
1567                    Some(vec![21, 22, 23, 24, 25, 26, 27, 28, 29, 30]),
1568                ]
1569                .into_iter(),
1570                10,
1571            )
1572            .unwrap(),
1573        ) as ArrayRef;
1574        let col16 = Arc::new(
1575            Decimal128Array::from(vec![Some(1), Some(2), None, Some(100)])
1576                .with_precision_and_scale(38, 5)
1577                .unwrap(),
1578        ) as ArrayRef;
1579        let to_write = RecordBatch::try_new(arrow_schema.clone(), vec![
1580            col0, col1, col2, col3, col4, col5, col6, col7, col8, col9, col10, col11, col12, col13,
1581            col14, col15, col16,
1582        ])
1583        .unwrap();
1584        let output_file = file_io.new_output(
1585            location_gen.generate_location(None, &file_name_gen.generate_file_name()),
1586        )?;
1587
1588        // write data
1589        let mut pw =
1590            ParquetWriterBuilder::new(WriterProperties::builder().build(), Arc::new(schema))
1591                .build(output_file)
1592                .await?;
1593        pw.write(&to_write).await?;
1594        let res = pw.close().await?;
1595        assert_eq!(res.len(), 1);
1596        let data_file = res
1597            .into_iter()
1598            .next()
1599            .unwrap()
1600            // Put dummy field for build successfully.
1601            .content(DataContentType::Data)
1602            .partition(Struct::empty())
1603            .partition_spec_id(0)
1604            .build()
1605            .unwrap();
1606
1607        // check data file
1608        assert_eq!(data_file.record_count(), 4);
1609        assert!(data_file.value_counts().iter().all(|(_, &v)| { v == 4 }));
1610        assert!(
1611            data_file
1612                .null_value_counts()
1613                .iter()
1614                .all(|(_, &v)| { v == 1 })
1615        );
1616        assert_eq!(
1617            *data_file.lower_bounds(),
1618            HashMap::from([
1619                (0, Datum::bool(false)),
1620                (1, Datum::int(1)),
1621                (2, Datum::long(1)),
1622                (3, Datum::float(0.5_f32)),
1623                (4, Datum::double(0.5)),
1624                (5, Datum::string("a")),
1625                (6, Datum::binary(vec![])),
1626                (7, Datum::date(0)),
1627                (8, Datum::time_micros(0).unwrap()),
1628                (9, Datum::timestamp_micros(0)),
1629                (10, Datum::timestamptz_micros(0)),
1630                (11, Datum::timestamp_nanos(0)),
1631                (12, Datum::timestamptz_nanos(0)),
1632                (
1633                    13,
1634                    Datum::new(
1635                        PrimitiveType::Decimal {
1636                            precision: 10,
1637                            scale: 5
1638                        },
1639                        PrimitiveLiteral::Int128(1)
1640                    )
1641                ),
1642                (14, Datum::uuid(Uuid::from_u128(0))),
1643                (15, Datum::fixed(vec![1, 2, 3, 4, 5, 6, 7, 8, 9, 10])),
1644                (
1645                    16,
1646                    Datum::new(
1647                        PrimitiveType::Decimal {
1648                            precision: 38,
1649                            scale: 5
1650                        },
1651                        PrimitiveLiteral::Int128(1)
1652                    )
1653                ),
1654            ])
1655        );
1656        assert_eq!(
1657            *data_file.upper_bounds(),
1658            HashMap::from([
1659                (0, Datum::bool(true)),
1660                (1, Datum::int(4)),
1661                (2, Datum::long(4)),
1662                (3, Datum::float(3.5_f32)),
1663                (4, Datum::double(3.5)),
1664                (5, Datum::string("d")),
1665                (6, Datum::binary(vec![122, 122, 122, 122])),
1666                (7, Datum::date(3)),
1667                (8, Datum::time_micros(3).unwrap()),
1668                (9, Datum::timestamp_micros(3)),
1669                (10, Datum::timestamptz_micros(3)),
1670                (11, Datum::timestamp_nanos(3)),
1671                (12, Datum::timestamptz_nanos(3)),
1672                (
1673                    13,
1674                    Datum::new(
1675                        PrimitiveType::Decimal {
1676                            precision: 10,
1677                            scale: 5
1678                        },
1679                        PrimitiveLiteral::Int128(100)
1680                    )
1681                ),
1682                (14, Datum::uuid(Uuid::from_u128(3))),
1683                (
1684                    15,
1685                    Datum::fixed(vec![21, 22, 23, 24, 25, 26, 27, 28, 29, 30])
1686                ),
1687                (
1688                    16,
1689                    Datum::new(
1690                        PrimitiveType::Decimal {
1691                            precision: 38,
1692                            scale: 5
1693                        },
1694                        PrimitiveLiteral::Int128(100)
1695                    )
1696                ),
1697            ])
1698        );
1699
1700        // check the written file
1701        check_parquet_data_file(&file_io, &data_file, &to_write).await;
1702
1703        Ok(())
1704    }
1705
1706    #[tokio::test]
1707    async fn test_decimal_bound() -> Result<()> {
1708        let temp_dir = TempDir::new().unwrap();
1709        let file_io = FileIO::new_with_fs();
1710        let location_gen = DefaultLocationGenerator::with_data_location(
1711            temp_dir.path().to_str().unwrap().to_string(),
1712        );
1713        let file_name_gen =
1714            DefaultFileNameGenerator::new("test".to_string(), None, DataFileFormat::Parquet);
1715
1716        // test 1.1 and 2.2
1717        let schema = Arc::new(
1718            Schema::builder()
1719                .with_fields(vec![
1720                    NestedField::optional(
1721                        0,
1722                        "decimal",
1723                        Type::Primitive(PrimitiveType::Decimal {
1724                            precision: 28,
1725                            scale: 10,
1726                        }),
1727                    )
1728                    .into(),
1729                ])
1730                .build()
1731                .unwrap(),
1732        );
1733        let arrow_schema: ArrowSchemaRef = Arc::new(schema_to_arrow_schema(&schema).unwrap());
1734        let output_file = file_io.new_output(
1735            location_gen.generate_location(None, &file_name_gen.generate_file_name()),
1736        )?;
1737        let mut pw = ParquetWriterBuilder::new(WriterProperties::builder().build(), schema.clone())
1738            .build(output_file)
1739            .await?;
1740        let col0 = Arc::new(
1741            Decimal128Array::from(vec![Some(22000000000), Some(11000000000)])
1742                .with_data_type(DataType::Decimal128(28, 10)),
1743        ) as ArrayRef;
1744        let to_write = RecordBatch::try_new(arrow_schema.clone(), vec![col0]).unwrap();
1745        pw.write(&to_write).await?;
1746        let res = pw.close().await?;
1747        assert_eq!(res.len(), 1);
1748        let data_file = res
1749            .into_iter()
1750            .next()
1751            .unwrap()
1752            .content(DataContentType::Data)
1753            .partition(Struct::empty())
1754            .partition_spec_id(0)
1755            .build()
1756            .unwrap();
1757        assert_eq!(
1758            data_file.upper_bounds().get(&0),
1759            Some(Datum::decimal_with_precision(decimal_new(22000000000_i64, 10), 28).unwrap())
1760                .as_ref()
1761        );
1762        assert_eq!(
1763            data_file.lower_bounds().get(&0),
1764            Some(Datum::decimal_with_precision(decimal_new(11000000000_i64, 10), 28).unwrap())
1765                .as_ref()
1766        );
1767
1768        // test -1.1 and -2.2
1769        let schema = Arc::new(
1770            Schema::builder()
1771                .with_fields(vec![
1772                    NestedField::optional(
1773                        0,
1774                        "decimal",
1775                        Type::Primitive(PrimitiveType::Decimal {
1776                            precision: 28,
1777                            scale: 10,
1778                        }),
1779                    )
1780                    .into(),
1781                ])
1782                .build()
1783                .unwrap(),
1784        );
1785        let arrow_schema: ArrowSchemaRef = Arc::new(schema_to_arrow_schema(&schema).unwrap());
1786        let output_file = file_io.new_output(
1787            location_gen.generate_location(None, &file_name_gen.generate_file_name()),
1788        )?;
1789        let mut pw = ParquetWriterBuilder::new(WriterProperties::builder().build(), schema.clone())
1790            .build(output_file)
1791            .await?;
1792        let col0 = Arc::new(
1793            Decimal128Array::from(vec![Some(-22000000000), Some(-11000000000)])
1794                .with_data_type(DataType::Decimal128(28, 10)),
1795        ) as ArrayRef;
1796        let to_write = RecordBatch::try_new(arrow_schema.clone(), vec![col0]).unwrap();
1797        pw.write(&to_write).await?;
1798        let res = pw.close().await?;
1799        assert_eq!(res.len(), 1);
1800        let data_file = res
1801            .into_iter()
1802            .next()
1803            .unwrap()
1804            .content(DataContentType::Data)
1805            .partition(Struct::empty())
1806            .partition_spec_id(0)
1807            .build()
1808            .unwrap();
1809        assert_eq!(
1810            data_file.upper_bounds().get(&0),
1811            Some(Datum::decimal_with_precision(decimal_new(-11000000000_i64, 10), 28).unwrap())
1812                .as_ref()
1813        );
1814        assert_eq!(
1815            data_file.lower_bounds().get(&0),
1816            Some(Datum::decimal_with_precision(decimal_new(-22000000000_i64, 10), 28).unwrap())
1817                .as_ref()
1818        );
1819
1820        // test 38-digit precision decimal values (Iceberg spec max)
1821        // Note: fastnum D128::MAX/MIN have impractical exponents, so we use meaningful values
1822        use crate::spec::decimal_utils::decimal_from_str_exact;
1823        let decimal_max = decimal_from_str_exact("99999999999999999999999999999999999999").unwrap();
1824        let decimal_min =
1825            decimal_from_str_exact("-99999999999999999999999999999999999999").unwrap();
1826        assert_eq!(decimal_scale(&decimal_max), decimal_scale(&decimal_min));
1827        let schema = Arc::new(
1828            Schema::builder()
1829                .with_fields(vec![
1830                    NestedField::optional(
1831                        0,
1832                        "decimal",
1833                        Type::Primitive(PrimitiveType::Decimal {
1834                            precision: 38,
1835                            scale: decimal_scale(&decimal_max),
1836                        }),
1837                    )
1838                    .into(),
1839                ])
1840                .build()
1841                .unwrap(),
1842        );
1843        let arrow_schema: ArrowSchemaRef = Arc::new(schema_to_arrow_schema(&schema).unwrap());
1844        let output_file = file_io.new_output(
1845            location_gen.generate_location(None, &file_name_gen.generate_file_name()),
1846        )?;
1847        let mut pw = ParquetWriterBuilder::new(WriterProperties::builder().build(), schema)
1848            .build(output_file)
1849            .await?;
1850        let col0 = Arc::new(
1851            Decimal128Array::from(vec![
1852                Some(decimal_mantissa(&decimal_max)),
1853                Some(decimal_mantissa(&decimal_min)),
1854            ])
1855            .with_data_type(DataType::Decimal128(38, 0)),
1856        ) as ArrayRef;
1857        let to_write = RecordBatch::try_new(arrow_schema.clone(), vec![col0]).unwrap();
1858        pw.write(&to_write).await?;
1859        let res = pw.close().await?;
1860        assert_eq!(res.len(), 1);
1861        let data_file = res
1862            .into_iter()
1863            .next()
1864            .unwrap()
1865            .content(DataContentType::Data)
1866            .partition(Struct::empty())
1867            .partition_spec_id(0)
1868            .build()
1869            .unwrap();
1870        assert_eq!(
1871            data_file.upper_bounds().get(&0),
1872            Some(Datum::decimal(decimal_max).unwrap()).as_ref()
1873        );
1874        assert_eq!(
1875            data_file.lower_bounds().get(&0),
1876            Some(Datum::decimal(decimal_min).unwrap()).as_ref()
1877        );
1878
1879        // test max and min for scale 38
1880        // # TODO
1881        // Readd this case after resolve https://github.com/apache/iceberg-rust/issues/669
1882        // let schema = Arc::new(
1883        //     Schema::builder()
1884        //         .with_fields(vec![NestedField::optional(
1885        //             0,
1886        //             "decimal",
1887        //             Type::Primitive(PrimitiveType::Decimal {
1888        //                 precision: 38,
1889        //                 scale: 0,
1890        //             }),
1891        //         )
1892        //         .into()])
1893        //         .build()
1894        //         .unwrap(),
1895        // );
1896        // let arrow_schema: ArrowSchemaRef = Arc::new(schema_to_arrow_schema(&schema).unwrap());
1897        // let mut pw = ParquetWriterBuilder::new(
1898        //     WriterProperties::builder().build(),
1899        //     schema,
1900        //     file_io.clone(),
1901        //     loccation_gen,
1902        //     file_name_gen,
1903        // )
1904        // .build()
1905        // .await?;
1906        // let col0 = Arc::new(
1907        //     Decimal128Array::from(vec![
1908        //         Some(99999999999999999999999999999999999999_i128),
1909        //         Some(-99999999999999999999999999999999999999_i128),
1910        //     ])
1911        //     .with_data_type(DataType::Decimal128(38, 0)),
1912        // ) as ArrayRef;
1913        // let to_write = RecordBatch::try_new(arrow_schema.clone(), vec![col0]).unwrap();
1914        // pw.write(&to_write).await?;
1915        // let res = pw.close().await?;
1916        // assert_eq!(res.len(), 1);
1917        // let data_file = res
1918        //     .into_iter()
1919        //     .next()
1920        //     .unwrap()
1921        //     .content(crate::spec::DataContentType::Data)
1922        //     .partition(Struct::empty())
1923        //     .build()
1924        //     .unwrap();
1925        // assert_eq!(
1926        //     data_file.upper_bounds().get(&0),
1927        //     Some(Datum::new(
1928        //         PrimitiveType::Decimal {
1929        //             precision: 38,
1930        //             scale: 0
1931        //         },
1932        //         PrimitiveLiteral::Int128(99999999999999999999999999999999999999_i128)
1933        //     ))
1934        //     .as_ref()
1935        // );
1936        // assert_eq!(
1937        //     data_file.lower_bounds().get(&0),
1938        //     Some(Datum::new(
1939        //         PrimitiveType::Decimal {
1940        //             precision: 38,
1941        //             scale: 0
1942        //         },
1943        //         PrimitiveLiteral::Int128(-99999999999999999999999999999999999999_i128)
1944        //     ))
1945        //     .as_ref()
1946        // );
1947
1948        Ok(())
1949    }
1950
1951    #[tokio::test]
1952    async fn test_empty_write() -> Result<()> {
1953        let temp_dir = TempDir::new().unwrap();
1954        let file_io = FileIO::new_with_fs();
1955        let location_gen = DefaultLocationGenerator::with_data_location(
1956            temp_dir.path().to_str().unwrap().to_string(),
1957        );
1958        let file_name_gen =
1959            DefaultFileNameGenerator::new("test".to_string(), None, DataFileFormat::Parquet);
1960
1961        // Test that file will create if data to write
1962        let schema = {
1963            let fields =
1964                vec![
1965                    Field::new("col", DataType::Int64, true).with_metadata(HashMap::from([(
1966                        PARQUET_FIELD_ID_META_KEY.to_string(),
1967                        "0".to_string(),
1968                    )])),
1969                ];
1970            Arc::new(arrow_schema::Schema::new(fields))
1971        };
1972        let col = Arc::new(Int64Array::from_iter_values(0..1024)) as ArrayRef;
1973        let to_write = RecordBatch::try_new(schema.clone(), vec![col]).unwrap();
1974        let file_path = location_gen.generate_location(None, &file_name_gen.generate_file_name());
1975        let output_file = file_io.new_output(&file_path)?;
1976        let mut pw = ParquetWriterBuilder::new(
1977            WriterProperties::builder().build(),
1978            Arc::new(to_write.schema().as_ref().try_into().unwrap()),
1979        )
1980        .build(output_file)
1981        .await?;
1982        pw.write(&to_write).await?;
1983        pw.close().await.unwrap();
1984        assert!(file_io.exists(&file_path).await.unwrap());
1985
1986        // Test that file will not create if no data to write
1987        let file_name_gen =
1988            DefaultFileNameGenerator::new("test_empty".to_string(), None, DataFileFormat::Parquet);
1989        let file_path = location_gen.generate_location(None, &file_name_gen.generate_file_name());
1990        let output_file = file_io.new_output(&file_path)?;
1991        let pw = ParquetWriterBuilder::new(
1992            WriterProperties::builder().build(),
1993            Arc::new(to_write.schema().as_ref().try_into().unwrap()),
1994        )
1995        .build(output_file)
1996        .await?;
1997        pw.close().await.unwrap();
1998        assert!(!file_io.exists(&file_path).await.unwrap());
1999
2000        Ok(())
2001    }
2002
2003    #[tokio::test]
2004    async fn test_nan_val_cnts_primitive_type() -> Result<()> {
2005        let temp_dir = TempDir::new().unwrap();
2006        let file_io = FileIO::new_with_fs();
2007        let location_gen = DefaultLocationGenerator::with_data_location(
2008            temp_dir.path().to_str().unwrap().to_string(),
2009        );
2010        let file_name_gen =
2011            DefaultFileNameGenerator::new("test".to_string(), None, DataFileFormat::Parquet);
2012
2013        // prepare data
2014        let arrow_schema = {
2015            let fields = vec![
2016                Field::new("col", DataType::Float32, false).with_metadata(HashMap::from([(
2017                    PARQUET_FIELD_ID_META_KEY.to_string(),
2018                    "0".to_string(),
2019                )])),
2020                Field::new("col2", DataType::Float64, false).with_metadata(HashMap::from([(
2021                    PARQUET_FIELD_ID_META_KEY.to_string(),
2022                    "1".to_string(),
2023                )])),
2024            ];
2025            Arc::new(arrow_schema::Schema::new(fields))
2026        };
2027
2028        let float_32_col = Arc::new(Float32Array::from_iter_values_with_nulls(
2029            [1.0_f32, f32::NAN, 2.0, 2.0].into_iter(),
2030            None,
2031        )) as ArrayRef;
2032
2033        let float_64_col = Arc::new(Float64Array::from_iter_values_with_nulls(
2034            [1.0_f64, f64::NAN, 2.0, 2.0].into_iter(),
2035            None,
2036        )) as ArrayRef;
2037
2038        let to_write =
2039            RecordBatch::try_new(arrow_schema.clone(), vec![float_32_col, float_64_col]).unwrap();
2040        let output_file = file_io.new_output(
2041            location_gen.generate_location(None, &file_name_gen.generate_file_name()),
2042        )?;
2043
2044        // write data
2045        let mut pw = ParquetWriterBuilder::new(
2046            WriterProperties::builder().build(),
2047            Arc::new(to_write.schema().as_ref().try_into().unwrap()),
2048        )
2049        .build(output_file)
2050        .await?;
2051
2052        pw.write(&to_write).await?;
2053        let res = pw.close().await?;
2054        assert_eq!(res.len(), 1);
2055        let data_file = res
2056            .into_iter()
2057            .next()
2058            .unwrap()
2059            // Put dummy field for build successfully.
2060            .content(DataContentType::Data)
2061            .partition(Struct::empty())
2062            .partition_spec_id(0)
2063            .build()
2064            .unwrap();
2065
2066        // check data file
2067        assert_eq!(data_file.record_count(), 4);
2068        assert_eq!(*data_file.value_counts(), HashMap::from([(0, 4), (1, 4)]));
2069        assert_eq!(
2070            *data_file.lower_bounds(),
2071            HashMap::from([(0, Datum::float(1.0_f32)), (1, Datum::double(1.0)),])
2072        );
2073        assert_eq!(
2074            *data_file.upper_bounds(),
2075            HashMap::from([(0, Datum::float(2.0_f32)), (1, Datum::double(2.0)),])
2076        );
2077        assert_eq!(
2078            *data_file.null_value_counts(),
2079            HashMap::from([(0, 0), (1, 0)])
2080        );
2081        assert_eq!(
2082            *data_file.nan_value_counts(),
2083            HashMap::from([(0, 1), (1, 1)])
2084        );
2085
2086        // check the written file
2087        let expect_batch = concat_batches(&arrow_schema, vec![&to_write]).unwrap();
2088        check_parquet_data_file(&file_io, &data_file, &expect_batch).await;
2089
2090        Ok(())
2091    }
2092
2093    #[tokio::test]
2094    async fn test_nan_val_cnts_struct_type() -> Result<()> {
2095        let temp_dir = TempDir::new().unwrap();
2096        let file_io = FileIO::new_with_fs();
2097        let location_gen = DefaultLocationGenerator::with_data_location(
2098            temp_dir.path().to_str().unwrap().to_string(),
2099        );
2100        let file_name_gen =
2101            DefaultFileNameGenerator::new("test".to_string(), None, DataFileFormat::Parquet);
2102
2103        let schema_struct_float_fields = Fields::from(vec![
2104            Field::new("col4", DataType::Float32, false).with_metadata(HashMap::from([(
2105                PARQUET_FIELD_ID_META_KEY.to_string(),
2106                "4".to_string(),
2107            )])),
2108        ]);
2109
2110        let schema_struct_nested_float_fields = Fields::from(vec![
2111            Field::new("col7", DataType::Float32, false).with_metadata(HashMap::from([(
2112                PARQUET_FIELD_ID_META_KEY.to_string(),
2113                "7".to_string(),
2114            )])),
2115        ]);
2116
2117        let schema_struct_nested_fields = Fields::from(vec![
2118            Field::new(
2119                "col6",
2120                DataType::Struct(schema_struct_nested_float_fields.clone()),
2121                false,
2122            )
2123            .with_metadata(HashMap::from([(
2124                PARQUET_FIELD_ID_META_KEY.to_string(),
2125                "6".to_string(),
2126            )])),
2127        ]);
2128
2129        // prepare data
2130        let arrow_schema = {
2131            let fields = vec![
2132                Field::new(
2133                    "col3",
2134                    DataType::Struct(schema_struct_float_fields.clone()),
2135                    false,
2136                )
2137                .with_metadata(HashMap::from([(
2138                    PARQUET_FIELD_ID_META_KEY.to_string(),
2139                    "3".to_string(),
2140                )])),
2141                Field::new(
2142                    "col5",
2143                    DataType::Struct(schema_struct_nested_fields.clone()),
2144                    false,
2145                )
2146                .with_metadata(HashMap::from([(
2147                    PARQUET_FIELD_ID_META_KEY.to_string(),
2148                    "5".to_string(),
2149                )])),
2150            ];
2151            Arc::new(arrow_schema::Schema::new(fields))
2152        };
2153
2154        let float_32_col = Arc::new(Float32Array::from_iter_values_with_nulls(
2155            [1.0_f32, f32::NAN, 2.0, 2.0].into_iter(),
2156            None,
2157        )) as ArrayRef;
2158
2159        let struct_float_field_col = Arc::new(StructArray::new(
2160            schema_struct_float_fields,
2161            vec![float_32_col.clone()],
2162            None,
2163        )) as ArrayRef;
2164
2165        let struct_nested_float_field_col = Arc::new(StructArray::new(
2166            schema_struct_nested_fields,
2167            vec![Arc::new(StructArray::new(
2168                schema_struct_nested_float_fields,
2169                vec![float_32_col.clone()],
2170                None,
2171            )) as ArrayRef],
2172            None,
2173        )) as ArrayRef;
2174
2175        let to_write = RecordBatch::try_new(arrow_schema.clone(), vec![
2176            struct_float_field_col,
2177            struct_nested_float_field_col,
2178        ])
2179        .unwrap();
2180        let output_file = file_io.new_output(
2181            location_gen.generate_location(None, &file_name_gen.generate_file_name()),
2182        )?;
2183
2184        // write data
2185        let mut pw = ParquetWriterBuilder::new(
2186            WriterProperties::builder().build(),
2187            Arc::new(to_write.schema().as_ref().try_into().unwrap()),
2188        )
2189        .build(output_file)
2190        .await?;
2191
2192        pw.write(&to_write).await?;
2193        let res = pw.close().await?;
2194        assert_eq!(res.len(), 1);
2195        let data_file = res
2196            .into_iter()
2197            .next()
2198            .unwrap()
2199            // Put dummy field for build successfully.
2200            .content(DataContentType::Data)
2201            .partition(Struct::empty())
2202            .partition_spec_id(0)
2203            .build()
2204            .unwrap();
2205
2206        // check data file
2207        assert_eq!(data_file.record_count(), 4);
2208        assert_eq!(*data_file.value_counts(), HashMap::from([(4, 4), (7, 4)]));
2209        assert_eq!(
2210            *data_file.lower_bounds(),
2211            HashMap::from([(4, Datum::float(1.0_f32)), (7, Datum::float(1.0_f32)),])
2212        );
2213        assert_eq!(
2214            *data_file.upper_bounds(),
2215            HashMap::from([(4, Datum::float(2.0_f32)), (7, Datum::float(2.0_f32)),])
2216        );
2217        assert_eq!(
2218            *data_file.null_value_counts(),
2219            HashMap::from([(4, 0), (7, 0)])
2220        );
2221        assert_eq!(
2222            *data_file.nan_value_counts(),
2223            HashMap::from([(4, 1), (7, 1)])
2224        );
2225
2226        // check the written file
2227        let expect_batch = concat_batches(&arrow_schema, vec![&to_write]).unwrap();
2228        check_parquet_data_file(&file_io, &data_file, &expect_batch).await;
2229
2230        Ok(())
2231    }
2232
2233    #[tokio::test]
2234    async fn test_nan_val_cnts_list_type() -> Result<()> {
2235        let temp_dir = TempDir::new().unwrap();
2236        let file_io = FileIO::new_with_fs();
2237        let location_gen = DefaultLocationGenerator::with_data_location(
2238            temp_dir.path().to_str().unwrap().to_string(),
2239        );
2240        let file_name_gen =
2241            DefaultFileNameGenerator::new("test".to_string(), None, DataFileFormat::Parquet);
2242
2243        let schema_list_float_field = Field::new("element", DataType::Float32, true).with_metadata(
2244            HashMap::from([(PARQUET_FIELD_ID_META_KEY.to_string(), "1".to_string())]),
2245        );
2246
2247        let schema_struct_list_float_field = Field::new("element", DataType::Float32, true)
2248            .with_metadata(HashMap::from([(
2249                PARQUET_FIELD_ID_META_KEY.to_string(),
2250                "4".to_string(),
2251            )]));
2252
2253        let schema_struct_list_field = Fields::from(vec![
2254            Field::new_list("col2", schema_struct_list_float_field.clone(), true).with_metadata(
2255                HashMap::from([(PARQUET_FIELD_ID_META_KEY.to_string(), "3".to_string())]),
2256            ),
2257        ]);
2258
2259        let arrow_schema = {
2260            let fields = vec![
2261                Field::new_list("col0", schema_list_float_field.clone(), true).with_metadata(
2262                    HashMap::from([(PARQUET_FIELD_ID_META_KEY.to_string(), "0".to_string())]),
2263                ),
2264                Field::new_struct("col1", schema_struct_list_field.clone(), true)
2265                    .with_metadata(HashMap::from([(
2266                        PARQUET_FIELD_ID_META_KEY.to_string(),
2267                        "2".to_string(),
2268                    )]))
2269                    .clone(),
2270                // Field::new_large_list("col3", schema_large_list_float_field.clone(), true).with_metadata(
2271                //     HashMap::from([(PARQUET_FIELD_ID_META_KEY.to_string(), "5".to_string())]),
2272                // ).clone(),
2273            ];
2274            Arc::new(arrow_schema::Schema::new(fields))
2275        };
2276
2277        let list_parts = ListArray::from_iter_primitive::<Float32Type, _, _>(vec![Some(vec![
2278            Some(1.0_f32),
2279            Some(f32::NAN),
2280            Some(2.0),
2281            Some(2.0),
2282        ])])
2283        .into_parts();
2284
2285        let list_float_field_col = Arc::new({
2286            let list_parts = list_parts.clone();
2287            ListArray::new(
2288                {
2289                    if let DataType::List(field) = arrow_schema.field(0).data_type() {
2290                        field.clone()
2291                    } else {
2292                        unreachable!()
2293                    }
2294                },
2295                list_parts.1,
2296                list_parts.2,
2297                list_parts.3,
2298            )
2299        }) as ArrayRef;
2300
2301        let struct_list_fields_schema =
2302            if let DataType::Struct(fields) = arrow_schema.field(1).data_type() {
2303                fields.clone()
2304            } else {
2305                unreachable!()
2306            };
2307
2308        let struct_list_float_field_col = Arc::new({
2309            ListArray::new(
2310                {
2311                    if let DataType::List(field) = struct_list_fields_schema
2312                        .first()
2313                        .expect("could not find first list field")
2314                        .data_type()
2315                    {
2316                        field.clone()
2317                    } else {
2318                        unreachable!()
2319                    }
2320                },
2321                list_parts.1,
2322                list_parts.2,
2323                list_parts.3,
2324            )
2325        }) as ArrayRef;
2326
2327        let struct_list_float_field_col = Arc::new(StructArray::new(
2328            struct_list_fields_schema,
2329            vec![struct_list_float_field_col.clone()],
2330            None,
2331        )) as ArrayRef;
2332
2333        let to_write = RecordBatch::try_new(arrow_schema.clone(), vec![
2334            list_float_field_col,
2335            struct_list_float_field_col,
2336            // large_list_float_field_col,
2337        ])
2338        .expect("Could not form record batch");
2339        let output_file = file_io.new_output(
2340            location_gen.generate_location(None, &file_name_gen.generate_file_name()),
2341        )?;
2342
2343        // write data
2344        let mut pw = ParquetWriterBuilder::new(
2345            WriterProperties::builder().build(),
2346            Arc::new(
2347                to_write
2348                    .schema()
2349                    .as_ref()
2350                    .try_into()
2351                    .expect("Could not convert iceberg schema"),
2352            ),
2353        )
2354        .build(output_file)
2355        .await?;
2356
2357        pw.write(&to_write).await?;
2358        let res = pw.close().await?;
2359        assert_eq!(res.len(), 1);
2360        let data_file = res
2361            .into_iter()
2362            .next()
2363            .unwrap()
2364            .content(DataContentType::Data)
2365            .partition(Struct::empty())
2366            .partition_spec_id(0)
2367            .build()
2368            .unwrap();
2369
2370        // check data file
2371        assert_eq!(data_file.record_count(), 1);
2372        assert_eq!(*data_file.value_counts(), HashMap::from([(1, 4), (4, 4)]));
2373        assert_eq!(
2374            *data_file.lower_bounds(),
2375            HashMap::from([(1, Datum::float(1.0_f32)), (4, Datum::float(1.0_f32))])
2376        );
2377        assert_eq!(
2378            *data_file.upper_bounds(),
2379            HashMap::from([(1, Datum::float(2.0_f32)), (4, Datum::float(2.0_f32))])
2380        );
2381        assert_eq!(
2382            *data_file.null_value_counts(),
2383            HashMap::from([(1, 0), (4, 0)])
2384        );
2385        assert_eq!(
2386            *data_file.nan_value_counts(),
2387            HashMap::from([(1, 1), (4, 1)])
2388        );
2389
2390        // check the written file
2391        let expect_batch = concat_batches(&arrow_schema, vec![&to_write]).unwrap();
2392        check_parquet_data_file(&file_io, &data_file, &expect_batch).await;
2393
2394        Ok(())
2395    }
2396
2397    macro_rules! construct_map_arr {
2398        ($map_key_field_schema:ident, $map_value_field_schema:ident) => {{
2399            let int_builder = Int32Builder::new();
2400            let float_builder = Float32Builder::with_capacity(4);
2401            let mut builder = MapBuilder::new(None, int_builder, float_builder);
2402            builder.keys().append_value(1);
2403            builder.values().append_value(1.0_f32);
2404            builder.append(true).unwrap();
2405            builder.keys().append_value(2);
2406            builder.values().append_value(f32::NAN);
2407            builder.append(true).unwrap();
2408            builder.keys().append_value(3);
2409            builder.values().append_value(2.0);
2410            builder.append(true).unwrap();
2411            builder.keys().append_value(4);
2412            builder.values().append_value(2.0);
2413            builder.append(true).unwrap();
2414            let array = builder.finish();
2415
2416            let (_field, offsets, entries, nulls, ordered) = array.into_parts();
2417            let new_struct_fields_schema =
2418                Fields::from(vec![$map_key_field_schema, $map_value_field_schema]);
2419
2420            let entries = {
2421                let (_, arrays, nulls) = entries.into_parts();
2422                StructArray::new(new_struct_fields_schema.clone(), arrays, nulls)
2423            };
2424
2425            let field = Arc::new(Field::new(
2426                DEFAULT_MAP_FIELD_NAME,
2427                DataType::Struct(new_struct_fields_schema),
2428                false,
2429            ));
2430
2431            Arc::new(MapArray::new(field, offsets, entries, nulls, ordered))
2432        }};
2433    }
2434
2435    #[tokio::test]
2436    async fn test_nan_val_cnts_map_type() -> Result<()> {
2437        let temp_dir = TempDir::new().unwrap();
2438        let file_io = FileIO::new_with_fs();
2439        let location_gen = DefaultLocationGenerator::with_data_location(
2440            temp_dir.path().to_str().unwrap().to_string(),
2441        );
2442        let file_name_gen =
2443            DefaultFileNameGenerator::new("test".to_string(), None, DataFileFormat::Parquet);
2444
2445        let map_key_field_schema =
2446            Field::new(MAP_KEY_FIELD_NAME, DataType::Int32, false).with_metadata(HashMap::from([
2447                (PARQUET_FIELD_ID_META_KEY.to_string(), "1".to_string()),
2448            ]));
2449
2450        let map_value_field_schema =
2451            Field::new(MAP_VALUE_FIELD_NAME, DataType::Float32, true).with_metadata(HashMap::from(
2452                [(PARQUET_FIELD_ID_META_KEY.to_string(), "2".to_string())],
2453            ));
2454
2455        let struct_map_key_field_schema =
2456            Field::new(MAP_KEY_FIELD_NAME, DataType::Int32, false).with_metadata(HashMap::from([
2457                (PARQUET_FIELD_ID_META_KEY.to_string(), "6".to_string()),
2458            ]));
2459
2460        let struct_map_value_field_schema =
2461            Field::new(MAP_VALUE_FIELD_NAME, DataType::Float32, true).with_metadata(HashMap::from(
2462                [(PARQUET_FIELD_ID_META_KEY.to_string(), "7".to_string())],
2463            ));
2464
2465        let schema_struct_map_field = Fields::from(vec![
2466            Field::new_map(
2467                "col3",
2468                DEFAULT_MAP_FIELD_NAME,
2469                struct_map_key_field_schema.clone(),
2470                struct_map_value_field_schema.clone(),
2471                false,
2472                false,
2473            )
2474            .with_metadata(HashMap::from([(
2475                PARQUET_FIELD_ID_META_KEY.to_string(),
2476                "5".to_string(),
2477            )])),
2478        ]);
2479
2480        let arrow_schema = {
2481            let fields = vec![
2482                Field::new_map(
2483                    "col0",
2484                    DEFAULT_MAP_FIELD_NAME,
2485                    map_key_field_schema.clone(),
2486                    map_value_field_schema.clone(),
2487                    false,
2488                    false,
2489                )
2490                .with_metadata(HashMap::from([(
2491                    PARQUET_FIELD_ID_META_KEY.to_string(),
2492                    "0".to_string(),
2493                )])),
2494                Field::new_struct("col1", schema_struct_map_field.clone(), true)
2495                    .with_metadata(HashMap::from([(
2496                        PARQUET_FIELD_ID_META_KEY.to_string(),
2497                        "3".to_string(),
2498                    )]))
2499                    .clone(),
2500            ];
2501            Arc::new(arrow_schema::Schema::new(fields))
2502        };
2503
2504        let map_array = construct_map_arr!(map_key_field_schema, map_value_field_schema);
2505
2506        let struct_map_arr =
2507            construct_map_arr!(struct_map_key_field_schema, struct_map_value_field_schema);
2508
2509        let struct_list_float_field_col = Arc::new(StructArray::new(
2510            schema_struct_map_field,
2511            vec![struct_map_arr],
2512            None,
2513        )) as ArrayRef;
2514
2515        let to_write = RecordBatch::try_new(arrow_schema.clone(), vec![
2516            map_array,
2517            struct_list_float_field_col,
2518        ])
2519        .expect("Could not form record batch");
2520        let output_file = file_io.new_output(
2521            location_gen.generate_location(None, &file_name_gen.generate_file_name()),
2522        )?;
2523
2524        // write data
2525        let mut pw = ParquetWriterBuilder::new(
2526            WriterProperties::builder().build(),
2527            Arc::new(
2528                to_write
2529                    .schema()
2530                    .as_ref()
2531                    .try_into()
2532                    .expect("Could not convert iceberg schema"),
2533            ),
2534        )
2535        .build(output_file)
2536        .await?;
2537
2538        pw.write(&to_write).await?;
2539        let res = pw.close().await?;
2540        assert_eq!(res.len(), 1);
2541        let data_file = res
2542            .into_iter()
2543            .next()
2544            .unwrap()
2545            .content(DataContentType::Data)
2546            .partition(Struct::empty())
2547            .partition_spec_id(0)
2548            .build()
2549            .unwrap();
2550
2551        // check data file
2552        assert_eq!(data_file.record_count(), 4);
2553        assert_eq!(
2554            *data_file.value_counts(),
2555            HashMap::from([(1, 4), (2, 4), (6, 4), (7, 4)])
2556        );
2557        assert_eq!(
2558            *data_file.lower_bounds(),
2559            HashMap::from([
2560                (1, Datum::int(1)),
2561                (2, Datum::float(1.0_f32)),
2562                (6, Datum::int(1)),
2563                (7, Datum::float(1.0_f32))
2564            ])
2565        );
2566        assert_eq!(
2567            *data_file.upper_bounds(),
2568            HashMap::from([
2569                (1, Datum::int(4)),
2570                (2, Datum::float(2.0_f32)),
2571                (6, Datum::int(4)),
2572                (7, Datum::float(2.0_f32))
2573            ])
2574        );
2575        assert_eq!(
2576            *data_file.null_value_counts(),
2577            HashMap::from([(1, 0), (2, 0), (6, 0), (7, 0)])
2578        );
2579        assert_eq!(
2580            *data_file.nan_value_counts(),
2581            HashMap::from([(2, 1), (7, 1)])
2582        );
2583
2584        // check the written file
2585        let expect_batch = concat_batches(&arrow_schema, vec![&to_write]).unwrap();
2586        check_parquet_data_file(&file_io, &data_file, &expect_batch).await;
2587
2588        Ok(())
2589    }
2590
2591    #[tokio::test]
2592    async fn test_write_empty_parquet_file() {
2593        let temp_dir = TempDir::new().unwrap();
2594        let file_io = FileIO::new_with_fs();
2595        let location_gen = DefaultLocationGenerator::with_data_location(
2596            temp_dir.path().to_str().unwrap().to_string(),
2597        );
2598        let file_name_gen =
2599            DefaultFileNameGenerator::new("test".to_string(), None, DataFileFormat::Parquet);
2600        let output_file = file_io
2601            .new_output(location_gen.generate_location(None, &file_name_gen.generate_file_name()))
2602            .unwrap();
2603
2604        // write data
2605        let pw = ParquetWriterBuilder::new(
2606            WriterProperties::builder().build(),
2607            Arc::new(
2608                Schema::builder()
2609                    .with_schema_id(1)
2610                    .with_fields(vec![
2611                        NestedField::required(0, "col", Type::Primitive(PrimitiveType::Long))
2612                            .with_id(0)
2613                            .into(),
2614                    ])
2615                    .build()
2616                    .expect("Failed to create schema"),
2617            ),
2618        )
2619        .build(output_file)
2620        .await
2621        .unwrap();
2622
2623        let res = pw.close().await.unwrap();
2624        assert_eq!(res.len(), 0);
2625
2626        // Check that file should have been deleted.
2627        assert_eq!(std::fs::read_dir(temp_dir.path()).unwrap().count(), 0);
2628    }
2629
2630    #[tokio::test]
2631    async fn test_parquet_writer_uuid_logical_type() {
2632        let temp_dir = TempDir::new().unwrap();
2633        let file_io = FileIO::new_with_fs();
2634        let location_gen = DefaultLocationGenerator::with_data_location(
2635            temp_dir.path().to_str().unwrap().to_string(),
2636        );
2637        let file_name_gen =
2638            DefaultFileNameGenerator::new("test".to_string(), None, DataFileFormat::Parquet);
2639        let output_file = file_io
2640            .new_output(location_gen.generate_location(None, &file_name_gen.generate_file_name()))
2641            .unwrap();
2642
2643        let schema = Arc::new(
2644            Schema::builder()
2645                .with_schema_id(1)
2646                .with_fields(vec![
2647                    NestedField::required(0, "uuid", Type::Primitive(PrimitiveType::Uuid)).into(),
2648                ])
2649                .build()
2650                .unwrap(),
2651        );
2652        let arrow_schema: ArrowSchemaRef = Arc::new(schema_to_arrow_schema(&schema).unwrap());
2653        let col = Arc::new(
2654            arrow_array::FixedSizeBinaryArray::try_from_iter(
2655                [Uuid::from_u128(0), Uuid::from_u128(1)]
2656                    .into_iter()
2657                    .map(|uuid| uuid.into_bytes()),
2658            )
2659            .unwrap(),
2660        ) as ArrayRef;
2661        let to_write = RecordBatch::try_new(arrow_schema, vec![col]).unwrap();
2662
2663        let mut pw = ParquetWriterBuilder::new(WriterProperties::builder().build(), schema)
2664            .build(output_file)
2665            .await
2666            .unwrap();
2667        pw.write(&to_write).await.unwrap();
2668        let data_file = pw
2669            .close()
2670            .await
2671            .unwrap()
2672            .into_iter()
2673            .next()
2674            .unwrap()
2675            .content(DataContentType::Data)
2676            .partition(Struct::empty())
2677            .partition_spec_id(0)
2678            .build()
2679            .unwrap();
2680
2681        let raw = file_io
2682            .new_input(data_file.file_path.clone())
2683            .unwrap()
2684            .read()
2685            .await
2686            .unwrap();
2687        let reader = ParquetRecordBatchReaderBuilder::try_new(raw).unwrap();
2688        let column = reader.metadata().file_metadata().schema_descr().column(0);
2689        assert_eq!(
2690            (
2691                column.physical_type(),
2692                column.type_length(),
2693                column.logical_type_ref()
2694            ),
2695            (
2696                parquet::basic::Type::FIXED_LEN_BYTE_ARRAY,
2697                16,
2698                Some(&LogicalType::Uuid)
2699            )
2700        );
2701    }
2702
2703    struct GeospatialTestFile {
2704        temp_dir: TempDir,
2705        file_io: FileIO,
2706        schema: SchemaRef,
2707        batch: RecordBatch,
2708        data_file: DataFile,
2709    }
2710
2711    async fn write_geospatial_test_file() -> Result<GeospatialTestFile> {
2712        let temp_dir = TempDir::new().unwrap();
2713        let file_io = FileIO::new_with_fs();
2714        let geom_type = PrimitiveType::Geometry(GeometryType::new(Some("srid:0".to_string()))?);
2715        let schema = Arc::new(
2716            Schema::builder()
2717                .with_fields(vec![
2718                    NestedField::required(1, "geom", Type::Primitive(geom_type)).into(),
2719                    NestedField::optional(
2720                        2,
2721                        "geog",
2722                        Type::Primitive(PrimitiveType::Geography(GeographyType::new(
2723                            None,
2724                            IcebergEdgeInterpolationAlgorithm::Karney,
2725                        )?)),
2726                    )
2727                    .into(),
2728                ])
2729                .build()?,
2730        );
2731        let geometry = [
2732            parquet_geospatial::testing::wkb_point_xyzm(1.0, 20.0, f64::NAN, 40.0),
2733            parquet_geospatial::testing::wkb_point_xyzm(-3.0, 4.0, f64::NAN, 70.0),
2734        ];
2735        let geography = wkb_point_xy(3.0, 4.0);
2736        let batch = RecordBatch::try_new(Arc::new(schema_to_arrow_schema(&schema)?), vec![
2737            Arc::new(arrow_array::LargeBinaryArray::from_iter_values(&geometry)),
2738            Arc::new(arrow_array::LargeBinaryArray::from(vec![
2739                Some(geography.as_slice()),
2740                None,
2741            ])),
2742        ])?;
2743        let output = file_io.new_output(temp_dir.path().join("geo.parquet").to_str().unwrap())?;
2744        let props = WriterProperties::builder()
2745            .set_max_row_group_row_count(Some(1))
2746            .build();
2747        let mut writer = ParquetWriterBuilder::new(props, schema.clone())
2748            .build(output)
2749            .await?;
2750        writer.write(&batch).await?;
2751        let data_file = writer
2752            .close()
2753            .await?
2754            .into_iter()
2755            .next()
2756            .unwrap()
2757            .content(DataContentType::Data)
2758            .partition(Struct::empty())
2759            .partition_spec_id(0)
2760            .build()
2761            .unwrap();
2762
2763        Ok(GeospatialTestFile {
2764            temp_dir,
2765            file_io,
2766            schema,
2767            batch,
2768            data_file,
2769        })
2770    }
2771
2772    #[tokio::test]
2773    async fn test_geospatial_parquet_roundtrip() -> Result<()> {
2774        let file = write_geospatial_test_file().await?;
2775        let input = file.file_io.new_input(file.data_file.file_path())?;
2776        let mut parquet_reader =
2777            ArrowFileReader::new(input.metadata().await?, input.reader().await?);
2778        let metadata = parquet_reader.get_metadata(None).await?;
2779        let schema_descr = metadata.file_metadata().schema_descr();
2780        assert_eq!(metadata.num_row_groups(), 2);
2781        let logical_types: Vec<_> = schema_descr
2782            .columns()
2783            .iter()
2784            .map(|column| column.logical_type_ref())
2785            .collect();
2786        let expected = [
2787            LogicalType::geometry(Some("srid:0".to_string())),
2788            LogicalType::geography(None, Some(EdgeInterpolationAlgorithm::KARNEY)),
2789        ];
2790        assert_eq!(logical_types, expected.iter().map(Some).collect::<Vec<_>>());
2791
2792        let arrow_schema = parquet::arrow::parquet_to_arrow_schema(
2793            schema_descr,
2794            metadata.file_metadata().key_value_metadata(),
2795        )?;
2796        let unset_metadata: serde_json::Value =
2797            serde_json::from_str(arrow_schema.field(0).extension_type_metadata().unwrap())?;
2798        assert!(unset_metadata.get("crs").is_none());
2799
2800        let task = FileScanTask::builder()
2801            .with_file_size_in_bytes(file.data_file.file_size_in_bytes())
2802            .with_start(0)
2803            .with_length(0)
2804            .with_data_file_path(file.data_file.file_path.clone())
2805            .with_data_file_format(DataFileFormat::Parquet)
2806            .with_schema(file.schema)
2807            .with_project_field_ids(vec![1, 2])
2808            .with_case_sensitive(false)
2809            .build()?;
2810        let tasks = Box::pin(futures::stream::iter(vec![Ok(task)])) as FileScanTaskStream;
2811        let reader = ArrowReaderBuilder::new(file.file_io, Runtime::current()).build();
2812        let batches: Vec<RecordBatch> = reader.read(tasks)?.stream().try_collect().await?;
2813        let read_back = concat_batches(batches[0].schema_ref(), &batches)?;
2814        assert_eq!(read_back.columns(), file.batch.columns());
2815        Ok(())
2816    }
2817
2818    fn coordinate_bytes(coordinates: &[f64]) -> Vec<u8> {
2819        coordinates
2820            .iter()
2821            .flat_map(|value| value.to_le_bytes())
2822            .collect()
2823    }
2824
2825    #[tokio::test]
2826    async fn test_geometry_bounds_manifest_roundtrip() -> Result<()> {
2827        let file = write_geospatial_test_file().await?;
2828        let path = file.temp_dir.path().join("manifest.avro");
2829        let output = file.file_io.new_output(path.to_str().unwrap())?;
2830        let mut writer = ManifestWriterBuilder::new(
2831            output,
2832            Some(1),
2833            file.schema.clone(),
2834            PartitionSpec::builder(file.schema).build()?,
2835        )
2836        .build_v3_data();
2837        writer.add_file(file.data_file, 1)?;
2838        writer.write_manifest_file().await?;
2839        let manifest = Manifest::parse_avro(&std::fs::read(path).unwrap())?;
2840        let data_file = manifest.entries()[0].data_file();
2841        let ty = PrimitiveType::Geometry(GeometryType::new(Some("srid:0".to_string()))?);
2842        let expected_lower = HashMap::from([(
2843            1,
2844            Datum::new(
2845                ty.clone(),
2846                PrimitiveLiteral::Binary(coordinate_bytes(&[-3.0, 4.0, f64::NAN, 40.0])),
2847            ),
2848        )]);
2849        let expected_upper = HashMap::from([(
2850            1,
2851            Datum::new(
2852                ty,
2853                PrimitiveLiteral::Binary(coordinate_bytes(&[1.0, 20.0, f64::NAN, 70.0])),
2854            ),
2855        )]);
2856        assert_eq!(
2857            (data_file.lower_bounds(), data_file.upper_bounds()),
2858            (&expected_lower, &expected_upper),
2859        );
2860        Ok(())
2861    }
2862
2863    #[tokio::test]
2864    async fn test_geometry_bounds_do_not_drive_scalar_metrics() -> Result<()> {
2865        use crate::expr::visitors::inclusive_metrics_evaluator::InclusiveMetricsEvaluator;
2866        use crate::expr::visitors::strict_metrics_evaluator::StrictMetricsEvaluator;
2867        use crate::expr::{Bind, Reference};
2868
2869        let file = write_geospatial_test_file().await?;
2870        let values = file
2871            .batch
2872            .column(0)
2873            .as_any()
2874            .downcast_ref::<arrow_array::LargeBinaryArray>()
2875            .unwrap();
2876        let literal = Datum::try_from_bytes(
2877            values.value(0),
2878            file.schema
2879                .field_by_id(1)
2880                .unwrap()
2881                .field_type
2882                .as_primitive_type()
2883                .unwrap()
2884                .clone(),
2885        )?;
2886        let inequality = Reference::new("geom")
2887            .less_than(literal.clone())
2888            .bind(file.schema.clone(), true)?;
2889        let exclusion = Reference::new("geom")
2890            .not_equal_to(literal)
2891            .bind(file.schema, true)?;
2892        assert_eq!(
2893            (
2894                InclusiveMetricsEvaluator::eval(&inequality, &file.data_file, true)?,
2895                StrictMetricsEvaluator::eval(&exclusion, &file.data_file)?,
2896            ),
2897            (true, false),
2898        );
2899        Ok(())
2900    }
2901
2902    fn geometry_bound_bytes(bbox: BoundingBox) -> (Vec<u8>, Vec<u8>) {
2903        let ty = PrimitiveType::Geometry(GeometryType::default());
2904        let (lower, upper) = GeospatialBounds::from(&bbox).into_datums(ty);
2905        (
2906            lower.to_bytes().unwrap().into_vec(),
2907            upper.to_bytes().unwrap().into_vec(),
2908        )
2909    }
2910
2911    #[test]
2912    fn test_geometry_xy_bounds() {
2913        assert_eq!(
2914            geometry_bound_bytes(BoundingBox::new(-3.0, 1.0, 4.0, 20.0)),
2915            (
2916                coordinate_bytes(&[-3.0, 4.0]),
2917                coordinate_bytes(&[1.0, 20.0])
2918            ),
2919        );
2920    }
2921
2922    #[test]
2923    fn test_geometry_xyz_bounds() {
2924        assert_eq!(
2925            geometry_bound_bytes(BoundingBox::new(-3.0, 1.0, 4.0, 20.0).with_zrange(2.0, 9.0)),
2926            (
2927                coordinate_bytes(&[-3.0, 4.0, 2.0]),
2928                coordinate_bytes(&[1.0, 20.0, 9.0])
2929            ),
2930        );
2931    }
2932
2933    #[test]
2934    fn test_wrapping_geometry_bounds_cover_all_x() {
2935        assert_eq!(
2936            geometry_bound_bytes(BoundingBox::new(170.0, -170.0, -10.0, 10.0)),
2937            (
2938                coordinate_bytes(&[f64::NEG_INFINITY, -10.0]),
2939                coordinate_bytes(&[f64::INFINITY, 10.0]),
2940            ),
2941        );
2942    }
2943
2944    fn geospatial_stats_fixture(ty: PrimitiveType) -> (SchemaRef, ColumnChunkMetaData) {
2945        let schema = Arc::new(
2946            Schema::builder()
2947                .with_fields(vec![
2948                    NestedField::optional(1, "geo", Type::Primitive(ty)).into(),
2949                ])
2950                .build()
2951                .unwrap(),
2952        );
2953        let parquet_schema = parquet::arrow::ArrowSchemaConverter::new()
2954            .convert(&schema_to_arrow_schema(&schema).unwrap())
2955            .unwrap();
2956        let column = ColumnChunkMetaData::builder(parquet_schema.column(0))
2957            .set_num_values(1)
2958            .build()
2959            .unwrap();
2960        (schema, column)
2961    }
2962
2963    fn with_geo_box(column: &ColumnChunkMetaData, bbox: BoundingBox) -> ColumnChunkMetaData {
2964        use parquet::geospatial::statistics::GeospatialStatistics;
2965        column
2966            .clone()
2967            .into_builder()
2968            .set_geo_statistics(Box::new(GeospatialStatistics::new(Some(bbox), None)))
2969            .build()
2970            .unwrap()
2971    }
2972
2973    #[test]
2974    fn test_geometry_bounds_merge_across_null_row_group() -> Result<()> {
2975        let (schema, column) =
2976            geospatial_stats_fixture(PrimitiveType::Geometry(GeometryType::default()));
2977        let first = with_geo_box(
2978            &column,
2979            BoundingBox::new(-3.0, 1.0, 4.0, 20.0).with_zrange(4.0, 9.0),
2980        );
2981        let second = with_geo_box(
2982            &column,
2983            BoundingBox::new(0.0, 5.0, -20.0, 5.0).with_mrange(1.0, 3.0),
2984        );
2985        let nulls = column
2986            .into_builder()
2987            .set_statistics(Statistics::ByteArray(ValueStatistics::new(
2988                None,
2989                None,
2990                None,
2991                Some(1),
2992                false,
2993            )))
2994            .build()?;
2995        let mut bounds = MinMaxColAggregator::new(schema);
2996        for column in [&first, &nulls, &second] {
2997            bounds.update(1, column)?;
2998        }
2999        let (lower, upper) = bounds.produce();
3000        assert_eq!(
3001            (
3002                lower[&1].to_bytes()?.into_vec(),
3003                upper[&1].to_bytes()?.into_vec()
3004            ),
3005            (
3006                coordinate_bytes(&[-3.0, -20.0, 4.0, 1.0]),
3007                coordinate_bytes(&[5.0, 20.0, 9.0, 3.0])
3008            ),
3009        );
3010        Ok(())
3011    }
3012
3013    #[test]
3014    fn test_geometry_bounds_require_all_non_null_row_groups() -> Result<()> {
3015        let (schema, unknown) =
3016            geospatial_stats_fixture(PrimitiveType::Geometry(GeometryType::default()));
3017        let known = with_geo_box(&unknown, BoundingBox::new(1.0, 2.0, 3.0, 4.0));
3018        let mut bounds = MinMaxColAggregator::new(schema);
3019        for column in [&known, &unknown, &known] {
3020            bounds.update(1, column)?;
3021        }
3022        assert_eq!(bounds.produce(), (HashMap::new(), HashMap::new()));
3023        Ok(())
3024    }
3025
3026    #[test]
3027    fn test_geography_bounds_merge_from_parquet_statistics() -> Result<()> {
3028        let (schema, column) =
3029            geospatial_stats_fixture(PrimitiveType::Geography(GeographyType::default()));
3030        let first = with_geo_box(&column, BoundingBox::new(170.0, -170.0, -20.0, 20.0));
3031        let second = with_geo_box(&column, BoundingBox::new(175.0, 178.0, -10.0, 30.0));
3032        let mut bounds = MinMaxColAggregator::new(schema);
3033        bounds.update(1, &first)?;
3034        bounds.update(1, &second)?;
3035        let (lower, upper) = bounds.produce();
3036        assert_eq!(
3037            (
3038                lower[&1].to_bytes()?.into_vec(),
3039                upper[&1].to_bytes()?.into_vec()
3040            ),
3041            (
3042                coordinate_bytes(&[170.0, -20.0]),
3043                coordinate_bytes(&[-170.0, 30.0])
3044            ),
3045        );
3046        Ok(())
3047    }
3048
3049    #[test]
3050    fn test_geography_bounds_full_longitude_range() -> Result<()> {
3051        let (schema, column) =
3052            geospatial_stats_fixture(PrimitiveType::Geography(GeographyType::default()));
3053        let first = with_geo_box(&column, BoundingBox::new(170.0, -170.0, -20.0, 20.0));
3054        let second = with_geo_box(&column, BoundingBox::new(-175.0, 175.0, -10.0, 30.0));
3055        let mut bounds = MinMaxColAggregator::new(schema);
3056        bounds.update(1, &first)?;
3057        bounds.update(1, &second)?;
3058        let (lower, upper) = bounds.produce();
3059        assert_eq!(
3060            (
3061                lower[&1].to_bytes()?.into_vec(),
3062                upper[&1].to_bytes()?.into_vec()
3063            ),
3064            (
3065                coordinate_bytes(&[-180.0, -20.0]),
3066                coordinate_bytes(&[180.0, 30.0])
3067            ),
3068        );
3069        Ok(())
3070    }
3071
3072    #[test]
3073    fn test_min_max_aggregator() {
3074        let schema = Arc::new(
3075            Schema::builder()
3076                .with_schema_id(1)
3077                .with_fields(vec![
3078                    NestedField::required(0, "col", Type::Primitive(PrimitiveType::Int))
3079                        .with_id(0)
3080                        .into(),
3081                ])
3082                .build()
3083                .expect("Failed to create schema"),
3084        );
3085        let parquet_schema = parquet::arrow::ArrowSchemaConverter::new()
3086            .convert(&schema_to_arrow_schema(&schema).unwrap())
3087            .unwrap();
3088        let mut min_max_agg = MinMaxColAggregator::new(schema);
3089        let create_statistics = |min, max| {
3090            ColumnChunkMetaData::builder(parquet_schema.column(0))
3091                .set_statistics(Statistics::Int32(ValueStatistics::new(
3092                    min, max, None, None, false,
3093                )))
3094                .build()
3095                .unwrap()
3096        };
3097        min_max_agg
3098            .update(0, &create_statistics(None, Some(42)))
3099            .unwrap();
3100        min_max_agg
3101            .update(0, &create_statistics(Some(0), Some(i32::MAX)))
3102            .unwrap();
3103        min_max_agg
3104            .update(0, &create_statistics(Some(i32::MIN), None))
3105            .unwrap();
3106        min_max_agg
3107            .update(0, &create_statistics(None, None))
3108            .unwrap();
3109
3110        let (lower_bounds, upper_bounds) = min_max_agg.produce();
3111
3112        assert_eq!(lower_bounds, HashMap::from([(0, Datum::int(i32::MIN))]));
3113        assert_eq!(upper_bounds, HashMap::from([(0, Datum::int(i32::MAX))]));
3114    }
3115
3116    #[test]
3117    fn test_min_max_aggregator_skips_nan_bounds() {
3118        let schema = Arc::new(
3119            Schema::builder()
3120                .with_schema_id(1)
3121                .with_fields(vec![
3122                    NestedField::required(0, "f", Type::Primitive(PrimitiveType::Float)).into(),
3123                    NestedField::required(1, "d", Type::Primitive(PrimitiveType::Double)).into(),
3124                ])
3125                .build()
3126                .expect("Failed to create schema"),
3127        );
3128        let parquet_schema = parquet::arrow::ArrowSchemaConverter::new()
3129            .convert(&schema_to_arrow_schema(&schema).unwrap())
3130            .unwrap();
3131        let mut min_max_agg = MinMaxColAggregator::new(schema);
3132        let float_stats = |min, max| {
3133            ColumnChunkMetaData::builder(parquet_schema.column(0))
3134                .set_statistics(Statistics::Float(ValueStatistics::new(
3135                    min, max, None, None, false,
3136                )))
3137                .build()
3138                .unwrap()
3139        };
3140        let double_stats = |min, max| {
3141            ColumnChunkMetaData::builder(parquet_schema.column(1))
3142                .set_statistics(Statistics::Double(ValueStatistics::new(
3143                    min, max, None, None, false,
3144                )))
3145                .build()
3146                .unwrap()
3147        };
3148
3149        min_max_agg
3150            .update(0, &float_stats(Some(-f32::NAN), Some(f32::NAN)))
3151            .unwrap();
3152        min_max_agg
3153            .update(0, &float_stats(Some(1.5), Some(2.5)))
3154            .unwrap();
3155        min_max_agg
3156            .update(1, &double_stats(Some(f64::NAN), Some(f64::NAN)))
3157            .unwrap();
3158
3159        let (lower_bounds, upper_bounds) = min_max_agg.produce();
3160
3161        assert_eq!(lower_bounds, HashMap::from([(0, Datum::float(1.5_f32))]));
3162        assert_eq!(upper_bounds, HashMap::from([(0, Datum::float(2.5_f32))]));
3163    }
3164
3165    fn cdc_test_schema() -> SchemaRef {
3166        Arc::new(
3167            Schema::builder()
3168                .with_schema_id(1)
3169                .with_fields(vec![
3170                    NestedField::required(1, "id", Type::Primitive(PrimitiveType::Long)).into(),
3171                    NestedField::required(2, "payload", Type::Primitive(PrimitiveType::String))
3172                        .into(),
3173                ])
3174                .build()
3175                .unwrap(),
3176        )
3177    }
3178
3179    #[test]
3180    fn test_from_table_properties_no_cdc_by_default() {
3181        let raw_properties = HashMap::new();
3182        let tp = TableProperties::new(&raw_properties);
3183        let builder = ParquetWriterBuilder::from_table_properties(&tp, cdc_test_schema()).unwrap();
3184        assert!(builder.props.content_defined_chunking().is_none());
3185    }
3186
3187    #[tokio::test]
3188    async fn test_from_table_properties_without_encryption_writes_plaintext() {
3189        let raw_properties = HashMap::new();
3190        let tp = TableProperties::new(&raw_properties);
3191        let tmp = TempDir::new().unwrap();
3192        let output = FileIO::new_with_fs()
3193            .new_output(format!("{}/plain.parquet", tmp.path().to_str().unwrap()))
3194            .unwrap();
3195        let writer = ParquetWriterBuilder::from_table_properties(&tp, cdc_test_schema())
3196            .unwrap()
3197            .build(output)
3198            .await
3199            .unwrap();
3200
3201        assert!(writer.key_metadata.is_none());
3202        assert!(
3203            writer
3204                .writer_properties
3205                .file_encryption_properties()
3206                .is_none()
3207        );
3208    }
3209
3210    #[tokio::test]
3211    async fn test_from_table_properties_propagate_to_writer() {
3212        // `build()` must carry the translated `WriterProperties` through to the
3213        // `ParquetWriter` unchanged — otherwise the `write.parquet.*` settings
3214        // derived in `from_table_properties` would never reach parquet-rs.
3215        //
3216        // Asserting on the writer's `WriterProperties` (rather than re-reading a
3217        // written file) keeps this a direct propagation check: every future
3218        // `write.parquet.*` option just adds an assertion on its corresponding
3219        // `WriterProperties` getter here.
3220        let raw_properties = HashMap::from([
3221            (
3222                TableProperties::PROPERTY_PARQUET_CDC_ENABLED.to_string(),
3223                "true".to_string(),
3224            ),
3225            (
3226                TableProperties::PROPERTY_PARQUET_CDC_MIN_CHUNK_SIZE.to_string(),
3227                "4096".to_string(),
3228            ),
3229            (
3230                TableProperties::PROPERTY_PARQUET_CDC_MAX_CHUNK_SIZE.to_string(),
3231                "8192".to_string(),
3232            ),
3233            (
3234                TableProperties::PROPERTY_PARQUET_CDC_NORM_LEVEL.to_string(),
3235                "2".to_string(),
3236            ),
3237        ]);
3238        let tp = TableProperties::new(&raw_properties);
3239
3240        let tmp = TempDir::new().unwrap();
3241        let output = FileIO::new_with_fs()
3242            .new_output(format!("{}/cdc.parquet", tmp.path().to_str().unwrap()))
3243            .unwrap();
3244        let writer = ParquetWriterBuilder::from_table_properties(&tp, cdc_test_schema())
3245            .unwrap()
3246            .build(output)
3247            .await
3248            .unwrap();
3249
3250        let cdc = writer
3251            .writer_properties
3252            .content_defined_chunking()
3253            .copied()
3254            .expect("CDC should be enabled on the built writer");
3255        assert_eq!(cdc.min_chunk_size, 4096);
3256        assert_eq!(cdc.max_chunk_size, 8192);
3257        assert_eq!(cdc.norm_level, 2);
3258    }
3259
3260    #[test]
3261    fn test_from_table_properties_sizing_defaults() {
3262        // With no properties set, the writer must use Iceberg's defaults (which
3263        // differ from parquet-rs's own defaults), not parquet-rs's.
3264        let raw_properties = HashMap::new();
3265        let tp = TableProperties::new(&raw_properties);
3266        let props = ParquetWriterBuilder::from_table_properties(&tp, cdc_test_schema())
3267            .unwrap()
3268            .props;
3269
3270        assert_eq!(
3271            props.max_row_group_bytes(),
3272            Some(TableProperties::PROPERTY_PARQUET_ROW_GROUP_SIZE_BYTES_DEFAULT)
3273        );
3274        assert_eq!(
3275            props.data_page_size_limit(),
3276            TableProperties::PROPERTY_PARQUET_PAGE_SIZE_BYTES_DEFAULT
3277        );
3278        assert_eq!(
3279            props.data_page_row_count_limit(),
3280            TableProperties::PROPERTY_PARQUET_PAGE_ROW_LIMIT_DEFAULT
3281        );
3282        assert_eq!(
3283            props.dictionary_page_size_limit(),
3284            TableProperties::PROPERTY_PARQUET_DICT_SIZE_BYTES_DEFAULT
3285        );
3286        // Default codec is zstd at the Java-aligned default level (3).
3287        assert_eq!(
3288            props.compression(&ColumnPath::from("id")),
3289            Compression::ZSTD(ZstdLevel::try_new(3).unwrap())
3290        );
3291    }
3292
3293    #[test]
3294    fn test_from_table_properties_sizing_and_compression_overrides() {
3295        let raw_properties = HashMap::from([
3296            (
3297                TableProperties::PROPERTY_PARQUET_ROW_GROUP_SIZE_BYTES.to_string(),
3298                "1048576".to_string(),
3299            ),
3300            (
3301                TableProperties::PROPERTY_PARQUET_PAGE_SIZE_BYTES.to_string(),
3302                "65536".to_string(),
3303            ),
3304            (
3305                TableProperties::PROPERTY_PARQUET_PAGE_ROW_LIMIT.to_string(),
3306                "5000".to_string(),
3307            ),
3308            (
3309                TableProperties::PROPERTY_PARQUET_DICT_SIZE_BYTES.to_string(),
3310                "131072".to_string(),
3311            ),
3312            (
3313                TableProperties::PROPERTY_PARQUET_COMPRESSION_CODEC.to_string(),
3314                "gzip".to_string(),
3315            ),
3316            (
3317                TableProperties::PROPERTY_PARQUET_COMPRESSION_LEVEL.to_string(),
3318                "9".to_string(),
3319            ),
3320        ]);
3321        let tp = TableProperties::new(&raw_properties);
3322        let props = ParquetWriterBuilder::from_table_properties(&tp, cdc_test_schema())
3323            .unwrap()
3324            .props;
3325
3326        assert_eq!(props.max_row_group_bytes(), Some(1048576));
3327        assert_eq!(props.data_page_size_limit(), 65536);
3328        assert_eq!(props.data_page_row_count_limit(), 5000);
3329        assert_eq!(props.dictionary_page_size_limit(), 131072);
3330        assert_eq!(
3331            props.compression(&ColumnPath::from("id")),
3332            Compression::GZIP(GzipLevel::try_new(9).unwrap())
3333        );
3334    }
3335
3336    #[test]
3337    fn test_from_table_properties_invalid_codec_errors() {
3338        let entries = HashMap::from([(
3339            TableProperties::PROPERTY_PARQUET_COMPRESSION_CODEC.to_string(),
3340            "bogus".to_string(),
3341        )]);
3342        let err = ParquetWriterBuilder::from_table_properties(
3343            &TableProperties::new(&entries),
3344            cdc_test_schema(),
3345        )
3346        .unwrap_err();
3347        assert_eq!(err.kind(), ErrorKind::DataInvalid);
3348        assert!(err.to_string().contains("bogus"));
3349    }
3350
3351    #[test]
3352    fn test_parquet_compression_mapping() {
3353        // Codecs without a level.
3354        assert_eq!(
3355            parquet_compression(CompressionCodec::None).unwrap(),
3356            Compression::UNCOMPRESSED
3357        );
3358        assert_eq!(
3359            parquet_compression(CompressionCodec::Snappy).unwrap(),
3360            Compression::SNAPPY
3361        );
3362        assert_eq!(
3363            parquet_compression(CompressionCodec::Lz4).unwrap(),
3364            Compression::LZ4
3365        );
3366        assert_eq!(
3367            parquet_compression(CompressionCodec::Lz4Raw).unwrap(),
3368            Compression::LZ4_RAW
3369        );
3370        assert_eq!(
3371            parquet_compression(CompressionCodec::Lzo).unwrap(),
3372            Compression::LZO
3373        );
3374
3375        // Level-carrying codecs at their defaults.
3376        assert_eq!(
3377            parquet_compression(CompressionCodec::zstd_default()).unwrap(),
3378            Compression::ZSTD(ZstdLevel::try_new(3).unwrap())
3379        );
3380        assert_eq!(
3381            parquet_compression(CompressionCodec::gzip_default()).unwrap(),
3382            Compression::GZIP(GzipLevel::try_new(6).unwrap())
3383        );
3384        assert_eq!(
3385            parquet_compression(CompressionCodec::brotli_default()).unwrap(),
3386            Compression::BROTLI(BrotliLevel::try_new(1).unwrap())
3387        );
3388
3389        // Explicit levels are honored.
3390        assert_eq!(
3391            parquet_compression(CompressionCodec::Zstd(10)).unwrap(),
3392            Compression::ZSTD(ZstdLevel::try_new(10).unwrap())
3393        );
3394    }
3395
3396    #[test]
3397    fn test_parquet_compression_invalid_level() {
3398        // zstd valid range is 1..=22; 99 is out of range.
3399        let err = parquet_compression(CompressionCodec::Zstd(99)).unwrap_err();
3400        assert_eq!(err.kind(), ErrorKind::DataInvalid);
3401        assert!(err.to_string().contains("zstd"));
3402    }
3403}