Skip to main content

iceberg/arrow/
schema.rs

1// Licensed to the Apache Software Foundation (ASF) under one
2// or more contributor license agreements.  See the NOTICE file
3// distributed with this work for additional information
4// regarding copyright ownership.  The ASF licenses this file
5// to you under the Apache License, Version 2.0 (the
6// "License"); you may not use this file except in compliance
7// with the License.  You may obtain a copy of the License at
8//
9//   http://www.apache.org/licenses/LICENSE-2.0
10//
11// Unless required by applicable law or agreed to in writing,
12// software distributed under the License is distributed on an
13// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14// KIND, either express or implied.  See the License for the
15// specific language governing permissions and limitations
16// under the License.
17
18//! Conversion between Arrow schema and Iceberg schema.
19
20use std::collections::HashMap;
21use std::sync::Arc;
22
23use arrow_array::types::{Decimal128Type, validate_decimal_precision_and_scale};
24use arrow_array::{
25    BinaryArray, BooleanArray, Date32Array, Datum as ArrowDatum, Decimal128Array,
26    FixedSizeBinaryArray, Float32Array, Float64Array, Int32Array, Int64Array, Scalar, StringArray,
27    TimestampMicrosecondArray, TimestampNanosecondArray,
28};
29use arrow_schema::extension::{ExtensionType, Uuid as UuidExtensionType};
30use arrow_schema::{
31    ArrowError, DataType, Field, FieldRef, Fields, Schema as ArrowSchema, TimeUnit,
32};
33use parquet::arrow::PARQUET_FIELD_ID_META_KEY;
34use parquet::file::statistics::Statistics;
35use parquet_geospatial::{WkbEdges, WkbMetadata, WkbType, WkbTypeHint};
36use uuid::Uuid;
37
38use crate::error::{Result, invalid_data};
39use crate::spec::decimal_utils::i128_from_be_bytes;
40use crate::spec::geospatial::EdgeInterpolationAlgorithm;
41use crate::spec::{
42    Datum, FIRST_FIELD_ID, GeographyType, GeometryType, LIST_FIELD_NAME, ListType,
43    MAP_KEY_FIELD_NAME, MAP_VALUE_FIELD_NAME, MapType, NestedField, NestedFieldRef,
44    PrimitiveLiteral, PrimitiveType, Schema, SchemaVisitor, StructType, Type, VariantType,
45};
46use crate::{Error, ErrorKind};
47
48/// When iceberg map type convert to Arrow map type, the default map field name is "key_value".
49pub const DEFAULT_MAP_FIELD_NAME: &str = "key_value";
50/// UTC time zone for Arrow timestamp type.
51pub const UTC_TIME_ZONE: &str = "+00:00";
52const UNSET_GEOSPATIAL_CRS: &str = "srid:0";
53
54/// The canonical Arrow [`arrow.parquet.variant`] extension type.
55///
56/// Iceberg stores a Variant as a `Struct { metadata: Binary, value: Binary }`. Attaching this
57/// extension type to the enclosing field marks that struct as a single logical Variant value,
58/// so Arrow consumers treat it as a Variant rather than an anonymous struct. It carries no
59/// metadata.
60///
61/// [`arrow.parquet.variant`]: https://arrow.apache.org/docs/format/CanonicalExtensions.html#parquet-variant
62#[derive(Debug, Clone, Copy, Default)]
63pub(crate) struct VariantExtensionType;
64
65impl ExtensionType for VariantExtensionType {
66    const NAME: &'static str = "arrow.parquet.variant";
67
68    type Metadata = ();
69
70    fn metadata(&self) -> &Self::Metadata {
71        &()
72    }
73
74    fn serialize_metadata(&self) -> Option<String> {
75        None
76    }
77
78    fn deserialize_metadata(
79        metadata: Option<&str>,
80    ) -> std::result::Result<Self::Metadata, ArrowError> {
81        match metadata {
82            None | Some("") => Ok(()),
83            Some(other) => Err(ArrowError::InvalidArgumentError(format!(
84                "arrow.parquet.variant extension type takes no metadata, got {other:?}"
85            ))),
86        }
87    }
88
89    fn supports_data_type(&self, data_type: &DataType) -> std::result::Result<(), ArrowError> {
90        match data_type {
91            DataType::Struct(_) => Ok(()),
92            other => Err(ArrowError::InvalidArgumentError(format!(
93                "arrow.parquet.variant extension type requires a Struct storage type, got {other}"
94            ))),
95        }
96    }
97
98    fn try_new(
99        data_type: &DataType,
100        _metadata: Self::Metadata,
101    ) -> std::result::Result<Self, ArrowError> {
102        Self.supports_data_type(data_type)?;
103        Ok(Self)
104    }
105}
106
107/// Whether `field` carries the [`VariantExtensionType`].
108pub(crate) fn is_variant_field(field: &Field) -> bool {
109    field.extension_type_name() == Some(VariantExtensionType::NAME)
110}
111
112/// Whether `data_type` can hold a variant's `metadata` or `value` bytes.
113fn is_variant_binary(data_type: &DataType) -> bool {
114    match data_type {
115        DataType::Binary | DataType::LargeBinary | DataType::BinaryView => true,
116        DataType::Dictionary(_, value_type) => is_variant_binary(value_type),
117        _ => false,
118    }
119}
120
121/// Checks that `field` stores a variant unshredded: `metadata` and `value` bytes, matched by
122/// name, with a required `metadata`.
123pub(crate) fn validate_unshredded_variant(field: &Field, variant_field_id: i32) -> Result<()> {
124    let invalid = |problem: String| {
125        invalid_data!(
126            "Variant column `{}` (field id {variant_field_id}) {problem}",
127            field.name()
128        )
129    };
130    let DataType::Struct(subfields) = field.data_type() else {
131        return Err(invalid(format!(
132            "must be stored as a group, found {}",
133            field.data_type()
134        )));
135    };
136    if subfields.find("typed_value").is_some() {
137        return Err(Error::new(
138            ErrorKind::FeatureUnsupported,
139            format!(
140                "Reading shredded variant column `{}` (field id {variant_field_id}) is not supported yet, see https://github.com/apache/iceberg-rust/issues/3606",
141                field.name()
142            ),
143        ));
144    }
145    for name in ["metadata", "value"] {
146        match subfields.find(name) {
147            Some((_, subfield)) if !is_variant_binary(subfield.data_type()) => {
148                return Err(invalid(format!(
149                    "has a non-binary `{name}` field: {}",
150                    subfield.data_type()
151                )));
152            }
153            // The Parquet and Arrow variant specs require `metadata`, and the table type reads
154            // it as non-nullable.
155            Some((_, subfield)) if name == "metadata" && subfield.is_nullable() => {
156                return Err(invalid(
157                    "has an optional `metadata` field, which must be required".to_string(),
158                ));
159            }
160            Some(_) => {}
161            None => return Err(invalid(format!("is missing the `{name}` field"))),
162        }
163    }
164    if subfields.len() != 2 {
165        return Err(invalid(
166            "has fields other than `metadata` and `value`".to_string(),
167        ));
168    }
169    Ok(())
170}
171
172fn to_arrow_wkb_edges(algorithm: EdgeInterpolationAlgorithm) -> WkbEdges {
173    match algorithm {
174        EdgeInterpolationAlgorithm::Spherical => WkbEdges::Spherical,
175        EdgeInterpolationAlgorithm::Vincenty => WkbEdges::Vincenty,
176        EdgeInterpolationAlgorithm::Thomas => WkbEdges::Thomas,
177        EdgeInterpolationAlgorithm::Andoyer => WkbEdges::Andoyer,
178        EdgeInterpolationAlgorithm::Karney => WkbEdges::Karney,
179    }
180}
181
182impl From<WkbEdges> for EdgeInterpolationAlgorithm {
183    fn from(edges: WkbEdges) -> Self {
184        match edges {
185            WkbEdges::Spherical => Self::Spherical,
186            WkbEdges::Vincenty => Self::Vincenty,
187            WkbEdges::Thomas => Self::Thomas,
188            WkbEdges::Andoyer => Self::Andoyer,
189            WkbEdges::Karney => Self::Karney,
190        }
191    }
192}
193
194fn iceberg_crs_from_wkb_metadata(metadata: &WkbMetadata) -> Result<String> {
195    match metadata.crs.as_ref() {
196        None => Ok(UNSET_GEOSPATIAL_CRS.to_string()),
197        Some(serde_json::Value::String(crs)) => Ok(crs.clone()),
198        Some(serde_json::Value::Object(crs)) => {
199            let id = crs.get("id");
200            let authority = id
201                .and_then(|id| id.get("authority"))
202                .and_then(serde_json::Value::as_str)
203                .filter(|authority| !authority.is_empty());
204            let code = match id.and_then(|id| id.get("code")) {
205                Some(serde_json::Value::String(code)) => Some(code.clone()),
206                Some(serde_json::Value::Number(code)) => Some(code.to_string()),
207                _ => None,
208            };
209
210            match (authority, code) {
211                (Some(authority), Some(code)) => Ok(format!("{authority}:{code}")),
212                _ => Err(Error::new(
213                    ErrorKind::DataInvalid,
214                    "Cannot determine Iceberg geospatial type from PROJJSON CRS without an embedded authority/code",
215                )),
216            }
217        }
218        Some(_) => Err(Error::new(
219            ErrorKind::DataInvalid,
220            "Geospatial CRS metadata must be a string or PROJJSON object",
221        )),
222    }
223}
224
225fn arrow_crs_from_iceberg(crs: &str) -> Option<&str> {
226    (!crs.eq_ignore_ascii_case(UNSET_GEOSPATIAL_CRS)).then_some(crs)
227}
228
229/// A post order arrow schema visitor.
230///
231/// For order of methods called, please refer to the internal `visit_schema` function.
232pub trait ArrowSchemaVisitor {
233    /// Return type of this visitor on arrow field.
234    type T;
235
236    /// Return type of this visitor on arrow schema.
237    type U;
238
239    /// Called before each field of the schema or of a struct, whatever its type.
240    /// List elements and map keys and values use [`Self::before_list_element`],
241    /// [`Self::before_map_key`] and [`Self::before_map_value`] instead.
242    fn before_field(&mut self, _field: &FieldRef) -> Result<()> {
243        Ok(())
244    }
245
246    /// Called after each field of the schema or of a struct, whatever its type.
247    /// List elements and map keys and values use [`Self::after_list_element`],
248    /// [`Self::after_map_key`] and [`Self::after_map_value`] instead.
249    fn after_field(&mut self, _field: &FieldRef) -> Result<()> {
250        Ok(())
251    }
252
253    /// Called before list element.
254    fn before_list_element(&mut self, _field: &FieldRef) -> Result<()> {
255        Ok(())
256    }
257
258    /// Called after list element.
259    fn after_list_element(&mut self, _field: &FieldRef) -> Result<()> {
260        Ok(())
261    }
262
263    /// Called before map key.
264    fn before_map_key(&mut self, _field: &FieldRef) -> Result<()> {
265        Ok(())
266    }
267
268    /// Called after map key.
269    fn after_map_key(&mut self, _field: &FieldRef) -> Result<()> {
270        Ok(())
271    }
272
273    /// Called before map value.
274    fn before_map_value(&mut self, _field: &FieldRef) -> Result<()> {
275        Ok(())
276    }
277
278    /// Called after map value.
279    fn after_map_value(&mut self, _field: &FieldRef) -> Result<()> {
280        Ok(())
281    }
282
283    /// Called after schema's type visited.
284    fn schema(&mut self, schema: &ArrowSchema, values: Vec<Self::T>) -> Result<Self::U>;
285
286    /// Called after struct's fields visited.
287    fn r#struct(&mut self, fields: &Fields, results: Vec<Self::T>) -> Result<Self::T>;
288
289    /// Called after list fields visited.
290    fn list(&mut self, list: &DataType, value: Self::T) -> Result<Self::T>;
291
292    /// Called after map's key and value fields visited.
293    fn map(&mut self, map: &DataType, key_value: Self::T, value: Self::T) -> Result<Self::T>;
294
295    /// Called when see a primitive type.
296    fn primitive(&mut self, p: &DataType) -> Result<Self::T>;
297
298    /// Called when a field carries the `arrow.parquet.variant` extension type.
299    ///
300    /// The default treats the variant as its underlying Arrow struct storage: it
301    /// re-enters normal traversal, preserving the behavior of visitors that don't
302    /// special-case variants. A visitor that produces Iceberg types (or otherwise
303    /// needs variant identity) should override this to fold the struct into a
304    /// single logical variant.
305    ///
306    /// Takes the `&FieldRef` rather than a `&DataType` because the variant signal
307    /// lives on the field's metadata, not on its data type.
308    fn variant(&mut self, field: &FieldRef) -> Result<Self::T>
309    where Self: Sized {
310        visit_type(field.data_type(), self)
311    }
312
313    /// Called when a field carries the `arrow.uuid` extension type.
314    ///
315    /// The default treats the field as its underlying Arrow storage, preserving
316    /// the behavior of visitors that don't special-case UUIDs. A visitor that
317    /// produces Iceberg types should override this to fold the storage into a
318    /// single logical UUID.
319    ///
320    /// Takes the `&FieldRef` rather than a `&DataType` because the UUID signal
321    /// lives on the field's metadata, not on its data type.
322    fn uuid(&mut self, field: &FieldRef) -> Result<Self::T>
323    where Self: Sized {
324        visit_type(field.data_type(), self)
325    }
326}
327
328/// Visiting a type in post order.
329fn visit_type<V: ArrowSchemaVisitor>(r#type: &DataType, visitor: &mut V) -> Result<V::T> {
330    match r#type {
331        p if p.is_primitive()
332            || matches!(
333                p,
334                DataType::Boolean
335                    | DataType::Utf8
336                    | DataType::LargeUtf8
337                    | DataType::Utf8View
338                    | DataType::Null
339                    | DataType::Binary
340                    | DataType::LargeBinary
341                    | DataType::BinaryView
342                    | DataType::FixedSizeBinary(_)
343            ) =>
344        {
345            visitor.primitive(p)
346        }
347        DataType::List(element_field) => visit_list(r#type, element_field, visitor),
348        DataType::LargeList(element_field) => visit_list(r#type, element_field, visitor),
349        DataType::FixedSizeList(element_field, _) => visit_list(r#type, element_field, visitor),
350        DataType::Map(field, _) => match field.data_type() {
351            DataType::Struct(fields) => {
352                if fields.len() != 2 {
353                    return Err(invalid_data!("Map field must have exactly 2 fields"));
354                }
355
356                let key_field = &fields[0];
357                let value_field = &fields[1];
358
359                let key_result = {
360                    visitor.before_map_key(key_field)?;
361                    let ret = visit_field(key_field, visitor)?;
362                    visitor.after_map_key(key_field)?;
363                    ret
364                };
365
366                let value_result = {
367                    visitor.before_map_value(value_field)?;
368                    let ret = visit_field(value_field, visitor)?;
369                    visitor.after_map_value(value_field)?;
370                    ret
371                };
372
373                visitor.map(r#type, key_result, value_result)
374            }
375            _ => Err(invalid_data!("Map field must have struct type")),
376        },
377        DataType::Struct(fields) => visit_struct(fields, visitor),
378        DataType::Dictionary(_key_type, value_type) => visit_type(value_type, visitor),
379        other => Err(invalid_data!("Cannot visit Arrow data type: {other}")),
380    }
381}
382
383/// Dispatch a field, and fold it into following extension types when carried:
384///  - `arrow.parquet.variant` -- fold into a variant,
385///  - `arrow.uuid` -- fold it into a UUID type,
386///  - and in all other cases we visit it's data.
387fn visit_field<V: ArrowSchemaVisitor>(field: &FieldRef, visitor: &mut V) -> Result<V::T> {
388    if field.extension_type_name() == Some(VariantExtensionType::NAME) {
389        visitor.variant(field)
390    } else if field.extension_type_name() == Some(UuidExtensionType::NAME) {
391        visitor.uuid(field)
392    } else {
393        visit_type(field.data_type(), visitor)
394    }
395}
396
397/// Visit list types in post order.
398fn visit_list<V: ArrowSchemaVisitor>(
399    data_type: &DataType,
400    element_field: &FieldRef,
401    visitor: &mut V,
402) -> Result<V::T> {
403    visitor.before_list_element(element_field)?;
404    let value = visit_field(element_field, visitor)?;
405    visitor.after_list_element(element_field)?;
406    visitor.list(data_type, value)
407}
408
409/// Visit struct type in post order.
410fn visit_struct<V: ArrowSchemaVisitor>(fields: &Fields, visitor: &mut V) -> Result<V::T> {
411    let mut results = Vec::with_capacity(fields.len());
412    for field in fields {
413        visitor.before_field(field)?;
414        let result = visit_field(field, visitor)?;
415        visitor.after_field(field)?;
416        results.push(result);
417    }
418
419    visitor.r#struct(fields, results)
420}
421
422/// Visit schema in post order.
423pub(crate) fn visit_schema<V: ArrowSchemaVisitor>(
424    schema: &ArrowSchema,
425    visitor: &mut V,
426) -> Result<V::U> {
427    let mut results = Vec::with_capacity(schema.fields().len());
428    for field in schema.fields() {
429        visitor.before_field(field)?;
430        let result = visit_field(field, visitor)?;
431        visitor.after_field(field)?;
432        results.push(result);
433    }
434    visitor.schema(schema, results)
435}
436
437/// Convert Arrow schema to Iceberg schema.
438///
439/// Iceberg schema fields require a unique field id, and this function assumes that each field
440/// in the provided Arrow schema contains a field id in its metadata. If the metadata is missing
441/// or the field id is not set, the conversion will fail
442pub fn arrow_schema_to_schema(schema: &ArrowSchema) -> Result<Schema> {
443    let mut visitor = ArrowSchemaConverter::new();
444    visit_schema(schema, &mut visitor)
445}
446
447/// Convert Arrow schema to Iceberg schema with automatically assigned field IDs.
448///
449/// Unlike [`arrow_schema_to_schema`], this function does not require field IDs in the Arrow
450/// schema metadata. Instead, it automatically assigns unique field IDs starting from 1,
451/// following Iceberg's field ID assignment rules.
452///
453/// This is useful when converting Arrow schemas that don't originate from Iceberg tables,
454/// such as schemas from DataFusion or other Arrow-based systems.
455pub fn arrow_schema_to_schema_auto_assign_ids(schema: &ArrowSchema) -> Result<Schema> {
456    let mut visitor = ArrowSchemaConverter::new_with_field_ids_from(FIRST_FIELD_ID);
457    visit_schema(schema, &mut visitor)
458}
459
460/// Convert Arrow type to iceberg type.
461pub fn arrow_type_to_type(ty: &DataType) -> Result<Type> {
462    let mut visitor = ArrowSchemaConverter::new();
463    visit_type(ty, &mut visitor)
464}
465
466const ARROW_FIELD_DOC_KEY: &str = "doc";
467
468pub(super) fn try_get_field_id_from_metadata(field: &Field) -> Result<Option<i32>> {
469    field
470        .metadata()
471        .get(PARQUET_FIELD_ID_META_KEY)
472        .map(|value| {
473            value.parse::<i32>().map_err(|e| {
474                invalid_data!("Failed to parse field id")
475                    .with_context("value", value)
476                    .with_source(e)
477            })
478        })
479        .transpose()
480}
481
482pub(super) fn get_field_id_from_metadata(field: &Field) -> Result<i32> {
483    try_get_field_id_from_metadata(field)?
484        .ok_or_else(|| invalid_data!("Field id not found in metadata"))
485}
486
487fn get_field_doc(field: &FieldRef) -> Option<String> {
488    if let Some(value) = field.metadata().get(ARROW_FIELD_DOC_KEY) {
489        return Some(value.clone());
490    }
491    None
492}
493
494struct ArrowSchemaConverter {
495    /// When set, the schema builder will reassign field IDs starting from this value
496    /// using level-order traversal (breadth-first).
497    reassign_field_ids_from: Option<i32>,
498    /// Generates unique placeholder IDs for fields before reassignment.
499    /// Required because `ReassignFieldIds` builds an old-to-new ID mapping
500    /// that expects unique input IDs.
501    next_field_id: i32,
502}
503
504impl ArrowSchemaConverter {
505    fn new() -> Self {
506        Self {
507            reassign_field_ids_from: None,
508            next_field_id: 0,
509        }
510    }
511
512    fn new_with_field_ids_from(start_from: i32) -> Self {
513        Self {
514            reassign_field_ids_from: Some(start_from),
515            next_field_id: 0,
516        }
517    }
518
519    fn get_field_id(&mut self, field: &FieldRef) -> Result<i32> {
520        if self.reassign_field_ids_from.is_some() {
521            // Field IDs will be reassigned by the schema builder.
522            // We need unique temporary IDs because ReassignFieldIds builds an
523            // old->new ID mapping that requires unique input IDs.
524            let temp_id = self.next_field_id;
525            self.next_field_id += 1;
526            Ok(temp_id)
527        } else {
528            // Get field ID from arrow field metadata
529            get_field_id_from_metadata(field)
530        }
531    }
532
533    fn convert_fields(
534        &mut self,
535        fields: &Fields,
536        field_results: &[Type],
537    ) -> Result<Vec<NestedFieldRef>> {
538        let mut results = Vec::with_capacity(fields.len());
539        for i in 0..fields.len() {
540            let field = &fields[i];
541            let field_type = self.apply_arrow_extension(field, field_results[i].clone())?;
542            let id = self.get_field_id(field)?;
543            let doc = get_field_doc(field);
544            let nested_field = NestedField {
545                id,
546                doc,
547                name: field.name().clone(),
548                required: !field.is_nullable(),
549                field_type: Box::new(field_type),
550                initial_default: None,
551                write_default: None,
552            };
553            results.push(Arc::new(nested_field));
554        }
555        Ok(results)
556    }
557
558    fn apply_arrow_extension(&self, arrow_field: &FieldRef, iceberg_type: Type) -> Result<Type> {
559        if arrow_field.extension_type_name() != Some(WkbType::NAME) {
560            return Ok(iceberg_type);
561        }
562
563        if !matches!(&iceberg_type, Type::Primitive(PrimitiveType::Binary)) {
564            return Err(Error::new(
565                ErrorKind::DataInvalid,
566                format!(
567                    "WKB extension type on field {} requires binary storage, got {iceberg_type}",
568                    arrow_field.name()
569                ),
570            ));
571        }
572
573        let wkb_type = arrow_field.try_extension_type::<WkbType>().map_err(|err| {
574            Error::new(
575                ErrorKind::DataInvalid,
576                format!(
577                    "Invalid geospatial Arrow extension metadata for field {}",
578                    arrow_field.name()
579                ),
580            )
581            .with_source(err)
582        })?;
583
584        let crs = iceberg_crs_from_wkb_metadata(wkb_type.metadata())?;
585
586        match wkb_type.metadata().type_hint() {
587            WkbTypeHint::Geometry => Ok(Type::Primitive(PrimitiveType::Geometry(
588                GeometryType::new(Some(crs))?,
589            ))),
590            WkbTypeHint::Geography => Ok(Type::Primitive(PrimitiveType::Geography(
591                GeographyType::new(
592                    Some(crs),
593                    wkb_type
594                        .metadata()
595                        .algorithm
596                        .unwrap_or(WkbEdges::Spherical)
597                        .into(),
598                )?,
599            ))),
600        }
601    }
602}
603
604impl ArrowSchemaVisitor for ArrowSchemaConverter {
605    type T = Type;
606    type U = Schema;
607
608    fn schema(&mut self, schema: &ArrowSchema, values: Vec<Self::T>) -> Result<Self::U> {
609        let fields = self.convert_fields(schema.fields(), &values)?;
610        let mut builder = Schema::builder().with_fields(fields);
611        if let Some(start_from) = self.reassign_field_ids_from {
612            builder = builder.with_reassigned_field_ids(start_from)
613        }
614        builder.build()
615    }
616
617    fn r#struct(&mut self, fields: &Fields, results: Vec<Self::T>) -> Result<Self::T> {
618        let fields = self.convert_fields(fields, &results)?;
619        Ok(Type::Struct(StructType::new(fields)))
620    }
621
622    fn list(&mut self, list: &DataType, value: Self::T) -> Result<Self::T> {
623        let element_field = match list {
624            DataType::List(element_field) => element_field,
625            DataType::LargeList(element_field) => element_field,
626            DataType::FixedSizeList(element_field, _) => element_field,
627            _ => {
628                return Err(invalid_data!("List type must have list data type"));
629            }
630        };
631
632        let value = self.apply_arrow_extension(element_field, value)?;
633        let id = self.get_field_id(element_field)?;
634        let doc = get_field_doc(element_field);
635        let mut element_field =
636            NestedField::list_element(id, value.clone(), !element_field.is_nullable());
637        if let Some(doc) = doc {
638            element_field = element_field.with_doc(doc);
639        }
640        let element_field = Arc::new(element_field);
641        Ok(Type::List(ListType { element_field }))
642    }
643
644    fn map(&mut self, map: &DataType, key_value: Self::T, value: Self::T) -> Result<Self::T> {
645        match map {
646            DataType::Map(field, _) => match field.data_type() {
647                DataType::Struct(fields) => {
648                    if fields.len() != 2 {
649                        return Err(invalid_data!("Map field must have exactly 2 fields"));
650                    }
651
652                    let key_field = &fields[0];
653                    let value_field = &fields[1];
654                    let key_value = self.apply_arrow_extension(key_field, key_value)?;
655                    let value = self.apply_arrow_extension(value_field, value)?;
656
657                    let key_id = self.get_field_id(key_field)?;
658                    let key_doc = get_field_doc(key_field);
659                    let mut key_field = NestedField::map_key_element(key_id, key_value.clone());
660                    if let Some(doc) = key_doc {
661                        key_field = key_field.with_doc(doc);
662                    }
663                    let key_field = Arc::new(key_field);
664
665                    let value_id = self.get_field_id(value_field)?;
666                    let value_doc = get_field_doc(value_field);
667                    let mut value_field = NestedField::map_value_element(
668                        value_id,
669                        value.clone(),
670                        !value_field.is_nullable(),
671                    );
672                    if let Some(doc) = value_doc {
673                        value_field = value_field.with_doc(doc);
674                    }
675                    let value_field = Arc::new(value_field);
676
677                    Ok(Type::Map(MapType {
678                        key_field,
679                        value_field,
680                    }))
681                }
682                _ => Err(invalid_data!("Map field must have struct type")),
683            },
684            _ => Err(invalid_data!("Map type must have map data type")),
685        }
686    }
687
688    fn primitive(&mut self, p: &DataType) -> Result<Self::T> {
689        match p {
690            DataType::Null => Ok(Type::Primitive(PrimitiveType::Unknown)),
691            DataType::Boolean => Ok(Type::Primitive(PrimitiveType::Boolean)),
692            DataType::Int8 | DataType::Int16 | DataType::Int32 => {
693                Ok(Type::Primitive(PrimitiveType::Int))
694            }
695            DataType::UInt8 | DataType::UInt16 => Ok(Type::Primitive(PrimitiveType::Int)),
696            DataType::UInt32 => Ok(Type::Primitive(PrimitiveType::Long)),
697            DataType::Int64 => Ok(Type::Primitive(PrimitiveType::Long)),
698            DataType::UInt64 => {
699                // Block uint64 - no safe casting option
700                Err(invalid_data!(
701                    "UInt64 is not supported. Use Int64 for values ≤ 9,223,372,036,854,775,807 or Decimal(20,0) for full uint64 range."
702                ))
703            }
704            DataType::Float32 => Ok(Type::Primitive(PrimitiveType::Float)),
705            DataType::Float64 => Ok(Type::Primitive(PrimitiveType::Double)),
706            DataType::Decimal128(p, s) => Type::decimal(*p as u32, *s as u32)
707                .map_err(|e| invalid_data!("Failed to create decimal type").with_source(e)),
708            DataType::Date32 => Ok(Type::Primitive(PrimitiveType::Date)),
709            DataType::Time64(unit) if unit == &TimeUnit::Microsecond => {
710                Ok(Type::Primitive(PrimitiveType::Time))
711            }
712            DataType::Timestamp(unit, None) if unit == &TimeUnit::Microsecond => {
713                Ok(Type::Primitive(PrimitiveType::Timestamp))
714            }
715            DataType::Timestamp(unit, None) if unit == &TimeUnit::Nanosecond => {
716                Ok(Type::Primitive(PrimitiveType::TimestampNs))
717            }
718            DataType::Timestamp(unit, Some(zone))
719                if unit == &TimeUnit::Microsecond
720                    && (zone.as_ref() == "UTC" || zone.as_ref() == "+00:00") =>
721            {
722                Ok(Type::Primitive(PrimitiveType::Timestamptz))
723            }
724            DataType::Timestamp(unit, Some(zone))
725                if unit == &TimeUnit::Nanosecond
726                    && (zone.as_ref() == "UTC" || zone.as_ref() == "+00:00") =>
727            {
728                Ok(Type::Primitive(PrimitiveType::TimestamptzNs))
729            }
730            DataType::Binary | DataType::LargeBinary | DataType::BinaryView => {
731                Ok(Type::Primitive(PrimitiveType::Binary))
732            }
733            DataType::FixedSizeBinary(width) => {
734                Ok(Type::Primitive(PrimitiveType::Fixed(*width as u64)))
735            }
736            DataType::Utf8View | DataType::Utf8 | DataType::LargeUtf8 => {
737                Ok(Type::Primitive(PrimitiveType::String))
738            }
739            _ => Err(invalid_data!("Unsupported Arrow data type: {p}")),
740        }
741    }
742
743    fn variant(&mut self, field: &FieldRef) -> Result<Self::T> {
744        // The extension may only sit on struct storage (mirrors
745        // `VariantExtensionType::supports_data_type`).
746        if !matches!(field.data_type(), DataType::Struct(_)) {
747            return Err(invalid_data!(
748                "arrow.parquet.variant extension requires Struct storage"
749            ));
750        }
751        // Fold the whole struct into a single logical variant without descending:
752        // the storage sub-fields carry no Iceberg field id, so visiting them would
753        // fail. The enclosing field's own id is read by the caller.
754        Ok(Type::Variant(VariantType))
755    }
756
757    fn uuid(&mut self, field: &FieldRef) -> Result<Self::T> {
758        // The extension expects a fixed size binary.
759        // See: Apache Arrow spec https://arrow.apache.org/docs/format/CanonicalExtensions.html#uuid
760        let field_data_type = field.data_type();
761        let expected_data_type = DataType::FixedSizeBinary(16);
762        if *field_data_type != expected_data_type {
763            let error_message = format!(
764                "{} extension requires {expected_data_type} storage, found {field_data_type}",
765                UuidExtensionType::NAME,
766            );
767
768            return Err(Error::new(ErrorKind::DataInvalid, error_message));
769        }
770
771        Ok(Type::Primitive(PrimitiveType::Uuid))
772    }
773}
774
775struct ToArrowSchemaConverter;
776
777enum ArrowSchemaOrFieldOrType {
778    Schema(ArrowSchema),
779    Field(Field),
780    Type(DataType),
781}
782
783impl SchemaVisitor for ToArrowSchemaConverter {
784    type T = ArrowSchemaOrFieldOrType;
785
786    fn schema(
787        &mut self,
788        _schema: &Schema,
789        value: ArrowSchemaOrFieldOrType,
790    ) -> Result<ArrowSchemaOrFieldOrType> {
791        let struct_type = match value {
792            ArrowSchemaOrFieldOrType::Type(DataType::Struct(fields)) => fields,
793            _ => unreachable!(),
794        };
795        Ok(ArrowSchemaOrFieldOrType::Schema(ArrowSchema::new(
796            struct_type,
797        )))
798    }
799
800    fn field(
801        &mut self,
802        field: &NestedFieldRef,
803        value: ArrowSchemaOrFieldOrType,
804    ) -> Result<ArrowSchemaOrFieldOrType> {
805        let ty = match value {
806            ArrowSchemaOrFieldOrType::Type(ty) => ty,
807            _ => unreachable!(),
808        };
809        let metadata = if let Some(doc) = &field.doc {
810            HashMap::from([
811                (PARQUET_FIELD_ID_META_KEY.to_string(), field.id.to_string()),
812                (ARROW_FIELD_DOC_KEY.to_string(), doc.clone()),
813            ])
814        } else {
815            HashMap::from([(PARQUET_FIELD_ID_META_KEY.to_string(), field.id.to_string())])
816        };
817        let mut arrow_field =
818            Field::new(field.name.clone(), ty, !field.required).with_metadata(metadata);
819
820        match field.field_type.as_ref() {
821            Type::Variant(_) => {
822                // A variant column's storage is a struct; tag the field with the canonical
823                // `arrow.parquet.variant` extension type so consumers read it as a Variant, not a struct.
824                arrow_field.try_with_extension_type(VariantExtensionType)?;
825            }
826            Type::Primitive(PrimitiveType::Uuid) => {
827                // Preserve the canonical UUID extension on its FixedSizeBinary(16) storage.
828                arrow_field.try_with_extension_type(UuidExtensionType)?;
829            }
830            Type::Primitive(PrimitiveType::Geometry(geometry)) => {
831                let metadata = WkbMetadata::new(arrow_crs_from_iceberg(geometry.crs()), None);
832                arrow_field.try_with_extension_type(WkbType::new(Some(metadata)))?;
833            }
834            Type::Primitive(PrimitiveType::Geography(geography)) => {
835                let metadata = WkbMetadata::new(
836                    arrow_crs_from_iceberg(geography.crs()),
837                    Some(to_arrow_wkb_edges(geography.algorithm())),
838                );
839                arrow_field.try_with_extension_type(WkbType::new(Some(metadata)))?;
840            }
841            _ => {}
842        }
843
844        Ok(ArrowSchemaOrFieldOrType::Field(arrow_field))
845    }
846
847    fn r#struct(
848        &mut self,
849        _: &StructType,
850        results: Vec<ArrowSchemaOrFieldOrType>,
851    ) -> Result<ArrowSchemaOrFieldOrType> {
852        let fields = results
853            .into_iter()
854            .map(|result| match result {
855                ArrowSchemaOrFieldOrType::Field(field) => field,
856                _ => unreachable!(),
857            })
858            .collect();
859        Ok(ArrowSchemaOrFieldOrType::Type(DataType::Struct(fields)))
860    }
861
862    fn list(&mut self, list: &ListType, value: ArrowSchemaOrFieldOrType) -> Result<Self::T> {
863        // `field` already carries the element's field id, doc, and — for a variant element —
864        // the arrow.parquet.variant extension type. Don't overwrite its metadata here (doing so
865        // would drop the extension type for `list<variant>`).
866        let field = match self.field(&list.element_field, value)? {
867            ArrowSchemaOrFieldOrType::Field(field) => field,
868            _ => unreachable!(),
869        };
870        Ok(ArrowSchemaOrFieldOrType::Type(DataType::List(Arc::new(
871            field,
872        ))))
873    }
874
875    fn map(
876        &mut self,
877        map: &MapType,
878        key_value: ArrowSchemaOrFieldOrType,
879        value: ArrowSchemaOrFieldOrType,
880    ) -> Result<ArrowSchemaOrFieldOrType> {
881        let key_field = match self.field(&map.key_field, key_value)? {
882            ArrowSchemaOrFieldOrType::Field(field) => field,
883            _ => unreachable!(),
884        };
885        let value_field = match self.field(&map.value_field, value)? {
886            ArrowSchemaOrFieldOrType::Field(field) => field,
887            _ => unreachable!(),
888        };
889        let field = Field::new(
890            DEFAULT_MAP_FIELD_NAME,
891            DataType::Struct(vec![key_field, value_field].into()),
892            // Map field is always not nullable
893            false,
894        );
895
896        Ok(ArrowSchemaOrFieldOrType::Type(DataType::Map(
897            field.into(),
898            false,
899        )))
900    }
901
902    fn primitive(&mut self, p: &PrimitiveType) -> Result<ArrowSchemaOrFieldOrType> {
903        match p {
904            PrimitiveType::Unknown => Ok(ArrowSchemaOrFieldOrType::Type(DataType::Null)),
905            PrimitiveType::Boolean => Ok(ArrowSchemaOrFieldOrType::Type(DataType::Boolean)),
906            PrimitiveType::Int => Ok(ArrowSchemaOrFieldOrType::Type(DataType::Int32)),
907            PrimitiveType::Long => Ok(ArrowSchemaOrFieldOrType::Type(DataType::Int64)),
908            PrimitiveType::Float => Ok(ArrowSchemaOrFieldOrType::Type(DataType::Float32)),
909            PrimitiveType::Double => Ok(ArrowSchemaOrFieldOrType::Type(DataType::Float64)),
910            PrimitiveType::Decimal { precision, scale } => {
911                let (precision, scale) = {
912                    let precision: u8 = precision.to_owned().try_into().map_err(|err| {
913                        invalid_data!("incompatible precision for decimal type convert")
914                            .with_source(err)
915                    })?;
916                    let scale = scale.to_owned().try_into().map_err(|err| {
917                        invalid_data!("incompatible scale for decimal type convert")
918                            .with_source(err)
919                    })?;
920                    (precision, scale)
921                };
922                validate_decimal_precision_and_scale::<Decimal128Type>(precision, scale).map_err(
923                    |err| {
924                        invalid_data!("incompatible precision and scale for decimal type convert")
925                            .with_source(err)
926                    },
927                )?;
928                Ok(ArrowSchemaOrFieldOrType::Type(DataType::Decimal128(
929                    precision, scale,
930                )))
931            }
932            PrimitiveType::Date => Ok(ArrowSchemaOrFieldOrType::Type(DataType::Date32)),
933            PrimitiveType::Time => Ok(ArrowSchemaOrFieldOrType::Type(DataType::Time64(
934                TimeUnit::Microsecond,
935            ))),
936            PrimitiveType::Timestamp => Ok(ArrowSchemaOrFieldOrType::Type(DataType::Timestamp(
937                TimeUnit::Microsecond,
938                None,
939            ))),
940            PrimitiveType::Timestamptz => Ok(ArrowSchemaOrFieldOrType::Type(
941                // Timestampz always stored as UTC
942                DataType::Timestamp(TimeUnit::Microsecond, Some(UTC_TIME_ZONE.into())),
943            )),
944            PrimitiveType::TimestampNs => Ok(ArrowSchemaOrFieldOrType::Type(DataType::Timestamp(
945                TimeUnit::Nanosecond,
946                None,
947            ))),
948            PrimitiveType::TimestamptzNs => Ok(ArrowSchemaOrFieldOrType::Type(
949                // Store timestamptz_ns as UTC
950                DataType::Timestamp(TimeUnit::Nanosecond, Some(UTC_TIME_ZONE.into())),
951            )),
952            PrimitiveType::String => Ok(ArrowSchemaOrFieldOrType::Type(DataType::Utf8)),
953            PrimitiveType::Uuid => Ok(ArrowSchemaOrFieldOrType::Type(DataType::FixedSizeBinary(
954                16,
955            ))),
956            PrimitiveType::Fixed(len) => Ok(ArrowSchemaOrFieldOrType::Type(
957                i32::try_from(*len)
958                    .ok()
959                    .map(DataType::FixedSizeBinary)
960                    .unwrap_or(DataType::LargeBinary),
961            )),
962            PrimitiveType::Binary => Ok(ArrowSchemaOrFieldOrType::Type(DataType::Binary)),
963            PrimitiveType::Geometry(_) | PrimitiveType::Geography(_) => {
964                Ok(ArrowSchemaOrFieldOrType::Type(DataType::LargeBinary))
965            }
966        }
967    }
968
969    fn variant(&mut self, _v: &VariantType) -> Result<ArrowSchemaOrFieldOrType> {
970        // Variant is stored as a struct of two binary sub-fields (no field IDs on sub-fields).
971        // Uses Binary (not LargeBinary) matching the Parquet BINARY primitive directly.
972        // `metadata` is always present; `value` is nullable, since in a shredded variant the
973        // value may be absent. The enclosing field carries the `arrow.parquet.variant` extension type
974        // (attached in `field`).
975        let metadata_field = Field::new("metadata", DataType::Binary, false);
976        let value_field = Field::new("value", DataType::Binary, true);
977        Ok(ArrowSchemaOrFieldOrType::Type(DataType::Struct(
978            vec![metadata_field, value_field].into(),
979        )))
980    }
981}
982
983/// Convert iceberg schema to an arrow schema.
984pub fn schema_to_arrow_schema(schema: &Schema) -> Result<ArrowSchema> {
985    let mut converter = ToArrowSchemaConverter;
986    match crate::spec::visit_schema(schema, &mut converter)? {
987        ArrowSchemaOrFieldOrType::Schema(schema) => Ok(schema),
988        _ => unreachable!(),
989    }
990}
991
992/// Convert iceberg type to an arrow type.
993pub fn type_to_arrow_type(ty: &Type) -> Result<DataType> {
994    let mut converter = ToArrowSchemaConverter;
995    match crate::spec::visit_type(ty, &mut converter)? {
996        ArrowSchemaOrFieldOrType::Type(ty) => Ok(ty),
997        _ => unreachable!(),
998    }
999}
1000
1001/// Convert Iceberg Datum to Arrow Datum.
1002pub(crate) fn get_arrow_datum(datum: &Datum) -> Result<Arc<dyn ArrowDatum + Send + Sync>> {
1003    match (datum.data_type(), datum.literal()) {
1004        (PrimitiveType::Boolean, PrimitiveLiteral::Boolean(value)) => {
1005            Ok(Arc::new(BooleanArray::new_scalar(*value)))
1006        }
1007        (PrimitiveType::Int, PrimitiveLiteral::Int(value)) => {
1008            Ok(Arc::new(Int32Array::new_scalar(*value)))
1009        }
1010        (PrimitiveType::Long, PrimitiveLiteral::Long(value)) => {
1011            Ok(Arc::new(Int64Array::new_scalar(*value)))
1012        }
1013        (PrimitiveType::Float, PrimitiveLiteral::Float(value)) => {
1014            Ok(Arc::new(Float32Array::new_scalar(value.into_inner())))
1015        }
1016        (PrimitiveType::Double, PrimitiveLiteral::Double(value)) => {
1017            Ok(Arc::new(Float64Array::new_scalar(value.into_inner())))
1018        }
1019        (PrimitiveType::String, PrimitiveLiteral::String(value)) => {
1020            Ok(Arc::new(StringArray::new_scalar(value.as_str())))
1021        }
1022        (PrimitiveType::Binary, PrimitiveLiteral::Binary(value)) => {
1023            Ok(Arc::new(BinaryArray::new_scalar(value.as_slice())))
1024        }
1025        (PrimitiveType::Date, PrimitiveLiteral::Int(value)) => {
1026            Ok(Arc::new(Date32Array::new_scalar(*value)))
1027        }
1028        (PrimitiveType::Timestamp, PrimitiveLiteral::Long(value)) => {
1029            Ok(Arc::new(TimestampMicrosecondArray::new_scalar(*value)))
1030        }
1031        (PrimitiveType::Timestamptz, PrimitiveLiteral::Long(value)) => Ok(Arc::new(Scalar::new(
1032            TimestampMicrosecondArray::new(vec![*value; 1].into(), None).with_timezone_utc(),
1033        ))),
1034        (PrimitiveType::TimestampNs, PrimitiveLiteral::Long(value)) => {
1035            Ok(Arc::new(TimestampNanosecondArray::new_scalar(*value)))
1036        }
1037        (PrimitiveType::TimestamptzNs, PrimitiveLiteral::Long(value)) => Ok(Arc::new(Scalar::new(
1038            TimestampNanosecondArray::new(vec![*value; 1].into(), None).with_timezone_utc(),
1039        ))),
1040        (PrimitiveType::Decimal { precision, scale }, PrimitiveLiteral::Int128(value)) => {
1041            let array = Decimal128Array::from_value(*value, 1)
1042                .with_precision_and_scale(*precision as _, *scale as _)
1043                .unwrap();
1044            Ok(Arc::new(Scalar::new(array)))
1045        }
1046        (PrimitiveType::Uuid, PrimitiveLiteral::UInt128(value)) => {
1047            let bytes = Uuid::from_u128(*value).into_bytes();
1048            let array = FixedSizeBinaryArray::try_from_iter(vec![bytes].into_iter()).unwrap();
1049            Ok(Arc::new(Scalar::new(array)))
1050        }
1051        (PrimitiveType::Fixed(_), PrimitiveLiteral::Binary(value)) => {
1052            let array = FixedSizeBinaryArray::try_from_iter(std::iter::once(value.as_slice()))
1053                .map_err(|e| invalid_data!("FixedSizeBinary conversion failed").with_source(e))?;
1054            Ok(Arc::new(Scalar::new(array)))
1055        }
1056
1057        (primitive_type, _) => Err(Error::new(
1058            ErrorKind::FeatureUnsupported,
1059            format!("Converting datum from type {primitive_type:?} to arrow not supported yet."),
1060        )),
1061    }
1062}
1063
1064pub(crate) fn get_parquet_stat_min_as_datum(
1065    primitive_type: &PrimitiveType,
1066    stats: &Statistics,
1067) -> Result<Option<Datum>> {
1068    Ok(match (primitive_type, stats) {
1069        (PrimitiveType::Boolean, Statistics::Boolean(stats)) => {
1070            stats.min_opt().map(|val| Datum::bool(*val))
1071        }
1072        (PrimitiveType::Int, Statistics::Int32(stats)) => {
1073            stats.min_opt().map(|val| Datum::int(*val))
1074        }
1075        (PrimitiveType::Date, Statistics::Int32(stats)) => {
1076            stats.min_opt().map(|val| Datum::date(*val))
1077        }
1078        (PrimitiveType::Long, Statistics::Int64(stats)) => {
1079            stats.min_opt().map(|val| Datum::long(*val))
1080        }
1081        (PrimitiveType::Time, Statistics::Int64(stats)) => {
1082            let Some(val) = stats.min_opt() else {
1083                return Ok(None);
1084            };
1085
1086            Some(Datum::time_micros(*val)?)
1087        }
1088        (PrimitiveType::Timestamp, Statistics::Int64(stats)) => {
1089            stats.min_opt().map(|val| Datum::timestamp_micros(*val))
1090        }
1091        (PrimitiveType::Timestamptz, Statistics::Int64(stats)) => {
1092            stats.min_opt().map(|val| Datum::timestamptz_micros(*val))
1093        }
1094        (PrimitiveType::TimestampNs, Statistics::Int64(stats)) => {
1095            stats.min_opt().map(|val| Datum::timestamp_nanos(*val))
1096        }
1097        (PrimitiveType::TimestamptzNs, Statistics::Int64(stats)) => {
1098            stats.min_opt().map(|val| Datum::timestamptz_nanos(*val))
1099        }
1100        (PrimitiveType::Float, Statistics::Float(stats)) => {
1101            stats.min_opt().map(|val| Datum::float(*val))
1102        }
1103        (PrimitiveType::Double, Statistics::Double(stats)) => {
1104            stats.min_opt().map(|val| Datum::double(*val))
1105        }
1106        (PrimitiveType::String, Statistics::ByteArray(stats)) => {
1107            let Some(val) = stats.min_opt() else {
1108                return Ok(None);
1109            };
1110
1111            Some(Datum::string(val.as_utf8()?))
1112        }
1113        (
1114            PrimitiveType::Decimal {
1115                precision: _,
1116                scale: _,
1117            },
1118            Statistics::ByteArray(stats),
1119        ) => {
1120            let Some(bytes) = stats.min_bytes_opt() else {
1121                return Ok(None);
1122            };
1123            Some(Datum::new(
1124                primitive_type.clone(),
1125                PrimitiveLiteral::Int128(
1126                    // An empty bound carries no value; reject it rather than
1127                    // decode it as zero, which could prune matching row groups.
1128                    i128_from_be_bytes(bytes)
1129                        .filter(|_| !bytes.is_empty())
1130                        .ok_or_else(|| invalid_data!("Can't convert bytes to i128: {bytes:?}"))?,
1131                ),
1132            ))
1133        }
1134        (
1135            PrimitiveType::Decimal {
1136                precision: _,
1137                scale: _,
1138            },
1139            Statistics::FixedLenByteArray(stats),
1140        ) => {
1141            let Some(bytes) = stats.min_bytes_opt() else {
1142                return Ok(None);
1143            };
1144            Some(Datum::new(
1145                primitive_type.clone(),
1146                PrimitiveLiteral::Int128(
1147                    i128_from_be_bytes(bytes)
1148                        .ok_or_else(|| invalid_data!("Can't convert bytes to i128: {bytes:?}"))?,
1149                ),
1150            ))
1151        }
1152        (
1153            PrimitiveType::Decimal {
1154                precision: _,
1155                scale: _,
1156            },
1157            Statistics::Int32(stats),
1158        ) => stats.min_opt().map(|val| {
1159            Datum::new(
1160                primitive_type.clone(),
1161                PrimitiveLiteral::Int128(i128::from(*val)),
1162            )
1163        }),
1164
1165        (
1166            PrimitiveType::Decimal {
1167                precision: _,
1168                scale: _,
1169            },
1170            Statistics::Int64(stats),
1171        ) => stats.min_opt().map(|val| {
1172            Datum::new(
1173                primitive_type.clone(),
1174                PrimitiveLiteral::Int128(i128::from(*val)),
1175            )
1176        }),
1177        (PrimitiveType::Uuid, Statistics::FixedLenByteArray(stats)) => {
1178            let Some(bytes) = stats.min_bytes_opt() else {
1179                return Ok(None);
1180            };
1181            if bytes.len() != 16 {
1182                return Err(Error::new(
1183                    ErrorKind::Unexpected,
1184                    "Invalid length of uuid bytes.",
1185                ));
1186            }
1187            Some(Datum::uuid(Uuid::from_bytes(
1188                bytes[..16].try_into().unwrap(),
1189            )))
1190        }
1191        (PrimitiveType::Fixed(len), Statistics::FixedLenByteArray(stat)) => {
1192            let Some(bytes) = stat.min_bytes_opt() else {
1193                return Ok(None);
1194            };
1195            if bytes.len() != *len as usize {
1196                return Err(Error::new(
1197                    ErrorKind::Unexpected,
1198                    "Invalid length of fixed bytes.",
1199                ));
1200            }
1201            Some(Datum::fixed(bytes.to_vec()))
1202        }
1203        (PrimitiveType::Binary, Statistics::ByteArray(stat)) => {
1204            return Ok(stat
1205                .min_bytes_opt()
1206                .map(|bytes| Datum::binary(bytes.to_vec())));
1207        }
1208        _ => {
1209            return Ok(None);
1210        }
1211    })
1212}
1213
1214pub(crate) fn get_parquet_stat_max_as_datum(
1215    primitive_type: &PrimitiveType,
1216    stats: &Statistics,
1217) -> Result<Option<Datum>> {
1218    Ok(match (primitive_type, stats) {
1219        (PrimitiveType::Boolean, Statistics::Boolean(stats)) => {
1220            stats.max_opt().map(|val| Datum::bool(*val))
1221        }
1222        (PrimitiveType::Int, Statistics::Int32(stats)) => {
1223            stats.max_opt().map(|val| Datum::int(*val))
1224        }
1225        (PrimitiveType::Date, Statistics::Int32(stats)) => {
1226            stats.max_opt().map(|val| Datum::date(*val))
1227        }
1228        (PrimitiveType::Long, Statistics::Int64(stats)) => {
1229            stats.max_opt().map(|val| Datum::long(*val))
1230        }
1231        (PrimitiveType::Time, Statistics::Int64(stats)) => {
1232            let Some(val) = stats.max_opt() else {
1233                return Ok(None);
1234            };
1235
1236            Some(Datum::time_micros(*val)?)
1237        }
1238        (PrimitiveType::Timestamp, Statistics::Int64(stats)) => {
1239            stats.max_opt().map(|val| Datum::timestamp_micros(*val))
1240        }
1241        (PrimitiveType::Timestamptz, Statistics::Int64(stats)) => {
1242            stats.max_opt().map(|val| Datum::timestamptz_micros(*val))
1243        }
1244        (PrimitiveType::TimestampNs, Statistics::Int64(stats)) => {
1245            stats.max_opt().map(|val| Datum::timestamp_nanos(*val))
1246        }
1247        (PrimitiveType::TimestamptzNs, Statistics::Int64(stats)) => {
1248            stats.max_opt().map(|val| Datum::timestamptz_nanos(*val))
1249        }
1250        (PrimitiveType::Float, Statistics::Float(stats)) => {
1251            stats.max_opt().map(|val| Datum::float(*val))
1252        }
1253        (PrimitiveType::Double, Statistics::Double(stats)) => {
1254            stats.max_opt().map(|val| Datum::double(*val))
1255        }
1256        (PrimitiveType::String, Statistics::ByteArray(stats)) => {
1257            let Some(val) = stats.max_opt() else {
1258                return Ok(None);
1259            };
1260
1261            Some(Datum::string(val.as_utf8()?))
1262        }
1263        (
1264            PrimitiveType::Decimal {
1265                precision: _,
1266                scale: _,
1267            },
1268            Statistics::ByteArray(stats),
1269        ) => {
1270            let Some(bytes) = stats.max_bytes_opt() else {
1271                return Ok(None);
1272            };
1273            Some(Datum::new(
1274                primitive_type.clone(),
1275                PrimitiveLiteral::Int128(
1276                    // An empty bound carries no value; reject it rather than
1277                    // decode it as zero, which could prune matching row groups.
1278                    i128_from_be_bytes(bytes)
1279                        .filter(|_| !bytes.is_empty())
1280                        .ok_or_else(|| invalid_data!("Can't convert bytes to i128: {bytes:?}"))?,
1281                ),
1282            ))
1283        }
1284        (
1285            PrimitiveType::Decimal {
1286                precision: _,
1287                scale: _,
1288            },
1289            Statistics::FixedLenByteArray(stats),
1290        ) => {
1291            let Some(bytes) = stats.max_bytes_opt() else {
1292                return Ok(None);
1293            };
1294            Some(Datum::new(
1295                primitive_type.clone(),
1296                PrimitiveLiteral::Int128(
1297                    i128_from_be_bytes(bytes)
1298                        .ok_or_else(|| invalid_data!("Can't convert bytes to i128: {bytes:?}"))?,
1299                ),
1300            ))
1301        }
1302        (
1303            PrimitiveType::Decimal {
1304                precision: _,
1305                scale: _,
1306            },
1307            Statistics::Int32(stats),
1308        ) => stats.max_opt().map(|val| {
1309            Datum::new(
1310                primitive_type.clone(),
1311                PrimitiveLiteral::Int128(i128::from(*val)),
1312            )
1313        }),
1314
1315        (
1316            PrimitiveType::Decimal {
1317                precision: _,
1318                scale: _,
1319            },
1320            Statistics::Int64(stats),
1321        ) => stats.max_opt().map(|val| {
1322            Datum::new(
1323                primitive_type.clone(),
1324                PrimitiveLiteral::Int128(i128::from(*val)),
1325            )
1326        }),
1327        (PrimitiveType::Uuid, Statistics::FixedLenByteArray(stats)) => {
1328            let Some(bytes) = stats.max_bytes_opt() else {
1329                return Ok(None);
1330            };
1331            if bytes.len() != 16 {
1332                return Err(Error::new(
1333                    ErrorKind::Unexpected,
1334                    "Invalid length of uuid bytes.",
1335                ));
1336            }
1337            Some(Datum::uuid(Uuid::from_bytes(
1338                bytes[..16].try_into().unwrap(),
1339            )))
1340        }
1341        (PrimitiveType::Fixed(len), Statistics::FixedLenByteArray(stat)) => {
1342            let Some(bytes) = stat.max_bytes_opt() else {
1343                return Ok(None);
1344            };
1345            if bytes.len() != *len as usize {
1346                return Err(Error::new(
1347                    ErrorKind::Unexpected,
1348                    "Invalid length of fixed bytes.",
1349                ));
1350            }
1351            Some(Datum::fixed(bytes.to_vec()))
1352        }
1353        (PrimitiveType::Binary, Statistics::ByteArray(stat)) => {
1354            return Ok(stat
1355                .max_bytes_opt()
1356                .map(|bytes| Datum::binary(bytes.to_vec())));
1357        }
1358        _ => {
1359            return Ok(None);
1360        }
1361    })
1362}
1363
1364impl TryFrom<&ArrowSchema> for Schema {
1365    type Error = Error;
1366
1367    fn try_from(schema: &ArrowSchema) -> Result<Self> {
1368        arrow_schema_to_schema(schema)
1369    }
1370}
1371
1372impl TryFrom<&Schema> for ArrowSchema {
1373    type Error = Error;
1374
1375    fn try_from(schema: &Schema) -> Result<Self> {
1376        schema_to_arrow_schema(schema)
1377    }
1378}
1379
1380/// Converts a Datum (Iceberg type + primitive literal) to its corresponding Arrow DataType
1381/// with Run-End Encoding (REE).
1382///
1383/// This function is used for constant fields in record batches, where all values are the same.
1384/// Run-End Encoding provides efficient storage for such constant columns.
1385///
1386/// # Arguments
1387/// * `datum` - The Datum to convert, which contains both type and value information
1388///
1389/// # Returns
1390/// Arrow DataType with Run-End Encoding applied
1391///
1392/// # Example
1393/// ```
1394/// use iceberg::arrow::datum_to_arrow_type_with_ree;
1395/// use iceberg::spec::Datum;
1396///
1397/// let datum = Datum::string("test_file.parquet");
1398/// let ree_type = datum_to_arrow_type_with_ree(&datum);
1399/// // Returns: RunEndEncoded(Int32, Utf8)
1400/// ```
1401pub fn datum_to_arrow_type_with_ree(datum: &Datum) -> DataType {
1402    primitive_type_to_arrow_type_with_ree(datum.data_type())
1403}
1404
1405/// Returns the run-end-encoded Arrow type used to materialize a per-file constant
1406/// column of the given Iceberg primitive type. Shared by both the value and the
1407/// all-null constant paths so a column keeps one Arrow type across files.
1408pub(crate) fn primitive_type_to_arrow_type_with_ree(primitive_type: &PrimitiveType) -> DataType {
1409    // Helper to create REE type with the given values type.
1410    // Note: values field is nullable as Arrow expects this when building the
1411    // final Arrow schema with `RunArray::try_new`.
1412    let make_ree = |values_type: DataType| -> DataType {
1413        let run_ends_field = Arc::new(Field::new("run_ends", DataType::Int32, false));
1414        let values_field = Arc::new(Field::new("values", values_type, true));
1415        DataType::RunEndEncoded(run_ends_field, values_field)
1416    };
1417
1418    match primitive_type {
1419        PrimitiveType::Unknown => make_ree(DataType::Null),
1420        PrimitiveType::Boolean => make_ree(DataType::Boolean),
1421        PrimitiveType::Int => make_ree(DataType::Int32),
1422        PrimitiveType::Long => make_ree(DataType::Int64),
1423        PrimitiveType::Float => make_ree(DataType::Float32),
1424        PrimitiveType::Double => make_ree(DataType::Float64),
1425        PrimitiveType::Date => make_ree(DataType::Date32),
1426        PrimitiveType::Time => make_ree(DataType::Int64),
1427        PrimitiveType::Timestamp => make_ree(DataType::Int64),
1428        PrimitiveType::Timestamptz => make_ree(DataType::Int64),
1429        PrimitiveType::TimestampNs => make_ree(DataType::Int64),
1430        PrimitiveType::TimestamptzNs => make_ree(DataType::Int64),
1431        PrimitiveType::String => make_ree(DataType::Utf8),
1432        PrimitiveType::Uuid => make_ree(DataType::Binary),
1433        PrimitiveType::Fixed(_) => make_ree(DataType::Binary),
1434        PrimitiveType::Binary => make_ree(DataType::Binary),
1435        // Match the LargeBinary storage used by schema_to_arrow_schema in the read layer.
1436        // Its 64-bit offsets cover the array's total WKB bytes, not only one geometry's size.
1437        PrimitiveType::Geometry(_) | PrimitiveType::Geography(_) => make_ree(DataType::LargeBinary),
1438        PrimitiveType::Decimal { precision, scale } => {
1439            make_ree(DataType::Decimal128(*precision as u8, *scale as i8))
1440        }
1441    }
1442}
1443
1444/// A visitor that strips metadata from an Arrow schema.
1445///
1446/// This visitor recursively removes all metadata from fields at every level of the schema,
1447/// including nested struct, list, and map fields. This is useful for schema comparison
1448/// where metadata differences should be ignored.
1449struct MetadataStripVisitor {
1450    /// Fields whose types are still being visited, innermost last
1451    pending_fields: Vec<PendingField>,
1452}
1453
1454/// What `MetadataStripVisitor` keeps of a field until its type is visited
1455struct PendingField {
1456    name: String,
1457    nullable: bool,
1458}
1459
1460impl MetadataStripVisitor {
1461    fn new() -> Self {
1462        Self {
1463            pending_fields: Vec::new(),
1464        }
1465    }
1466
1467    /// Records a field whose type is about to be visited.
1468    fn start_field(&mut self, name: &str, nullable: bool) -> Result<()> {
1469        self.pending_fields.push(PendingField {
1470            name: name.to_string(),
1471            nullable,
1472        });
1473        Ok(())
1474    }
1475
1476    /// Builds the innermost pending field with `data_type` and no metadata.
1477    /// `kind` names the type in the error for an unbalanced stack.
1478    fn finish_field(&mut self, data_type: DataType, kind: &str) -> Result<Field> {
1479        let PendingField { name, nullable } = self.pending_fields.pop().ok_or_else(|| {
1480            Error::new(
1481                ErrorKind::Unexpected,
1482                format!("Field stack underflow in {kind}"),
1483            )
1484        })?;
1485        Ok(Field::new(name, data_type, nullable))
1486    }
1487}
1488
1489impl ArrowSchemaVisitor for MetadataStripVisitor {
1490    type T = Field;
1491    type U = ArrowSchema;
1492
1493    fn before_field(&mut self, field: &FieldRef) -> Result<()> {
1494        self.start_field(field.name(), field.is_nullable())
1495    }
1496
1497    fn before_list_element(&mut self, field: &FieldRef) -> Result<()> {
1498        self.start_field(LIST_FIELD_NAME, field.is_nullable())
1499    }
1500
1501    fn before_map_key(&mut self, _field: &FieldRef) -> Result<()> {
1502        // Map keys are always required, as `schema_to_arrow_schema` builds them.
1503        self.start_field(MAP_KEY_FIELD_NAME, false)
1504    }
1505
1506    fn before_map_value(&mut self, field: &FieldRef) -> Result<()> {
1507        self.start_field(MAP_VALUE_FIELD_NAME, field.is_nullable())
1508    }
1509
1510    fn schema(&mut self, _schema: &ArrowSchema, values: Vec<Self::T>) -> Result<Self::U> {
1511        if !self.pending_fields.is_empty() {
1512            return Err(Error::new(
1513                ErrorKind::Unexpected,
1514                format!(
1515                    "{} fields left on the field stack after traversal",
1516                    self.pending_fields.len()
1517                ),
1518            ));
1519        }
1520        Ok(ArrowSchema::new(values))
1521    }
1522
1523    fn r#struct(&mut self, _fields: &Fields, results: Vec<Self::T>) -> Result<Self::T> {
1524        self.finish_field(DataType::Struct(Fields::from(results)), "struct")
1525    }
1526
1527    fn list(&mut self, list: &DataType, value: Self::T) -> Result<Self::T> {
1528        let list_type = match list {
1529            DataType::List(_) => DataType::List(Arc::new(value)),
1530            DataType::LargeList(_) => DataType::LargeList(Arc::new(value)),
1531            DataType::FixedSizeList(_, size) => DataType::FixedSizeList(Arc::new(value), *size),
1532            _ => {
1533                return Err(Error::new(
1534                    ErrorKind::Unexpected,
1535                    format!("Expected list type, got {list}"),
1536                ));
1537            }
1538        };
1539
1540        self.finish_field(list_type, "list")
1541    }
1542
1543    fn map(&mut self, map: &DataType, key_value: Self::T, value: Self::T) -> Result<Self::T> {
1544        // Reconstruct the map's struct field (contains key and value)
1545        let struct_field = Field::new(
1546            DEFAULT_MAP_FIELD_NAME,
1547            DataType::Struct(Fields::from(vec![key_value, value])),
1548            false,
1549        );
1550
1551        // Get the sorted flag from the original map type
1552        let sorted = match map {
1553            DataType::Map(_, sorted) => *sorted,
1554            _ => {
1555                return Err(Error::new(
1556                    ErrorKind::Unexpected,
1557                    format!("Expected map type, got {map}"),
1558                ));
1559            }
1560        };
1561
1562        self.finish_field(DataType::Map(Arc::new(struct_field), sorted), "map")
1563    }
1564
1565    fn primitive(&mut self, p: &DataType) -> Result<Self::T> {
1566        self.finish_field(p.clone(), "primitive")
1567    }
1568}
1569
1570/// Strips all metadata from an Arrow schema and its nested fields.
1571///
1572/// This function recursively removes metadata from all fields at every level of the schema,
1573/// including nested struct, list, and map fields. This is useful for schema comparison
1574/// where metadata differences should be ignored.
1575///
1576/// Some other differences are normalized away too, so that schemas that differ only in
1577/// them compare equal once stripped. The fields inside lists and maps are renamed as
1578/// [`schema_to_arrow_schema`] names them, and dictionaries are unwrapped:
1579/// - A list's element field is renamed to [`LIST_FIELD_NAME`], so `item` (the arrow-rs
1580///   default) becomes `element`.
1581/// - A map's entries field is renamed to [`DEFAULT_MAP_FIELD_NAME`] and made non-nullable.
1582/// - A map's key and value fields are renamed to [`MAP_KEY_FIELD_NAME`] and
1583///   [`MAP_VALUE_FIELD_NAME`], so `keys` and `values` (the arrow-rs `MapBuilder` default)
1584///   become `key` and `value`, and the key is made non-nullable.
1585/// - A dictionary-encoded field becomes a field of the dictionary's value type.
1586///
1587/// The names of the schema's fields and of struct fields are kept. Extension type
1588/// metadata (`ARROW:extension:*`) is dropped with the rest, so a variant field becomes a
1589/// plain struct of its `metadata` and `value` fields.
1590///
1591/// # Arguments
1592/// * `schema` - The Arrow schema to strip metadata from
1593///
1594/// # Returns
1595/// A new Arrow schema with all metadata removed, or an error if the schema structure
1596/// is invalid.
1597///
1598/// # Example
1599/// ```
1600/// use std::collections::HashMap;
1601///
1602/// use arrow_schema::{DataType, Field, Schema as ArrowSchema};
1603/// use iceberg::arrow::strip_metadata_from_schema;
1604///
1605/// let mut metadata = HashMap::new();
1606/// metadata.insert("key".to_string(), "value".to_string());
1607///
1608/// let field = Field::new("col1", DataType::Int32, false).with_metadata(metadata);
1609/// let schema = ArrowSchema::new(vec![field]);
1610///
1611/// let stripped = strip_metadata_from_schema(&schema).unwrap();
1612/// assert!(stripped.field(0).metadata().is_empty());
1613/// ```
1614pub fn strip_metadata_from_schema(schema: &ArrowSchema) -> Result<ArrowSchema> {
1615    let mut visitor = MetadataStripVisitor::new();
1616    visit_schema(schema, &mut visitor)
1617}
1618
1619#[cfg(test)]
1620mod tests {
1621    use std::collections::HashMap;
1622    use std::sync::Arc;
1623
1624    use arrow_schema::extension::ExtensionType;
1625    use arrow_schema::{DataType, Field, Schema as ArrowSchema, TimeUnit};
1626    use parquet_geospatial::WkbEdges;
1627
1628    use super::*;
1629    use crate::spec::decimal_utils::decimal_new;
1630    use crate::spec::geospatial::{
1631        DEFAULT_GEOSPATIAL_CRS, EdgeInterpolationAlgorithm as IcebergEdgeInterpolationAlgorithm,
1632    };
1633    use crate::spec::{Literal, Schema};
1634
1635    /// Create a simple field with metadata.
1636    fn simple_field(name: &str, ty: DataType, nullable: bool, value: &str) -> Field {
1637        Field::new(name, ty, nullable).with_metadata(HashMap::from([(
1638            PARQUET_FIELD_ID_META_KEY.to_string(),
1639            value.to_string(),
1640        )]))
1641    }
1642
1643    fn arrow_schema_for_arrow_schema_to_schema_test() -> ArrowSchema {
1644        let fields = Fields::from(vec![
1645            simple_field("key", DataType::Int32, false, "28"),
1646            simple_field("value", DataType::Utf8, true, "29"),
1647        ]);
1648
1649        let r#struct = DataType::Struct(fields);
1650        let map = DataType::Map(
1651            Arc::new(simple_field(DEFAULT_MAP_FIELD_NAME, r#struct, false, "17")),
1652            false,
1653        );
1654        let dictionary = DataType::Dictionary(Box::new(DataType::Int32), Box::new(DataType::Utf8));
1655
1656        let fields = Fields::from(vec![
1657            simple_field("aa", DataType::Int32, false, "18"),
1658            simple_field("bb", DataType::Utf8, true, "19"),
1659            simple_field(
1660                "cc",
1661                DataType::Timestamp(TimeUnit::Microsecond, None),
1662                false,
1663                "20",
1664            ),
1665        ]);
1666
1667        let r#struct = DataType::Struct(fields);
1668
1669        ArrowSchema::new(vec![
1670            simple_field("a", DataType::Int32, false, "2"),
1671            simple_field("b", DataType::Int64, false, "1"),
1672            simple_field("c", DataType::Utf8, false, "3"),
1673            simple_field("n", DataType::Utf8, false, "21"),
1674            simple_field(
1675                "d",
1676                DataType::Timestamp(TimeUnit::Microsecond, None),
1677                true,
1678                "4",
1679            ),
1680            simple_field("e", DataType::Boolean, true, "6"),
1681            simple_field("f", DataType::Float32, false, "5"),
1682            simple_field("g", DataType::Float64, false, "7"),
1683            simple_field("p", DataType::Decimal128(10, 2), false, "27"),
1684            simple_field("h", DataType::Date32, false, "8"),
1685            simple_field("i", DataType::Time64(TimeUnit::Microsecond), false, "9"),
1686            simple_field(
1687                "j",
1688                DataType::Timestamp(TimeUnit::Microsecond, Some("UTC".into())),
1689                false,
1690                "10",
1691            ),
1692            simple_field(
1693                "k",
1694                DataType::Timestamp(TimeUnit::Microsecond, Some("+00:00".into())),
1695                false,
1696                "12",
1697            ),
1698            simple_field("l", DataType::Binary, false, "13"),
1699            simple_field("o", DataType::LargeBinary, false, "22"),
1700            simple_field("m", DataType::FixedSizeBinary(10), false, "11"),
1701            simple_field(
1702                "list",
1703                DataType::List(Arc::new(simple_field(
1704                    "element",
1705                    DataType::Int32,
1706                    false,
1707                    "15",
1708                ))),
1709                true,
1710                "14",
1711            ),
1712            simple_field(
1713                "large_list",
1714                DataType::LargeList(Arc::new(simple_field(
1715                    "element",
1716                    DataType::Utf8,
1717                    false,
1718                    "23",
1719                ))),
1720                true,
1721                "24",
1722            ),
1723            simple_field(
1724                "fixed_list",
1725                DataType::FixedSizeList(
1726                    Arc::new(simple_field("element", DataType::Binary, false, "26")),
1727                    10,
1728                ),
1729                true,
1730                "25",
1731            ),
1732            simple_field("map", map, false, "16"),
1733            simple_field("struct", r#struct, false, "17"),
1734            simple_field("dictionary", dictionary, false, "30"),
1735        ])
1736    }
1737
1738    fn iceberg_schema_for_arrow_schema_to_schema_test() -> Schema {
1739        let schema_json = r#"{
1740            "type":"struct",
1741            "schema-id":0,
1742            "fields":[
1743                {
1744                    "id":2,
1745                    "name":"a",
1746                    "required":true,
1747                    "type":"int"
1748                },
1749                {
1750                    "id":1,
1751                    "name":"b",
1752                    "required":true,
1753                    "type":"long"
1754                },
1755                {
1756                    "id":3,
1757                    "name":"c",
1758                    "required":true,
1759                    "type":"string"
1760                },
1761                {
1762                    "id":21,
1763                    "name":"n",
1764                    "required":true,
1765                    "type":"string"
1766                },
1767                {
1768                    "id":4,
1769                    "name":"d",
1770                    "required":false,
1771                    "type":"timestamp"
1772                },
1773                {
1774                    "id":6,
1775                    "name":"e",
1776                    "required":false,
1777                    "type":"boolean"
1778                },
1779                {
1780                    "id":5,
1781                    "name":"f",
1782                    "required":true,
1783                    "type":"float"
1784                },
1785                {
1786                    "id":7,
1787                    "name":"g",
1788                    "required":true,
1789                    "type":"double"
1790                },
1791                {
1792                    "id":27,
1793                    "name":"p",
1794                    "required":true,
1795                    "type":"decimal(10,2)"
1796                },
1797                {
1798                    "id":8,
1799                    "name":"h",
1800                    "required":true,
1801                    "type":"date"
1802                },
1803                {
1804                    "id":9,
1805                    "name":"i",
1806                    "required":true,
1807                    "type":"time"
1808                },
1809                {
1810                    "id":10,
1811                    "name":"j",
1812                    "required":true,
1813                    "type":"timestamptz"
1814                },
1815                {
1816                    "id":12,
1817                    "name":"k",
1818                    "required":true,
1819                    "type":"timestamptz"
1820                },
1821                {
1822                    "id":13,
1823                    "name":"l",
1824                    "required":true,
1825                    "type":"binary"
1826                },
1827                {
1828                    "id":22,
1829                    "name":"o",
1830                    "required":true,
1831                    "type":"binary"
1832                },
1833                {
1834                    "id":11,
1835                    "name":"m",
1836                    "required":true,
1837                    "type":"fixed[10]"
1838                },
1839                {
1840                    "id":14,
1841                    "name":"list",
1842                    "required": false,
1843                    "type": {
1844                        "type": "list",
1845                        "element-id": 15,
1846                        "element-required": true,
1847                        "element": "int"
1848                    }
1849                },
1850                {
1851                    "id":24,
1852                    "name":"large_list",
1853                    "required": false,
1854                    "type": {
1855                        "type": "list",
1856                        "element-id": 23,
1857                        "element-required": true,
1858                        "element": "string"
1859                    }
1860                },
1861                {
1862                    "id":25,
1863                    "name":"fixed_list",
1864                    "required": false,
1865                    "type": {
1866                        "type": "list",
1867                        "element-id": 26,
1868                        "element-required": true,
1869                        "element": "binary"
1870                    }
1871                },
1872                {
1873                    "id":16,
1874                    "name":"map",
1875                    "required": true,
1876                    "type": {
1877                        "type": "map",
1878                        "key-id": 28,
1879                        "key": "int",
1880                        "value-id": 29,
1881                        "value-required": false,
1882                        "value": "string"
1883                    }
1884                },
1885                {
1886                    "id":17,
1887                    "name":"struct",
1888                    "required": true,
1889                    "type": {
1890                        "type": "struct",
1891                        "fields": [
1892                            {
1893                                "id":18,
1894                                "name":"aa",
1895                                "required":true,
1896                                "type":"int"
1897                            },
1898                            {
1899                                "id":19,
1900                                "name":"bb",
1901                                "required":false,
1902                                "type":"string"
1903                            },
1904                            {
1905                                "id":20,
1906                                "name":"cc",
1907                                "required":true,
1908                                "type":"timestamp"
1909                            }
1910                        ]
1911                    }
1912                },
1913                {
1914                    "id":30,
1915                    "name":"dictionary",
1916                    "required":true,
1917                    "type":"string"
1918                }
1919            ],
1920            "identifier-field-ids":[]
1921        }"#;
1922
1923        let schema: Schema = serde_json::from_str(schema_json).unwrap();
1924        schema
1925    }
1926
1927    #[test]
1928    fn test_arrow_schema_to_schema() {
1929        let arrow_schema = arrow_schema_for_arrow_schema_to_schema_test();
1930        let schema = iceberg_schema_for_arrow_schema_to_schema_test();
1931        let converted_schema = arrow_schema_to_schema(&arrow_schema).unwrap();
1932        pretty_assertions::assert_eq!(converted_schema, schema);
1933    }
1934
1935    #[test]
1936    fn test_converting_uuid_from_arrow_to_iceberg_to_arrow_should_give_uuid() {
1937        let arrow_schema = ArrowSchema::new(vec![
1938            simple_field("uuid_field", DataType::FixedSizeBinary(16), false, "1")
1939                .with_extension_type(UuidExtensionType),
1940        ]);
1941        let output =
1942            schema_to_arrow_schema(&arrow_schema_to_schema(&arrow_schema).unwrap()).unwrap();
1943
1944        assert_eq!(arrow_schema, output);
1945    }
1946
1947    #[test]
1948    fn test_converting_uuid_from_iceberg_to_arrow_to_iceberg_should_give_uuid() {
1949        let iceberg_schema = Schema::builder()
1950            .with_fields(vec![
1951                NestedField::optional(1, "uuid_field", Type::Primitive(PrimitiveType::Uuid)).into(),
1952            ])
1953            .build()
1954            .unwrap();
1955        let output =
1956            arrow_schema_to_schema(&schema_to_arrow_schema(&iceberg_schema).unwrap()).unwrap();
1957
1958        assert_eq!(iceberg_schema, output);
1959    }
1960
1961    #[test]
1962    fn test_arrow_schema_to_schema_should_convert_uuid_when_fixed_size_binary() {
1963        let converted_schema = arrow_schema_to_schema(&ArrowSchema::new(vec![
1964            simple_field("uuid_field", DataType::FixedSizeBinary(16), false, "1")
1965                .with_extension_type(UuidExtensionType),
1966        ]))
1967        .unwrap();
1968
1969        let expected = Schema::builder()
1970            .with_fields([NestedField::required(
1971                1,
1972                "uuid_field",
1973                Type::Primitive(PrimitiveType::Uuid),
1974            )
1975            .into()])
1976            .build()
1977            .unwrap();
1978
1979        pretty_assertions::assert_eq!(expected, converted_schema);
1980    }
1981
1982    #[test]
1983    fn test_arrow_schema_to_schema_should_reject_uuid_when_not_a_fixed_size_binary() {
1984        // The field must be built correctly, and then changed to the wrong type,
1985        // to avoid Arrow's own validation and panic.
1986        let field = simple_field_with_uuid_ext(DataType::Utf8);
1987
1988        let error = arrow_schema_to_schema(&ArrowSchema::new(vec![field])).unwrap_err();
1989
1990        pretty_assertions::assert_eq!(
1991            "arrow.uuid extension requires FixedSizeBinary(16) storage, found Utf8",
1992            error.message()
1993        );
1994    }
1995
1996    #[test]
1997    fn test_arrow_schema_to_schema_should_reject_uuid_when_fixed_size_binary_that_is_less_than_16()
1998    {
1999        let field = simple_field_with_uuid_ext(DataType::FixedSizeBinary(8));
2000        let error = arrow_schema_to_schema(&ArrowSchema::new(vec![field])).unwrap_err();
2001
2002        pretty_assertions::assert_eq!(
2003            "arrow.uuid extension requires FixedSizeBinary(16) storage, found FixedSizeBinary(8)",
2004            error.message()
2005        );
2006    }
2007
2008    #[test]
2009    fn test_arrow_schema_to_schema_should_reject_uuid_when_fixed_size_binary_that_is_more_than_16()
2010    {
2011        let field = simple_field_with_uuid_ext(DataType::FixedSizeBinary(17));
2012        let error = arrow_schema_to_schema(&ArrowSchema::new(vec![field])).unwrap_err();
2013
2014        pretty_assertions::assert_eq!(
2015            "arrow.uuid extension requires FixedSizeBinary(16) storage, found FixedSizeBinary(17)",
2016            error.message()
2017        );
2018    }
2019
2020    fn simple_field_with_uuid_ext(field_data_type: DataType) -> Field {
2021        let mut field = simple_field("uuid_field", DataType::FixedSizeBinary(16), false, "1")
2022            .with_extension_type(UuidExtensionType);
2023
2024        // We want to allow setting a UUID extension to the wrong data type.
2025        // This is to test our logic against bad schemas.
2026        //
2027        // To avoid Arrow's own internal validation and panic, we need to build
2028        // the field correctly above, and then manually override the type here.
2029        field = field.with_data_type(field_data_type);
2030
2031        field
2032    }
2033
2034    fn arrow_schema_for_schema_to_arrow_schema_test() -> ArrowSchema {
2035        let fields = Fields::from(vec![
2036            simple_field("key", DataType::Int32, false, "28"),
2037            simple_field("value", DataType::Utf8, true, "29"),
2038        ]);
2039
2040        let r#struct = DataType::Struct(fields);
2041        let map = DataType::Map(
2042            Arc::new(Field::new(DEFAULT_MAP_FIELD_NAME, r#struct, false)),
2043            false,
2044        );
2045
2046        let fields = Fields::from(vec![
2047            simple_field("aa", DataType::Int32, false, "18"),
2048            simple_field("bb", DataType::Utf8, true, "19"),
2049            simple_field(
2050                "cc",
2051                DataType::Timestamp(TimeUnit::Microsecond, None),
2052                false,
2053                "20",
2054            ),
2055        ]);
2056
2057        let r#struct = DataType::Struct(fields);
2058
2059        ArrowSchema::new(vec![
2060            simple_field("a", DataType::Int32, false, "2"),
2061            simple_field("b", DataType::Int64, false, "1"),
2062            simple_field("c", DataType::Utf8, false, "3"),
2063            simple_field("n", DataType::Utf8, false, "21"),
2064            simple_field(
2065                "d",
2066                DataType::Timestamp(TimeUnit::Microsecond, None),
2067                true,
2068                "4",
2069            ),
2070            simple_field("e", DataType::Boolean, true, "6"),
2071            simple_field("f", DataType::Float32, false, "5"),
2072            simple_field("g", DataType::Float64, false, "7"),
2073            simple_field("p", DataType::Decimal128(10, 2), false, "27"),
2074            simple_field("h", DataType::Date32, false, "8"),
2075            simple_field("i", DataType::Time64(TimeUnit::Microsecond), false, "9"),
2076            simple_field(
2077                "j",
2078                DataType::Timestamp(TimeUnit::Microsecond, Some("+00:00".into())),
2079                false,
2080                "10",
2081            ),
2082            simple_field(
2083                "k",
2084                DataType::Timestamp(TimeUnit::Microsecond, Some("+00:00".into())),
2085                false,
2086                "12",
2087            ),
2088            simple_field("l", DataType::Binary, false, "13"),
2089            simple_field("o", DataType::Binary, false, "22"),
2090            simple_field("m", DataType::FixedSizeBinary(10), false, "11"),
2091            simple_field(
2092                "list",
2093                DataType::List(Arc::new(simple_field(
2094                    "element",
2095                    DataType::Int32,
2096                    false,
2097                    "15",
2098                ))),
2099                true,
2100                "14",
2101            ),
2102            simple_field(
2103                "large_list",
2104                DataType::List(Arc::new(simple_field(
2105                    "element",
2106                    DataType::Utf8,
2107                    false,
2108                    "23",
2109                ))),
2110                true,
2111                "24",
2112            ),
2113            simple_field(
2114                "fixed_list",
2115                DataType::List(Arc::new(simple_field(
2116                    "element",
2117                    DataType::Binary,
2118                    false,
2119                    "26",
2120                ))),
2121                true,
2122                "25",
2123            ),
2124            simple_field("map", map, false, "16"),
2125            simple_field("struct", r#struct, false, "17"),
2126            simple_field("uuid", DataType::FixedSizeBinary(16), false, "30")
2127                .with_extension_type(UuidExtensionType),
2128            Field::new(
2129                "v",
2130                DataType::Struct(Fields::from(vec![
2131                    Field::new("metadata", DataType::Binary, false),
2132                    Field::new("value", DataType::Binary, true),
2133                ])),
2134                true,
2135            )
2136            .with_metadata(HashMap::from([
2137                (PARQUET_FIELD_ID_META_KEY.to_string(), "31".to_string()),
2138                (
2139                    arrow_schema::extension::EXTENSION_TYPE_NAME_KEY.to_string(),
2140                    "arrow.parquet.variant".to_string(),
2141                ),
2142            ])),
2143        ])
2144    }
2145
2146    fn iceberg_schema_for_schema_to_arrow_schema() -> Schema {
2147        let schema_json = r#"{
2148            "type":"struct",
2149            "schema-id":0,
2150            "fields":[
2151                {
2152                    "id":2,
2153                    "name":"a",
2154                    "required":true,
2155                    "type":"int"
2156                },
2157                {
2158                    "id":1,
2159                    "name":"b",
2160                    "required":true,
2161                    "type":"long"
2162                },
2163                {
2164                    "id":3,
2165                    "name":"c",
2166                    "required":true,
2167                    "type":"string"
2168                },
2169                {
2170                    "id":21,
2171                    "name":"n",
2172                    "required":true,
2173                    "type":"string"
2174                },
2175                {
2176                    "id":4,
2177                    "name":"d",
2178                    "required":false,
2179                    "type":"timestamp"
2180                },
2181                {
2182                    "id":6,
2183                    "name":"e",
2184                    "required":false,
2185                    "type":"boolean"
2186                },
2187                {
2188                    "id":5,
2189                    "name":"f",
2190                    "required":true,
2191                    "type":"float"
2192                },
2193                {
2194                    "id":7,
2195                    "name":"g",
2196                    "required":true,
2197                    "type":"double"
2198                },
2199                {
2200                    "id":27,
2201                    "name":"p",
2202                    "required":true,
2203                    "type":"decimal(10,2)"
2204                },
2205                {
2206                    "id":8,
2207                    "name":"h",
2208                    "required":true,
2209                    "type":"date"
2210                },
2211                {
2212                    "id":9,
2213                    "name":"i",
2214                    "required":true,
2215                    "type":"time"
2216                },
2217                {
2218                    "id":10,
2219                    "name":"j",
2220                    "required":true,
2221                    "type":"timestamptz"
2222                },
2223                {
2224                    "id":12,
2225                    "name":"k",
2226                    "required":true,
2227                    "type":"timestamptz"
2228                },
2229                {
2230                    "id":13,
2231                    "name":"l",
2232                    "required":true,
2233                    "type":"binary"
2234                },
2235                {
2236                    "id":22,
2237                    "name":"o",
2238                    "required":true,
2239                    "type":"binary"
2240                },
2241                {
2242                    "id":11,
2243                    "name":"m",
2244                    "required":true,
2245                    "type":"fixed[10]"
2246                },
2247                {
2248                    "id":14,
2249                    "name":"list",
2250                    "required": false,
2251                    "type": {
2252                        "type": "list",
2253                        "element-id": 15,
2254                        "element-required": true,
2255                        "element": "int"
2256                    }
2257                },
2258                {
2259                    "id":24,
2260                    "name":"large_list",
2261                    "required": false,
2262                    "type": {
2263                        "type": "list",
2264                        "element-id": 23,
2265                        "element-required": true,
2266                        "element": "string"
2267                    }
2268                },
2269                {
2270                    "id":25,
2271                    "name":"fixed_list",
2272                    "required": false,
2273                    "type": {
2274                        "type": "list",
2275                        "element-id": 26,
2276                        "element-required": true,
2277                        "element": "binary"
2278                    }
2279                },
2280                {
2281                    "id":16,
2282                    "name":"map",
2283                    "required": true,
2284                    "type": {
2285                        "type": "map",
2286                        "key-id": 28,
2287                        "key": "int",
2288                        "value-id": 29,
2289                        "value-required": false,
2290                        "value": "string"
2291                    }
2292                },
2293                {
2294                    "id":17,
2295                    "name":"struct",
2296                    "required": true,
2297                    "type": {
2298                        "type": "struct",
2299                        "fields": [
2300                            {
2301                                "id":18,
2302                                "name":"aa",
2303                                "required":true,
2304                                "type":"int"
2305                            },
2306                            {
2307                                "id":19,
2308                                "name":"bb",
2309                                "required":false,
2310                                "type":"string"
2311                            },
2312                            {
2313                                "id":20,
2314                                "name":"cc",
2315                                "required":true,
2316                                "type":"timestamp"
2317                            }
2318                        ]
2319                    }
2320                },
2321                {
2322                    "id":30,
2323                    "name":"uuid",
2324                    "required":true,
2325                    "type":"uuid"
2326                },
2327                {
2328                    "id":31,
2329                    "name":"v",
2330                    "required":false,
2331                    "type":"variant"
2332                }
2333            ],
2334            "identifier-field-ids":[]
2335        }"#;
2336
2337        let schema: Schema = serde_json::from_str(schema_json).unwrap();
2338        schema
2339    }
2340
2341    #[test]
2342    fn test_schema_to_arrow_schema() {
2343        let expected_arrow_schema = arrow_schema_for_schema_to_arrow_schema_test();
2344        let schema = iceberg_schema_for_schema_to_arrow_schema();
2345        let converted_arrow_schema = schema_to_arrow_schema(&schema).unwrap();
2346        assert_eq!(converted_arrow_schema, expected_arrow_schema);
2347    }
2348
2349    #[test]
2350    fn test_variant_type_to_arrow_type() {
2351        // Variant maps to a struct with a required `metadata` and a nullable `value` binary
2352        // field, with no field ids on the sub-fields, matching the Parquet BINARY layout.
2353        let arrow_type = type_to_arrow_type(&Type::Variant(VariantType)).unwrap();
2354        assert_eq!(
2355            arrow_type,
2356            DataType::Struct(Fields::from(vec![
2357                Field::new("metadata", DataType::Binary, false),
2358                Field::new("value", DataType::Binary, true),
2359            ]))
2360        );
2361    }
2362
2363    #[test]
2364    fn test_variant_field_carries_arrow_extension_type() {
2365        // Converting a schema with a variant column tags the column's field with the
2366        // canonical `arrow.parquet.variant` extension type (the struct storage stays as-is).
2367        let schema = Schema::builder()
2368            .with_fields(vec![
2369                NestedField::optional(1, "v", Type::Variant(VariantType)).into(),
2370            ])
2371            .build()
2372            .unwrap();
2373
2374        let arrow_schema = schema_to_arrow_schema(&schema).unwrap();
2375        let field = arrow_schema.field_with_name("v").unwrap();
2376
2377        assert_eq!(field.extension_type_name(), Some("arrow.parquet.variant"));
2378        // Attaching the extension type must not clobber the Iceberg field id.
2379        assert_eq!(
2380            field.metadata().get(PARQUET_FIELD_ID_META_KEY),
2381            Some(&"1".to_string())
2382        );
2383        assert_eq!(
2384            field.data_type(),
2385            &DataType::Struct(Fields::from(vec![
2386                Field::new("metadata", DataType::Binary, false),
2387                Field::new("value", DataType::Binary, true),
2388            ]))
2389        );
2390    }
2391
2392    #[test]
2393    fn test_schema_to_arrow_schema_should_convert_uuid_to_arrow_uuid() {
2394        let schema = Schema::builder()
2395            .with_fields(vec![
2396                NestedField::optional(1, "uuid_field", Type::Primitive(PrimitiveType::Uuid)).into(),
2397            ])
2398            .build()
2399            .unwrap();
2400
2401        let arrow_schema = schema_to_arrow_schema(&schema).unwrap();
2402        let field = arrow_schema.field_with_name("uuid_field").unwrap();
2403
2404        assert_eq!(field.extension_type_name(), Some(UuidExtensionType::NAME));
2405        // Attaching the extension type must not clobber the Iceberg field id.
2406        assert_eq!(
2407            field.metadata().get(PARQUET_FIELD_ID_META_KEY),
2408            Some(&"1".to_string())
2409        );
2410        assert_eq!(*field.data_type(), DataType::FixedSizeBinary(16));
2411    }
2412
2413    #[test]
2414    fn test_variant_nested_in_list_and_map_carries_arrow_extension_type() {
2415        // A variant nested in a list element or map value keeps the arrow.parquet.variant
2416        // extension type. Regression guard: the list converter must not overwrite the
2417        // element field's metadata (which would drop the extension type).
2418        let schema = Schema::builder()
2419            .with_fields(vec![
2420                NestedField::optional(
2421                    1,
2422                    "l",
2423                    Type::List(ListType::new(
2424                        NestedField::optional(2, "element", Type::Variant(VariantType)).into(),
2425                    )),
2426                )
2427                .into(),
2428                NestedField::optional(
2429                    3,
2430                    "m",
2431                    Type::Map(MapType::new(
2432                        NestedField::map_key_element(4, Type::Primitive(PrimitiveType::String))
2433                            .into(),
2434                        NestedField::map_value_element(5, Type::Variant(VariantType), false).into(),
2435                    )),
2436                )
2437                .into(),
2438            ])
2439            .build()
2440            .unwrap();
2441
2442        let arrow_schema = schema_to_arrow_schema(&schema).unwrap();
2443
2444        let DataType::List(element) = arrow_schema.field_with_name("l").unwrap().data_type() else {
2445            panic!("expected a list");
2446        };
2447        assert_eq!(element.extension_type_name(), Some("arrow.parquet.variant"));
2448
2449        let DataType::Map(entries, _) = arrow_schema.field_with_name("m").unwrap().data_type()
2450        else {
2451            panic!("expected a map");
2452        };
2453        let DataType::Struct(kv) = entries.data_type() else {
2454            panic!("expected a key_value struct");
2455        };
2456        let value = kv.iter().find(|f| f.name() == "value").unwrap();
2457        assert_eq!(value.extension_type_name(), Some("arrow.parquet.variant"));
2458    }
2459
2460    /// The unshredded Arrow storage of a variant: `metadata` (required) + `value`
2461    /// (nullable) binary, with no field ids on the sub-fields.
2462    fn variant_storage() -> DataType {
2463        DataType::Struct(Fields::from(vec![
2464            Field::new("metadata", DataType::Binary, false),
2465            Field::new("value", DataType::Binary, true),
2466        ]))
2467    }
2468
2469    #[test]
2470    fn test_variant_arrow_field_folds_to_iceberg_variant() {
2471        // A field tagged with the arrow.parquet.variant extension is folded into an
2472        // atomic Type::Variant; its storage sub-fields (which carry no field id) are
2473        // never descended into.
2474        let field = simple_field("v", variant_storage(), true, "1")
2475            .with_extension_type(VariantExtensionType);
2476        let arrow_schema = ArrowSchema::new(vec![field]);
2477
2478        let converted = arrow_schema_to_schema(&arrow_schema).unwrap();
2479        let expected = Schema::builder()
2480            .with_fields(vec![
2481                NestedField::optional(1, "v", Type::Variant(VariantType)).into(),
2482            ])
2483            .build()
2484            .unwrap();
2485        pretty_assertions::assert_eq!(converted, expected);
2486    }
2487
2488    #[test]
2489    fn test_variant_schema_round_trips() {
2490        // Iceberg -> Arrow -> Iceberg is the identity for variants at every position:
2491        // top-level, nested in a struct, as a list element, and as a map value.
2492        let schema = Schema::builder()
2493            .with_fields(vec![
2494                NestedField::optional(1, "v", Type::Variant(VariantType)).into(),
2495                NestedField::optional(
2496                    2,
2497                    "s",
2498                    Type::Struct(StructType::new(vec![
2499                        NestedField::optional(3, "sv", Type::Variant(VariantType)).into(),
2500                    ])),
2501                )
2502                .into(),
2503                NestedField::optional(
2504                    4,
2505                    "l",
2506                    Type::List(ListType::new(
2507                        NestedField::optional(5, "element", Type::Variant(VariantType)).into(),
2508                    )),
2509                )
2510                .into(),
2511                NestedField::optional(
2512                    6,
2513                    "m",
2514                    Type::Map(MapType::new(
2515                        NestedField::map_key_element(7, Type::Primitive(PrimitiveType::String))
2516                            .into(),
2517                        NestedField::map_value_element(8, Type::Variant(VariantType), false).into(),
2518                    )),
2519                )
2520                .into(),
2521            ])
2522            .build()
2523            .unwrap();
2524
2525        let arrow_schema = schema_to_arrow_schema(&schema).unwrap();
2526        let round_tripped = arrow_schema_to_schema(&arrow_schema).unwrap();
2527        pretty_assertions::assert_eq!(round_tripped, schema);
2528    }
2529
2530    #[test]
2531    fn test_variant_recognized_with_auto_assigned_ids() {
2532        // Recognition also works when the Arrow schema has no field ids: the variant
2533        // field gets an auto-assigned id and its storage is still not descended into.
2534        let field =
2535            Field::new("v", variant_storage(), true).with_extension_type(VariantExtensionType);
2536        let arrow_schema = ArrowSchema::new(vec![field]);
2537
2538        let converted = arrow_schema_to_schema_auto_assign_ids(&arrow_schema).unwrap();
2539        let expected = Schema::builder()
2540            .with_fields(vec![
2541                NestedField::optional(1, "v", Type::Variant(VariantType)).into(),
2542            ])
2543            .build()
2544            .unwrap();
2545        pretty_assertions::assert_eq!(converted, expected);
2546    }
2547
2548    #[test]
2549    fn test_variant_extension_on_non_struct_storage_is_rejected() {
2550        // The extension may only sit on struct storage. A hand-injected tag on a
2551        // non-struct field is rejected rather than silently reinterpreted as a variant.
2552        let field = Field::new("v", DataType::Int32, true).with_metadata(HashMap::from([
2553            (PARQUET_FIELD_ID_META_KEY.to_string(), "1".to_string()),
2554            (
2555                arrow_schema::extension::EXTENSION_TYPE_NAME_KEY.to_string(),
2556                VariantExtensionType::NAME.to_string(),
2557            ),
2558        ]));
2559        let arrow_schema = ArrowSchema::new(vec![field]);
2560
2561        let err = arrow_schema_to_schema(&arrow_schema).unwrap_err();
2562        assert!(
2563            err.to_string().contains("requires Struct storage"),
2564            "unexpected error: {err}"
2565        );
2566    }
2567
2568    #[test]
2569    fn test_geospatial_arrow_schema_roundtrip() {
2570        let schema = Schema::builder()
2571            .with_schema_id(1)
2572            .with_fields(vec![
2573                NestedField::required(
2574                    1,
2575                    "geom",
2576                    Type::Primitive(PrimitiveType::Geometry(
2577                        GeometryType::new(Some("EPSG:3857".to_string())).unwrap(),
2578                    )),
2579                )
2580                .into(),
2581                NestedField::optional(
2582                    2,
2583                    "geog",
2584                    Type::Primitive(PrimitiveType::Geography(
2585                        GeographyType::new(
2586                            Some("OGC:CRS27".to_string()),
2587                            IcebergEdgeInterpolationAlgorithm::Karney,
2588                        )
2589                        .unwrap(),
2590                    )),
2591                )
2592                .into(),
2593                NestedField::optional(
2594                    3,
2595                    "geom_list",
2596                    Type::List(ListType::new(
2597                        NestedField::list_element(
2598                            4,
2599                            Type::Primitive(PrimitiveType::Geometry(GeometryType::default())),
2600                            true,
2601                        )
2602                        .into(),
2603                    )),
2604                )
2605                .into(),
2606            ])
2607            .build()
2608            .unwrap();
2609
2610        let arrow_schema = schema_to_arrow_schema(&schema).unwrap();
2611        let geom = arrow_schema.field(0);
2612        assert_eq!(geom.data_type(), &DataType::LargeBinary);
2613
2614        let geog = arrow_schema.field(1);
2615        let geog_wkb = geog.try_extension_type::<WkbType>().unwrap();
2616        assert_eq!(geog_wkb.metadata().algorithm, Some(WkbEdges::Karney));
2617        let geog_metadata: serde_json::Value =
2618            serde_json::from_str(&geog_wkb.serialize_metadata().unwrap()).unwrap();
2619        assert_eq!(geog_metadata.get("edges").unwrap(), "karney");
2620        assert!(geog_metadata.get("algorithm").is_none());
2621
2622        let list = arrow_schema.field(2);
2623        let DataType::List(element) = list.data_type() else {
2624            panic!("Expected list field");
2625        };
2626        assert!(
2627            matches!(
2628                element
2629                    .try_extension_type::<WkbType>()
2630                    .unwrap()
2631                    .metadata()
2632                    .type_hint(),
2633                WkbTypeHint::Geometry
2634            ),
2635            "Expected list element to retain WKB extension metadata"
2636        );
2637        assert_eq!(
2638            element
2639                .try_extension_type::<WkbType>()
2640                .unwrap()
2641                .metadata()
2642                .crs
2643                .as_ref()
2644                .and_then(serde_json::Value::as_str),
2645            Some(DEFAULT_GEOSPATIAL_CRS)
2646        );
2647
2648        let converted = arrow_schema_to_schema(&arrow_schema).unwrap();
2649        assert_eq!(converted.as_struct().fields(), schema.as_struct().fields());
2650    }
2651
2652    #[test]
2653    fn test_geospatial_arrow_crs_import() {
2654        let mut geom = simple_field("geom", DataType::LargeBinary, true, "1");
2655        geom.try_with_extension_type(WkbType::new(Some(WkbMetadata::new(
2656            Some(r#"{"id":{"authority":"EPSG","code":3857}}"#),
2657            None,
2658        ))))
2659        .unwrap();
2660        let mut unset_geom = simple_field("unset_geom", DataType::LargeBinary, true, "2");
2661        unset_geom
2662            .try_with_extension_type(WkbType::new(Some(WkbMetadata::new(None, None))))
2663            .unwrap();
2664
2665        let arrow_schema = ArrowSchema::new(vec![geom, unset_geom]);
2666        let schema = arrow_schema_to_schema(&arrow_schema).unwrap();
2667
2668        assert_eq!(
2669            schema.field_by_id(1).unwrap().field_type.as_ref(),
2670            &Type::Primitive(PrimitiveType::Geometry(
2671                GeometryType::new(Some("EPSG:3857".to_string())).unwrap()
2672            ))
2673        );
2674        assert_eq!(
2675            schema.field_by_id(2).unwrap().field_type.as_ref(),
2676            &Type::Primitive(PrimitiveType::Geometry(
2677                GeometryType::new(Some(UNSET_GEOSPATIAL_CRS.to_string())).unwrap()
2678            ))
2679        );
2680    }
2681
2682    #[test]
2683    fn test_geospatial_arrow_unset_crs_export() {
2684        let schema = Schema::builder()
2685            .with_fields(vec![
2686                NestedField::optional(
2687                    1,
2688                    "geom",
2689                    Type::Primitive(PrimitiveType::Geometry(
2690                        GeometryType::new(Some(UNSET_GEOSPATIAL_CRS.to_string())).unwrap(),
2691                    )),
2692                )
2693                .into(),
2694                NestedField::optional(
2695                    2,
2696                    "geog",
2697                    Type::Primitive(PrimitiveType::Geography(
2698                        GeographyType::new(
2699                            Some(UNSET_GEOSPATIAL_CRS.to_string()),
2700                            IcebergEdgeInterpolationAlgorithm::Spherical,
2701                        )
2702                        .unwrap(),
2703                    )),
2704                )
2705                .into(),
2706            ])
2707            .build()
2708            .unwrap();
2709        let arrow_schema = schema_to_arrow_schema(&schema).unwrap();
2710        for field in arrow_schema.fields() {
2711            assert!(
2712                field
2713                    .try_extension_type::<WkbType>()
2714                    .unwrap()
2715                    .metadata()
2716                    .crs
2717                    .is_none()
2718            );
2719        }
2720        assert_eq!(arrow_schema_to_schema(&arrow_schema).unwrap(), schema);
2721    }
2722
2723    #[test]
2724    fn test_geospatial_arrow_invalid_crs_is_rejected() {
2725        for crs in [r#"{}"#, "42"] {
2726            let mut field = simple_field("geom", DataType::LargeBinary, true, "1");
2727            field
2728                .try_with_extension_type(WkbType::new(Some(WkbMetadata::new(Some(crs), None))))
2729                .unwrap();
2730
2731            let err = arrow_schema_to_schema(&ArrowSchema::new(vec![field])).unwrap_err();
2732            assert_eq!(err.kind(), ErrorKind::DataInvalid, "{err}");
2733        }
2734    }
2735
2736    #[test]
2737    fn test_geospatial_arrow_extension_requires_binary_storage() {
2738        use arrow_schema::extension::{EXTENSION_TYPE_METADATA_KEY, EXTENSION_TYPE_NAME_KEY};
2739
2740        let metadata = WkbType::new(Some(WkbMetadata::new(None, None)))
2741            .serialize_metadata()
2742            .unwrap();
2743        let field =
2744            simple_field("geom", DataType::Int32, true, "1").with_metadata(HashMap::from([
2745                (PARQUET_FIELD_ID_META_KEY.to_string(), "1".to_string()),
2746                (
2747                    EXTENSION_TYPE_NAME_KEY.to_string(),
2748                    WkbType::NAME.to_string(),
2749                ),
2750                (EXTENSION_TYPE_METADATA_KEY.to_string(), metadata),
2751            ]));
2752
2753        let err = arrow_schema_to_schema(&ArrowSchema::new(vec![field])).unwrap_err();
2754        assert_eq!(err.kind(), ErrorKind::DataInvalid, "{err}");
2755        assert!(
2756            err.message().contains("requires binary storage"),
2757            "unexpected error: {err}"
2758        );
2759    }
2760
2761    #[test]
2762    fn test_geospatial_parquet_statistics_are_ignored() {
2763        use parquet::data_type::ByteArray;
2764        use parquet::file::statistics::ValueStatistics;
2765
2766        let stats = Statistics::ByteArray(ValueStatistics::new(
2767            Some(ByteArray::from(vec![1, 2, 3])),
2768            Some(ByteArray::from(vec![4, 5, 6])),
2769            None,
2770            None,
2771            false,
2772        ));
2773
2774        for primitive_type in [
2775            PrimitiveType::Geometry(GeometryType::default()),
2776            PrimitiveType::Geography(GeographyType::default()),
2777        ] {
2778            assert_eq!(
2779                get_parquet_stat_min_as_datum(&primitive_type, &stats).unwrap(),
2780                None
2781            );
2782            assert_eq!(
2783                get_parquet_stat_max_as_datum(&primitive_type, &stats).unwrap(),
2784                None
2785            );
2786        }
2787    }
2788
2789    #[test]
2790    fn test_type_conversion() {
2791        // test primitive type
2792        {
2793            let arrow_type = DataType::Int32;
2794            let iceberg_type = Type::Primitive(PrimitiveType::Int);
2795            assert_eq!(arrow_type, type_to_arrow_type(&iceberg_type).unwrap());
2796            assert_eq!(iceberg_type, arrow_type_to_type(&arrow_type).unwrap());
2797        }
2798
2799        {
2800            let arrow_type = DataType::Null;
2801            let iceberg_type = Type::Primitive(PrimitiveType::Unknown);
2802            assert_eq!(arrow_type, type_to_arrow_type(&iceberg_type).unwrap());
2803            assert_eq!(iceberg_type, arrow_type_to_type(&arrow_type).unwrap());
2804        }
2805
2806        // test struct type
2807        {
2808            // no metadata will cause error
2809            let arrow_type = DataType::Struct(Fields::from(vec![
2810                Field::new("a", DataType::Int64, false),
2811                Field::new("b", DataType::Utf8, true),
2812            ]));
2813            assert_eq!(
2814                &arrow_type_to_type(&arrow_type).unwrap_err().to_string(),
2815                "DataInvalid => Field id not found in metadata"
2816            );
2817
2818            let arrow_type = DataType::Struct(Fields::from(vec![
2819                Field::new("a", DataType::Int64, false).with_metadata(HashMap::from_iter([(
2820                    PARQUET_FIELD_ID_META_KEY.to_string(),
2821                    1.to_string(),
2822                )])),
2823                Field::new("b", DataType::Utf8, true).with_metadata(HashMap::from_iter([(
2824                    PARQUET_FIELD_ID_META_KEY.to_string(),
2825                    2.to_string(),
2826                )])),
2827            ]));
2828            let iceberg_type = Type::Struct(StructType::new(vec![
2829                NestedField {
2830                    id: 1,
2831                    doc: None,
2832                    name: "a".to_string(),
2833                    required: true,
2834                    field_type: Box::new(Type::Primitive(PrimitiveType::Long)),
2835                    initial_default: None,
2836                    write_default: None,
2837                }
2838                .into(),
2839                NestedField {
2840                    id: 2,
2841                    doc: None,
2842                    name: "b".to_string(),
2843                    required: false,
2844                    field_type: Box::new(Type::Primitive(PrimitiveType::String)),
2845                    initial_default: None,
2846                    write_default: None,
2847                }
2848                .into(),
2849            ]));
2850            assert_eq!(iceberg_type, arrow_type_to_type(&arrow_type).unwrap());
2851            assert_eq!(arrow_type, type_to_arrow_type(&iceberg_type).unwrap());
2852
2853            // initial_default and write_default is ignored
2854            let iceberg_type = Type::Struct(StructType::new(vec![
2855                NestedField {
2856                    id: 1,
2857                    doc: None,
2858                    name: "a".to_string(),
2859                    required: true,
2860                    field_type: Box::new(Type::Primitive(PrimitiveType::Long)),
2861                    initial_default: Some(Literal::Primitive(PrimitiveLiteral::Int(114514))),
2862                    write_default: None,
2863                }
2864                .into(),
2865                NestedField {
2866                    id: 2,
2867                    doc: None,
2868                    name: "b".to_string(),
2869                    required: false,
2870                    field_type: Box::new(Type::Primitive(PrimitiveType::String)),
2871                    initial_default: None,
2872                    write_default: Some(Literal::Primitive(PrimitiveLiteral::String(
2873                        "514".to_string(),
2874                    ))),
2875                }
2876                .into(),
2877            ]));
2878            assert_eq!(arrow_type, type_to_arrow_type(&iceberg_type).unwrap());
2879        }
2880
2881        // test dictionary type
2882        {
2883            let arrow_type =
2884                DataType::Dictionary(Box::new(DataType::Int32), Box::new(DataType::Int8));
2885            let iceberg_type = Type::Primitive(PrimitiveType::Int);
2886            assert_eq!(
2887                iceberg_type,
2888                arrow_type_to_type(&arrow_type).unwrap(),
2889                "Expected dictionary conversion to use the contained value"
2890            );
2891
2892            let arrow_type =
2893                DataType::Dictionary(Box::new(DataType::Utf8), Box::new(DataType::Boolean));
2894            let iceberg_type = Type::Primitive(PrimitiveType::Boolean);
2895            assert_eq!(iceberg_type, arrow_type_to_type(&arrow_type).unwrap());
2896        }
2897    }
2898
2899    #[test]
2900    fn test_unsigned_integer_type_conversion() {
2901        let test_cases = vec![
2902            (DataType::UInt8, PrimitiveType::Int),
2903            (DataType::UInt16, PrimitiveType::Int),
2904            (DataType::UInt32, PrimitiveType::Long),
2905        ];
2906
2907        for (arrow_type, expected_iceberg_type) in test_cases {
2908            let arrow_field = Field::new("test", arrow_type.clone(), false).with_metadata(
2909                HashMap::from([(PARQUET_FIELD_ID_META_KEY.to_string(), "1".to_string())]),
2910            );
2911            let arrow_schema = ArrowSchema::new(vec![arrow_field]);
2912
2913            let iceberg_schema = arrow_schema_to_schema(&arrow_schema).unwrap();
2914            let iceberg_field = iceberg_schema.as_struct().fields().first().unwrap();
2915
2916            assert!(
2917                matches!(iceberg_field.field_type.as_ref(), Type::Primitive(t) if *t == expected_iceberg_type),
2918                "Expected {arrow_type:?} to map to {expected_iceberg_type:?}"
2919            );
2920        }
2921
2922        // Test UInt64 blocking
2923        {
2924            let arrow_field = Field::new("test", DataType::UInt64, false).with_metadata(
2925                HashMap::from([(PARQUET_FIELD_ID_META_KEY.to_string(), "1".to_string())]),
2926            );
2927            let arrow_schema = ArrowSchema::new(vec![arrow_field]);
2928
2929            let result = arrow_schema_to_schema(&arrow_schema);
2930            assert!(result.is_err());
2931            assert!(
2932                result
2933                    .unwrap_err()
2934                    .to_string()
2935                    .contains("UInt64 is not supported")
2936            );
2937        }
2938    }
2939
2940    #[test]
2941    fn test_datum_conversion() {
2942        {
2943            let datum = Datum::bool(true);
2944            let arrow_datum = get_arrow_datum(&datum).unwrap();
2945            let (array, is_scalar) = arrow_datum.get();
2946            let array = array.as_any().downcast_ref::<BooleanArray>().unwrap();
2947            assert!(is_scalar);
2948            assert!(array.value(0));
2949        }
2950        {
2951            let datum = Datum::int(42);
2952            let arrow_datum = get_arrow_datum(&datum).unwrap();
2953            let (array, is_scalar) = arrow_datum.get();
2954            let array = array.as_any().downcast_ref::<Int32Array>().unwrap();
2955            assert!(is_scalar);
2956            assert_eq!(array.value(0), 42);
2957        }
2958        {
2959            let datum = Datum::long(42);
2960            let arrow_datum = get_arrow_datum(&datum).unwrap();
2961            let (array, is_scalar) = arrow_datum.get();
2962            let array = array.as_any().downcast_ref::<Int64Array>().unwrap();
2963            assert!(is_scalar);
2964            assert_eq!(array.value(0), 42);
2965        }
2966        {
2967            let datum = Datum::float(42.42_f32);
2968            let arrow_datum = get_arrow_datum(&datum).unwrap();
2969            let (array, is_scalar) = arrow_datum.get();
2970            let array = array.as_any().downcast_ref::<Float32Array>().unwrap();
2971            assert!(is_scalar);
2972            assert_eq!(array.value(0), 42.42);
2973        }
2974        {
2975            let datum = Datum::double(42.42);
2976            let arrow_datum = get_arrow_datum(&datum).unwrap();
2977            let (array, is_scalar) = arrow_datum.get();
2978            let array = array.as_any().downcast_ref::<Float64Array>().unwrap();
2979            assert!(is_scalar);
2980            assert_eq!(array.value(0), 42.42);
2981        }
2982        {
2983            let datum = Datum::string("abc");
2984            let arrow_datum = get_arrow_datum(&datum).unwrap();
2985            let (array, is_scalar) = arrow_datum.get();
2986            let array = array.as_any().downcast_ref::<StringArray>().unwrap();
2987            assert!(is_scalar);
2988            assert_eq!(array.value(0), "abc");
2989        }
2990        {
2991            let datum = Datum::binary(vec![1, 2, 3, 4]);
2992            let arrow_datum = get_arrow_datum(&datum).unwrap();
2993            let (array, is_scalar) = arrow_datum.get();
2994            let array = array.as_any().downcast_ref::<BinaryArray>().unwrap();
2995            assert!(is_scalar);
2996            assert_eq!(array.value(0), &[1, 2, 3, 4]);
2997        }
2998        {
2999            let datum = Datum::date(42);
3000            let arrow_datum = get_arrow_datum(&datum).unwrap();
3001            let (array, is_scalar) = arrow_datum.get();
3002            let array = array.as_any().downcast_ref::<Date32Array>().unwrap();
3003            assert!(is_scalar);
3004            assert_eq!(array.value(0), 42);
3005        }
3006        {
3007            let datum = Datum::timestamp_micros(42);
3008            let arrow_datum = get_arrow_datum(&datum).unwrap();
3009            let (array, is_scalar) = arrow_datum.get();
3010            let array = array
3011                .as_any()
3012                .downcast_ref::<TimestampMicrosecondArray>()
3013                .unwrap();
3014            assert!(is_scalar);
3015            assert_eq!(array.value(0), 42);
3016        }
3017        {
3018            let datum = Datum::timestamptz_micros(42);
3019            let arrow_datum = get_arrow_datum(&datum).unwrap();
3020            let (array, is_scalar) = arrow_datum.get();
3021            let array = array
3022                .as_any()
3023                .downcast_ref::<TimestampMicrosecondArray>()
3024                .unwrap();
3025            assert!(is_scalar);
3026            assert_eq!(array.timezone(), Some("+00:00"));
3027            assert_eq!(array.value(0), 42);
3028        }
3029        {
3030            let datum = Datum::decimal_with_precision(decimal_new(123, 2), 30).unwrap();
3031            let arrow_datum = get_arrow_datum(&datum).unwrap();
3032            let (array, is_scalar) = arrow_datum.get();
3033            let array = array.as_any().downcast_ref::<Decimal128Array>().unwrap();
3034            assert!(is_scalar);
3035            assert_eq!(array.precision(), 30);
3036            assert_eq!(array.scale(), 2);
3037            assert_eq!(array.value(0), 123);
3038        }
3039        {
3040            let datum = Datum::uuid_from_str("42424242-4242-4242-4242-424242424242").unwrap();
3041            let arrow_datum = get_arrow_datum(&datum).unwrap();
3042            let (array, is_scalar) = arrow_datum.get();
3043            let array = array
3044                .as_any()
3045                .downcast_ref::<FixedSizeBinaryArray>()
3046                .unwrap();
3047            assert!(is_scalar);
3048            assert_eq!(array.value(0), [66u8; 16]);
3049        }
3050        {
3051            let datum = Datum::fixed(vec![1u8, 2, 3, 4, 5, 6, 7, 8]);
3052            let arrow_datum = get_arrow_datum(&datum).unwrap();
3053            let (array, is_scalar) = arrow_datum.get();
3054            let array = array
3055                .as_any()
3056                .downcast_ref::<FixedSizeBinaryArray>()
3057                .unwrap();
3058            assert!(is_scalar);
3059            assert_eq!(array.value_length(), 8);
3060            assert_eq!(array.value(0), &[1u8, 2, 3, 4, 5, 6, 7, 8]);
3061        }
3062    }
3063
3064    #[test]
3065    fn test_arrow_schema_to_schema_with_field_id() {
3066        // Create a complex Arrow schema without field ID metadata
3067        // Including: primitives, list, nested struct, map, and nested list of structs
3068        let arrow_schema = ArrowSchema::new(vec![
3069            Field::new("id", DataType::Int64, false),
3070            Field::new("name", DataType::Utf8, true),
3071            Field::new("price", DataType::Decimal128(10, 2), false),
3072            Field::new("uuid", DataType::FixedSizeBinary(16), false)
3073                .with_extension_type(UuidExtensionType),
3074            Field::new(
3075                "created_at",
3076                DataType::Timestamp(TimeUnit::Microsecond, Some("+00:00".into())),
3077                true,
3078            ),
3079            Field::new(
3080                "tags",
3081                DataType::List(Arc::new(Field::new("item", DataType::Utf8, true))),
3082                true,
3083            ),
3084            Field::new(
3085                "address",
3086                DataType::Struct(Fields::from(vec![
3087                    Field::new("street", DataType::Utf8, true),
3088                    Field::new("city", DataType::Utf8, false),
3089                    Field::new("zip", DataType::Int32, true),
3090                ])),
3091                true,
3092            ),
3093            Field::new(
3094                "attributes",
3095                DataType::Map(
3096                    Arc::new(Field::new(
3097                        DEFAULT_MAP_FIELD_NAME,
3098                        DataType::Struct(Fields::from(vec![
3099                            Field::new("key", DataType::Utf8, false),
3100                            Field::new("value", DataType::Utf8, true),
3101                        ])),
3102                        false,
3103                    )),
3104                    false,
3105                ),
3106                true,
3107            ),
3108            Field::new(
3109                "orders",
3110                DataType::List(Arc::new(Field::new(
3111                    "element",
3112                    DataType::Struct(Fields::from(vec![
3113                        Field::new("order_id", DataType::Int64, false),
3114                        Field::new("amount", DataType::Float64, false),
3115                    ])),
3116                    true,
3117                ))),
3118                true,
3119            ),
3120        ]);
3121
3122        let schema = arrow_schema_to_schema_auto_assign_ids(&arrow_schema).unwrap();
3123
3124        // Build expected schema with exact field IDs following level-order assignment:
3125        // Level 0: id=1, name=2, price=3, uuid=4, created_at=5, tags=6, address=7, attributes=8, orders=9
3126        // Level 1: tags.element=10, address.{street=11,city=12,zip=13}, attributes.{key=14,value=15}, orders.element=16
3127        // Level 2: orders.element.{order_id=17,amount=18}
3128        let expected = Schema::builder()
3129            .with_fields(vec![
3130                NestedField::required(1, "id", Type::Primitive(PrimitiveType::Long)).into(),
3131                NestedField::optional(2, "name", Type::Primitive(PrimitiveType::String)).into(),
3132                NestedField::required(
3133                    3,
3134                    "price",
3135                    Type::Primitive(PrimitiveType::Decimal {
3136                        precision: 10,
3137                        scale: 2,
3138                    }),
3139                )
3140                .into(),
3141                NestedField::required(4, "uuid", Type::Primitive(PrimitiveType::Uuid)).into(),
3142                NestedField::optional(5, "created_at", Type::Primitive(PrimitiveType::Timestamptz))
3143                    .into(),
3144                NestedField::optional(
3145                    6,
3146                    "tags",
3147                    Type::List(ListType {
3148                        element_field: NestedField::list_element(
3149                            10,
3150                            Type::Primitive(PrimitiveType::String),
3151                            false,
3152                        )
3153                        .into(),
3154                    }),
3155                )
3156                .into(),
3157                NestedField::optional(
3158                    7,
3159                    "address",
3160                    Type::Struct(StructType::new(vec![
3161                        NestedField::optional(11, "street", Type::Primitive(PrimitiveType::String))
3162                            .into(),
3163                        NestedField::required(12, "city", Type::Primitive(PrimitiveType::String))
3164                            .into(),
3165                        NestedField::optional(13, "zip", Type::Primitive(PrimitiveType::Int))
3166                            .into(),
3167                    ])),
3168                )
3169                .into(),
3170                NestedField::optional(
3171                    8,
3172                    "attributes",
3173                    Type::Map(MapType {
3174                        key_field: NestedField::map_key_element(
3175                            14,
3176                            Type::Primitive(PrimitiveType::String),
3177                        )
3178                        .into(),
3179                        value_field: NestedField::map_value_element(
3180                            15,
3181                            Type::Primitive(PrimitiveType::String),
3182                            false,
3183                        )
3184                        .into(),
3185                    }),
3186                )
3187                .into(),
3188                NestedField::optional(
3189                    9,
3190                    "orders",
3191                    Type::List(ListType {
3192                        element_field: NestedField::list_element(
3193                            16,
3194                            Type::Struct(StructType::new(vec![
3195                                NestedField::required(
3196                                    17,
3197                                    "order_id",
3198                                    Type::Primitive(PrimitiveType::Long),
3199                                )
3200                                .into(),
3201                                NestedField::required(
3202                                    18,
3203                                    "amount",
3204                                    Type::Primitive(PrimitiveType::Double),
3205                                )
3206                                .into(),
3207                            ])),
3208                            false,
3209                        )
3210                        .into(),
3211                    }),
3212                )
3213                .into(),
3214            ])
3215            .build()
3216            .unwrap();
3217
3218        pretty_assertions::assert_eq!(schema, expected);
3219        assert_eq!(schema.highest_field_id(), 18);
3220    }
3221
3222    fn decimal_byte_array_stats(min: Vec<u8>, max: Vec<u8>) -> Statistics {
3223        Statistics::byte_array(
3224            Some(parquet::data_type::ByteArray::from(min)),
3225            Some(parquet::data_type::ByteArray::from(max)),
3226            None,
3227            Some(0),
3228            false,
3229        )
3230    }
3231
3232    fn decimal_bounds(stats: &Statistics) -> Result<(Option<Datum>, Option<Datum>)> {
3233        let ty = PrimitiveType::Decimal {
3234            precision: 38,
3235            scale: 2,
3236        };
3237        Ok((
3238            get_parquet_stat_min_as_datum(&ty, stats)?,
3239            get_parquet_stat_max_as_datum(&ty, stats)?,
3240        ))
3241    }
3242
3243    fn decimal_datum(value: i128) -> Datum {
3244        Datum::new(
3245            PrimitiveType::Decimal {
3246                precision: 38,
3247                scale: 2,
3248            },
3249            PrimitiveLiteral::Int128(value),
3250        )
3251    }
3252
3253    #[test]
3254    fn test_byte_array_decimal_stats_short_bounds() {
3255        // Minimal-length big-endian two's-complement: -1234 and 1234.
3256        let stats = decimal_byte_array_stats(vec![0xFB, 0x2E], vec![0x04, 0xD2]);
3257        let (min, max) = decimal_bounds(&stats).unwrap();
3258        assert_eq!(min, Some(decimal_datum(-1234)));
3259        assert_eq!(max, Some(decimal_datum(1234)));
3260
3261        // Single-byte bounds: sign bit set must sign-extend, not zero-extend.
3262        let stats = decimal_byte_array_stats(vec![0x80], vec![0x7F]);
3263        let (min, max) = decimal_bounds(&stats).unwrap();
3264        assert_eq!(min, Some(decimal_datum(-128)));
3265        assert_eq!(max, Some(decimal_datum(127)));
3266    }
3267
3268    #[test]
3269    fn test_byte_array_decimal_stats_full_width_bounds() {
3270        let stats = decimal_byte_array_stats(
3271            i128::MIN.to_be_bytes().to_vec(),
3272            i128::MAX.to_be_bytes().to_vec(),
3273        );
3274        let (min, max) = decimal_bounds(&stats).unwrap();
3275        assert_eq!(min, Some(decimal_datum(i128::MIN)));
3276        assert_eq!(max, Some(decimal_datum(i128::MAX)));
3277    }
3278
3279    #[test]
3280    fn test_byte_array_decimal_stats_empty_bound_errors() {
3281        let stats = decimal_byte_array_stats(vec![], vec![0x01]);
3282        let err = decimal_bounds(&stats).unwrap_err();
3283        assert_eq!(err.kind(), ErrorKind::DataInvalid);
3284
3285        let stats = decimal_byte_array_stats(vec![0x01], vec![]);
3286        let err = decimal_bounds(&stats).unwrap_err();
3287        assert_eq!(err.kind(), ErrorKind::DataInvalid);
3288    }
3289
3290    #[test]
3291    fn test_byte_array_decimal_stats_oversized_bound_errors() {
3292        let stats = decimal_byte_array_stats(vec![0; 17], vec![0x01]);
3293        let err = decimal_bounds(&stats).unwrap_err();
3294        assert_eq!(err.kind(), ErrorKind::DataInvalid);
3295
3296        let stats = decimal_byte_array_stats(vec![0x01], vec![0; 17]);
3297        let err = decimal_bounds(&stats).unwrap_err();
3298        assert_eq!(err.kind(), ErrorKind::DataInvalid);
3299    }
3300
3301    /// Every field is rebuilt without metadata, at any depth and inside any
3302    /// container. Fields are rebuilt in nesting order rather than looked up by
3303    /// name, so a path that repeats names (`a.b.a.b`) keeps each level's name
3304    /// and nullability.
3305    #[test]
3306    fn test_strip_metadata_from_nested_schema() {
3307        // The same schema, built with and without metadata on the schema and every field.
3308        let schema = |with_metadata: bool| {
3309            let metadata = |value: &str| {
3310                if with_metadata {
3311                    HashMap::from([("k".to_string(), value.to_string())])
3312                } else {
3313                    HashMap::new()
3314                }
3315            };
3316            let field = |name: &str, ty: DataType, nullable: bool| {
3317                Field::new(name, ty, nullable).with_metadata(metadata(name))
3318            };
3319            let element = |ty: DataType| Arc::new(field(LIST_FIELD_NAME, ty, false));
3320            let ints = || DataType::List(element(DataType::Int32));
3321            let map = |value: DataType, sorted: bool| {
3322                let entries = Fields::from(vec![
3323                    field(MAP_KEY_FIELD_NAME, DataType::Utf8, false),
3324                    field(MAP_VALUE_FIELD_NAME, value, true),
3325                ]);
3326                let entries = field(DEFAULT_MAP_FIELD_NAME, DataType::Struct(entries), false);
3327                DataType::Map(Arc::new(entries), sorted)
3328            };
3329            let struct_of = |fields: Vec<Field>| DataType::Struct(Fields::from(fields));
3330
3331            let a_b_a_b = struct_of(vec![field(
3332                "b",
3333                struct_of(vec![field(
3334                    "a",
3335                    struct_of(vec![field("b", DataType::Int32, true)]),
3336                    false,
3337                )]),
3338                false,
3339            )]);
3340            ArrowSchema::new(vec![
3341                field("list", ints(), true),
3342                field(
3343                    "large_list",
3344                    DataType::LargeList(element(DataType::Int32)),
3345                    false,
3346                ),
3347                field(
3348                    "fixed_list",
3349                    DataType::FixedSizeList(element(DataType::Int32), 2),
3350                    true,
3351                ),
3352                field("list_of_list", DataType::List(element(ints())), false),
3353                field("sorted_map", map(ints(), true), false),
3354                field(
3355                    "map_of_struct",
3356                    map(
3357                        struct_of(vec![
3358                            field("a", DataType::Int32, true),
3359                            field("b", ints(), false),
3360                        ]),
3361                        false,
3362                    ),
3363                    true,
3364                ),
3365                field(
3366                    "list_of_struct",
3367                    DataType::List(element(struct_of(vec![field(
3368                        "m",
3369                        map(ints(), false),
3370                        true,
3371                    )]))),
3372                    true,
3373                ),
3374                field("a", a_b_a_b, true),
3375            ])
3376            .with_metadata(metadata("schema"))
3377        };
3378
3379        let (input, expected) = (schema(true), schema(false));
3380        // Otherwise a builder that stopped adding metadata would pass trivially.
3381        assert_ne!(input, expected);
3382        pretty_assertions::assert_eq!(strip_metadata_from_schema(&input).unwrap(), expected);
3383    }
3384
3385    /// arrow-rs names a list's element `item`, and an arrow-rs `MapBuilder` names
3386    /// a map's fields `entries`, `keys` and `values`. Once stripped, a schema
3387    /// using those names, with a nullable map key, compares equal to the same
3388    /// schema from `schema_to_arrow_schema`.
3389    #[test]
3390    fn test_strip_metadata_normalizes_container_field_names() {
3391        // A list of ints and a map from string to a list of ints, with the given
3392        // names for the list elements and the map's entries, key and value.
3393        let schema = |[element, entries, key, value]: [&str; 4], key_nullable: bool| {
3394            let ints =
3395                |nullable| DataType::List(Arc::new(Field::new(element, DataType::Int32, nullable)));
3396            let key_value = Fields::from(vec![
3397                Field::new(key, DataType::Utf8, key_nullable),
3398                Field::new(value, ints(false), true),
3399            ]);
3400            let entries = Field::new(entries, DataType::Struct(key_value), false);
3401            ArrowSchema::new(vec![
3402                Field::new("l", ints(true), true),
3403                Field::new("m", DataType::Map(Arc::new(entries), false), true),
3404            ])
3405        };
3406        let arrow_rs = schema(["item", "entries", "keys", "values"], true);
3407        let expected = schema(
3408            [
3409                LIST_FIELD_NAME,
3410                DEFAULT_MAP_FIELD_NAME,
3411                MAP_KEY_FIELD_NAME,
3412                MAP_VALUE_FIELD_NAME,
3413            ],
3414            false,
3415        );
3416
3417        // The same schema as an Iceberg table's.
3418        let int_list = |id, required| {
3419            let int = Type::Primitive(PrimitiveType::Int);
3420            Type::List(ListType::new(
3421                NestedField::list_element(id, int, required).into(),
3422            ))
3423        };
3424        let string_to_int_list = MapType::new(
3425            NestedField::map_key_element(4, Type::Primitive(PrimitiveType::String)).into(),
3426            NestedField::map_value_element(5, int_list(6, true), false).into(),
3427        );
3428        let iceberg = Schema::builder()
3429            .with_fields(vec![
3430                NestedField::optional(1, "l", int_list(2, false)).into(),
3431                NestedField::optional(3, "m", Type::Map(string_to_int_list)).into(),
3432            ])
3433            .build()
3434            .unwrap();
3435
3436        pretty_assertions::assert_eq!(strip_metadata_from_schema(&arrow_rs).unwrap(), expected);
3437        pretty_assertions::assert_eq!(
3438            strip_metadata_from_schema(&schema_to_arrow_schema(&iceberg).unwrap()).unwrap(),
3439            expected
3440        );
3441    }
3442
3443    /// Names and nullability that `schema_to_arrow_schema` doesn't fix, such as a
3444    /// struct field's name or a list element's nullability, still tell schemas apart.
3445    #[test]
3446    fn test_strip_metadata_keeps_other_differences() {
3447        let schema = |child: &str, element_nullable: bool| {
3448            let child = Field::new(child, DataType::Int32, true);
3449            let element = Field::new("item", DataType::Int32, element_nullable);
3450            ArrowSchema::new(vec![
3451                Field::new("s", DataType::Struct(Fields::from(vec![child])), true),
3452                Field::new("l", DataType::List(Arc::new(element)), true),
3453            ])
3454        };
3455        let strip = |schema: ArrowSchema| strip_metadata_from_schema(&schema).unwrap();
3456
3457        assert_ne!(strip(schema("a", true)), strip(schema("b", true)));
3458        assert_ne!(strip(schema("a", true)), strip(schema("a", false)));
3459    }
3460
3461    /// Stripping drops the `arrow.parquet.variant` extension type with the rest of
3462    /// the metadata, so a variant becomes a plain struct, in a list or map too.
3463    #[test]
3464    fn test_strip_metadata_turns_variants_into_structs() {
3465        let variant = || Type::Variant(VariantType);
3466        let schema = Schema::builder()
3467            .with_fields(vec![
3468                NestedField::optional(1, "v", variant()).into(),
3469                NestedField::optional(
3470                    2,
3471                    "l",
3472                    Type::List(ListType::new(
3473                        NestedField::list_element(3, variant(), false).into(),
3474                    )),
3475                )
3476                .into(),
3477                NestedField::optional(
3478                    4,
3479                    "m",
3480                    Type::Map(MapType::new(
3481                        NestedField::map_key_element(5, Type::Primitive(PrimitiveType::String))
3482                            .into(),
3483                        NestedField::map_value_element(6, variant(), false).into(),
3484                    )),
3485                )
3486                .into(),
3487            ])
3488            .build()
3489            .unwrap();
3490
3491        let storage = || {
3492            DataType::Struct(Fields::from(vec![
3493                Field::new("metadata", DataType::Binary, false),
3494                Field::new("value", DataType::Binary, true),
3495            ]))
3496        };
3497        let entries = Fields::from(vec![
3498            Field::new(MAP_KEY_FIELD_NAME, DataType::Utf8, false),
3499            Field::new(MAP_VALUE_FIELD_NAME, storage(), true),
3500        ]);
3501        let entries = Field::new(DEFAULT_MAP_FIELD_NAME, DataType::Struct(entries), false);
3502        let expected = ArrowSchema::new(vec![
3503            Field::new("v", storage(), true),
3504            Field::new(
3505                "l",
3506                DataType::List(Arc::new(Field::new(LIST_FIELD_NAME, storage(), true))),
3507                true,
3508            ),
3509            Field::new("m", DataType::Map(Arc::new(entries), false), true),
3510        ]);
3511
3512        pretty_assertions::assert_eq!(
3513            strip_metadata_from_schema(&schema_to_arrow_schema(&schema).unwrap()).unwrap(),
3514            expected
3515        );
3516    }
3517
3518    /// A dictionary-encoded field compares equal to a plain field of its value
3519    /// type once stripped, at any depth.
3520    #[test]
3521    fn test_strip_metadata_unwraps_dictionaries() {
3522        let schema = |dictionary_encoded: bool| {
3523            let maybe_dictionary = |key: DataType, value: DataType| {
3524                if dictionary_encoded {
3525                    DataType::Dictionary(Box::new(key), Box::new(value))
3526                } else {
3527                    value
3528                }
3529            };
3530            let element = Field::new(
3531                LIST_FIELD_NAME,
3532                maybe_dictionary(DataType::Int8, DataType::LargeUtf8),
3533                false,
3534            );
3535            ArrowSchema::new(vec![
3536                Field::new("d", maybe_dictionary(DataType::Int32, DataType::Utf8), true),
3537                Field::new("l", DataType::List(Arc::new(element)), true),
3538            ])
3539        };
3540
3541        pretty_assertions::assert_eq!(
3542            strip_metadata_from_schema(&schema(true)).unwrap(),
3543            schema(false)
3544        );
3545    }
3546}