1use std::collections::HashMap;
21use std::sync::Arc;
22
23use arrow_array::types::{Decimal128Type, validate_decimal_precision_and_scale};
24use arrow_array::{
25 BinaryArray, BooleanArray, Date32Array, Datum as ArrowDatum, Decimal128Array,
26 FixedSizeBinaryArray, Float32Array, Float64Array, Int32Array, Int64Array, Scalar, StringArray,
27 TimestampMicrosecondArray, TimestampNanosecondArray,
28};
29use arrow_schema::extension::{ExtensionType, Uuid as UuidExtensionType};
30use arrow_schema::{
31 ArrowError, DataType, Field, FieldRef, Fields, Schema as ArrowSchema, TimeUnit,
32};
33use parquet::arrow::PARQUET_FIELD_ID_META_KEY;
34use parquet::file::statistics::Statistics;
35use parquet_geospatial::{WkbEdges, WkbMetadata, WkbType, WkbTypeHint};
36use uuid::Uuid;
37
38use crate::error::{Result, invalid_data};
39use crate::spec::decimal_utils::i128_from_be_bytes;
40use crate::spec::geospatial::EdgeInterpolationAlgorithm;
41use crate::spec::{
42 Datum, FIRST_FIELD_ID, GeographyType, GeometryType, LIST_FIELD_NAME, ListType,
43 MAP_KEY_FIELD_NAME, MAP_VALUE_FIELD_NAME, MapType, NestedField, NestedFieldRef,
44 PrimitiveLiteral, PrimitiveType, Schema, SchemaVisitor, StructType, Type, VariantType,
45};
46use crate::{Error, ErrorKind};
47
48pub const DEFAULT_MAP_FIELD_NAME: &str = "key_value";
50pub const UTC_TIME_ZONE: &str = "+00:00";
52const UNSET_GEOSPATIAL_CRS: &str = "srid:0";
53
54#[derive(Debug, Clone, Copy, Default)]
63pub(crate) struct VariantExtensionType;
64
65impl ExtensionType for VariantExtensionType {
66 const NAME: &'static str = "arrow.parquet.variant";
67
68 type Metadata = ();
69
70 fn metadata(&self) -> &Self::Metadata {
71 &()
72 }
73
74 fn serialize_metadata(&self) -> Option<String> {
75 None
76 }
77
78 fn deserialize_metadata(
79 metadata: Option<&str>,
80 ) -> std::result::Result<Self::Metadata, ArrowError> {
81 match metadata {
82 None | Some("") => Ok(()),
83 Some(other) => Err(ArrowError::InvalidArgumentError(format!(
84 "arrow.parquet.variant extension type takes no metadata, got {other:?}"
85 ))),
86 }
87 }
88
89 fn supports_data_type(&self, data_type: &DataType) -> std::result::Result<(), ArrowError> {
90 match data_type {
91 DataType::Struct(_) => Ok(()),
92 other => Err(ArrowError::InvalidArgumentError(format!(
93 "arrow.parquet.variant extension type requires a Struct storage type, got {other}"
94 ))),
95 }
96 }
97
98 fn try_new(
99 data_type: &DataType,
100 _metadata: Self::Metadata,
101 ) -> std::result::Result<Self, ArrowError> {
102 Self.supports_data_type(data_type)?;
103 Ok(Self)
104 }
105}
106
107pub(crate) fn is_variant_field(field: &Field) -> bool {
109 field.extension_type_name() == Some(VariantExtensionType::NAME)
110}
111
112fn is_variant_binary(data_type: &DataType) -> bool {
114 match data_type {
115 DataType::Binary | DataType::LargeBinary | DataType::BinaryView => true,
116 DataType::Dictionary(_, value_type) => is_variant_binary(value_type),
117 _ => false,
118 }
119}
120
121pub(crate) fn validate_unshredded_variant(field: &Field, variant_field_id: i32) -> Result<()> {
124 let invalid = |problem: String| {
125 invalid_data!(
126 "Variant column `{}` (field id {variant_field_id}) {problem}",
127 field.name()
128 )
129 };
130 let DataType::Struct(subfields) = field.data_type() else {
131 return Err(invalid(format!(
132 "must be stored as a group, found {}",
133 field.data_type()
134 )));
135 };
136 if subfields.find("typed_value").is_some() {
137 return Err(Error::new(
138 ErrorKind::FeatureUnsupported,
139 format!(
140 "Reading shredded variant column `{}` (field id {variant_field_id}) is not supported yet, see https://github.com/apache/iceberg-rust/issues/3606",
141 field.name()
142 ),
143 ));
144 }
145 for name in ["metadata", "value"] {
146 match subfields.find(name) {
147 Some((_, subfield)) if !is_variant_binary(subfield.data_type()) => {
148 return Err(invalid(format!(
149 "has a non-binary `{name}` field: {}",
150 subfield.data_type()
151 )));
152 }
153 Some((_, subfield)) if name == "metadata" && subfield.is_nullable() => {
156 return Err(invalid(
157 "has an optional `metadata` field, which must be required".to_string(),
158 ));
159 }
160 Some(_) => {}
161 None => return Err(invalid(format!("is missing the `{name}` field"))),
162 }
163 }
164 if subfields.len() != 2 {
165 return Err(invalid(
166 "has fields other than `metadata` and `value`".to_string(),
167 ));
168 }
169 Ok(())
170}
171
172fn to_arrow_wkb_edges(algorithm: EdgeInterpolationAlgorithm) -> WkbEdges {
173 match algorithm {
174 EdgeInterpolationAlgorithm::Spherical => WkbEdges::Spherical,
175 EdgeInterpolationAlgorithm::Vincenty => WkbEdges::Vincenty,
176 EdgeInterpolationAlgorithm::Thomas => WkbEdges::Thomas,
177 EdgeInterpolationAlgorithm::Andoyer => WkbEdges::Andoyer,
178 EdgeInterpolationAlgorithm::Karney => WkbEdges::Karney,
179 }
180}
181
182impl From<WkbEdges> for EdgeInterpolationAlgorithm {
183 fn from(edges: WkbEdges) -> Self {
184 match edges {
185 WkbEdges::Spherical => Self::Spherical,
186 WkbEdges::Vincenty => Self::Vincenty,
187 WkbEdges::Thomas => Self::Thomas,
188 WkbEdges::Andoyer => Self::Andoyer,
189 WkbEdges::Karney => Self::Karney,
190 }
191 }
192}
193
194fn iceberg_crs_from_wkb_metadata(metadata: &WkbMetadata) -> Result<String> {
195 match metadata.crs.as_ref() {
196 None => Ok(UNSET_GEOSPATIAL_CRS.to_string()),
197 Some(serde_json::Value::String(crs)) => Ok(crs.clone()),
198 Some(serde_json::Value::Object(crs)) => {
199 let id = crs.get("id");
200 let authority = id
201 .and_then(|id| id.get("authority"))
202 .and_then(serde_json::Value::as_str)
203 .filter(|authority| !authority.is_empty());
204 let code = match id.and_then(|id| id.get("code")) {
205 Some(serde_json::Value::String(code)) => Some(code.clone()),
206 Some(serde_json::Value::Number(code)) => Some(code.to_string()),
207 _ => None,
208 };
209
210 match (authority, code) {
211 (Some(authority), Some(code)) => Ok(format!("{authority}:{code}")),
212 _ => Err(Error::new(
213 ErrorKind::DataInvalid,
214 "Cannot determine Iceberg geospatial type from PROJJSON CRS without an embedded authority/code",
215 )),
216 }
217 }
218 Some(_) => Err(Error::new(
219 ErrorKind::DataInvalid,
220 "Geospatial CRS metadata must be a string or PROJJSON object",
221 )),
222 }
223}
224
225fn arrow_crs_from_iceberg(crs: &str) -> Option<&str> {
226 (!crs.eq_ignore_ascii_case(UNSET_GEOSPATIAL_CRS)).then_some(crs)
227}
228
229pub trait ArrowSchemaVisitor {
233 type T;
235
236 type U;
238
239 fn before_field(&mut self, _field: &FieldRef) -> Result<()> {
243 Ok(())
244 }
245
246 fn after_field(&mut self, _field: &FieldRef) -> Result<()> {
250 Ok(())
251 }
252
253 fn before_list_element(&mut self, _field: &FieldRef) -> Result<()> {
255 Ok(())
256 }
257
258 fn after_list_element(&mut self, _field: &FieldRef) -> Result<()> {
260 Ok(())
261 }
262
263 fn before_map_key(&mut self, _field: &FieldRef) -> Result<()> {
265 Ok(())
266 }
267
268 fn after_map_key(&mut self, _field: &FieldRef) -> Result<()> {
270 Ok(())
271 }
272
273 fn before_map_value(&mut self, _field: &FieldRef) -> Result<()> {
275 Ok(())
276 }
277
278 fn after_map_value(&mut self, _field: &FieldRef) -> Result<()> {
280 Ok(())
281 }
282
283 fn schema(&mut self, schema: &ArrowSchema, values: Vec<Self::T>) -> Result<Self::U>;
285
286 fn r#struct(&mut self, fields: &Fields, results: Vec<Self::T>) -> Result<Self::T>;
288
289 fn list(&mut self, list: &DataType, value: Self::T) -> Result<Self::T>;
291
292 fn map(&mut self, map: &DataType, key_value: Self::T, value: Self::T) -> Result<Self::T>;
294
295 fn primitive(&mut self, p: &DataType) -> Result<Self::T>;
297
298 fn variant(&mut self, field: &FieldRef) -> Result<Self::T>
309 where Self: Sized {
310 visit_type(field.data_type(), self)
311 }
312
313 fn uuid(&mut self, field: &FieldRef) -> Result<Self::T>
323 where Self: Sized {
324 visit_type(field.data_type(), self)
325 }
326}
327
328fn visit_type<V: ArrowSchemaVisitor>(r#type: &DataType, visitor: &mut V) -> Result<V::T> {
330 match r#type {
331 p if p.is_primitive()
332 || matches!(
333 p,
334 DataType::Boolean
335 | DataType::Utf8
336 | DataType::LargeUtf8
337 | DataType::Utf8View
338 | DataType::Null
339 | DataType::Binary
340 | DataType::LargeBinary
341 | DataType::BinaryView
342 | DataType::FixedSizeBinary(_)
343 ) =>
344 {
345 visitor.primitive(p)
346 }
347 DataType::List(element_field) => visit_list(r#type, element_field, visitor),
348 DataType::LargeList(element_field) => visit_list(r#type, element_field, visitor),
349 DataType::FixedSizeList(element_field, _) => visit_list(r#type, element_field, visitor),
350 DataType::Map(field, _) => match field.data_type() {
351 DataType::Struct(fields) => {
352 if fields.len() != 2 {
353 return Err(invalid_data!("Map field must have exactly 2 fields"));
354 }
355
356 let key_field = &fields[0];
357 let value_field = &fields[1];
358
359 let key_result = {
360 visitor.before_map_key(key_field)?;
361 let ret = visit_field(key_field, visitor)?;
362 visitor.after_map_key(key_field)?;
363 ret
364 };
365
366 let value_result = {
367 visitor.before_map_value(value_field)?;
368 let ret = visit_field(value_field, visitor)?;
369 visitor.after_map_value(value_field)?;
370 ret
371 };
372
373 visitor.map(r#type, key_result, value_result)
374 }
375 _ => Err(invalid_data!("Map field must have struct type")),
376 },
377 DataType::Struct(fields) => visit_struct(fields, visitor),
378 DataType::Dictionary(_key_type, value_type) => visit_type(value_type, visitor),
379 other => Err(invalid_data!("Cannot visit Arrow data type: {other}")),
380 }
381}
382
383fn visit_field<V: ArrowSchemaVisitor>(field: &FieldRef, visitor: &mut V) -> Result<V::T> {
388 if field.extension_type_name() == Some(VariantExtensionType::NAME) {
389 visitor.variant(field)
390 } else if field.extension_type_name() == Some(UuidExtensionType::NAME) {
391 visitor.uuid(field)
392 } else {
393 visit_type(field.data_type(), visitor)
394 }
395}
396
397fn visit_list<V: ArrowSchemaVisitor>(
399 data_type: &DataType,
400 element_field: &FieldRef,
401 visitor: &mut V,
402) -> Result<V::T> {
403 visitor.before_list_element(element_field)?;
404 let value = visit_field(element_field, visitor)?;
405 visitor.after_list_element(element_field)?;
406 visitor.list(data_type, value)
407}
408
409fn visit_struct<V: ArrowSchemaVisitor>(fields: &Fields, visitor: &mut V) -> Result<V::T> {
411 let mut results = Vec::with_capacity(fields.len());
412 for field in fields {
413 visitor.before_field(field)?;
414 let result = visit_field(field, visitor)?;
415 visitor.after_field(field)?;
416 results.push(result);
417 }
418
419 visitor.r#struct(fields, results)
420}
421
422pub(crate) fn visit_schema<V: ArrowSchemaVisitor>(
424 schema: &ArrowSchema,
425 visitor: &mut V,
426) -> Result<V::U> {
427 let mut results = Vec::with_capacity(schema.fields().len());
428 for field in schema.fields() {
429 visitor.before_field(field)?;
430 let result = visit_field(field, visitor)?;
431 visitor.after_field(field)?;
432 results.push(result);
433 }
434 visitor.schema(schema, results)
435}
436
437pub fn arrow_schema_to_schema(schema: &ArrowSchema) -> Result<Schema> {
443 let mut visitor = ArrowSchemaConverter::new();
444 visit_schema(schema, &mut visitor)
445}
446
447pub fn arrow_schema_to_schema_auto_assign_ids(schema: &ArrowSchema) -> Result<Schema> {
456 let mut visitor = ArrowSchemaConverter::new_with_field_ids_from(FIRST_FIELD_ID);
457 visit_schema(schema, &mut visitor)
458}
459
460pub fn arrow_type_to_type(ty: &DataType) -> Result<Type> {
462 let mut visitor = ArrowSchemaConverter::new();
463 visit_type(ty, &mut visitor)
464}
465
466const ARROW_FIELD_DOC_KEY: &str = "doc";
467
468pub(super) fn try_get_field_id_from_metadata(field: &Field) -> Result<Option<i32>> {
469 field
470 .metadata()
471 .get(PARQUET_FIELD_ID_META_KEY)
472 .map(|value| {
473 value.parse::<i32>().map_err(|e| {
474 invalid_data!("Failed to parse field id")
475 .with_context("value", value)
476 .with_source(e)
477 })
478 })
479 .transpose()
480}
481
482pub(super) fn get_field_id_from_metadata(field: &Field) -> Result<i32> {
483 try_get_field_id_from_metadata(field)?
484 .ok_or_else(|| invalid_data!("Field id not found in metadata"))
485}
486
487fn get_field_doc(field: &FieldRef) -> Option<String> {
488 if let Some(value) = field.metadata().get(ARROW_FIELD_DOC_KEY) {
489 return Some(value.clone());
490 }
491 None
492}
493
494struct ArrowSchemaConverter {
495 reassign_field_ids_from: Option<i32>,
498 next_field_id: i32,
502}
503
504impl ArrowSchemaConverter {
505 fn new() -> Self {
506 Self {
507 reassign_field_ids_from: None,
508 next_field_id: 0,
509 }
510 }
511
512 fn new_with_field_ids_from(start_from: i32) -> Self {
513 Self {
514 reassign_field_ids_from: Some(start_from),
515 next_field_id: 0,
516 }
517 }
518
519 fn get_field_id(&mut self, field: &FieldRef) -> Result<i32> {
520 if self.reassign_field_ids_from.is_some() {
521 let temp_id = self.next_field_id;
525 self.next_field_id += 1;
526 Ok(temp_id)
527 } else {
528 get_field_id_from_metadata(field)
530 }
531 }
532
533 fn convert_fields(
534 &mut self,
535 fields: &Fields,
536 field_results: &[Type],
537 ) -> Result<Vec<NestedFieldRef>> {
538 let mut results = Vec::with_capacity(fields.len());
539 for i in 0..fields.len() {
540 let field = &fields[i];
541 let field_type = self.apply_arrow_extension(field, field_results[i].clone())?;
542 let id = self.get_field_id(field)?;
543 let doc = get_field_doc(field);
544 let nested_field = NestedField {
545 id,
546 doc,
547 name: field.name().clone(),
548 required: !field.is_nullable(),
549 field_type: Box::new(field_type),
550 initial_default: None,
551 write_default: None,
552 };
553 results.push(Arc::new(nested_field));
554 }
555 Ok(results)
556 }
557
558 fn apply_arrow_extension(&self, arrow_field: &FieldRef, iceberg_type: Type) -> Result<Type> {
559 if arrow_field.extension_type_name() != Some(WkbType::NAME) {
560 return Ok(iceberg_type);
561 }
562
563 if !matches!(&iceberg_type, Type::Primitive(PrimitiveType::Binary)) {
564 return Err(Error::new(
565 ErrorKind::DataInvalid,
566 format!(
567 "WKB extension type on field {} requires binary storage, got {iceberg_type}",
568 arrow_field.name()
569 ),
570 ));
571 }
572
573 let wkb_type = arrow_field.try_extension_type::<WkbType>().map_err(|err| {
574 Error::new(
575 ErrorKind::DataInvalid,
576 format!(
577 "Invalid geospatial Arrow extension metadata for field {}",
578 arrow_field.name()
579 ),
580 )
581 .with_source(err)
582 })?;
583
584 let crs = iceberg_crs_from_wkb_metadata(wkb_type.metadata())?;
585
586 match wkb_type.metadata().type_hint() {
587 WkbTypeHint::Geometry => Ok(Type::Primitive(PrimitiveType::Geometry(
588 GeometryType::new(Some(crs))?,
589 ))),
590 WkbTypeHint::Geography => Ok(Type::Primitive(PrimitiveType::Geography(
591 GeographyType::new(
592 Some(crs),
593 wkb_type
594 .metadata()
595 .algorithm
596 .unwrap_or(WkbEdges::Spherical)
597 .into(),
598 )?,
599 ))),
600 }
601 }
602}
603
604impl ArrowSchemaVisitor for ArrowSchemaConverter {
605 type T = Type;
606 type U = Schema;
607
608 fn schema(&mut self, schema: &ArrowSchema, values: Vec<Self::T>) -> Result<Self::U> {
609 let fields = self.convert_fields(schema.fields(), &values)?;
610 let mut builder = Schema::builder().with_fields(fields);
611 if let Some(start_from) = self.reassign_field_ids_from {
612 builder = builder.with_reassigned_field_ids(start_from)
613 }
614 builder.build()
615 }
616
617 fn r#struct(&mut self, fields: &Fields, results: Vec<Self::T>) -> Result<Self::T> {
618 let fields = self.convert_fields(fields, &results)?;
619 Ok(Type::Struct(StructType::new(fields)))
620 }
621
622 fn list(&mut self, list: &DataType, value: Self::T) -> Result<Self::T> {
623 let element_field = match list {
624 DataType::List(element_field) => element_field,
625 DataType::LargeList(element_field) => element_field,
626 DataType::FixedSizeList(element_field, _) => element_field,
627 _ => {
628 return Err(invalid_data!("List type must have list data type"));
629 }
630 };
631
632 let value = self.apply_arrow_extension(element_field, value)?;
633 let id = self.get_field_id(element_field)?;
634 let doc = get_field_doc(element_field);
635 let mut element_field =
636 NestedField::list_element(id, value.clone(), !element_field.is_nullable());
637 if let Some(doc) = doc {
638 element_field = element_field.with_doc(doc);
639 }
640 let element_field = Arc::new(element_field);
641 Ok(Type::List(ListType { element_field }))
642 }
643
644 fn map(&mut self, map: &DataType, key_value: Self::T, value: Self::T) -> Result<Self::T> {
645 match map {
646 DataType::Map(field, _) => match field.data_type() {
647 DataType::Struct(fields) => {
648 if fields.len() != 2 {
649 return Err(invalid_data!("Map field must have exactly 2 fields"));
650 }
651
652 let key_field = &fields[0];
653 let value_field = &fields[1];
654 let key_value = self.apply_arrow_extension(key_field, key_value)?;
655 let value = self.apply_arrow_extension(value_field, value)?;
656
657 let key_id = self.get_field_id(key_field)?;
658 let key_doc = get_field_doc(key_field);
659 let mut key_field = NestedField::map_key_element(key_id, key_value.clone());
660 if let Some(doc) = key_doc {
661 key_field = key_field.with_doc(doc);
662 }
663 let key_field = Arc::new(key_field);
664
665 let value_id = self.get_field_id(value_field)?;
666 let value_doc = get_field_doc(value_field);
667 let mut value_field = NestedField::map_value_element(
668 value_id,
669 value.clone(),
670 !value_field.is_nullable(),
671 );
672 if let Some(doc) = value_doc {
673 value_field = value_field.with_doc(doc);
674 }
675 let value_field = Arc::new(value_field);
676
677 Ok(Type::Map(MapType {
678 key_field,
679 value_field,
680 }))
681 }
682 _ => Err(invalid_data!("Map field must have struct type")),
683 },
684 _ => Err(invalid_data!("Map type must have map data type")),
685 }
686 }
687
688 fn primitive(&mut self, p: &DataType) -> Result<Self::T> {
689 match p {
690 DataType::Null => Ok(Type::Primitive(PrimitiveType::Unknown)),
691 DataType::Boolean => Ok(Type::Primitive(PrimitiveType::Boolean)),
692 DataType::Int8 | DataType::Int16 | DataType::Int32 => {
693 Ok(Type::Primitive(PrimitiveType::Int))
694 }
695 DataType::UInt8 | DataType::UInt16 => Ok(Type::Primitive(PrimitiveType::Int)),
696 DataType::UInt32 => Ok(Type::Primitive(PrimitiveType::Long)),
697 DataType::Int64 => Ok(Type::Primitive(PrimitiveType::Long)),
698 DataType::UInt64 => {
699 Err(invalid_data!(
701 "UInt64 is not supported. Use Int64 for values ≤ 9,223,372,036,854,775,807 or Decimal(20,0) for full uint64 range."
702 ))
703 }
704 DataType::Float32 => Ok(Type::Primitive(PrimitiveType::Float)),
705 DataType::Float64 => Ok(Type::Primitive(PrimitiveType::Double)),
706 DataType::Decimal128(p, s) => Type::decimal(*p as u32, *s as u32)
707 .map_err(|e| invalid_data!("Failed to create decimal type").with_source(e)),
708 DataType::Date32 => Ok(Type::Primitive(PrimitiveType::Date)),
709 DataType::Time64(unit) if unit == &TimeUnit::Microsecond => {
710 Ok(Type::Primitive(PrimitiveType::Time))
711 }
712 DataType::Timestamp(unit, None) if unit == &TimeUnit::Microsecond => {
713 Ok(Type::Primitive(PrimitiveType::Timestamp))
714 }
715 DataType::Timestamp(unit, None) if unit == &TimeUnit::Nanosecond => {
716 Ok(Type::Primitive(PrimitiveType::TimestampNs))
717 }
718 DataType::Timestamp(unit, Some(zone))
719 if unit == &TimeUnit::Microsecond
720 && (zone.as_ref() == "UTC" || zone.as_ref() == "+00:00") =>
721 {
722 Ok(Type::Primitive(PrimitiveType::Timestamptz))
723 }
724 DataType::Timestamp(unit, Some(zone))
725 if unit == &TimeUnit::Nanosecond
726 && (zone.as_ref() == "UTC" || zone.as_ref() == "+00:00") =>
727 {
728 Ok(Type::Primitive(PrimitiveType::TimestamptzNs))
729 }
730 DataType::Binary | DataType::LargeBinary | DataType::BinaryView => {
731 Ok(Type::Primitive(PrimitiveType::Binary))
732 }
733 DataType::FixedSizeBinary(width) => {
734 Ok(Type::Primitive(PrimitiveType::Fixed(*width as u64)))
735 }
736 DataType::Utf8View | DataType::Utf8 | DataType::LargeUtf8 => {
737 Ok(Type::Primitive(PrimitiveType::String))
738 }
739 _ => Err(invalid_data!("Unsupported Arrow data type: {p}")),
740 }
741 }
742
743 fn variant(&mut self, field: &FieldRef) -> Result<Self::T> {
744 if !matches!(field.data_type(), DataType::Struct(_)) {
747 return Err(invalid_data!(
748 "arrow.parquet.variant extension requires Struct storage"
749 ));
750 }
751 Ok(Type::Variant(VariantType))
755 }
756
757 fn uuid(&mut self, field: &FieldRef) -> Result<Self::T> {
758 let field_data_type = field.data_type();
761 let expected_data_type = DataType::FixedSizeBinary(16);
762 if *field_data_type != expected_data_type {
763 let error_message = format!(
764 "{} extension requires {expected_data_type} storage, found {field_data_type}",
765 UuidExtensionType::NAME,
766 );
767
768 return Err(Error::new(ErrorKind::DataInvalid, error_message));
769 }
770
771 Ok(Type::Primitive(PrimitiveType::Uuid))
772 }
773}
774
775struct ToArrowSchemaConverter;
776
777enum ArrowSchemaOrFieldOrType {
778 Schema(ArrowSchema),
779 Field(Field),
780 Type(DataType),
781}
782
783impl SchemaVisitor for ToArrowSchemaConverter {
784 type T = ArrowSchemaOrFieldOrType;
785
786 fn schema(
787 &mut self,
788 _schema: &Schema,
789 value: ArrowSchemaOrFieldOrType,
790 ) -> Result<ArrowSchemaOrFieldOrType> {
791 let struct_type = match value {
792 ArrowSchemaOrFieldOrType::Type(DataType::Struct(fields)) => fields,
793 _ => unreachable!(),
794 };
795 Ok(ArrowSchemaOrFieldOrType::Schema(ArrowSchema::new(
796 struct_type,
797 )))
798 }
799
800 fn field(
801 &mut self,
802 field: &NestedFieldRef,
803 value: ArrowSchemaOrFieldOrType,
804 ) -> Result<ArrowSchemaOrFieldOrType> {
805 let ty = match value {
806 ArrowSchemaOrFieldOrType::Type(ty) => ty,
807 _ => unreachable!(),
808 };
809 let metadata = if let Some(doc) = &field.doc {
810 HashMap::from([
811 (PARQUET_FIELD_ID_META_KEY.to_string(), field.id.to_string()),
812 (ARROW_FIELD_DOC_KEY.to_string(), doc.clone()),
813 ])
814 } else {
815 HashMap::from([(PARQUET_FIELD_ID_META_KEY.to_string(), field.id.to_string())])
816 };
817 let mut arrow_field =
818 Field::new(field.name.clone(), ty, !field.required).with_metadata(metadata);
819
820 match field.field_type.as_ref() {
821 Type::Variant(_) => {
822 arrow_field.try_with_extension_type(VariantExtensionType)?;
825 }
826 Type::Primitive(PrimitiveType::Uuid) => {
827 arrow_field.try_with_extension_type(UuidExtensionType)?;
829 }
830 Type::Primitive(PrimitiveType::Geometry(geometry)) => {
831 let metadata = WkbMetadata::new(arrow_crs_from_iceberg(geometry.crs()), None);
832 arrow_field.try_with_extension_type(WkbType::new(Some(metadata)))?;
833 }
834 Type::Primitive(PrimitiveType::Geography(geography)) => {
835 let metadata = WkbMetadata::new(
836 arrow_crs_from_iceberg(geography.crs()),
837 Some(to_arrow_wkb_edges(geography.algorithm())),
838 );
839 arrow_field.try_with_extension_type(WkbType::new(Some(metadata)))?;
840 }
841 _ => {}
842 }
843
844 Ok(ArrowSchemaOrFieldOrType::Field(arrow_field))
845 }
846
847 fn r#struct(
848 &mut self,
849 _: &StructType,
850 results: Vec<ArrowSchemaOrFieldOrType>,
851 ) -> Result<ArrowSchemaOrFieldOrType> {
852 let fields = results
853 .into_iter()
854 .map(|result| match result {
855 ArrowSchemaOrFieldOrType::Field(field) => field,
856 _ => unreachable!(),
857 })
858 .collect();
859 Ok(ArrowSchemaOrFieldOrType::Type(DataType::Struct(fields)))
860 }
861
862 fn list(&mut self, list: &ListType, value: ArrowSchemaOrFieldOrType) -> Result<Self::T> {
863 let field = match self.field(&list.element_field, value)? {
867 ArrowSchemaOrFieldOrType::Field(field) => field,
868 _ => unreachable!(),
869 };
870 Ok(ArrowSchemaOrFieldOrType::Type(DataType::List(Arc::new(
871 field,
872 ))))
873 }
874
875 fn map(
876 &mut self,
877 map: &MapType,
878 key_value: ArrowSchemaOrFieldOrType,
879 value: ArrowSchemaOrFieldOrType,
880 ) -> Result<ArrowSchemaOrFieldOrType> {
881 let key_field = match self.field(&map.key_field, key_value)? {
882 ArrowSchemaOrFieldOrType::Field(field) => field,
883 _ => unreachable!(),
884 };
885 let value_field = match self.field(&map.value_field, value)? {
886 ArrowSchemaOrFieldOrType::Field(field) => field,
887 _ => unreachable!(),
888 };
889 let field = Field::new(
890 DEFAULT_MAP_FIELD_NAME,
891 DataType::Struct(vec![key_field, value_field].into()),
892 false,
894 );
895
896 Ok(ArrowSchemaOrFieldOrType::Type(DataType::Map(
897 field.into(),
898 false,
899 )))
900 }
901
902 fn primitive(&mut self, p: &PrimitiveType) -> Result<ArrowSchemaOrFieldOrType> {
903 match p {
904 PrimitiveType::Unknown => Ok(ArrowSchemaOrFieldOrType::Type(DataType::Null)),
905 PrimitiveType::Boolean => Ok(ArrowSchemaOrFieldOrType::Type(DataType::Boolean)),
906 PrimitiveType::Int => Ok(ArrowSchemaOrFieldOrType::Type(DataType::Int32)),
907 PrimitiveType::Long => Ok(ArrowSchemaOrFieldOrType::Type(DataType::Int64)),
908 PrimitiveType::Float => Ok(ArrowSchemaOrFieldOrType::Type(DataType::Float32)),
909 PrimitiveType::Double => Ok(ArrowSchemaOrFieldOrType::Type(DataType::Float64)),
910 PrimitiveType::Decimal { precision, scale } => {
911 let (precision, scale) = {
912 let precision: u8 = precision.to_owned().try_into().map_err(|err| {
913 invalid_data!("incompatible precision for decimal type convert")
914 .with_source(err)
915 })?;
916 let scale = scale.to_owned().try_into().map_err(|err| {
917 invalid_data!("incompatible scale for decimal type convert")
918 .with_source(err)
919 })?;
920 (precision, scale)
921 };
922 validate_decimal_precision_and_scale::<Decimal128Type>(precision, scale).map_err(
923 |err| {
924 invalid_data!("incompatible precision and scale for decimal type convert")
925 .with_source(err)
926 },
927 )?;
928 Ok(ArrowSchemaOrFieldOrType::Type(DataType::Decimal128(
929 precision, scale,
930 )))
931 }
932 PrimitiveType::Date => Ok(ArrowSchemaOrFieldOrType::Type(DataType::Date32)),
933 PrimitiveType::Time => Ok(ArrowSchemaOrFieldOrType::Type(DataType::Time64(
934 TimeUnit::Microsecond,
935 ))),
936 PrimitiveType::Timestamp => Ok(ArrowSchemaOrFieldOrType::Type(DataType::Timestamp(
937 TimeUnit::Microsecond,
938 None,
939 ))),
940 PrimitiveType::Timestamptz => Ok(ArrowSchemaOrFieldOrType::Type(
941 DataType::Timestamp(TimeUnit::Microsecond, Some(UTC_TIME_ZONE.into())),
943 )),
944 PrimitiveType::TimestampNs => Ok(ArrowSchemaOrFieldOrType::Type(DataType::Timestamp(
945 TimeUnit::Nanosecond,
946 None,
947 ))),
948 PrimitiveType::TimestamptzNs => Ok(ArrowSchemaOrFieldOrType::Type(
949 DataType::Timestamp(TimeUnit::Nanosecond, Some(UTC_TIME_ZONE.into())),
951 )),
952 PrimitiveType::String => Ok(ArrowSchemaOrFieldOrType::Type(DataType::Utf8)),
953 PrimitiveType::Uuid => Ok(ArrowSchemaOrFieldOrType::Type(DataType::FixedSizeBinary(
954 16,
955 ))),
956 PrimitiveType::Fixed(len) => Ok(ArrowSchemaOrFieldOrType::Type(
957 i32::try_from(*len)
958 .ok()
959 .map(DataType::FixedSizeBinary)
960 .unwrap_or(DataType::LargeBinary),
961 )),
962 PrimitiveType::Binary => Ok(ArrowSchemaOrFieldOrType::Type(DataType::Binary)),
963 PrimitiveType::Geometry(_) | PrimitiveType::Geography(_) => {
964 Ok(ArrowSchemaOrFieldOrType::Type(DataType::LargeBinary))
965 }
966 }
967 }
968
969 fn variant(&mut self, _v: &VariantType) -> Result<ArrowSchemaOrFieldOrType> {
970 let metadata_field = Field::new("metadata", DataType::Binary, false);
976 let value_field = Field::new("value", DataType::Binary, true);
977 Ok(ArrowSchemaOrFieldOrType::Type(DataType::Struct(
978 vec![metadata_field, value_field].into(),
979 )))
980 }
981}
982
983pub fn schema_to_arrow_schema(schema: &Schema) -> Result<ArrowSchema> {
985 let mut converter = ToArrowSchemaConverter;
986 match crate::spec::visit_schema(schema, &mut converter)? {
987 ArrowSchemaOrFieldOrType::Schema(schema) => Ok(schema),
988 _ => unreachable!(),
989 }
990}
991
992pub fn type_to_arrow_type(ty: &Type) -> Result<DataType> {
994 let mut converter = ToArrowSchemaConverter;
995 match crate::spec::visit_type(ty, &mut converter)? {
996 ArrowSchemaOrFieldOrType::Type(ty) => Ok(ty),
997 _ => unreachable!(),
998 }
999}
1000
1001pub(crate) fn get_arrow_datum(datum: &Datum) -> Result<Arc<dyn ArrowDatum + Send + Sync>> {
1003 match (datum.data_type(), datum.literal()) {
1004 (PrimitiveType::Boolean, PrimitiveLiteral::Boolean(value)) => {
1005 Ok(Arc::new(BooleanArray::new_scalar(*value)))
1006 }
1007 (PrimitiveType::Int, PrimitiveLiteral::Int(value)) => {
1008 Ok(Arc::new(Int32Array::new_scalar(*value)))
1009 }
1010 (PrimitiveType::Long, PrimitiveLiteral::Long(value)) => {
1011 Ok(Arc::new(Int64Array::new_scalar(*value)))
1012 }
1013 (PrimitiveType::Float, PrimitiveLiteral::Float(value)) => {
1014 Ok(Arc::new(Float32Array::new_scalar(value.into_inner())))
1015 }
1016 (PrimitiveType::Double, PrimitiveLiteral::Double(value)) => {
1017 Ok(Arc::new(Float64Array::new_scalar(value.into_inner())))
1018 }
1019 (PrimitiveType::String, PrimitiveLiteral::String(value)) => {
1020 Ok(Arc::new(StringArray::new_scalar(value.as_str())))
1021 }
1022 (PrimitiveType::Binary, PrimitiveLiteral::Binary(value)) => {
1023 Ok(Arc::new(BinaryArray::new_scalar(value.as_slice())))
1024 }
1025 (PrimitiveType::Date, PrimitiveLiteral::Int(value)) => {
1026 Ok(Arc::new(Date32Array::new_scalar(*value)))
1027 }
1028 (PrimitiveType::Timestamp, PrimitiveLiteral::Long(value)) => {
1029 Ok(Arc::new(TimestampMicrosecondArray::new_scalar(*value)))
1030 }
1031 (PrimitiveType::Timestamptz, PrimitiveLiteral::Long(value)) => Ok(Arc::new(Scalar::new(
1032 TimestampMicrosecondArray::new(vec![*value; 1].into(), None).with_timezone_utc(),
1033 ))),
1034 (PrimitiveType::TimestampNs, PrimitiveLiteral::Long(value)) => {
1035 Ok(Arc::new(TimestampNanosecondArray::new_scalar(*value)))
1036 }
1037 (PrimitiveType::TimestamptzNs, PrimitiveLiteral::Long(value)) => Ok(Arc::new(Scalar::new(
1038 TimestampNanosecondArray::new(vec![*value; 1].into(), None).with_timezone_utc(),
1039 ))),
1040 (PrimitiveType::Decimal { precision, scale }, PrimitiveLiteral::Int128(value)) => {
1041 let array = Decimal128Array::from_value(*value, 1)
1042 .with_precision_and_scale(*precision as _, *scale as _)
1043 .unwrap();
1044 Ok(Arc::new(Scalar::new(array)))
1045 }
1046 (PrimitiveType::Uuid, PrimitiveLiteral::UInt128(value)) => {
1047 let bytes = Uuid::from_u128(*value).into_bytes();
1048 let array = FixedSizeBinaryArray::try_from_iter(vec![bytes].into_iter()).unwrap();
1049 Ok(Arc::new(Scalar::new(array)))
1050 }
1051 (PrimitiveType::Fixed(_), PrimitiveLiteral::Binary(value)) => {
1052 let array = FixedSizeBinaryArray::try_from_iter(std::iter::once(value.as_slice()))
1053 .map_err(|e| invalid_data!("FixedSizeBinary conversion failed").with_source(e))?;
1054 Ok(Arc::new(Scalar::new(array)))
1055 }
1056
1057 (primitive_type, _) => Err(Error::new(
1058 ErrorKind::FeatureUnsupported,
1059 format!("Converting datum from type {primitive_type:?} to arrow not supported yet."),
1060 )),
1061 }
1062}
1063
1064pub(crate) fn get_parquet_stat_min_as_datum(
1065 primitive_type: &PrimitiveType,
1066 stats: &Statistics,
1067) -> Result<Option<Datum>> {
1068 Ok(match (primitive_type, stats) {
1069 (PrimitiveType::Boolean, Statistics::Boolean(stats)) => {
1070 stats.min_opt().map(|val| Datum::bool(*val))
1071 }
1072 (PrimitiveType::Int, Statistics::Int32(stats)) => {
1073 stats.min_opt().map(|val| Datum::int(*val))
1074 }
1075 (PrimitiveType::Date, Statistics::Int32(stats)) => {
1076 stats.min_opt().map(|val| Datum::date(*val))
1077 }
1078 (PrimitiveType::Long, Statistics::Int64(stats)) => {
1079 stats.min_opt().map(|val| Datum::long(*val))
1080 }
1081 (PrimitiveType::Time, Statistics::Int64(stats)) => {
1082 let Some(val) = stats.min_opt() else {
1083 return Ok(None);
1084 };
1085
1086 Some(Datum::time_micros(*val)?)
1087 }
1088 (PrimitiveType::Timestamp, Statistics::Int64(stats)) => {
1089 stats.min_opt().map(|val| Datum::timestamp_micros(*val))
1090 }
1091 (PrimitiveType::Timestamptz, Statistics::Int64(stats)) => {
1092 stats.min_opt().map(|val| Datum::timestamptz_micros(*val))
1093 }
1094 (PrimitiveType::TimestampNs, Statistics::Int64(stats)) => {
1095 stats.min_opt().map(|val| Datum::timestamp_nanos(*val))
1096 }
1097 (PrimitiveType::TimestamptzNs, Statistics::Int64(stats)) => {
1098 stats.min_opt().map(|val| Datum::timestamptz_nanos(*val))
1099 }
1100 (PrimitiveType::Float, Statistics::Float(stats)) => {
1101 stats.min_opt().map(|val| Datum::float(*val))
1102 }
1103 (PrimitiveType::Double, Statistics::Double(stats)) => {
1104 stats.min_opt().map(|val| Datum::double(*val))
1105 }
1106 (PrimitiveType::String, Statistics::ByteArray(stats)) => {
1107 let Some(val) = stats.min_opt() else {
1108 return Ok(None);
1109 };
1110
1111 Some(Datum::string(val.as_utf8()?))
1112 }
1113 (
1114 PrimitiveType::Decimal {
1115 precision: _,
1116 scale: _,
1117 },
1118 Statistics::ByteArray(stats),
1119 ) => {
1120 let Some(bytes) = stats.min_bytes_opt() else {
1121 return Ok(None);
1122 };
1123 Some(Datum::new(
1124 primitive_type.clone(),
1125 PrimitiveLiteral::Int128(
1126 i128_from_be_bytes(bytes)
1129 .filter(|_| !bytes.is_empty())
1130 .ok_or_else(|| invalid_data!("Can't convert bytes to i128: {bytes:?}"))?,
1131 ),
1132 ))
1133 }
1134 (
1135 PrimitiveType::Decimal {
1136 precision: _,
1137 scale: _,
1138 },
1139 Statistics::FixedLenByteArray(stats),
1140 ) => {
1141 let Some(bytes) = stats.min_bytes_opt() else {
1142 return Ok(None);
1143 };
1144 Some(Datum::new(
1145 primitive_type.clone(),
1146 PrimitiveLiteral::Int128(
1147 i128_from_be_bytes(bytes)
1148 .ok_or_else(|| invalid_data!("Can't convert bytes to i128: {bytes:?}"))?,
1149 ),
1150 ))
1151 }
1152 (
1153 PrimitiveType::Decimal {
1154 precision: _,
1155 scale: _,
1156 },
1157 Statistics::Int32(stats),
1158 ) => stats.min_opt().map(|val| {
1159 Datum::new(
1160 primitive_type.clone(),
1161 PrimitiveLiteral::Int128(i128::from(*val)),
1162 )
1163 }),
1164
1165 (
1166 PrimitiveType::Decimal {
1167 precision: _,
1168 scale: _,
1169 },
1170 Statistics::Int64(stats),
1171 ) => stats.min_opt().map(|val| {
1172 Datum::new(
1173 primitive_type.clone(),
1174 PrimitiveLiteral::Int128(i128::from(*val)),
1175 )
1176 }),
1177 (PrimitiveType::Uuid, Statistics::FixedLenByteArray(stats)) => {
1178 let Some(bytes) = stats.min_bytes_opt() else {
1179 return Ok(None);
1180 };
1181 if bytes.len() != 16 {
1182 return Err(Error::new(
1183 ErrorKind::Unexpected,
1184 "Invalid length of uuid bytes.",
1185 ));
1186 }
1187 Some(Datum::uuid(Uuid::from_bytes(
1188 bytes[..16].try_into().unwrap(),
1189 )))
1190 }
1191 (PrimitiveType::Fixed(len), Statistics::FixedLenByteArray(stat)) => {
1192 let Some(bytes) = stat.min_bytes_opt() else {
1193 return Ok(None);
1194 };
1195 if bytes.len() != *len as usize {
1196 return Err(Error::new(
1197 ErrorKind::Unexpected,
1198 "Invalid length of fixed bytes.",
1199 ));
1200 }
1201 Some(Datum::fixed(bytes.to_vec()))
1202 }
1203 (PrimitiveType::Binary, Statistics::ByteArray(stat)) => {
1204 return Ok(stat
1205 .min_bytes_opt()
1206 .map(|bytes| Datum::binary(bytes.to_vec())));
1207 }
1208 _ => {
1209 return Ok(None);
1210 }
1211 })
1212}
1213
1214pub(crate) fn get_parquet_stat_max_as_datum(
1215 primitive_type: &PrimitiveType,
1216 stats: &Statistics,
1217) -> Result<Option<Datum>> {
1218 Ok(match (primitive_type, stats) {
1219 (PrimitiveType::Boolean, Statistics::Boolean(stats)) => {
1220 stats.max_opt().map(|val| Datum::bool(*val))
1221 }
1222 (PrimitiveType::Int, Statistics::Int32(stats)) => {
1223 stats.max_opt().map(|val| Datum::int(*val))
1224 }
1225 (PrimitiveType::Date, Statistics::Int32(stats)) => {
1226 stats.max_opt().map(|val| Datum::date(*val))
1227 }
1228 (PrimitiveType::Long, Statistics::Int64(stats)) => {
1229 stats.max_opt().map(|val| Datum::long(*val))
1230 }
1231 (PrimitiveType::Time, Statistics::Int64(stats)) => {
1232 let Some(val) = stats.max_opt() else {
1233 return Ok(None);
1234 };
1235
1236 Some(Datum::time_micros(*val)?)
1237 }
1238 (PrimitiveType::Timestamp, Statistics::Int64(stats)) => {
1239 stats.max_opt().map(|val| Datum::timestamp_micros(*val))
1240 }
1241 (PrimitiveType::Timestamptz, Statistics::Int64(stats)) => {
1242 stats.max_opt().map(|val| Datum::timestamptz_micros(*val))
1243 }
1244 (PrimitiveType::TimestampNs, Statistics::Int64(stats)) => {
1245 stats.max_opt().map(|val| Datum::timestamp_nanos(*val))
1246 }
1247 (PrimitiveType::TimestamptzNs, Statistics::Int64(stats)) => {
1248 stats.max_opt().map(|val| Datum::timestamptz_nanos(*val))
1249 }
1250 (PrimitiveType::Float, Statistics::Float(stats)) => {
1251 stats.max_opt().map(|val| Datum::float(*val))
1252 }
1253 (PrimitiveType::Double, Statistics::Double(stats)) => {
1254 stats.max_opt().map(|val| Datum::double(*val))
1255 }
1256 (PrimitiveType::String, Statistics::ByteArray(stats)) => {
1257 let Some(val) = stats.max_opt() else {
1258 return Ok(None);
1259 };
1260
1261 Some(Datum::string(val.as_utf8()?))
1262 }
1263 (
1264 PrimitiveType::Decimal {
1265 precision: _,
1266 scale: _,
1267 },
1268 Statistics::ByteArray(stats),
1269 ) => {
1270 let Some(bytes) = stats.max_bytes_opt() else {
1271 return Ok(None);
1272 };
1273 Some(Datum::new(
1274 primitive_type.clone(),
1275 PrimitiveLiteral::Int128(
1276 i128_from_be_bytes(bytes)
1279 .filter(|_| !bytes.is_empty())
1280 .ok_or_else(|| invalid_data!("Can't convert bytes to i128: {bytes:?}"))?,
1281 ),
1282 ))
1283 }
1284 (
1285 PrimitiveType::Decimal {
1286 precision: _,
1287 scale: _,
1288 },
1289 Statistics::FixedLenByteArray(stats),
1290 ) => {
1291 let Some(bytes) = stats.max_bytes_opt() else {
1292 return Ok(None);
1293 };
1294 Some(Datum::new(
1295 primitive_type.clone(),
1296 PrimitiveLiteral::Int128(
1297 i128_from_be_bytes(bytes)
1298 .ok_or_else(|| invalid_data!("Can't convert bytes to i128: {bytes:?}"))?,
1299 ),
1300 ))
1301 }
1302 (
1303 PrimitiveType::Decimal {
1304 precision: _,
1305 scale: _,
1306 },
1307 Statistics::Int32(stats),
1308 ) => stats.max_opt().map(|val| {
1309 Datum::new(
1310 primitive_type.clone(),
1311 PrimitiveLiteral::Int128(i128::from(*val)),
1312 )
1313 }),
1314
1315 (
1316 PrimitiveType::Decimal {
1317 precision: _,
1318 scale: _,
1319 },
1320 Statistics::Int64(stats),
1321 ) => stats.max_opt().map(|val| {
1322 Datum::new(
1323 primitive_type.clone(),
1324 PrimitiveLiteral::Int128(i128::from(*val)),
1325 )
1326 }),
1327 (PrimitiveType::Uuid, Statistics::FixedLenByteArray(stats)) => {
1328 let Some(bytes) = stats.max_bytes_opt() else {
1329 return Ok(None);
1330 };
1331 if bytes.len() != 16 {
1332 return Err(Error::new(
1333 ErrorKind::Unexpected,
1334 "Invalid length of uuid bytes.",
1335 ));
1336 }
1337 Some(Datum::uuid(Uuid::from_bytes(
1338 bytes[..16].try_into().unwrap(),
1339 )))
1340 }
1341 (PrimitiveType::Fixed(len), Statistics::FixedLenByteArray(stat)) => {
1342 let Some(bytes) = stat.max_bytes_opt() else {
1343 return Ok(None);
1344 };
1345 if bytes.len() != *len as usize {
1346 return Err(Error::new(
1347 ErrorKind::Unexpected,
1348 "Invalid length of fixed bytes.",
1349 ));
1350 }
1351 Some(Datum::fixed(bytes.to_vec()))
1352 }
1353 (PrimitiveType::Binary, Statistics::ByteArray(stat)) => {
1354 return Ok(stat
1355 .max_bytes_opt()
1356 .map(|bytes| Datum::binary(bytes.to_vec())));
1357 }
1358 _ => {
1359 return Ok(None);
1360 }
1361 })
1362}
1363
1364impl TryFrom<&ArrowSchema> for Schema {
1365 type Error = Error;
1366
1367 fn try_from(schema: &ArrowSchema) -> Result<Self> {
1368 arrow_schema_to_schema(schema)
1369 }
1370}
1371
1372impl TryFrom<&Schema> for ArrowSchema {
1373 type Error = Error;
1374
1375 fn try_from(schema: &Schema) -> Result<Self> {
1376 schema_to_arrow_schema(schema)
1377 }
1378}
1379
1380pub fn datum_to_arrow_type_with_ree(datum: &Datum) -> DataType {
1402 primitive_type_to_arrow_type_with_ree(datum.data_type())
1403}
1404
1405pub(crate) fn primitive_type_to_arrow_type_with_ree(primitive_type: &PrimitiveType) -> DataType {
1409 let make_ree = |values_type: DataType| -> DataType {
1413 let run_ends_field = Arc::new(Field::new("run_ends", DataType::Int32, false));
1414 let values_field = Arc::new(Field::new("values", values_type, true));
1415 DataType::RunEndEncoded(run_ends_field, values_field)
1416 };
1417
1418 match primitive_type {
1419 PrimitiveType::Unknown => make_ree(DataType::Null),
1420 PrimitiveType::Boolean => make_ree(DataType::Boolean),
1421 PrimitiveType::Int => make_ree(DataType::Int32),
1422 PrimitiveType::Long => make_ree(DataType::Int64),
1423 PrimitiveType::Float => make_ree(DataType::Float32),
1424 PrimitiveType::Double => make_ree(DataType::Float64),
1425 PrimitiveType::Date => make_ree(DataType::Date32),
1426 PrimitiveType::Time => make_ree(DataType::Int64),
1427 PrimitiveType::Timestamp => make_ree(DataType::Int64),
1428 PrimitiveType::Timestamptz => make_ree(DataType::Int64),
1429 PrimitiveType::TimestampNs => make_ree(DataType::Int64),
1430 PrimitiveType::TimestamptzNs => make_ree(DataType::Int64),
1431 PrimitiveType::String => make_ree(DataType::Utf8),
1432 PrimitiveType::Uuid => make_ree(DataType::Binary),
1433 PrimitiveType::Fixed(_) => make_ree(DataType::Binary),
1434 PrimitiveType::Binary => make_ree(DataType::Binary),
1435 PrimitiveType::Geometry(_) | PrimitiveType::Geography(_) => make_ree(DataType::LargeBinary),
1438 PrimitiveType::Decimal { precision, scale } => {
1439 make_ree(DataType::Decimal128(*precision as u8, *scale as i8))
1440 }
1441 }
1442}
1443
1444struct MetadataStripVisitor {
1450 pending_fields: Vec<PendingField>,
1452}
1453
1454struct PendingField {
1456 name: String,
1457 nullable: bool,
1458}
1459
1460impl MetadataStripVisitor {
1461 fn new() -> Self {
1462 Self {
1463 pending_fields: Vec::new(),
1464 }
1465 }
1466
1467 fn start_field(&mut self, name: &str, nullable: bool) -> Result<()> {
1469 self.pending_fields.push(PendingField {
1470 name: name.to_string(),
1471 nullable,
1472 });
1473 Ok(())
1474 }
1475
1476 fn finish_field(&mut self, data_type: DataType, kind: &str) -> Result<Field> {
1479 let PendingField { name, nullable } = self.pending_fields.pop().ok_or_else(|| {
1480 Error::new(
1481 ErrorKind::Unexpected,
1482 format!("Field stack underflow in {kind}"),
1483 )
1484 })?;
1485 Ok(Field::new(name, data_type, nullable))
1486 }
1487}
1488
1489impl ArrowSchemaVisitor for MetadataStripVisitor {
1490 type T = Field;
1491 type U = ArrowSchema;
1492
1493 fn before_field(&mut self, field: &FieldRef) -> Result<()> {
1494 self.start_field(field.name(), field.is_nullable())
1495 }
1496
1497 fn before_list_element(&mut self, field: &FieldRef) -> Result<()> {
1498 self.start_field(LIST_FIELD_NAME, field.is_nullable())
1499 }
1500
1501 fn before_map_key(&mut self, _field: &FieldRef) -> Result<()> {
1502 self.start_field(MAP_KEY_FIELD_NAME, false)
1504 }
1505
1506 fn before_map_value(&mut self, field: &FieldRef) -> Result<()> {
1507 self.start_field(MAP_VALUE_FIELD_NAME, field.is_nullable())
1508 }
1509
1510 fn schema(&mut self, _schema: &ArrowSchema, values: Vec<Self::T>) -> Result<Self::U> {
1511 if !self.pending_fields.is_empty() {
1512 return Err(Error::new(
1513 ErrorKind::Unexpected,
1514 format!(
1515 "{} fields left on the field stack after traversal",
1516 self.pending_fields.len()
1517 ),
1518 ));
1519 }
1520 Ok(ArrowSchema::new(values))
1521 }
1522
1523 fn r#struct(&mut self, _fields: &Fields, results: Vec<Self::T>) -> Result<Self::T> {
1524 self.finish_field(DataType::Struct(Fields::from(results)), "struct")
1525 }
1526
1527 fn list(&mut self, list: &DataType, value: Self::T) -> Result<Self::T> {
1528 let list_type = match list {
1529 DataType::List(_) => DataType::List(Arc::new(value)),
1530 DataType::LargeList(_) => DataType::LargeList(Arc::new(value)),
1531 DataType::FixedSizeList(_, size) => DataType::FixedSizeList(Arc::new(value), *size),
1532 _ => {
1533 return Err(Error::new(
1534 ErrorKind::Unexpected,
1535 format!("Expected list type, got {list}"),
1536 ));
1537 }
1538 };
1539
1540 self.finish_field(list_type, "list")
1541 }
1542
1543 fn map(&mut self, map: &DataType, key_value: Self::T, value: Self::T) -> Result<Self::T> {
1544 let struct_field = Field::new(
1546 DEFAULT_MAP_FIELD_NAME,
1547 DataType::Struct(Fields::from(vec![key_value, value])),
1548 false,
1549 );
1550
1551 let sorted = match map {
1553 DataType::Map(_, sorted) => *sorted,
1554 _ => {
1555 return Err(Error::new(
1556 ErrorKind::Unexpected,
1557 format!("Expected map type, got {map}"),
1558 ));
1559 }
1560 };
1561
1562 self.finish_field(DataType::Map(Arc::new(struct_field), sorted), "map")
1563 }
1564
1565 fn primitive(&mut self, p: &DataType) -> Result<Self::T> {
1566 self.finish_field(p.clone(), "primitive")
1567 }
1568}
1569
1570pub fn strip_metadata_from_schema(schema: &ArrowSchema) -> Result<ArrowSchema> {
1615 let mut visitor = MetadataStripVisitor::new();
1616 visit_schema(schema, &mut visitor)
1617}
1618
1619#[cfg(test)]
1620mod tests {
1621 use std::collections::HashMap;
1622 use std::sync::Arc;
1623
1624 use arrow_schema::extension::ExtensionType;
1625 use arrow_schema::{DataType, Field, Schema as ArrowSchema, TimeUnit};
1626 use parquet_geospatial::WkbEdges;
1627
1628 use super::*;
1629 use crate::spec::decimal_utils::decimal_new;
1630 use crate::spec::geospatial::{
1631 DEFAULT_GEOSPATIAL_CRS, EdgeInterpolationAlgorithm as IcebergEdgeInterpolationAlgorithm,
1632 };
1633 use crate::spec::{Literal, Schema};
1634
1635 fn simple_field(name: &str, ty: DataType, nullable: bool, value: &str) -> Field {
1637 Field::new(name, ty, nullable).with_metadata(HashMap::from([(
1638 PARQUET_FIELD_ID_META_KEY.to_string(),
1639 value.to_string(),
1640 )]))
1641 }
1642
1643 fn arrow_schema_for_arrow_schema_to_schema_test() -> ArrowSchema {
1644 let fields = Fields::from(vec![
1645 simple_field("key", DataType::Int32, false, "28"),
1646 simple_field("value", DataType::Utf8, true, "29"),
1647 ]);
1648
1649 let r#struct = DataType::Struct(fields);
1650 let map = DataType::Map(
1651 Arc::new(simple_field(DEFAULT_MAP_FIELD_NAME, r#struct, false, "17")),
1652 false,
1653 );
1654 let dictionary = DataType::Dictionary(Box::new(DataType::Int32), Box::new(DataType::Utf8));
1655
1656 let fields = Fields::from(vec![
1657 simple_field("aa", DataType::Int32, false, "18"),
1658 simple_field("bb", DataType::Utf8, true, "19"),
1659 simple_field(
1660 "cc",
1661 DataType::Timestamp(TimeUnit::Microsecond, None),
1662 false,
1663 "20",
1664 ),
1665 ]);
1666
1667 let r#struct = DataType::Struct(fields);
1668
1669 ArrowSchema::new(vec![
1670 simple_field("a", DataType::Int32, false, "2"),
1671 simple_field("b", DataType::Int64, false, "1"),
1672 simple_field("c", DataType::Utf8, false, "3"),
1673 simple_field("n", DataType::Utf8, false, "21"),
1674 simple_field(
1675 "d",
1676 DataType::Timestamp(TimeUnit::Microsecond, None),
1677 true,
1678 "4",
1679 ),
1680 simple_field("e", DataType::Boolean, true, "6"),
1681 simple_field("f", DataType::Float32, false, "5"),
1682 simple_field("g", DataType::Float64, false, "7"),
1683 simple_field("p", DataType::Decimal128(10, 2), false, "27"),
1684 simple_field("h", DataType::Date32, false, "8"),
1685 simple_field("i", DataType::Time64(TimeUnit::Microsecond), false, "9"),
1686 simple_field(
1687 "j",
1688 DataType::Timestamp(TimeUnit::Microsecond, Some("UTC".into())),
1689 false,
1690 "10",
1691 ),
1692 simple_field(
1693 "k",
1694 DataType::Timestamp(TimeUnit::Microsecond, Some("+00:00".into())),
1695 false,
1696 "12",
1697 ),
1698 simple_field("l", DataType::Binary, false, "13"),
1699 simple_field("o", DataType::LargeBinary, false, "22"),
1700 simple_field("m", DataType::FixedSizeBinary(10), false, "11"),
1701 simple_field(
1702 "list",
1703 DataType::List(Arc::new(simple_field(
1704 "element",
1705 DataType::Int32,
1706 false,
1707 "15",
1708 ))),
1709 true,
1710 "14",
1711 ),
1712 simple_field(
1713 "large_list",
1714 DataType::LargeList(Arc::new(simple_field(
1715 "element",
1716 DataType::Utf8,
1717 false,
1718 "23",
1719 ))),
1720 true,
1721 "24",
1722 ),
1723 simple_field(
1724 "fixed_list",
1725 DataType::FixedSizeList(
1726 Arc::new(simple_field("element", DataType::Binary, false, "26")),
1727 10,
1728 ),
1729 true,
1730 "25",
1731 ),
1732 simple_field("map", map, false, "16"),
1733 simple_field("struct", r#struct, false, "17"),
1734 simple_field("dictionary", dictionary, false, "30"),
1735 ])
1736 }
1737
1738 fn iceberg_schema_for_arrow_schema_to_schema_test() -> Schema {
1739 let schema_json = r#"{
1740 "type":"struct",
1741 "schema-id":0,
1742 "fields":[
1743 {
1744 "id":2,
1745 "name":"a",
1746 "required":true,
1747 "type":"int"
1748 },
1749 {
1750 "id":1,
1751 "name":"b",
1752 "required":true,
1753 "type":"long"
1754 },
1755 {
1756 "id":3,
1757 "name":"c",
1758 "required":true,
1759 "type":"string"
1760 },
1761 {
1762 "id":21,
1763 "name":"n",
1764 "required":true,
1765 "type":"string"
1766 },
1767 {
1768 "id":4,
1769 "name":"d",
1770 "required":false,
1771 "type":"timestamp"
1772 },
1773 {
1774 "id":6,
1775 "name":"e",
1776 "required":false,
1777 "type":"boolean"
1778 },
1779 {
1780 "id":5,
1781 "name":"f",
1782 "required":true,
1783 "type":"float"
1784 },
1785 {
1786 "id":7,
1787 "name":"g",
1788 "required":true,
1789 "type":"double"
1790 },
1791 {
1792 "id":27,
1793 "name":"p",
1794 "required":true,
1795 "type":"decimal(10,2)"
1796 },
1797 {
1798 "id":8,
1799 "name":"h",
1800 "required":true,
1801 "type":"date"
1802 },
1803 {
1804 "id":9,
1805 "name":"i",
1806 "required":true,
1807 "type":"time"
1808 },
1809 {
1810 "id":10,
1811 "name":"j",
1812 "required":true,
1813 "type":"timestamptz"
1814 },
1815 {
1816 "id":12,
1817 "name":"k",
1818 "required":true,
1819 "type":"timestamptz"
1820 },
1821 {
1822 "id":13,
1823 "name":"l",
1824 "required":true,
1825 "type":"binary"
1826 },
1827 {
1828 "id":22,
1829 "name":"o",
1830 "required":true,
1831 "type":"binary"
1832 },
1833 {
1834 "id":11,
1835 "name":"m",
1836 "required":true,
1837 "type":"fixed[10]"
1838 },
1839 {
1840 "id":14,
1841 "name":"list",
1842 "required": false,
1843 "type": {
1844 "type": "list",
1845 "element-id": 15,
1846 "element-required": true,
1847 "element": "int"
1848 }
1849 },
1850 {
1851 "id":24,
1852 "name":"large_list",
1853 "required": false,
1854 "type": {
1855 "type": "list",
1856 "element-id": 23,
1857 "element-required": true,
1858 "element": "string"
1859 }
1860 },
1861 {
1862 "id":25,
1863 "name":"fixed_list",
1864 "required": false,
1865 "type": {
1866 "type": "list",
1867 "element-id": 26,
1868 "element-required": true,
1869 "element": "binary"
1870 }
1871 },
1872 {
1873 "id":16,
1874 "name":"map",
1875 "required": true,
1876 "type": {
1877 "type": "map",
1878 "key-id": 28,
1879 "key": "int",
1880 "value-id": 29,
1881 "value-required": false,
1882 "value": "string"
1883 }
1884 },
1885 {
1886 "id":17,
1887 "name":"struct",
1888 "required": true,
1889 "type": {
1890 "type": "struct",
1891 "fields": [
1892 {
1893 "id":18,
1894 "name":"aa",
1895 "required":true,
1896 "type":"int"
1897 },
1898 {
1899 "id":19,
1900 "name":"bb",
1901 "required":false,
1902 "type":"string"
1903 },
1904 {
1905 "id":20,
1906 "name":"cc",
1907 "required":true,
1908 "type":"timestamp"
1909 }
1910 ]
1911 }
1912 },
1913 {
1914 "id":30,
1915 "name":"dictionary",
1916 "required":true,
1917 "type":"string"
1918 }
1919 ],
1920 "identifier-field-ids":[]
1921 }"#;
1922
1923 let schema: Schema = serde_json::from_str(schema_json).unwrap();
1924 schema
1925 }
1926
1927 #[test]
1928 fn test_arrow_schema_to_schema() {
1929 let arrow_schema = arrow_schema_for_arrow_schema_to_schema_test();
1930 let schema = iceberg_schema_for_arrow_schema_to_schema_test();
1931 let converted_schema = arrow_schema_to_schema(&arrow_schema).unwrap();
1932 pretty_assertions::assert_eq!(converted_schema, schema);
1933 }
1934
1935 #[test]
1936 fn test_converting_uuid_from_arrow_to_iceberg_to_arrow_should_give_uuid() {
1937 let arrow_schema = ArrowSchema::new(vec![
1938 simple_field("uuid_field", DataType::FixedSizeBinary(16), false, "1")
1939 .with_extension_type(UuidExtensionType),
1940 ]);
1941 let output =
1942 schema_to_arrow_schema(&arrow_schema_to_schema(&arrow_schema).unwrap()).unwrap();
1943
1944 assert_eq!(arrow_schema, output);
1945 }
1946
1947 #[test]
1948 fn test_converting_uuid_from_iceberg_to_arrow_to_iceberg_should_give_uuid() {
1949 let iceberg_schema = Schema::builder()
1950 .with_fields(vec![
1951 NestedField::optional(1, "uuid_field", Type::Primitive(PrimitiveType::Uuid)).into(),
1952 ])
1953 .build()
1954 .unwrap();
1955 let output =
1956 arrow_schema_to_schema(&schema_to_arrow_schema(&iceberg_schema).unwrap()).unwrap();
1957
1958 assert_eq!(iceberg_schema, output);
1959 }
1960
1961 #[test]
1962 fn test_arrow_schema_to_schema_should_convert_uuid_when_fixed_size_binary() {
1963 let converted_schema = arrow_schema_to_schema(&ArrowSchema::new(vec![
1964 simple_field("uuid_field", DataType::FixedSizeBinary(16), false, "1")
1965 .with_extension_type(UuidExtensionType),
1966 ]))
1967 .unwrap();
1968
1969 let expected = Schema::builder()
1970 .with_fields([NestedField::required(
1971 1,
1972 "uuid_field",
1973 Type::Primitive(PrimitiveType::Uuid),
1974 )
1975 .into()])
1976 .build()
1977 .unwrap();
1978
1979 pretty_assertions::assert_eq!(expected, converted_schema);
1980 }
1981
1982 #[test]
1983 fn test_arrow_schema_to_schema_should_reject_uuid_when_not_a_fixed_size_binary() {
1984 let field = simple_field_with_uuid_ext(DataType::Utf8);
1987
1988 let error = arrow_schema_to_schema(&ArrowSchema::new(vec![field])).unwrap_err();
1989
1990 pretty_assertions::assert_eq!(
1991 "arrow.uuid extension requires FixedSizeBinary(16) storage, found Utf8",
1992 error.message()
1993 );
1994 }
1995
1996 #[test]
1997 fn test_arrow_schema_to_schema_should_reject_uuid_when_fixed_size_binary_that_is_less_than_16()
1998 {
1999 let field = simple_field_with_uuid_ext(DataType::FixedSizeBinary(8));
2000 let error = arrow_schema_to_schema(&ArrowSchema::new(vec![field])).unwrap_err();
2001
2002 pretty_assertions::assert_eq!(
2003 "arrow.uuid extension requires FixedSizeBinary(16) storage, found FixedSizeBinary(8)",
2004 error.message()
2005 );
2006 }
2007
2008 #[test]
2009 fn test_arrow_schema_to_schema_should_reject_uuid_when_fixed_size_binary_that_is_more_than_16()
2010 {
2011 let field = simple_field_with_uuid_ext(DataType::FixedSizeBinary(17));
2012 let error = arrow_schema_to_schema(&ArrowSchema::new(vec![field])).unwrap_err();
2013
2014 pretty_assertions::assert_eq!(
2015 "arrow.uuid extension requires FixedSizeBinary(16) storage, found FixedSizeBinary(17)",
2016 error.message()
2017 );
2018 }
2019
2020 fn simple_field_with_uuid_ext(field_data_type: DataType) -> Field {
2021 let mut field = simple_field("uuid_field", DataType::FixedSizeBinary(16), false, "1")
2022 .with_extension_type(UuidExtensionType);
2023
2024 field = field.with_data_type(field_data_type);
2030
2031 field
2032 }
2033
2034 fn arrow_schema_for_schema_to_arrow_schema_test() -> ArrowSchema {
2035 let fields = Fields::from(vec![
2036 simple_field("key", DataType::Int32, false, "28"),
2037 simple_field("value", DataType::Utf8, true, "29"),
2038 ]);
2039
2040 let r#struct = DataType::Struct(fields);
2041 let map = DataType::Map(
2042 Arc::new(Field::new(DEFAULT_MAP_FIELD_NAME, r#struct, false)),
2043 false,
2044 );
2045
2046 let fields = Fields::from(vec![
2047 simple_field("aa", DataType::Int32, false, "18"),
2048 simple_field("bb", DataType::Utf8, true, "19"),
2049 simple_field(
2050 "cc",
2051 DataType::Timestamp(TimeUnit::Microsecond, None),
2052 false,
2053 "20",
2054 ),
2055 ]);
2056
2057 let r#struct = DataType::Struct(fields);
2058
2059 ArrowSchema::new(vec![
2060 simple_field("a", DataType::Int32, false, "2"),
2061 simple_field("b", DataType::Int64, false, "1"),
2062 simple_field("c", DataType::Utf8, false, "3"),
2063 simple_field("n", DataType::Utf8, false, "21"),
2064 simple_field(
2065 "d",
2066 DataType::Timestamp(TimeUnit::Microsecond, None),
2067 true,
2068 "4",
2069 ),
2070 simple_field("e", DataType::Boolean, true, "6"),
2071 simple_field("f", DataType::Float32, false, "5"),
2072 simple_field("g", DataType::Float64, false, "7"),
2073 simple_field("p", DataType::Decimal128(10, 2), false, "27"),
2074 simple_field("h", DataType::Date32, false, "8"),
2075 simple_field("i", DataType::Time64(TimeUnit::Microsecond), false, "9"),
2076 simple_field(
2077 "j",
2078 DataType::Timestamp(TimeUnit::Microsecond, Some("+00:00".into())),
2079 false,
2080 "10",
2081 ),
2082 simple_field(
2083 "k",
2084 DataType::Timestamp(TimeUnit::Microsecond, Some("+00:00".into())),
2085 false,
2086 "12",
2087 ),
2088 simple_field("l", DataType::Binary, false, "13"),
2089 simple_field("o", DataType::Binary, false, "22"),
2090 simple_field("m", DataType::FixedSizeBinary(10), false, "11"),
2091 simple_field(
2092 "list",
2093 DataType::List(Arc::new(simple_field(
2094 "element",
2095 DataType::Int32,
2096 false,
2097 "15",
2098 ))),
2099 true,
2100 "14",
2101 ),
2102 simple_field(
2103 "large_list",
2104 DataType::List(Arc::new(simple_field(
2105 "element",
2106 DataType::Utf8,
2107 false,
2108 "23",
2109 ))),
2110 true,
2111 "24",
2112 ),
2113 simple_field(
2114 "fixed_list",
2115 DataType::List(Arc::new(simple_field(
2116 "element",
2117 DataType::Binary,
2118 false,
2119 "26",
2120 ))),
2121 true,
2122 "25",
2123 ),
2124 simple_field("map", map, false, "16"),
2125 simple_field("struct", r#struct, false, "17"),
2126 simple_field("uuid", DataType::FixedSizeBinary(16), false, "30")
2127 .with_extension_type(UuidExtensionType),
2128 Field::new(
2129 "v",
2130 DataType::Struct(Fields::from(vec![
2131 Field::new("metadata", DataType::Binary, false),
2132 Field::new("value", DataType::Binary, true),
2133 ])),
2134 true,
2135 )
2136 .with_metadata(HashMap::from([
2137 (PARQUET_FIELD_ID_META_KEY.to_string(), "31".to_string()),
2138 (
2139 arrow_schema::extension::EXTENSION_TYPE_NAME_KEY.to_string(),
2140 "arrow.parquet.variant".to_string(),
2141 ),
2142 ])),
2143 ])
2144 }
2145
2146 fn iceberg_schema_for_schema_to_arrow_schema() -> Schema {
2147 let schema_json = r#"{
2148 "type":"struct",
2149 "schema-id":0,
2150 "fields":[
2151 {
2152 "id":2,
2153 "name":"a",
2154 "required":true,
2155 "type":"int"
2156 },
2157 {
2158 "id":1,
2159 "name":"b",
2160 "required":true,
2161 "type":"long"
2162 },
2163 {
2164 "id":3,
2165 "name":"c",
2166 "required":true,
2167 "type":"string"
2168 },
2169 {
2170 "id":21,
2171 "name":"n",
2172 "required":true,
2173 "type":"string"
2174 },
2175 {
2176 "id":4,
2177 "name":"d",
2178 "required":false,
2179 "type":"timestamp"
2180 },
2181 {
2182 "id":6,
2183 "name":"e",
2184 "required":false,
2185 "type":"boolean"
2186 },
2187 {
2188 "id":5,
2189 "name":"f",
2190 "required":true,
2191 "type":"float"
2192 },
2193 {
2194 "id":7,
2195 "name":"g",
2196 "required":true,
2197 "type":"double"
2198 },
2199 {
2200 "id":27,
2201 "name":"p",
2202 "required":true,
2203 "type":"decimal(10,2)"
2204 },
2205 {
2206 "id":8,
2207 "name":"h",
2208 "required":true,
2209 "type":"date"
2210 },
2211 {
2212 "id":9,
2213 "name":"i",
2214 "required":true,
2215 "type":"time"
2216 },
2217 {
2218 "id":10,
2219 "name":"j",
2220 "required":true,
2221 "type":"timestamptz"
2222 },
2223 {
2224 "id":12,
2225 "name":"k",
2226 "required":true,
2227 "type":"timestamptz"
2228 },
2229 {
2230 "id":13,
2231 "name":"l",
2232 "required":true,
2233 "type":"binary"
2234 },
2235 {
2236 "id":22,
2237 "name":"o",
2238 "required":true,
2239 "type":"binary"
2240 },
2241 {
2242 "id":11,
2243 "name":"m",
2244 "required":true,
2245 "type":"fixed[10]"
2246 },
2247 {
2248 "id":14,
2249 "name":"list",
2250 "required": false,
2251 "type": {
2252 "type": "list",
2253 "element-id": 15,
2254 "element-required": true,
2255 "element": "int"
2256 }
2257 },
2258 {
2259 "id":24,
2260 "name":"large_list",
2261 "required": false,
2262 "type": {
2263 "type": "list",
2264 "element-id": 23,
2265 "element-required": true,
2266 "element": "string"
2267 }
2268 },
2269 {
2270 "id":25,
2271 "name":"fixed_list",
2272 "required": false,
2273 "type": {
2274 "type": "list",
2275 "element-id": 26,
2276 "element-required": true,
2277 "element": "binary"
2278 }
2279 },
2280 {
2281 "id":16,
2282 "name":"map",
2283 "required": true,
2284 "type": {
2285 "type": "map",
2286 "key-id": 28,
2287 "key": "int",
2288 "value-id": 29,
2289 "value-required": false,
2290 "value": "string"
2291 }
2292 },
2293 {
2294 "id":17,
2295 "name":"struct",
2296 "required": true,
2297 "type": {
2298 "type": "struct",
2299 "fields": [
2300 {
2301 "id":18,
2302 "name":"aa",
2303 "required":true,
2304 "type":"int"
2305 },
2306 {
2307 "id":19,
2308 "name":"bb",
2309 "required":false,
2310 "type":"string"
2311 },
2312 {
2313 "id":20,
2314 "name":"cc",
2315 "required":true,
2316 "type":"timestamp"
2317 }
2318 ]
2319 }
2320 },
2321 {
2322 "id":30,
2323 "name":"uuid",
2324 "required":true,
2325 "type":"uuid"
2326 },
2327 {
2328 "id":31,
2329 "name":"v",
2330 "required":false,
2331 "type":"variant"
2332 }
2333 ],
2334 "identifier-field-ids":[]
2335 }"#;
2336
2337 let schema: Schema = serde_json::from_str(schema_json).unwrap();
2338 schema
2339 }
2340
2341 #[test]
2342 fn test_schema_to_arrow_schema() {
2343 let expected_arrow_schema = arrow_schema_for_schema_to_arrow_schema_test();
2344 let schema = iceberg_schema_for_schema_to_arrow_schema();
2345 let converted_arrow_schema = schema_to_arrow_schema(&schema).unwrap();
2346 assert_eq!(converted_arrow_schema, expected_arrow_schema);
2347 }
2348
2349 #[test]
2350 fn test_variant_type_to_arrow_type() {
2351 let arrow_type = type_to_arrow_type(&Type::Variant(VariantType)).unwrap();
2354 assert_eq!(
2355 arrow_type,
2356 DataType::Struct(Fields::from(vec![
2357 Field::new("metadata", DataType::Binary, false),
2358 Field::new("value", DataType::Binary, true),
2359 ]))
2360 );
2361 }
2362
2363 #[test]
2364 fn test_variant_field_carries_arrow_extension_type() {
2365 let schema = Schema::builder()
2368 .with_fields(vec![
2369 NestedField::optional(1, "v", Type::Variant(VariantType)).into(),
2370 ])
2371 .build()
2372 .unwrap();
2373
2374 let arrow_schema = schema_to_arrow_schema(&schema).unwrap();
2375 let field = arrow_schema.field_with_name("v").unwrap();
2376
2377 assert_eq!(field.extension_type_name(), Some("arrow.parquet.variant"));
2378 assert_eq!(
2380 field.metadata().get(PARQUET_FIELD_ID_META_KEY),
2381 Some(&"1".to_string())
2382 );
2383 assert_eq!(
2384 field.data_type(),
2385 &DataType::Struct(Fields::from(vec![
2386 Field::new("metadata", DataType::Binary, false),
2387 Field::new("value", DataType::Binary, true),
2388 ]))
2389 );
2390 }
2391
2392 #[test]
2393 fn test_schema_to_arrow_schema_should_convert_uuid_to_arrow_uuid() {
2394 let schema = Schema::builder()
2395 .with_fields(vec![
2396 NestedField::optional(1, "uuid_field", Type::Primitive(PrimitiveType::Uuid)).into(),
2397 ])
2398 .build()
2399 .unwrap();
2400
2401 let arrow_schema = schema_to_arrow_schema(&schema).unwrap();
2402 let field = arrow_schema.field_with_name("uuid_field").unwrap();
2403
2404 assert_eq!(field.extension_type_name(), Some(UuidExtensionType::NAME));
2405 assert_eq!(
2407 field.metadata().get(PARQUET_FIELD_ID_META_KEY),
2408 Some(&"1".to_string())
2409 );
2410 assert_eq!(*field.data_type(), DataType::FixedSizeBinary(16));
2411 }
2412
2413 #[test]
2414 fn test_variant_nested_in_list_and_map_carries_arrow_extension_type() {
2415 let schema = Schema::builder()
2419 .with_fields(vec![
2420 NestedField::optional(
2421 1,
2422 "l",
2423 Type::List(ListType::new(
2424 NestedField::optional(2, "element", Type::Variant(VariantType)).into(),
2425 )),
2426 )
2427 .into(),
2428 NestedField::optional(
2429 3,
2430 "m",
2431 Type::Map(MapType::new(
2432 NestedField::map_key_element(4, Type::Primitive(PrimitiveType::String))
2433 .into(),
2434 NestedField::map_value_element(5, Type::Variant(VariantType), false).into(),
2435 )),
2436 )
2437 .into(),
2438 ])
2439 .build()
2440 .unwrap();
2441
2442 let arrow_schema = schema_to_arrow_schema(&schema).unwrap();
2443
2444 let DataType::List(element) = arrow_schema.field_with_name("l").unwrap().data_type() else {
2445 panic!("expected a list");
2446 };
2447 assert_eq!(element.extension_type_name(), Some("arrow.parquet.variant"));
2448
2449 let DataType::Map(entries, _) = arrow_schema.field_with_name("m").unwrap().data_type()
2450 else {
2451 panic!("expected a map");
2452 };
2453 let DataType::Struct(kv) = entries.data_type() else {
2454 panic!("expected a key_value struct");
2455 };
2456 let value = kv.iter().find(|f| f.name() == "value").unwrap();
2457 assert_eq!(value.extension_type_name(), Some("arrow.parquet.variant"));
2458 }
2459
2460 fn variant_storage() -> DataType {
2463 DataType::Struct(Fields::from(vec![
2464 Field::new("metadata", DataType::Binary, false),
2465 Field::new("value", DataType::Binary, true),
2466 ]))
2467 }
2468
2469 #[test]
2470 fn test_variant_arrow_field_folds_to_iceberg_variant() {
2471 let field = simple_field("v", variant_storage(), true, "1")
2475 .with_extension_type(VariantExtensionType);
2476 let arrow_schema = ArrowSchema::new(vec![field]);
2477
2478 let converted = arrow_schema_to_schema(&arrow_schema).unwrap();
2479 let expected = Schema::builder()
2480 .with_fields(vec![
2481 NestedField::optional(1, "v", Type::Variant(VariantType)).into(),
2482 ])
2483 .build()
2484 .unwrap();
2485 pretty_assertions::assert_eq!(converted, expected);
2486 }
2487
2488 #[test]
2489 fn test_variant_schema_round_trips() {
2490 let schema = Schema::builder()
2493 .with_fields(vec![
2494 NestedField::optional(1, "v", Type::Variant(VariantType)).into(),
2495 NestedField::optional(
2496 2,
2497 "s",
2498 Type::Struct(StructType::new(vec![
2499 NestedField::optional(3, "sv", Type::Variant(VariantType)).into(),
2500 ])),
2501 )
2502 .into(),
2503 NestedField::optional(
2504 4,
2505 "l",
2506 Type::List(ListType::new(
2507 NestedField::optional(5, "element", Type::Variant(VariantType)).into(),
2508 )),
2509 )
2510 .into(),
2511 NestedField::optional(
2512 6,
2513 "m",
2514 Type::Map(MapType::new(
2515 NestedField::map_key_element(7, Type::Primitive(PrimitiveType::String))
2516 .into(),
2517 NestedField::map_value_element(8, Type::Variant(VariantType), false).into(),
2518 )),
2519 )
2520 .into(),
2521 ])
2522 .build()
2523 .unwrap();
2524
2525 let arrow_schema = schema_to_arrow_schema(&schema).unwrap();
2526 let round_tripped = arrow_schema_to_schema(&arrow_schema).unwrap();
2527 pretty_assertions::assert_eq!(round_tripped, schema);
2528 }
2529
2530 #[test]
2531 fn test_variant_recognized_with_auto_assigned_ids() {
2532 let field =
2535 Field::new("v", variant_storage(), true).with_extension_type(VariantExtensionType);
2536 let arrow_schema = ArrowSchema::new(vec![field]);
2537
2538 let converted = arrow_schema_to_schema_auto_assign_ids(&arrow_schema).unwrap();
2539 let expected = Schema::builder()
2540 .with_fields(vec![
2541 NestedField::optional(1, "v", Type::Variant(VariantType)).into(),
2542 ])
2543 .build()
2544 .unwrap();
2545 pretty_assertions::assert_eq!(converted, expected);
2546 }
2547
2548 #[test]
2549 fn test_variant_extension_on_non_struct_storage_is_rejected() {
2550 let field = Field::new("v", DataType::Int32, true).with_metadata(HashMap::from([
2553 (PARQUET_FIELD_ID_META_KEY.to_string(), "1".to_string()),
2554 (
2555 arrow_schema::extension::EXTENSION_TYPE_NAME_KEY.to_string(),
2556 VariantExtensionType::NAME.to_string(),
2557 ),
2558 ]));
2559 let arrow_schema = ArrowSchema::new(vec![field]);
2560
2561 let err = arrow_schema_to_schema(&arrow_schema).unwrap_err();
2562 assert!(
2563 err.to_string().contains("requires Struct storage"),
2564 "unexpected error: {err}"
2565 );
2566 }
2567
2568 #[test]
2569 fn test_geospatial_arrow_schema_roundtrip() {
2570 let schema = Schema::builder()
2571 .with_schema_id(1)
2572 .with_fields(vec![
2573 NestedField::required(
2574 1,
2575 "geom",
2576 Type::Primitive(PrimitiveType::Geometry(
2577 GeometryType::new(Some("EPSG:3857".to_string())).unwrap(),
2578 )),
2579 )
2580 .into(),
2581 NestedField::optional(
2582 2,
2583 "geog",
2584 Type::Primitive(PrimitiveType::Geography(
2585 GeographyType::new(
2586 Some("OGC:CRS27".to_string()),
2587 IcebergEdgeInterpolationAlgorithm::Karney,
2588 )
2589 .unwrap(),
2590 )),
2591 )
2592 .into(),
2593 NestedField::optional(
2594 3,
2595 "geom_list",
2596 Type::List(ListType::new(
2597 NestedField::list_element(
2598 4,
2599 Type::Primitive(PrimitiveType::Geometry(GeometryType::default())),
2600 true,
2601 )
2602 .into(),
2603 )),
2604 )
2605 .into(),
2606 ])
2607 .build()
2608 .unwrap();
2609
2610 let arrow_schema = schema_to_arrow_schema(&schema).unwrap();
2611 let geom = arrow_schema.field(0);
2612 assert_eq!(geom.data_type(), &DataType::LargeBinary);
2613
2614 let geog = arrow_schema.field(1);
2615 let geog_wkb = geog.try_extension_type::<WkbType>().unwrap();
2616 assert_eq!(geog_wkb.metadata().algorithm, Some(WkbEdges::Karney));
2617 let geog_metadata: serde_json::Value =
2618 serde_json::from_str(&geog_wkb.serialize_metadata().unwrap()).unwrap();
2619 assert_eq!(geog_metadata.get("edges").unwrap(), "karney");
2620 assert!(geog_metadata.get("algorithm").is_none());
2621
2622 let list = arrow_schema.field(2);
2623 let DataType::List(element) = list.data_type() else {
2624 panic!("Expected list field");
2625 };
2626 assert!(
2627 matches!(
2628 element
2629 .try_extension_type::<WkbType>()
2630 .unwrap()
2631 .metadata()
2632 .type_hint(),
2633 WkbTypeHint::Geometry
2634 ),
2635 "Expected list element to retain WKB extension metadata"
2636 );
2637 assert_eq!(
2638 element
2639 .try_extension_type::<WkbType>()
2640 .unwrap()
2641 .metadata()
2642 .crs
2643 .as_ref()
2644 .and_then(serde_json::Value::as_str),
2645 Some(DEFAULT_GEOSPATIAL_CRS)
2646 );
2647
2648 let converted = arrow_schema_to_schema(&arrow_schema).unwrap();
2649 assert_eq!(converted.as_struct().fields(), schema.as_struct().fields());
2650 }
2651
2652 #[test]
2653 fn test_geospatial_arrow_crs_import() {
2654 let mut geom = simple_field("geom", DataType::LargeBinary, true, "1");
2655 geom.try_with_extension_type(WkbType::new(Some(WkbMetadata::new(
2656 Some(r#"{"id":{"authority":"EPSG","code":3857}}"#),
2657 None,
2658 ))))
2659 .unwrap();
2660 let mut unset_geom = simple_field("unset_geom", DataType::LargeBinary, true, "2");
2661 unset_geom
2662 .try_with_extension_type(WkbType::new(Some(WkbMetadata::new(None, None))))
2663 .unwrap();
2664
2665 let arrow_schema = ArrowSchema::new(vec![geom, unset_geom]);
2666 let schema = arrow_schema_to_schema(&arrow_schema).unwrap();
2667
2668 assert_eq!(
2669 schema.field_by_id(1).unwrap().field_type.as_ref(),
2670 &Type::Primitive(PrimitiveType::Geometry(
2671 GeometryType::new(Some("EPSG:3857".to_string())).unwrap()
2672 ))
2673 );
2674 assert_eq!(
2675 schema.field_by_id(2).unwrap().field_type.as_ref(),
2676 &Type::Primitive(PrimitiveType::Geometry(
2677 GeometryType::new(Some(UNSET_GEOSPATIAL_CRS.to_string())).unwrap()
2678 ))
2679 );
2680 }
2681
2682 #[test]
2683 fn test_geospatial_arrow_unset_crs_export() {
2684 let schema = Schema::builder()
2685 .with_fields(vec![
2686 NestedField::optional(
2687 1,
2688 "geom",
2689 Type::Primitive(PrimitiveType::Geometry(
2690 GeometryType::new(Some(UNSET_GEOSPATIAL_CRS.to_string())).unwrap(),
2691 )),
2692 )
2693 .into(),
2694 NestedField::optional(
2695 2,
2696 "geog",
2697 Type::Primitive(PrimitiveType::Geography(
2698 GeographyType::new(
2699 Some(UNSET_GEOSPATIAL_CRS.to_string()),
2700 IcebergEdgeInterpolationAlgorithm::Spherical,
2701 )
2702 .unwrap(),
2703 )),
2704 )
2705 .into(),
2706 ])
2707 .build()
2708 .unwrap();
2709 let arrow_schema = schema_to_arrow_schema(&schema).unwrap();
2710 for field in arrow_schema.fields() {
2711 assert!(
2712 field
2713 .try_extension_type::<WkbType>()
2714 .unwrap()
2715 .metadata()
2716 .crs
2717 .is_none()
2718 );
2719 }
2720 assert_eq!(arrow_schema_to_schema(&arrow_schema).unwrap(), schema);
2721 }
2722
2723 #[test]
2724 fn test_geospatial_arrow_invalid_crs_is_rejected() {
2725 for crs in [r#"{}"#, "42"] {
2726 let mut field = simple_field("geom", DataType::LargeBinary, true, "1");
2727 field
2728 .try_with_extension_type(WkbType::new(Some(WkbMetadata::new(Some(crs), None))))
2729 .unwrap();
2730
2731 let err = arrow_schema_to_schema(&ArrowSchema::new(vec![field])).unwrap_err();
2732 assert_eq!(err.kind(), ErrorKind::DataInvalid, "{err}");
2733 }
2734 }
2735
2736 #[test]
2737 fn test_geospatial_arrow_extension_requires_binary_storage() {
2738 use arrow_schema::extension::{EXTENSION_TYPE_METADATA_KEY, EXTENSION_TYPE_NAME_KEY};
2739
2740 let metadata = WkbType::new(Some(WkbMetadata::new(None, None)))
2741 .serialize_metadata()
2742 .unwrap();
2743 let field =
2744 simple_field("geom", DataType::Int32, true, "1").with_metadata(HashMap::from([
2745 (PARQUET_FIELD_ID_META_KEY.to_string(), "1".to_string()),
2746 (
2747 EXTENSION_TYPE_NAME_KEY.to_string(),
2748 WkbType::NAME.to_string(),
2749 ),
2750 (EXTENSION_TYPE_METADATA_KEY.to_string(), metadata),
2751 ]));
2752
2753 let err = arrow_schema_to_schema(&ArrowSchema::new(vec![field])).unwrap_err();
2754 assert_eq!(err.kind(), ErrorKind::DataInvalid, "{err}");
2755 assert!(
2756 err.message().contains("requires binary storage"),
2757 "unexpected error: {err}"
2758 );
2759 }
2760
2761 #[test]
2762 fn test_geospatial_parquet_statistics_are_ignored() {
2763 use parquet::data_type::ByteArray;
2764 use parquet::file::statistics::ValueStatistics;
2765
2766 let stats = Statistics::ByteArray(ValueStatistics::new(
2767 Some(ByteArray::from(vec![1, 2, 3])),
2768 Some(ByteArray::from(vec![4, 5, 6])),
2769 None,
2770 None,
2771 false,
2772 ));
2773
2774 for primitive_type in [
2775 PrimitiveType::Geometry(GeometryType::default()),
2776 PrimitiveType::Geography(GeographyType::default()),
2777 ] {
2778 assert_eq!(
2779 get_parquet_stat_min_as_datum(&primitive_type, &stats).unwrap(),
2780 None
2781 );
2782 assert_eq!(
2783 get_parquet_stat_max_as_datum(&primitive_type, &stats).unwrap(),
2784 None
2785 );
2786 }
2787 }
2788
2789 #[test]
2790 fn test_type_conversion() {
2791 {
2793 let arrow_type = DataType::Int32;
2794 let iceberg_type = Type::Primitive(PrimitiveType::Int);
2795 assert_eq!(arrow_type, type_to_arrow_type(&iceberg_type).unwrap());
2796 assert_eq!(iceberg_type, arrow_type_to_type(&arrow_type).unwrap());
2797 }
2798
2799 {
2800 let arrow_type = DataType::Null;
2801 let iceberg_type = Type::Primitive(PrimitiveType::Unknown);
2802 assert_eq!(arrow_type, type_to_arrow_type(&iceberg_type).unwrap());
2803 assert_eq!(iceberg_type, arrow_type_to_type(&arrow_type).unwrap());
2804 }
2805
2806 {
2808 let arrow_type = DataType::Struct(Fields::from(vec![
2810 Field::new("a", DataType::Int64, false),
2811 Field::new("b", DataType::Utf8, true),
2812 ]));
2813 assert_eq!(
2814 &arrow_type_to_type(&arrow_type).unwrap_err().to_string(),
2815 "DataInvalid => Field id not found in metadata"
2816 );
2817
2818 let arrow_type = DataType::Struct(Fields::from(vec![
2819 Field::new("a", DataType::Int64, false).with_metadata(HashMap::from_iter([(
2820 PARQUET_FIELD_ID_META_KEY.to_string(),
2821 1.to_string(),
2822 )])),
2823 Field::new("b", DataType::Utf8, true).with_metadata(HashMap::from_iter([(
2824 PARQUET_FIELD_ID_META_KEY.to_string(),
2825 2.to_string(),
2826 )])),
2827 ]));
2828 let iceberg_type = Type::Struct(StructType::new(vec![
2829 NestedField {
2830 id: 1,
2831 doc: None,
2832 name: "a".to_string(),
2833 required: true,
2834 field_type: Box::new(Type::Primitive(PrimitiveType::Long)),
2835 initial_default: None,
2836 write_default: None,
2837 }
2838 .into(),
2839 NestedField {
2840 id: 2,
2841 doc: None,
2842 name: "b".to_string(),
2843 required: false,
2844 field_type: Box::new(Type::Primitive(PrimitiveType::String)),
2845 initial_default: None,
2846 write_default: None,
2847 }
2848 .into(),
2849 ]));
2850 assert_eq!(iceberg_type, arrow_type_to_type(&arrow_type).unwrap());
2851 assert_eq!(arrow_type, type_to_arrow_type(&iceberg_type).unwrap());
2852
2853 let iceberg_type = Type::Struct(StructType::new(vec![
2855 NestedField {
2856 id: 1,
2857 doc: None,
2858 name: "a".to_string(),
2859 required: true,
2860 field_type: Box::new(Type::Primitive(PrimitiveType::Long)),
2861 initial_default: Some(Literal::Primitive(PrimitiveLiteral::Int(114514))),
2862 write_default: None,
2863 }
2864 .into(),
2865 NestedField {
2866 id: 2,
2867 doc: None,
2868 name: "b".to_string(),
2869 required: false,
2870 field_type: Box::new(Type::Primitive(PrimitiveType::String)),
2871 initial_default: None,
2872 write_default: Some(Literal::Primitive(PrimitiveLiteral::String(
2873 "514".to_string(),
2874 ))),
2875 }
2876 .into(),
2877 ]));
2878 assert_eq!(arrow_type, type_to_arrow_type(&iceberg_type).unwrap());
2879 }
2880
2881 {
2883 let arrow_type =
2884 DataType::Dictionary(Box::new(DataType::Int32), Box::new(DataType::Int8));
2885 let iceberg_type = Type::Primitive(PrimitiveType::Int);
2886 assert_eq!(
2887 iceberg_type,
2888 arrow_type_to_type(&arrow_type).unwrap(),
2889 "Expected dictionary conversion to use the contained value"
2890 );
2891
2892 let arrow_type =
2893 DataType::Dictionary(Box::new(DataType::Utf8), Box::new(DataType::Boolean));
2894 let iceberg_type = Type::Primitive(PrimitiveType::Boolean);
2895 assert_eq!(iceberg_type, arrow_type_to_type(&arrow_type).unwrap());
2896 }
2897 }
2898
2899 #[test]
2900 fn test_unsigned_integer_type_conversion() {
2901 let test_cases = vec![
2902 (DataType::UInt8, PrimitiveType::Int),
2903 (DataType::UInt16, PrimitiveType::Int),
2904 (DataType::UInt32, PrimitiveType::Long),
2905 ];
2906
2907 for (arrow_type, expected_iceberg_type) in test_cases {
2908 let arrow_field = Field::new("test", arrow_type.clone(), false).with_metadata(
2909 HashMap::from([(PARQUET_FIELD_ID_META_KEY.to_string(), "1".to_string())]),
2910 );
2911 let arrow_schema = ArrowSchema::new(vec![arrow_field]);
2912
2913 let iceberg_schema = arrow_schema_to_schema(&arrow_schema).unwrap();
2914 let iceberg_field = iceberg_schema.as_struct().fields().first().unwrap();
2915
2916 assert!(
2917 matches!(iceberg_field.field_type.as_ref(), Type::Primitive(t) if *t == expected_iceberg_type),
2918 "Expected {arrow_type:?} to map to {expected_iceberg_type:?}"
2919 );
2920 }
2921
2922 {
2924 let arrow_field = Field::new("test", DataType::UInt64, false).with_metadata(
2925 HashMap::from([(PARQUET_FIELD_ID_META_KEY.to_string(), "1".to_string())]),
2926 );
2927 let arrow_schema = ArrowSchema::new(vec![arrow_field]);
2928
2929 let result = arrow_schema_to_schema(&arrow_schema);
2930 assert!(result.is_err());
2931 assert!(
2932 result
2933 .unwrap_err()
2934 .to_string()
2935 .contains("UInt64 is not supported")
2936 );
2937 }
2938 }
2939
2940 #[test]
2941 fn test_datum_conversion() {
2942 {
2943 let datum = Datum::bool(true);
2944 let arrow_datum = get_arrow_datum(&datum).unwrap();
2945 let (array, is_scalar) = arrow_datum.get();
2946 let array = array.as_any().downcast_ref::<BooleanArray>().unwrap();
2947 assert!(is_scalar);
2948 assert!(array.value(0));
2949 }
2950 {
2951 let datum = Datum::int(42);
2952 let arrow_datum = get_arrow_datum(&datum).unwrap();
2953 let (array, is_scalar) = arrow_datum.get();
2954 let array = array.as_any().downcast_ref::<Int32Array>().unwrap();
2955 assert!(is_scalar);
2956 assert_eq!(array.value(0), 42);
2957 }
2958 {
2959 let datum = Datum::long(42);
2960 let arrow_datum = get_arrow_datum(&datum).unwrap();
2961 let (array, is_scalar) = arrow_datum.get();
2962 let array = array.as_any().downcast_ref::<Int64Array>().unwrap();
2963 assert!(is_scalar);
2964 assert_eq!(array.value(0), 42);
2965 }
2966 {
2967 let datum = Datum::float(42.42_f32);
2968 let arrow_datum = get_arrow_datum(&datum).unwrap();
2969 let (array, is_scalar) = arrow_datum.get();
2970 let array = array.as_any().downcast_ref::<Float32Array>().unwrap();
2971 assert!(is_scalar);
2972 assert_eq!(array.value(0), 42.42);
2973 }
2974 {
2975 let datum = Datum::double(42.42);
2976 let arrow_datum = get_arrow_datum(&datum).unwrap();
2977 let (array, is_scalar) = arrow_datum.get();
2978 let array = array.as_any().downcast_ref::<Float64Array>().unwrap();
2979 assert!(is_scalar);
2980 assert_eq!(array.value(0), 42.42);
2981 }
2982 {
2983 let datum = Datum::string("abc");
2984 let arrow_datum = get_arrow_datum(&datum).unwrap();
2985 let (array, is_scalar) = arrow_datum.get();
2986 let array = array.as_any().downcast_ref::<StringArray>().unwrap();
2987 assert!(is_scalar);
2988 assert_eq!(array.value(0), "abc");
2989 }
2990 {
2991 let datum = Datum::binary(vec![1, 2, 3, 4]);
2992 let arrow_datum = get_arrow_datum(&datum).unwrap();
2993 let (array, is_scalar) = arrow_datum.get();
2994 let array = array.as_any().downcast_ref::<BinaryArray>().unwrap();
2995 assert!(is_scalar);
2996 assert_eq!(array.value(0), &[1, 2, 3, 4]);
2997 }
2998 {
2999 let datum = Datum::date(42);
3000 let arrow_datum = get_arrow_datum(&datum).unwrap();
3001 let (array, is_scalar) = arrow_datum.get();
3002 let array = array.as_any().downcast_ref::<Date32Array>().unwrap();
3003 assert!(is_scalar);
3004 assert_eq!(array.value(0), 42);
3005 }
3006 {
3007 let datum = Datum::timestamp_micros(42);
3008 let arrow_datum = get_arrow_datum(&datum).unwrap();
3009 let (array, is_scalar) = arrow_datum.get();
3010 let array = array
3011 .as_any()
3012 .downcast_ref::<TimestampMicrosecondArray>()
3013 .unwrap();
3014 assert!(is_scalar);
3015 assert_eq!(array.value(0), 42);
3016 }
3017 {
3018 let datum = Datum::timestamptz_micros(42);
3019 let arrow_datum = get_arrow_datum(&datum).unwrap();
3020 let (array, is_scalar) = arrow_datum.get();
3021 let array = array
3022 .as_any()
3023 .downcast_ref::<TimestampMicrosecondArray>()
3024 .unwrap();
3025 assert!(is_scalar);
3026 assert_eq!(array.timezone(), Some("+00:00"));
3027 assert_eq!(array.value(0), 42);
3028 }
3029 {
3030 let datum = Datum::decimal_with_precision(decimal_new(123, 2), 30).unwrap();
3031 let arrow_datum = get_arrow_datum(&datum).unwrap();
3032 let (array, is_scalar) = arrow_datum.get();
3033 let array = array.as_any().downcast_ref::<Decimal128Array>().unwrap();
3034 assert!(is_scalar);
3035 assert_eq!(array.precision(), 30);
3036 assert_eq!(array.scale(), 2);
3037 assert_eq!(array.value(0), 123);
3038 }
3039 {
3040 let datum = Datum::uuid_from_str("42424242-4242-4242-4242-424242424242").unwrap();
3041 let arrow_datum = get_arrow_datum(&datum).unwrap();
3042 let (array, is_scalar) = arrow_datum.get();
3043 let array = array
3044 .as_any()
3045 .downcast_ref::<FixedSizeBinaryArray>()
3046 .unwrap();
3047 assert!(is_scalar);
3048 assert_eq!(array.value(0), [66u8; 16]);
3049 }
3050 {
3051 let datum = Datum::fixed(vec![1u8, 2, 3, 4, 5, 6, 7, 8]);
3052 let arrow_datum = get_arrow_datum(&datum).unwrap();
3053 let (array, is_scalar) = arrow_datum.get();
3054 let array = array
3055 .as_any()
3056 .downcast_ref::<FixedSizeBinaryArray>()
3057 .unwrap();
3058 assert!(is_scalar);
3059 assert_eq!(array.value_length(), 8);
3060 assert_eq!(array.value(0), &[1u8, 2, 3, 4, 5, 6, 7, 8]);
3061 }
3062 }
3063
3064 #[test]
3065 fn test_arrow_schema_to_schema_with_field_id() {
3066 let arrow_schema = ArrowSchema::new(vec![
3069 Field::new("id", DataType::Int64, false),
3070 Field::new("name", DataType::Utf8, true),
3071 Field::new("price", DataType::Decimal128(10, 2), false),
3072 Field::new("uuid", DataType::FixedSizeBinary(16), false)
3073 .with_extension_type(UuidExtensionType),
3074 Field::new(
3075 "created_at",
3076 DataType::Timestamp(TimeUnit::Microsecond, Some("+00:00".into())),
3077 true,
3078 ),
3079 Field::new(
3080 "tags",
3081 DataType::List(Arc::new(Field::new("item", DataType::Utf8, true))),
3082 true,
3083 ),
3084 Field::new(
3085 "address",
3086 DataType::Struct(Fields::from(vec![
3087 Field::new("street", DataType::Utf8, true),
3088 Field::new("city", DataType::Utf8, false),
3089 Field::new("zip", DataType::Int32, true),
3090 ])),
3091 true,
3092 ),
3093 Field::new(
3094 "attributes",
3095 DataType::Map(
3096 Arc::new(Field::new(
3097 DEFAULT_MAP_FIELD_NAME,
3098 DataType::Struct(Fields::from(vec![
3099 Field::new("key", DataType::Utf8, false),
3100 Field::new("value", DataType::Utf8, true),
3101 ])),
3102 false,
3103 )),
3104 false,
3105 ),
3106 true,
3107 ),
3108 Field::new(
3109 "orders",
3110 DataType::List(Arc::new(Field::new(
3111 "element",
3112 DataType::Struct(Fields::from(vec![
3113 Field::new("order_id", DataType::Int64, false),
3114 Field::new("amount", DataType::Float64, false),
3115 ])),
3116 true,
3117 ))),
3118 true,
3119 ),
3120 ]);
3121
3122 let schema = arrow_schema_to_schema_auto_assign_ids(&arrow_schema).unwrap();
3123
3124 let expected = Schema::builder()
3129 .with_fields(vec![
3130 NestedField::required(1, "id", Type::Primitive(PrimitiveType::Long)).into(),
3131 NestedField::optional(2, "name", Type::Primitive(PrimitiveType::String)).into(),
3132 NestedField::required(
3133 3,
3134 "price",
3135 Type::Primitive(PrimitiveType::Decimal {
3136 precision: 10,
3137 scale: 2,
3138 }),
3139 )
3140 .into(),
3141 NestedField::required(4, "uuid", Type::Primitive(PrimitiveType::Uuid)).into(),
3142 NestedField::optional(5, "created_at", Type::Primitive(PrimitiveType::Timestamptz))
3143 .into(),
3144 NestedField::optional(
3145 6,
3146 "tags",
3147 Type::List(ListType {
3148 element_field: NestedField::list_element(
3149 10,
3150 Type::Primitive(PrimitiveType::String),
3151 false,
3152 )
3153 .into(),
3154 }),
3155 )
3156 .into(),
3157 NestedField::optional(
3158 7,
3159 "address",
3160 Type::Struct(StructType::new(vec![
3161 NestedField::optional(11, "street", Type::Primitive(PrimitiveType::String))
3162 .into(),
3163 NestedField::required(12, "city", Type::Primitive(PrimitiveType::String))
3164 .into(),
3165 NestedField::optional(13, "zip", Type::Primitive(PrimitiveType::Int))
3166 .into(),
3167 ])),
3168 )
3169 .into(),
3170 NestedField::optional(
3171 8,
3172 "attributes",
3173 Type::Map(MapType {
3174 key_field: NestedField::map_key_element(
3175 14,
3176 Type::Primitive(PrimitiveType::String),
3177 )
3178 .into(),
3179 value_field: NestedField::map_value_element(
3180 15,
3181 Type::Primitive(PrimitiveType::String),
3182 false,
3183 )
3184 .into(),
3185 }),
3186 )
3187 .into(),
3188 NestedField::optional(
3189 9,
3190 "orders",
3191 Type::List(ListType {
3192 element_field: NestedField::list_element(
3193 16,
3194 Type::Struct(StructType::new(vec![
3195 NestedField::required(
3196 17,
3197 "order_id",
3198 Type::Primitive(PrimitiveType::Long),
3199 )
3200 .into(),
3201 NestedField::required(
3202 18,
3203 "amount",
3204 Type::Primitive(PrimitiveType::Double),
3205 )
3206 .into(),
3207 ])),
3208 false,
3209 )
3210 .into(),
3211 }),
3212 )
3213 .into(),
3214 ])
3215 .build()
3216 .unwrap();
3217
3218 pretty_assertions::assert_eq!(schema, expected);
3219 assert_eq!(schema.highest_field_id(), 18);
3220 }
3221
3222 fn decimal_byte_array_stats(min: Vec<u8>, max: Vec<u8>) -> Statistics {
3223 Statistics::byte_array(
3224 Some(parquet::data_type::ByteArray::from(min)),
3225 Some(parquet::data_type::ByteArray::from(max)),
3226 None,
3227 Some(0),
3228 false,
3229 )
3230 }
3231
3232 fn decimal_bounds(stats: &Statistics) -> Result<(Option<Datum>, Option<Datum>)> {
3233 let ty = PrimitiveType::Decimal {
3234 precision: 38,
3235 scale: 2,
3236 };
3237 Ok((
3238 get_parquet_stat_min_as_datum(&ty, stats)?,
3239 get_parquet_stat_max_as_datum(&ty, stats)?,
3240 ))
3241 }
3242
3243 fn decimal_datum(value: i128) -> Datum {
3244 Datum::new(
3245 PrimitiveType::Decimal {
3246 precision: 38,
3247 scale: 2,
3248 },
3249 PrimitiveLiteral::Int128(value),
3250 )
3251 }
3252
3253 #[test]
3254 fn test_byte_array_decimal_stats_short_bounds() {
3255 let stats = decimal_byte_array_stats(vec![0xFB, 0x2E], vec![0x04, 0xD2]);
3257 let (min, max) = decimal_bounds(&stats).unwrap();
3258 assert_eq!(min, Some(decimal_datum(-1234)));
3259 assert_eq!(max, Some(decimal_datum(1234)));
3260
3261 let stats = decimal_byte_array_stats(vec![0x80], vec![0x7F]);
3263 let (min, max) = decimal_bounds(&stats).unwrap();
3264 assert_eq!(min, Some(decimal_datum(-128)));
3265 assert_eq!(max, Some(decimal_datum(127)));
3266 }
3267
3268 #[test]
3269 fn test_byte_array_decimal_stats_full_width_bounds() {
3270 let stats = decimal_byte_array_stats(
3271 i128::MIN.to_be_bytes().to_vec(),
3272 i128::MAX.to_be_bytes().to_vec(),
3273 );
3274 let (min, max) = decimal_bounds(&stats).unwrap();
3275 assert_eq!(min, Some(decimal_datum(i128::MIN)));
3276 assert_eq!(max, Some(decimal_datum(i128::MAX)));
3277 }
3278
3279 #[test]
3280 fn test_byte_array_decimal_stats_empty_bound_errors() {
3281 let stats = decimal_byte_array_stats(vec![], vec![0x01]);
3282 let err = decimal_bounds(&stats).unwrap_err();
3283 assert_eq!(err.kind(), ErrorKind::DataInvalid);
3284
3285 let stats = decimal_byte_array_stats(vec![0x01], vec![]);
3286 let err = decimal_bounds(&stats).unwrap_err();
3287 assert_eq!(err.kind(), ErrorKind::DataInvalid);
3288 }
3289
3290 #[test]
3291 fn test_byte_array_decimal_stats_oversized_bound_errors() {
3292 let stats = decimal_byte_array_stats(vec![0; 17], vec![0x01]);
3293 let err = decimal_bounds(&stats).unwrap_err();
3294 assert_eq!(err.kind(), ErrorKind::DataInvalid);
3295
3296 let stats = decimal_byte_array_stats(vec![0x01], vec![0; 17]);
3297 let err = decimal_bounds(&stats).unwrap_err();
3298 assert_eq!(err.kind(), ErrorKind::DataInvalid);
3299 }
3300
3301 #[test]
3306 fn test_strip_metadata_from_nested_schema() {
3307 let schema = |with_metadata: bool| {
3309 let metadata = |value: &str| {
3310 if with_metadata {
3311 HashMap::from([("k".to_string(), value.to_string())])
3312 } else {
3313 HashMap::new()
3314 }
3315 };
3316 let field = |name: &str, ty: DataType, nullable: bool| {
3317 Field::new(name, ty, nullable).with_metadata(metadata(name))
3318 };
3319 let element = |ty: DataType| Arc::new(field(LIST_FIELD_NAME, ty, false));
3320 let ints = || DataType::List(element(DataType::Int32));
3321 let map = |value: DataType, sorted: bool| {
3322 let entries = Fields::from(vec![
3323 field(MAP_KEY_FIELD_NAME, DataType::Utf8, false),
3324 field(MAP_VALUE_FIELD_NAME, value, true),
3325 ]);
3326 let entries = field(DEFAULT_MAP_FIELD_NAME, DataType::Struct(entries), false);
3327 DataType::Map(Arc::new(entries), sorted)
3328 };
3329 let struct_of = |fields: Vec<Field>| DataType::Struct(Fields::from(fields));
3330
3331 let a_b_a_b = struct_of(vec![field(
3332 "b",
3333 struct_of(vec![field(
3334 "a",
3335 struct_of(vec![field("b", DataType::Int32, true)]),
3336 false,
3337 )]),
3338 false,
3339 )]);
3340 ArrowSchema::new(vec![
3341 field("list", ints(), true),
3342 field(
3343 "large_list",
3344 DataType::LargeList(element(DataType::Int32)),
3345 false,
3346 ),
3347 field(
3348 "fixed_list",
3349 DataType::FixedSizeList(element(DataType::Int32), 2),
3350 true,
3351 ),
3352 field("list_of_list", DataType::List(element(ints())), false),
3353 field("sorted_map", map(ints(), true), false),
3354 field(
3355 "map_of_struct",
3356 map(
3357 struct_of(vec![
3358 field("a", DataType::Int32, true),
3359 field("b", ints(), false),
3360 ]),
3361 false,
3362 ),
3363 true,
3364 ),
3365 field(
3366 "list_of_struct",
3367 DataType::List(element(struct_of(vec![field(
3368 "m",
3369 map(ints(), false),
3370 true,
3371 )]))),
3372 true,
3373 ),
3374 field("a", a_b_a_b, true),
3375 ])
3376 .with_metadata(metadata("schema"))
3377 };
3378
3379 let (input, expected) = (schema(true), schema(false));
3380 assert_ne!(input, expected);
3382 pretty_assertions::assert_eq!(strip_metadata_from_schema(&input).unwrap(), expected);
3383 }
3384
3385 #[test]
3390 fn test_strip_metadata_normalizes_container_field_names() {
3391 let schema = |[element, entries, key, value]: [&str; 4], key_nullable: bool| {
3394 let ints =
3395 |nullable| DataType::List(Arc::new(Field::new(element, DataType::Int32, nullable)));
3396 let key_value = Fields::from(vec![
3397 Field::new(key, DataType::Utf8, key_nullable),
3398 Field::new(value, ints(false), true),
3399 ]);
3400 let entries = Field::new(entries, DataType::Struct(key_value), false);
3401 ArrowSchema::new(vec![
3402 Field::new("l", ints(true), true),
3403 Field::new("m", DataType::Map(Arc::new(entries), false), true),
3404 ])
3405 };
3406 let arrow_rs = schema(["item", "entries", "keys", "values"], true);
3407 let expected = schema(
3408 [
3409 LIST_FIELD_NAME,
3410 DEFAULT_MAP_FIELD_NAME,
3411 MAP_KEY_FIELD_NAME,
3412 MAP_VALUE_FIELD_NAME,
3413 ],
3414 false,
3415 );
3416
3417 let int_list = |id, required| {
3419 let int = Type::Primitive(PrimitiveType::Int);
3420 Type::List(ListType::new(
3421 NestedField::list_element(id, int, required).into(),
3422 ))
3423 };
3424 let string_to_int_list = MapType::new(
3425 NestedField::map_key_element(4, Type::Primitive(PrimitiveType::String)).into(),
3426 NestedField::map_value_element(5, int_list(6, true), false).into(),
3427 );
3428 let iceberg = Schema::builder()
3429 .with_fields(vec![
3430 NestedField::optional(1, "l", int_list(2, false)).into(),
3431 NestedField::optional(3, "m", Type::Map(string_to_int_list)).into(),
3432 ])
3433 .build()
3434 .unwrap();
3435
3436 pretty_assertions::assert_eq!(strip_metadata_from_schema(&arrow_rs).unwrap(), expected);
3437 pretty_assertions::assert_eq!(
3438 strip_metadata_from_schema(&schema_to_arrow_schema(&iceberg).unwrap()).unwrap(),
3439 expected
3440 );
3441 }
3442
3443 #[test]
3446 fn test_strip_metadata_keeps_other_differences() {
3447 let schema = |child: &str, element_nullable: bool| {
3448 let child = Field::new(child, DataType::Int32, true);
3449 let element = Field::new("item", DataType::Int32, element_nullable);
3450 ArrowSchema::new(vec![
3451 Field::new("s", DataType::Struct(Fields::from(vec![child])), true),
3452 Field::new("l", DataType::List(Arc::new(element)), true),
3453 ])
3454 };
3455 let strip = |schema: ArrowSchema| strip_metadata_from_schema(&schema).unwrap();
3456
3457 assert_ne!(strip(schema("a", true)), strip(schema("b", true)));
3458 assert_ne!(strip(schema("a", true)), strip(schema("a", false)));
3459 }
3460
3461 #[test]
3464 fn test_strip_metadata_turns_variants_into_structs() {
3465 let variant = || Type::Variant(VariantType);
3466 let schema = Schema::builder()
3467 .with_fields(vec![
3468 NestedField::optional(1, "v", variant()).into(),
3469 NestedField::optional(
3470 2,
3471 "l",
3472 Type::List(ListType::new(
3473 NestedField::list_element(3, variant(), false).into(),
3474 )),
3475 )
3476 .into(),
3477 NestedField::optional(
3478 4,
3479 "m",
3480 Type::Map(MapType::new(
3481 NestedField::map_key_element(5, Type::Primitive(PrimitiveType::String))
3482 .into(),
3483 NestedField::map_value_element(6, variant(), false).into(),
3484 )),
3485 )
3486 .into(),
3487 ])
3488 .build()
3489 .unwrap();
3490
3491 let storage = || {
3492 DataType::Struct(Fields::from(vec![
3493 Field::new("metadata", DataType::Binary, false),
3494 Field::new("value", DataType::Binary, true),
3495 ]))
3496 };
3497 let entries = Fields::from(vec![
3498 Field::new(MAP_KEY_FIELD_NAME, DataType::Utf8, false),
3499 Field::new(MAP_VALUE_FIELD_NAME, storage(), true),
3500 ]);
3501 let entries = Field::new(DEFAULT_MAP_FIELD_NAME, DataType::Struct(entries), false);
3502 let expected = ArrowSchema::new(vec![
3503 Field::new("v", storage(), true),
3504 Field::new(
3505 "l",
3506 DataType::List(Arc::new(Field::new(LIST_FIELD_NAME, storage(), true))),
3507 true,
3508 ),
3509 Field::new("m", DataType::Map(Arc::new(entries), false), true),
3510 ]);
3511
3512 pretty_assertions::assert_eq!(
3513 strip_metadata_from_schema(&schema_to_arrow_schema(&schema).unwrap()).unwrap(),
3514 expected
3515 );
3516 }
3517
3518 #[test]
3521 fn test_strip_metadata_unwraps_dictionaries() {
3522 let schema = |dictionary_encoded: bool| {
3523 let maybe_dictionary = |key: DataType, value: DataType| {
3524 if dictionary_encoded {
3525 DataType::Dictionary(Box::new(key), Box::new(value))
3526 } else {
3527 value
3528 }
3529 };
3530 let element = Field::new(
3531 LIST_FIELD_NAME,
3532 maybe_dictionary(DataType::Int8, DataType::LargeUtf8),
3533 false,
3534 );
3535 ArrowSchema::new(vec![
3536 Field::new("d", maybe_dictionary(DataType::Int32, DataType::Utf8), true),
3537 Field::new("l", DataType::List(Arc::new(element)), true),
3538 ])
3539 };
3540
3541 pretty_assertions::assert_eq!(
3542 strip_metadata_from_schema(&schema(true)).unwrap(),
3543 schema(false)
3544 );
3545 }
3546}