1use std::sync::Arc;
19
20use arrow_array::{
21 Array, ArrayRef, BinaryArray, BinaryViewArray, BooleanArray, Date32Array, Decimal128Array,
22 FixedSizeBinaryArray, FixedSizeListArray, Float32Array, Float64Array, Int32Array, Int64Array,
23 LargeBinaryArray, LargeListArray, LargeStringArray, ListArray, MapArray, StringArray,
24 StructArray, Time64MicrosecondArray, TimestampMicrosecondArray, TimestampNanosecondArray,
25 new_null_array,
26};
27use arrow_buffer::BooleanBuffer;
28use arrow_schema::{DataType, FieldRef, TimeUnit};
29use uuid::Uuid;
30
31use super::get_field_id_from_metadata;
32use crate::error::invalid_data;
33use crate::spec::{
34 ListType, Literal, Map, MapType, NestedField, PartnerAccessor, PrimitiveLiteral, PrimitiveType,
35 SchemaWithPartnerVisitor, Struct, StructType, Type, VariantType, visit_struct_with_partner,
36 visit_type_with_partner,
37};
38use crate::{Error, ErrorKind, Result};
39
40struct ArrowArrayToIcebergStructConverter;
41
42impl SchemaWithPartnerVisitor<ArrayRef> for ArrowArrayToIcebergStructConverter {
43 type T = Vec<Option<Literal>>;
44
45 fn schema(
46 &mut self,
47 _schema: &crate::spec::Schema,
48 _partner: &ArrayRef,
49 value: Vec<Option<Literal>>,
50 ) -> Result<Vec<Option<Literal>>> {
51 Ok(value)
52 }
53
54 fn field(
55 &mut self,
56 field: &crate::spec::NestedFieldRef,
57 _partner: &ArrayRef,
58 value: Vec<Option<Literal>>,
59 ) -> Result<Vec<Option<Literal>>> {
60 if field.required && value.iter().any(Option::is_none) {
62 return Err(invalid_data!("The field is required but has null value")
63 .with_context("field_id", field.id.to_string())
64 .with_context("field_name", &field.name));
65 }
66 Ok(value)
67 }
68
69 fn r#struct(
70 &mut self,
71 _struct: &StructType,
72 array: &ArrayRef,
73 results: Vec<Vec<Option<Literal>>>,
74 ) -> Result<Vec<Option<Literal>>> {
75 let row_len = results.first().map(|column| column.len()).unwrap_or(0);
76 if let Some(col) = results.iter().find(|col| col.len() != row_len) {
77 return Err(
78 invalid_data!("The struct columns have different row length")
79 .with_context("first col length", row_len.to_string())
80 .with_context("actual col length", col.len().to_string()),
81 );
82 }
83
84 let mut struct_literals = Vec::with_capacity(row_len);
85 let mut columns_iters = results
86 .into_iter()
87 .map(|column| column.into_iter())
88 .collect::<Vec<_>>();
89
90 for i in 0..row_len {
91 let mut literals = Vec::with_capacity(columns_iters.len());
92 for column_iter in columns_iters.iter_mut() {
93 literals.push(column_iter.next().unwrap());
94 }
95 if array.is_null(i) {
96 struct_literals.push(None);
97 } else {
98 struct_literals.push(Some(Literal::Struct(Struct::from_iter(literals))));
99 }
100 }
101
102 Ok(struct_literals)
103 }
104
105 fn list(
106 &mut self,
107 list: &ListType,
108 array: &ArrayRef,
109 elements: Vec<Option<Literal>>,
110 ) -> Result<Vec<Option<Literal>>> {
111 if list.element_field.required && elements.iter().any(Option::is_none) {
112 return Err(invalid_data!("The list should not have null value"));
113 }
114 match array.data_type() {
115 DataType::List(_) => {
116 let offset = array
117 .as_any()
118 .downcast_ref::<ListArray>()
119 .ok_or_else(|| invalid_data!("The partner is not a list array"))?
120 .offsets();
121 let mut result = Vec::with_capacity(offset.len() - 1);
123 for i in 0..offset.len() - 1 {
124 let start = offset[i] as usize;
125 let end = offset[i + 1] as usize;
126 result.push(Some(Literal::List(elements[start..end].to_vec())));
127 }
128 Ok(result)
129 }
130 DataType::LargeList(_) => {
131 let offset = array
132 .as_any()
133 .downcast_ref::<LargeListArray>()
134 .ok_or_else(|| invalid_data!("The partner is not a large list array"))?
135 .offsets();
136 let mut result = Vec::with_capacity(offset.len() - 1);
138 for i in 0..offset.len() - 1 {
139 let start = offset[i] as usize;
140 let end = offset[i + 1] as usize;
141 result.push(Some(Literal::List(elements[start..end].to_vec())));
142 }
143 Ok(result)
144 }
145 DataType::FixedSizeList(_, len) => {
146 let mut result = Vec::with_capacity(elements.len() / *len as usize);
147 for i in 0..elements.len() / *len as usize {
148 let start = i * *len as usize;
149 let end = (i + 1) * *len as usize;
150 result.push(Some(Literal::List(elements[start..end].to_vec())));
151 }
152 Ok(result)
153 }
154 _ => Err(invalid_data!("The partner is not a list type")),
155 }
156 }
157
158 fn map(
159 &mut self,
160 _map: &MapType,
161 partner: &ArrayRef,
162 key_values: Vec<Option<Literal>>,
163 values: Vec<Option<Literal>>,
164 ) -> Result<Vec<Option<Literal>>> {
165 if key_values.len() != values.len() {
167 return Err(invalid_data!(
168 "The key value and value of map should have the same row length"
169 ));
170 }
171
172 let offsets = partner
173 .as_any()
174 .downcast_ref::<MapArray>()
175 .ok_or_else(|| invalid_data!("The partner is not a map array"))?
176 .offsets();
177 let mut result = Vec::with_capacity(offsets.len() - 1);
179 for i in 0..offsets.len() - 1 {
180 let start = offsets[i] as usize;
181 let end = offsets[i + 1] as usize;
182 let mut map = Map::new();
183 for (key, value) in key_values[start..end].iter().zip(values[start..end].iter()) {
184 map.insert(key.clone().unwrap(), value.clone());
185 }
186 result.push(Some(Literal::Map(map)));
187 }
188 Ok(result)
189 }
190
191 fn primitive(&mut self, p: &PrimitiveType, partner: &ArrayRef) -> Result<Vec<Option<Literal>>> {
192 match p {
193 PrimitiveType::Unknown => Ok(vec![None; partner.len()]),
194 PrimitiveType::Boolean => {
195 let array = partner
196 .as_any()
197 .downcast_ref::<BooleanArray>()
198 .ok_or_else(|| invalid_data!("The partner is not a boolean array"))?;
199 Ok(array.iter().map(|v| v.map(Literal::bool)).collect())
200 }
201 PrimitiveType::Int => {
202 let array = partner
203 .as_any()
204 .downcast_ref::<Int32Array>()
205 .ok_or_else(|| invalid_data!("The partner is not a int32 array"))?;
206 Ok(array.iter().map(|v| v.map(Literal::int)).collect())
207 }
208 PrimitiveType::Long => {
209 let array = partner
210 .as_any()
211 .downcast_ref::<Int64Array>()
212 .ok_or_else(|| invalid_data!("The partner is not a int64 array"))?;
213 Ok(array.iter().map(|v| v.map(Literal::long)).collect())
214 }
215 PrimitiveType::Float => {
216 let array = partner
217 .as_any()
218 .downcast_ref::<Float32Array>()
219 .ok_or_else(|| invalid_data!("The partner is not a float32 array"))?;
220 Ok(array.iter().map(|v| v.map(Literal::float)).collect())
221 }
222 PrimitiveType::Double => {
223 let array = partner
224 .as_any()
225 .downcast_ref::<Float64Array>()
226 .ok_or_else(|| invalid_data!("The partner is not a float64 array"))?;
227 Ok(array.iter().map(|v| v.map(Literal::double)).collect())
228 }
229 PrimitiveType::Decimal { precision, scale } => {
230 let array = partner
231 .as_any()
232 .downcast_ref::<Decimal128Array>()
233 .ok_or_else(|| invalid_data!("The partner is not a decimal128 array"))?;
234 if let DataType::Decimal128(arrow_precision, arrow_scale) = array.data_type()
235 && (*arrow_precision as u32 != *precision || *arrow_scale as u32 != *scale)
236 {
237 return Err(invalid_data!(
238 "The precision or scale ({arrow_precision},{arrow_scale}) of arrow decimal128 array is not compatible with iceberg decimal type ({precision},{scale})"
239 ));
240 }
241 Ok(array.iter().map(|v| v.map(Literal::decimal)).collect())
242 }
243 PrimitiveType::Date => {
244 let array = partner
245 .as_any()
246 .downcast_ref::<Date32Array>()
247 .ok_or_else(|| invalid_data!("The partner is not a date32 array"))?;
248 Ok(array.iter().map(|v| v.map(Literal::date)).collect())
249 }
250 PrimitiveType::Time => {
251 let array = partner
252 .as_any()
253 .downcast_ref::<Time64MicrosecondArray>()
254 .ok_or_else(|| invalid_data!("The partner is not a time64 array"))?;
255 Ok(array.iter().map(|v| v.map(Literal::time)).collect())
256 }
257 PrimitiveType::Timestamp => {
258 let array = partner
259 .as_any()
260 .downcast_ref::<TimestampMicrosecondArray>()
261 .ok_or_else(|| invalid_data!("The partner is not a timestamp array"))?;
262 Ok(array.iter().map(|v| v.map(Literal::timestamp)).collect())
263 }
264 PrimitiveType::Timestamptz => {
265 let array = partner
266 .as_any()
267 .downcast_ref::<TimestampMicrosecondArray>()
268 .ok_or_else(|| invalid_data!("The partner is not a timestamptz array"))?;
269 Ok(array.iter().map(|v| v.map(Literal::timestamptz)).collect())
270 }
271 PrimitiveType::TimestampNs => {
272 let array = partner
273 .as_any()
274 .downcast_ref::<TimestampNanosecondArray>()
275 .ok_or_else(|| invalid_data!("The partner is not a timestamp_ns array"))?;
276 Ok(array
277 .iter()
278 .map(|v| v.map(Literal::timestamp_nano))
279 .collect())
280 }
281 PrimitiveType::TimestamptzNs => {
282 let array = partner
283 .as_any()
284 .downcast_ref::<TimestampNanosecondArray>()
285 .ok_or_else(|| invalid_data!("The partner is not a timestamptz_ns array"))?;
286 Ok(array
287 .iter()
288 .map(|v| v.map(Literal::timestamptz_nano))
289 .collect())
290 }
291 PrimitiveType::String => {
292 if let Some(array) = partner.as_any().downcast_ref::<LargeStringArray>() {
293 Ok(array.iter().map(|v| v.map(Literal::string)).collect())
294 } else if let Some(array) = partner.as_any().downcast_ref::<StringArray>() {
295 Ok(array.iter().map(|v| v.map(Literal::string)).collect())
296 } else {
297 Err(invalid_data!("The partner is not a string array"))
298 }
299 }
300 PrimitiveType::Uuid => {
301 if let Some(array) = partner.as_any().downcast_ref::<FixedSizeBinaryArray>() {
302 if array.value_length() != 16 {
303 return Err(invalid_data!("The partner is not a uuid array"));
304 }
305 Ok(array
306 .iter()
307 .map(|v| {
308 v.map(|v| {
309 Ok(Literal::uuid(Uuid::from_bytes(v.try_into().map_err(
310 |_| invalid_data!("Failed to convert binary to uuid"),
311 )?)))
312 })
313 .transpose()
314 })
315 .collect::<Result<Vec<_>>>()?)
316 } else {
317 Err(invalid_data!("The partner is not a uuid array"))
318 }
319 }
320 PrimitiveType::Fixed(len) => {
321 let array = partner
322 .as_any()
323 .downcast_ref::<FixedSizeBinaryArray>()
324 .ok_or_else(|| invalid_data!("The partner is not a fixed array"))?;
325 if array.value_length() != *len as i32 {
326 return Err(invalid_data!(
327 "The length of fixed size binary array is not compatible with iceberg fixed type"
328 ));
329 }
330 Ok(array
331 .iter()
332 .map(|v| v.map(|v| Literal::fixed(v.iter().cloned())))
333 .collect())
334 }
335 PrimitiveType::Binary | PrimitiveType::Geometry(_) | PrimitiveType::Geography(_) => {
336 if let Some(array) = partner.as_any().downcast_ref::<LargeBinaryArray>() {
337 Ok(array
338 .iter()
339 .map(|v| v.map(|v| Literal::binary(v.to_vec())))
340 .collect())
341 } else if let Some(array) = partner.as_any().downcast_ref::<BinaryArray>() {
342 Ok(array
343 .iter()
344 .map(|v| v.map(|v| Literal::binary(v.to_vec())))
345 .collect())
346 } else if let Some(array) = partner.as_any().downcast_ref::<BinaryViewArray>() {
347 Ok(array
348 .iter()
349 .map(|v| v.map(|v| Literal::binary(v.to_vec())))
350 .collect())
351 } else {
352 Err(invalid_data!("The partner is not a binary array"))
353 }
354 }
355 }
356 }
357
358 fn variant(&mut self, _v: &VariantType, _partner: &ArrayRef) -> Result<Vec<Option<Literal>>> {
359 Err(Error::new(
360 ErrorKind::FeatureUnsupported,
361 "Converting variant Arrow array to Iceberg literal is not supported yet",
362 ))
363 }
364}
365
366#[derive(Clone, Copy, Debug)]
377pub enum FieldMatchMode {
378 Id,
380 Name,
382}
383
384impl FieldMatchMode {
385 pub fn match_field(&self, arrow_field: &FieldRef, iceberg_field: &NestedField) -> bool {
387 match self {
388 FieldMatchMode::Id => get_field_id_from_metadata(arrow_field)
389 .map(|id| id == iceberg_field.id)
390 .unwrap_or(false),
391 FieldMatchMode::Name => arrow_field.name() == &iceberg_field.name,
392 }
393 }
394}
395
396pub struct ArrowArrayAccessor {
398 match_mode: FieldMatchMode,
399}
400
401impl ArrowArrayAccessor {
402 pub fn new() -> Self {
404 Self {
405 match_mode: FieldMatchMode::Id,
406 }
407 }
408
409 pub fn new_with_match_mode(match_mode: FieldMatchMode) -> Self {
411 Self { match_mode }
412 }
413}
414
415impl Default for ArrowArrayAccessor {
416 fn default() -> Self {
417 Self::new()
418 }
419}
420
421impl PartnerAccessor<ArrayRef> for ArrowArrayAccessor {
422 fn struct_partner<'a>(&self, schema_partner: &'a ArrayRef) -> Result<&'a ArrayRef> {
423 if !matches!(schema_partner.data_type(), DataType::Struct(_)) {
424 return Err(invalid_data!("The schema partner is not a struct type"));
425 }
426
427 Ok(schema_partner)
428 }
429
430 fn field_partner<'a>(
431 &self,
432 struct_partner: &'a ArrayRef,
433 field: &NestedField,
434 ) -> Result<&'a ArrayRef> {
435 let struct_array = struct_partner
436 .as_any()
437 .downcast_ref::<StructArray>()
438 .ok_or_else(|| {
439 invalid_data!(
440 "The struct partner is not a struct array, partner: {struct_partner:?}"
441 )
442 })?;
443
444 let field_pos = struct_array
445 .fields()
446 .iter()
447 .position(|arrow_field| self.match_mode.match_field(arrow_field, field))
448 .ok_or_else(|| invalid_data!("Field id {} not found in struct array", field.id))?;
449
450 Ok(struct_array.column(field_pos))
451 }
452
453 fn list_element_partner<'a>(&self, list_partner: &'a ArrayRef) -> Result<&'a ArrayRef> {
454 match list_partner.data_type() {
455 DataType::List(_) => {
456 let list_array = list_partner
457 .as_any()
458 .downcast_ref::<ListArray>()
459 .ok_or_else(|| invalid_data!("The list partner is not a list array"))?;
460 Ok(list_array.values())
461 }
462 DataType::LargeList(_) => {
463 let list_array = list_partner
464 .as_any()
465 .downcast_ref::<LargeListArray>()
466 .ok_or_else(|| invalid_data!("The list partner is not a large list array"))?;
467 Ok(list_array.values())
468 }
469 DataType::FixedSizeList(_, _) => {
470 let list_array = list_partner
471 .as_any()
472 .downcast_ref::<FixedSizeListArray>()
473 .ok_or_else(|| {
474 invalid_data!("The list partner is not a fixed size list array")
475 })?;
476 Ok(list_array.values())
477 }
478 _ => Err(invalid_data!("The list partner is not a list type")),
479 }
480 }
481
482 fn map_key_partner<'a>(&self, map_partner: &'a ArrayRef) -> Result<&'a ArrayRef> {
483 let map_array = map_partner
484 .as_any()
485 .downcast_ref::<MapArray>()
486 .ok_or_else(|| invalid_data!("The map partner is not a map array"))?;
487 Ok(map_array.keys())
488 }
489
490 fn map_value_partner<'a>(&self, map_partner: &'a ArrayRef) -> Result<&'a ArrayRef> {
491 let map_array = map_partner
492 .as_any()
493 .downcast_ref::<MapArray>()
494 .ok_or_else(|| invalid_data!("The map partner is not a map array"))?;
495 Ok(map_array.values())
496 }
497}
498
499pub fn arrow_struct_to_literal(
502 struct_array: &ArrayRef,
503 ty: &StructType,
504) -> Result<Vec<Option<Literal>>> {
505 visit_struct_with_partner(
506 ty,
507 struct_array,
508 &mut ArrowArrayToIcebergStructConverter,
509 &ArrowArrayAccessor::new(),
510 )
511}
512
513pub fn arrow_primitive_to_literal(
516 primitive_array: &ArrayRef,
517 ty: &Type,
518) -> Result<Vec<Option<Literal>>> {
519 visit_type_with_partner(
520 ty,
521 primitive_array,
522 &mut ArrowArrayToIcebergStructConverter,
523 &ArrowArrayAccessor::new(),
524 )
525}
526
527pub(crate) fn create_primitive_array_single_element(
532 data_type: &DataType,
533 prim_lit: Option<&PrimitiveLiteral>,
534) -> Result<ArrayRef> {
535 if prim_lit.is_none() {
537 return Ok(new_null_array(data_type, 1));
538 }
539 match (data_type, prim_lit) {
540 (DataType::Boolean, Some(PrimitiveLiteral::Boolean(v))) => {
541 Ok(Arc::new(BooleanArray::from(vec![*v])))
542 }
543 (DataType::Int32, Some(PrimitiveLiteral::Int(v))) => {
544 Ok(Arc::new(Int32Array::from(vec![*v])))
545 }
546 (DataType::Date32, Some(PrimitiveLiteral::Int(v))) => {
547 Ok(Arc::new(Date32Array::from(vec![*v])))
548 }
549 (DataType::Int64, Some(PrimitiveLiteral::Long(v))) => {
550 Ok(Arc::new(Int64Array::from(vec![*v])))
551 }
552 (DataType::Timestamp(TimeUnit::Microsecond, timezone), Some(PrimitiveLiteral::Long(v))) => {
553 let array = TimestampMicrosecondArray::from(vec![*v]);
554 if let Some(timezone) = timezone {
555 Ok(Arc::new(array.with_timezone(timezone.clone())))
556 } else {
557 Ok(Arc::new(array))
558 }
559 }
560 (DataType::Timestamp(TimeUnit::Nanosecond, timezone), Some(PrimitiveLiteral::Long(v))) => {
561 let array = TimestampNanosecondArray::from(vec![*v]);
562 if let Some(timezone) = timezone {
563 Ok(Arc::new(array.with_timezone(timezone.clone())))
564 } else {
565 Ok(Arc::new(array))
566 }
567 }
568 (DataType::Float32, Some(PrimitiveLiteral::Float(v))) => {
569 Ok(Arc::new(Float32Array::from(vec![v.0])))
570 }
571 (DataType::Float64, Some(PrimitiveLiteral::Double(v))) => {
572 Ok(Arc::new(Float64Array::from(vec![v.0])))
573 }
574 (DataType::Utf8, Some(PrimitiveLiteral::String(v))) => {
575 Ok(Arc::new(StringArray::from(vec![v.as_str()])))
576 }
577 (DataType::Binary, Some(PrimitiveLiteral::Binary(v))) => {
578 Ok(Arc::new(BinaryArray::from_vec(vec![v.as_slice()])))
579 }
580 (DataType::LargeBinary, Some(PrimitiveLiteral::Binary(v))) => {
581 Ok(Arc::new(LargeBinaryArray::from_vec(vec![v.as_slice()])))
582 }
583 (DataType::Decimal128(precision, scale), Some(PrimitiveLiteral::Int128(v))) => {
584 let array = Decimal128Array::from(vec![{ *v }])
585 .with_precision_and_scale(*precision, *scale)
586 .map_err(|e| {
587 invalid_data!(
588 "Failed to create Decimal128Array with precision {precision} and scale {scale}: {e}"
589 )
590 })?;
591 Ok(Arc::new(array))
592 }
593 (DataType::Decimal128(precision, scale), Some(PrimitiveLiteral::UInt128(v))) => {
594 let array = Decimal128Array::from(vec![*v as i128])
595 .with_precision_and_scale(*precision, *scale)
596 .map_err(|e| {
597 invalid_data!(
598 "Failed to create Decimal128Array with precision {precision} and scale {scale}: {e}"
599 )
600 })?;
601 Ok(Arc::new(array))
602 }
603 _ => Err(Error::new(
604 ErrorKind::Unexpected,
605 format!("Unsupported constant type combination: {data_type:?} with {prim_lit:?}"),
606 )),
607 }
608}
609
610pub(crate) fn create_primitive_array_repeated(
615 data_type: &DataType,
616 prim_lit: Option<&PrimitiveLiteral>,
617 num_rows: usize,
618) -> Result<ArrayRef> {
619 if prim_lit.is_none() {
621 return Ok(new_null_array(data_type, num_rows));
622 }
623 Ok(match (data_type, prim_lit) {
624 (DataType::Boolean, Some(PrimitiveLiteral::Boolean(value))) => {
626 let buffer = if *value {
627 BooleanBuffer::new_set(num_rows)
628 } else {
629 BooleanBuffer::new_unset(num_rows)
630 };
631 Arc::new(BooleanArray::new(buffer, None))
632 }
633 (DataType::Int32, Some(PrimitiveLiteral::Int(value))) => {
634 Arc::new(Int32Array::from(vec![*value; num_rows]))
635 }
636 (DataType::Date32, Some(PrimitiveLiteral::Int(value))) => {
637 Arc::new(Date32Array::from(vec![*value; num_rows]))
638 }
639 (DataType::Int64, Some(PrimitiveLiteral::Int(value))) => {
640 Arc::new(Int64Array::from(vec![i64::from(*value); num_rows]))
641 }
642 (DataType::Int64, Some(PrimitiveLiteral::Long(value))) => {
643 Arc::new(Int64Array::from(vec![*value; num_rows]))
644 }
645 (
646 DataType::Timestamp(TimeUnit::Microsecond, timezone),
647 Some(PrimitiveLiteral::Long(value)),
648 ) => {
649 let array = TimestampMicrosecondArray::from(vec![*value; num_rows]);
650 if let Some(timezone) = timezone {
651 Arc::new(array.with_timezone(timezone.clone()))
652 } else {
653 Arc::new(array)
654 }
655 }
656 (
657 DataType::Timestamp(TimeUnit::Nanosecond, timezone),
658 Some(PrimitiveLiteral::Long(value)),
659 ) => {
660 let array = TimestampNanosecondArray::from(vec![*value; num_rows]);
661 if let Some(timezone) = timezone {
662 Arc::new(array.with_timezone(timezone.clone()))
663 } else {
664 Arc::new(array)
665 }
666 }
667 (DataType::Float32, Some(PrimitiveLiteral::Float(value))) => {
668 Arc::new(Float32Array::from(vec![value.0; num_rows]))
669 }
670 (DataType::Float64, Some(PrimitiveLiteral::Double(value))) => {
671 Arc::new(Float64Array::from(vec![value.0; num_rows]))
672 }
673 (DataType::Utf8, Some(PrimitiveLiteral::String(value))) => Arc::new(
674 StringArray::from_iter_values(std::iter::repeat_n(value.as_str(), num_rows)),
675 ),
676 (DataType::Binary, Some(PrimitiveLiteral::Binary(value))) => Arc::new(
677 BinaryArray::from_iter_values(std::iter::repeat_n(value.as_slice(), num_rows)),
678 ),
679 (DataType::LargeBinary, Some(PrimitiveLiteral::Binary(value))) => Arc::new(
680 LargeBinaryArray::from_iter_values(std::iter::repeat_n(value.as_slice(), num_rows)),
681 ),
682 (DataType::FixedSizeBinary(len), Some(PrimitiveLiteral::Binary(value))) => {
683 if value.len() != *len as usize {
687 return Err(invalid_data!(
688 "FixedSizeBinary literal length {} does not match declared width {len}",
689 value.len()
690 ));
691 }
692
693 Arc::new(
694 FixedSizeBinaryArray::try_from_iter(std::iter::repeat_n(value.as_slice(), num_rows))
695 .map_err(|e| {
696 invalid_data!("Failed to create FixedSizeBinary({len}) array: {e}")
697 })?,
698 )
699 }
700 (DataType::Time64(TimeUnit::Microsecond), Some(PrimitiveLiteral::Long(value))) => {
701 Arc::new(Time64MicrosecondArray::from(vec![*value; num_rows]))
702 }
703 (DataType::Decimal128(precision, scale), Some(PrimitiveLiteral::Int128(value))) => {
704 Arc::new(
705 Decimal128Array::from(vec![*value; num_rows])
706 .with_precision_and_scale(*precision, *scale)
707 .map_err(|e| {
708 invalid_data!(
709 "Failed to create Decimal128Array with precision {precision} and scale {scale}: {e}"
710 )
711 })?,
712 )
713 }
714 (DataType::Decimal128(precision, scale), Some(PrimitiveLiteral::UInt128(value))) => {
715 Arc::new(
716 Decimal128Array::from(vec![*value as i128; num_rows])
717 .with_precision_and_scale(*precision, *scale)
718 .map_err(|e| {
719 invalid_data!(
720 "Failed to create Decimal128Array with precision {precision} and scale {scale}: {e}"
721 )
722 })?,
723 )
724 }
725
726 (dt, _) => {
727 return Err(Error::new(
728 ErrorKind::Unexpected,
729 format!("unexpected target column type {dt}, prim_lit {prim_lit:?}"),
730 ));
731 }
732 })
733}
734
735#[cfg(test)]
736mod test {
737 use std::collections::HashMap;
738 use std::sync::Arc;
739
740 use arrow_array::builder::{
741 Int32Builder, ListBuilder, MapBuilder, MapFieldNames, StructBuilder,
742 };
743 use arrow_array::{
744 ArrayRef, BinaryArray, BooleanArray, Date32Array, Decimal128Array, Float32Array,
745 Float64Array, Int32Array, Int64Array, StringArray, StructArray, Time64MicrosecondArray,
746 TimestampMicrosecondArray, TimestampNanosecondArray,
747 };
748 use arrow_schema::{DataType, Field, Fields, TimeUnit};
749 use parquet::arrow::PARQUET_FIELD_ID_META_KEY;
750
751 use super::*;
752 use crate::spec::{ListType, Literal, MapType, NestedField, PrimitiveType, StructType, Type};
753
754 #[test]
755 fn test_arrow_struct_to_iceberg_struct() {
756 let bool_array = BooleanArray::from(vec![Some(true), Some(false), None]);
757 let int32_array = Int32Array::from(vec![Some(3), Some(4), None]);
758 let int64_array = Int64Array::from(vec![Some(5), Some(6), None]);
759 let float32_array = Float32Array::from(vec![Some(1.1), Some(2.2), None]);
760 let float64_array = Float64Array::from(vec![Some(3.3), Some(4.4), None]);
761 let decimal_array = Decimal128Array::from(vec![Some(1000), Some(2000), None])
762 .with_precision_and_scale(10, 2)
763 .unwrap();
764 let date_array = Date32Array::from(vec![Some(18628), Some(18629), None]);
765 let time_array = Time64MicrosecondArray::from(vec![Some(123456789), Some(987654321), None]);
766 let timestamp_micro_array = TimestampMicrosecondArray::from(vec![
767 Some(1622548800000000),
768 Some(1622635200000000),
769 None,
770 ]);
771 let timestamp_nano_array = TimestampNanosecondArray::from(vec![
772 Some(1622548800000000000),
773 Some(1622635200000000000),
774 None,
775 ]);
776 let string_array = StringArray::from(vec![Some("a"), Some("b"), None]);
777 let binary_array =
778 BinaryArray::from(vec![Some(b"abc".as_ref()), Some(b"def".as_ref()), None]);
779
780 let struct_array = Arc::new(StructArray::from(vec![
781 (
782 Arc::new(
783 Field::new("bool_field", DataType::Boolean, true).with_metadata(HashMap::from(
784 [(PARQUET_FIELD_ID_META_KEY.to_string(), "0".to_string())],
785 )),
786 ),
787 Arc::new(bool_array) as ArrayRef,
788 ),
789 (
790 Arc::new(
791 Field::new("int32_field", DataType::Int32, true).with_metadata(HashMap::from(
792 [(PARQUET_FIELD_ID_META_KEY.to_string(), "2".to_string())],
793 )),
794 ),
795 Arc::new(int32_array) as ArrayRef,
796 ),
797 (
798 Arc::new(
799 Field::new("int64_field", DataType::Int64, true).with_metadata(HashMap::from(
800 [(PARQUET_FIELD_ID_META_KEY.to_string(), "3".to_string())],
801 )),
802 ),
803 Arc::new(int64_array) as ArrayRef,
804 ),
805 (
806 Arc::new(
807 Field::new("float32_field", DataType::Float32, true).with_metadata(
808 HashMap::from([(PARQUET_FIELD_ID_META_KEY.to_string(), "4".to_string())]),
809 ),
810 ),
811 Arc::new(float32_array) as ArrayRef,
812 ),
813 (
814 Arc::new(
815 Field::new("float64_field", DataType::Float64, true).with_metadata(
816 HashMap::from([(PARQUET_FIELD_ID_META_KEY.to_string(), "5".to_string())]),
817 ),
818 ),
819 Arc::new(float64_array) as ArrayRef,
820 ),
821 (
822 Arc::new(
823 Field::new("decimal_field", DataType::Decimal128(10, 2), true).with_metadata(
824 HashMap::from([(PARQUET_FIELD_ID_META_KEY.to_string(), "6".to_string())]),
825 ),
826 ),
827 Arc::new(decimal_array) as ArrayRef,
828 ),
829 (
830 Arc::new(
831 Field::new("date_field", DataType::Date32, true).with_metadata(HashMap::from(
832 [(PARQUET_FIELD_ID_META_KEY.to_string(), "7".to_string())],
833 )),
834 ),
835 Arc::new(date_array) as ArrayRef,
836 ),
837 (
838 Arc::new(
839 Field::new("time_field", DataType::Time64(TimeUnit::Microsecond), true)
840 .with_metadata(HashMap::from([(
841 PARQUET_FIELD_ID_META_KEY.to_string(),
842 "8".to_string(),
843 )])),
844 ),
845 Arc::new(time_array) as ArrayRef,
846 ),
847 (
848 Arc::new(
849 Field::new(
850 "timestamp_micro_field",
851 DataType::Timestamp(TimeUnit::Microsecond, None),
852 true,
853 )
854 .with_metadata(HashMap::from([(
855 PARQUET_FIELD_ID_META_KEY.to_string(),
856 "9".to_string(),
857 )])),
858 ),
859 Arc::new(timestamp_micro_array) as ArrayRef,
860 ),
861 (
862 Arc::new(
863 Field::new(
864 "timestamp_nano_field",
865 DataType::Timestamp(TimeUnit::Nanosecond, None),
866 true,
867 )
868 .with_metadata(HashMap::from([(
869 PARQUET_FIELD_ID_META_KEY.to_string(),
870 "10".to_string(),
871 )])),
872 ),
873 Arc::new(timestamp_nano_array) as ArrayRef,
874 ),
875 (
876 Arc::new(
877 Field::new("string_field", DataType::Utf8, true).with_metadata(HashMap::from(
878 [(PARQUET_FIELD_ID_META_KEY.to_string(), "11".to_string())],
879 )),
880 ),
881 Arc::new(string_array) as ArrayRef,
882 ),
883 (
884 Arc::new(
885 Field::new("binary_field", DataType::Binary, true).with_metadata(
886 HashMap::from([(PARQUET_FIELD_ID_META_KEY.to_string(), "12".to_string())]),
887 ),
888 ),
889 Arc::new(binary_array) as ArrayRef,
890 ),
891 ])) as ArrayRef;
892
893 let iceberg_struct_type = StructType::new(vec![
894 Arc::new(NestedField::optional(
895 0,
896 "bool_field",
897 Type::Primitive(PrimitiveType::Boolean),
898 )),
899 Arc::new(NestedField::optional(
900 2,
901 "int32_field",
902 Type::Primitive(PrimitiveType::Int),
903 )),
904 Arc::new(NestedField::optional(
905 3,
906 "int64_field",
907 Type::Primitive(PrimitiveType::Long),
908 )),
909 Arc::new(NestedField::optional(
910 4,
911 "float32_field",
912 Type::Primitive(PrimitiveType::Float),
913 )),
914 Arc::new(NestedField::optional(
915 5,
916 "float64_field",
917 Type::Primitive(PrimitiveType::Double),
918 )),
919 Arc::new(NestedField::optional(
920 6,
921 "decimal_field",
922 Type::Primitive(PrimitiveType::Decimal {
923 precision: 10,
924 scale: 2,
925 }),
926 )),
927 Arc::new(NestedField::optional(
928 7,
929 "date_field",
930 Type::Primitive(PrimitiveType::Date),
931 )),
932 Arc::new(NestedField::optional(
933 8,
934 "time_field",
935 Type::Primitive(PrimitiveType::Time),
936 )),
937 Arc::new(NestedField::optional(
938 9,
939 "timestamp_micro_field",
940 Type::Primitive(PrimitiveType::Timestamp),
941 )),
942 Arc::new(NestedField::optional(
943 10,
944 "timestamp_nao_field",
945 Type::Primitive(PrimitiveType::TimestampNs),
946 )),
947 Arc::new(NestedField::optional(
948 11,
949 "string_field",
950 Type::Primitive(PrimitiveType::String),
951 )),
952 Arc::new(NestedField::optional(
953 12,
954 "binary_field",
955 Type::Primitive(PrimitiveType::Binary),
956 )),
957 ]);
958
959 let result = arrow_struct_to_literal(&struct_array, &iceberg_struct_type).unwrap();
960
961 assert_eq!(result, vec![
962 Some(Literal::Struct(Struct::from_iter(vec![
963 Some(Literal::bool(true)),
964 Some(Literal::int(3)),
965 Some(Literal::long(5)),
966 Some(Literal::float(1.1_f32)),
967 Some(Literal::double(3.3)),
968 Some(Literal::decimal(1000)),
969 Some(Literal::date(18628)),
970 Some(Literal::time(123456789)),
971 Some(Literal::timestamp(1622548800000000)),
972 Some(Literal::timestamp_nano(1622548800000000000)),
973 Some(Literal::string("a".to_string())),
974 Some(Literal::binary(b"abc".to_vec())),
975 ]))),
976 Some(Literal::Struct(Struct::from_iter(vec![
977 Some(Literal::bool(false)),
978 Some(Literal::int(4)),
979 Some(Literal::long(6)),
980 Some(Literal::float(2.2_f32)),
981 Some(Literal::double(4.4)),
982 Some(Literal::decimal(2000)),
983 Some(Literal::date(18629)),
984 Some(Literal::time(987654321)),
985 Some(Literal::timestamp(1622635200000000)),
986 Some(Literal::timestamp_nano(1622635200000000000)),
987 Some(Literal::string("b".to_string())),
988 Some(Literal::binary(b"def".to_vec())),
989 ]))),
990 Some(Literal::Struct(Struct::from_iter(vec![
991 None, None, None, None, None, None, None, None, None, None, None, None,
992 ]))),
993 ]);
994 }
995
996 #[test]
997 fn test_nullable_struct() {
998 let struct_array = {
1005 let mut builder = StructBuilder::from_fields(
1006 Fields::from(vec![
1007 Field::new("a", DataType::Int32, true).with_metadata(HashMap::from([(
1008 PARQUET_FIELD_ID_META_KEY.to_string(),
1009 "0".to_string(),
1010 )])),
1011 Field::new("b", DataType::Int32, true).with_metadata(HashMap::from([(
1012 PARQUET_FIELD_ID_META_KEY.to_string(),
1013 "1".to_string(),
1014 )])),
1015 ]),
1016 3,
1017 );
1018 builder
1019 .field_builder::<Int32Builder>(0)
1020 .unwrap()
1021 .append_null();
1022 builder
1023 .field_builder::<Int32Builder>(1)
1024 .unwrap()
1025 .append_null();
1026 builder.append(true);
1027
1028 builder
1029 .field_builder::<Int32Builder>(0)
1030 .unwrap()
1031 .append_value(1);
1032 builder
1033 .field_builder::<Int32Builder>(1)
1034 .unwrap()
1035 .append_null();
1036 builder.append(true);
1037
1038 builder
1039 .field_builder::<Int32Builder>(0)
1040 .unwrap()
1041 .append_value(1);
1042 builder
1043 .field_builder::<Int32Builder>(1)
1044 .unwrap()
1045 .append_value(1);
1046 builder.append_null();
1047
1048 Arc::new(builder.finish()) as ArrayRef
1049 };
1050
1051 let iceberg_struct_type = StructType::new(vec![
1052 Arc::new(NestedField::optional(
1053 0,
1054 "a",
1055 Type::Primitive(PrimitiveType::Int),
1056 )),
1057 Arc::new(NestedField::optional(
1058 1,
1059 "b",
1060 Type::Primitive(PrimitiveType::Int),
1061 )),
1062 ]);
1063
1064 let result = arrow_struct_to_literal(&struct_array, &iceberg_struct_type).unwrap();
1065 assert_eq!(result, vec![
1066 Some(Literal::Struct(Struct::from_iter(vec![None, None,]))),
1067 Some(Literal::Struct(Struct::from_iter(vec![
1068 Some(Literal::int(1)),
1069 None,
1070 ]))),
1071 None,
1072 ]);
1073 }
1074
1075 #[test]
1076 fn test_empty_struct() {
1077 let struct_array = Arc::new(StructArray::new_null(Fields::empty(), 3)) as ArrayRef;
1078 let iceberg_struct_type = StructType::new(vec![]);
1079 let result = arrow_struct_to_literal(&struct_array, &iceberg_struct_type).unwrap();
1080 assert_eq!(result, vec![None; 0]);
1081 }
1082
1083 #[test]
1084 fn test_arrow_variant_to_literal_is_unsupported() {
1085 let variant_child = Arc::new(StructArray::from(vec![
1088 (
1089 Arc::new(Field::new("metadata", DataType::Binary, false)),
1090 Arc::new(BinaryArray::from(vec![Some(b"m".as_ref())])) as ArrayRef,
1091 ),
1092 (
1093 Arc::new(Field::new("value", DataType::Binary, false)),
1094 Arc::new(BinaryArray::from(vec![Some(b"v".as_ref())])) as ArrayRef,
1095 ),
1096 ])) as ArrayRef;
1097
1098 let struct_array = Arc::new(StructArray::from(vec![(
1099 Arc::new(
1100 Field::new("v", variant_child.data_type().clone(), false).with_metadata(
1101 HashMap::from([(PARQUET_FIELD_ID_META_KEY.to_string(), "1".to_string())]),
1102 ),
1103 ),
1104 variant_child,
1105 )])) as ArrayRef;
1106
1107 let ty = StructType::new(vec![
1108 NestedField::required(1, "v", Type::Variant(VariantType)).into(),
1109 ]);
1110
1111 let err = arrow_struct_to_literal(&struct_array, &ty).unwrap_err();
1112 assert_eq!(err.kind(), ErrorKind::FeatureUnsupported);
1113 assert!(
1114 err.to_string()
1115 .contains("Converting variant Arrow array to Iceberg literal is not supported yet"),
1116 "{err}"
1117 );
1118 }
1119
1120 #[test]
1121 fn test_arrow_geospatial_to_literal() {
1122 let values = Arc::new(LargeBinaryArray::from(vec![
1123 Some(b"wkb-1".as_slice()),
1124 None,
1125 Some(b"wkb-2".as_slice()),
1126 ])) as ArrayRef;
1127 let mut converter = ArrowArrayToIcebergStructConverter;
1128
1129 let result = converter
1130 .primitive(&PrimitiveType::Geometry(Default::default()), &values)
1131 .unwrap();
1132
1133 assert_eq!(result, vec![
1134 Some(Literal::binary(b"wkb-1".to_vec())),
1135 None,
1136 Some(Literal::binary(b"wkb-2".to_vec())),
1137 ]);
1138 }
1139
1140 #[test]
1141 fn test_arrow_geospatial_binary_view_to_literal() {
1142 let wkb = parquet_geospatial::testing::wkb_point_xy(1.0, 2.0);
1143 let values = Arc::new(BinaryViewArray::from(vec![Some(wkb.as_slice()), None])) as ArrayRef;
1144 let ty = Type::Primitive(PrimitiveType::Geography(Default::default()));
1145
1146 let result = arrow_primitive_to_literal(&values, &ty).unwrap();
1147
1148 assert_eq!(result, vec![Some(Literal::binary(wkb)), None]);
1149 }
1150
1151 #[test]
1152 fn test_find_field_by_id() {
1153 let field_a_array = Int32Array::from(vec![Some(42), Some(43), None]);
1155 let field_b_array = StringArray::from(vec![Some("value1"), Some("value2"), None]);
1156
1157 let nested_struct_array =
1159 Arc::new(StructArray::from(vec![
1160 (
1161 Arc::new(Field::new("field_a", DataType::Int32, true).with_metadata(
1162 HashMap::from([(PARQUET_FIELD_ID_META_KEY.to_string(), "1".to_string())]),
1163 )),
1164 Arc::new(field_a_array) as ArrayRef,
1165 ),
1166 (
1167 Arc::new(Field::new("field_b", DataType::Utf8, true).with_metadata(
1168 HashMap::from([(PARQUET_FIELD_ID_META_KEY.to_string(), "2".to_string())]),
1169 )),
1170 Arc::new(field_b_array) as ArrayRef,
1171 ),
1172 ])) as ArrayRef;
1173
1174 let field_c_array = Int32Array::from(vec![Some(100), Some(200), None]);
1175
1176 let struct_array = Arc::new(StructArray::from(vec![
1178 (
1179 Arc::new(
1180 Field::new(
1181 "nested_struct",
1182 DataType::Struct(Fields::from(vec![
1183 Field::new("field_a", DataType::Int32, true).with_metadata(
1184 HashMap::from([(
1185 PARQUET_FIELD_ID_META_KEY.to_string(),
1186 "1".to_string(),
1187 )]),
1188 ),
1189 Field::new("field_b", DataType::Utf8, true).with_metadata(
1190 HashMap::from([(
1191 PARQUET_FIELD_ID_META_KEY.to_string(),
1192 "2".to_string(),
1193 )]),
1194 ),
1195 ])),
1196 true,
1197 )
1198 .with_metadata(HashMap::from([(
1199 PARQUET_FIELD_ID_META_KEY.to_string(),
1200 "3".to_string(),
1201 )])),
1202 ),
1203 nested_struct_array,
1204 ),
1205 (
1206 Arc::new(Field::new("field_c", DataType::Int32, true).with_metadata(
1207 HashMap::from([(PARQUET_FIELD_ID_META_KEY.to_string(), "4".to_string())]),
1208 )),
1209 Arc::new(field_c_array) as ArrayRef,
1210 ),
1211 ])) as ArrayRef;
1212
1213 let accessor = ArrowArrayAccessor::new_with_match_mode(FieldMatchMode::Id);
1215
1216 let nested_field = NestedField::optional(
1218 3,
1219 "nested_struct",
1220 Type::Struct(StructType::new(vec![
1221 Arc::new(NestedField::optional(
1222 1,
1223 "field_a",
1224 Type::Primitive(PrimitiveType::Int),
1225 )),
1226 Arc::new(NestedField::optional(
1227 2,
1228 "field_b",
1229 Type::Primitive(PrimitiveType::String),
1230 )),
1231 ])),
1232 );
1233 let nested_partner = accessor
1234 .field_partner(&struct_array, &nested_field)
1235 .unwrap();
1236
1237 let field_a = NestedField::optional(1, "field_a", Type::Primitive(PrimitiveType::Int));
1239 let field_a_partner = accessor.field_partner(nested_partner, &field_a).unwrap();
1240
1241 let int_array = field_a_partner
1243 .as_any()
1244 .downcast_ref::<Int32Array>()
1245 .unwrap();
1246 assert_eq!(int_array.value(0), 42);
1247 assert_eq!(int_array.value(1), 43);
1248 assert!(int_array.is_null(2));
1249 }
1250
1251 #[test]
1252 fn test_find_field_by_name() {
1253 let field_a_array = Int32Array::from(vec![Some(42), Some(43), None]);
1255 let field_b_array = StringArray::from(vec![Some("value1"), Some("value2"), None]);
1256
1257 let nested_struct_array = Arc::new(StructArray::from(vec![
1259 (
1260 Arc::new(Field::new("field_a", DataType::Int32, true)),
1261 Arc::new(field_a_array) as ArrayRef,
1262 ),
1263 (
1264 Arc::new(Field::new("field_b", DataType::Utf8, true)),
1265 Arc::new(field_b_array) as ArrayRef,
1266 ),
1267 ])) as ArrayRef;
1268
1269 let field_c_array = Int32Array::from(vec![Some(100), Some(200), None]);
1270
1271 let struct_array = Arc::new(StructArray::from(vec![
1273 (
1274 Arc::new(Field::new(
1275 "nested_struct",
1276 DataType::Struct(Fields::from(vec![
1277 Field::new("field_a", DataType::Int32, true),
1278 Field::new("field_b", DataType::Utf8, true),
1279 ])),
1280 true,
1281 )),
1282 nested_struct_array,
1283 ),
1284 (
1285 Arc::new(Field::new("field_c", DataType::Int32, true)),
1286 Arc::new(field_c_array) as ArrayRef,
1287 ),
1288 ])) as ArrayRef;
1289
1290 let accessor = ArrowArrayAccessor::new_with_match_mode(FieldMatchMode::Name);
1292
1293 let nested_field = NestedField::optional(
1295 3,
1296 "nested_struct",
1297 Type::Struct(StructType::new(vec![
1298 Arc::new(NestedField::optional(
1299 1,
1300 "field_a",
1301 Type::Primitive(PrimitiveType::Int),
1302 )),
1303 Arc::new(NestedField::optional(
1304 2,
1305 "field_b",
1306 Type::Primitive(PrimitiveType::String),
1307 )),
1308 ])),
1309 );
1310 let nested_partner = accessor
1311 .field_partner(&struct_array, &nested_field)
1312 .unwrap();
1313
1314 let field_a = NestedField::optional(1, "field_a", Type::Primitive(PrimitiveType::Int));
1316 let field_a_partner = accessor.field_partner(nested_partner, &field_a).unwrap();
1317
1318 let int_array = field_a_partner
1320 .as_any()
1321 .downcast_ref::<Int32Array>()
1322 .unwrap();
1323 assert_eq!(int_array.value(0), 42);
1324 assert_eq!(int_array.value(1), 43);
1325 assert!(int_array.is_null(2));
1326 }
1327
1328 #[test]
1329 fn test_complex_nested() {
1330 let struct_type = StructType::new(vec![
1337 Arc::new(NestedField::required(
1338 0,
1339 "A",
1340 Type::List(ListType::new(Arc::new(NestedField::required(
1341 1,
1342 "item",
1343 Type::Struct(StructType::new(vec![
1344 Arc::new(NestedField::required(
1345 2,
1346 "a1",
1347 Type::Primitive(PrimitiveType::Int),
1348 )),
1349 Arc::new(NestedField::required(
1350 3,
1351 "a2",
1352 Type::Primitive(PrimitiveType::Int),
1353 )),
1354 ])),
1355 )))),
1356 )),
1357 Arc::new(NestedField::required(
1358 4,
1359 "B",
1360 Type::List(ListType::new(Arc::new(NestedField::required(
1361 5,
1362 "item",
1363 Type::Map(MapType::new(
1364 NestedField::optional(6, "keys", Type::Primitive(PrimitiveType::Int))
1365 .into(),
1366 NestedField::optional(7, "values", Type::Primitive(PrimitiveType::Int))
1367 .into(),
1368 )),
1369 )))),
1370 )),
1371 Arc::new(NestedField::required(
1372 8,
1373 "C",
1374 Type::List(ListType::new(Arc::new(NestedField::required(
1375 9,
1376 "item",
1377 Type::List(ListType::new(Arc::new(NestedField::optional(
1378 10,
1379 "item",
1380 Type::Primitive(PrimitiveType::Int),
1381 )))),
1382 )))),
1383 )),
1384 ]);
1385
1386 let struct_array =
1392 {
1393 let a_struct_a1_builder = Int32Builder::new();
1394 let a_struct_a2_builder = Int32Builder::new();
1395 let a_struct_builder =
1396 StructBuilder::new(
1397 vec![
1398 Field::new("a1", DataType::Int32, false).with_metadata(HashMap::from(
1399 [(PARQUET_FIELD_ID_META_KEY.to_string(), "2".to_string())],
1400 )),
1401 Field::new("a2", DataType::Int32, false).with_metadata(HashMap::from(
1402 [(PARQUET_FIELD_ID_META_KEY.to_string(), "3".to_string())],
1403 )),
1404 ],
1405 vec![Box::new(a_struct_a1_builder), Box::new(a_struct_a2_builder)],
1406 );
1407 let a_builder = ListBuilder::new(a_struct_builder);
1408
1409 let map_key_builder = Int32Builder::new();
1410 let map_value_builder = Int32Builder::new();
1411 let map_builder = MapBuilder::new(
1414 Some(MapFieldNames {
1415 entry: "entries".to_string(),
1416 key: "keys".to_string(),
1417 value: "values".to_string(),
1418 }),
1419 map_key_builder,
1420 map_value_builder,
1421 );
1422 let b_builder = ListBuilder::new(map_builder);
1423
1424 let inner_list_item_builder = Int32Builder::new();
1425 let inner_list_builder = ListBuilder::new(inner_list_item_builder);
1426 let c_builder = ListBuilder::new(inner_list_builder);
1427
1428 let mut top_struct_builder = {
1429 let a_struct_type =
1430 DataType::Struct(Fields::from(vec![
1431 Field::new("a1", DataType::Int32, false).with_metadata(HashMap::from(
1432 [(PARQUET_FIELD_ID_META_KEY.to_string(), "2".to_string())],
1433 )),
1434 Field::new("a2", DataType::Int32, false).with_metadata(HashMap::from(
1435 [(PARQUET_FIELD_ID_META_KEY.to_string(), "3".to_string())],
1436 )),
1437 ]));
1438 let a_type =
1439 DataType::List(Arc::new(Field::new("item", a_struct_type.clone(), true)));
1440
1441 let b_map_entry_struct = Field::new(
1442 "entries",
1443 DataType::Struct(Fields::from(vec![
1444 Field::new("keys", DataType::Int32, false),
1445 Field::new("values", DataType::Int32, true),
1446 ])),
1447 false,
1448 );
1449 let b_map_type =
1450 DataType::Map(Arc::new(b_map_entry_struct), false);
1451 let b_type =
1452 DataType::List(Arc::new(Field::new("item", b_map_type.clone(), true)));
1453
1454 let c_inner_list_type =
1455 DataType::List(Arc::new(Field::new("item", DataType::Int32, true)));
1456 let c_type = DataType::List(Arc::new(Field::new(
1457 "item",
1458 c_inner_list_type.clone(),
1459 true,
1460 )));
1461 StructBuilder::new(
1462 Fields::from(vec![
1463 Field::new("A", a_type.clone(), false).with_metadata(HashMap::from([
1464 (PARQUET_FIELD_ID_META_KEY.to_string(), "0".to_string()),
1465 ])),
1466 Field::new("B", b_type.clone(), false).with_metadata(HashMap::from([
1467 (PARQUET_FIELD_ID_META_KEY.to_string(), "4".to_string()),
1468 ])),
1469 Field::new("C", c_type.clone(), false).with_metadata(HashMap::from([
1470 (PARQUET_FIELD_ID_META_KEY.to_string(), "8".to_string()),
1471 ])),
1472 ]),
1473 vec![
1474 Box::new(a_builder),
1475 Box::new(b_builder),
1476 Box::new(c_builder),
1477 ],
1478 )
1479 };
1480
1481 {
1484 let a_builder = top_struct_builder
1485 .field_builder::<ListBuilder<StructBuilder>>(0)
1486 .unwrap();
1487 let struct_builder = a_builder.values();
1488 struct_builder
1489 .field_builder::<Int32Builder>(0)
1490 .unwrap()
1491 .append_value(10);
1492 struct_builder
1493 .field_builder::<Int32Builder>(1)
1494 .unwrap()
1495 .append_value(20);
1496 struct_builder.append(true);
1497 let struct_builder = a_builder.values();
1498 struct_builder
1499 .field_builder::<Int32Builder>(0)
1500 .unwrap()
1501 .append_value(11);
1502 struct_builder
1503 .field_builder::<Int32Builder>(1)
1504 .unwrap()
1505 .append_value(21);
1506 struct_builder.append(true);
1507 a_builder.append(true);
1508 }
1509 {
1510 let b_builder = top_struct_builder
1511 .field_builder::<ListBuilder<MapBuilder<Int32Builder, Int32Builder>>>(1)
1512 .unwrap();
1513 let map_builder = b_builder.values();
1514 map_builder.keys().append_value(1);
1515 map_builder.values().append_value(100);
1516 map_builder.keys().append_value(3);
1517 map_builder.values().append_value(300);
1518 map_builder.append(true).unwrap();
1519
1520 map_builder.keys().append_value(2);
1521 map_builder.values().append_value(200);
1522 map_builder.append(true).unwrap();
1523
1524 b_builder.append(true);
1525 }
1526 {
1527 let c_builder = top_struct_builder
1528 .field_builder::<ListBuilder<ListBuilder<Int32Builder>>>(2)
1529 .unwrap();
1530 let inner_list_builder = c_builder.values();
1531 inner_list_builder.values().append_value(100);
1532 inner_list_builder.values().append_value(101);
1533 inner_list_builder.values().append_value(102);
1534 inner_list_builder.append(true);
1535 let inner_list_builder = c_builder.values();
1536 inner_list_builder.values().append_value(200);
1537 inner_list_builder.values().append_value(201);
1538 inner_list_builder.append(true);
1539 c_builder.append(true);
1540 }
1541 top_struct_builder.append(true);
1542
1543 {
1546 let a_builder = top_struct_builder
1547 .field_builder::<ListBuilder<StructBuilder>>(0)
1548 .unwrap();
1549 let struct_builder = a_builder.values();
1550 struct_builder
1551 .field_builder::<Int32Builder>(0)
1552 .unwrap()
1553 .append_value(12);
1554 struct_builder
1555 .field_builder::<Int32Builder>(1)
1556 .unwrap()
1557 .append_value(22);
1558 struct_builder.append(true);
1559 let struct_builder = a_builder.values();
1560 struct_builder
1561 .field_builder::<Int32Builder>(0)
1562 .unwrap()
1563 .append_value(13);
1564 struct_builder
1565 .field_builder::<Int32Builder>(1)
1566 .unwrap()
1567 .append_value(23);
1568 struct_builder.append(true);
1569 a_builder.append(true);
1570 }
1571 {
1572 let b_builder = top_struct_builder
1573 .field_builder::<ListBuilder<MapBuilder<Int32Builder, Int32Builder>>>(1)
1574 .unwrap();
1575 let map_builder = b_builder.values();
1576 map_builder.keys().append_value(3);
1577 map_builder.values().append_value(300);
1578 map_builder.append(true).unwrap();
1579
1580 b_builder.append(true);
1581 }
1582 {
1583 let c_builder = top_struct_builder
1584 .field_builder::<ListBuilder<ListBuilder<Int32Builder>>>(2)
1585 .unwrap();
1586 let inner_list_builder = c_builder.values();
1587 inner_list_builder.values().append_value(300);
1588 inner_list_builder.values().append_value(301);
1589 inner_list_builder.values().append_value(302);
1590 inner_list_builder.append(true);
1591 let inner_list_builder = c_builder.values();
1592 inner_list_builder.values().append_value(400);
1593 inner_list_builder.values().append_value(401);
1594 inner_list_builder.append(true);
1595 c_builder.append(true);
1596 }
1597 top_struct_builder.append(true);
1598
1599 Arc::new(top_struct_builder.finish()) as ArrayRef
1600 };
1601
1602 let result = arrow_struct_to_literal(&struct_array, &struct_type).unwrap();
1603 assert_eq!(result, vec![
1604 Some(Literal::Struct(Struct::from_iter(vec![
1605 Some(Literal::List(vec![
1606 Some(Literal::Struct(Struct::from_iter(vec![
1607 Some(Literal::int(10)),
1608 Some(Literal::int(20)),
1609 ]))),
1610 Some(Literal::Struct(Struct::from_iter(vec![
1611 Some(Literal::int(11)),
1612 Some(Literal::int(21)),
1613 ]))),
1614 ])),
1615 Some(Literal::List(vec![
1616 Some(Literal::Map(Map::from_iter(vec![
1617 (Literal::int(1), Some(Literal::int(100))),
1618 (Literal::int(3), Some(Literal::int(300))),
1619 ]))),
1620 Some(Literal::Map(Map::from_iter(vec![(
1621 Literal::int(2),
1622 Some(Literal::int(200))
1623 ),]))),
1624 ])),
1625 Some(Literal::List(vec![
1626 Some(Literal::List(vec![
1627 Some(Literal::int(100)),
1628 Some(Literal::int(101)),
1629 Some(Literal::int(102)),
1630 ])),
1631 Some(Literal::List(vec![
1632 Some(Literal::int(200)),
1633 Some(Literal::int(201)),
1634 ])),
1635 ])),
1636 ]))),
1637 Some(Literal::Struct(Struct::from_iter(vec![
1638 Some(Literal::List(vec![
1639 Some(Literal::Struct(Struct::from_iter(vec![
1640 Some(Literal::int(12)),
1641 Some(Literal::int(22)),
1642 ]))),
1643 Some(Literal::Struct(Struct::from_iter(vec![
1644 Some(Literal::int(13)),
1645 Some(Literal::int(23)),
1646 ]))),
1647 ])),
1648 Some(Literal::List(vec![Some(Literal::Map(Map::from_iter(
1649 vec![(Literal::int(3), Some(Literal::int(300))),]
1650 ))),])),
1651 Some(Literal::List(vec![
1652 Some(Literal::List(vec![
1653 Some(Literal::int(300)),
1654 Some(Literal::int(301)),
1655 Some(Literal::int(302)),
1656 ])),
1657 Some(Literal::List(vec![
1658 Some(Literal::int(400)),
1659 Some(Literal::int(401)),
1660 ])),
1661 ])),
1662 ]))),
1663 ]);
1664 }
1665
1666 #[test]
1667 fn test_create_decimal_array_respects_precision() {
1668 let target_precision = 18u8;
1671 let target_scale = 10i8;
1672 let target_type = DataType::Decimal128(target_precision, target_scale);
1673 let value = PrimitiveLiteral::Int128(10000000000);
1674
1675 let array = create_primitive_array_single_element(&target_type, Some(&value))
1676 .expect("Failed to create decimal array");
1677
1678 match array.data_type() {
1679 DataType::Decimal128(precision, scale) => {
1680 assert_eq!(*precision, target_precision);
1681 assert_eq!(*scale, target_scale);
1682 }
1683 other => panic!("Expected Decimal128, got {other:?}"),
1684 }
1685 }
1686
1687 #[test]
1688 fn test_create_null_arrays_preserve_type_and_length() {
1689 let data_types = [
1690 DataType::Decimal128(10, 2),
1691 DataType::Timestamp(TimeUnit::Microsecond, Some("+00:00".into())),
1692 DataType::Struct(
1693 vec![
1694 Field::new("a", DataType::Utf8, true),
1695 Field::new(
1696 "ys",
1697 DataType::List(Arc::new(Field::new("element", DataType::Int64, true))),
1698 true,
1699 ),
1700 ]
1701 .into(),
1702 ),
1703 DataType::Null,
1704 ];
1705
1706 for data_type in data_types {
1708 let single = create_primitive_array_single_element(&data_type, None)
1709 .unwrap_or_else(|err| panic!("single, type={data_type:?}: {err}"));
1710 assert_eq!(single.data_type(), &data_type, "single, type={data_type:?}");
1711 assert_eq!(single.len(), 1, "single, type={data_type:?}");
1712 if data_type != DataType::Null {
1713 assert_eq!(single.null_count(), 1, "single, type={data_type:?}");
1714 }
1715 assert_eq!(single.logical_null_count(), 1, "single, type={data_type:?}");
1716
1717 for num_rows in [0, 1, 3] {
1718 let repeated = create_primitive_array_repeated(&data_type, None, num_rows)
1719 .unwrap_or_else(|err| {
1720 panic!("repeated, type={data_type:?}, rows={num_rows}: {err}")
1721 });
1722 assert_eq!(
1723 repeated.data_type(),
1724 &data_type,
1725 "repeated, type={data_type:?}, rows={num_rows}"
1726 );
1727 assert_eq!(
1728 repeated.len(),
1729 num_rows,
1730 "repeated, type={data_type:?}, rows={num_rows}"
1731 );
1732 if data_type != DataType::Null {
1733 assert_eq!(
1734 repeated.null_count(),
1735 num_rows,
1736 "repeated, type={data_type:?}, rows={num_rows}"
1737 );
1738 }
1739 assert_eq!(
1740 repeated.logical_null_count(),
1741 num_rows,
1742 "repeated, type={data_type:?}, rows={num_rows}"
1743 );
1744 }
1745 }
1746 }
1747
1748 #[test]
1749 fn test_create_null_array_rejects_non_null_literal() {
1750 let literal = Some(PrimitiveLiteral::Int(1));
1751
1752 assert!(create_primitive_array_single_element(&DataType::Null, literal.as_ref()).is_err());
1753 assert!(create_primitive_array_repeated(&DataType::Null, literal.as_ref(), 2).is_err());
1754 assert_eq!(
1755 create_primitive_array_single_element(&DataType::Null, None)
1756 .unwrap()
1757 .len(),
1758 1
1759 );
1760 assert_eq!(
1761 create_primitive_array_repeated(&DataType::Null, None, 2)
1762 .unwrap()
1763 .len(),
1764 2
1765 );
1766 }
1767
1768 #[test]
1769 fn test_create_decimal_array_repeated_respects_precision() {
1770 let target_precision = 18u8;
1772 let target_scale = 10i8;
1773 let target_type = DataType::Decimal128(target_precision, target_scale);
1774 let value = PrimitiveLiteral::Int128(10000000000);
1775 let num_rows = 5;
1776
1777 let array = create_primitive_array_repeated(&target_type, Some(&value), num_rows)
1778 .expect("Failed to create repeated decimal array");
1779
1780 match array.data_type() {
1781 DataType::Decimal128(precision, scale) => {
1782 assert_eq!(*precision, target_precision);
1783 assert_eq!(*scale, target_scale);
1784 }
1785 other => panic!("Expected Decimal128, got {other:?}"),
1786 }
1787
1788 assert_eq!(array.len(), num_rows);
1789 }
1790
1791 #[test]
1792 fn test_create_timestamp_microsecond_array_repeated() {
1793 let target_type = DataType::Timestamp(TimeUnit::Microsecond, None);
1794 let value = PrimitiveLiteral::Long(1_740_600_000_000_000);
1795 let num_rows = 3;
1796
1797 let array = create_primitive_array_repeated(&target_type, Some(&value), num_rows)
1798 .expect("Failed to create repeated timestamp microsecond array");
1799
1800 assert_eq!(array.data_type(), &target_type);
1801 assert_eq!(array.len(), num_rows);
1802 }
1803
1804 #[test]
1805 fn test_create_timestamp_microsecond_with_timezone_array_repeated() {
1806 let target_type = DataType::Timestamp(TimeUnit::Microsecond, Some("UTC".into()));
1807 let value = PrimitiveLiteral::Long(1_740_600_000_000_000);
1808 let num_rows = 2;
1809
1810 let array = create_primitive_array_repeated(&target_type, Some(&value), num_rows)
1811 .expect("Failed to create repeated timestamp microsecond array with timezone");
1812
1813 assert_eq!(array.data_type(), &target_type);
1814 assert_eq!(array.len(), num_rows);
1815 }
1816
1817 #[test]
1818 fn test_create_string_and_binary_arrays_repeated() {
1819 let text = "partition-value-2026";
1820 let bytes: Vec<u8> = vec![0xDE, 0xAD, 0xBE, 0xEF];
1821 let num_rows = 4;
1822
1823 let utf8 = create_primitive_array_repeated(
1824 &DataType::Utf8,
1825 Some(&PrimitiveLiteral::String(text.to_string())),
1826 num_rows,
1827 )
1828 .unwrap();
1829 let utf8 = utf8.as_any().downcast_ref::<StringArray>().unwrap();
1830 assert_eq!(utf8.len(), num_rows);
1831 assert!((0..num_rows).all(|i| utf8.value(i) == text));
1832
1833 let binary = create_primitive_array_repeated(
1834 &DataType::Binary,
1835 Some(&PrimitiveLiteral::Binary(bytes.clone())),
1836 num_rows,
1837 )
1838 .unwrap();
1839 let binary = binary.as_any().downcast_ref::<BinaryArray>().unwrap();
1840 assert_eq!(binary.len(), num_rows);
1841 assert!((0..num_rows).all(|i| binary.value(i) == bytes.as_slice()));
1842
1843 let large = create_primitive_array_repeated(
1844 &DataType::LargeBinary,
1845 Some(&PrimitiveLiteral::Binary(bytes.clone())),
1846 num_rows,
1847 )
1848 .unwrap();
1849 let large = large.as_any().downcast_ref::<LargeBinaryArray>().unwrap();
1850 assert_eq!(large.len(), num_rows);
1851 assert!((0..num_rows).all(|i| large.value(i) == bytes.as_slice()));
1852
1853 let fixed = create_primitive_array_repeated(
1854 &DataType::FixedSizeBinary(bytes.len() as i32),
1855 Some(&PrimitiveLiteral::Binary(bytes.clone())),
1856 num_rows,
1857 )
1858 .unwrap();
1859 let fixed = fixed
1860 .as_any()
1861 .downcast_ref::<FixedSizeBinaryArray>()
1862 .unwrap();
1863 assert_eq!(fixed.len(), num_rows);
1864 assert!((0..num_rows).all(|i| fixed.value(i) == bytes.as_slice()));
1865 }
1866
1867 #[test]
1868 fn test_create_boolean_array_repeated() {
1869 let num_rows = 4;
1870
1871 for value in [true, false] {
1872 let array = create_primitive_array_repeated(
1873 &DataType::Boolean,
1874 Some(&PrimitiveLiteral::Boolean(value)),
1875 num_rows,
1876 )
1877 .unwrap();
1878 let array = array.as_any().downcast_ref::<BooleanArray>().unwrap();
1879 assert_eq!(array.len(), num_rows);
1880 assert_eq!(array.null_count(), 0);
1881 assert!((0..num_rows).all(|i| array.value(i) == value));
1882 }
1883
1884 let empty = create_primitive_array_repeated(
1886 &DataType::Boolean,
1887 Some(&PrimitiveLiteral::Boolean(true)),
1888 0,
1889 )
1890 .unwrap();
1891 assert_eq!(empty.len(), 0);
1892 assert_eq!(empty.null_count(), 0);
1893 }
1894
1895 #[test]
1896 fn test_create_string_array_repeated_empty() {
1897 let array = create_primitive_array_repeated(
1899 &DataType::Utf8,
1900 Some(&PrimitiveLiteral::String("x".to_string())),
1901 0,
1902 )
1903 .unwrap();
1904 assert_eq!(array.len(), 0);
1905 }
1906
1907 #[test]
1908 fn test_create_fixed_size_binary_repeated_wrong_width_errors() {
1909 let err = create_primitive_array_repeated(
1912 &DataType::FixedSizeBinary(4),
1913 Some(&PrimitiveLiteral::Binary(vec![0x01, 0x02, 0x03])),
1914 2,
1915 )
1916 .unwrap_err();
1917 assert!(err.to_string().contains("does not match declared width"));
1918 }
1919
1920 #[test]
1921 fn test_create_fixed_size_binary_repeated_empty_errors() {
1922 let result = create_primitive_array_repeated(
1926 &DataType::FixedSizeBinary(4),
1927 Some(&PrimitiveLiteral::Binary(vec![0xDE, 0xAD, 0xBE, 0xEF])),
1928 0,
1929 );
1930 assert!(result.is_err());
1931 }
1932}