1use std::sync::Arc;
19
20use arrow_array::{Array, ArrayRef, RecordBatch, StructArray, make_array};
21use arrow_buffer::NullBuffer;
22use arrow_cast::cast;
23use arrow_schema::{DataType, Field, FieldRef, Fields, Schema, SchemaRef};
24
25use crate::arrow::schema::{schema_to_arrow_schema, try_get_field_id_from_metadata};
26use crate::error::{Result, invalid_data};
27use crate::spec::Schema as IcebergSchema;
28use crate::{Error, ErrorKind};
29
30#[derive(Clone, Debug, PartialEq, Eq)]
32pub struct RecordBatchProjector {
33 field_indices: Vec<Vec<usize>>,
37 projected_schema: SchemaRef,
39}
40
41impl RecordBatchProjector {
42 pub(crate) fn new<F1, F2>(
49 original_schema: SchemaRef,
50 field_ids: &[i32],
51 field_id_fetch_func: F1,
52 searchable_field_func: F2,
53 ) -> Result<Self>
54 where
55 F1: Fn(&Field) -> Result<Option<i64>>,
56 F2: Fn(&Field) -> bool,
57 {
58 let mut field_indices = Vec::with_capacity(field_ids.len());
59 let mut fields = Vec::with_capacity(field_ids.len());
60 for &id in field_ids {
61 let mut field_index = vec![];
62 let field = Self::fetch_field_index(
63 original_schema.fields(),
64 &mut field_index,
65 id as i64,
66 &field_id_fetch_func,
67 &searchable_field_func,
68 )?
69 .ok_or_else(|| {
70 Error::new(ErrorKind::Unexpected, "Field not found")
71 .with_context("field_id", id.to_string())
72 })?;
73 fields.push(field.clone());
74 field_indices.push(field_index);
75 }
76 let delete_arrow_schema = Arc::new(Schema::new(fields));
77 Ok(Self {
78 field_indices,
79 projected_schema: delete_arrow_schema,
80 })
81 }
82
83 pub fn from_iceberg_schema(
92 iceberg_schema: Arc<IcebergSchema>,
93 target_field_ids: &[i32],
94 ) -> Result<Self> {
95 let arrow_schema_with_ids = Arc::new(schema_to_arrow_schema(&iceberg_schema)?);
96
97 let field_id_fetch_func = |field: &Field| -> Result<Option<i64>> {
98 Ok(try_get_field_id_from_metadata(field)?.map(|id| id as i64))
99 };
100
101 let searchable_field_func = |_field: &Field| -> bool { true };
102
103 Self::new(
104 arrow_schema_with_ids,
105 target_field_ids,
106 field_id_fetch_func,
107 searchable_field_func,
108 )
109 }
110
111 fn fetch_field_index<F1, F2>(
112 fields: &Fields,
113 index_vec: &mut Vec<usize>,
114 target_field_id: i64,
115 field_id_fetch_func: &F1,
116 searchable_field_func: &F2,
117 ) -> Result<Option<FieldRef>>
118 where
119 F1: Fn(&Field) -> Result<Option<i64>>,
120 F2: Fn(&Field) -> bool,
121 {
122 for (pos, field) in fields.iter().enumerate() {
123 let id = field_id_fetch_func(field)?;
124 if let Some(id) = id
125 && target_field_id == id
126 {
127 index_vec.push(pos);
128 return Ok(Some(field.clone()));
129 }
130 if let DataType::Struct(inner) = field.data_type()
131 && searchable_field_func(field)
132 && let Some(res) = Self::fetch_field_index(
133 inner,
134 index_vec,
135 target_field_id,
136 field_id_fetch_func,
137 searchable_field_func,
138 )?
139 {
140 index_vec.push(pos);
141 return Ok(Some(res));
142 }
143 }
144 Ok(None)
145 }
146
147 pub(crate) fn projected_schema_ref(&self) -> &SchemaRef {
149 &self.projected_schema
150 }
151
152 pub(crate) fn project_batch(&self, batch: RecordBatch) -> Result<RecordBatch> {
154 RecordBatch::try_new(
155 self.projected_schema.clone(),
156 self.project_column(batch.columns())?,
157 )
158 .map_err(|err| invalid_data!("{err}"))
159 }
160
161 pub fn project_column(&self, batch: &[ArrayRef]) -> Result<Vec<ArrayRef>> {
163 let mut columns = Vec::with_capacity(self.field_indices.len());
164 for (index_vec, field) in self
165 .field_indices
166 .iter()
167 .zip(self.projected_schema.fields())
168 {
169 let column = Self::get_column_by_field_index(batch, index_vec)?;
170 let column = match (column.data_type(), field.data_type()) {
171 (DataType::LargeBinary, DataType::Binary) => cast(&column, &DataType::Binary)
172 .map_err(|err| {
173 invalid_data!("Failed to cast column {} to Binary", field.name())
174 .with_source(err)
175 })?,
176 _ => column,
177 };
178 columns.push(column);
179 }
180 Ok(columns)
181 }
182
183 fn get_column_by_field_index(batch: &[ArrayRef], field_index: &[usize]) -> Result<ArrayRef> {
184 let mut rev_iterator = field_index.iter().rev();
185 let mut array = &batch[*rev_iterator.next().unwrap()];
186 let mut ancestor_nulls = None;
187 for idx in rev_iterator {
188 let struct_array = array
189 .as_any()
190 .downcast_ref::<StructArray>()
191 .ok_or_else(|| {
192 Error::new(ErrorKind::Unexpected, "Cannot convert Array to StructArray")
193 })?;
194 ancestor_nulls = NullBuffer::union(ancestor_nulls.as_ref(), struct_array.nulls());
195 array = struct_array.column(*idx);
196 }
197 let Some(ancestor_nulls) = ancestor_nulls else {
198 return Ok(Arc::clone(array));
201 };
202 let null_buffer = NullBuffer::union(Some(&ancestor_nulls), array.logical_nulls().as_ref());
203 Ok(make_array(
204 array.to_data().into_builder().nulls(null_buffer).build()?,
205 ))
206 }
207}
208
209#[cfg(test)]
210mod test {
211 use std::sync::Arc;
212
213 use arrow_array::{Array, ArrayRef, Int32Array, RecordBatch, StringArray, StructArray};
214 use arrow_buffer::NullBuffer;
215 use arrow_schema::{DataType, Field, Fields, Schema};
216
217 use crate::arrow::record_batch_projector::RecordBatchProjector;
218 use crate::spec::{NestedField, PrimitiveType, Schema as IcebergSchema, Type};
219 use crate::{Error, ErrorKind};
220
221 fn nested_projector() -> (RecordBatchProjector, Field, Field) {
222 let leaf_field = Field::new("leaf", DataType::Int32, true);
223 let inner_field = Field::new(
224 "inner",
225 DataType::Struct(Fields::from(vec![leaf_field.clone()])),
226 true,
227 );
228 let outer_field = Field::new(
229 "outer",
230 DataType::Struct(Fields::from(vec![inner_field.clone()])),
231 true,
232 );
233 let schema = Arc::new(Schema::new(vec![outer_field]));
234 let field_id_fetch_func = |field: &Field| -> crate::Result<Option<i64>> {
235 match field.name().as_str() {
236 "outer" => Ok(Some(1)),
237 "inner" => Ok(Some(2)),
238 "leaf" => Ok(Some(3)),
239 _ => Ok(None),
240 }
241 };
242 let projector =
243 RecordBatchProjector::new(schema, &[3], field_id_fetch_func, |_| true).unwrap();
244 (projector, inner_field, leaf_field)
245 }
246
247 #[test]
248 fn test_record_batch_projector_nested_level() {
249 let inner_fields = vec![
250 Field::new("inner_field1", DataType::Int32, false),
251 Field::new("inner_field2", DataType::Utf8, false),
252 ];
253 let fields = vec![
254 Field::new("field1", DataType::Int32, false),
255 Field::new(
256 "field2",
257 DataType::Struct(Fields::from(inner_fields.clone())),
258 false,
259 ),
260 ];
261 let schema = Arc::new(Schema::new(fields));
262
263 let field_id_fetch_func = |field: &Field| match field.name().as_str() {
264 "field1" => Ok(Some(1)),
265 "field2" => Ok(Some(2)),
266 "inner_field1" => Ok(Some(3)),
267 "inner_field2" => Ok(Some(4)),
268 _ => Err(Error::new(ErrorKind::Unexpected, "Field id not found")),
269 };
270 let projector =
271 RecordBatchProjector::new(schema.clone(), &[1, 3], field_id_fetch_func, |_| true)
272 .unwrap();
273
274 assert_eq!(projector.field_indices.len(), 2);
275 assert_eq!(projector.field_indices[0], vec![0]);
276 assert_eq!(projector.field_indices[1], vec![0, 1]);
277
278 let int_array = Arc::new(Int32Array::from(vec![1, 2, 3])) as ArrayRef;
279 let inner_int_array = Arc::new(Int32Array::from(vec![4, 5, 6])) as ArrayRef;
280 let inner_string_array = Arc::new(StringArray::from(vec!["x", "y", "z"])) as ArrayRef;
281 let struct_array = Arc::new(StructArray::from(vec![
282 (
283 Arc::new(inner_fields[0].clone()),
284 inner_int_array as ArrayRef,
285 ),
286 (
287 Arc::new(inner_fields[1].clone()),
288 inner_string_array as ArrayRef,
289 ),
290 ])) as ArrayRef;
291 let batch = RecordBatch::try_new(schema, vec![int_array, struct_array]).unwrap();
292
293 let projected_batch = projector.project_batch(batch).unwrap();
294 assert_eq!(projected_batch.num_columns(), 2);
295 let projected_int_array = projected_batch
296 .column(0)
297 .as_any()
298 .downcast_ref::<Int32Array>()
299 .unwrap();
300 let projected_inner_int_array = projected_batch
301 .column(1)
302 .as_any()
303 .downcast_ref::<Int32Array>()
304 .unwrap();
305
306 assert_eq!(projected_int_array.values(), &[1, 2, 3]);
307 assert_eq!(projected_inner_int_array.values(), &[4, 5, 6]);
308 }
309
310 #[test]
311 fn test_record_batch_projector_top_level_nullable_column() {
312 let iceberg_schema = IcebergSchema::builder()
313 .with_schema_id(0)
314 .with_fields(vec![
315 NestedField::optional(1, "id", Type::Primitive(PrimitiveType::Int)).into(),
316 ])
317 .build()
318 .unwrap();
319 let projector =
320 RecordBatchProjector::from_iceberg_schema(Arc::new(iceberg_schema), &[1]).unwrap();
321 let input = Arc::new(Int32Array::from(vec![Some(10), None, Some(30)])) as ArrayRef;
322
323 let projected = projector
324 .project_column(std::slice::from_ref(&input))
325 .unwrap();
326 assert!(Arc::ptr_eq(&projected[0], &input));
327 let projected_array = projected[0].as_any().downcast_ref::<Int32Array>().unwrap();
328
329 assert_eq!(projected_array.value(0), 10);
330 assert_eq!(projected_array.null_count(), 1);
331 assert!(projected_array.is_null(1));
332 assert_eq!(projected_array.value(2), 30);
333 }
334
335 #[test]
336 fn test_record_batch_projector_nested_nullable_leaf_without_parent_nulls() {
337 let (projector, inner_field, leaf_field) = nested_projector();
338 let leaf = Arc::new(Int32Array::from(vec![Some(10), None, Some(30)])) as ArrayRef;
339 let inner = Arc::new(StructArray::new(
340 Fields::from(vec![leaf_field]),
341 vec![leaf.clone()],
342 None,
343 )) as ArrayRef;
344 let outer = Arc::new(StructArray::new(
345 Fields::from(vec![inner_field]),
346 vec![inner],
347 None,
348 )) as ArrayRef;
349
350 let projected = projector.project_column(&[outer]).unwrap();
351 assert!(Arc::ptr_eq(&projected[0], &leaf));
352 }
353
354 #[test]
355 fn test_record_batch_projector_nested_all_valid_slice_reuses_leaf() {
356 let (projector, inner_field, leaf_field) = nested_projector();
357 let leaf = Arc::new(Int32Array::from(vec![Some(0), Some(10), None, Some(30)])) as ArrayRef;
358 let inner = Arc::new(StructArray::new(
359 Fields::from(vec![leaf_field]),
360 vec![leaf],
361 None,
362 )) as ArrayRef;
363 let outer = StructArray::new(
364 Fields::from(vec![inner_field]),
365 vec![inner],
366 Some(NullBuffer::from(vec![false, true, true, true])),
367 )
368 .slice(1, 3);
369 assert_eq!(outer.null_count(), 0);
370 assert!(outer.nulls().is_some());
371
372 let sliced_inner = outer
373 .column(0)
374 .as_any()
375 .downcast_ref::<StructArray>()
376 .unwrap();
377 let sliced_leaf = sliced_inner.column(0).clone();
378 let projected = projector
379 .project_column(&[Arc::new(outer) as ArrayRef])
380 .unwrap();
381 assert!(Arc::ptr_eq(&projected[0], &sliced_leaf));
382 }
383
384 #[test]
385 fn test_record_batch_projector_propagates_nested_parent_nulls() {
386 let (projector, inner_field, leaf_field) = nested_projector();
387 let leaf = Arc::new(Int32Array::from(vec![Some(10), Some(20), Some(30), None])) as ArrayRef;
388 let inner = Arc::new(StructArray::new(
389 Fields::from(vec![leaf_field]),
390 vec![leaf],
391 Some(NullBuffer::from(vec![true, true, false, true])),
392 )) as ArrayRef;
393 let outer = Arc::new(StructArray::new(
394 Fields::from(vec![inner_field]),
395 vec![inner],
396 Some(NullBuffer::from(vec![true, false, true, true])),
397 )) as ArrayRef;
398
399 let projected = projector.project_column(&[outer]).unwrap();
400 assert_eq!(
401 projected[0].as_ref(),
402 &Int32Array::from(vec![Some(10), None, None, None])
403 );
404 }
405
406 #[test]
407 fn test_field_not_found() {
408 let inner_fields = vec![
409 Field::new("inner_field1", DataType::Int32, false),
410 Field::new("inner_field2", DataType::Utf8, false),
411 ];
412
413 let fields = vec![
414 Field::new("field1", DataType::Int32, false),
415 Field::new(
416 "field2",
417 DataType::Struct(Fields::from(inner_fields.clone())),
418 false,
419 ),
420 ];
421 let schema = Arc::new(Schema::new(fields));
422
423 let field_id_fetch_func = |field: &Field| match field.name().as_str() {
424 "field1" => Ok(Some(1)),
425 "field2" => Ok(Some(2)),
426 "inner_field1" => Ok(Some(3)),
427 "inner_field2" => Ok(Some(4)),
428 _ => Err(Error::new(ErrorKind::Unexpected, "Field id not found")),
429 };
430 let projector =
431 RecordBatchProjector::new(schema.clone(), &[1, 5], field_id_fetch_func, |_| true);
432
433 assert!(projector.is_err());
434 }
435
436 #[test]
437 fn test_field_not_reachable() {
438 let inner_fields = vec![
439 Field::new("inner_field1", DataType::Int32, false),
440 Field::new("inner_field2", DataType::Utf8, false),
441 ];
442
443 let fields = vec![
444 Field::new("field1", DataType::Int32, false),
445 Field::new(
446 "field2",
447 DataType::Struct(Fields::from(inner_fields.clone())),
448 false,
449 ),
450 ];
451 let schema = Arc::new(Schema::new(fields));
452
453 let field_id_fetch_func = |field: &Field| match field.name().as_str() {
454 "field1" => Ok(Some(1)),
455 "field2" => Ok(Some(2)),
456 "inner_field1" => Ok(Some(3)),
457 "inner_field2" => Ok(Some(4)),
458 _ => Err(Error::new(ErrorKind::Unexpected, "Field id not found")),
459 };
460 let projector =
461 RecordBatchProjector::new(schema.clone(), &[3], field_id_fetch_func, |_| false);
462 assert!(projector.is_err());
463
464 let projector =
465 RecordBatchProjector::new(schema.clone(), &[3], field_id_fetch_func, |_| true);
466 assert!(projector.is_ok());
467 }
468
469 #[test]
470 fn test_from_iceberg_schema() {
471 let iceberg_schema = IcebergSchema::builder()
472 .with_schema_id(0)
473 .with_fields(vec![
474 NestedField::required(1, "id", Type::Primitive(PrimitiveType::Int)).into(),
475 NestedField::required(2, "name", Type::Primitive(PrimitiveType::String)).into(),
476 NestedField::optional(3, "age", Type::Primitive(PrimitiveType::Int)).into(),
477 ])
478 .build()
479 .unwrap();
480
481 let projector =
482 RecordBatchProjector::from_iceberg_schema(Arc::new(iceberg_schema), &[1, 3]).unwrap();
483
484 assert_eq!(projector.field_indices.len(), 2);
485 assert_eq!(projector.projected_schema_ref().fields().len(), 2);
486 assert_eq!(projector.projected_schema_ref().field(0).name(), "id");
487 assert_eq!(projector.projected_schema_ref().field(1).name(), "age");
488 }
489}