Skip to main content

iceberg/arrow/
record_batch_projector.rs

1// Licensed to the Apache Software Foundation (ASF) under one
2// or more contributor license agreements.  See the NOTICE file
3// distributed with this work for additional information
4// regarding copyright ownership.  The ASF licenses this file
5// to you under the Apache License, Version 2.0 (the
6// "License"); you may not use this file except in compliance
7// with the License.  You may obtain a copy of the License at
8//
9//   http://www.apache.org/licenses/LICENSE-2.0
10//
11// Unless required by applicable law or agreed to in writing,
12// software distributed under the License is distributed on an
13// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14// KIND, either express or implied.  See the License for the
15// specific language governing permissions and limitations
16// under the License.
17
18use std::sync::Arc;
19
20use arrow_array::{Array, ArrayRef, RecordBatch, StructArray, make_array};
21use arrow_buffer::NullBuffer;
22use arrow_cast::cast;
23use arrow_schema::{DataType, Field, FieldRef, Fields, Schema, SchemaRef};
24
25use crate::arrow::schema::{schema_to_arrow_schema, try_get_field_id_from_metadata};
26use crate::error::{Result, invalid_data};
27use crate::spec::Schema as IcebergSchema;
28use crate::{Error, ErrorKind};
29
30/// Help to project specific field from `RecordBatch`` according to the fields id.
31#[derive(Clone, Debug, PartialEq, Eq)]
32pub struct RecordBatchProjector {
33    // A vector of vectors, where each inner vector represents the index path to access a specific field in a nested structure.
34    // E.g. [[0], [1, 2]] means the first field is accessed directly from the first column,
35    // while the second field is accessed from the second column and then from its third subcolumn (second column must be a struct column).
36    field_indices: Vec<Vec<usize>>,
37    // The schema reference after projection. This schema is derived from the original schema based on the given field IDs.
38    projected_schema: SchemaRef,
39}
40
41impl RecordBatchProjector {
42    /// Init ArrowFieldProjector
43    ///
44    /// This function will iterate through the field and fetch the field from the original schema according to the field ids.
45    /// The function to fetch the field id from the field is provided by `field_id_fetch_func`, return None if the field need to be skipped.
46    /// This function will iterate through the nested fields if the field is a struct, `searchable_field_func` can be used to control whether
47    /// iterate into the nested fields.
48    pub(crate) fn new<F1, F2>(
49        original_schema: SchemaRef,
50        field_ids: &[i32],
51        field_id_fetch_func: F1,
52        searchable_field_func: F2,
53    ) -> Result<Self>
54    where
55        F1: Fn(&Field) -> Result<Option<i64>>,
56        F2: Fn(&Field) -> bool,
57    {
58        let mut field_indices = Vec::with_capacity(field_ids.len());
59        let mut fields = Vec::with_capacity(field_ids.len());
60        for &id in field_ids {
61            let mut field_index = vec![];
62            let field = Self::fetch_field_index(
63                original_schema.fields(),
64                &mut field_index,
65                id as i64,
66                &field_id_fetch_func,
67                &searchable_field_func,
68            )?
69            .ok_or_else(|| {
70                Error::new(ErrorKind::Unexpected, "Field not found")
71                    .with_context("field_id", id.to_string())
72            })?;
73            fields.push(field.clone());
74            field_indices.push(field_index);
75        }
76        let delete_arrow_schema = Arc::new(Schema::new(fields));
77        Ok(Self {
78            field_indices,
79            projected_schema: delete_arrow_schema,
80        })
81    }
82
83    /// Create RecordBatchProjector using Iceberg schema.
84    ///
85    /// This constructor converts the Iceberg schema to Arrow schema with field ID metadata,
86    /// then uses the standard field ID lookup for projection.
87    ///
88    /// # Arguments
89    /// * `iceberg_schema` - The Iceberg schema for field ID mapping  
90    /// * `target_field_ids` - The field IDs to project
91    pub fn from_iceberg_schema(
92        iceberg_schema: Arc<IcebergSchema>,
93        target_field_ids: &[i32],
94    ) -> Result<Self> {
95        let arrow_schema_with_ids = Arc::new(schema_to_arrow_schema(&iceberg_schema)?);
96
97        let field_id_fetch_func = |field: &Field| -> Result<Option<i64>> {
98            Ok(try_get_field_id_from_metadata(field)?.map(|id| id as i64))
99        };
100
101        let searchable_field_func = |_field: &Field| -> bool { true };
102
103        Self::new(
104            arrow_schema_with_ids,
105            target_field_ids,
106            field_id_fetch_func,
107            searchable_field_func,
108        )
109    }
110
111    fn fetch_field_index<F1, F2>(
112        fields: &Fields,
113        index_vec: &mut Vec<usize>,
114        target_field_id: i64,
115        field_id_fetch_func: &F1,
116        searchable_field_func: &F2,
117    ) -> Result<Option<FieldRef>>
118    where
119        F1: Fn(&Field) -> Result<Option<i64>>,
120        F2: Fn(&Field) -> bool,
121    {
122        for (pos, field) in fields.iter().enumerate() {
123            let id = field_id_fetch_func(field)?;
124            if let Some(id) = id
125                && target_field_id == id
126            {
127                index_vec.push(pos);
128                return Ok(Some(field.clone()));
129            }
130            if let DataType::Struct(inner) = field.data_type()
131                && searchable_field_func(field)
132                && let Some(res) = Self::fetch_field_index(
133                    inner,
134                    index_vec,
135                    target_field_id,
136                    field_id_fetch_func,
137                    searchable_field_func,
138                )?
139            {
140                index_vec.push(pos);
141                return Ok(Some(res));
142            }
143        }
144        Ok(None)
145    }
146
147    /// Return the reference of projected schema
148    pub(crate) fn projected_schema_ref(&self) -> &SchemaRef {
149        &self.projected_schema
150    }
151
152    /// Do projection with record batch
153    pub(crate) fn project_batch(&self, batch: RecordBatch) -> Result<RecordBatch> {
154        RecordBatch::try_new(
155            self.projected_schema.clone(),
156            self.project_column(batch.columns())?,
157        )
158        .map_err(|err| invalid_data!("{err}"))
159    }
160
161    /// Do projection with columns
162    pub fn project_column(&self, batch: &[ArrayRef]) -> Result<Vec<ArrayRef>> {
163        let mut columns = Vec::with_capacity(self.field_indices.len());
164        for (index_vec, field) in self
165            .field_indices
166            .iter()
167            .zip(self.projected_schema.fields())
168        {
169            let column = Self::get_column_by_field_index(batch, index_vec)?;
170            let column = match (column.data_type(), field.data_type()) {
171                (DataType::LargeBinary, DataType::Binary) => cast(&column, &DataType::Binary)
172                    .map_err(|err| {
173                        invalid_data!("Failed to cast column {} to Binary", field.name())
174                            .with_source(err)
175                    })?,
176                _ => column,
177            };
178            columns.push(column);
179        }
180        Ok(columns)
181    }
182
183    fn get_column_by_field_index(batch: &[ArrayRef], field_index: &[usize]) -> Result<ArrayRef> {
184        let mut rev_iterator = field_index.iter().rev();
185        let mut array = &batch[*rev_iterator.next().unwrap()];
186        let mut ancestor_nulls = None;
187        for idx in rev_iterator {
188            let struct_array = array
189                .as_any()
190                .downcast_ref::<StructArray>()
191                .ok_or_else(|| {
192                    Error::new(ErrorKind::Unexpected, "Cannot convert Array to StructArray")
193                })?;
194            ancestor_nulls = NullBuffer::union(ancestor_nulls.as_ref(), struct_array.nulls());
195            array = struct_array.column(*idx);
196        }
197        let Some(ancestor_nulls) = ancestor_nulls else {
198            // Iceberg leaf types handled here have logical_nulls() == nulls(), so no null
199            // materialization is needed when no ancestor contributes nulls.
200            return Ok(Arc::clone(array));
201        };
202        let null_buffer = NullBuffer::union(Some(&ancestor_nulls), array.logical_nulls().as_ref());
203        Ok(make_array(
204            array.to_data().into_builder().nulls(null_buffer).build()?,
205        ))
206    }
207}
208
209#[cfg(test)]
210mod test {
211    use std::sync::Arc;
212
213    use arrow_array::{Array, ArrayRef, Int32Array, RecordBatch, StringArray, StructArray};
214    use arrow_buffer::NullBuffer;
215    use arrow_schema::{DataType, Field, Fields, Schema};
216
217    use crate::arrow::record_batch_projector::RecordBatchProjector;
218    use crate::spec::{NestedField, PrimitiveType, Schema as IcebergSchema, Type};
219    use crate::{Error, ErrorKind};
220
221    fn nested_projector() -> (RecordBatchProjector, Field, Field) {
222        let leaf_field = Field::new("leaf", DataType::Int32, true);
223        let inner_field = Field::new(
224            "inner",
225            DataType::Struct(Fields::from(vec![leaf_field.clone()])),
226            true,
227        );
228        let outer_field = Field::new(
229            "outer",
230            DataType::Struct(Fields::from(vec![inner_field.clone()])),
231            true,
232        );
233        let schema = Arc::new(Schema::new(vec![outer_field]));
234        let field_id_fetch_func = |field: &Field| -> crate::Result<Option<i64>> {
235            match field.name().as_str() {
236                "outer" => Ok(Some(1)),
237                "inner" => Ok(Some(2)),
238                "leaf" => Ok(Some(3)),
239                _ => Ok(None),
240            }
241        };
242        let projector =
243            RecordBatchProjector::new(schema, &[3], field_id_fetch_func, |_| true).unwrap();
244        (projector, inner_field, leaf_field)
245    }
246
247    #[test]
248    fn test_record_batch_projector_nested_level() {
249        let inner_fields = vec![
250            Field::new("inner_field1", DataType::Int32, false),
251            Field::new("inner_field2", DataType::Utf8, false),
252        ];
253        let fields = vec![
254            Field::new("field1", DataType::Int32, false),
255            Field::new(
256                "field2",
257                DataType::Struct(Fields::from(inner_fields.clone())),
258                false,
259            ),
260        ];
261        let schema = Arc::new(Schema::new(fields));
262
263        let field_id_fetch_func = |field: &Field| match field.name().as_str() {
264            "field1" => Ok(Some(1)),
265            "field2" => Ok(Some(2)),
266            "inner_field1" => Ok(Some(3)),
267            "inner_field2" => Ok(Some(4)),
268            _ => Err(Error::new(ErrorKind::Unexpected, "Field id not found")),
269        };
270        let projector =
271            RecordBatchProjector::new(schema.clone(), &[1, 3], field_id_fetch_func, |_| true)
272                .unwrap();
273
274        assert_eq!(projector.field_indices.len(), 2);
275        assert_eq!(projector.field_indices[0], vec![0]);
276        assert_eq!(projector.field_indices[1], vec![0, 1]);
277
278        let int_array = Arc::new(Int32Array::from(vec![1, 2, 3])) as ArrayRef;
279        let inner_int_array = Arc::new(Int32Array::from(vec![4, 5, 6])) as ArrayRef;
280        let inner_string_array = Arc::new(StringArray::from(vec!["x", "y", "z"])) as ArrayRef;
281        let struct_array = Arc::new(StructArray::from(vec![
282            (
283                Arc::new(inner_fields[0].clone()),
284                inner_int_array as ArrayRef,
285            ),
286            (
287                Arc::new(inner_fields[1].clone()),
288                inner_string_array as ArrayRef,
289            ),
290        ])) as ArrayRef;
291        let batch = RecordBatch::try_new(schema, vec![int_array, struct_array]).unwrap();
292
293        let projected_batch = projector.project_batch(batch).unwrap();
294        assert_eq!(projected_batch.num_columns(), 2);
295        let projected_int_array = projected_batch
296            .column(0)
297            .as_any()
298            .downcast_ref::<Int32Array>()
299            .unwrap();
300        let projected_inner_int_array = projected_batch
301            .column(1)
302            .as_any()
303            .downcast_ref::<Int32Array>()
304            .unwrap();
305
306        assert_eq!(projected_int_array.values(), &[1, 2, 3]);
307        assert_eq!(projected_inner_int_array.values(), &[4, 5, 6]);
308    }
309
310    #[test]
311    fn test_record_batch_projector_top_level_nullable_column() {
312        let iceberg_schema = IcebergSchema::builder()
313            .with_schema_id(0)
314            .with_fields(vec![
315                NestedField::optional(1, "id", Type::Primitive(PrimitiveType::Int)).into(),
316            ])
317            .build()
318            .unwrap();
319        let projector =
320            RecordBatchProjector::from_iceberg_schema(Arc::new(iceberg_schema), &[1]).unwrap();
321        let input = Arc::new(Int32Array::from(vec![Some(10), None, Some(30)])) as ArrayRef;
322
323        let projected = projector
324            .project_column(std::slice::from_ref(&input))
325            .unwrap();
326        assert!(Arc::ptr_eq(&projected[0], &input));
327        let projected_array = projected[0].as_any().downcast_ref::<Int32Array>().unwrap();
328
329        assert_eq!(projected_array.value(0), 10);
330        assert_eq!(projected_array.null_count(), 1);
331        assert!(projected_array.is_null(1));
332        assert_eq!(projected_array.value(2), 30);
333    }
334
335    #[test]
336    fn test_record_batch_projector_nested_nullable_leaf_without_parent_nulls() {
337        let (projector, inner_field, leaf_field) = nested_projector();
338        let leaf = Arc::new(Int32Array::from(vec![Some(10), None, Some(30)])) as ArrayRef;
339        let inner = Arc::new(StructArray::new(
340            Fields::from(vec![leaf_field]),
341            vec![leaf.clone()],
342            None,
343        )) as ArrayRef;
344        let outer = Arc::new(StructArray::new(
345            Fields::from(vec![inner_field]),
346            vec![inner],
347            None,
348        )) as ArrayRef;
349
350        let projected = projector.project_column(&[outer]).unwrap();
351        assert!(Arc::ptr_eq(&projected[0], &leaf));
352    }
353
354    #[test]
355    fn test_record_batch_projector_nested_all_valid_slice_reuses_leaf() {
356        let (projector, inner_field, leaf_field) = nested_projector();
357        let leaf = Arc::new(Int32Array::from(vec![Some(0), Some(10), None, Some(30)])) as ArrayRef;
358        let inner = Arc::new(StructArray::new(
359            Fields::from(vec![leaf_field]),
360            vec![leaf],
361            None,
362        )) as ArrayRef;
363        let outer = StructArray::new(
364            Fields::from(vec![inner_field]),
365            vec![inner],
366            Some(NullBuffer::from(vec![false, true, true, true])),
367        )
368        .slice(1, 3);
369        assert_eq!(outer.null_count(), 0);
370        assert!(outer.nulls().is_some());
371
372        let sliced_inner = outer
373            .column(0)
374            .as_any()
375            .downcast_ref::<StructArray>()
376            .unwrap();
377        let sliced_leaf = sliced_inner.column(0).clone();
378        let projected = projector
379            .project_column(&[Arc::new(outer) as ArrayRef])
380            .unwrap();
381        assert!(Arc::ptr_eq(&projected[0], &sliced_leaf));
382    }
383
384    #[test]
385    fn test_record_batch_projector_propagates_nested_parent_nulls() {
386        let (projector, inner_field, leaf_field) = nested_projector();
387        let leaf = Arc::new(Int32Array::from(vec![Some(10), Some(20), Some(30), None])) as ArrayRef;
388        let inner = Arc::new(StructArray::new(
389            Fields::from(vec![leaf_field]),
390            vec![leaf],
391            Some(NullBuffer::from(vec![true, true, false, true])),
392        )) as ArrayRef;
393        let outer = Arc::new(StructArray::new(
394            Fields::from(vec![inner_field]),
395            vec![inner],
396            Some(NullBuffer::from(vec![true, false, true, true])),
397        )) as ArrayRef;
398
399        let projected = projector.project_column(&[outer]).unwrap();
400        assert_eq!(
401            projected[0].as_ref(),
402            &Int32Array::from(vec![Some(10), None, None, None])
403        );
404    }
405
406    #[test]
407    fn test_field_not_found() {
408        let inner_fields = vec![
409            Field::new("inner_field1", DataType::Int32, false),
410            Field::new("inner_field2", DataType::Utf8, false),
411        ];
412
413        let fields = vec![
414            Field::new("field1", DataType::Int32, false),
415            Field::new(
416                "field2",
417                DataType::Struct(Fields::from(inner_fields.clone())),
418                false,
419            ),
420        ];
421        let schema = Arc::new(Schema::new(fields));
422
423        let field_id_fetch_func = |field: &Field| match field.name().as_str() {
424            "field1" => Ok(Some(1)),
425            "field2" => Ok(Some(2)),
426            "inner_field1" => Ok(Some(3)),
427            "inner_field2" => Ok(Some(4)),
428            _ => Err(Error::new(ErrorKind::Unexpected, "Field id not found")),
429        };
430        let projector =
431            RecordBatchProjector::new(schema.clone(), &[1, 5], field_id_fetch_func, |_| true);
432
433        assert!(projector.is_err());
434    }
435
436    #[test]
437    fn test_field_not_reachable() {
438        let inner_fields = vec![
439            Field::new("inner_field1", DataType::Int32, false),
440            Field::new("inner_field2", DataType::Utf8, false),
441        ];
442
443        let fields = vec![
444            Field::new("field1", DataType::Int32, false),
445            Field::new(
446                "field2",
447                DataType::Struct(Fields::from(inner_fields.clone())),
448                false,
449            ),
450        ];
451        let schema = Arc::new(Schema::new(fields));
452
453        let field_id_fetch_func = |field: &Field| match field.name().as_str() {
454            "field1" => Ok(Some(1)),
455            "field2" => Ok(Some(2)),
456            "inner_field1" => Ok(Some(3)),
457            "inner_field2" => Ok(Some(4)),
458            _ => Err(Error::new(ErrorKind::Unexpected, "Field id not found")),
459        };
460        let projector =
461            RecordBatchProjector::new(schema.clone(), &[3], field_id_fetch_func, |_| false);
462        assert!(projector.is_err());
463
464        let projector =
465            RecordBatchProjector::new(schema.clone(), &[3], field_id_fetch_func, |_| true);
466        assert!(projector.is_ok());
467    }
468
469    #[test]
470    fn test_from_iceberg_schema() {
471        let iceberg_schema = IcebergSchema::builder()
472            .with_schema_id(0)
473            .with_fields(vec![
474                NestedField::required(1, "id", Type::Primitive(PrimitiveType::Int)).into(),
475                NestedField::required(2, "name", Type::Primitive(PrimitiveType::String)).into(),
476                NestedField::optional(3, "age", Type::Primitive(PrimitiveType::Int)).into(),
477            ])
478            .build()
479            .unwrap();
480
481        let projector =
482            RecordBatchProjector::from_iceberg_schema(Arc::new(iceberg_schema), &[1, 3]).unwrap();
483
484        assert_eq!(projector.field_indices.len(), 2);
485        assert_eq!(projector.projected_schema_ref().fields().len(), 2);
486        assert_eq!(projector.projected_schema_ref().field(0).name(), "id");
487        assert_eq!(projector.projected_schema_ref().field(1).name(), "age");
488    }
489}