Skip to main content

iceberg/arrow/
delete_file_loader.rs

1// Licensed to the Apache Software Foundation (ASF) under one
2// or more contributor license agreements.  See the NOTICE file
3// distributed with this work for additional information
4// regarding copyright ownership.  The ASF licenses this file
5// to you under the Apache License, Version 2.0 (the
6// "License"); you may not use this file except in compliance
7// with the License.  You may obtain a copy of the License at
8//
9//   http://www.apache.org/licenses/LICENSE-2.0
10//
11// Unless required by applicable law or agreed to in writing,
12// software distributed under the License is distributed on an
13// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14// KIND, either express or implied.  See the License for the
15// specific language governing permissions and limitations
16// under the License.
17
18use std::collections::{HashMap, HashSet};
19use std::sync::Arc;
20
21use arrow_array::{ArrayRef, RecordBatch, RecordBatchOptions, StructArray};
22use arrow_schema::{FieldRef, Fields, Schema as ArrowSchema, SchemaRef as ArrowSchemaRef};
23use futures::{StreamExt, TryStreamExt};
24use parquet::arrow::{PARQUET_FIELD_ID_META_KEY, ParquetRecordBatchStreamBuilder};
25
26use crate::arrow::ArrowReader;
27use crate::arrow::reader::ParquetReadOptions;
28use crate::arrow::record_batch_transformer::RecordBatchTransformerBuilder;
29use crate::arrow::scan_metrics::ScanMetrics;
30use crate::arrow::schema::try_get_field_id_from_metadata;
31use crate::io::FileIO;
32use crate::scan::{ArrowRecordBatchStream, FileScanTaskDeleteFile};
33use crate::spec::{Schema, SchemaRef, prune_columns};
34use crate::{Error, ErrorKind, Result};
35
36/// Delete File Loader
37#[allow(unused)]
38#[async_trait::async_trait]
39pub trait DeleteFileLoader {
40    /// Read the delete file referred to in the task
41    ///
42    /// Returns the contents of the delete file as a RecordBatch stream. Applies schema evolution.
43    async fn read_delete_file(
44        &self,
45        task: &FileScanTaskDeleteFile,
46        schema: SchemaRef,
47    ) -> Result<ArrowRecordBatchStream>;
48}
49
50#[derive(Clone, Debug)]
51pub(crate) struct BasicDeleteFileLoader {
52    file_io: FileIO,
53    scan_metrics: ScanMetrics,
54}
55
56#[allow(unused_variables)]
57impl BasicDeleteFileLoader {
58    pub fn new(file_io: FileIO, scan_metrics: ScanMetrics) -> Self {
59        BasicDeleteFileLoader {
60            file_io,
61            scan_metrics,
62        }
63    }
64
65    pub(crate) fn file_io(&self) -> &FileIO {
66        &self.file_io
67    }
68
69    /// Loads a RecordBatchStream for a given datafile.
70    pub(crate) async fn parquet_to_batch_stream(
71        &self,
72        data_file_path: &str,
73        file_size_in_bytes: Option<u64>,
74        key_metadata: Option<&[u8]>,
75    ) -> Result<ArrowRecordBatchStream> {
76        /*
77           Essentially a super-cut-down ArrowReader. We can't use ArrowReader directly
78           as that introduces a circular dependency.
79        */
80        // `None` means the task did not carry a size (see `FileScanTaskDeleteFile`).
81        // Resolve it when the file is first loaded.
82        let file_size_in_bytes = match file_size_in_bytes {
83            Some(file_size_in_bytes) => file_size_in_bytes,
84            None => {
85                self.file_io
86                    .new_input(data_file_path)?
87                    .metadata()
88                    .await
89                    .map_err(|error| {
90                        // Keep the storage error's kind and retryability.
91                        Error::new(
92                            error.kind(),
93                            format!("Failed to stat delete file '{data_file_path}'"),
94                        )
95                        .with_retryable(error.retryable())
96                        .with_source(error)
97                    })?
98                    .size
99            }
100        };
101        let parquet_read_options = ParquetReadOptions::builder().build();
102
103        let (parquet_file_reader, arrow_metadata) = ArrowReader::open_parquet_file(
104            data_file_path,
105            &self.file_io,
106            file_size_in_bytes,
107            parquet_read_options,
108            self.scan_metrics.bytes_read_counter(),
109            key_metadata,
110        )
111        .await?;
112
113        let record_batch_stream =
114            ParquetRecordBatchStreamBuilder::new_with_metadata(parquet_file_reader, arrow_metadata)
115                .build()?
116                .map_err(|e| Error::new(ErrorKind::Unexpected, format!("{e}")));
117
118        Ok(Box::pin(record_batch_stream) as ArrowRecordBatchStream)
119    }
120
121    /// Evolves the schema of the RecordBatches from an equality delete file.
122    ///
123    /// Per the [Iceberg spec](https://iceberg.apache.org/spec/#equality-delete-files),
124    /// only evolves the specified `equality_ids` columns, not all table columns.
125    pub(crate) async fn evolve_schema(
126        record_batch_stream: ArrowRecordBatchStream,
127        target_schema: Arc<Schema>,
128        equality_ids: &[i32],
129    ) -> Result<ArrowRecordBatchStream> {
130        let mut record_batch_transformer =
131            RecordBatchTransformerBuilder::new(target_schema.clone(), equality_ids).build();
132
133        let record_batch_stream = record_batch_stream.map(move |record_batch| {
134            record_batch.and_then(|record_batch| {
135                record_batch_transformer.process_record_batch(record_batch)
136            })
137        });
138
139        Ok(Box::pin(record_batch_stream) as ArrowRecordBatchStream)
140    }
141}
142
143/// Checks that every equality id is a field of the scan schema. Java's `DeleteFilter` fails the
144/// same way rather than matching rows on fewer columns.
145pub(crate) fn validate_equality_ids(schema: &Schema, equality_ids: &[i32]) -> Result<()> {
146    match equality_ids
147        .iter()
148        .find(|&&id| schema.field_by_id(id).is_none())
149    {
150        Some(id) => Err(Error::new(
151            ErrorKind::DataInvalid,
152            format!("Equality delete field id {id} is not in the scan schema"),
153        )),
154        None => Ok(()),
155    }
156}
157
158/// Reads an equality delete file as `schema` pruned to its equality fields, the way Java's
159/// `DeleteFilter` selects them, so a field nested in a struct keeps its parent struct.
160///
161/// Java writes a nested equality field under its parent structs, while `EqualityDeleteWriter`
162/// writes it as a top-level column. Such a column is put back under its parents before schema
163/// evolution, so both layouts read the same way.
164pub(crate) fn evolve_equality_delete_stream(
165    record_batch_stream: ArrowRecordBatchStream,
166    schema: &Schema,
167    equality_ids: &[i32],
168) -> Result<ArrowRecordBatchStream> {
169    validate_equality_ids(schema, equality_ids)?;
170    let crate::spec::Type::Struct(pruned) =
171        prune_columns(schema, equality_ids.iter().copied(), false)?
172    else {
173        return Err(Error::new(
174            ErrorKind::Unexpected,
175            "pruning a schema did not return a struct",
176        ));
177    };
178    let projection: Vec<i32> = pruned.fields().iter().map(|field| field.id).collect();
179    let pruned = Arc::new(
180        Schema::builder()
181            .with_schema_id(schema.schema_id())
182            .with_fields(pruned.fields().to_vec())
183            .build()?,
184    );
185    let mut transformer = RecordBatchTransformerBuilder::new(pruned.clone(), &projection).build();
186    // The batches of a file share its schema, so the nesting is worked out once and only
187    // rebuilt if a batch comes with a different schema.
188    let mut nesting: Option<EqualityColumnNesting> = None;
189    Ok(Box::pin(record_batch_stream.map(move |batch| {
190        let batch = batch?;
191        let nesting = match &mut nesting {
192            Some(nesting) if nesting.input_schema == batch.schema() => nesting,
193            slot => slot.insert(EqualityColumnNesting::try_new(batch.schema(), &pruned)?),
194        };
195        transformer.process_record_batch(nesting.apply(batch)?)
196    })) as ArrowRecordBatchStream)
197}
198
199/// How to build the columns of an equality delete batch once the top-level columns of fields
200/// nested in the pruned schema are moved back under their parent structs.
201struct EqualityColumnNesting {
202    input_schema: ArrowSchemaRef,
203    /// The output schema and columns, or `None` when no column has to move.
204    output: Option<(ArrowSchemaRef, Vec<NestedColumn>)>,
205}
206
207/// An output column: a column of the input batch, or a struct built from such columns.
208enum NestedColumn {
209    Input(usize),
210    Struct(Fields, Vec<NestedColumn>),
211}
212
213impl NestedColumn {
214    fn build(&self, batch: &RecordBatch) -> ArrayRef {
215        match self {
216            NestedColumn::Input(index) => batch.column(*index).clone(),
217            NestedColumn::Struct(fields, children) => Arc::new(StructArray::new(
218                fields.clone(),
219                children.iter().map(|child| child.build(batch)).collect(),
220                None,
221            )),
222        }
223    }
224}
225
226impl EqualityColumnNesting {
227    fn try_new(input_schema: ArrowSchemaRef, schema: &Schema) -> Result<Self> {
228        let top_level_ids: HashSet<i32> = schema
229            .as_struct()
230            .fields()
231            .iter()
232            .map(|field| field.id)
233            .collect();
234        let mut flat = HashMap::new();
235        for (index, field) in input_schema.fields().iter().enumerate() {
236            if let Some(id) = try_get_field_id_from_metadata(field)?
237                && !top_level_ids.contains(&id)
238                && schema.field_by_id(id).is_some()
239            {
240                flat.insert(id, index);
241            }
242        }
243        if flat.is_empty() {
244            return Ok(Self {
245                input_schema,
246                output: None,
247            });
248        }
249
250        fn nest(
251            field: &crate::spec::NestedField,
252            input_schema: &ArrowSchema,
253            flat: &HashMap<i32, usize>,
254        ) -> Option<(FieldRef, NestedColumn)> {
255            if let Some(&index) = flat.get(&field.id) {
256                return Some((
257                    input_schema.fields()[index].clone(),
258                    NestedColumn::Input(index),
259                ));
260            }
261            let crate::spec::Type::Struct(struct_type) = field.field_type.as_ref() else {
262                return None;
263            };
264            let (fields, children): (Vec<FieldRef>, Vec<NestedColumn>) = struct_type
265                .fields()
266                .iter()
267                .filter_map(|child| nest(child, input_schema, flat))
268                .unzip();
269            if fields.is_empty() {
270                return None;
271            }
272            let fields = Fields::from(fields);
273            let arrow_field = arrow_schema::Field::new(
274                &field.name,
275                arrow_schema::DataType::Struct(fields.clone()),
276                !field.required,
277            )
278            .with_metadata(HashMap::from([(
279                PARQUET_FIELD_ID_META_KEY.to_string(),
280                field.id.to_string(),
281            )]));
282            Some((
283                Arc::new(arrow_field),
284                NestedColumn::Struct(fields, children),
285            ))
286        }
287
288        let moved: HashSet<usize> = flat.values().copied().collect();
289        let mut fields: Vec<FieldRef> = Vec::new();
290        let mut columns: Vec<NestedColumn> = Vec::new();
291        for (index, field) in input_schema.fields().iter().enumerate() {
292            if !moved.contains(&index) {
293                fields.push(field.clone());
294                columns.push(NestedColumn::Input(index));
295            }
296        }
297        let present: HashSet<i32> = fields
298            .iter()
299            .map(|field| try_get_field_id_from_metadata(field))
300            .collect::<Result<Vec<_>>>()?
301            .into_iter()
302            .flatten()
303            .collect();
304        for field in schema.as_struct().fields() {
305            if present.contains(&field.id) {
306                continue;
307            }
308            if let Some((arrow_field, column)) = nest(field, &input_schema, &flat) {
309                fields.push(arrow_field);
310                columns.push(column);
311            }
312        }
313        Ok(Self {
314            input_schema,
315            output: Some((Arc::new(ArrowSchema::new(fields)), columns)),
316        })
317    }
318
319    fn apply(&self, batch: RecordBatch) -> Result<RecordBatch> {
320        let Some((schema, columns)) = &self.output else {
321            return Ok(batch);
322        };
323        let options = RecordBatchOptions::new().with_row_count(Some(batch.num_rows()));
324        Ok(RecordBatch::try_new_with_options(
325            schema.clone(),
326            columns.iter().map(|column| column.build(&batch)).collect(),
327            &options,
328        )?)
329    }
330}
331
332#[async_trait::async_trait]
333impl DeleteFileLoader for BasicDeleteFileLoader {
334    async fn read_delete_file(
335        &self,
336        task: &FileScanTaskDeleteFile,
337        schema: SchemaRef,
338    ) -> Result<ArrowRecordBatchStream> {
339        let raw_batch_stream = self
340            .parquet_to_batch_stream(
341                task.file_path(),
342                task.file_size_in_bytes(),
343                task.key_metadata(),
344            )
345            .await?;
346
347        // For equality deletes, only evolve the columns holding the equality_ids.
348        // For positional deletes (equality_ids is None), use all field IDs.
349        match task.equality_ids() {
350            Some(ids) => evolve_equality_delete_stream(raw_batch_stream, &schema, ids),
351            None => {
352                let field_ids: Vec<i32> = schema.field_id_to_name_map().keys().cloned().collect();
353                Self::evolve_schema(raw_batch_stream, schema, &field_ids).await
354            }
355        }
356    }
357}
358
359#[cfg(test)]
360mod tests {
361    use tempfile::TempDir;
362
363    use super::*;
364    use crate::arrow::delete_filter::tests::setup;
365    use crate::arrow::test_utils::write_encrypted_parquet;
366
367    #[tokio::test]
368    async fn test_basic_delete_file_loader_read_delete_file() {
369        let tmp_dir = TempDir::new().unwrap();
370        let table_location = tmp_dir.path();
371        let file_io = FileIO::new_with_fs();
372
373        let scan_metrics = ScanMetrics::new();
374        let delete_file_loader = BasicDeleteFileLoader::new(file_io.clone(), scan_metrics);
375
376        let file_scan_tasks = setup(table_location);
377
378        let result = delete_file_loader
379            .read_delete_file(
380                &file_scan_tasks[0].deletes()[0],
381                file_scan_tasks[0].schema_ref(),
382            )
383            .await
384            .unwrap();
385
386        let result = result.try_collect::<Vec<_>>().await.unwrap();
387
388        assert_eq!(result.len(), 1);
389    }
390
391    #[tokio::test]
392    async fn test_read_encrypted_positional_delete_file() {
393        use std::sync::Arc;
394
395        use arrow_array::{Int64Array, RecordBatch, StringArray};
396
397        use crate::arrow::delete_filter::tests::create_pos_del_schema;
398        use crate::encryption::StandardKeyMetadata;
399        use crate::scan::FileScanTaskDeleteFile;
400        use crate::spec::{DataContentType, DataFileFormat};
401
402        let encryption_key = b"0123456789abcdef";
403        let aad_prefix = b"aad_prefix";
404
405        let tmp_dir = TempDir::new().unwrap();
406        let table_location = tmp_dir.path().to_str().unwrap();
407        let file_io = FileIO::new_with_fs();
408
409        let positional_delete_schema = create_pos_del_schema();
410        let file_path_col = Arc::new(StringArray::from_iter_values(vec!["data.parquet"; 4]));
411        let pos_col = Arc::new(Int64Array::from(vec![0i64, 1, 5, 10]));
412        let batch = RecordBatch::try_new(positional_delete_schema.clone(), vec![
413            file_path_col,
414            pos_col,
415        ])
416        .unwrap();
417
418        let del_path = format!("{table_location}/encrypted-pos-del.parquet");
419        write_encrypted_parquet(&del_path, &batch, encryption_key, Some(aad_prefix));
420
421        let key_metadata = StandardKeyMetadata::try_new(encryption_key)
422            .unwrap()
423            .with_aad_prefix(aad_prefix)
424            .encode()
425            .unwrap();
426
427        let schema = Arc::new(
428            Schema::builder()
429                .with_schema_id(1)
430                .with_fields(vec![
431                    crate::spec::NestedField::required(
432                        2147483546,
433                        "file_path",
434                        crate::spec::Type::Primitive(crate::spec::PrimitiveType::String),
435                    )
436                    .into(),
437                    crate::spec::NestedField::required(
438                        2147483545,
439                        "pos",
440                        crate::spec::Type::Primitive(crate::spec::PrimitiveType::Long),
441                    )
442                    .into(),
443                ])
444                .build()
445                .unwrap(),
446        );
447
448        let scan_metrics = ScanMetrics::new();
449        let delete_file_loader = BasicDeleteFileLoader::new(file_io, scan_metrics);
450
451        // A known size, and an unknown size resolved before decryption.
452        for file_size_in_bytes in [Some(std::fs::metadata(&del_path).unwrap().len()), None] {
453            let task = FileScanTaskDeleteFile::builder()
454                .with_file_path(del_path.clone())
455                .with_file_size_in_bytes_opt(file_size_in_bytes)
456                .with_file_type(DataContentType::PositionDeletes)
457                .with_file_format(DataFileFormat::Parquet)
458                .with_partition_spec_id(0)
459                .with_key_metadata(Some(Box::from(key_metadata.as_ref())))
460                .build()
461                .unwrap();
462
463            let result = delete_file_loader
464                .read_delete_file(&task, Arc::clone(&schema))
465                .await
466                .unwrap();
467
468            let batches: Vec<_> = result.try_collect().await.unwrap();
469            assert_eq!(batches.len(), 1);
470            assert_eq!(batches[0].num_rows(), 4);
471        }
472    }
473
474    #[tokio::test]
475    async fn test_read_encrypted_equality_delete_file() {
476        use std::collections::HashMap;
477        use std::sync::Arc;
478
479        use arrow_array::{Int64Array, RecordBatch};
480        use parquet::arrow::PARQUET_FIELD_ID_META_KEY;
481
482        use crate::encryption::StandardKeyMetadata;
483        use crate::scan::FileScanTaskDeleteFile;
484        use crate::spec::{DataContentType, DataFileFormat};
485
486        let encryption_key = b"0123456789abcdef";
487        let aad_prefix = b"my-table-uuid!!";
488
489        let tmp_dir = TempDir::new().unwrap();
490        let table_location = tmp_dir.path().to_str().unwrap();
491        let file_io = FileIO::new_with_fs();
492
493        let arrow_schema = Arc::new(arrow_schema::Schema::new(vec![
494            arrow_schema::Field::new("id", arrow_schema::DataType::Int64, false).with_metadata(
495                HashMap::from([(PARQUET_FIELD_ID_META_KEY.to_string(), "1".to_string())]),
496            ),
497        ]));
498
499        let id_col = Arc::new(Int64Array::from(vec![100i64, 200, 300]));
500        let batch = RecordBatch::try_new(arrow_schema.clone(), vec![id_col]).unwrap();
501
502        let del_path = format!("{table_location}/encrypted-eq-del.parquet");
503        write_encrypted_parquet(&del_path, &batch, encryption_key, Some(aad_prefix));
504
505        let key_metadata = StandardKeyMetadata::try_new(encryption_key)
506            .unwrap()
507            .with_aad_prefix(aad_prefix)
508            .encode()
509            .unwrap();
510
511        let schema = Arc::new(
512            Schema::builder()
513                .with_schema_id(1)
514                .with_fields(vec![
515                    crate::spec::NestedField::required(
516                        1,
517                        "id",
518                        crate::spec::Type::Primitive(crate::spec::PrimitiveType::Long),
519                    )
520                    .into(),
521                ])
522                .build()
523                .unwrap(),
524        );
525
526        let task = FileScanTaskDeleteFile::builder()
527            .with_file_path(del_path.clone())
528            .with_file_size_in_bytes(std::fs::metadata(&del_path).unwrap().len())
529            .with_file_type(DataContentType::EqualityDeletes)
530            .with_file_format(DataFileFormat::Parquet)
531            .with_partition_spec_id(0)
532            .with_equality_ids(Some(vec![1]))
533            .with_key_metadata(Some(Box::from(key_metadata.as_ref())))
534            .build()
535            .unwrap();
536
537        let scan_metrics = ScanMetrics::new();
538        let delete_file_loader = BasicDeleteFileLoader::new(file_io, scan_metrics);
539
540        let result = delete_file_loader
541            .read_delete_file(&task, schema)
542            .await
543            .unwrap();
544
545        let batches: Vec<_> = result.try_collect().await.unwrap();
546        assert_eq!(batches.len(), 1);
547        assert_eq!(batches[0].num_rows(), 3);
548    }
549
550    #[tokio::test]
551    async fn test_delete_file_too_small_for_footer_fails() {
552        let temp = TempDir::new().unwrap();
553        let loader = BasicDeleteFileLoader::new(FileIO::new_with_fs(), ScanMetrics::new());
554
555        // A size too small for a Parquet footer fails the read, whether it was
556        // resolved from storage or supplied by the task.
557        let empty = temp.path().join("empty.parquet");
558        std::fs::write(&empty, []).unwrap();
559        for size in [None, Some(0), Some(4)] {
560            let error = loader
561                .parquet_to_batch_stream(empty.to_str().unwrap(), size, None)
562                .await
563                .err()
564                .unwrap();
565            assert!(error.to_string().contains("is less than footer"), "{error}");
566        }
567    }
568}