1use std::collections::{HashMap, HashSet};
19use std::sync::Arc;
20
21use arrow_array::{ArrayRef, RecordBatch, RecordBatchOptions, StructArray};
22use arrow_schema::{FieldRef, Fields, Schema as ArrowSchema, SchemaRef as ArrowSchemaRef};
23use futures::{StreamExt, TryStreamExt};
24use parquet::arrow::{PARQUET_FIELD_ID_META_KEY, ParquetRecordBatchStreamBuilder};
25
26use crate::arrow::ArrowReader;
27use crate::arrow::reader::ParquetReadOptions;
28use crate::arrow::record_batch_transformer::RecordBatchTransformerBuilder;
29use crate::arrow::scan_metrics::ScanMetrics;
30use crate::arrow::schema::try_get_field_id_from_metadata;
31use crate::io::FileIO;
32use crate::scan::{ArrowRecordBatchStream, FileScanTaskDeleteFile};
33use crate::spec::{Schema, SchemaRef, prune_columns};
34use crate::{Error, ErrorKind, Result};
35
36#[allow(unused)]
38#[async_trait::async_trait]
39pub trait DeleteFileLoader {
40 async fn read_delete_file(
44 &self,
45 task: &FileScanTaskDeleteFile,
46 schema: SchemaRef,
47 ) -> Result<ArrowRecordBatchStream>;
48}
49
50#[derive(Clone, Debug)]
51pub(crate) struct BasicDeleteFileLoader {
52 file_io: FileIO,
53 scan_metrics: ScanMetrics,
54}
55
56#[allow(unused_variables)]
57impl BasicDeleteFileLoader {
58 pub fn new(file_io: FileIO, scan_metrics: ScanMetrics) -> Self {
59 BasicDeleteFileLoader {
60 file_io,
61 scan_metrics,
62 }
63 }
64
65 pub(crate) fn file_io(&self) -> &FileIO {
66 &self.file_io
67 }
68
69 pub(crate) async fn parquet_to_batch_stream(
71 &self,
72 data_file_path: &str,
73 file_size_in_bytes: Option<u64>,
74 key_metadata: Option<&[u8]>,
75 ) -> Result<ArrowRecordBatchStream> {
76 let file_size_in_bytes = match file_size_in_bytes {
83 Some(file_size_in_bytes) => file_size_in_bytes,
84 None => {
85 self.file_io
86 .new_input(data_file_path)?
87 .metadata()
88 .await
89 .map_err(|error| {
90 Error::new(
92 error.kind(),
93 format!("Failed to stat delete file '{data_file_path}'"),
94 )
95 .with_retryable(error.retryable())
96 .with_source(error)
97 })?
98 .size
99 }
100 };
101 let parquet_read_options = ParquetReadOptions::builder().build();
102
103 let (parquet_file_reader, arrow_metadata) = ArrowReader::open_parquet_file(
104 data_file_path,
105 &self.file_io,
106 file_size_in_bytes,
107 parquet_read_options,
108 self.scan_metrics.bytes_read_counter(),
109 key_metadata,
110 )
111 .await?;
112
113 let record_batch_stream =
114 ParquetRecordBatchStreamBuilder::new_with_metadata(parquet_file_reader, arrow_metadata)
115 .build()?
116 .map_err(|e| Error::new(ErrorKind::Unexpected, format!("{e}")));
117
118 Ok(Box::pin(record_batch_stream) as ArrowRecordBatchStream)
119 }
120
121 pub(crate) async fn evolve_schema(
126 record_batch_stream: ArrowRecordBatchStream,
127 target_schema: Arc<Schema>,
128 equality_ids: &[i32],
129 ) -> Result<ArrowRecordBatchStream> {
130 let mut record_batch_transformer =
131 RecordBatchTransformerBuilder::new(target_schema.clone(), equality_ids).build();
132
133 let record_batch_stream = record_batch_stream.map(move |record_batch| {
134 record_batch.and_then(|record_batch| {
135 record_batch_transformer.process_record_batch(record_batch)
136 })
137 });
138
139 Ok(Box::pin(record_batch_stream) as ArrowRecordBatchStream)
140 }
141}
142
143pub(crate) fn validate_equality_ids(schema: &Schema, equality_ids: &[i32]) -> Result<()> {
146 match equality_ids
147 .iter()
148 .find(|&&id| schema.field_by_id(id).is_none())
149 {
150 Some(id) => Err(Error::new(
151 ErrorKind::DataInvalid,
152 format!("Equality delete field id {id} is not in the scan schema"),
153 )),
154 None => Ok(()),
155 }
156}
157
158pub(crate) fn evolve_equality_delete_stream(
165 record_batch_stream: ArrowRecordBatchStream,
166 schema: &Schema,
167 equality_ids: &[i32],
168) -> Result<ArrowRecordBatchStream> {
169 validate_equality_ids(schema, equality_ids)?;
170 let crate::spec::Type::Struct(pruned) =
171 prune_columns(schema, equality_ids.iter().copied(), false)?
172 else {
173 return Err(Error::new(
174 ErrorKind::Unexpected,
175 "pruning a schema did not return a struct",
176 ));
177 };
178 let projection: Vec<i32> = pruned.fields().iter().map(|field| field.id).collect();
179 let pruned = Arc::new(
180 Schema::builder()
181 .with_schema_id(schema.schema_id())
182 .with_fields(pruned.fields().to_vec())
183 .build()?,
184 );
185 let mut transformer = RecordBatchTransformerBuilder::new(pruned.clone(), &projection).build();
186 let mut nesting: Option<EqualityColumnNesting> = None;
189 Ok(Box::pin(record_batch_stream.map(move |batch| {
190 let batch = batch?;
191 let nesting = match &mut nesting {
192 Some(nesting) if nesting.input_schema == batch.schema() => nesting,
193 slot => slot.insert(EqualityColumnNesting::try_new(batch.schema(), &pruned)?),
194 };
195 transformer.process_record_batch(nesting.apply(batch)?)
196 })) as ArrowRecordBatchStream)
197}
198
199struct EqualityColumnNesting {
202 input_schema: ArrowSchemaRef,
203 output: Option<(ArrowSchemaRef, Vec<NestedColumn>)>,
205}
206
207enum NestedColumn {
209 Input(usize),
210 Struct(Fields, Vec<NestedColumn>),
211}
212
213impl NestedColumn {
214 fn build(&self, batch: &RecordBatch) -> ArrayRef {
215 match self {
216 NestedColumn::Input(index) => batch.column(*index).clone(),
217 NestedColumn::Struct(fields, children) => Arc::new(StructArray::new(
218 fields.clone(),
219 children.iter().map(|child| child.build(batch)).collect(),
220 None,
221 )),
222 }
223 }
224}
225
226impl EqualityColumnNesting {
227 fn try_new(input_schema: ArrowSchemaRef, schema: &Schema) -> Result<Self> {
228 let top_level_ids: HashSet<i32> = schema
229 .as_struct()
230 .fields()
231 .iter()
232 .map(|field| field.id)
233 .collect();
234 let mut flat = HashMap::new();
235 for (index, field) in input_schema.fields().iter().enumerate() {
236 if let Some(id) = try_get_field_id_from_metadata(field)?
237 && !top_level_ids.contains(&id)
238 && schema.field_by_id(id).is_some()
239 {
240 flat.insert(id, index);
241 }
242 }
243 if flat.is_empty() {
244 return Ok(Self {
245 input_schema,
246 output: None,
247 });
248 }
249
250 fn nest(
251 field: &crate::spec::NestedField,
252 input_schema: &ArrowSchema,
253 flat: &HashMap<i32, usize>,
254 ) -> Option<(FieldRef, NestedColumn)> {
255 if let Some(&index) = flat.get(&field.id) {
256 return Some((
257 input_schema.fields()[index].clone(),
258 NestedColumn::Input(index),
259 ));
260 }
261 let crate::spec::Type::Struct(struct_type) = field.field_type.as_ref() else {
262 return None;
263 };
264 let (fields, children): (Vec<FieldRef>, Vec<NestedColumn>) = struct_type
265 .fields()
266 .iter()
267 .filter_map(|child| nest(child, input_schema, flat))
268 .unzip();
269 if fields.is_empty() {
270 return None;
271 }
272 let fields = Fields::from(fields);
273 let arrow_field = arrow_schema::Field::new(
274 &field.name,
275 arrow_schema::DataType::Struct(fields.clone()),
276 !field.required,
277 )
278 .with_metadata(HashMap::from([(
279 PARQUET_FIELD_ID_META_KEY.to_string(),
280 field.id.to_string(),
281 )]));
282 Some((
283 Arc::new(arrow_field),
284 NestedColumn::Struct(fields, children),
285 ))
286 }
287
288 let moved: HashSet<usize> = flat.values().copied().collect();
289 let mut fields: Vec<FieldRef> = Vec::new();
290 let mut columns: Vec<NestedColumn> = Vec::new();
291 for (index, field) in input_schema.fields().iter().enumerate() {
292 if !moved.contains(&index) {
293 fields.push(field.clone());
294 columns.push(NestedColumn::Input(index));
295 }
296 }
297 let present: HashSet<i32> = fields
298 .iter()
299 .map(|field| try_get_field_id_from_metadata(field))
300 .collect::<Result<Vec<_>>>()?
301 .into_iter()
302 .flatten()
303 .collect();
304 for field in schema.as_struct().fields() {
305 if present.contains(&field.id) {
306 continue;
307 }
308 if let Some((arrow_field, column)) = nest(field, &input_schema, &flat) {
309 fields.push(arrow_field);
310 columns.push(column);
311 }
312 }
313 Ok(Self {
314 input_schema,
315 output: Some((Arc::new(ArrowSchema::new(fields)), columns)),
316 })
317 }
318
319 fn apply(&self, batch: RecordBatch) -> Result<RecordBatch> {
320 let Some((schema, columns)) = &self.output else {
321 return Ok(batch);
322 };
323 let options = RecordBatchOptions::new().with_row_count(Some(batch.num_rows()));
324 Ok(RecordBatch::try_new_with_options(
325 schema.clone(),
326 columns.iter().map(|column| column.build(&batch)).collect(),
327 &options,
328 )?)
329 }
330}
331
332#[async_trait::async_trait]
333impl DeleteFileLoader for BasicDeleteFileLoader {
334 async fn read_delete_file(
335 &self,
336 task: &FileScanTaskDeleteFile,
337 schema: SchemaRef,
338 ) -> Result<ArrowRecordBatchStream> {
339 let raw_batch_stream = self
340 .parquet_to_batch_stream(
341 task.file_path(),
342 task.file_size_in_bytes(),
343 task.key_metadata(),
344 )
345 .await?;
346
347 match task.equality_ids() {
350 Some(ids) => evolve_equality_delete_stream(raw_batch_stream, &schema, ids),
351 None => {
352 let field_ids: Vec<i32> = schema.field_id_to_name_map().keys().cloned().collect();
353 Self::evolve_schema(raw_batch_stream, schema, &field_ids).await
354 }
355 }
356 }
357}
358
359#[cfg(test)]
360mod tests {
361 use tempfile::TempDir;
362
363 use super::*;
364 use crate::arrow::delete_filter::tests::setup;
365 use crate::arrow::test_utils::write_encrypted_parquet;
366
367 #[tokio::test]
368 async fn test_basic_delete_file_loader_read_delete_file() {
369 let tmp_dir = TempDir::new().unwrap();
370 let table_location = tmp_dir.path();
371 let file_io = FileIO::new_with_fs();
372
373 let scan_metrics = ScanMetrics::new();
374 let delete_file_loader = BasicDeleteFileLoader::new(file_io.clone(), scan_metrics);
375
376 let file_scan_tasks = setup(table_location);
377
378 let result = delete_file_loader
379 .read_delete_file(
380 &file_scan_tasks[0].deletes()[0],
381 file_scan_tasks[0].schema_ref(),
382 )
383 .await
384 .unwrap();
385
386 let result = result.try_collect::<Vec<_>>().await.unwrap();
387
388 assert_eq!(result.len(), 1);
389 }
390
391 #[tokio::test]
392 async fn test_read_encrypted_positional_delete_file() {
393 use std::sync::Arc;
394
395 use arrow_array::{Int64Array, RecordBatch, StringArray};
396
397 use crate::arrow::delete_filter::tests::create_pos_del_schema;
398 use crate::encryption::StandardKeyMetadata;
399 use crate::scan::FileScanTaskDeleteFile;
400 use crate::spec::{DataContentType, DataFileFormat};
401
402 let encryption_key = b"0123456789abcdef";
403 let aad_prefix = b"aad_prefix";
404
405 let tmp_dir = TempDir::new().unwrap();
406 let table_location = tmp_dir.path().to_str().unwrap();
407 let file_io = FileIO::new_with_fs();
408
409 let positional_delete_schema = create_pos_del_schema();
410 let file_path_col = Arc::new(StringArray::from_iter_values(vec!["data.parquet"; 4]));
411 let pos_col = Arc::new(Int64Array::from(vec![0i64, 1, 5, 10]));
412 let batch = RecordBatch::try_new(positional_delete_schema.clone(), vec![
413 file_path_col,
414 pos_col,
415 ])
416 .unwrap();
417
418 let del_path = format!("{table_location}/encrypted-pos-del.parquet");
419 write_encrypted_parquet(&del_path, &batch, encryption_key, Some(aad_prefix));
420
421 let key_metadata = StandardKeyMetadata::try_new(encryption_key)
422 .unwrap()
423 .with_aad_prefix(aad_prefix)
424 .encode()
425 .unwrap();
426
427 let schema = Arc::new(
428 Schema::builder()
429 .with_schema_id(1)
430 .with_fields(vec![
431 crate::spec::NestedField::required(
432 2147483546,
433 "file_path",
434 crate::spec::Type::Primitive(crate::spec::PrimitiveType::String),
435 )
436 .into(),
437 crate::spec::NestedField::required(
438 2147483545,
439 "pos",
440 crate::spec::Type::Primitive(crate::spec::PrimitiveType::Long),
441 )
442 .into(),
443 ])
444 .build()
445 .unwrap(),
446 );
447
448 let scan_metrics = ScanMetrics::new();
449 let delete_file_loader = BasicDeleteFileLoader::new(file_io, scan_metrics);
450
451 for file_size_in_bytes in [Some(std::fs::metadata(&del_path).unwrap().len()), None] {
453 let task = FileScanTaskDeleteFile::builder()
454 .with_file_path(del_path.clone())
455 .with_file_size_in_bytes_opt(file_size_in_bytes)
456 .with_file_type(DataContentType::PositionDeletes)
457 .with_file_format(DataFileFormat::Parquet)
458 .with_partition_spec_id(0)
459 .with_key_metadata(Some(Box::from(key_metadata.as_ref())))
460 .build()
461 .unwrap();
462
463 let result = delete_file_loader
464 .read_delete_file(&task, Arc::clone(&schema))
465 .await
466 .unwrap();
467
468 let batches: Vec<_> = result.try_collect().await.unwrap();
469 assert_eq!(batches.len(), 1);
470 assert_eq!(batches[0].num_rows(), 4);
471 }
472 }
473
474 #[tokio::test]
475 async fn test_read_encrypted_equality_delete_file() {
476 use std::collections::HashMap;
477 use std::sync::Arc;
478
479 use arrow_array::{Int64Array, RecordBatch};
480 use parquet::arrow::PARQUET_FIELD_ID_META_KEY;
481
482 use crate::encryption::StandardKeyMetadata;
483 use crate::scan::FileScanTaskDeleteFile;
484 use crate::spec::{DataContentType, DataFileFormat};
485
486 let encryption_key = b"0123456789abcdef";
487 let aad_prefix = b"my-table-uuid!!";
488
489 let tmp_dir = TempDir::new().unwrap();
490 let table_location = tmp_dir.path().to_str().unwrap();
491 let file_io = FileIO::new_with_fs();
492
493 let arrow_schema = Arc::new(arrow_schema::Schema::new(vec![
494 arrow_schema::Field::new("id", arrow_schema::DataType::Int64, false).with_metadata(
495 HashMap::from([(PARQUET_FIELD_ID_META_KEY.to_string(), "1".to_string())]),
496 ),
497 ]));
498
499 let id_col = Arc::new(Int64Array::from(vec![100i64, 200, 300]));
500 let batch = RecordBatch::try_new(arrow_schema.clone(), vec![id_col]).unwrap();
501
502 let del_path = format!("{table_location}/encrypted-eq-del.parquet");
503 write_encrypted_parquet(&del_path, &batch, encryption_key, Some(aad_prefix));
504
505 let key_metadata = StandardKeyMetadata::try_new(encryption_key)
506 .unwrap()
507 .with_aad_prefix(aad_prefix)
508 .encode()
509 .unwrap();
510
511 let schema = Arc::new(
512 Schema::builder()
513 .with_schema_id(1)
514 .with_fields(vec![
515 crate::spec::NestedField::required(
516 1,
517 "id",
518 crate::spec::Type::Primitive(crate::spec::PrimitiveType::Long),
519 )
520 .into(),
521 ])
522 .build()
523 .unwrap(),
524 );
525
526 let task = FileScanTaskDeleteFile::builder()
527 .with_file_path(del_path.clone())
528 .with_file_size_in_bytes(std::fs::metadata(&del_path).unwrap().len())
529 .with_file_type(DataContentType::EqualityDeletes)
530 .with_file_format(DataFileFormat::Parquet)
531 .with_partition_spec_id(0)
532 .with_equality_ids(Some(vec![1]))
533 .with_key_metadata(Some(Box::from(key_metadata.as_ref())))
534 .build()
535 .unwrap();
536
537 let scan_metrics = ScanMetrics::new();
538 let delete_file_loader = BasicDeleteFileLoader::new(file_io, scan_metrics);
539
540 let result = delete_file_loader
541 .read_delete_file(&task, schema)
542 .await
543 .unwrap();
544
545 let batches: Vec<_> = result.try_collect().await.unwrap();
546 assert_eq!(batches.len(), 1);
547 assert_eq!(batches[0].num_rows(), 3);
548 }
549
550 #[tokio::test]
551 async fn test_delete_file_too_small_for_footer_fails() {
552 let temp = TempDir::new().unwrap();
553 let loader = BasicDeleteFileLoader::new(FileIO::new_with_fs(), ScanMetrics::new());
554
555 let empty = temp.path().join("empty.parquet");
558 std::fs::write(&empty, []).unwrap();
559 for size in [None, Some(0), Some(4)] {
560 let error = loader
561 .parquet_to_batch_stream(empty.to_str().unwrap(), size, None)
562 .await
563 .err()
564 .unwrap();
565 assert!(error.to_string().contains("is less than footer"), "{error}");
566 }
567 }
568}