1use std::collections::HashMap;
21use std::io::Cursor;
22use std::sync::Arc;
23
24use arrow_arith::boolean::{and, is_null};
25use arrow_array::builder::Int64Builder;
26use arrow_array::cast::AsArray;
27use arrow_array::types::{TimestampMicrosecondType, TimestampNanosecondType};
28use arrow_array::{
29 Array, ArrayRef, BooleanArray, Int32Array, Int64Array, ListArray, MapArray, RecordBatch,
30 StructArray, new_null_array,
31};
32use arrow_avro::reader::{Reader, ReaderBuilder};
33use arrow_cast::{CastOptions, cast_with_options};
34use arrow_ord::cmp::eq;
35use arrow_schema::{ArrowError, DataType, Field, Fields, SchemaRef, TimeUnit};
36use arrow_select::zip::zip;
37use parquet::arrow::PARQUET_FIELD_ID_META_KEY;
38use serde_json::Value as JsonValue;
39
40use crate::arrow::schema_to_arrow_schema;
41use crate::error::invalid_data;
42use crate::spec::{
43 DATA_FILE_CONTENT_FIELD_ID, FILE_SEQUENCE_NUMBER_FIELD_ID, FormatVersion,
44 MANIFEST_CONTENT_FIELD_ID, MANIFEST_SEQUENCE_NUMBER_FIELD_ID, MIN_SEQUENCE_NUMBER_FIELD_ID,
45 ManifestContentType, ManifestFile, ManifestMetadata, ManifestStatus, SEQUENCE_NUMBER_FIELD_ID,
46 Schema, manifest_entry_fields, manifest_file_fields,
47};
48use crate::{Error, Result};
49
50const BATCH_SIZE: usize = 1024;
54
55pub fn read_manifest_entries<B: AsRef<[u8]>>(
117 bytes: B,
118 manifest_file: &ManifestFile,
119) -> Result<ManifestEntryBatchReader<B>> {
120 ManifestEntryBatchReader::try_new(bytes, manifest_file, BATCH_SIZE)
121}
122
123pub struct ManifestEntryBatchReader<B: AsRef<[u8]>> {
126 inner: ProjectingReader<B>,
127 inheritance: Inheritance,
128}
129
130struct Inheritance {
132 manifest_path: String,
133 added_snapshot_id: i64,
134 sequence_number: i64,
135 content: ManifestContentType,
136 next_row_id: Option<i64>,
139}
140
141impl<B: AsRef<[u8]>> ManifestEntryBatchReader<B> {
142 fn try_new(bytes: B, manifest_file: &ManifestFile, batch_size: usize) -> Result<Self> {
143 let reader = ReaderBuilder::new()
144 .with_batch_size(batch_size)
145 .build(Cursor::new(bytes))
146 .map_err(invalid_manifest)?;
147 let header = reader.avro_header();
148 let metadata: HashMap<String, Vec<u8>> = header
149 .metadata()
150 .map(|(key, value)| (String::from_utf8_lossy(key).into_owned(), value.to_vec()))
151 .collect();
152 let metadata = ManifestMetadata::parse(&metadata)?;
153 let avro_schema: JsonValue = serde_json::from_slice(
154 header
155 .get("avro.schema")
156 .ok_or_else(|| invalid_data!("manifest has no avro.schema"))?,
157 )?;
158 let partition_type = metadata
159 .partition_spec()
160 .partition_type(metadata.schema())?;
161 let schema = Arc::new(schema_to_arrow_schema(
162 &Schema::builder()
163 .with_fields(manifest_entry_fields(&partition_type))
164 .build()?,
165 )?);
166
167 let zero_defaults: &[i32] = match metadata.format_version() {
171 FormatVersion::V1 => &[
172 DATA_FILE_CONTENT_FIELD_ID,
173 SEQUENCE_NUMBER_FIELD_ID,
174 FILE_SEQUENCE_NUMBER_FIELD_ID,
175 ],
176 _ => &[DATA_FILE_CONTENT_FIELD_ID],
177 };
178 Ok(Self {
179 inner: ProjectingReader {
180 reader,
181 schema,
182 avro_schema,
183 zero_defaults,
184 invalid: invalid_manifest,
185 finished: false,
186 },
187 inheritance: Inheritance {
188 manifest_path: manifest_file.manifest_path.clone(),
189 added_snapshot_id: manifest_file.added_snapshot_id,
190 sequence_number: manifest_file.sequence_number,
191 content: manifest_file.content,
192 next_row_id: manifest_file.first_row_id.map(i64::try_from).transpose()?,
193 },
194 })
195 }
196
197 pub fn schema(&self) -> SchemaRef {
199 self.inner.schema.clone()
200 }
201}
202
203impl<B: AsRef<[u8]>> Iterator for ManifestEntryBatchReader<B> {
204 type Item = Result<RecordBatch>;
205
206 fn next(&mut self) -> Option<Self::Item> {
207 let inheritance = &mut self.inheritance;
208 self.inner
209 .next_batch(|entries| inherit(entries, inheritance))
210 }
211}
212
213pub fn read_manifest_list<B: AsRef<[u8]>>(bytes: B) -> Result<ManifestListBatchReader<B>> {
252 ManifestListBatchReader::try_new(bytes, BATCH_SIZE)
253}
254
255pub struct ManifestListBatchReader<B: AsRef<[u8]>> {
258 inner: ProjectingReader<B>,
259}
260
261impl<B: AsRef<[u8]>> ManifestListBatchReader<B> {
262 fn try_new(bytes: B, batch_size: usize) -> Result<Self> {
263 let reader = ReaderBuilder::new()
264 .with_batch_size(batch_size)
265 .build(Cursor::new(bytes))
266 .map_err(invalid_manifest_list)?;
267 let header = reader.avro_header();
268 let avro_schema: JsonValue = serde_json::from_slice(
269 header
270 .get("avro.schema")
271 .ok_or_else(|| invalid_data!("manifest list has no avro.schema"))?,
272 )?;
273 let format_version = match header.get("format-version") {
277 Some(bytes) => serde_json::from_slice::<FormatVersion>(bytes).map_err(|err| {
278 invalid_data!("failed to parse format version in manifest list metadata")
279 .with_source(err)
280 })?,
281 None => FormatVersion::V1,
282 };
283 let schema = Arc::new(schema_to_arrow_schema(
284 &Schema::builder()
285 .with_fields(manifest_file_fields())
286 .build()?,
287 )?);
288 let zero_defaults: &[i32] = match format_version {
291 FormatVersion::V1 => &[
292 MANIFEST_CONTENT_FIELD_ID,
293 MANIFEST_SEQUENCE_NUMBER_FIELD_ID,
294 MIN_SEQUENCE_NUMBER_FIELD_ID,
295 ],
296 _ => &[],
297 };
298 Ok(Self {
299 inner: ProjectingReader {
300 reader,
301 schema,
302 avro_schema,
303 zero_defaults,
304 invalid: invalid_manifest_list,
305 finished: false,
306 },
307 })
308 }
309
310 pub fn schema(&self) -> SchemaRef {
312 self.inner.schema.clone()
313 }
314}
315
316impl<B: AsRef<[u8]>> Iterator for ManifestListBatchReader<B> {
317 type Item = Result<RecordBatch>;
318
319 fn next(&mut self) -> Option<Self::Item> {
320 self.inner.next_batch(Ok)
321 }
322}
323
324struct ProjectingReader<B: AsRef<[u8]>> {
326 reader: Reader<Cursor<B>>,
327 schema: SchemaRef,
328 avro_schema: JsonValue,
329 zero_defaults: &'static [i32],
330 invalid: fn(ArrowError) -> Error,
331 finished: bool,
334}
335
336impl<B: AsRef<[u8]>> ProjectingReader<B> {
337 fn next_batch(
339 &mut self,
340 finish: impl FnOnce(StructArray) -> Result<StructArray>,
341 ) -> Option<Result<RecordBatch>> {
342 if self.finished {
343 return None;
344 }
345 let batch = self.reader.next()?.map_err(self.invalid).and_then(|batch| {
346 let rows = project_struct(
347 &StructArray::from(batch),
348 &self.avro_schema,
349 self.schema.fields(),
350 self.zero_defaults,
351 )?;
352 Ok(RecordBatch::from(finish(rows)?))
353 });
354 self.finished = batch.is_err();
355 Some(batch)
356 }
357}
358
359fn invalid_manifest(err: ArrowError) -> Error {
360 invalid_data!("failed to read manifest").with_source(err)
361}
362
363fn invalid_manifest_list(err: ArrowError) -> Error {
364 invalid_data!("failed to read manifest list").with_source(err)
365}
366
367fn invalid_column(err: ArrowError) -> Error {
368 invalid_data!("failed to build Arrow column from Avro").with_source(err)
369}
370
371fn field_id(field: &Field) -> i32 {
375 field.metadata()[PARQUET_FIELD_ID_META_KEY].parse().unwrap()
376}
377
378fn non_null_type(avro_type: &JsonValue) -> &JsonValue {
380 match avro_type.as_array().map(Vec::as_slice) {
381 Some([JsonValue::String(null), branch] | [branch, JsonValue::String(null)])
382 if null == "null" =>
383 {
384 branch
385 }
386 _ => avro_type,
387 }
388}
389
390fn project_struct(
394 array: &StructArray,
395 record: &JsonValue,
396 fields: &Fields,
397 zero_defaults: &[i32],
398) -> Result<StructArray> {
399 let avro_fields = record["fields"]
400 .as_array()
401 .ok_or_else(|| invalid_data!("Avro type {record} is not a record"))?;
402 let mut columns = Vec::with_capacity(fields.len());
403 for field in fields {
404 let id = field_id(field);
405 columns.push(
406 match avro_fields
407 .iter()
408 .position(|avro_field| avro_field["field-id"].as_i64() == Some(id.into()))
409 {
410 Some(index) => project(
411 array.column(index),
412 non_null_type(&avro_fields[index]["type"]),
413 field,
414 zero_defaults,
415 )?,
416 None => missing_field(field, array.len(), zero_defaults)?,
417 },
418 );
419 }
420 StructArray::try_new_with_length(fields.clone(), columns, array.nulls().cloned(), array.len())
421 .map_err(invalid_column)
422}
423
424fn missing_field(field: &Field, len: usize, zero_defaults: &[i32]) -> Result<ArrayRef> {
426 let id = field_id(field);
427 let zero = zero_defaults.contains(&id);
428 match field.data_type() {
429 DataType::Int32 if zero => Ok(Arc::new(Int32Array::from_value(0, len))),
430 DataType::Int64 if zero => Ok(Arc::new(Int64Array::from_value(0, len))),
431 _ if field.is_nullable() => Ok(new_null_array(field.data_type(), len)),
432 _ => Err(invalid_data!(
433 "Avro file has no required field {} with field ID {id}",
434 field.name()
435 )),
436 }
437}
438
439fn project(
442 array: &ArrayRef,
443 avro_type: &JsonValue,
444 field: &Field,
445 zero_defaults: &[i32],
446) -> Result<ArrayRef> {
447 let type_mismatch = || {
448 invalid_data!(
449 "field {} has Avro type {avro_type}, which doesn't match {}",
450 field.name(),
451 field.data_type()
452 )
453 };
454 Ok(match field.data_type() {
455 DataType::Struct(fields) => Arc::new(project_struct(
456 array.as_struct_opt().ok_or_else(type_mismatch)?,
457 avro_type,
458 fields,
459 zero_defaults,
460 )?),
461 DataType::List(element) => {
462 let list = array.as_list_opt::<i32>().ok_or_else(type_mismatch)?;
463 let values = project(
464 list.values(),
465 non_null_type(&avro_type["items"]),
466 element,
467 zero_defaults,
468 )?;
469 Arc::new(
470 ListArray::try_new(
471 element.clone(),
472 list.offsets().clone(),
473 values,
474 list.nulls().cloned(),
475 )
476 .map_err(invalid_column)?,
477 )
478 }
479 DataType::Map(entries, _) => {
481 let list = array.as_list_opt::<i32>().ok_or_else(type_mismatch)?;
482 let DataType::Struct(key_value) = entries.data_type() else {
483 unreachable!()
484 };
485 let entries_array = project_struct(
486 list.values().as_struct_opt().ok_or_else(type_mismatch)?,
487 &avro_type["items"],
488 key_value,
489 zero_defaults,
490 )?;
491 Arc::new(
492 MapArray::try_new(
493 entries.clone(),
494 list.offsets().clone(),
495 entries_array,
496 list.nulls().cloned(),
497 false,
498 )
499 .map_err(invalid_column)?,
500 )
501 }
502 DataType::Timestamp(TimeUnit::Microsecond, time_zone) => Arc::new(
505 array
506 .as_primitive_opt::<TimestampMicrosecondType>()
507 .ok_or_else(type_mismatch)?
508 .clone()
509 .with_timezone_opt(time_zone.clone()),
510 ),
511 DataType::Timestamp(TimeUnit::Nanosecond, time_zone) => Arc::new(
512 array
513 .as_primitive_opt::<TimestampNanosecondType>()
514 .ok_or_else(type_mismatch)?
515 .clone()
516 .with_timezone_opt(time_zone.clone()),
517 ),
518 data_type if array.data_type() == data_type => array.clone(),
519 data_type => cast_with_options(array, data_type, &CastOptions {
522 safe: false,
523 ..Default::default()
524 })
525 .map_err(invalid_column)?,
526 })
527}
528
529fn inherit(entries: StructArray, inheritance: &mut Inheritance) -> Result<StructArray> {
532 let (fields, mut columns, nulls) = entries.into_parts();
533 let index = |name: &str| fields.find(name).unwrap().0;
534 let (status, snapshot_id, sequence_number, file_sequence_number, data_file) = (
535 index("status"),
536 index("snapshot_id"),
537 index("sequence_number"),
538 index("file_sequence_number"),
539 index("data_file"),
540 );
541
542 columns[snapshot_id] = fill_nulls(&columns[snapshot_id], None, inheritance.added_snapshot_id)?;
543 if columns[sequence_number].null_count() > 0 || columns[file_sequence_number].null_count() > 0 {
544 let added = eq(
545 &columns[status],
546 &Int32Array::new_scalar(ManifestStatus::Added as i32),
547 )?;
548 for column in [sequence_number, file_sequence_number] {
549 columns[column] =
550 fill_nulls(&columns[column], Some(&added), inheritance.sequence_number)?;
551 }
552 }
553 if inheritance.content == ManifestContentType::Data {
554 let (data_file_fields, mut data_file_columns, data_file_nulls) =
555 columns[data_file].as_struct().clone().into_parts();
556 let index = |name: &str| data_file_fields.find(name).unwrap().0;
557 let first_row_id = index("first_row_id");
558 data_file_columns[first_row_id] = inherit_first_row_ids(
559 data_file_columns[first_row_id].as_primitive(),
560 columns[status].as_primitive(),
561 data_file_columns[index("record_count")].as_primitive(),
562 inheritance,
563 )?;
564 columns[data_file] = Arc::new(StructArray::new(
565 data_file_fields,
566 data_file_columns,
567 data_file_nulls,
568 ));
569 }
570 Ok(StructArray::new(fields, columns, nulls))
571}
572
573fn fill_nulls(array: &ArrayRef, rows: Option<&BooleanArray>, value: i64) -> Result<ArrayRef> {
576 if array.null_count() == 0 {
577 return Ok(array.clone());
578 }
579 let mask = match rows {
580 Some(rows) => and(&is_null(array)?, rows)?,
581 None => is_null(array)?,
582 };
583 Ok(zip(&mask, &Int64Array::new_scalar(value), array)?)
584}
585
586fn inherit_first_row_ids(
593 first_row_ids: &Int64Array,
594 status: &Int32Array,
595 record_counts: &Int64Array,
596 inheritance: &mut Inheritance,
597) -> Result<ArrayRef> {
598 let Some(next_row_id) = inheritance.next_row_id.as_mut() else {
599 return Ok(new_null_array(&DataType::Int64, first_row_ids.len()));
600 };
601 if first_row_ids.null_count() == 0 {
602 return Ok(Arc::new(first_row_ids.clone()));
603 }
604 let mut builder = Int64Builder::with_capacity(first_row_ids.len());
605 for ((first_row_id, &status), &record_count) in first_row_ids
606 .iter()
607 .zip(status.values())
608 .zip(record_counts.values())
609 {
610 match first_row_id {
611 None if status != ManifestStatus::Deleted as i32 => {
612 if record_count < 0 {
613 return Err(invalid_data!(
614 "negative record_count {record_count} assigning first row IDs in {}",
615 inheritance.manifest_path
616 ));
617 }
618 builder.append_value(*next_row_id);
619 *next_row_id = next_row_id.checked_add(record_count).ok_or_else(|| {
620 invalid_data!(
621 "row ID overflow assigning first row IDs in {}",
622 inheritance.manifest_path
623 )
624 })?;
625 }
626 first_row_id => builder.append_option(first_row_id),
627 }
628 }
629 Ok(Arc::new(builder.finish()))
630}
631
632#[cfg(test)]
633mod tests {
634 use std::collections::HashMap;
635 use std::sync::Arc;
636
637 use apache_avro::types::Value as AvroValue;
638 use apache_avro::{Codec, DeflateSettings, ZstandardSettings};
639 use arrow_array::cast::AsArray;
640 use arrow_array::types::{Int32Type, Int64Type};
641 use arrow_array::{
642 Array, ArrayRef, BinaryArray, BooleanArray, Date32Array, Decimal128Array,
643 FixedSizeBinaryArray, Float32Array, Float64Array, Int32Array, Int64Array, StringArray,
644 Time64MicrosecondArray, TimestampMicrosecondArray, TimestampNanosecondArray,
645 };
646 use arrow_buffer::{NullBuffer, OffsetBuffer};
647 use arrow_schema::SchemaRef;
648 use arrow_select::concat::concat_batches;
649
650 use super::*;
651 use crate::ErrorKind;
652 use crate::arrow::arrow_schema_to_schema;
653 use crate::io::FileIO;
654 use crate::spec::{
655 DataContentType, DataFile, DataFileBuilder, DataFileFormat, Datum, FieldSummary, ListType,
656 Literal, Manifest, ManifestEntry, ManifestList, ManifestListWriter, ManifestStatus,
657 ManifestWriter, ManifestWriterBuilder, MapType, NestedField, PartitionSpec,
658 PrimitiveLiteral, PrimitiveType, Schema, Struct, StructType, Transform, Type,
659 UNASSIGNED_SEQUENCE_NUMBER,
660 };
661
662 fn table_schema() -> Schema {
663 let columns = [
664 (1, "b", PrimitiveType::Boolean),
665 (2, "i", PrimitiveType::Int),
666 (3, "l", PrimitiveType::Long),
667 (4, "f", PrimitiveType::Float),
668 (5, "d", PrimitiveType::Double),
669 (6, "dec", PrimitiveType::Decimal {
670 precision: 38,
671 scale: 10,
672 }),
673 (7, "date", PrimitiveType::Date),
674 (8, "time", PrimitiveType::Time),
675 (9, "ts", PrimitiveType::Timestamp),
676 (10, "tstz", PrimitiveType::Timestamptz),
677 (11, "ts_ns", PrimitiveType::TimestampNs),
678 (12, "tstz_ns", PrimitiveType::TimestamptzNs),
679 (13, "s", PrimitiveType::String),
680 (14, "fx", PrimitiveType::Fixed(3)),
681 (15, "bin", PrimitiveType::Binary),
682 ];
683 Schema::builder()
684 .with_fields(
685 columns.into_iter().map(|(id, name, ty)| {
686 NestedField::optional(id, name, Type::Primitive(ty)).into()
687 }),
688 )
689 .build()
690 .unwrap()
691 }
692
693 fn partition_spec(fields: &[(&str, Transform)]) -> PartitionSpec {
694 fields
695 .iter()
696 .fold(
697 PartitionSpec::builder(table_schema()).with_spec_id(0),
698 |builder, (source, transform)| {
699 builder
700 .add_partition_field(*source, format!("{source}_{transform}"), *transform)
701 .unwrap()
702 },
703 )
704 .build()
705 .unwrap()
706 }
707
708 fn data_file(content: DataContentType, path: &str) -> DataFileBuilder {
709 let mut builder = DataFileBuilder::default();
710 builder
711 .content(content)
712 .file_path(path.to_string())
713 .file_format(DataFileFormat::Parquet)
714 .record_count(1)
715 .file_size_in_bytes(10);
716 builder
717 }
718
719 async fn write_manifest(
722 spec: PartitionSpec,
723 snapshot_id: Option<i64>,
724 build: impl FnOnce(ManifestWriterBuilder) -> ManifestWriter,
725 add: impl FnOnce(&mut ManifestWriter),
726 ) -> (Vec<u8>, ManifestFile) {
727 let io = FileIO::new_with_memory();
728 let output = io.new_output("memory:///manifest.avro").unwrap();
729 let mut writer = build(ManifestWriterBuilder::new(
730 output,
731 snapshot_id,
732 Arc::new(table_schema()),
733 spec,
734 ));
735 add(&mut writer);
736 let manifest_file = writer.write_manifest_file().await.unwrap();
737 let bytes = io
738 .new_input("memory:///manifest.avro")
739 .unwrap()
740 .read()
741 .await
742 .unwrap()
743 .to_vec();
744 (bytes, manifest_file)
745 }
746
747 fn concat(
748 schema: SchemaRef,
749 batches: impl Iterator<Item = Result<RecordBatch>>,
750 ) -> Result<RecordBatch> {
751 let batches = batches.collect::<Result<Vec<_>>>()?;
752 Ok(concat_batches(&schema, &batches)?)
753 }
754
755 fn read_entries(bytes: &[u8], manifest_file: &ManifestFile) -> Result<RecordBatch> {
756 let reader = read_manifest_entries(bytes, manifest_file)?;
757 concat(reader.schema(), reader)
758 }
759
760 fn read_list(bytes: &[u8]) -> Result<RecordBatch> {
761 let reader = read_manifest_list(bytes)?;
762 concat(reader.schema(), reader)
763 }
764
765 fn assert_batches_match_whole<I: Iterator<Item = Result<RecordBatch>>>(
768 read: impl Fn(usize) -> (SchemaRef, I),
769 ) {
770 let (schema, batches) = read(usize::MAX);
771 let whole = concat(schema.clone(), batches).unwrap();
772 for batch_size in 1..=3 {
773 let (batch_schema, batches) = read(batch_size);
774 assert_eq!(batch_schema, schema);
775 let batches: Vec<_> = batches.map(Result::unwrap).collect();
776 for batch in &batches {
777 assert_eq!(batch.schema(), schema, "batch size {batch_size}");
778 assert!(
779 (1..=batch_size).contains(&batch.num_rows()),
780 "batch size {batch_size}"
781 );
782 }
783 assert_eq!(
784 concat_batches(&schema, &batches).unwrap(),
785 whole,
786 "batch size {batch_size}"
787 );
788 }
789 }
790
791 fn column(batch: &RecordBatch, name: &str) -> ArrayRef {
792 batch.column_by_name(name).unwrap().clone()
793 }
794
795 fn data_file_column(batch: &RecordBatch, name: &str) -> ArrayRef {
796 column(batch, "data_file")
797 .as_struct()
798 .column_by_name(name)
799 .unwrap()
800 .clone()
801 }
802
803 fn long_map(column: &ArrayRef, row: usize) -> Vec<(i32, i64)> {
805 let entries = column.as_map().value(row);
806 let keys = entries.column(0).as_primitive::<Int32Type>();
807 let values = entries.column(1).as_primitive::<Int64Type>();
808 let mut pairs: Vec<_> = keys
809 .values()
810 .iter()
811 .copied()
812 .zip(values.values().iter().copied())
813 .collect();
814 pairs.sort_unstable();
815 pairs
816 }
817
818 fn binary_map(column: &ArrayRef, row: usize) -> Vec<(i32, Vec<u8>)> {
820 let entries = column.as_map().value(row);
821 let keys = entries.column(0).as_primitive::<Int32Type>();
822 let values = entries.column(1).as_binary::<i32>();
823 let mut pairs: Vec<_> = keys
824 .values()
825 .iter()
826 .copied()
827 .zip(values.iter().map(|v| v.unwrap().to_vec()))
828 .collect();
829 pairs.sort_unstable();
830 pairs
831 }
832
833 fn int_map_type(key_id: i32, value_id: i32, value: PrimitiveType) -> Type {
834 Type::Map(MapType::new(
835 NestedField::map_key_element(key_id, Type::Primitive(PrimitiveType::Int)).into(),
836 NestedField::map_value_element(value_id, Type::Primitive(value), true).into(),
837 ))
838 }
839
840 fn list_type(element_id: i32, element: PrimitiveType) -> Type {
841 Type::List(ListType::new(
842 NestedField::list_element(element_id, Type::Primitive(element), true).into(),
843 ))
844 }
845
846 fn set_avro_metadata(bytes: &[u8], key: &str, value: Option<&str>) -> Vec<u8> {
849 let reader = apache_avro::Reader::new(bytes).unwrap();
850 let schema = reader.writer_schema().clone();
851 let mut metadata = reader.user_metadata().clone();
852 match value {
853 Some(value) => metadata.insert(key.to_string(), value.as_bytes().to_vec()),
854 None => metadata.remove(key),
855 };
856 let mut writer = apache_avro::Writer::new(&schema, Vec::new()).unwrap();
857 for (key, value) in metadata {
858 writer.add_user_metadata(key, value).unwrap();
859 }
860 for record in reader {
861 writer.append_value(record.unwrap()).unwrap();
862 }
863 writer.into_inner().unwrap()
864 }
865
866 fn rewrite_manifest(
869 bytes: &[u8],
870 codec: Codec,
871 edit_schema: impl Fn(&mut serde_json::Value),
872 edit_record: impl Fn(&mut AvroValue),
873 ) -> Vec<u8> {
874 let reader = apache_avro::Reader::new(bytes).unwrap();
875 let mut schema = serde_json::to_value(reader.writer_schema()).unwrap();
876 edit_schema(&mut schema);
877 let schema = apache_avro::Schema::parse(&schema).unwrap();
878 let metadata = reader.user_metadata().clone();
879 let mut writer = apache_avro::Writer::with_codec(&schema, Vec::new(), codec).unwrap();
880 for (key, value) in metadata {
881 writer.add_user_metadata(key, value).unwrap();
882 }
883 for record in reader {
884 let mut record = record.unwrap();
885 edit_record(&mut record);
886 writer.append_value(record).unwrap();
887 }
888 writer.into_inner().unwrap()
889 }
890
891 fn for_each_avro_field(schema: &mut serde_json::Value, edit: &impl Fn(&mut serde_json::Value)) {
893 match schema {
894 serde_json::Value::Object(object) => {
895 if let Some(serde_json::Value::Array(fields)) = object.get_mut("fields") {
896 for field in fields {
897 edit(field);
898 for_each_avro_field(field.get_mut("type").unwrap(), edit);
899 }
900 }
901 if let Some(items) = object.get_mut("items") {
902 for_each_avro_field(items, edit);
903 }
904 }
905 serde_json::Value::Array(branches) => {
906 for branch in branches {
907 for_each_avro_field(branch, edit);
908 }
909 }
910 _ => {}
911 }
912 }
913
914 fn rename_avro_field(value: &mut AvroValue, from: &str, to: &str) {
916 match value {
917 AvroValue::Record(fields) => {
918 for (name, field) in fields {
919 if name == from {
920 *name = to.to_string();
921 }
922 rename_avro_field(field, from, to);
923 }
924 }
925 AvroValue::Union(_, value) => rename_avro_field(value, from, to),
926 AvroValue::Array(items) => {
927 for item in items {
928 rename_avro_field(item, from, to);
929 }
930 }
931 _ => {}
932 }
933 }
934
935 fn rename_fields(bytes: &[u8], renames: &[(&str, &str)]) -> Vec<u8> {
938 rewrite_manifest(
939 bytes,
940 Codec::Null,
941 |schema| {
942 for_each_avro_field(schema, &|field| {
943 for (from, to) in renames {
944 if field["name"] == *from {
945 field["name"] = (*to).into();
946 }
947 }
948 })
949 },
950 |record| {
951 for (from, to) in renames {
952 rename_avro_field(record, from, to);
953 }
954 },
955 )
956 }
957
958 #[tokio::test]
959 async fn test_schema_is_spec_manifest_entry() {
960 let spec = partition_spec(&[("i", Transform::Identity), ("s", Transform::Truncate(2))]);
961 let (bytes, manifest_file) =
962 write_manifest(spec, Some(1), |b| b.build_v2_data(), |_| {}).await;
963 let batch = read_entries(&bytes, &manifest_file).unwrap();
964 assert_eq!(batch.num_rows(), 0);
965
966 let long = || Type::Primitive(PrimitiveType::Long);
967 let int = || Type::Primitive(PrimitiveType::Int);
968 let string = || Type::Primitive(PrimitiveType::String);
969 let data_file = StructType::new(vec![
970 NestedField::required(134, "content", int()).into(),
971 NestedField::required(100, "file_path", string()).into(),
972 NestedField::required(101, "file_format", string()).into(),
973 NestedField::required(
974 102,
975 "partition",
976 Type::Struct(StructType::new(vec![
977 NestedField::optional(1000, "i_identity", int()).into(),
978 NestedField::optional(1001, "s_truncate[2]", string()).into(),
979 ])),
980 )
981 .into(),
982 NestedField::required(103, "record_count", long()).into(),
983 NestedField::required(104, "file_size_in_bytes", long()).into(),
984 NestedField::optional(
985 108,
986 "column_sizes",
987 int_map_type(117, 118, PrimitiveType::Long),
988 )
989 .into(),
990 NestedField::optional(
991 109,
992 "value_counts",
993 int_map_type(119, 120, PrimitiveType::Long),
994 )
995 .into(),
996 NestedField::optional(
997 110,
998 "null_value_counts",
999 int_map_type(121, 122, PrimitiveType::Long),
1000 )
1001 .into(),
1002 NestedField::optional(
1003 137,
1004 "nan_value_counts",
1005 int_map_type(138, 139, PrimitiveType::Long),
1006 )
1007 .into(),
1008 NestedField::optional(
1009 125,
1010 "lower_bounds",
1011 int_map_type(126, 127, PrimitiveType::Binary),
1012 )
1013 .into(),
1014 NestedField::optional(
1015 128,
1016 "upper_bounds",
1017 int_map_type(129, 130, PrimitiveType::Binary),
1018 )
1019 .into(),
1020 NestedField::optional(131, "key_metadata", Type::Primitive(PrimitiveType::Binary))
1021 .into(),
1022 NestedField::optional(132, "split_offsets", list_type(133, PrimitiveType::Long)).into(),
1023 NestedField::optional(135, "equality_ids", list_type(136, PrimitiveType::Int)).into(),
1024 NestedField::optional(140, "sort_order_id", int()).into(),
1025 NestedField::optional(142, "first_row_id", long()).into(),
1026 NestedField::optional(143, "referenced_data_file", string()).into(),
1027 NestedField::optional(144, "content_offset", long()).into(),
1028 NestedField::optional(145, "content_size_in_bytes", long()).into(),
1029 ]);
1030 let expected = Schema::builder()
1031 .with_fields(vec![
1032 NestedField::required(0, "status", int()).into(),
1033 NestedField::optional(1, "snapshot_id", long()).into(),
1034 NestedField::optional(3, "sequence_number", long()).into(),
1035 NestedField::optional(4, "file_sequence_number", long()).into(),
1036 NestedField::required(2, "data_file", Type::Struct(data_file)).into(),
1037 ])
1038 .build()
1039 .unwrap();
1040 assert_eq!(
1041 arrow_schema_to_schema(&batch.schema()).unwrap().as_struct(),
1042 expected.as_struct()
1043 );
1044 }
1045
1046 #[tokio::test]
1047 async fn test_data_file_columns() {
1048 let added = DataFile {
1049 column_sizes: HashMap::from([(1, 11)]),
1050 value_counts: HashMap::from([(1, 10), (3, 9)]),
1051 null_value_counts: HashMap::from([(1, 1)]),
1052 nan_value_counts: HashMap::from([(4, 0)]),
1053 lower_bounds: HashMap::from([(2, Datum::int(-1)), (13, Datum::string("abc"))]),
1054 upper_bounds: HashMap::from([(2, Datum::int(7))]),
1055 key_metadata: Some(vec![1, 2]),
1056 split_offsets: Some(vec![4, 100]),
1057 sort_order_id: Some(0),
1058 ..data_file(DataContentType::Data, "s3://b/data/0.parquet")
1059 .record_count(10)
1060 .file_size_in_bytes(100)
1061 .build()
1062 .unwrap()
1063 };
1064 let existing = data_file(DataContentType::Data, "s3://b/data/1.parquet")
1065 .build()
1066 .unwrap();
1067 let (bytes, manifest_file) = write_manifest(
1068 partition_spec(&[]),
1069 Some(1),
1070 |b| b.build_v2_data(),
1071 |writer| {
1072 writer.add_file(added, 2).unwrap();
1073 writer.add_existing_file(existing, 5, 3, Some(3)).unwrap();
1074 },
1075 )
1076 .await;
1077 let batch = read_entries(&bytes, &manifest_file).unwrap();
1078 assert_eq!(batch.num_rows(), 2);
1079
1080 assert_eq!(
1081 column(&batch, "status").as_ref(),
1082 &Int32Array::from(vec![1, 0]) as &dyn Array
1083 );
1084 assert_eq!(
1085 column(&batch, "snapshot_id").as_ref(),
1086 &Int64Array::from(vec![1, 5]) as &dyn Array
1087 );
1088 assert_eq!(
1089 column(&batch, "sequence_number").as_ref(),
1090 &Int64Array::from(vec![2, 3]) as &dyn Array
1091 );
1092
1093 let column = |name: &str| data_file_column(&batch, name);
1094 assert_eq!(
1095 column("content").as_ref(),
1096 &Int32Array::from(vec![0, 0]) as &dyn Array
1097 );
1098 assert_eq!(
1099 column("file_path").as_ref(),
1100 &StringArray::from(vec!["s3://b/data/0.parquet", "s3://b/data/1.parquet"])
1101 as &dyn Array
1102 );
1103 assert_eq!(
1104 column("file_format").as_ref(),
1105 &StringArray::from(vec!["PARQUET", "PARQUET"]) as &dyn Array
1106 );
1107 assert_eq!(
1108 column("record_count").as_ref(),
1109 &Int64Array::from(vec![10, 1]) as &dyn Array
1110 );
1111 assert_eq!(
1112 column("file_size_in_bytes").as_ref(),
1113 &Int64Array::from(vec![100, 10]) as &dyn Array
1114 );
1115 assert_eq!(long_map(&column("column_sizes"), 0), vec![(1, 11)]);
1116 assert_eq!(long_map(&column("value_counts"), 0), vec![(1, 10), (3, 9)]);
1117 assert_eq!(long_map(&column("null_value_counts"), 0), vec![(1, 1)]);
1118 assert_eq!(long_map(&column("nan_value_counts"), 0), vec![(4, 0)]);
1119 assert_eq!(long_map(&column("value_counts"), 1), vec![]);
1120 assert_eq!(binary_map(&column("lower_bounds"), 0), vec![
1121 (2, (-1i32).to_le_bytes().to_vec()),
1122 (13, b"abc".to_vec()),
1123 ]);
1124 assert_eq!(binary_map(&column("upper_bounds"), 0), vec![(
1125 2,
1126 7i32.to_le_bytes().to_vec()
1127 )]);
1128 assert_eq!(
1129 column("key_metadata").as_ref(),
1130 &BinaryArray::from(vec![Some(&[1u8, 2][..]), None]) as &dyn Array
1131 );
1132 let split_offsets = column("split_offsets");
1133 assert_eq!(
1134 split_offsets.as_list::<i32>().value(0).as_ref(),
1135 &Int64Array::from(vec![4, 100]) as &dyn Array
1136 );
1137 assert_eq!(
1138 column("sort_order_id").as_ref(),
1139 &Int32Array::from(vec![Some(0), None]) as &dyn Array
1140 );
1141 }
1142
1143 #[tokio::test]
1144 async fn test_delete_file_columns() {
1145 let position_deletes = DataFile {
1146 file_format: DataFileFormat::Puffin,
1147 referenced_data_file: Some("s3://b/data/0.parquet".to_string()),
1148 content_offset: Some(4),
1149 content_size_in_bytes: Some(40),
1150 ..data_file(DataContentType::PositionDeletes, "s3://b/data/1.puffin")
1151 .build()
1152 .unwrap()
1153 };
1154 let equality_deletes = DataFile {
1155 equality_ids: Some(vec![1, 3]),
1156 ..data_file(DataContentType::EqualityDeletes, "s3://b/data/2.parquet")
1157 .build()
1158 .unwrap()
1159 };
1160 let (bytes, manifest_file) = write_manifest(
1161 partition_spec(&[]),
1162 Some(1),
1163 |b| b.build_v3_deletes(),
1164 |writer| {
1165 writer.add_file(position_deletes, 2).unwrap();
1166 writer.add_file(equality_deletes, 2).unwrap();
1167 },
1168 )
1169 .await;
1170 let batch = read_entries(&bytes, &manifest_file).unwrap();
1171
1172 let column = |name: &str| data_file_column(&batch, name);
1173 assert_eq!(
1174 column("content").as_ref(),
1175 &Int32Array::from(vec![1, 2]) as &dyn Array
1176 );
1177 assert_eq!(
1178 column("file_format").as_ref(),
1179 &StringArray::from(vec!["PUFFIN", "PARQUET"]) as &dyn Array
1180 );
1181 assert_eq!(
1182 column("referenced_data_file").as_ref(),
1183 &StringArray::from(vec![Some("s3://b/data/0.parquet"), None]) as &dyn Array
1184 );
1185 assert_eq!(
1186 column("content_offset").as_ref(),
1187 &Int64Array::from(vec![Some(4), None]) as &dyn Array
1188 );
1189 assert_eq!(
1190 column("content_size_in_bytes").as_ref(),
1191 &Int64Array::from(vec![Some(40), None]) as &dyn Array
1192 );
1193 let equality_ids = column("equality_ids");
1194 let equality_ids = equality_ids.as_list::<i32>();
1195 assert!(equality_ids.is_null(0));
1196 assert_eq!(
1197 equality_ids.value(1).as_ref(),
1198 &Int32Array::from(vec![1, 3]) as &dyn Array
1199 );
1200 }
1201
1202 #[tokio::test]
1203 async fn test_partition_values() {
1204 let identity_columns = [
1205 "b", "i", "l", "f", "d", "dec", "date", "time", "ts", "tstz", "ts_ns", "tstz_ns", "s",
1206 "fx", "bin",
1207 ];
1208 let mut fields: Vec<_> = identity_columns
1209 .iter()
1210 .map(|column| (*column, Transform::Identity))
1211 .collect();
1212 fields.extend([
1213 ("i", Transform::Bucket(4)),
1214 ("ts", Transform::Day),
1215 ("s", Transform::Truncate(2)),
1216 ("l", Transform::Void),
1217 ]);
1218 let spec = partition_spec(&fields);
1219
1220 let values = Struct::from_iter([
1221 Some(Literal::bool(true)),
1222 Some(Literal::int(-7)),
1223 Some(Literal::long(1i64 << 40)),
1224 Some(Literal::float(1.5f32)),
1225 Some(Literal::double(-2.25)),
1226 Some(Literal::decimal(-12_345)),
1227 Some(Literal::date(19_000)),
1228 Some(Literal::time(3_600_000_000)),
1229 Some(Literal::timestamp(1_700_000_000_000_000)),
1230 Some(Literal::timestamptz(1_700_000_000_000_001)),
1231 Some(Literal::timestamp_nano(1_700_000_000_000_000_002)),
1232 Some(Literal::timestamptz_nano(1_700_000_000_000_000_003)),
1233 Some(Literal::string("abc")),
1234 Some(Literal::fixed([1u8, 2, 3])),
1235 Some(Literal::binary([9u8])),
1236 Some(Literal::int(3)),
1237 Some(Literal::date(19_675)),
1238 Some(Literal::string("ab")),
1239 None,
1240 ]);
1241 let nulls = Struct::from_iter(fields.iter().map(|_| None));
1242 let (bytes, manifest_file) = write_manifest(
1243 spec,
1244 Some(1),
1245 |b| b.build_v2_data(),
1246 |writer| {
1247 for (i, partition) in [values, nulls].into_iter().enumerate() {
1248 let file = DataFile {
1249 partition,
1250 ..data_file(DataContentType::Data, &format!("s3://b/data/{i}.parquet"))
1251 .build()
1252 .unwrap()
1253 };
1254 writer.add_file(file, 1).unwrap();
1255 }
1256 },
1257 )
1258 .await;
1259 let batch = read_entries(&bytes, &manifest_file).unwrap();
1260
1261 let partition = data_file_column(&batch, "partition");
1262 let partition = partition.as_struct();
1263 let expected: Vec<ArrayRef> = vec![
1264 Arc::new(BooleanArray::from(vec![Some(true), None])),
1265 Arc::new(Int32Array::from(vec![Some(-7), None])),
1266 Arc::new(Int64Array::from(vec![Some(1i64 << 40), None])),
1267 Arc::new(Float32Array::from(vec![Some(1.5), None])),
1268 Arc::new(Float64Array::from(vec![Some(-2.25), None])),
1269 Arc::new(
1270 Decimal128Array::from(vec![Some(-12_345), None])
1271 .with_precision_and_scale(38, 10)
1272 .unwrap(),
1273 ),
1274 Arc::new(Date32Array::from(vec![Some(19_000), None])),
1275 Arc::new(Time64MicrosecondArray::from(vec![
1276 Some(3_600_000_000),
1277 None,
1278 ])),
1279 Arc::new(TimestampMicrosecondArray::from(vec![
1280 Some(1_700_000_000_000_000),
1281 None,
1282 ])),
1283 Arc::new(
1284 TimestampMicrosecondArray::from(vec![Some(1_700_000_000_000_001), None])
1285 .with_timezone("+00:00"),
1286 ),
1287 Arc::new(TimestampNanosecondArray::from(vec![
1288 Some(1_700_000_000_000_000_002),
1289 None,
1290 ])),
1291 Arc::new(
1292 TimestampNanosecondArray::from(vec![Some(1_700_000_000_000_000_003), None])
1293 .with_timezone("+00:00"),
1294 ),
1295 Arc::new(StringArray::from(vec![Some("abc"), None])),
1296 Arc::new(
1297 FixedSizeBinaryArray::try_from_sparse_iter_with_size(
1298 [Some(vec![1u8, 2, 3]), None].into_iter(),
1299 3,
1300 )
1301 .unwrap(),
1302 ),
1303 Arc::new(BinaryArray::from(vec![Some(&[9u8][..]), None])),
1304 Arc::new(Int32Array::from(vec![Some(3), None])),
1305 Arc::new(Date32Array::from(vec![Some(19_675), None])),
1306 Arc::new(StringArray::from(vec![Some("ab"), None])),
1307 Arc::new(Int64Array::from(vec![None, None])),
1308 ];
1309 assert_eq!(partition.num_columns(), expected.len());
1310 for (index, expected) in expected.iter().enumerate() {
1311 assert_eq!(
1312 partition.column(index).as_ref(),
1313 expected.as_ref(),
1314 "partition field {}",
1315 partition.fields()[index].name()
1316 );
1317 }
1318 }
1319
1320 #[tokio::test]
1321 async fn test_v1_manifest_reads_with_v2_defaults() {
1322 let file = data_file(DataContentType::Data, "s3://b/data/0.parquet")
1323 .build()
1324 .unwrap();
1325 let (bytes, mut manifest_file) = write_manifest(
1326 partition_spec(&[]),
1327 Some(1),
1328 |b| b.build_v1(),
1329 |writer| writer.add_file(file, -1).unwrap(),
1330 )
1331 .await;
1332 manifest_file.sequence_number = 9;
1335 let batch = read_entries(&bytes, &manifest_file).unwrap();
1336
1337 assert_eq!(
1338 data_file_column(&batch, "content").as_ref(),
1339 &Int32Array::from(vec![0]) as &dyn Array
1340 );
1341 for name in ["sequence_number", "file_sequence_number"] {
1342 assert_eq!(
1343 column(&batch, name).as_ref(),
1344 &Int64Array::from(vec![0]) as &dyn Array,
1345 "{name}"
1346 );
1347 }
1348 assert_eq!(
1349 column(&batch, "snapshot_id").as_ref(),
1350 &Int64Array::from(vec![1]) as &dyn Array
1351 );
1352 }
1353
1354 #[tokio::test]
1355 async fn test_snapshot_id_and_sequence_number_inheritance() {
1356 let file = |i: usize| {
1357 data_file(DataContentType::Data, &format!("s3://b/data/{i}.parquet"))
1358 .build()
1359 .unwrap()
1360 };
1361 let (bytes, mut manifest_file) = write_manifest(
1362 partition_spec(&[]),
1363 None,
1364 |b| b.build_v2_data(),
1365 |writer| {
1366 writer.add_file(file(0), -1).unwrap();
1367 writer.add_delete_file(file(1), 4, Some(4)).unwrap();
1368 writer.add_existing_file(file(2), 3, 5, Some(5)).unwrap();
1369 },
1370 )
1371 .await;
1372 let bytes = rewrite_manifest(
1374 &bytes,
1375 Codec::Null,
1376 |_| {},
1377 |record| {
1378 let AvroValue::Record(fields) = record else {
1379 panic!("manifest entry is not a record");
1380 };
1381 if fields.contains(&("status".to_string(), AvroValue::Int(0))) {
1382 for (name, value) in fields {
1383 if name.ends_with("sequence_number") {
1384 *value = AvroValue::Union(0, Box::new(AvroValue::Null));
1385 }
1386 }
1387 }
1388 },
1389 );
1390 manifest_file.added_snapshot_id = 42;
1391 manifest_file.sequence_number = 7;
1392 let batch = read_entries(&bytes, &manifest_file).unwrap();
1393
1394 assert_eq!(
1395 column(&batch, "snapshot_id").as_ref(),
1396 &Int64Array::from(vec![42, 42, 3]) as &dyn Array
1397 );
1398 for name in ["sequence_number", "file_sequence_number"] {
1399 assert_eq!(
1400 column(&batch, name).as_ref(),
1401 &Int64Array::from(vec![Some(7), Some(4), None]) as &dyn Array,
1402 "{name}"
1403 );
1404 }
1405 }
1406
1407 #[tokio::test]
1408 async fn test_v2_manifest_without_file_sequence_number_inherits() {
1409 let file = |i: usize| {
1410 data_file(DataContentType::Data, &format!("s3://b/data/{i}.parquet"))
1411 .build()
1412 .unwrap()
1413 };
1414 let (bytes, mut manifest_file) = write_manifest(
1415 partition_spec(&[]),
1416 Some(1),
1417 |b| b.build_v2_data(),
1418 |writer| {
1419 writer.add_file(file(0), -1).unwrap();
1420 writer.add_existing_file(file(1), 3, 5, Some(5)).unwrap();
1421 },
1422 )
1423 .await;
1424 let bytes = rewrite_manifest(
1426 &bytes,
1427 Codec::Null,
1428 |schema| {
1429 schema["fields"]
1430 .as_array_mut()
1431 .unwrap()
1432 .retain(|field| field["name"] != "file_sequence_number")
1433 },
1434 |record| {
1435 let AvroValue::Record(fields) = record else {
1436 panic!("manifest entry is not a record");
1437 };
1438 fields.retain(|(name, _)| name != "file_sequence_number");
1439 },
1440 );
1441 manifest_file.sequence_number = 7;
1442 let batch = read_entries(&bytes, &manifest_file).unwrap();
1443
1444 assert_eq!(
1445 column(&batch, "file_sequence_number").as_ref(),
1446 &Int64Array::from(vec![Some(7), None]) as &dyn Array
1447 );
1448 }
1449
1450 async fn first_row_id_manifest() -> (Vec<u8>, ManifestFile) {
1451 let file = |i: usize, record_count: u64, first_row_id: Option<i64>| DataFile {
1452 first_row_id,
1453 ..data_file(DataContentType::Data, &format!("s3://b/data/{i}.parquet"))
1454 .record_count(record_count)
1455 .build()
1456 .unwrap()
1457 };
1458 let entry = |status, data_file| ManifestEntry {
1459 status,
1460 snapshot_id: Some(1),
1461 sequence_number: Some(1),
1462 file_sequence_number: Some(1),
1463 data_file,
1464 };
1465 write_manifest(
1466 partition_spec(&[]),
1467 Some(1),
1468 |b| b.build_v3_data(),
1469 |writer| {
1470 writer.add_file(file(0, 10, None), 1).unwrap();
1471 writer
1472 .add_existing_entry(entry(ManifestStatus::Existing, file(1, 3, Some(5))))
1473 .unwrap();
1474 writer
1475 .add_delete_entry(entry(ManifestStatus::Deleted, file(2, 4, None)))
1476 .unwrap();
1477 writer
1478 .add_existing_entry(entry(ManifestStatus::Existing, file(3, 6, None)))
1479 .unwrap();
1480 writer.add_file(file(4, 2, None), 1).unwrap();
1481 },
1482 )
1483 .await
1484 }
1485
1486 #[tokio::test]
1487 async fn test_first_row_id_inheritance() {
1488 let (bytes, mut manifest_file) = first_row_id_manifest().await;
1489 manifest_file.first_row_id = Some(100);
1490 let batch = read_entries(&bytes, &manifest_file).unwrap();
1491 assert_eq!(
1493 data_file_column(&batch, "first_row_id").as_ref(),
1494 &Int64Array::from(vec![Some(100), Some(5), None, Some(110), Some(116)]) as &dyn Array
1495 );
1496
1497 manifest_file.first_row_id = None;
1498 let batch = read_entries(&bytes, &manifest_file).unwrap();
1499 assert_eq!(data_file_column(&batch, "first_row_id").null_count(), 5);
1500
1501 manifest_file.first_row_id = Some(i64::MAX as u64 - 5);
1502 let err = read_entries(&bytes, &manifest_file).unwrap_err();
1503 assert_eq!(err.kind(), ErrorKind::DataInvalid);
1504
1505 let bytes = negate_record_count(&bytes, 10);
1507 manifest_file.first_row_id = Some(100);
1508 let err = read_entries(&bytes, &manifest_file).unwrap_err();
1509 assert_eq!(err.kind(), ErrorKind::DataInvalid);
1510 }
1511
1512 fn negate_record_count(bytes: &[u8], record_count: i64) -> Vec<u8> {
1515 rewrite_manifest(
1516 bytes,
1517 Codec::Null,
1518 |_| {},
1519 |record| {
1520 let AvroValue::Record(fields) = record else {
1521 panic!("manifest entry is not a record");
1522 };
1523 let Some((_, AvroValue::Record(data_file))) =
1524 fields.iter_mut().find(|(name, _)| name == "data_file")
1525 else {
1526 panic!("manifest entry has no data_file record");
1527 };
1528 for (name, value) in data_file {
1529 if name == "record_count" && *value == AvroValue::Long(record_count) {
1530 *value = AvroValue::Long(-record_count);
1531 }
1532 }
1533 },
1534 )
1535 }
1536
1537 #[tokio::test]
1538 async fn test_manifest_batches_match_whole_manifest() {
1539 let (first_row_id_bytes, mut first_row_id_manifest_file) = first_row_id_manifest().await;
1540 first_row_id_manifest_file.first_row_id = Some(100);
1541 let v1_files = (0..3).map(|i| {
1542 data_file(DataContentType::Data, &format!("s3://b/data/{i}.parquet"))
1543 .build()
1544 .unwrap()
1545 });
1546 let v1 = write_manifest(
1547 partition_spec(&[]),
1548 Some(1),
1549 |b| b.build_v1(),
1550 |writer| v1_files.for_each(|file| writer.add_file(file, -1).unwrap()),
1551 )
1552 .await;
1553 let empty = write_manifest(
1554 partition_spec(&[("i", Transform::Identity)]),
1555 Some(1),
1556 |b| b.build_v2_data(),
1557 |_| {},
1558 )
1559 .await;
1560 let pyiceberg_bytes = std::fs::read(format!(
1561 "{}/testdata/manifests/pyiceberg-v2-data.avro",
1562 env!("CARGO_MANIFEST_DIR")
1563 ))
1564 .unwrap();
1565 let pyiceberg_manifest_file = ManifestFile {
1566 sequence_number: 1,
1567 ..listed_manifest("s3://b/metadata/0.avro", ManifestContentType::Data)
1568 };
1569
1570 for (bytes, manifest_file) in [
1571 (first_row_id_bytes, first_row_id_manifest_file),
1572 v1,
1573 empty,
1574 (pyiceberg_bytes, pyiceberg_manifest_file),
1575 ] {
1576 assert_batches_match_whole(|batch_size| {
1577 let reader =
1578 ManifestEntryBatchReader::try_new(&bytes, &manifest_file, batch_size).unwrap();
1579 (reader.schema(), reader)
1580 });
1581 }
1582 }
1583
1584 #[tokio::test]
1585 async fn test_unknown_manifest_format_version_is_an_error() {
1586 let (bytes, manifest_file) = first_row_id_manifest().await;
1587 let bytes = set_avro_metadata(&bytes, "format-version", Some("4"));
1588 let Err(err) = read_manifest_entries(&bytes, &manifest_file) else {
1589 panic!("a v4 manifest read as an older version");
1590 };
1591 assert_eq!(err.kind(), ErrorKind::DataInvalid);
1592 }
1593
1594 #[tokio::test]
1595 async fn test_manifest_error_ends_the_batches() {
1596 let (bytes, mut manifest_file) = first_row_id_manifest().await;
1597 manifest_file.first_row_id = Some(100);
1598 let bytes = negate_record_count(&bytes, 2);
1601 let mut reader = ManifestEntryBatchReader::try_new(&bytes, &manifest_file, 2).unwrap();
1602 assert_eq!(reader.next().unwrap().unwrap().num_rows(), 2);
1603 assert_eq!(reader.next().unwrap().unwrap().num_rows(), 2);
1604 assert_eq!(
1605 reader.next().unwrap().unwrap_err().kind(),
1606 ErrorKind::DataInvalid
1607 );
1608 assert!(reader.next().is_none());
1609 }
1610
1611 #[tokio::test]
1612 async fn test_compressed_manifests() {
1613 let (bytes, manifest_file) = first_row_id_manifest().await;
1614 let expected = read_entries(&bytes, &manifest_file).unwrap();
1615 for codec in [
1616 Codec::Deflate(DeflateSettings::default()),
1617 Codec::Snappy,
1618 Codec::Zstandard(ZstandardSettings::default()),
1619 ] {
1620 let bytes = rewrite_manifest(&bytes, codec, |_| {}, |_| {});
1621 assert_eq!(
1622 read_entries(&bytes, &manifest_file).unwrap(),
1623 expected,
1624 "{codec:?}"
1625 );
1626 }
1627 }
1628
1629 #[tokio::test]
1630 async fn test_fields_are_matched_by_field_id() {
1631 let file = DataFile {
1632 partition: Struct::from_iter([Some(Literal::int(7))]),
1633 ..data_file(DataContentType::Data, "s3://b/data/0.parquet")
1634 .build()
1635 .unwrap()
1636 };
1637 let (bytes, manifest_file) = write_manifest(
1638 partition_spec(&[("i", Transform::Identity)]),
1639 Some(1),
1640 |b| b.build_v2_data(),
1641 |writer| writer.add_file(file, 1).unwrap(),
1642 )
1643 .await;
1644 let bytes = rename_fields(&bytes, &[
1645 ("file_path", "path"),
1646 ("i_identity", "renamed_partition"),
1647 ]);
1648 let batch = read_entries(&bytes, &manifest_file).unwrap();
1649
1650 assert_eq!(
1651 data_file_column(&batch, "file_path").as_ref(),
1652 &StringArray::from(vec!["s3://b/data/0.parquet"]) as &dyn Array
1653 );
1654 let partition = data_file_column(&batch, "partition");
1655 assert_eq!(
1656 partition.as_struct().column(0).as_ref(),
1657 &Int32Array::from(vec![7]) as &dyn Array
1658 );
1659 }
1660
1661 #[tokio::test]
1662 async fn test_missing_required_field_is_an_error() {
1663 let file = data_file(DataContentType::Data, "s3://b/data/0.parquet")
1664 .build()
1665 .unwrap();
1666 let (bytes, manifest_file) = write_manifest(
1667 partition_spec(&[]),
1668 Some(1),
1669 |b| b.build_v2_data(),
1670 |writer| writer.add_file(file, 1).unwrap(),
1671 )
1672 .await;
1673 let bytes = rewrite_manifest(
1675 &bytes,
1676 Codec::Null,
1677 |schema| {
1678 for_each_avro_field(schema, &|field| {
1679 if field["name"] == "file_path" {
1680 field["field-id"] = 999.into();
1681 }
1682 })
1683 },
1684 |_| {},
1685 );
1686 let err = read_entries(&bytes, &manifest_file).unwrap_err();
1687 assert_eq!(err.kind(), ErrorKind::DataInvalid);
1688 }
1689
1690 #[test]
1691 fn test_manifest_written_by_pyiceberg() {
1692 let bytes = std::fs::read(format!(
1693 "{}/testdata/manifests/pyiceberg-v2-data.avro",
1694 env!("CARGO_MANIFEST_DIR")
1695 ))
1696 .unwrap();
1697 let manifest = Manifest::parse_avro(&bytes).unwrap();
1698 let manifest_file = ManifestFile {
1699 manifest_path: "s3://bucket/metadata/manifest.avro".to_string(),
1700 manifest_length: bytes.len() as i64,
1701 partition_spec_id: 0,
1702 content: ManifestContentType::Data,
1703 sequence_number: 1,
1704 min_sequence_number: 1,
1705 added_snapshot_id: 1,
1706 added_files_count: None,
1707 existing_files_count: None,
1708 deleted_files_count: None,
1709 added_rows_count: None,
1710 existing_rows_count: None,
1711 deleted_rows_count: None,
1712 partitions: None,
1713 key_metadata: None,
1714 first_row_id: None,
1715 };
1716 let batch = read_entries(&bytes, &manifest_file).unwrap();
1717
1718 assert_eq!(batch.num_rows(), manifest.entries().len());
1719 let paths = data_file_column(&batch, "file_path");
1720 let partition = data_file_column(&batch, "partition");
1721 let category = partition.as_struct().column(0).as_string::<i32>().clone();
1722 for (row, entry) in manifest.entries().iter().enumerate() {
1723 let data_file = entry.data_file();
1724 assert_eq!(paths.as_string::<i32>().value(row), data_file.file_path());
1725 let Some(Literal::Primitive(PrimitiveLiteral::String(expected))) =
1726 &data_file.partition()[0]
1727 else {
1728 panic!("partition value is not a string");
1729 };
1730 assert_eq!(category.value(row), expected);
1731 let mut lower_bounds: Vec<_> = data_file
1732 .lower_bounds()
1733 .iter()
1734 .map(|(id, bound)| (*id, bound.to_bytes().unwrap().to_vec()))
1735 .collect();
1736 lower_bounds.sort_unstable();
1737 assert!(!lower_bounds.is_empty());
1738 assert_eq!(
1739 binary_map(&data_file_column(&batch, "lower_bounds"), row),
1740 lower_bounds
1741 );
1742 }
1743 }
1744
1745 fn listed_manifest(path: &str, content: ManifestContentType) -> ManifestFile {
1748 ManifestFile {
1749 manifest_path: path.to_string(),
1750 manifest_length: 100,
1751 partition_spec_id: 0,
1752 content,
1753 sequence_number: UNASSIGNED_SEQUENCE_NUMBER,
1754 min_sequence_number: UNASSIGNED_SEQUENCE_NUMBER,
1755 added_snapshot_id: 1,
1756 added_files_count: Some(1),
1757 existing_files_count: Some(2),
1758 deleted_files_count: Some(3),
1759 added_rows_count: Some(10),
1760 existing_rows_count: Some(20),
1761 deleted_rows_count: Some(30),
1762 partitions: None,
1763 key_metadata: None,
1764 first_row_id: None,
1765 }
1766 }
1767
1768 async fn write_manifest_list(version: FormatVersion, manifests: Vec<ManifestFile>) -> Vec<u8> {
1771 let io = FileIO::new_with_memory();
1772 let path = "memory:///manifest-list.avro";
1773 let output = io.new_output(path).unwrap().writer().await.unwrap();
1774 let mut writer = match version {
1775 FormatVersion::V1 => ManifestListWriter::v1(output, 1, None),
1776 FormatVersion::V2 => ManifestListWriter::v2(output, 1, None, 5),
1777 FormatVersion::V3 => ManifestListWriter::v3(output, 1, None, 5, Some(100)),
1778 };
1779 writer.add_manifests(manifests.into_iter()).unwrap();
1780 writer.close().await.unwrap();
1781 io.new_input(path).unwrap().read().await.unwrap().to_vec()
1782 }
1783
1784 #[tokio::test]
1785 async fn test_schema_is_spec_manifest_file() {
1786 let bytes = write_manifest_list(FormatVersion::V3, vec![]).await;
1787 let batch = read_list(&bytes).unwrap();
1788 assert_eq!(batch.num_rows(), 0);
1789
1790 let long = || Type::Primitive(PrimitiveType::Long);
1791 let int = || Type::Primitive(PrimitiveType::Int);
1792 let boolean = || Type::Primitive(PrimitiveType::Boolean);
1793 let binary = || Type::Primitive(PrimitiveType::Binary);
1794 let field_summary = StructType::new(vec![
1795 NestedField::required(509, "contains_null", boolean()).into(),
1796 NestedField::optional(518, "contains_nan", boolean()).into(),
1797 NestedField::optional(510, "lower_bound", binary()).into(),
1798 NestedField::optional(511, "upper_bound", binary()).into(),
1799 ]);
1800 let expected = Schema::builder()
1801 .with_fields(vec![
1802 NestedField::required(500, "manifest_path", Type::Primitive(PrimitiveType::String))
1803 .into(),
1804 NestedField::required(501, "manifest_length", long()).into(),
1805 NestedField::required(502, "partition_spec_id", int()).into(),
1806 NestedField::required(517, "content", int()).into(),
1807 NestedField::required(515, "sequence_number", long()).into(),
1808 NestedField::required(516, "min_sequence_number", long()).into(),
1809 NestedField::required(503, "added_snapshot_id", long()).into(),
1810 NestedField::optional(504, "added_files_count", int()).into(),
1811 NestedField::optional(505, "existing_files_count", int()).into(),
1812 NestedField::optional(506, "deleted_files_count", int()).into(),
1813 NestedField::optional(512, "added_rows_count", long()).into(),
1814 NestedField::optional(513, "existing_rows_count", long()).into(),
1815 NestedField::optional(514, "deleted_rows_count", long()).into(),
1816 NestedField::optional(
1817 507,
1818 "partitions",
1819 Type::List(ListType::new(
1820 NestedField::list_element(508, Type::Struct(field_summary), true).into(),
1821 )),
1822 )
1823 .into(),
1824 NestedField::optional(519, "key_metadata", binary()).into(),
1825 NestedField::optional(520, "first_row_id", long()).into(),
1826 ])
1827 .build()
1828 .unwrap();
1829 assert_eq!(
1830 arrow_schema_to_schema(&batch.schema()).unwrap().as_struct(),
1831 expected.as_struct()
1832 );
1833 }
1834
1835 #[tokio::test]
1836 async fn test_manifest_list_columns() {
1837 let data = ManifestFile {
1838 partitions: Some(vec![FieldSummary {
1839 contains_null: true,
1840 contains_nan: Some(false),
1841 lower_bound: Some(Datum::int(1).to_bytes().unwrap()),
1842 upper_bound: Some(Datum::int(9).to_bytes().unwrap()),
1843 }]),
1844 key_metadata: Some(vec![1, 2]),
1845 ..listed_manifest("s3://b/metadata/data.avro", ManifestContentType::Data)
1846 };
1847 let existing_deletes = ManifestFile {
1848 partition_spec_id: 1,
1849 sequence_number: 3,
1850 min_sequence_number: 2,
1851 added_snapshot_id: 0,
1852 added_files_count: Some(0),
1853 ..listed_manifest("s3://b/metadata/deletes.avro", ManifestContentType::Deletes)
1854 };
1855 let bytes = write_manifest_list(FormatVersion::V3, vec![data, existing_deletes]).await;
1856 let batch = read_list(&bytes).unwrap();
1857
1858 let schema = batch.schema();
1859 let DataType::List(summary) = schema.field_with_name("partitions").unwrap().data_type()
1860 else {
1861 panic!("partitions is not a list");
1862 };
1863 let DataType::Struct(summary_fields) = summary.data_type() else {
1864 panic!("partitions element is not a struct");
1865 };
1866 let summaries = StructArray::new(
1867 summary_fields.clone(),
1868 vec![
1869 Arc::new(BooleanArray::from(vec![true])),
1870 Arc::new(BooleanArray::from(vec![Some(false)])),
1871 Arc::new(BinaryArray::from(vec![&1i32.to_le_bytes()[..]])),
1872 Arc::new(BinaryArray::from(vec![&9i32.to_le_bytes()[..]])),
1873 ],
1874 None,
1875 );
1876 let columns: Vec<ArrayRef> = vec![
1877 Arc::new(StringArray::from(vec![
1878 "s3://b/metadata/data.avro",
1879 "s3://b/metadata/deletes.avro",
1880 ])),
1881 Arc::new(Int64Array::from(vec![100, 100])),
1882 Arc::new(Int32Array::from(vec![0, 1])),
1883 Arc::new(Int32Array::from(vec![0, 1])),
1884 Arc::new(Int64Array::from(vec![5, 3])),
1885 Arc::new(Int64Array::from(vec![5, 2])),
1886 Arc::new(Int64Array::from(vec![1, 0])),
1887 Arc::new(Int32Array::from(vec![1, 0])),
1888 Arc::new(Int32Array::from(vec![2, 2])),
1889 Arc::new(Int32Array::from(vec![3, 3])),
1890 Arc::new(Int64Array::from(vec![10, 10])),
1891 Arc::new(Int64Array::from(vec![20, 20])),
1892 Arc::new(Int64Array::from(vec![30, 30])),
1893 Arc::new(ListArray::new(
1894 summary.clone(),
1895 OffsetBuffer::from_lengths([1, 0]),
1896 Arc::new(summaries),
1897 Some(NullBuffer::from(vec![true, false])),
1898 )),
1899 Arc::new(BinaryArray::from(vec![Some(&[1u8, 2][..]), None])),
1900 Arc::new(Int64Array::from(vec![Some(100), None])),
1902 ];
1903 assert_eq!(batch, RecordBatch::try_new(schema, columns).unwrap());
1904 }
1905
1906 #[tokio::test]
1907 async fn test_v1_manifest_list_reads_with_v2_defaults() {
1908 let unknown_counts = ManifestFile {
1909 added_files_count: None,
1910 existing_files_count: None,
1911 deleted_files_count: None,
1912 added_rows_count: None,
1913 existing_rows_count: None,
1914 deleted_rows_count: None,
1915 ..listed_manifest("s3://b/metadata/1.avro", ManifestContentType::Data)
1916 };
1917 let bytes = write_manifest_list(FormatVersion::V1, vec![
1918 listed_manifest("s3://b/metadata/0.avro", ManifestContentType::Data),
1919 unknown_counts,
1920 ])
1921 .await;
1922 let without_format_version = set_avro_metadata(&bytes, "format-version", None);
1924 for bytes in [bytes, without_format_version] {
1925 let batch = read_list(&bytes).unwrap();
1926
1927 assert_eq!(
1928 column(&batch, "content").as_ref(),
1929 &Int32Array::from(vec![0, 0]) as &dyn Array
1930 );
1931 for name in ["sequence_number", "min_sequence_number"] {
1932 assert_eq!(
1933 column(&batch, name).as_ref(),
1934 &Int64Array::from(vec![0, 0]) as &dyn Array,
1935 "{name}"
1936 );
1937 }
1938 assert_eq!(
1939 column(&batch, "added_files_count").as_ref(),
1940 &Int32Array::from(vec![Some(1), None]) as &dyn Array
1941 );
1942 assert_eq!(
1943 column(&batch, "deleted_rows_count").as_ref(),
1944 &Int64Array::from(vec![Some(30), None]) as &dyn Array
1945 );
1946 assert_eq!(column(&batch, "first_row_id").null_count(), 2);
1947 }
1948 }
1949
1950 #[tokio::test]
1951 async fn test_manifest_list_fields_are_matched_by_field_id() {
1952 let manifest = ManifestFile {
1953 partitions: Some(vec![FieldSummary {
1954 contains_null: true,
1955 contains_nan: None,
1956 lower_bound: None,
1957 upper_bound: None,
1958 }]),
1959 ..listed_manifest("s3://b/metadata/0.avro", ManifestContentType::Data)
1960 };
1961 let bytes = write_manifest_list(FormatVersion::V2, vec![manifest]).await;
1962 let bytes = rename_fields(&bytes, &[
1963 ("manifest_path", "path"),
1964 ("added_files_count", "added_data_files_count"),
1965 ("contains_null", "has_null"),
1966 ]);
1967 let batch = read_list(&bytes).unwrap();
1968
1969 assert_eq!(
1970 column(&batch, "manifest_path").as_ref(),
1971 &StringArray::from(vec!["s3://b/metadata/0.avro"]) as &dyn Array
1972 );
1973 assert_eq!(
1974 column(&batch, "added_files_count").as_ref(),
1975 &Int32Array::from(vec![1]) as &dyn Array
1976 );
1977 let partitions = column(&batch, "partitions");
1978 assert_eq!(
1979 partitions
1980 .as_list::<i32>()
1981 .value(0)
1982 .as_struct()
1983 .column(0)
1984 .as_ref(),
1985 &BooleanArray::from(vec![true]) as &dyn Array
1986 );
1987 }
1988
1989 #[tokio::test]
1990 async fn test_invalid_manifest_list_is_an_error() {
1991 let err = read_list(&[]).unwrap_err();
1992 assert_eq!(err.kind(), ErrorKind::DataInvalid);
1993
1994 let bytes = write_manifest_list(FormatVersion::V2, vec![listed_manifest(
1995 "s3://b/metadata/0.avro",
1996 ManifestContentType::Data,
1997 )])
1998 .await;
1999 let moved_manifest_path = rewrite_manifest(
2002 &bytes,
2003 Codec::Null,
2004 |schema| {
2005 for_each_avro_field(schema, &|field| {
2006 if field["name"] == "manifest_path" {
2007 field["field-id"] = 999.into();
2008 }
2009 })
2010 },
2011 |_| {},
2012 );
2013 let without_sequence_number = rewrite_manifest(
2015 &bytes,
2016 Codec::Null,
2017 |schema| {
2018 schema["fields"]
2019 .as_array_mut()
2020 .unwrap()
2021 .retain(|field| field["name"] != "sequence_number")
2022 },
2023 |record| {
2024 let AvroValue::Record(fields) = record else {
2025 panic!("manifest list entry is not a record");
2026 };
2027 fields.retain(|(name, _)| name != "sequence_number");
2028 },
2029 );
2030 let unknown_format_version = set_avro_metadata(&bytes, "format-version", Some("4"));
2031 for bytes in [
2032 moved_manifest_path,
2033 without_sequence_number,
2034 unknown_format_version,
2035 ] {
2036 let err = read_list(&bytes).unwrap_err();
2037 assert_eq!(err.kind(), ErrorKind::DataInvalid);
2038 }
2039 }
2040
2041 #[test]
2042 fn test_encrypted_bytes_are_an_error() {
2043 let bytes = std::fs::read(format!(
2045 "{}/testdata/manifests_lists/manifest-list-v3-encrypted.avro",
2046 env!("CARGO_MANIFEST_DIR")
2047 ))
2048 .unwrap();
2049 let manifest_file = listed_manifest("s3://b/metadata/0.avro", ManifestContentType::Data);
2050 let Err(err) = read_manifest_entries(&bytes, &manifest_file) else {
2051 panic!("an encrypted manifest read as plaintext");
2052 };
2053 assert_eq!(err.kind(), ErrorKind::DataInvalid);
2054 let Err(err) = read_manifest_list(&bytes) else {
2055 panic!("an encrypted manifest list read as plaintext");
2056 };
2057 assert_eq!(err.kind(), ErrorKind::DataInvalid);
2058 }
2059
2060 #[test]
2061 fn test_manifest_lists_written_by_java() {
2062 for name in ["manifest-list-v2-1.avro", "manifest-list-v2-2.avro"] {
2064 let bytes = std::fs::read(format!(
2065 "{}/testdata/manifests_lists/{name}",
2066 env!("CARGO_MANIFEST_DIR")
2067 ))
2068 .unwrap();
2069 let manifest_list =
2070 ManifestList::parse_with_version(&bytes, FormatVersion::V2).unwrap();
2071 let batch = read_list(&bytes).unwrap();
2072
2073 assert_eq!(batch.num_rows(), manifest_list.entries().len(), "{name}");
2074 let paths = column(&batch, "manifest_path");
2075 let sequence_numbers = column(&batch, "sequence_number");
2076 let added_files_counts = column(&batch, "added_files_count");
2077 let partitions = column(&batch, "partitions");
2078 for (row, manifest) in manifest_list.entries().iter().enumerate() {
2079 assert_eq!(
2080 paths.as_string::<i32>().value(row),
2081 manifest.manifest_path,
2082 "{name}"
2083 );
2084 assert_eq!(
2085 sequence_numbers.as_primitive::<Int64Type>().value(row),
2086 manifest.sequence_number,
2087 "{name}"
2088 );
2089 assert_eq!(
2090 added_files_counts.as_primitive::<Int32Type>().value(row),
2091 manifest.added_files_count.unwrap() as i32,
2092 "{name}"
2093 );
2094 let summaries = partitions.as_list::<i32>().value(row);
2095 let lower_bounds: Vec<_> = summaries
2096 .as_struct()
2097 .column_by_name("lower_bound")
2098 .unwrap()
2099 .as_binary::<i32>()
2100 .iter()
2101 .map(|bound| bound.map(<[u8]>::to_vec))
2102 .collect();
2103 let expected: Vec<_> = manifest
2104 .partitions
2105 .iter()
2106 .flatten()
2107 .map(|summary| summary.lower_bound.as_ref().map(|bound| bound.to_vec()))
2108 .collect();
2109 assert_eq!(lower_bounds, expected, "{name}");
2110 }
2111 }
2112 }
2113
2114 #[tokio::test]
2115 async fn test_manifest_list_batches_match_whole_list() {
2116 for version in [FormatVersion::V1, FormatVersion::V2, FormatVersion::V3] {
2117 for len in [0, 3] {
2118 let manifests = (0..len)
2119 .map(|i| {
2120 listed_manifest(
2121 &format!("s3://b/metadata/{i}.avro"),
2122 ManifestContentType::Data,
2123 )
2124 })
2125 .collect();
2126 let bytes = write_manifest_list(version, manifests).await;
2127 assert_batches_match_whole(|batch_size| {
2128 let reader = ManifestListBatchReader::try_new(&bytes, batch_size).unwrap();
2129 (reader.schema(), reader)
2130 });
2131 }
2132 }
2133 }
2134}