Skip to main content

iceberg/arrow/
manifest.rs

1// Licensed to the Apache Software Foundation (ASF) under one
2// or more contributor license agreements.  See the NOTICE file
3// distributed with this work for additional information
4// regarding copyright ownership.  The ASF licenses this file
5// to you under the Apache License, Version 2.0 (the
6// "License"); you may not use this file except in compliance
7// with the License.  You may obtain a copy of the License at
8//
9//   http://www.apache.org/licenses/LICENSE-2.0
10//
11// Unless required by applicable law or agreed to in writing,
12// software distributed under the License is distributed on an
13// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14// KIND, either express or implied.  See the License for the
15// specific language governing permissions and limitations
16// under the License.
17
18//! Reading manifests and manifest lists into Arrow.
19
20use std::collections::HashMap;
21use std::io::Cursor;
22use std::sync::Arc;
23
24use arrow_arith::boolean::{and, is_null};
25use arrow_array::builder::Int64Builder;
26use arrow_array::cast::AsArray;
27use arrow_array::types::{TimestampMicrosecondType, TimestampNanosecondType};
28use arrow_array::{
29    Array, ArrayRef, BooleanArray, Int32Array, Int64Array, ListArray, MapArray, RecordBatch,
30    StructArray, new_null_array,
31};
32use arrow_avro::reader::{Reader, ReaderBuilder};
33use arrow_cast::{CastOptions, cast_with_options};
34use arrow_ord::cmp::eq;
35use arrow_schema::{ArrowError, DataType, Field, Fields, SchemaRef, TimeUnit};
36use arrow_select::zip::zip;
37use parquet::arrow::PARQUET_FIELD_ID_META_KEY;
38use serde_json::Value as JsonValue;
39
40use crate::arrow::schema_to_arrow_schema;
41use crate::error::invalid_data;
42use crate::spec::{
43    DATA_FILE_CONTENT_FIELD_ID, FILE_SEQUENCE_NUMBER_FIELD_ID, FormatVersion,
44    MANIFEST_CONTENT_FIELD_ID, MANIFEST_SEQUENCE_NUMBER_FIELD_ID, MIN_SEQUENCE_NUMBER_FIELD_ID,
45    ManifestContentType, ManifestFile, ManifestMetadata, ManifestStatus, SEQUENCE_NUMBER_FIELD_ID,
46    Schema, manifest_entry_fields, manifest_file_fields,
47};
48use crate::{Error, Result};
49
50/// Rows per batch. A row's size grows with the table's column count through
51/// the column stats maps. With stats on every column of a 100-column table of
52/// longs, 1024 rows take about 9 MB.
53const BATCH_SIZE: usize = 1024;
54
55/// Reads the entries of a manifest file as batches with one row per entry.
56///
57/// `bytes` is the manifest file and `manifest_file` is its entry in the
58/// manifest list. The schema is the spec's `manifest_entry` struct for the
59/// latest format version, with field IDs in the Arrow field metadata, so
60/// manifests of every format version read into the same schema apart from
61/// `data_file.partition`, which follows the manifest's partition spec.
62///
63/// Fields are matched to the manifest's Avro schema by field ID. A v1
64/// manifest reads with the spec's v2+ defaults, and null snapshot IDs,
65/// sequence numbers, and first row IDs are inherited from `manifest_file`.
66/// Lower and upper bounds keep the binary single-value serialization they were
67/// written with.
68///
69/// If `manifest_file.key_metadata` is set, the manifest is encrypted and
70/// `bytes` must be decrypted first.
71/// [`Table::manifest_reader`](crate::table::Table::manifest_reader) returns a
72/// reader that decrypts.
73///
74/// # Errors
75///
76/// Returns an error if the manifest's header can't be read or names an
77/// unsupported format version. Encrypted bytes fail at the header. Errors in
78/// the entries come from the reader, which yields no batches after one.
79///
80/// # Example
81///
82/// ```
83/// use iceberg::arrow::read_manifest_entries;
84/// use iceberg::spec::{ManifestContentType, ManifestFile};
85///
86/// let bytes = std::fs::read(concat!(
87///     env!("CARGO_MANIFEST_DIR"),
88///     "/testdata/manifests/pyiceberg-v2-data.avro"
89/// ))?;
90/// let manifest_file = ManifestFile {
91///     manifest_path: "s3://bucket/metadata/manifest.avro".to_string(),
92///     manifest_length: bytes.len() as i64,
93///     partition_spec_id: 0,
94///     content: ManifestContentType::Data,
95///     sequence_number: 1,
96///     min_sequence_number: 1,
97///     added_snapshot_id: 1,
98///     added_files_count: None,
99///     existing_files_count: None,
100///     deleted_files_count: None,
101///     added_rows_count: None,
102///     existing_rows_count: None,
103///     deleted_rows_count: None,
104///     partitions: None,
105///     key_metadata: None,
106///     first_row_id: None,
107/// };
108///
109/// let mut rows = 0;
110/// for batch in read_manifest_entries(&bytes, &manifest_file)? {
111///     rows += batch?.num_rows();
112/// }
113/// assert_eq!(rows, 2);
114/// # Ok::<(), Box<dyn std::error::Error>>(())
115/// ```
116pub fn read_manifest_entries<B: AsRef<[u8]>>(
117    bytes: B,
118    manifest_file: &ManifestFile,
119) -> Result<ManifestEntryBatchReader<B>> {
120    ManifestEntryBatchReader::try_new(bytes, manifest_file, BATCH_SIZE)
121}
122
123/// Batches of the entries of a manifest file, returned by
124/// [`read_manifest_entries`].
125pub struct ManifestEntryBatchReader<B: AsRef<[u8]>> {
126    inner: ProjectingReader<B>,
127    inheritance: Inheritance,
128}
129
130/// The values that a manifest's entries inherit from the manifest list.
131struct Inheritance {
132    manifest_path: String,
133    added_snapshot_id: i64,
134    sequence_number: i64,
135    content: ManifestContentType,
136    /// The first row ID of the next entry that inherits one, which advances
137    /// across batches. `None` for a manifest without a first row ID.
138    next_row_id: Option<i64>,
139}
140
141impl<B: AsRef<[u8]>> ManifestEntryBatchReader<B> {
142    fn try_new(bytes: B, manifest_file: &ManifestFile, batch_size: usize) -> Result<Self> {
143        let reader = ReaderBuilder::new()
144            .with_batch_size(batch_size)
145            .build(Cursor::new(bytes))
146            .map_err(invalid_manifest)?;
147        let header = reader.avro_header();
148        let metadata: HashMap<String, Vec<u8>> = header
149            .metadata()
150            .map(|(key, value)| (String::from_utf8_lossy(key).into_owned(), value.to_vec()))
151            .collect();
152        let metadata = ManifestMetadata::parse(&metadata)?;
153        let avro_schema: JsonValue = serde_json::from_slice(
154            header
155                .get("avro.schema")
156                .ok_or_else(|| invalid_data!("manifest has no avro.schema"))?,
157        )?;
158        let partition_type = metadata
159            .partition_spec()
160            .partition_type(metadata.schema())?;
161        let schema = Arc::new(schema_to_arrow_schema(
162            &Schema::builder()
163                .with_fields(manifest_entry_fields(&partition_type))
164                .build()?,
165        )?);
166
167        // v1 manifests have no content or sequence number columns, which v2+
168        // readers read as 0. A v2+ manifest may lack file_sequence_number (Java
169        // wrote none before 1.1.0), which reads as null and is inherited.
170        let zero_defaults: &[i32] = match metadata.format_version() {
171            FormatVersion::V1 => &[
172                DATA_FILE_CONTENT_FIELD_ID,
173                SEQUENCE_NUMBER_FIELD_ID,
174                FILE_SEQUENCE_NUMBER_FIELD_ID,
175            ],
176            _ => &[DATA_FILE_CONTENT_FIELD_ID],
177        };
178        Ok(Self {
179            inner: ProjectingReader {
180                reader,
181                schema,
182                avro_schema,
183                zero_defaults,
184                invalid: invalid_manifest,
185                finished: false,
186            },
187            inheritance: Inheritance {
188                manifest_path: manifest_file.manifest_path.clone(),
189                added_snapshot_id: manifest_file.added_snapshot_id,
190                sequence_number: manifest_file.sequence_number,
191                content: manifest_file.content,
192                next_row_id: manifest_file.first_row_id.map(i64::try_from).transpose()?,
193            },
194        })
195    }
196
197    /// Returns the schema of every batch.
198    pub fn schema(&self) -> SchemaRef {
199        self.inner.schema.clone()
200    }
201}
202
203impl<B: AsRef<[u8]>> Iterator for ManifestEntryBatchReader<B> {
204    type Item = Result<RecordBatch>;
205
206    fn next(&mut self) -> Option<Self::Item> {
207        let inheritance = &mut self.inheritance;
208        self.inner
209            .next_batch(|entries| inherit(entries, inheritance))
210    }
211}
212
213/// Reads the entries of a manifest list as batches with one row per manifest.
214///
215/// The schema is the spec's `manifest_file` struct for the latest format
216/// version, with field IDs in the Arrow field metadata, so manifest lists of
217/// every format version read into the same schema. Fields are matched to the
218/// manifest list's Avro schema by field ID. The format version comes from the
219/// `format-version` key that writers put in the file's metadata, and a
220/// manifest list without one is v1. A v1 manifest list reads with the spec's
221/// v2+ defaults. Partition summary bounds keep the binary single-value
222/// serialization they were written with.
223///
224/// If the snapshot has an `encryption_key_id`, the manifest list is encrypted
225/// and `bytes` must be decrypted first.
226/// [`Table::manifest_list_reader`](crate::table::Table::manifest_list_reader)
227/// returns a reader that decrypts.
228///
229/// # Errors
230///
231/// Returns an error if the manifest list's header can't be read or names an
232/// unsupported format version. Encrypted bytes fail at the header. Errors in
233/// the entries come from the reader, which yields no batches after one.
234///
235/// # Example
236///
237/// ```
238/// use iceberg::arrow::read_manifest_list;
239///
240/// let bytes = std::fs::read(concat!(
241///     env!("CARGO_MANIFEST_DIR"),
242///     "/testdata/manifests_lists/manifest-list-v2-2.avro"
243/// ))?;
244/// let mut rows = 0;
245/// for batch in read_manifest_list(&bytes)? {
246///     rows += batch?.num_rows();
247/// }
248/// assert_eq!(rows, 1);
249/// # Ok::<(), Box<dyn std::error::Error>>(())
250/// ```
251pub fn read_manifest_list<B: AsRef<[u8]>>(bytes: B) -> Result<ManifestListBatchReader<B>> {
252    ManifestListBatchReader::try_new(bytes, BATCH_SIZE)
253}
254
255/// Batches of the entries of a manifest list, returned by
256/// [`read_manifest_list`].
257pub struct ManifestListBatchReader<B: AsRef<[u8]>> {
258    inner: ProjectingReader<B>,
259}
260
261impl<B: AsRef<[u8]>> ManifestListBatchReader<B> {
262    fn try_new(bytes: B, batch_size: usize) -> Result<Self> {
263        let reader = ReaderBuilder::new()
264            .with_batch_size(batch_size)
265            .build(Cursor::new(bytes))
266            .map_err(invalid_manifest_list)?;
267        let header = reader.avro_header();
268        let avro_schema: JsonValue = serde_json::from_slice(
269            header
270                .get("avro.schema")
271                .ok_or_else(|| invalid_data!("manifest list has no avro.schema"))?,
272        )?;
273        // The spec doesn't define this key, but Java, PyIceberg, and iceberg-rust
274        // all write it, and rejecting an unknown version keeps a newer manifest
275        // list from reading as an older one.
276        let format_version = match header.get("format-version") {
277            Some(bytes) => serde_json::from_slice::<FormatVersion>(bytes).map_err(|err| {
278                invalid_data!("failed to parse format version in manifest list metadata")
279                    .with_source(err)
280            })?,
281            None => FormatVersion::V1,
282        };
283        let schema = Arc::new(schema_to_arrow_schema(
284            &Schema::builder()
285                .with_fields(manifest_file_fields())
286                .build()?,
287        )?);
288        // v1 manifest lists have no content or sequence number columns, which v2+
289        // readers read as 0.
290        let zero_defaults: &[i32] = match format_version {
291            FormatVersion::V1 => &[
292                MANIFEST_CONTENT_FIELD_ID,
293                MANIFEST_SEQUENCE_NUMBER_FIELD_ID,
294                MIN_SEQUENCE_NUMBER_FIELD_ID,
295            ],
296            _ => &[],
297        };
298        Ok(Self {
299            inner: ProjectingReader {
300                reader,
301                schema,
302                avro_schema,
303                zero_defaults,
304                invalid: invalid_manifest_list,
305                finished: false,
306            },
307        })
308    }
309
310    /// Returns the schema of every batch.
311    pub fn schema(&self) -> SchemaRef {
312        self.inner.schema.clone()
313    }
314}
315
316impl<B: AsRef<[u8]>> Iterator for ManifestListBatchReader<B> {
317    type Item = Result<RecordBatch>;
318
319    fn next(&mut self) -> Option<Self::Item> {
320        self.inner.next_batch(Ok)
321    }
322}
323
324/// Decodes an Avro file in batches and projects them onto `schema`.
325struct ProjectingReader<B: AsRef<[u8]>> {
326    reader: Reader<Cursor<B>>,
327    schema: SchemaRef,
328    avro_schema: JsonValue,
329    zero_defaults: &'static [i32],
330    invalid: fn(ArrowError) -> Error,
331    /// Set after an error. First row ID inheritance can stop partway through
332    /// a batch, so the batches after an error would get the wrong row IDs.
333    finished: bool,
334}
335
336impl<B: AsRef<[u8]>> ProjectingReader<B> {
337    /// Returns the next batch with `finish` applied to its projected rows.
338    fn next_batch(
339        &mut self,
340        finish: impl FnOnce(StructArray) -> Result<StructArray>,
341    ) -> Option<Result<RecordBatch>> {
342        if self.finished {
343            return None;
344        }
345        let batch = self.reader.next()?.map_err(self.invalid).and_then(|batch| {
346            let rows = project_struct(
347                &StructArray::from(batch),
348                &self.avro_schema,
349                self.schema.fields(),
350                self.zero_defaults,
351            )?;
352            Ok(RecordBatch::from(finish(rows)?))
353        });
354        self.finished = batch.is_err();
355        Some(batch)
356    }
357}
358
359fn invalid_manifest(err: ArrowError) -> Error {
360    invalid_data!("failed to read manifest").with_source(err)
361}
362
363fn invalid_manifest_list(err: ArrowError) -> Error {
364    invalid_data!("failed to read manifest list").with_source(err)
365}
366
367fn invalid_column(err: ArrowError) -> Error {
368    invalid_data!("failed to build Arrow column from Avro").with_source(err)
369}
370
371/// Returns the field ID of a field in the schema built from
372/// `manifest_entry_fields` or `manifest_file_fields`, which give every field
373/// one.
374fn field_id(field: &Field) -> i32 {
375    field.metadata()[PARQUET_FIELD_ID_META_KEY].parse().unwrap()
376}
377
378/// Returns the type of a nullable Avro field, `["null", T]`, as `T`.
379fn non_null_type(avro_type: &JsonValue) -> &JsonValue {
380    match avro_type.as_array().map(Vec::as_slice) {
381        Some([JsonValue::String(null), branch] | [branch, JsonValue::String(null)])
382            if null == "null" =>
383        {
384            branch
385        }
386        _ => avro_type,
387    }
388}
389
390/// Builds each of `fields` from the column of `array` that the Avro record
391/// `record` gives the same field ID. A field that `record` lacks reads as 0 if
392/// its ID is in `zero_defaults`, and as null otherwise.
393fn project_struct(
394    array: &StructArray,
395    record: &JsonValue,
396    fields: &Fields,
397    zero_defaults: &[i32],
398) -> Result<StructArray> {
399    let avro_fields = record["fields"]
400        .as_array()
401        .ok_or_else(|| invalid_data!("Avro type {record} is not a record"))?;
402    let mut columns = Vec::with_capacity(fields.len());
403    for field in fields {
404        let id = field_id(field);
405        columns.push(
406            match avro_fields
407                .iter()
408                .position(|avro_field| avro_field["field-id"].as_i64() == Some(id.into()))
409            {
410                Some(index) => project(
411                    array.column(index),
412                    non_null_type(&avro_fields[index]["type"]),
413                    field,
414                    zero_defaults,
415                )?,
416                None => missing_field(field, array.len(), zero_defaults)?,
417            },
418        );
419    }
420    StructArray::try_new_with_length(fields.clone(), columns, array.nulls().cloned(), array.len())
421        .map_err(invalid_column)
422}
423
424/// Builds the column for a field that the Avro file doesn't have.
425fn missing_field(field: &Field, len: usize, zero_defaults: &[i32]) -> Result<ArrayRef> {
426    let id = field_id(field);
427    let zero = zero_defaults.contains(&id);
428    match field.data_type() {
429        DataType::Int32 if zero => Ok(Arc::new(Int32Array::from_value(0, len))),
430        DataType::Int64 if zero => Ok(Arc::new(Int64Array::from_value(0, len))),
431        _ if field.is_nullable() => Ok(new_null_array(field.data_type(), len)),
432        _ => Err(invalid_data!(
433            "Avro file has no required field {} with field ID {id}",
434            field.name()
435        )),
436    }
437}
438
439/// Converts the decoded `array` of Avro type `avro_type` to `field`, the
440/// spec's Arrow field for it.
441fn project(
442    array: &ArrayRef,
443    avro_type: &JsonValue,
444    field: &Field,
445    zero_defaults: &[i32],
446) -> Result<ArrayRef> {
447    let type_mismatch = || {
448        invalid_data!(
449            "field {} has Avro type {avro_type}, which doesn't match {}",
450            field.name(),
451            field.data_type()
452        )
453    };
454    Ok(match field.data_type() {
455        DataType::Struct(fields) => Arc::new(project_struct(
456            array.as_struct_opt().ok_or_else(type_mismatch)?,
457            avro_type,
458            fields,
459            zero_defaults,
460        )?),
461        DataType::List(element) => {
462            let list = array.as_list_opt::<i32>().ok_or_else(type_mismatch)?;
463            let values = project(
464                list.values(),
465                non_null_type(&avro_type["items"]),
466                element,
467                zero_defaults,
468            )?;
469            Arc::new(
470                ListArray::try_new(
471                    element.clone(),
472                    list.offsets().clone(),
473                    values,
474                    list.nulls().cloned(),
475                )
476                .map_err(invalid_column)?,
477            )
478        }
479        // Avro stores maps with int keys as arrays of key-value records.
480        DataType::Map(entries, _) => {
481            let list = array.as_list_opt::<i32>().ok_or_else(type_mismatch)?;
482            let DataType::Struct(key_value) = entries.data_type() else {
483                unreachable!()
484            };
485            let entries_array = project_struct(
486                list.values().as_struct_opt().ok_or_else(type_mismatch)?,
487                &avro_type["items"],
488                key_value,
489                zero_defaults,
490            )?;
491            Arc::new(
492                MapArray::try_new(
493                    entries.clone(),
494                    list.offsets().clone(),
495                    entries_array,
496                    list.nulls().cloned(),
497                    false,
498                )
499                .map_err(invalid_column)?,
500            )
501        }
502        // arrow-avro gives every timestamp a UTC time zone, so only the type
503        // changes.
504        DataType::Timestamp(TimeUnit::Microsecond, time_zone) => Arc::new(
505            array
506                .as_primitive_opt::<TimestampMicrosecondType>()
507                .ok_or_else(type_mismatch)?
508                .clone()
509                .with_timezone_opt(time_zone.clone()),
510        ),
511        DataType::Timestamp(TimeUnit::Nanosecond, time_zone) => Arc::new(
512            array
513                .as_primitive_opt::<TimestampNanosecondType>()
514                .ok_or_else(type_mismatch)?
515                .clone()
516                .with_timezone_opt(time_zone.clone()),
517        ),
518        data_type if array.data_type() == data_type => array.clone(),
519        // Without `safe: false`, a value that doesn't fit the spec's type would
520        // become null instead of an error.
521        data_type => cast_with_options(array, data_type, &CastOptions {
522            safe: false,
523            ..Default::default()
524        })
525        .map_err(invalid_column)?,
526    })
527}
528
529/// Fills in the values that the spec says entries inherit from the manifest
530/// list.
531fn inherit(entries: StructArray, inheritance: &mut Inheritance) -> Result<StructArray> {
532    let (fields, mut columns, nulls) = entries.into_parts();
533    let index = |name: &str| fields.find(name).unwrap().0;
534    let (status, snapshot_id, sequence_number, file_sequence_number, data_file) = (
535        index("status"),
536        index("snapshot_id"),
537        index("sequence_number"),
538        index("file_sequence_number"),
539        index("data_file"),
540    );
541
542    columns[snapshot_id] = fill_nulls(&columns[snapshot_id], None, inheritance.added_snapshot_id)?;
543    if columns[sequence_number].null_count() > 0 || columns[file_sequence_number].null_count() > 0 {
544        let added = eq(
545            &columns[status],
546            &Int32Array::new_scalar(ManifestStatus::Added as i32),
547        )?;
548        for column in [sequence_number, file_sequence_number] {
549            columns[column] =
550                fill_nulls(&columns[column], Some(&added), inheritance.sequence_number)?;
551        }
552    }
553    if inheritance.content == ManifestContentType::Data {
554        let (data_file_fields, mut data_file_columns, data_file_nulls) =
555            columns[data_file].as_struct().clone().into_parts();
556        let index = |name: &str| data_file_fields.find(name).unwrap().0;
557        let first_row_id = index("first_row_id");
558        data_file_columns[first_row_id] = inherit_first_row_ids(
559            data_file_columns[first_row_id].as_primitive(),
560            columns[status].as_primitive(),
561            data_file_columns[index("record_count")].as_primitive(),
562            inheritance,
563        )?;
564        columns[data_file] = Arc::new(StructArray::new(
565            data_file_fields,
566            data_file_columns,
567            data_file_nulls,
568        ));
569    }
570    Ok(StructArray::new(fields, columns, nulls))
571}
572
573/// Replaces the nulls of `array` with `value`, only in the rows that `rows`
574/// selects when given.
575fn fill_nulls(array: &ArrayRef, rows: Option<&BooleanArray>, value: i64) -> Result<ArrayRef> {
576    if array.null_count() == 0 {
577        return Ok(array.clone());
578    }
579    let mask = match rows {
580        Some(rows) => and(&is_null(array)?, rows)?,
581        None => is_null(array)?,
582    };
583    Ok(zip(&mask, &Int64Array::new_scalar(value), array)?)
584}
585
586/// Assigns first row IDs to the data files that lack one, following first row
587/// ID inheritance. Deleted entries take no row IDs, because the spec's
588/// suggested manifest `first_row_id` increment, `added_rows_count` plus
589/// `existing_rows_count`, leaves no room for them. A manifest without a first
590/// row ID belongs to a snapshot without row lineage, so all its row IDs are
591/// null.
592fn inherit_first_row_ids(
593    first_row_ids: &Int64Array,
594    status: &Int32Array,
595    record_counts: &Int64Array,
596    inheritance: &mut Inheritance,
597) -> Result<ArrayRef> {
598    let Some(next_row_id) = inheritance.next_row_id.as_mut() else {
599        return Ok(new_null_array(&DataType::Int64, first_row_ids.len()));
600    };
601    if first_row_ids.null_count() == 0 {
602        return Ok(Arc::new(first_row_ids.clone()));
603    }
604    let mut builder = Int64Builder::with_capacity(first_row_ids.len());
605    for ((first_row_id, &status), &record_count) in first_row_ids
606        .iter()
607        .zip(status.values())
608        .zip(record_counts.values())
609    {
610        match first_row_id {
611            None if status != ManifestStatus::Deleted as i32 => {
612                if record_count < 0 {
613                    return Err(invalid_data!(
614                        "negative record_count {record_count} assigning first row IDs in {}",
615                        inheritance.manifest_path
616                    ));
617                }
618                builder.append_value(*next_row_id);
619                *next_row_id = next_row_id.checked_add(record_count).ok_or_else(|| {
620                    invalid_data!(
621                        "row ID overflow assigning first row IDs in {}",
622                        inheritance.manifest_path
623                    )
624                })?;
625            }
626            first_row_id => builder.append_option(first_row_id),
627        }
628    }
629    Ok(Arc::new(builder.finish()))
630}
631
632#[cfg(test)]
633mod tests {
634    use std::collections::HashMap;
635    use std::sync::Arc;
636
637    use apache_avro::types::Value as AvroValue;
638    use apache_avro::{Codec, DeflateSettings, ZstandardSettings};
639    use arrow_array::cast::AsArray;
640    use arrow_array::types::{Int32Type, Int64Type};
641    use arrow_array::{
642        Array, ArrayRef, BinaryArray, BooleanArray, Date32Array, Decimal128Array,
643        FixedSizeBinaryArray, Float32Array, Float64Array, Int32Array, Int64Array, StringArray,
644        Time64MicrosecondArray, TimestampMicrosecondArray, TimestampNanosecondArray,
645    };
646    use arrow_buffer::{NullBuffer, OffsetBuffer};
647    use arrow_schema::SchemaRef;
648    use arrow_select::concat::concat_batches;
649
650    use super::*;
651    use crate::ErrorKind;
652    use crate::arrow::arrow_schema_to_schema;
653    use crate::io::FileIO;
654    use crate::spec::{
655        DataContentType, DataFile, DataFileBuilder, DataFileFormat, Datum, FieldSummary, ListType,
656        Literal, Manifest, ManifestEntry, ManifestList, ManifestListWriter, ManifestStatus,
657        ManifestWriter, ManifestWriterBuilder, MapType, NestedField, PartitionSpec,
658        PrimitiveLiteral, PrimitiveType, Schema, Struct, StructType, Transform, Type,
659        UNASSIGNED_SEQUENCE_NUMBER,
660    };
661
662    fn table_schema() -> Schema {
663        let columns = [
664            (1, "b", PrimitiveType::Boolean),
665            (2, "i", PrimitiveType::Int),
666            (3, "l", PrimitiveType::Long),
667            (4, "f", PrimitiveType::Float),
668            (5, "d", PrimitiveType::Double),
669            (6, "dec", PrimitiveType::Decimal {
670                precision: 38,
671                scale: 10,
672            }),
673            (7, "date", PrimitiveType::Date),
674            (8, "time", PrimitiveType::Time),
675            (9, "ts", PrimitiveType::Timestamp),
676            (10, "tstz", PrimitiveType::Timestamptz),
677            (11, "ts_ns", PrimitiveType::TimestampNs),
678            (12, "tstz_ns", PrimitiveType::TimestamptzNs),
679            (13, "s", PrimitiveType::String),
680            (14, "fx", PrimitiveType::Fixed(3)),
681            (15, "bin", PrimitiveType::Binary),
682        ];
683        Schema::builder()
684            .with_fields(
685                columns.into_iter().map(|(id, name, ty)| {
686                    NestedField::optional(id, name, Type::Primitive(ty)).into()
687                }),
688            )
689            .build()
690            .unwrap()
691    }
692
693    fn partition_spec(fields: &[(&str, Transform)]) -> PartitionSpec {
694        fields
695            .iter()
696            .fold(
697                PartitionSpec::builder(table_schema()).with_spec_id(0),
698                |builder, (source, transform)| {
699                    builder
700                        .add_partition_field(*source, format!("{source}_{transform}"), *transform)
701                        .unwrap()
702                },
703            )
704            .build()
705            .unwrap()
706    }
707
708    fn data_file(content: DataContentType, path: &str) -> DataFileBuilder {
709        let mut builder = DataFileBuilder::default();
710        builder
711            .content(content)
712            .file_path(path.to_string())
713            .file_format(DataFileFormat::Parquet)
714            .record_count(1)
715            .file_size_in_bytes(10);
716        builder
717    }
718
719    /// Writes a manifest with `ManifestWriter` and returns its bytes and its
720    /// manifest list entry.
721    async fn write_manifest(
722        spec: PartitionSpec,
723        snapshot_id: Option<i64>,
724        build: impl FnOnce(ManifestWriterBuilder) -> ManifestWriter,
725        add: impl FnOnce(&mut ManifestWriter),
726    ) -> (Vec<u8>, ManifestFile) {
727        let io = FileIO::new_with_memory();
728        let output = io.new_output("memory:///manifest.avro").unwrap();
729        let mut writer = build(ManifestWriterBuilder::new(
730            output,
731            snapshot_id,
732            Arc::new(table_schema()),
733            spec,
734        ));
735        add(&mut writer);
736        let manifest_file = writer.write_manifest_file().await.unwrap();
737        let bytes = io
738            .new_input("memory:///manifest.avro")
739            .unwrap()
740            .read()
741            .await
742            .unwrap()
743            .to_vec();
744        (bytes, manifest_file)
745    }
746
747    fn concat(
748        schema: SchemaRef,
749        batches: impl Iterator<Item = Result<RecordBatch>>,
750    ) -> Result<RecordBatch> {
751        let batches = batches.collect::<Result<Vec<_>>>()?;
752        Ok(concat_batches(&schema, &batches)?)
753    }
754
755    fn read_entries(bytes: &[u8], manifest_file: &ManifestFile) -> Result<RecordBatch> {
756        let reader = read_manifest_entries(bytes, manifest_file)?;
757        concat(reader.schema(), reader)
758    }
759
760    fn read_list(bytes: &[u8]) -> Result<RecordBatch> {
761        let reader = read_manifest_list(bytes)?;
762        concat(reader.schema(), reader)
763    }
764
765    /// Checks that reading in batches of 1 to 3 rows gives nonempty batches of
766    /// the reader's schema that concatenate to the file read as one batch.
767    fn assert_batches_match_whole<I: Iterator<Item = Result<RecordBatch>>>(
768        read: impl Fn(usize) -> (SchemaRef, I),
769    ) {
770        let (schema, batches) = read(usize::MAX);
771        let whole = concat(schema.clone(), batches).unwrap();
772        for batch_size in 1..=3 {
773            let (batch_schema, batches) = read(batch_size);
774            assert_eq!(batch_schema, schema);
775            let batches: Vec<_> = batches.map(Result::unwrap).collect();
776            for batch in &batches {
777                assert_eq!(batch.schema(), schema, "batch size {batch_size}");
778                assert!(
779                    (1..=batch_size).contains(&batch.num_rows()),
780                    "batch size {batch_size}"
781                );
782            }
783            assert_eq!(
784                concat_batches(&schema, &batches).unwrap(),
785                whole,
786                "batch size {batch_size}"
787            );
788        }
789    }
790
791    fn column(batch: &RecordBatch, name: &str) -> ArrayRef {
792        batch.column_by_name(name).unwrap().clone()
793    }
794
795    fn data_file_column(batch: &RecordBatch, name: &str) -> ArrayRef {
796        column(batch, "data_file")
797            .as_struct()
798            .column_by_name(name)
799            .unwrap()
800            .clone()
801    }
802
803    /// Entries of the map in `row` of a `map<int, long>` column, sorted by key.
804    fn long_map(column: &ArrayRef, row: usize) -> Vec<(i32, i64)> {
805        let entries = column.as_map().value(row);
806        let keys = entries.column(0).as_primitive::<Int32Type>();
807        let values = entries.column(1).as_primitive::<Int64Type>();
808        let mut pairs: Vec<_> = keys
809            .values()
810            .iter()
811            .copied()
812            .zip(values.values().iter().copied())
813            .collect();
814        pairs.sort_unstable();
815        pairs
816    }
817
818    /// Entries of the map in `row` of a `map<int, binary>` column, sorted by key.
819    fn binary_map(column: &ArrayRef, row: usize) -> Vec<(i32, Vec<u8>)> {
820        let entries = column.as_map().value(row);
821        let keys = entries.column(0).as_primitive::<Int32Type>();
822        let values = entries.column(1).as_binary::<i32>();
823        let mut pairs: Vec<_> = keys
824            .values()
825            .iter()
826            .copied()
827            .zip(values.iter().map(|v| v.unwrap().to_vec()))
828            .collect();
829        pairs.sort_unstable();
830        pairs
831    }
832
833    fn int_map_type(key_id: i32, value_id: i32, value: PrimitiveType) -> Type {
834        Type::Map(MapType::new(
835            NestedField::map_key_element(key_id, Type::Primitive(PrimitiveType::Int)).into(),
836            NestedField::map_value_element(value_id, Type::Primitive(value), true).into(),
837        ))
838    }
839
840    fn list_type(element_id: i32, element: PrimitiveType) -> Type {
841        Type::List(ListType::new(
842            NestedField::list_element(element_id, Type::Primitive(element), true).into(),
843        ))
844    }
845
846    /// Rewrites an Avro file with the user metadata `key` set to `value`, or
847    /// without it if `value` is `None`.
848    fn set_avro_metadata(bytes: &[u8], key: &str, value: Option<&str>) -> Vec<u8> {
849        let reader = apache_avro::Reader::new(bytes).unwrap();
850        let schema = reader.writer_schema().clone();
851        let mut metadata = reader.user_metadata().clone();
852        match value {
853            Some(value) => metadata.insert(key.to_string(), value.as_bytes().to_vec()),
854            None => metadata.remove(key),
855        };
856        let mut writer = apache_avro::Writer::new(&schema, Vec::new()).unwrap();
857        for (key, value) in metadata {
858            writer.add_user_metadata(key, value).unwrap();
859        }
860        for record in reader {
861            writer.append_value(record.unwrap()).unwrap();
862        }
863        writer.into_inner().unwrap()
864    }
865
866    /// Rewrites a manifest with `codec`, `edit_schema` applied to its Avro
867    /// schema, and `edit_record` applied to each record.
868    fn rewrite_manifest(
869        bytes: &[u8],
870        codec: Codec,
871        edit_schema: impl Fn(&mut serde_json::Value),
872        edit_record: impl Fn(&mut AvroValue),
873    ) -> Vec<u8> {
874        let reader = apache_avro::Reader::new(bytes).unwrap();
875        let mut schema = serde_json::to_value(reader.writer_schema()).unwrap();
876        edit_schema(&mut schema);
877        let schema = apache_avro::Schema::parse(&schema).unwrap();
878        let metadata = reader.user_metadata().clone();
879        let mut writer = apache_avro::Writer::with_codec(&schema, Vec::new(), codec).unwrap();
880        for (key, value) in metadata {
881            writer.add_user_metadata(key, value).unwrap();
882        }
883        for record in reader {
884            let mut record = record.unwrap();
885            edit_record(&mut record);
886            writer.append_value(record).unwrap();
887        }
888        writer.into_inner().unwrap()
889    }
890
891    /// Calls `edit` on every record field object in an Avro schema.
892    fn for_each_avro_field(schema: &mut serde_json::Value, edit: &impl Fn(&mut serde_json::Value)) {
893        match schema {
894            serde_json::Value::Object(object) => {
895                if let Some(serde_json::Value::Array(fields)) = object.get_mut("fields") {
896                    for field in fields {
897                        edit(field);
898                        for_each_avro_field(field.get_mut("type").unwrap(), edit);
899                    }
900                }
901                if let Some(items) = object.get_mut("items") {
902                    for_each_avro_field(items, edit);
903                }
904            }
905            serde_json::Value::Array(branches) => {
906                for branch in branches {
907                    for_each_avro_field(branch, edit);
908                }
909            }
910            _ => {}
911        }
912    }
913
914    /// Renames the record field `from` to `to` in every Avro record value.
915    fn rename_avro_field(value: &mut AvroValue, from: &str, to: &str) {
916        match value {
917            AvroValue::Record(fields) => {
918                for (name, field) in fields {
919                    if name == from {
920                        *name = to.to_string();
921                    }
922                    rename_avro_field(field, from, to);
923                }
924            }
925            AvroValue::Union(_, value) => rename_avro_field(value, from, to),
926            AvroValue::Array(items) => {
927                for item in items {
928                    rename_avro_field(item, from, to);
929                }
930            }
931            _ => {}
932        }
933    }
934
935    /// Renames record fields in an Avro file's schema and records, keeping
936    /// their field IDs.
937    fn rename_fields(bytes: &[u8], renames: &[(&str, &str)]) -> Vec<u8> {
938        rewrite_manifest(
939            bytes,
940            Codec::Null,
941            |schema| {
942                for_each_avro_field(schema, &|field| {
943                    for (from, to) in renames {
944                        if field["name"] == *from {
945                            field["name"] = (*to).into();
946                        }
947                    }
948                })
949            },
950            |record| {
951                for (from, to) in renames {
952                    rename_avro_field(record, from, to);
953                }
954            },
955        )
956    }
957
958    #[tokio::test]
959    async fn test_schema_is_spec_manifest_entry() {
960        let spec = partition_spec(&[("i", Transform::Identity), ("s", Transform::Truncate(2))]);
961        let (bytes, manifest_file) =
962            write_manifest(spec, Some(1), |b| b.build_v2_data(), |_| {}).await;
963        let batch = read_entries(&bytes, &manifest_file).unwrap();
964        assert_eq!(batch.num_rows(), 0);
965
966        let long = || Type::Primitive(PrimitiveType::Long);
967        let int = || Type::Primitive(PrimitiveType::Int);
968        let string = || Type::Primitive(PrimitiveType::String);
969        let data_file = StructType::new(vec![
970            NestedField::required(134, "content", int()).into(),
971            NestedField::required(100, "file_path", string()).into(),
972            NestedField::required(101, "file_format", string()).into(),
973            NestedField::required(
974                102,
975                "partition",
976                Type::Struct(StructType::new(vec![
977                    NestedField::optional(1000, "i_identity", int()).into(),
978                    NestedField::optional(1001, "s_truncate[2]", string()).into(),
979                ])),
980            )
981            .into(),
982            NestedField::required(103, "record_count", long()).into(),
983            NestedField::required(104, "file_size_in_bytes", long()).into(),
984            NestedField::optional(
985                108,
986                "column_sizes",
987                int_map_type(117, 118, PrimitiveType::Long),
988            )
989            .into(),
990            NestedField::optional(
991                109,
992                "value_counts",
993                int_map_type(119, 120, PrimitiveType::Long),
994            )
995            .into(),
996            NestedField::optional(
997                110,
998                "null_value_counts",
999                int_map_type(121, 122, PrimitiveType::Long),
1000            )
1001            .into(),
1002            NestedField::optional(
1003                137,
1004                "nan_value_counts",
1005                int_map_type(138, 139, PrimitiveType::Long),
1006            )
1007            .into(),
1008            NestedField::optional(
1009                125,
1010                "lower_bounds",
1011                int_map_type(126, 127, PrimitiveType::Binary),
1012            )
1013            .into(),
1014            NestedField::optional(
1015                128,
1016                "upper_bounds",
1017                int_map_type(129, 130, PrimitiveType::Binary),
1018            )
1019            .into(),
1020            NestedField::optional(131, "key_metadata", Type::Primitive(PrimitiveType::Binary))
1021                .into(),
1022            NestedField::optional(132, "split_offsets", list_type(133, PrimitiveType::Long)).into(),
1023            NestedField::optional(135, "equality_ids", list_type(136, PrimitiveType::Int)).into(),
1024            NestedField::optional(140, "sort_order_id", int()).into(),
1025            NestedField::optional(142, "first_row_id", long()).into(),
1026            NestedField::optional(143, "referenced_data_file", string()).into(),
1027            NestedField::optional(144, "content_offset", long()).into(),
1028            NestedField::optional(145, "content_size_in_bytes", long()).into(),
1029        ]);
1030        let expected = Schema::builder()
1031            .with_fields(vec![
1032                NestedField::required(0, "status", int()).into(),
1033                NestedField::optional(1, "snapshot_id", long()).into(),
1034                NestedField::optional(3, "sequence_number", long()).into(),
1035                NestedField::optional(4, "file_sequence_number", long()).into(),
1036                NestedField::required(2, "data_file", Type::Struct(data_file)).into(),
1037            ])
1038            .build()
1039            .unwrap();
1040        assert_eq!(
1041            arrow_schema_to_schema(&batch.schema()).unwrap().as_struct(),
1042            expected.as_struct()
1043        );
1044    }
1045
1046    #[tokio::test]
1047    async fn test_data_file_columns() {
1048        let added = DataFile {
1049            column_sizes: HashMap::from([(1, 11)]),
1050            value_counts: HashMap::from([(1, 10), (3, 9)]),
1051            null_value_counts: HashMap::from([(1, 1)]),
1052            nan_value_counts: HashMap::from([(4, 0)]),
1053            lower_bounds: HashMap::from([(2, Datum::int(-1)), (13, Datum::string("abc"))]),
1054            upper_bounds: HashMap::from([(2, Datum::int(7))]),
1055            key_metadata: Some(vec![1, 2]),
1056            split_offsets: Some(vec![4, 100]),
1057            sort_order_id: Some(0),
1058            ..data_file(DataContentType::Data, "s3://b/data/0.parquet")
1059                .record_count(10)
1060                .file_size_in_bytes(100)
1061                .build()
1062                .unwrap()
1063        };
1064        let existing = data_file(DataContentType::Data, "s3://b/data/1.parquet")
1065            .build()
1066            .unwrap();
1067        let (bytes, manifest_file) = write_manifest(
1068            partition_spec(&[]),
1069            Some(1),
1070            |b| b.build_v2_data(),
1071            |writer| {
1072                writer.add_file(added, 2).unwrap();
1073                writer.add_existing_file(existing, 5, 3, Some(3)).unwrap();
1074            },
1075        )
1076        .await;
1077        let batch = read_entries(&bytes, &manifest_file).unwrap();
1078        assert_eq!(batch.num_rows(), 2);
1079
1080        assert_eq!(
1081            column(&batch, "status").as_ref(),
1082            &Int32Array::from(vec![1, 0]) as &dyn Array
1083        );
1084        assert_eq!(
1085            column(&batch, "snapshot_id").as_ref(),
1086            &Int64Array::from(vec![1, 5]) as &dyn Array
1087        );
1088        assert_eq!(
1089            column(&batch, "sequence_number").as_ref(),
1090            &Int64Array::from(vec![2, 3]) as &dyn Array
1091        );
1092
1093        let column = |name: &str| data_file_column(&batch, name);
1094        assert_eq!(
1095            column("content").as_ref(),
1096            &Int32Array::from(vec![0, 0]) as &dyn Array
1097        );
1098        assert_eq!(
1099            column("file_path").as_ref(),
1100            &StringArray::from(vec!["s3://b/data/0.parquet", "s3://b/data/1.parquet"])
1101                as &dyn Array
1102        );
1103        assert_eq!(
1104            column("file_format").as_ref(),
1105            &StringArray::from(vec!["PARQUET", "PARQUET"]) as &dyn Array
1106        );
1107        assert_eq!(
1108            column("record_count").as_ref(),
1109            &Int64Array::from(vec![10, 1]) as &dyn Array
1110        );
1111        assert_eq!(
1112            column("file_size_in_bytes").as_ref(),
1113            &Int64Array::from(vec![100, 10]) as &dyn Array
1114        );
1115        assert_eq!(long_map(&column("column_sizes"), 0), vec![(1, 11)]);
1116        assert_eq!(long_map(&column("value_counts"), 0), vec![(1, 10), (3, 9)]);
1117        assert_eq!(long_map(&column("null_value_counts"), 0), vec![(1, 1)]);
1118        assert_eq!(long_map(&column("nan_value_counts"), 0), vec![(4, 0)]);
1119        assert_eq!(long_map(&column("value_counts"), 1), vec![]);
1120        assert_eq!(binary_map(&column("lower_bounds"), 0), vec![
1121            (2, (-1i32).to_le_bytes().to_vec()),
1122            (13, b"abc".to_vec()),
1123        ]);
1124        assert_eq!(binary_map(&column("upper_bounds"), 0), vec![(
1125            2,
1126            7i32.to_le_bytes().to_vec()
1127        )]);
1128        assert_eq!(
1129            column("key_metadata").as_ref(),
1130            &BinaryArray::from(vec![Some(&[1u8, 2][..]), None]) as &dyn Array
1131        );
1132        let split_offsets = column("split_offsets");
1133        assert_eq!(
1134            split_offsets.as_list::<i32>().value(0).as_ref(),
1135            &Int64Array::from(vec![4, 100]) as &dyn Array
1136        );
1137        assert_eq!(
1138            column("sort_order_id").as_ref(),
1139            &Int32Array::from(vec![Some(0), None]) as &dyn Array
1140        );
1141    }
1142
1143    #[tokio::test]
1144    async fn test_delete_file_columns() {
1145        let position_deletes = DataFile {
1146            file_format: DataFileFormat::Puffin,
1147            referenced_data_file: Some("s3://b/data/0.parquet".to_string()),
1148            content_offset: Some(4),
1149            content_size_in_bytes: Some(40),
1150            ..data_file(DataContentType::PositionDeletes, "s3://b/data/1.puffin")
1151                .build()
1152                .unwrap()
1153        };
1154        let equality_deletes = DataFile {
1155            equality_ids: Some(vec![1, 3]),
1156            ..data_file(DataContentType::EqualityDeletes, "s3://b/data/2.parquet")
1157                .build()
1158                .unwrap()
1159        };
1160        let (bytes, manifest_file) = write_manifest(
1161            partition_spec(&[]),
1162            Some(1),
1163            |b| b.build_v3_deletes(),
1164            |writer| {
1165                writer.add_file(position_deletes, 2).unwrap();
1166                writer.add_file(equality_deletes, 2).unwrap();
1167            },
1168        )
1169        .await;
1170        let batch = read_entries(&bytes, &manifest_file).unwrap();
1171
1172        let column = |name: &str| data_file_column(&batch, name);
1173        assert_eq!(
1174            column("content").as_ref(),
1175            &Int32Array::from(vec![1, 2]) as &dyn Array
1176        );
1177        assert_eq!(
1178            column("file_format").as_ref(),
1179            &StringArray::from(vec!["PUFFIN", "PARQUET"]) as &dyn Array
1180        );
1181        assert_eq!(
1182            column("referenced_data_file").as_ref(),
1183            &StringArray::from(vec![Some("s3://b/data/0.parquet"), None]) as &dyn Array
1184        );
1185        assert_eq!(
1186            column("content_offset").as_ref(),
1187            &Int64Array::from(vec![Some(4), None]) as &dyn Array
1188        );
1189        assert_eq!(
1190            column("content_size_in_bytes").as_ref(),
1191            &Int64Array::from(vec![Some(40), None]) as &dyn Array
1192        );
1193        let equality_ids = column("equality_ids");
1194        let equality_ids = equality_ids.as_list::<i32>();
1195        assert!(equality_ids.is_null(0));
1196        assert_eq!(
1197            equality_ids.value(1).as_ref(),
1198            &Int32Array::from(vec![1, 3]) as &dyn Array
1199        );
1200    }
1201
1202    #[tokio::test]
1203    async fn test_partition_values() {
1204        let identity_columns = [
1205            "b", "i", "l", "f", "d", "dec", "date", "time", "ts", "tstz", "ts_ns", "tstz_ns", "s",
1206            "fx", "bin",
1207        ];
1208        let mut fields: Vec<_> = identity_columns
1209            .iter()
1210            .map(|column| (*column, Transform::Identity))
1211            .collect();
1212        fields.extend([
1213            ("i", Transform::Bucket(4)),
1214            ("ts", Transform::Day),
1215            ("s", Transform::Truncate(2)),
1216            ("l", Transform::Void),
1217        ]);
1218        let spec = partition_spec(&fields);
1219
1220        let values = Struct::from_iter([
1221            Some(Literal::bool(true)),
1222            Some(Literal::int(-7)),
1223            Some(Literal::long(1i64 << 40)),
1224            Some(Literal::float(1.5f32)),
1225            Some(Literal::double(-2.25)),
1226            Some(Literal::decimal(-12_345)),
1227            Some(Literal::date(19_000)),
1228            Some(Literal::time(3_600_000_000)),
1229            Some(Literal::timestamp(1_700_000_000_000_000)),
1230            Some(Literal::timestamptz(1_700_000_000_000_001)),
1231            Some(Literal::timestamp_nano(1_700_000_000_000_000_002)),
1232            Some(Literal::timestamptz_nano(1_700_000_000_000_000_003)),
1233            Some(Literal::string("abc")),
1234            Some(Literal::fixed([1u8, 2, 3])),
1235            Some(Literal::binary([9u8])),
1236            Some(Literal::int(3)),
1237            Some(Literal::date(19_675)),
1238            Some(Literal::string("ab")),
1239            None,
1240        ]);
1241        let nulls = Struct::from_iter(fields.iter().map(|_| None));
1242        let (bytes, manifest_file) = write_manifest(
1243            spec,
1244            Some(1),
1245            |b| b.build_v2_data(),
1246            |writer| {
1247                for (i, partition) in [values, nulls].into_iter().enumerate() {
1248                    let file = DataFile {
1249                        partition,
1250                        ..data_file(DataContentType::Data, &format!("s3://b/data/{i}.parquet"))
1251                            .build()
1252                            .unwrap()
1253                    };
1254                    writer.add_file(file, 1).unwrap();
1255                }
1256            },
1257        )
1258        .await;
1259        let batch = read_entries(&bytes, &manifest_file).unwrap();
1260
1261        let partition = data_file_column(&batch, "partition");
1262        let partition = partition.as_struct();
1263        let expected: Vec<ArrayRef> = vec![
1264            Arc::new(BooleanArray::from(vec![Some(true), None])),
1265            Arc::new(Int32Array::from(vec![Some(-7), None])),
1266            Arc::new(Int64Array::from(vec![Some(1i64 << 40), None])),
1267            Arc::new(Float32Array::from(vec![Some(1.5), None])),
1268            Arc::new(Float64Array::from(vec![Some(-2.25), None])),
1269            Arc::new(
1270                Decimal128Array::from(vec![Some(-12_345), None])
1271                    .with_precision_and_scale(38, 10)
1272                    .unwrap(),
1273            ),
1274            Arc::new(Date32Array::from(vec![Some(19_000), None])),
1275            Arc::new(Time64MicrosecondArray::from(vec![
1276                Some(3_600_000_000),
1277                None,
1278            ])),
1279            Arc::new(TimestampMicrosecondArray::from(vec![
1280                Some(1_700_000_000_000_000),
1281                None,
1282            ])),
1283            Arc::new(
1284                TimestampMicrosecondArray::from(vec![Some(1_700_000_000_000_001), None])
1285                    .with_timezone("+00:00"),
1286            ),
1287            Arc::new(TimestampNanosecondArray::from(vec![
1288                Some(1_700_000_000_000_000_002),
1289                None,
1290            ])),
1291            Arc::new(
1292                TimestampNanosecondArray::from(vec![Some(1_700_000_000_000_000_003), None])
1293                    .with_timezone("+00:00"),
1294            ),
1295            Arc::new(StringArray::from(vec![Some("abc"), None])),
1296            Arc::new(
1297                FixedSizeBinaryArray::try_from_sparse_iter_with_size(
1298                    [Some(vec![1u8, 2, 3]), None].into_iter(),
1299                    3,
1300                )
1301                .unwrap(),
1302            ),
1303            Arc::new(BinaryArray::from(vec![Some(&[9u8][..]), None])),
1304            Arc::new(Int32Array::from(vec![Some(3), None])),
1305            Arc::new(Date32Array::from(vec![Some(19_675), None])),
1306            Arc::new(StringArray::from(vec![Some("ab"), None])),
1307            Arc::new(Int64Array::from(vec![None, None])),
1308        ];
1309        assert_eq!(partition.num_columns(), expected.len());
1310        for (index, expected) in expected.iter().enumerate() {
1311            assert_eq!(
1312                partition.column(index).as_ref(),
1313                expected.as_ref(),
1314                "partition field {}",
1315                partition.fields()[index].name()
1316            );
1317        }
1318    }
1319
1320    #[tokio::test]
1321    async fn test_v1_manifest_reads_with_v2_defaults() {
1322        let file = data_file(DataContentType::Data, "s3://b/data/0.parquet")
1323            .build()
1324            .unwrap();
1325        let (bytes, mut manifest_file) = write_manifest(
1326            partition_spec(&[]),
1327            Some(1),
1328            |b| b.build_v1(),
1329            |writer| writer.add_file(file, -1).unwrap(),
1330        )
1331        .await;
1332        // The v1 default must win over inheritance, so give the manifest list
1333        // entry a sequence number that inheritance would use.
1334        manifest_file.sequence_number = 9;
1335        let batch = read_entries(&bytes, &manifest_file).unwrap();
1336
1337        assert_eq!(
1338            data_file_column(&batch, "content").as_ref(),
1339            &Int32Array::from(vec![0]) as &dyn Array
1340        );
1341        for name in ["sequence_number", "file_sequence_number"] {
1342            assert_eq!(
1343                column(&batch, name).as_ref(),
1344                &Int64Array::from(vec![0]) as &dyn Array,
1345                "{name}"
1346            );
1347        }
1348        assert_eq!(
1349            column(&batch, "snapshot_id").as_ref(),
1350            &Int64Array::from(vec![1]) as &dyn Array
1351        );
1352    }
1353
1354    #[tokio::test]
1355    async fn test_snapshot_id_and_sequence_number_inheritance() {
1356        let file = |i: usize| {
1357            data_file(DataContentType::Data, &format!("s3://b/data/{i}.parquet"))
1358                .build()
1359                .unwrap()
1360        };
1361        let (bytes, mut manifest_file) = write_manifest(
1362            partition_spec(&[]),
1363            None,
1364            |b| b.build_v2_data(),
1365            |writer| {
1366                writer.add_file(file(0), -1).unwrap();
1367                writer.add_delete_file(file(1), 4, Some(4)).unwrap();
1368                writer.add_existing_file(file(2), 3, 5, Some(5)).unwrap();
1369            },
1370        )
1371        .await;
1372        // ManifestWriter rejects an existing entry without sequence numbers.
1373        let bytes = rewrite_manifest(
1374            &bytes,
1375            Codec::Null,
1376            |_| {},
1377            |record| {
1378                let AvroValue::Record(fields) = record else {
1379                    panic!("manifest entry is not a record");
1380                };
1381                if fields.contains(&("status".to_string(), AvroValue::Int(0))) {
1382                    for (name, value) in fields {
1383                        if name.ends_with("sequence_number") {
1384                            *value = AvroValue::Union(0, Box::new(AvroValue::Null));
1385                        }
1386                    }
1387                }
1388            },
1389        );
1390        manifest_file.added_snapshot_id = 42;
1391        manifest_file.sequence_number = 7;
1392        let batch = read_entries(&bytes, &manifest_file).unwrap();
1393
1394        assert_eq!(
1395            column(&batch, "snapshot_id").as_ref(),
1396            &Int64Array::from(vec![42, 42, 3]) as &dyn Array
1397        );
1398        for name in ["sequence_number", "file_sequence_number"] {
1399            assert_eq!(
1400                column(&batch, name).as_ref(),
1401                &Int64Array::from(vec![Some(7), Some(4), None]) as &dyn Array,
1402                "{name}"
1403            );
1404        }
1405    }
1406
1407    #[tokio::test]
1408    async fn test_v2_manifest_without_file_sequence_number_inherits() {
1409        let file = |i: usize| {
1410            data_file(DataContentType::Data, &format!("s3://b/data/{i}.parquet"))
1411                .build()
1412                .unwrap()
1413        };
1414        let (bytes, mut manifest_file) = write_manifest(
1415            partition_spec(&[]),
1416            Some(1),
1417            |b| b.build_v2_data(),
1418            |writer| {
1419                writer.add_file(file(0), -1).unwrap();
1420                writer.add_existing_file(file(1), 3, 5, Some(5)).unwrap();
1421            },
1422        )
1423        .await;
1424        // Java wrote v2 manifests without file_sequence_number before 1.1.0.
1425        let bytes = rewrite_manifest(
1426            &bytes,
1427            Codec::Null,
1428            |schema| {
1429                schema["fields"]
1430                    .as_array_mut()
1431                    .unwrap()
1432                    .retain(|field| field["name"] != "file_sequence_number")
1433            },
1434            |record| {
1435                let AvroValue::Record(fields) = record else {
1436                    panic!("manifest entry is not a record");
1437                };
1438                fields.retain(|(name, _)| name != "file_sequence_number");
1439            },
1440        );
1441        manifest_file.sequence_number = 7;
1442        let batch = read_entries(&bytes, &manifest_file).unwrap();
1443
1444        assert_eq!(
1445            column(&batch, "file_sequence_number").as_ref(),
1446            &Int64Array::from(vec![Some(7), None]) as &dyn Array
1447        );
1448    }
1449
1450    async fn first_row_id_manifest() -> (Vec<u8>, ManifestFile) {
1451        let file = |i: usize, record_count: u64, first_row_id: Option<i64>| DataFile {
1452            first_row_id,
1453            ..data_file(DataContentType::Data, &format!("s3://b/data/{i}.parquet"))
1454                .record_count(record_count)
1455                .build()
1456                .unwrap()
1457        };
1458        let entry = |status, data_file| ManifestEntry {
1459            status,
1460            snapshot_id: Some(1),
1461            sequence_number: Some(1),
1462            file_sequence_number: Some(1),
1463            data_file,
1464        };
1465        write_manifest(
1466            partition_spec(&[]),
1467            Some(1),
1468            |b| b.build_v3_data(),
1469            |writer| {
1470                writer.add_file(file(0, 10, None), 1).unwrap();
1471                writer
1472                    .add_existing_entry(entry(ManifestStatus::Existing, file(1, 3, Some(5))))
1473                    .unwrap();
1474                writer
1475                    .add_delete_entry(entry(ManifestStatus::Deleted, file(2, 4, None)))
1476                    .unwrap();
1477                writer
1478                    .add_existing_entry(entry(ManifestStatus::Existing, file(3, 6, None)))
1479                    .unwrap();
1480                writer.add_file(file(4, 2, None), 1).unwrap();
1481            },
1482        )
1483        .await
1484    }
1485
1486    #[tokio::test]
1487    async fn test_first_row_id_inheritance() {
1488        let (bytes, mut manifest_file) = first_row_id_manifest().await;
1489        manifest_file.first_row_id = Some(100);
1490        let batch = read_entries(&bytes, &manifest_file).unwrap();
1491        // Deleted entries don't take row IDs, and entries with a first_row_id keep it.
1492        assert_eq!(
1493            data_file_column(&batch, "first_row_id").as_ref(),
1494            &Int64Array::from(vec![Some(100), Some(5), None, Some(110), Some(116)]) as &dyn Array
1495        );
1496
1497        manifest_file.first_row_id = None;
1498        let batch = read_entries(&bytes, &manifest_file).unwrap();
1499        assert_eq!(data_file_column(&batch, "first_row_id").null_count(), 5);
1500
1501        manifest_file.first_row_id = Some(i64::MAX as u64 - 5);
1502        let err = read_entries(&bytes, &manifest_file).unwrap_err();
1503        assert_eq!(err.kind(), ErrorKind::DataInvalid);
1504
1505        // A negative record_count would assign overlapping row IDs.
1506        let bytes = negate_record_count(&bytes, 10);
1507        manifest_file.first_row_id = Some(100);
1508        let err = read_entries(&bytes, &manifest_file).unwrap_err();
1509        assert_eq!(err.kind(), ErrorKind::DataInvalid);
1510    }
1511
1512    /// Negates the `record_count` of the data files whose count is
1513    /// `record_count`.
1514    fn negate_record_count(bytes: &[u8], record_count: i64) -> Vec<u8> {
1515        rewrite_manifest(
1516            bytes,
1517            Codec::Null,
1518            |_| {},
1519            |record| {
1520                let AvroValue::Record(fields) = record else {
1521                    panic!("manifest entry is not a record");
1522                };
1523                let Some((_, AvroValue::Record(data_file))) =
1524                    fields.iter_mut().find(|(name, _)| name == "data_file")
1525                else {
1526                    panic!("manifest entry has no data_file record");
1527                };
1528                for (name, value) in data_file {
1529                    if name == "record_count" && *value == AvroValue::Long(record_count) {
1530                        *value = AvroValue::Long(-record_count);
1531                    }
1532                }
1533            },
1534        )
1535    }
1536
1537    #[tokio::test]
1538    async fn test_manifest_batches_match_whole_manifest() {
1539        let (first_row_id_bytes, mut first_row_id_manifest_file) = first_row_id_manifest().await;
1540        first_row_id_manifest_file.first_row_id = Some(100);
1541        let v1_files = (0..3).map(|i| {
1542            data_file(DataContentType::Data, &format!("s3://b/data/{i}.parquet"))
1543                .build()
1544                .unwrap()
1545        });
1546        let v1 = write_manifest(
1547            partition_spec(&[]),
1548            Some(1),
1549            |b| b.build_v1(),
1550            |writer| v1_files.for_each(|file| writer.add_file(file, -1).unwrap()),
1551        )
1552        .await;
1553        let empty = write_manifest(
1554            partition_spec(&[("i", Transform::Identity)]),
1555            Some(1),
1556            |b| b.build_v2_data(),
1557            |_| {},
1558        )
1559        .await;
1560        let pyiceberg_bytes = std::fs::read(format!(
1561            "{}/testdata/manifests/pyiceberg-v2-data.avro",
1562            env!("CARGO_MANIFEST_DIR")
1563        ))
1564        .unwrap();
1565        let pyiceberg_manifest_file = ManifestFile {
1566            sequence_number: 1,
1567            ..listed_manifest("s3://b/metadata/0.avro", ManifestContentType::Data)
1568        };
1569
1570        for (bytes, manifest_file) in [
1571            (first_row_id_bytes, first_row_id_manifest_file),
1572            v1,
1573            empty,
1574            (pyiceberg_bytes, pyiceberg_manifest_file),
1575        ] {
1576            assert_batches_match_whole(|batch_size| {
1577                let reader =
1578                    ManifestEntryBatchReader::try_new(&bytes, &manifest_file, batch_size).unwrap();
1579                (reader.schema(), reader)
1580            });
1581        }
1582    }
1583
1584    #[tokio::test]
1585    async fn test_unknown_manifest_format_version_is_an_error() {
1586        let (bytes, manifest_file) = first_row_id_manifest().await;
1587        let bytes = set_avro_metadata(&bytes, "format-version", Some("4"));
1588        let Err(err) = read_manifest_entries(&bytes, &manifest_file) else {
1589            panic!("a v4 manifest read as an older version");
1590        };
1591        assert_eq!(err.kind(), ErrorKind::DataInvalid);
1592    }
1593
1594    #[tokio::test]
1595    async fn test_manifest_error_ends_the_batches() {
1596        let (bytes, mut manifest_file) = first_row_id_manifest().await;
1597        manifest_file.first_row_id = Some(100);
1598        // The last of the five entries fails first row ID inheritance, so the
1599        // third batch of two is an error.
1600        let bytes = negate_record_count(&bytes, 2);
1601        let mut reader = ManifestEntryBatchReader::try_new(&bytes, &manifest_file, 2).unwrap();
1602        assert_eq!(reader.next().unwrap().unwrap().num_rows(), 2);
1603        assert_eq!(reader.next().unwrap().unwrap().num_rows(), 2);
1604        assert_eq!(
1605            reader.next().unwrap().unwrap_err().kind(),
1606            ErrorKind::DataInvalid
1607        );
1608        assert!(reader.next().is_none());
1609    }
1610
1611    #[tokio::test]
1612    async fn test_compressed_manifests() {
1613        let (bytes, manifest_file) = first_row_id_manifest().await;
1614        let expected = read_entries(&bytes, &manifest_file).unwrap();
1615        for codec in [
1616            Codec::Deflate(DeflateSettings::default()),
1617            Codec::Snappy,
1618            Codec::Zstandard(ZstandardSettings::default()),
1619        ] {
1620            let bytes = rewrite_manifest(&bytes, codec, |_| {}, |_| {});
1621            assert_eq!(
1622                read_entries(&bytes, &manifest_file).unwrap(),
1623                expected,
1624                "{codec:?}"
1625            );
1626        }
1627    }
1628
1629    #[tokio::test]
1630    async fn test_fields_are_matched_by_field_id() {
1631        let file = DataFile {
1632            partition: Struct::from_iter([Some(Literal::int(7))]),
1633            ..data_file(DataContentType::Data, "s3://b/data/0.parquet")
1634                .build()
1635                .unwrap()
1636        };
1637        let (bytes, manifest_file) = write_manifest(
1638            partition_spec(&[("i", Transform::Identity)]),
1639            Some(1),
1640            |b| b.build_v2_data(),
1641            |writer| writer.add_file(file, 1).unwrap(),
1642        )
1643        .await;
1644        let bytes = rename_fields(&bytes, &[
1645            ("file_path", "path"),
1646            ("i_identity", "renamed_partition"),
1647        ]);
1648        let batch = read_entries(&bytes, &manifest_file).unwrap();
1649
1650        assert_eq!(
1651            data_file_column(&batch, "file_path").as_ref(),
1652            &StringArray::from(vec!["s3://b/data/0.parquet"]) as &dyn Array
1653        );
1654        let partition = data_file_column(&batch, "partition");
1655        assert_eq!(
1656            partition.as_struct().column(0).as_ref(),
1657            &Int32Array::from(vec![7]) as &dyn Array
1658        );
1659    }
1660
1661    #[tokio::test]
1662    async fn test_missing_required_field_is_an_error() {
1663        let file = data_file(DataContentType::Data, "s3://b/data/0.parquet")
1664            .build()
1665            .unwrap();
1666        let (bytes, manifest_file) = write_manifest(
1667            partition_spec(&[]),
1668            Some(1),
1669            |b| b.build_v2_data(),
1670            |writer| writer.add_file(file, 1).unwrap(),
1671        )
1672        .await;
1673        // A file_path under another field ID leaves the required file_path missing.
1674        let bytes = rewrite_manifest(
1675            &bytes,
1676            Codec::Null,
1677            |schema| {
1678                for_each_avro_field(schema, &|field| {
1679                    if field["name"] == "file_path" {
1680                        field["field-id"] = 999.into();
1681                    }
1682                })
1683            },
1684            |_| {},
1685        );
1686        let err = read_entries(&bytes, &manifest_file).unwrap_err();
1687        assert_eq!(err.kind(), ErrorKind::DataInvalid);
1688    }
1689
1690    #[test]
1691    fn test_manifest_written_by_pyiceberg() {
1692        let bytes = std::fs::read(format!(
1693            "{}/testdata/manifests/pyiceberg-v2-data.avro",
1694            env!("CARGO_MANIFEST_DIR")
1695        ))
1696        .unwrap();
1697        let manifest = Manifest::parse_avro(&bytes).unwrap();
1698        let manifest_file = ManifestFile {
1699            manifest_path: "s3://bucket/metadata/manifest.avro".to_string(),
1700            manifest_length: bytes.len() as i64,
1701            partition_spec_id: 0,
1702            content: ManifestContentType::Data,
1703            sequence_number: 1,
1704            min_sequence_number: 1,
1705            added_snapshot_id: 1,
1706            added_files_count: None,
1707            existing_files_count: None,
1708            deleted_files_count: None,
1709            added_rows_count: None,
1710            existing_rows_count: None,
1711            deleted_rows_count: None,
1712            partitions: None,
1713            key_metadata: None,
1714            first_row_id: None,
1715        };
1716        let batch = read_entries(&bytes, &manifest_file).unwrap();
1717
1718        assert_eq!(batch.num_rows(), manifest.entries().len());
1719        let paths = data_file_column(&batch, "file_path");
1720        let partition = data_file_column(&batch, "partition");
1721        let category = partition.as_struct().column(0).as_string::<i32>().clone();
1722        for (row, entry) in manifest.entries().iter().enumerate() {
1723            let data_file = entry.data_file();
1724            assert_eq!(paths.as_string::<i32>().value(row), data_file.file_path());
1725            let Some(Literal::Primitive(PrimitiveLiteral::String(expected))) =
1726                &data_file.partition()[0]
1727            else {
1728                panic!("partition value is not a string");
1729            };
1730            assert_eq!(category.value(row), expected);
1731            let mut lower_bounds: Vec<_> = data_file
1732                .lower_bounds()
1733                .iter()
1734                .map(|(id, bound)| (*id, bound.to_bytes().unwrap().to_vec()))
1735                .collect();
1736            lower_bounds.sort_unstable();
1737            assert!(!lower_bounds.is_empty());
1738            assert_eq!(
1739                binary_map(&data_file_column(&batch, "lower_bounds"), row),
1740                lower_bounds
1741            );
1742        }
1743    }
1744
1745    /// A manifest list entry that a manifest list writer for snapshot 1
1746    /// assigns sequence numbers to.
1747    fn listed_manifest(path: &str, content: ManifestContentType) -> ManifestFile {
1748        ManifestFile {
1749            manifest_path: path.to_string(),
1750            manifest_length: 100,
1751            partition_spec_id: 0,
1752            content,
1753            sequence_number: UNASSIGNED_SEQUENCE_NUMBER,
1754            min_sequence_number: UNASSIGNED_SEQUENCE_NUMBER,
1755            added_snapshot_id: 1,
1756            added_files_count: Some(1),
1757            existing_files_count: Some(2),
1758            deleted_files_count: Some(3),
1759            added_rows_count: Some(10),
1760            existing_rows_count: Some(20),
1761            deleted_rows_count: Some(30),
1762            partitions: None,
1763            key_metadata: None,
1764            first_row_id: None,
1765        }
1766    }
1767
1768    /// Writes a manifest list for snapshot 1 with sequence number 5 and, in
1769    /// v3, first row ID 100.
1770    async fn write_manifest_list(version: FormatVersion, manifests: Vec<ManifestFile>) -> Vec<u8> {
1771        let io = FileIO::new_with_memory();
1772        let path = "memory:///manifest-list.avro";
1773        let output = io.new_output(path).unwrap().writer().await.unwrap();
1774        let mut writer = match version {
1775            FormatVersion::V1 => ManifestListWriter::v1(output, 1, None),
1776            FormatVersion::V2 => ManifestListWriter::v2(output, 1, None, 5),
1777            FormatVersion::V3 => ManifestListWriter::v3(output, 1, None, 5, Some(100)),
1778        };
1779        writer.add_manifests(manifests.into_iter()).unwrap();
1780        writer.close().await.unwrap();
1781        io.new_input(path).unwrap().read().await.unwrap().to_vec()
1782    }
1783
1784    #[tokio::test]
1785    async fn test_schema_is_spec_manifest_file() {
1786        let bytes = write_manifest_list(FormatVersion::V3, vec![]).await;
1787        let batch = read_list(&bytes).unwrap();
1788        assert_eq!(batch.num_rows(), 0);
1789
1790        let long = || Type::Primitive(PrimitiveType::Long);
1791        let int = || Type::Primitive(PrimitiveType::Int);
1792        let boolean = || Type::Primitive(PrimitiveType::Boolean);
1793        let binary = || Type::Primitive(PrimitiveType::Binary);
1794        let field_summary = StructType::new(vec![
1795            NestedField::required(509, "contains_null", boolean()).into(),
1796            NestedField::optional(518, "contains_nan", boolean()).into(),
1797            NestedField::optional(510, "lower_bound", binary()).into(),
1798            NestedField::optional(511, "upper_bound", binary()).into(),
1799        ]);
1800        let expected = Schema::builder()
1801            .with_fields(vec![
1802                NestedField::required(500, "manifest_path", Type::Primitive(PrimitiveType::String))
1803                    .into(),
1804                NestedField::required(501, "manifest_length", long()).into(),
1805                NestedField::required(502, "partition_spec_id", int()).into(),
1806                NestedField::required(517, "content", int()).into(),
1807                NestedField::required(515, "sequence_number", long()).into(),
1808                NestedField::required(516, "min_sequence_number", long()).into(),
1809                NestedField::required(503, "added_snapshot_id", long()).into(),
1810                NestedField::optional(504, "added_files_count", int()).into(),
1811                NestedField::optional(505, "existing_files_count", int()).into(),
1812                NestedField::optional(506, "deleted_files_count", int()).into(),
1813                NestedField::optional(512, "added_rows_count", long()).into(),
1814                NestedField::optional(513, "existing_rows_count", long()).into(),
1815                NestedField::optional(514, "deleted_rows_count", long()).into(),
1816                NestedField::optional(
1817                    507,
1818                    "partitions",
1819                    Type::List(ListType::new(
1820                        NestedField::list_element(508, Type::Struct(field_summary), true).into(),
1821                    )),
1822                )
1823                .into(),
1824                NestedField::optional(519, "key_metadata", binary()).into(),
1825                NestedField::optional(520, "first_row_id", long()).into(),
1826            ])
1827            .build()
1828            .unwrap();
1829        assert_eq!(
1830            arrow_schema_to_schema(&batch.schema()).unwrap().as_struct(),
1831            expected.as_struct()
1832        );
1833    }
1834
1835    #[tokio::test]
1836    async fn test_manifest_list_columns() {
1837        let data = ManifestFile {
1838            partitions: Some(vec![FieldSummary {
1839                contains_null: true,
1840                contains_nan: Some(false),
1841                lower_bound: Some(Datum::int(1).to_bytes().unwrap()),
1842                upper_bound: Some(Datum::int(9).to_bytes().unwrap()),
1843            }]),
1844            key_metadata: Some(vec![1, 2]),
1845            ..listed_manifest("s3://b/metadata/data.avro", ManifestContentType::Data)
1846        };
1847        let existing_deletes = ManifestFile {
1848            partition_spec_id: 1,
1849            sequence_number: 3,
1850            min_sequence_number: 2,
1851            added_snapshot_id: 0,
1852            added_files_count: Some(0),
1853            ..listed_manifest("s3://b/metadata/deletes.avro", ManifestContentType::Deletes)
1854        };
1855        let bytes = write_manifest_list(FormatVersion::V3, vec![data, existing_deletes]).await;
1856        let batch = read_list(&bytes).unwrap();
1857
1858        let schema = batch.schema();
1859        let DataType::List(summary) = schema.field_with_name("partitions").unwrap().data_type()
1860        else {
1861            panic!("partitions is not a list");
1862        };
1863        let DataType::Struct(summary_fields) = summary.data_type() else {
1864            panic!("partitions element is not a struct");
1865        };
1866        let summaries = StructArray::new(
1867            summary_fields.clone(),
1868            vec![
1869                Arc::new(BooleanArray::from(vec![true])),
1870                Arc::new(BooleanArray::from(vec![Some(false)])),
1871                Arc::new(BinaryArray::from(vec![&1i32.to_le_bytes()[..]])),
1872                Arc::new(BinaryArray::from(vec![&9i32.to_le_bytes()[..]])),
1873            ],
1874            None,
1875        );
1876        let columns: Vec<ArrayRef> = vec![
1877            Arc::new(StringArray::from(vec![
1878                "s3://b/metadata/data.avro",
1879                "s3://b/metadata/deletes.avro",
1880            ])),
1881            Arc::new(Int64Array::from(vec![100, 100])),
1882            Arc::new(Int32Array::from(vec![0, 1])),
1883            Arc::new(Int32Array::from(vec![0, 1])),
1884            Arc::new(Int64Array::from(vec![5, 3])),
1885            Arc::new(Int64Array::from(vec![5, 2])),
1886            Arc::new(Int64Array::from(vec![1, 0])),
1887            Arc::new(Int32Array::from(vec![1, 0])),
1888            Arc::new(Int32Array::from(vec![2, 2])),
1889            Arc::new(Int32Array::from(vec![3, 3])),
1890            Arc::new(Int64Array::from(vec![10, 10])),
1891            Arc::new(Int64Array::from(vec![20, 20])),
1892            Arc::new(Int64Array::from(vec![30, 30])),
1893            Arc::new(ListArray::new(
1894                summary.clone(),
1895                OffsetBuffer::from_lengths([1, 0]),
1896                Arc::new(summaries),
1897                Some(NullBuffer::from(vec![true, false])),
1898            )),
1899            Arc::new(BinaryArray::from(vec![Some(&[1u8, 2][..]), None])),
1900            // The writer assigns the data manifest the snapshot's first row ID.
1901            Arc::new(Int64Array::from(vec![Some(100), None])),
1902        ];
1903        assert_eq!(batch, RecordBatch::try_new(schema, columns).unwrap());
1904    }
1905
1906    #[tokio::test]
1907    async fn test_v1_manifest_list_reads_with_v2_defaults() {
1908        let unknown_counts = ManifestFile {
1909            added_files_count: None,
1910            existing_files_count: None,
1911            deleted_files_count: None,
1912            added_rows_count: None,
1913            existing_rows_count: None,
1914            deleted_rows_count: None,
1915            ..listed_manifest("s3://b/metadata/1.avro", ManifestContentType::Data)
1916        };
1917        let bytes = write_manifest_list(FormatVersion::V1, vec![
1918            listed_manifest("s3://b/metadata/0.avro", ManifestContentType::Data),
1919            unknown_counts,
1920        ])
1921        .await;
1922        // A manifest list without a format version reads as v1.
1923        let without_format_version = set_avro_metadata(&bytes, "format-version", None);
1924        for bytes in [bytes, without_format_version] {
1925            let batch = read_list(&bytes).unwrap();
1926
1927            assert_eq!(
1928                column(&batch, "content").as_ref(),
1929                &Int32Array::from(vec![0, 0]) as &dyn Array
1930            );
1931            for name in ["sequence_number", "min_sequence_number"] {
1932                assert_eq!(
1933                    column(&batch, name).as_ref(),
1934                    &Int64Array::from(vec![0, 0]) as &dyn Array,
1935                    "{name}"
1936                );
1937            }
1938            assert_eq!(
1939                column(&batch, "added_files_count").as_ref(),
1940                &Int32Array::from(vec![Some(1), None]) as &dyn Array
1941            );
1942            assert_eq!(
1943                column(&batch, "deleted_rows_count").as_ref(),
1944                &Int64Array::from(vec![Some(30), None]) as &dyn Array
1945            );
1946            assert_eq!(column(&batch, "first_row_id").null_count(), 2);
1947        }
1948    }
1949
1950    #[tokio::test]
1951    async fn test_manifest_list_fields_are_matched_by_field_id() {
1952        let manifest = ManifestFile {
1953            partitions: Some(vec![FieldSummary {
1954                contains_null: true,
1955                contains_nan: None,
1956                lower_bound: None,
1957                upper_bound: None,
1958            }]),
1959            ..listed_manifest("s3://b/metadata/0.avro", ManifestContentType::Data)
1960        };
1961        let bytes = write_manifest_list(FormatVersion::V2, vec![manifest]).await;
1962        let bytes = rename_fields(&bytes, &[
1963            ("manifest_path", "path"),
1964            ("added_files_count", "added_data_files_count"),
1965            ("contains_null", "has_null"),
1966        ]);
1967        let batch = read_list(&bytes).unwrap();
1968
1969        assert_eq!(
1970            column(&batch, "manifest_path").as_ref(),
1971            &StringArray::from(vec!["s3://b/metadata/0.avro"]) as &dyn Array
1972        );
1973        assert_eq!(
1974            column(&batch, "added_files_count").as_ref(),
1975            &Int32Array::from(vec![1]) as &dyn Array
1976        );
1977        let partitions = column(&batch, "partitions");
1978        assert_eq!(
1979            partitions
1980                .as_list::<i32>()
1981                .value(0)
1982                .as_struct()
1983                .column(0)
1984                .as_ref(),
1985            &BooleanArray::from(vec![true]) as &dyn Array
1986        );
1987    }
1988
1989    #[tokio::test]
1990    async fn test_invalid_manifest_list_is_an_error() {
1991        let err = read_list(&[]).unwrap_err();
1992        assert_eq!(err.kind(), ErrorKind::DataInvalid);
1993
1994        let bytes = write_manifest_list(FormatVersion::V2, vec![listed_manifest(
1995            "s3://b/metadata/0.avro",
1996            ManifestContentType::Data,
1997        )])
1998        .await;
1999        // A manifest_path under another field ID leaves the required
2000        // manifest_path missing.
2001        let moved_manifest_path = rewrite_manifest(
2002            &bytes,
2003            Codec::Null,
2004            |schema| {
2005                for_each_avro_field(schema, &|field| {
2006                    if field["name"] == "manifest_path" {
2007                        field["field-id"] = 999.into();
2008                    }
2009                })
2010            },
2011            |_| {},
2012        );
2013        // Only v1 manifest lists read a missing sequence number as 0.
2014        let without_sequence_number = rewrite_manifest(
2015            &bytes,
2016            Codec::Null,
2017            |schema| {
2018                schema["fields"]
2019                    .as_array_mut()
2020                    .unwrap()
2021                    .retain(|field| field["name"] != "sequence_number")
2022            },
2023            |record| {
2024                let AvroValue::Record(fields) = record else {
2025                    panic!("manifest list entry is not a record");
2026                };
2027                fields.retain(|(name, _)| name != "sequence_number");
2028            },
2029        );
2030        let unknown_format_version = set_avro_metadata(&bytes, "format-version", Some("4"));
2031        for bytes in [
2032            moved_manifest_path,
2033            without_sequence_number,
2034            unknown_format_version,
2035        ] {
2036            let err = read_list(&bytes).unwrap_err();
2037            assert_eq!(err.kind(), ErrorKind::DataInvalid);
2038        }
2039    }
2040
2041    #[test]
2042    fn test_encrypted_bytes_are_an_error() {
2043        // An AES GCM stream, which starts with AGS1 instead of Avro's magic.
2044        let bytes = std::fs::read(format!(
2045            "{}/testdata/manifests_lists/manifest-list-v3-encrypted.avro",
2046            env!("CARGO_MANIFEST_DIR")
2047        ))
2048        .unwrap();
2049        let manifest_file = listed_manifest("s3://b/metadata/0.avro", ManifestContentType::Data);
2050        let Err(err) = read_manifest_entries(&bytes, &manifest_file) else {
2051            panic!("an encrypted manifest read as plaintext");
2052        };
2053        assert_eq!(err.kind(), ErrorKind::DataInvalid);
2054        let Err(err) = read_manifest_list(&bytes) else {
2055            panic!("an encrypted manifest list read as plaintext");
2056        };
2057        assert_eq!(err.kind(), ErrorKind::DataInvalid);
2058    }
2059
2060    #[test]
2061    fn test_manifest_lists_written_by_java() {
2062        // Written by Java 1.4.0, with the v1 file count names, and 1.5.0.
2063        for name in ["manifest-list-v2-1.avro", "manifest-list-v2-2.avro"] {
2064            let bytes = std::fs::read(format!(
2065                "{}/testdata/manifests_lists/{name}",
2066                env!("CARGO_MANIFEST_DIR")
2067            ))
2068            .unwrap();
2069            let manifest_list =
2070                ManifestList::parse_with_version(&bytes, FormatVersion::V2).unwrap();
2071            let batch = read_list(&bytes).unwrap();
2072
2073            assert_eq!(batch.num_rows(), manifest_list.entries().len(), "{name}");
2074            let paths = column(&batch, "manifest_path");
2075            let sequence_numbers = column(&batch, "sequence_number");
2076            let added_files_counts = column(&batch, "added_files_count");
2077            let partitions = column(&batch, "partitions");
2078            for (row, manifest) in manifest_list.entries().iter().enumerate() {
2079                assert_eq!(
2080                    paths.as_string::<i32>().value(row),
2081                    manifest.manifest_path,
2082                    "{name}"
2083                );
2084                assert_eq!(
2085                    sequence_numbers.as_primitive::<Int64Type>().value(row),
2086                    manifest.sequence_number,
2087                    "{name}"
2088                );
2089                assert_eq!(
2090                    added_files_counts.as_primitive::<Int32Type>().value(row),
2091                    manifest.added_files_count.unwrap() as i32,
2092                    "{name}"
2093                );
2094                let summaries = partitions.as_list::<i32>().value(row);
2095                let lower_bounds: Vec<_> = summaries
2096                    .as_struct()
2097                    .column_by_name("lower_bound")
2098                    .unwrap()
2099                    .as_binary::<i32>()
2100                    .iter()
2101                    .map(|bound| bound.map(<[u8]>::to_vec))
2102                    .collect();
2103                let expected: Vec<_> = manifest
2104                    .partitions
2105                    .iter()
2106                    .flatten()
2107                    .map(|summary| summary.lower_bound.as_ref().map(|bound| bound.to_vec()))
2108                    .collect();
2109                assert_eq!(lower_bounds, expected, "{name}");
2110            }
2111        }
2112    }
2113
2114    #[tokio::test]
2115    async fn test_manifest_list_batches_match_whole_list() {
2116        for version in [FormatVersion::V1, FormatVersion::V2, FormatVersion::V3] {
2117            for len in [0, 3] {
2118                let manifests = (0..len)
2119                    .map(|i| {
2120                        listed_manifest(
2121                            &format!("s3://b/metadata/{i}.avro"),
2122                            ManifestContentType::Data,
2123                        )
2124                    })
2125                    .collect();
2126                let bytes = write_manifest_list(version, manifests).await;
2127                assert_batches_match_whole(|batch_size| {
2128                    let reader = ManifestListBatchReader::try_new(&bytes, batch_size).unwrap();
2129                    (reader.schema(), reader)
2130                });
2131            }
2132        }
2133    }
2134}