Skip to main content

iceberg/spec/manifest/
mod.rs

1// Licensed to the Apache Software Foundation (ASF) under one
2// or more contributor license agreements.  See the NOTICE file
3// distributed with this work for additional information
4// regarding copyright ownership.  The ASF licenses this file
5// to you under the Apache License, Version 2.0 (the
6// "License"); you may not use this file except in compliance
7// with the License.  You may obtain a copy of the License at
8//
9//   http://www.apache.org/licenses/LICENSE-2.0
10//
11// Unless required by applicable law or agreed to in writing,
12// software distributed under the License is distributed on an
13// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14// KIND, either express or implied.  See the License for the
15// specific language governing permissions and limitations
16// under the License.
17
18mod _serde;
19
20mod data_file;
21pub use data_file::*;
22mod entry;
23pub use entry::*;
24mod metadata;
25pub use metadata::*;
26mod reader;
27pub use reader::*;
28mod writer;
29use std::sync::Arc;
30use std::sync::atomic::{AtomicBool, Ordering};
31
32use apache_avro::Reader as AvroReader;
33use apache_avro::error::Details;
34pub use writer::*;
35
36use super::{
37    Datum, FormatVersion, ManifestContentType, PartitionSpec, PrimitiveType, Schema, Struct, Type,
38    UNASSIGNED_SEQUENCE_NUMBER,
39};
40use crate::avro::{Resolved, define_named_types_once};
41use crate::error::{Error, Result, invalid_data};
42
43/// Whether a manifest with repeated Avro named type definitions was logged at
44/// warn level. A table written by an affected release can have many of them.
45static WARNED_REPEATED_DEFINITIONS: AtomicBool = AtomicBool::new(false);
46
47/// A manifest contains metadata and a list of entries.
48#[derive(Debug, PartialEq, Eq, Clone)]
49pub struct Manifest {
50    metadata: ManifestMetadata,
51    entries: Vec<ManifestEntryRef>,
52}
53
54impl Manifest {
55    /// Parse manifest metadata and entries from bytes of avro file. `location`
56    /// names the manifest in warnings.
57    pub(crate) fn try_from_avro_bytes(
58        bs: &[u8],
59        location: Option<&str>,
60    ) -> Result<(ManifestMetadata, Vec<ManifestEntry>)> {
61        let rewritten;
62        let reader = match AvroReader::new(bs) {
63            Ok(reader) => reader,
64            // iceberg-rust repeated `decimal` definitions before
65            // `schema_to_avro_schema` defined each named type once, so this
66            // fallback stays while tables can contain manifests it wrote.
67            Err(e) if matches!(e.details(), Details::AmbiguousSchemaDefinition(_)) => {
68                let (bs, repeated) = match define_named_types_once(bs) {
69                    Ok(Some(rewrite)) => rewrite,
70                    Ok(None) => return Err(e.into()),
71                    Err(rewrite_error) => {
72                        return Err(Error::from(e).with_context(
73                            "error defining each named type once",
74                            rewrite_error.to_string(),
75                        ));
76                    }
77                };
78                rewritten = bs;
79                let reader = AvroReader::new(rewritten.as_slice()).map_err(|retry_error| {
80                    Error::from(e).with_context(
81                        "error after defining each named type once",
82                        retry_error.to_string(),
83                    )
84                })?;
85                let location = location.unwrap_or("<unknown location>");
86                if WARNED_REPEATED_DEFINITIONS.swap(true, Ordering::Relaxed) {
87                    tracing::debug!(
88                        "Manifest {location} defines Avro named types {repeated:?} more than once."
89                    );
90                } else {
91                    tracing::warn!(
92                        "Manifest {location} defines Avro named types {repeated:?} more than once, \
93                         which the Avro specification doesn't allow. Reading it with each repeated \
94                         definition replaced by a reference to the first. Later manifests like \
95                         this are logged at debug level."
96                    );
97                }
98                reader
99            }
100            Err(e) => return Err(e.into()),
101        };
102
103        // Parse manifest metadata
104        let meta = reader.user_metadata();
105        let metadata = ManifestMetadata::parse(meta)?;
106
107        // Parse manifest entries
108        let partition_type = metadata.partition_spec.partition_type(&metadata.schema)?;
109        // Wrap the partition type once and share it across all entries: the
110        // per-entry conversion needs a `&Type`, and building it here keeps the
111        // lazily-populated field-name lookup from being rebuilt for every entry.
112        let partition_struct_type = Type::Struct(partition_type.clone());
113
114        let entries = match metadata.format_version {
115            FormatVersion::V1 => reader
116                .into_deser_iter::<Resolved<_serde::ManifestEntryV1>>()
117                .map(|entry| {
118                    entry?.0.try_into(
119                        metadata.partition_spec.spec_id(),
120                        &partition_struct_type,
121                        &metadata.schema,
122                    )
123                })
124                .collect::<Result<Vec<_>>>()?,
125            FormatVersion::V2 => reader
126                .into_deser_iter::<Resolved<_serde::ManifestEntryV2>>()
127                .map(|entry| {
128                    entry?.0.try_into(
129                        metadata.partition_spec.spec_id(),
130                        &partition_struct_type,
131                        &metadata.schema,
132                    )
133                })
134                .collect::<Result<Vec<_>>>()?,
135            FormatVersion::V3 => reader
136                .into_deser_iter::<Resolved<_serde::ManifestEntryV3>>()
137                .map(|entry| {
138                    entry?.0.try_into(
139                        metadata.partition_spec.spec_id(),
140                        &partition_struct_type,
141                        &metadata.schema,
142                    )
143                })
144                .collect::<Result<Vec<_>>>()?,
145        };
146
147        Ok((metadata, entries))
148    }
149
150    /// Parse manifest from bytes of avro file.
151    pub fn parse_avro(bs: &[u8]) -> Result<Self> {
152        let (metadata, entries) = Self::try_from_avro_bytes(bs, None)?;
153        Ok(Self::new(metadata, entries))
154    }
155
156    /// Entries slice.
157    pub fn entries(&self) -> &[ManifestEntryRef] {
158        &self.entries
159    }
160
161    /// Get metadata.
162    pub fn metadata(&self) -> &ManifestMetadata {
163        &self.metadata
164    }
165
166    /// Consume this Manifest, returning its constituent parts
167    pub fn into_parts(self) -> (Vec<ManifestEntryRef>, ManifestMetadata) {
168        let Self { entries, metadata } = self;
169        (entries, metadata)
170    }
171
172    /// Constructor from [`ManifestMetadata`] and [`ManifestEntry`]s.
173    pub fn new(metadata: ManifestMetadata, entries: Vec<ManifestEntry>) -> Self {
174        Self {
175            metadata,
176            entries: entries.into_iter().map(Arc::new).collect(),
177        }
178    }
179}
180
181/// Serialize a DataFile to a JSON string.
182pub fn serialize_data_file_to_json(
183    data_file: DataFile,
184    partition_type: &super::StructType,
185    format_version: FormatVersion,
186) -> Result<String> {
187    let partition_struct_type = Type::Struct(partition_type.clone());
188    let serde = _serde::DataFileSerde::try_from(data_file, &partition_struct_type, format_version)?;
189    serde_json::to_string(&serde)
190        .map_err(|e| invalid_data!("Failed to serialize DataFile to JSON!").with_source(e))
191}
192
193/// Deserialize a DataFile from a JSON string.
194pub fn deserialize_data_file_from_json(
195    json: &str,
196    partition_spec_id: i32,
197    partition_type: &super::StructType,
198    schema: &Schema,
199) -> Result<DataFile> {
200    let serde = serde_json::from_str::<_serde::DataFileSerde>(json)
201        .map_err(|e| invalid_data!("Failed to deserialize JSON to DataFile!").with_source(e))?;
202
203    let partition_struct_type = Type::Struct(partition_type.clone());
204    serde.try_into(partition_spec_id, &partition_struct_type, schema)
205}
206
207#[cfg(test)]
208mod tests {
209    use std::collections::HashMap;
210    use std::fs;
211    use std::sync::Arc;
212
213    use apache_avro::types::Value as AvroValue;
214    use apache_avro::{Codec, Writer, to_value};
215    use serde_json::{Value, to_vec};
216    use tempfile::TempDir;
217    use tracing::Level;
218    use tracing::subscriber::with_default;
219
220    use super::*;
221    use crate::ErrorKind;
222    use crate::io::FileIO;
223    use crate::spec::{Literal, NestedField, PrimitiveType, Struct, Transform, Type};
224
225    #[tokio::test]
226    async fn test_parse_manifest_v2_unpartition() {
227        let schema = Arc::new(
228            Schema::builder()
229                .with_fields(vec![
230                    // id v_int v_long v_float v_double v_varchar v_bool v_date v_timestamp v_decimal v_ts_ntz
231                    Arc::new(NestedField::optional(
232                        1,
233                        "id",
234                        Type::Primitive(PrimitiveType::Long),
235                    )),
236                    Arc::new(NestedField::optional(
237                        2,
238                        "v_int",
239                        Type::Primitive(PrimitiveType::Int),
240                    )),
241                    Arc::new(NestedField::optional(
242                        3,
243                        "v_long",
244                        Type::Primitive(PrimitiveType::Long),
245                    )),
246                    Arc::new(NestedField::optional(
247                        4,
248                        "v_float",
249                        Type::Primitive(PrimitiveType::Float),
250                    )),
251                    Arc::new(NestedField::optional(
252                        5,
253                        "v_double",
254                        Type::Primitive(PrimitiveType::Double),
255                    )),
256                    Arc::new(NestedField::optional(
257                        6,
258                        "v_varchar",
259                        Type::Primitive(PrimitiveType::String),
260                    )),
261                    Arc::new(NestedField::optional(
262                        7,
263                        "v_bool",
264                        Type::Primitive(PrimitiveType::Boolean),
265                    )),
266                    Arc::new(NestedField::optional(
267                        8,
268                        "v_date",
269                        Type::Primitive(PrimitiveType::Date),
270                    )),
271                    Arc::new(NestedField::optional(
272                        9,
273                        "v_timestamp",
274                        Type::Primitive(PrimitiveType::Timestamptz),
275                    )),
276                    Arc::new(NestedField::optional(
277                        10,
278                        "v_decimal",
279                        Type::Primitive(PrimitiveType::Decimal {
280                            precision: 36,
281                            scale: 10,
282                        }),
283                    )),
284                    Arc::new(NestedField::optional(
285                        11,
286                        "v_ts_ntz",
287                        Type::Primitive(PrimitiveType::Timestamp),
288                    )),
289                    Arc::new(NestedField::optional(
290                        12,
291                        "v_ts_ns_ntz",
292                        Type::Primitive(PrimitiveType::TimestampNs),
293                    )),
294                ])
295                .build()
296                .unwrap(),
297        );
298        let metadata = ManifestMetadata {
299            schema_id: 0,
300            schema: schema.clone(),
301            partition_spec: PartitionSpec::builder(schema)
302                .with_spec_id(0)
303                .build()
304                .unwrap(),
305            content: ManifestContentType::Data,
306            format_version: FormatVersion::V2,
307        };
308        let mut entries = vec![
309                ManifestEntry {
310                    status: ManifestStatus::Added,
311                    snapshot_id: None,
312                    sequence_number: None,
313                    file_sequence_number: None,
314                    data_file: DataFile {content:DataContentType::Data,file_path:"s3a://icebergdata/demo/s1/t1/data/00000-0-ba56fbfa-f2ff-40c9-bb27-565ad6dc2be8-00000.parquet".to_string(),file_format:DataFileFormat::Parquet,partition:Struct::empty(),record_count:1,file_size_in_bytes:5442,column_sizes:HashMap::from([(0,73),(6,34),(2,73),(7,61),(3,61),(5,62),(9,79),(10,73),(1,61),(4,73),(8,73)]),value_counts:HashMap::from([(4,1),(5,1),(2,1),(0,1),(3,1),(6,1),(8,1),(1,1),(10,1),(7,1),(9,1)]),null_value_counts:HashMap::from([(1,0),(6,0),(2,0),(8,0),(0,0),(3,0),(5,0),(9,0),(7,0),(4,0),(10,0)]),nan_value_counts:HashMap::new(),lower_bounds:HashMap::new(),upper_bounds:HashMap::new(),key_metadata:None,split_offsets:Some(vec![4]),equality_ids:Some(Vec::new()),sort_order_id:None, partition_spec_id: 0,first_row_id: None,referenced_data_file: None,content_offset: None,content_size_in_bytes: None }
315                }
316            ];
317
318        // write manifest to file
319        let tmp_dir = TempDir::new().unwrap();
320        let path = tmp_dir.path().join("test_manifest.avro");
321        let io = FileIO::new_with_fs();
322        let output_file = io.new_output(path.to_str().unwrap()).unwrap();
323        let mut writer = ManifestWriterBuilder::new(
324            output_file,
325            Some(1),
326            metadata.schema.clone(),
327            metadata.partition_spec.clone(),
328        )
329        .build_v2_data();
330        for entry in &entries {
331            writer.add_entry(entry.clone()).unwrap();
332        }
333        writer.write_manifest_file().await.unwrap();
334
335        // read back the manifest file and check the content
336        let actual_manifest =
337            Manifest::parse_avro(fs::read(path).expect("read_file must succeed").as_slice())
338                .unwrap();
339        // The snapshot id is assigned when the entry is added to the manifest.
340        entries[0].snapshot_id = Some(1);
341        assert_eq!(actual_manifest, Manifest::new(metadata, entries));
342    }
343
344    #[test]
345    fn test_parse_snappy_manifest_v2() {
346        let schema = Arc::new(
347            Schema::builder()
348                .with_fields(vec![Arc::new(NestedField::optional(
349                    1,
350                    "id",
351                    Type::Primitive(PrimitiveType::Long),
352                ))])
353                .build()
354                .unwrap(),
355        );
356        let partition_spec = PartitionSpec::builder(schema.clone())
357            .with_spec_id(0)
358            .build()
359            .unwrap();
360
361        for (manifest_content, file_content, file_path) in [
362            (
363                ManifestContentType::Data,
364                DataContentType::Data,
365                "s3://bucket/table/data/data.parquet",
366            ),
367            (
368                ManifestContentType::Deletes,
369                DataContentType::PositionDeletes,
370                "s3://bucket/table/data/delete.parquet",
371            ),
372        ] {
373            let metadata = ManifestMetadata {
374                schema_id: 0,
375                schema: schema.clone(),
376                partition_spec: partition_spec.clone(),
377                content: manifest_content,
378                format_version: FormatVersion::V2,
379            };
380            let entry = ManifestEntry {
381                status: ManifestStatus::Added,
382                snapshot_id: Some(1),
383                sequence_number: None,
384                file_sequence_number: None,
385                data_file: DataFile {
386                    content: file_content,
387                    file_path: file_path.to_string(),
388                    file_format: DataFileFormat::Parquet,
389                    partition: Struct::empty(),
390                    record_count: 1,
391                    file_size_in_bytes: 1024,
392                    column_sizes: HashMap::new(),
393                    value_counts: HashMap::new(),
394                    null_value_counts: HashMap::new(),
395                    nan_value_counts: HashMap::new(),
396                    lower_bounds: HashMap::new(),
397                    upper_bounds: HashMap::new(),
398                    key_metadata: None,
399                    split_offsets: None,
400                    equality_ids: None,
401                    sort_order_id: None,
402                    partition_spec_id: 0,
403                    first_row_id: None,
404                    referenced_data_file: None,
405                    content_offset: None,
406                    content_size_in_bytes: None,
407                },
408            };
409
410            let partition_type = metadata
411                .partition_spec
412                .partition_type(&metadata.schema)
413                .unwrap();
414            let avro_schema = manifest_schema_v2(&partition_type).unwrap();
415            let mut writer = Writer::with_codec(&avro_schema, Vec::new(), Codec::Snappy).unwrap();
416            writer
417                .add_user_metadata("schema".to_string(), to_vec(&metadata.schema).unwrap())
418                .unwrap();
419            writer
420                .add_user_metadata(
421                    "schema-id".to_string(),
422                    metadata.schema.schema_id().to_string(),
423                )
424                .unwrap();
425            writer
426                .add_user_metadata(
427                    "partition-spec".to_string(),
428                    to_vec(&metadata.partition_spec.fields()).unwrap(),
429                )
430                .unwrap();
431            writer
432                .add_user_metadata(
433                    "partition-spec-id".to_string(),
434                    metadata.partition_spec.spec_id().to_string(),
435                )
436                .unwrap();
437            writer
438                .add_user_metadata(
439                    "format-version".to_string(),
440                    (metadata.format_version as u8).to_string(),
441                )
442                .unwrap();
443            writer
444                .add_user_metadata("content".to_string(), metadata.content.to_string())
445                .unwrap();
446            let value = to_value(
447                _serde::ManifestEntryV2::try_from(
448                    entry.clone(),
449                    &Type::Struct(partition_type.clone()),
450                )
451                .unwrap(),
452            )
453            .unwrap()
454            .resolve(&avro_schema)
455            .unwrap();
456            writer.append_value(value).unwrap();
457            let bs = writer.into_inner().unwrap();
458
459            let parsed_manifest = Manifest::parse_avro(&bs).unwrap();
460
461            assert_eq!(parsed_manifest, Manifest::new(metadata, vec![entry]));
462        }
463    }
464
465    #[tokio::test]
466    async fn test_parse_manifest_v2_partition() {
467        let schema = Arc::new(
468            Schema::builder()
469                .with_fields(vec![
470                    Arc::new(NestedField::optional(
471                        1,
472                        "id",
473                        Type::Primitive(PrimitiveType::Long),
474                    )),
475                    Arc::new(NestedField::optional(
476                        2,
477                        "v_int",
478                        Type::Primitive(PrimitiveType::Int),
479                    )),
480                    Arc::new(NestedField::optional(
481                        3,
482                        "v_long",
483                        Type::Primitive(PrimitiveType::Long),
484                    )),
485                    Arc::new(NestedField::optional(
486                        4,
487                        "v_float",
488                        Type::Primitive(PrimitiveType::Float),
489                    )),
490                    Arc::new(NestedField::optional(
491                        5,
492                        "v_double",
493                        Type::Primitive(PrimitiveType::Double),
494                    )),
495                    Arc::new(NestedField::optional(
496                        6,
497                        "v_varchar",
498                        Type::Primitive(PrimitiveType::String),
499                    )),
500                    Arc::new(NestedField::optional(
501                        7,
502                        "v_bool",
503                        Type::Primitive(PrimitiveType::Boolean),
504                    )),
505                    Arc::new(NestedField::optional(
506                        8,
507                        "v_date",
508                        Type::Primitive(PrimitiveType::Date),
509                    )),
510                    Arc::new(NestedField::optional(
511                        9,
512                        "v_timestamp",
513                        Type::Primitive(PrimitiveType::Timestamptz),
514                    )),
515                    Arc::new(NestedField::optional(
516                        10,
517                        "v_decimal",
518                        Type::Primitive(PrimitiveType::Decimal {
519                            precision: 36,
520                            scale: 10,
521                        }),
522                    )),
523                    Arc::new(NestedField::optional(
524                        11,
525                        "v_ts_ntz",
526                        Type::Primitive(PrimitiveType::Timestamp),
527                    )),
528                    Arc::new(NestedField::optional(
529                        12,
530                        "v_ts_ns_ntz",
531                        Type::Primitive(PrimitiveType::TimestampNs),
532                    )),
533                ])
534                .build()
535                .unwrap(),
536        );
537        let metadata = ManifestMetadata {
538            schema_id: 0,
539            schema: schema.clone(),
540            partition_spec: PartitionSpec::builder(schema)
541                .with_spec_id(0)
542                .add_partition_field("v_int", "v_int", Transform::Identity)
543                .unwrap()
544                .add_partition_field("v_long", "v_long", Transform::Identity)
545                .unwrap()
546                .build()
547                .unwrap(),
548            content: ManifestContentType::Data,
549            format_version: FormatVersion::V2,
550        };
551        let mut entries = vec![ManifestEntry {
552                status: ManifestStatus::Added,
553                snapshot_id: None,
554                sequence_number: None,
555                file_sequence_number: None,
556                data_file: DataFile {
557                    content: DataContentType::Data,
558                    file_format: DataFileFormat::Parquet,
559                    file_path: "s3a://icebergdata/demo/s1/t1/data/00000-0-378b56f5-5c52-4102-a2c2-f05f8a7cbe4a-00000.parquet".to_string(),
560                    partition: Struct::from_iter(
561                        vec![
562                            Some(Literal::int(1)),
563                            Some(Literal::long(1000)),
564                        ]
565                            .into_iter()
566                    ),
567                    record_count: 1,
568                    file_size_in_bytes: 5442,
569                    column_sizes: HashMap::from([
570                        (0, 73),
571                        (6, 34),
572                        (2, 73),
573                        (7, 61),
574                        (3, 61),
575                        (5, 62),
576                        (9, 79),
577                        (10, 73),
578                        (1, 61),
579                        (4, 73),
580                        (8, 73)
581                    ]),
582                    value_counts: HashMap::from([
583                        (4, 1),
584                        (5, 1),
585                        (2, 1),
586                        (0, 1),
587                        (3, 1),
588                        (6, 1),
589                        (8, 1),
590                        (1, 1),
591                        (10, 1),
592                        (7, 1),
593                        (9, 1)
594                    ]),
595                    null_value_counts: HashMap::from([
596                        (1, 0),
597                        (6, 0),
598                        (2, 0),
599                        (8, 0),
600                        (0, 0),
601                        (3, 0),
602                        (5, 0),
603                        (9, 0),
604                        (7, 0),
605                        (4, 0),
606                        (10, 0)
607                    ]),
608                    nan_value_counts: HashMap::new(),
609                    lower_bounds: HashMap::new(),
610                    upper_bounds: HashMap::new(),
611                    key_metadata: None,
612                    split_offsets: Some(vec![4]),
613                    equality_ids: Some(Vec::new()),
614                    sort_order_id: None,
615                    partition_spec_id: 0,
616                    first_row_id: None,
617                    referenced_data_file: None,
618                    content_offset: None,
619                    content_size_in_bytes: None,
620                },
621            }];
622
623        // write manifest to file and check the return manifest file.
624        let tmp_dir = TempDir::new().unwrap();
625        let path = tmp_dir.path().join("test_manifest.avro");
626        let io = FileIO::new_with_fs();
627        let output_file = io.new_output(path.to_str().unwrap()).unwrap();
628        let mut writer = ManifestWriterBuilder::new(
629            output_file,
630            Some(2),
631            metadata.schema.clone(),
632            metadata.partition_spec.clone(),
633        )
634        .build_v2_data();
635        for entry in &entries {
636            writer.add_entry(entry.clone()).unwrap();
637        }
638        let manifest_file = writer.write_manifest_file().await.unwrap();
639        assert_eq!(manifest_file.sequence_number, UNASSIGNED_SEQUENCE_NUMBER);
640        assert_eq!(
641            manifest_file.min_sequence_number,
642            UNASSIGNED_SEQUENCE_NUMBER
643        );
644
645        // read back the manifest file and check the content
646        let actual_manifest =
647            Manifest::parse_avro(fs::read(path).expect("read_file must succeed").as_slice())
648                .unwrap();
649        // The snapshot id is assigned when the entry is added to the manifest.
650        entries[0].snapshot_id = Some(2);
651        assert_eq!(actual_manifest, Manifest::new(metadata, entries));
652    }
653
654    #[tokio::test]
655    async fn test_parse_manifest_v1_unpartition() {
656        let schema = Arc::new(
657            Schema::builder()
658                .with_schema_id(1)
659                .with_fields(vec![
660                    Arc::new(NestedField::optional(
661                        1,
662                        "id",
663                        Type::Primitive(PrimitiveType::Int),
664                    )),
665                    Arc::new(NestedField::optional(
666                        2,
667                        "data",
668                        Type::Primitive(PrimitiveType::String),
669                    )),
670                    Arc::new(NestedField::optional(
671                        3,
672                        "comment",
673                        Type::Primitive(PrimitiveType::String),
674                    )),
675                ])
676                .build()
677                .unwrap(),
678        );
679        let metadata = ManifestMetadata {
680            schema_id: 1,
681            schema: schema.clone(),
682            partition_spec: PartitionSpec::builder(schema)
683                .with_spec_id(0)
684                .build()
685                .unwrap(),
686            content: ManifestContentType::Data,
687            format_version: FormatVersion::V1,
688        };
689        let mut entries = vec![ManifestEntry {
690                status: ManifestStatus::Added,
691                snapshot_id: Some(0),
692                sequence_number: Some(0),
693                file_sequence_number: Some(0),
694                data_file: DataFile {
695                    content: DataContentType::Data,
696                    file_path: "s3://testbucket/iceberg_data/iceberg_ctl/iceberg_db/iceberg_tbl/data/00000-7-45268d71-54eb-476c-b42c-942d880c04a1-00001.parquet".to_string(),
697                    file_format: DataFileFormat::Parquet,
698                    partition: Struct::empty(),
699                    record_count: 1,
700                    file_size_in_bytes: 875,
701                    column_sizes: HashMap::from([(1,47),(2,48),(3,52)]),
702                    value_counts: HashMap::from([(1,1),(2,1),(3,1)]),
703                    null_value_counts: HashMap::from([(1,0),(2,0),(3,0)]),
704                    nan_value_counts: HashMap::new(),
705                    lower_bounds: HashMap::from([(1,Datum::int(1)),(2,Datum::string("a")),(3,Datum::string("AC/DC"))]),
706                    upper_bounds: HashMap::from([(1,Datum::int(1)),(2,Datum::string("a")),(3,Datum::string("AC/DC"))]),
707                    key_metadata: None,
708                    split_offsets: Some(vec![4]),
709                    equality_ids: None,
710                    sort_order_id: Some(0),
711                    partition_spec_id: 0,
712                    first_row_id: None,
713                    referenced_data_file: None,
714                    content_offset: None,
715                    content_size_in_bytes: None,
716                }
717            }];
718
719        // write manifest to file
720        let tmp_dir = TempDir::new().unwrap();
721        let path = tmp_dir.path().join("test_manifest.avro");
722        let io = FileIO::new_with_fs();
723        let output_file = io.new_output(path.to_str().unwrap()).unwrap();
724        let mut writer = ManifestWriterBuilder::new(
725            output_file,
726            Some(3),
727            metadata.schema.clone(),
728            metadata.partition_spec.clone(),
729        )
730        .build_v1();
731        for entry in &entries {
732            writer.add_entry(entry.clone()).unwrap();
733        }
734        writer.write_manifest_file().await.unwrap();
735
736        // read back the manifest file and check the content
737        let actual_manifest =
738            Manifest::parse_avro(fs::read(path).expect("read_file must succeed").as_slice())
739                .unwrap();
740        // The snapshot id is assigned when the entry is added to the manifest.
741        entries[0].snapshot_id = Some(3);
742        assert_eq!(actual_manifest, Manifest::new(metadata, entries));
743    }
744
745    #[tokio::test]
746    async fn test_parse_manifest_v1_partition() {
747        let schema = Arc::new(
748            Schema::builder()
749                .with_fields(vec![
750                    Arc::new(NestedField::optional(
751                        1,
752                        "id",
753                        Type::Primitive(PrimitiveType::Long),
754                    )),
755                    Arc::new(NestedField::optional(
756                        2,
757                        "data",
758                        Type::Primitive(PrimitiveType::String),
759                    )),
760                    Arc::new(NestedField::optional(
761                        3,
762                        "category",
763                        Type::Primitive(PrimitiveType::String),
764                    )),
765                ])
766                .build()
767                .unwrap(),
768        );
769        let metadata = ManifestMetadata {
770            schema_id: 0,
771            schema: schema.clone(),
772            partition_spec: PartitionSpec::builder(schema)
773                .add_partition_field("category", "category", Transform::Identity)
774                .unwrap()
775                .build()
776                .unwrap(),
777            content: ManifestContentType::Data,
778            format_version: FormatVersion::V1,
779        };
780        let mut entries = vec![
781                ManifestEntry {
782                    status: ManifestStatus::Added,
783                    snapshot_id: Some(0),
784                    sequence_number: Some(0),
785                    file_sequence_number: Some(0),
786                    data_file: DataFile {
787                        content: DataContentType::Data,
788                        file_path: "s3://testbucket/prod/db/sample/data/category=x/00010-1-d5c93668-1e52-41ac-92a6-bba590cbf249-00001.parquet".to_string(),
789                        file_format: DataFileFormat::Parquet,
790                        partition: Struct::from_iter(
791                            vec![
792                                Some(
793                                    Literal::string("x"),
794                                ),
795                            ]
796                                .into_iter()
797                        ),
798                        record_count: 1,
799                        file_size_in_bytes: 874,
800                        column_sizes: HashMap::from([(1, 46), (2, 48), (3, 48)]),
801                        value_counts: HashMap::from([(1, 1), (2, 1), (3, 1)]),
802                        null_value_counts: HashMap::from([(1, 0), (2, 0), (3, 0)]),
803                        nan_value_counts: HashMap::new(),
804                        lower_bounds: HashMap::from([
805                        (1, Datum::long(1)),
806                        (2, Datum::string("a")),
807                        (3, Datum::string("x"))
808                        ]),
809                        upper_bounds: HashMap::from([
810                        (1, Datum::long(1)),
811                        (2, Datum::string("a")),
812                        (3, Datum::string("x"))
813                        ]),
814                        key_metadata: None,
815                        split_offsets: Some(vec![4]),
816                        equality_ids: None,
817                        sort_order_id: Some(0),
818                        partition_spec_id: 0,
819                        first_row_id: None,
820                        referenced_data_file: None,
821                        content_offset: None,
822                        content_size_in_bytes: None,
823                    },
824                }
825            ];
826
827        // write manifest to file
828        let tmp_dir = TempDir::new().unwrap();
829        let path = tmp_dir.path().join("test_manifest.avro");
830        let io = FileIO::new_with_fs();
831        let output_file = io.new_output(path.to_str().unwrap()).unwrap();
832        let mut writer = ManifestWriterBuilder::new(
833            output_file,
834            Some(2),
835            metadata.schema.clone(),
836            metadata.partition_spec.clone(),
837        )
838        .build_v1();
839        for entry in &entries {
840            writer.add_entry(entry.clone()).unwrap();
841        }
842        let manifest_file = writer.write_manifest_file().await.unwrap();
843        let partitions = manifest_file.partitions.unwrap();
844        assert_eq!(partitions.len(), 1);
845        assert_eq!(
846            partitions[0].clone().lower_bound.unwrap(),
847            Datum::string("x").to_bytes().unwrap()
848        );
849        assert_eq!(
850            partitions[0].clone().upper_bound.unwrap(),
851            Datum::string("x").to_bytes().unwrap()
852        );
853
854        // read back the manifest file and check the content
855        let actual_manifest =
856            Manifest::parse_avro(fs::read(path).expect("read_file must succeed").as_slice())
857                .unwrap();
858        // The snapshot id is assigned when the entry is added to the manifest.
859        entries[0].snapshot_id = Some(2);
860        assert_eq!(actual_manifest, Manifest::new(metadata, entries));
861    }
862
863    #[tokio::test]
864    async fn test_parse_manifest_with_schema_evolution() {
865        let schema = Arc::new(
866            Schema::builder()
867                .with_fields(vec![
868                    Arc::new(NestedField::optional(
869                        1,
870                        "id",
871                        Type::Primitive(PrimitiveType::Long),
872                    )),
873                    Arc::new(NestedField::optional(
874                        2,
875                        "v_int",
876                        Type::Primitive(PrimitiveType::Int),
877                    )),
878                ])
879                .build()
880                .unwrap(),
881        );
882        let metadata = ManifestMetadata {
883            schema_id: 0,
884            schema: schema.clone(),
885            partition_spec: PartitionSpec::builder(schema)
886                .with_spec_id(0)
887                .build()
888                .unwrap(),
889            content: ManifestContentType::Data,
890            format_version: FormatVersion::V2,
891        };
892        let entries = vec![ManifestEntry {
893                status: ManifestStatus::Added,
894                snapshot_id: None,
895                sequence_number: None,
896                file_sequence_number: None,
897                data_file: DataFile {
898                    content: DataContentType::Data,
899                    file_format: DataFileFormat::Parquet,
900                    file_path: "s3a://icebergdata/demo/s1/t1/data/00000-0-378b56f5-5c52-4102-a2c2-f05f8a7cbe4a-00000.parquet".to_string(),
901                    partition: Struct::empty(),
902                    record_count: 1,
903                    file_size_in_bytes: 5442,
904                    column_sizes: HashMap::from([
905                        (1, 61),
906                        (2, 73),
907                        (3, 61),
908                    ]),
909                    value_counts: HashMap::default(),
910                    null_value_counts: HashMap::default(),
911                    nan_value_counts: HashMap::new(),
912                    lower_bounds: HashMap::from([
913                        (1, Datum::long(1)),
914                        (2, Datum::int(2)),
915                        (3, Datum::string("x"))
916                    ]),
917                    upper_bounds: HashMap::from([
918                        (1, Datum::long(1)),
919                        (2, Datum::int(2)),
920                        (3, Datum::string("x"))
921                    ]),
922                    key_metadata: None,
923                    split_offsets: Some(vec![4]),
924                    equality_ids: None,
925                    sort_order_id: None,
926                    partition_spec_id: 0,
927                    first_row_id: None,
928                    referenced_data_file: None,
929                    content_offset: None,
930                    content_size_in_bytes: None,
931                },
932            }];
933
934        // write manifest to file
935        let tmp_dir = TempDir::new().unwrap();
936        let path = tmp_dir.path().join("test_manifest.avro");
937        let io = FileIO::new_with_fs();
938        let output_file = io.new_output(path.to_str().unwrap()).unwrap();
939        let mut writer = ManifestWriterBuilder::new(
940            output_file,
941            Some(2),
942            metadata.schema.clone(),
943            metadata.partition_spec.clone(),
944        )
945        .build_v2_data();
946        for entry in &entries {
947            writer.add_entry(entry.clone()).unwrap();
948        }
949        writer.write_manifest_file().await.unwrap();
950
951        // read back the manifest file and check the content
952        let actual_manifest =
953            Manifest::parse_avro(fs::read(path).expect("read_file must succeed").as_slice())
954                .unwrap();
955
956        // Compared with original manifest, the lower_bounds and upper_bounds no longer has data for field 3, and
957        // other parts should be same.
958        // The snapshot id is assigned when the entry is added to the manifest.
959        let schema = Arc::new(
960            Schema::builder()
961                .with_fields(vec![
962                    Arc::new(NestedField::optional(
963                        1,
964                        "id",
965                        Type::Primitive(PrimitiveType::Long),
966                    )),
967                    Arc::new(NestedField::optional(
968                        2,
969                        "v_int",
970                        Type::Primitive(PrimitiveType::Int),
971                    )),
972                ])
973                .build()
974                .unwrap(),
975        );
976        let expected_manifest = Manifest {
977            metadata: ManifestMetadata {
978                schema_id: 0,
979                schema: schema.clone(),
980                partition_spec: PartitionSpec::builder(schema).with_spec_id(0).build().unwrap(),
981                content: ManifestContentType::Data,
982                format_version: FormatVersion::V2,
983            },
984            entries: vec![Arc::new(ManifestEntry {
985                status: ManifestStatus::Added,
986                snapshot_id: Some(2),
987                sequence_number: None,
988                file_sequence_number: None,
989                data_file: DataFile {
990                    content: DataContentType::Data,
991                    file_format: DataFileFormat::Parquet,
992                    file_path: "s3a://icebergdata/demo/s1/t1/data/00000-0-378b56f5-5c52-4102-a2c2-f05f8a7cbe4a-00000.parquet".to_string(),
993                    partition: Struct::empty(),
994                    record_count: 1,
995                    file_size_in_bytes: 5442,
996                    column_sizes: HashMap::from([
997                        (1, 61),
998                        (2, 73),
999                        (3, 61),
1000                    ]),
1001                    value_counts: HashMap::default(),
1002                    null_value_counts: HashMap::default(),
1003                    nan_value_counts: HashMap::new(),
1004                    lower_bounds: HashMap::from([
1005                        (1, Datum::long(1)),
1006                        (2, Datum::int(2)),
1007                    ]),
1008                    upper_bounds: HashMap::from([
1009                        (1, Datum::long(1)),
1010                        (2, Datum::int(2)),
1011                    ]),
1012                    key_metadata: None,
1013                    split_offsets: Some(vec![4]),
1014                    equality_ids: None,
1015                    sort_order_id: None,
1016                    partition_spec_id: 0,
1017                    first_row_id: None,
1018                    referenced_data_file: None,
1019                    content_offset: None,
1020                    content_size_in_bytes: None,
1021                },
1022            })],
1023        };
1024
1025        assert_eq!(actual_manifest, expected_manifest);
1026    }
1027
1028    #[tokio::test]
1029    async fn test_manifest_summary() {
1030        let schema = Arc::new(
1031            Schema::builder()
1032                .with_fields(vec![
1033                    Arc::new(NestedField::optional(
1034                        1,
1035                        "time",
1036                        Type::Primitive(PrimitiveType::Date),
1037                    )),
1038                    Arc::new(NestedField::optional(
1039                        2,
1040                        "v_float",
1041                        Type::Primitive(PrimitiveType::Float),
1042                    )),
1043                    Arc::new(NestedField::optional(
1044                        3,
1045                        "v_double",
1046                        Type::Primitive(PrimitiveType::Double),
1047                    )),
1048                ])
1049                .build()
1050                .unwrap(),
1051        );
1052        let partition_spec = PartitionSpec::builder(schema.clone())
1053            .with_spec_id(0)
1054            .add_partition_field("time", "year_of_time", Transform::Year)
1055            .unwrap()
1056            .add_partition_field("v_float", "f", Transform::Identity)
1057            .unwrap()
1058            .add_partition_field("v_double", "d", Transform::Identity)
1059            .unwrap()
1060            .build()
1061            .unwrap();
1062        let metadata = ManifestMetadata {
1063            schema_id: 0,
1064            schema,
1065            partition_spec,
1066            content: ManifestContentType::Data,
1067            format_version: FormatVersion::V2,
1068        };
1069        let entries = vec![
1070            ManifestEntry {
1071                status: ManifestStatus::Added,
1072                snapshot_id: None,
1073                sequence_number: None,
1074                file_sequence_number: None,
1075                data_file: DataFile {
1076                    content: DataContentType::Data,
1077                    file_path: "s3a://icebergdata/demo/s1/t1/data/00000-0-ba56fbfa-f2ff-40c9-bb27-565ad6dc2be8-00000.parquet".to_string(),
1078                    file_format: DataFileFormat::Parquet,
1079                    partition: Struct::from_iter(
1080                        vec![
1081                            Some(Literal::int(2021)),
1082                            Some(Literal::float(1.0_f32)),
1083                            Some(Literal::double(2.0)),
1084                        ]
1085                    ),
1086                    record_count: 1,
1087                    file_size_in_bytes: 5442,
1088                    column_sizes: HashMap::from([(0,73),(6,34),(2,73),(7,61),(3,61),(5,62),(9,79),(10,73),(1,61),(4,73),(8,73)]),
1089                    value_counts: HashMap::from([(4,1),(5,1),(2,1),(0,1),(3,1),(6,1),(8,1),(1,1),(10,1),(7,1),(9,1)]),
1090                    null_value_counts: HashMap::from([(1,0),(6,0),(2,0),(8,0),(0,0),(3,0),(5,0),(9,0),(7,0),(4,0),(10,0)]),
1091                    nan_value_counts: HashMap::new(),
1092                    lower_bounds: HashMap::new(),
1093                    upper_bounds: HashMap::new(),
1094                    key_metadata: None,
1095                    split_offsets: Some(vec![4]),
1096                    equality_ids: None,
1097                    sort_order_id: None,
1098                    partition_spec_id: 0,
1099                    first_row_id: None,
1100                    referenced_data_file: None,
1101                    content_offset: None,
1102                    content_size_in_bytes: None,
1103                }
1104            },
1105                ManifestEntry {
1106                    status: ManifestStatus::Added,
1107                    snapshot_id: None,
1108                    sequence_number: None,
1109                    file_sequence_number: None,
1110                    data_file: DataFile {
1111                        content: DataContentType::Data,
1112                        file_path: "s3a://icebergdata/demo/s1/t1/data/00000-0-ba56fbfa-f2ff-40c9-bb27-565ad6dc2be8-00000.parquet".to_string(),
1113                        file_format: DataFileFormat::Parquet,
1114                        partition: Struct::from_iter(
1115                            vec![
1116                                Some(Literal::int(1111)),
1117                                Some(Literal::float(15.5_f32)),
1118                                Some(Literal::double(25.5)),
1119                            ]
1120                        ),
1121                        record_count: 1,
1122                        file_size_in_bytes: 5442,
1123                        column_sizes: HashMap::from([(0,73),(6,34),(2,73),(7,61),(3,61),(5,62),(9,79),(10,73),(1,61),(4,73),(8,73)]),
1124                        value_counts: HashMap::from([(4,1),(5,1),(2,1),(0,1),(3,1),(6,1),(8,1),(1,1),(10,1),(7,1),(9,1)]),
1125                        null_value_counts: HashMap::from([(1,0),(6,0),(2,0),(8,0),(0,0),(3,0),(5,0),(9,0),(7,0),(4,0),(10,0)]),
1126                        nan_value_counts: HashMap::new(),
1127                        lower_bounds: HashMap::new(),
1128                        upper_bounds: HashMap::new(),
1129                        key_metadata: None,
1130                        split_offsets: Some(vec![4]),
1131                        equality_ids: None,
1132                        sort_order_id: None,
1133                        partition_spec_id: 0,
1134                        first_row_id: None,
1135                        referenced_data_file: None,
1136                        content_offset: None,
1137                        content_size_in_bytes: None,
1138                    }
1139                },
1140                ManifestEntry {
1141                    status: ManifestStatus::Added,
1142                    snapshot_id: None,
1143                    sequence_number: None,
1144                    file_sequence_number: None,
1145                    data_file: DataFile {
1146                        content: DataContentType::Data,
1147                        file_path: "s3a://icebergdata/demo/s1/t1/data/00000-0-ba56fbfa-f2ff-40c9-bb27-565ad6dc2be8-00000.parquet".to_string(),
1148                        file_format: DataFileFormat::Parquet,
1149                        partition: Struct::from_iter(
1150                            vec![
1151                                Some(Literal::int(1211)),
1152                                Some(Literal::float(f32::NAN)),
1153                                Some(Literal::double(1.0)),
1154                            ]
1155                        ),
1156                        record_count: 1,
1157                        file_size_in_bytes: 5442,
1158                        column_sizes: HashMap::from([(0,73),(6,34),(2,73),(7,61),(3,61),(5,62),(9,79),(10,73),(1,61),(4,73),(8,73)]),
1159                        value_counts: HashMap::from([(4,1),(5,1),(2,1),(0,1),(3,1),(6,1),(8,1),(1,1),(10,1),(7,1),(9,1)]),
1160                        null_value_counts: HashMap::from([(1,0),(6,0),(2,0),(8,0),(0,0),(3,0),(5,0),(9,0),(7,0),(4,0),(10,0)]),
1161                        nan_value_counts: HashMap::new(),
1162                        lower_bounds: HashMap::new(),
1163                        upper_bounds: HashMap::new(),
1164                        key_metadata: None,
1165                        split_offsets: Some(vec![4]),
1166                        equality_ids: None,
1167                        sort_order_id: None,
1168                        partition_spec_id: 0,
1169                        first_row_id: None,
1170                        referenced_data_file: None,
1171                        content_offset: None,
1172                        content_size_in_bytes: None,
1173                    }
1174                },
1175                ManifestEntry {
1176                    status: ManifestStatus::Added,
1177                    snapshot_id: None,
1178                    sequence_number: None,
1179                    file_sequence_number: None,
1180                    data_file: DataFile {
1181                        content: DataContentType::Data,
1182                        file_path: "s3a://icebergdata/demo/s1/t1/data/00000-0-ba56fbfa-f2ff-40c9-bb27-565ad6dc2be8-00000.parquet".to_string(),
1183                        file_format: DataFileFormat::Parquet,
1184                        partition: Struct::from_iter(
1185                            vec![
1186                                Some(Literal::int(1111)),
1187                                None,
1188                                Some(Literal::double(11.0)),
1189                            ]
1190                        ),
1191                        record_count: 1,
1192                        file_size_in_bytes: 5442,
1193                        column_sizes: HashMap::from([(0,73),(6,34),(2,73),(7,61),(3,61),(5,62),(9,79),(10,73),(1,61),(4,73),(8,73)]),
1194                        value_counts: HashMap::from([(4,1),(5,1),(2,1),(0,1),(3,1),(6,1),(8,1),(1,1),(10,1),(7,1),(9,1)]),
1195                        null_value_counts: HashMap::from([(1,0),(6,0),(2,0),(8,0),(0,0),(3,0),(5,0),(9,0),(7,0),(4,0),(10,0)]),
1196                        nan_value_counts: HashMap::new(),
1197                        lower_bounds: HashMap::new(),
1198                        upper_bounds: HashMap::new(),
1199                        key_metadata: None,
1200                        split_offsets: Some(vec![4]),
1201                        equality_ids: None,
1202                        sort_order_id: None,
1203                        partition_spec_id: 0,
1204                        first_row_id: None,
1205                        referenced_data_file: None,
1206                        content_offset: None,
1207                        content_size_in_bytes: None,
1208                    }
1209                },
1210        ];
1211
1212        // write manifest to file
1213        let tmp_dir = TempDir::new().unwrap();
1214        let path = tmp_dir.path().join("test_manifest.avro");
1215        let io = FileIO::new_with_fs();
1216        let output_file = io.new_output(path.to_str().unwrap()).unwrap();
1217        let mut writer = ManifestWriterBuilder::new(
1218            output_file,
1219            Some(1),
1220            metadata.schema.clone(),
1221            metadata.partition_spec.clone(),
1222        )
1223        .build_v2_data();
1224        for entry in &entries {
1225            writer.add_entry(entry.clone()).unwrap();
1226        }
1227        let res = writer.write_manifest_file().await.unwrap();
1228
1229        let partitions = res.partitions.unwrap();
1230
1231        assert_eq!(partitions.len(), 3);
1232        assert_eq!(
1233            partitions[0].clone().lower_bound.unwrap(),
1234            Datum::int(1111).to_bytes().unwrap()
1235        );
1236        assert_eq!(
1237            partitions[0].clone().upper_bound.unwrap(),
1238            Datum::int(2021).to_bytes().unwrap()
1239        );
1240        assert!(!partitions[0].clone().contains_null);
1241        assert_eq!(partitions[0].clone().contains_nan, Some(false));
1242
1243        assert_eq!(
1244            partitions[1].clone().lower_bound.unwrap(),
1245            Datum::float(1.0_f32).to_bytes().unwrap()
1246        );
1247        assert_eq!(
1248            partitions[1].clone().upper_bound.unwrap(),
1249            Datum::float(15.5_f32).to_bytes().unwrap()
1250        );
1251        assert!(partitions[1].clone().contains_null);
1252        assert_eq!(partitions[1].clone().contains_nan, Some(true));
1253
1254        assert_eq!(
1255            partitions[2].clone().lower_bound.unwrap(),
1256            Datum::double(1.0).to_bytes().unwrap()
1257        );
1258        assert_eq!(
1259            partitions[2].clone().upper_bound.unwrap(),
1260            Datum::double(25.5).to_bytes().unwrap()
1261        );
1262        assert!(!partitions[2].clone().contains_null);
1263        assert_eq!(partitions[2].clone().contains_nan, Some(false));
1264    }
1265
1266    #[test]
1267    fn test_data_file_serialization() {
1268        // Create a simple schema
1269        let schema = Schema::builder()
1270            .with_schema_id(1)
1271            .with_identifier_field_ids(vec![1])
1272            .with_fields(vec![
1273                NestedField::required(1, "id", Type::Primitive(PrimitiveType::Long)).into(),
1274                NestedField::required(2, "name", Type::Primitive(PrimitiveType::String)).into(),
1275            ])
1276            .build()
1277            .unwrap();
1278
1279        // Create a partition spec
1280        let partition_spec = PartitionSpec::builder(schema.clone())
1281            .with_spec_id(1)
1282            .add_partition_field("id", "id_partition", Transform::Identity)
1283            .unwrap()
1284            .build()
1285            .unwrap();
1286
1287        // Get partition type from the partition spec
1288        let partition_type = partition_spec.partition_type(&schema).unwrap();
1289
1290        // Create a vector of DataFile objects
1291        let data_files = vec![
1292            DataFileBuilder::default()
1293                .content(DataContentType::Data)
1294                .file_format(DataFileFormat::Parquet)
1295                .file_path("path/to/file1.parquet".to_string())
1296                .file_size_in_bytes(1024)
1297                .record_count(100)
1298                .partition_spec_id(1)
1299                .partition(Struct::empty())
1300                .column_sizes(HashMap::from([(1, 512), (2, 1024)]))
1301                .value_counts(HashMap::from([(1, 100), (2, 500)]))
1302                .null_value_counts(HashMap::from([(1, 0), (2, 1)]))
1303                .build()
1304                .unwrap(),
1305            DataFileBuilder::default()
1306                .content(DataContentType::Data)
1307                .file_format(DataFileFormat::Parquet)
1308                .file_path("path/to/file2.parquet".to_string())
1309                .file_size_in_bytes(2048)
1310                .record_count(200)
1311                .partition_spec_id(1)
1312                .partition(Struct::empty())
1313                .column_sizes(HashMap::from([(1, 1024), (2, 2048)]))
1314                .value_counts(HashMap::from([(1, 200), (2, 600)]))
1315                .null_value_counts(HashMap::from([(1, 10), (2, 999)]))
1316                .build()
1317                .unwrap(),
1318        ];
1319
1320        // Serialize the DataFile objects
1321        let serialized_files = data_files
1322            .clone()
1323            .into_iter()
1324            .map(|f| serialize_data_file_to_json(f, &partition_type, FormatVersion::V2).unwrap())
1325            .collect::<Vec<String>>();
1326
1327        // Verify we have the expected serialized files
1328        assert_eq!(serialized_files.len(), 2);
1329        let pretty_json1: Value = serde_json::from_str(serialized_files.first().unwrap()).unwrap();
1330        let pretty_json2: Value = serde_json::from_str(serialized_files.get(1).unwrap()).unwrap();
1331        let expected_serialized_file1 = serde_json::json!({
1332            "content": 0,
1333            "file_path": "path/to/file1.parquet",
1334            "file_format": "PARQUET",
1335            "partition": {},
1336            "record_count": 100,
1337            "file_size_in_bytes": 1024,
1338            "column_sizes": [
1339                { "key": 1, "value": 512 },
1340                { "key": 2, "value": 1024 }
1341            ],
1342            "value_counts": [
1343                { "key": 1, "value": 100 },
1344                { "key": 2, "value": 500 }
1345            ],
1346            "null_value_counts": [
1347                { "key": 1, "value": 0 },
1348                { "key": 2, "value": 1 }
1349            ],
1350            "nan_value_counts": [],
1351            "lower_bounds": [],
1352            "upper_bounds": [],
1353            "key_metadata": null,
1354            "split_offsets": null,
1355            "equality_ids": null,
1356            "sort_order_id": null,
1357            "first_row_id": null,
1358            "referenced_data_file": null,
1359            "content_offset": null,
1360            "content_size_in_bytes": null
1361        });
1362        let expected_serialized_file2 = serde_json::json!({
1363            "content": 0,
1364            "file_path": "path/to/file2.parquet",
1365            "file_format": "PARQUET",
1366            "partition": {},
1367            "record_count": 200,
1368            "file_size_in_bytes": 2048,
1369            "column_sizes": [
1370                { "key": 1, "value": 1024 },
1371                { "key": 2, "value": 2048 }
1372            ],
1373            "value_counts": [
1374                { "key": 1, "value": 200 },
1375                { "key": 2, "value": 600 }
1376            ],
1377            "null_value_counts": [
1378                { "key": 1, "value": 10 },
1379                { "key": 2, "value": 999 }
1380            ],
1381            "nan_value_counts": [],
1382            "lower_bounds": [],
1383            "upper_bounds": [],
1384            "key_metadata": null,
1385            "split_offsets": null,
1386            "equality_ids": null,
1387            "sort_order_id": null,
1388            "first_row_id": null,
1389            "referenced_data_file": null,
1390            "content_offset": null,
1391            "content_size_in_bytes": null
1392        });
1393        assert_eq!(pretty_json1, expected_serialized_file1);
1394        assert_eq!(pretty_json2, expected_serialized_file2);
1395
1396        // Now deserialize the JSON strings back into DataFile objects
1397        let deserialized_files: Vec<DataFile> = serialized_files
1398            .into_iter()
1399            .map(|json| {
1400                deserialize_data_file_from_json(
1401                    &json,
1402                    partition_spec.spec_id(),
1403                    &partition_type,
1404                    &schema,
1405                )
1406                .unwrap()
1407            })
1408            .collect();
1409
1410        // Verify we have the expected number of deserialized files
1411        assert_eq!(deserialized_files.len(), 2);
1412        let deserialized_data_file1 = deserialized_files.first().unwrap();
1413        let deserialized_data_file2 = deserialized_files.get(1).unwrap();
1414        let original_data_file1 = data_files.first().unwrap();
1415        let original_data_file2 = data_files.get(1).unwrap();
1416
1417        assert_eq!(deserialized_data_file1, original_data_file1);
1418        assert_eq!(deserialized_data_file2, original_data_file2);
1419    }
1420
1421    /// Metadata for the writer schema tests: a data manifest of `format_version`
1422    /// whose spec has identity partitions on a long, a string, and a double column.
1423    fn writer_schema_test_metadata(format_version: FormatVersion) -> ManifestMetadata {
1424        let schema = Arc::new(
1425            Schema::builder()
1426                .with_fields(vec![
1427                    NestedField::optional(1, "id", Type::Primitive(PrimitiveType::Long)).into(),
1428                    NestedField::optional(2, "name", Type::Primitive(PrimitiveType::String)).into(),
1429                    NestedField::optional(3, "score", Type::Primitive(PrimitiveType::Double))
1430                        .into(),
1431                ])
1432                .build()
1433                .unwrap(),
1434        );
1435        let partition_spec = PartitionSpec::builder(schema.clone())
1436            .with_spec_id(0)
1437            .add_partition_field("id", "id", Transform::Identity)
1438            .unwrap()
1439            .add_partition_field("name", "name", Transform::Identity)
1440            .unwrap()
1441            .add_partition_field("score", "score", Transform::Identity)
1442            .unwrap()
1443            .build()
1444            .unwrap();
1445        ManifestMetadata {
1446            schema_id: 0,
1447            schema,
1448            partition_spec,
1449            content: ManifestContentType::Data,
1450            format_version,
1451        }
1452    }
1453
1454    /// A V2 manifest entry writer schema with Java's record names and only the
1455    /// required `data_file` fields.
1456    fn v2_writer_schema(partition_fields: Value) -> Value {
1457        serde_json::json!({
1458            "type": "record",
1459            "name": "manifest_entry",
1460            "fields": [
1461                {"name": "status", "type": "int", "field-id": 0},
1462                {"name": "snapshot_id", "type": ["null", "long"], "default": null, "field-id": 1},
1463                {"name": "sequence_number", "type": ["null", "long"], "default": null, "field-id": 3},
1464                {"name": "file_sequence_number", "type": ["null", "long"], "default": null, "field-id": 4},
1465                {"name": "data_file", "field-id": 2, "type": {
1466                    "type": "record",
1467                    "name": "r2",
1468                    "fields": [
1469                        {"name": "content", "type": "int", "field-id": 134},
1470                        {"name": "file_path", "type": "string", "field-id": 100},
1471                        {"name": "file_format", "type": "string", "field-id": 101},
1472                        {"name": "partition", "field-id": 102, "type": {
1473                            "type": "record",
1474                            "name": "r102",
1475                            "fields": partition_fields,
1476                        }},
1477                        {"name": "record_count", "type": "long", "field-id": 103},
1478                        {"name": "file_size_in_bytes", "type": "long", "field-id": 104},
1479                    ],
1480                }},
1481            ],
1482        })
1483    }
1484
1485    /// The field named by `path` in a record schema, descending through nested
1486    /// record types.
1487    fn writer_schema_field<'a>(record: &'a mut Value, path: &[&str]) -> &'a mut Value {
1488        let (name, rest) = path.split_first().unwrap();
1489        let field = record["fields"]
1490            .as_array_mut()
1491            .unwrap()
1492            .iter_mut()
1493            .find(|field| field["name"] == *name)
1494            .unwrap();
1495        if rest.is_empty() {
1496            field
1497        } else {
1498            writer_schema_field(&mut field["type"], rest)
1499        }
1500    }
1501
1502    fn v2_entry(partition: Value) -> Value {
1503        serde_json::json!({
1504            "status": 1,
1505            "snapshot_id": 7,
1506            "sequence_number": null,
1507            "file_sequence_number": null,
1508            "data_file": {
1509                "content": 0,
1510                "file_path": "s3://bucket/table/data/a.parquet",
1511                "file_format": "PARQUET",
1512                "partition": partition,
1513                "record_count": 10,
1514                "file_size_in_bytes": 100,
1515            },
1516        })
1517    }
1518
1519    fn expected_entry(partition: Struct) -> ManifestEntry {
1520        ManifestEntry {
1521            status: ManifestStatus::Added,
1522            snapshot_id: Some(7),
1523            sequence_number: None,
1524            file_sequence_number: None,
1525            data_file: DataFile {
1526                content: DataContentType::Data,
1527                file_path: "s3://bucket/table/data/a.parquet".to_string(),
1528                file_format: DataFileFormat::Parquet,
1529                partition,
1530                record_count: 10,
1531                file_size_in_bytes: 100,
1532                column_sizes: HashMap::new(),
1533                value_counts: HashMap::new(),
1534                null_value_counts: HashMap::new(),
1535                nan_value_counts: HashMap::new(),
1536                lower_bounds: HashMap::new(),
1537                upper_bounds: HashMap::new(),
1538                key_metadata: None,
1539                split_offsets: None,
1540                equality_ids: None,
1541                sort_order_id: None,
1542                partition_spec_id: 0,
1543                first_row_id: None,
1544                referenced_data_file: None,
1545                content_offset: None,
1546                content_size_in_bytes: None,
1547            },
1548        }
1549    }
1550
1551    /// Writes `entries` as a manifest with the given writer schema, encoding each
1552    /// JSON value with the writer schema's types.
1553    fn write_with_writer_schema(
1554        metadata: &ManifestMetadata,
1555        writer_schema: &Value,
1556        entries: Vec<Value>,
1557    ) -> Vec<u8> {
1558        write_avro_values_with_writer_schema(
1559            metadata,
1560            writer_schema,
1561            entries
1562                .into_iter()
1563                .map(|entry| AvroValue::try_from(entry).unwrap())
1564                .collect(),
1565        )
1566    }
1567
1568    /// Like [`write_with_writer_schema`], for entries that JSON can't express.
1569    fn write_avro_values_with_writer_schema(
1570        metadata: &ManifestMetadata,
1571        writer_schema: &Value,
1572        entries: Vec<AvroValue>,
1573    ) -> Vec<u8> {
1574        let avro_schema = apache_avro::Schema::parse(writer_schema).unwrap();
1575        let mut writer = Writer::new(&avro_schema, Vec::new()).unwrap();
1576        for (key, value) in [
1577            ("schema", to_vec(&metadata.schema).unwrap()),
1578            ("schema-id", metadata.schema_id.to_string().into_bytes()),
1579            (
1580                "partition-spec",
1581                to_vec(&metadata.partition_spec.fields()).unwrap(),
1582            ),
1583            (
1584                "partition-spec-id",
1585                metadata.partition_spec.spec_id().to_string().into_bytes(),
1586            ),
1587            (
1588                "format-version",
1589                (metadata.format_version as u8).to_string().into_bytes(),
1590            ),
1591            ("content", metadata.content.to_string().into_bytes()),
1592        ] {
1593            writer.add_user_metadata(key.to_string(), value).unwrap();
1594        }
1595        for entry in entries {
1596            writer
1597                .append_value(entry.resolve(&avro_schema).unwrap())
1598                .unwrap();
1599        }
1600        writer.into_inner().unwrap()
1601    }
1602
1603    /// Removes the field named by `path` from a record schema and from a record
1604    /// value of that schema.
1605    fn remove_writer_field(schema: &mut Value, entry: &mut AvroValue, path: &[&str]) {
1606        let (name, rest) = path.split_first().unwrap();
1607        let AvroValue::Record(values) = entry else {
1608            unreachable!("the entry is a record");
1609        };
1610        let fields = schema["fields"].as_array_mut().unwrap();
1611        if rest.is_empty() {
1612            fields.retain(|field| field["name"] != *name);
1613            values.retain(|(value_name, _)| value_name != name);
1614        } else {
1615            let field = fields.iter_mut().find(|field| field["name"] == *name);
1616            let value = values.iter_mut().find(|(value_name, _)| value_name == name);
1617            remove_writer_field(&mut field.unwrap()["type"], &mut value.unwrap().1, rest);
1618        }
1619    }
1620
1621    type ResetField = fn(&mut ManifestEntry);
1622
1623    /// Removes each field in `optional` and `required` in turn from a manifest
1624    /// that `full_schema` and `full_value` write as `full_entry`. Checks that the
1625    /// manifest reads with an optional field reset and fails naming a required one.
1626    fn assert_reads_without_each_field(
1627        metadata: &ManifestMetadata,
1628        full_schema: &Value,
1629        full_value: &AvroValue,
1630        full_entry: &ManifestEntry,
1631        optional: &[(&[&str], ResetField)],
1632        required: &[&[&str]],
1633    ) {
1634        for (path, reset) in optional {
1635            let (mut schema, mut value) = (full_schema.clone(), full_value.clone());
1636            remove_writer_field(&mut schema, &mut value, path);
1637            let bs = write_avro_values_with_writer_schema(metadata, &schema, vec![value]);
1638
1639            let manifest = Manifest::parse_avro(&bs).unwrap();
1640
1641            let mut expected = full_entry.clone();
1642            reset(&mut expected);
1643            assert_eq!(
1644                manifest,
1645                Manifest::new(metadata.clone(), vec![expected]),
1646                "{path:?}"
1647            );
1648        }
1649
1650        for path in required {
1651            let (mut schema, mut value) = (full_schema.clone(), full_value.clone());
1652            remove_writer_field(&mut schema, &mut value, path);
1653            let bs = write_avro_values_with_writer_schema(metadata, &schema, vec![value]);
1654
1655            let err = Manifest::parse_avro(&bs).unwrap_err();
1656
1657            assert!(
1658                err.to_string().contains(path.last().unwrap()),
1659                "{path:?}: {err}"
1660            );
1661        }
1662    }
1663
1664    #[test]
1665    fn test_parse_manifest_v3_without_each_field() {
1666        // Fields that the spec doesn't require in every version must read as
1667        // their default when the writer omits them.
1668        let metadata = writer_schema_test_metadata(FormatVersion::V3);
1669        let partition_type = metadata
1670            .partition_spec
1671            .partition_type(&metadata.schema)
1672            .unwrap();
1673        let full_schema =
1674            serde_json::to_value(manifest_schema_v3(&partition_type).unwrap()).unwrap();
1675        let mut full_entry = expected_entry(Struct::from_iter([
1676            Some(Literal::long(5)),
1677            Some(Literal::string("a")),
1678            Some(Literal::double(2.5)),
1679        ]));
1680        full_entry.sequence_number = Some(3);
1681        full_entry.file_sequence_number = Some(4);
1682        let data_file = &mut full_entry.data_file;
1683        data_file.content = DataContentType::PositionDeletes;
1684        data_file.column_sizes = HashMap::from([(1, 40)]);
1685        data_file.value_counts = HashMap::from([(1, 10)]);
1686        data_file.null_value_counts = HashMap::from([(1, 1)]);
1687        data_file.nan_value_counts = HashMap::from([(3, 2)]);
1688        data_file.lower_bounds = HashMap::from([(1, Datum::long(1))]);
1689        data_file.upper_bounds = HashMap::from([(1, Datum::long(9))]);
1690        data_file.key_metadata = Some(vec![1, 2]);
1691        data_file.split_offsets = Some(vec![4]);
1692        data_file.equality_ids = Some(vec![1]);
1693        data_file.sort_order_id = Some(0);
1694        data_file.first_row_id = Some(100);
1695        data_file.referenced_data_file = Some("s3://bucket/table/data/b.parquet".to_string());
1696        data_file.content_offset = Some(4);
1697        data_file.content_size_in_bytes = Some(8);
1698        let full_value = to_value(
1699            _serde::ManifestEntryV3::try_from(
1700                full_entry.clone(),
1701                &Type::Struct(partition_type.clone()),
1702            )
1703            .unwrap(),
1704        )
1705        .unwrap();
1706
1707        let optional: [(&[&str], ResetField); 18] = [
1708            (&["snapshot_id"], |e| e.snapshot_id = None),
1709            (&["sequence_number"], |e| e.sequence_number = None),
1710            (&["file_sequence_number"], |e| e.file_sequence_number = None),
1711            (&["data_file", "content"], |e| {
1712                e.data_file.content = DataContentType::Data
1713            }),
1714            (&["data_file", "column_sizes"], |e| {
1715                e.data_file.column_sizes.clear()
1716            }),
1717            (&["data_file", "value_counts"], |e| {
1718                e.data_file.value_counts.clear()
1719            }),
1720            (&["data_file", "null_value_counts"], |e| {
1721                e.data_file.null_value_counts.clear()
1722            }),
1723            (&["data_file", "nan_value_counts"], |e| {
1724                e.data_file.nan_value_counts.clear()
1725            }),
1726            (&["data_file", "lower_bounds"], |e| {
1727                e.data_file.lower_bounds.clear()
1728            }),
1729            (&["data_file", "upper_bounds"], |e| {
1730                e.data_file.upper_bounds.clear()
1731            }),
1732            (&["data_file", "key_metadata"], |e| {
1733                e.data_file.key_metadata = None
1734            }),
1735            (&["data_file", "split_offsets"], |e| {
1736                e.data_file.split_offsets = None
1737            }),
1738            (&["data_file", "equality_ids"], |e| {
1739                e.data_file.equality_ids = None
1740            }),
1741            (&["data_file", "sort_order_id"], |e| {
1742                e.data_file.sort_order_id = None
1743            }),
1744            (&["data_file", "first_row_id"], |e| {
1745                e.data_file.first_row_id = None
1746            }),
1747            (&["data_file", "referenced_data_file"], |e| {
1748                e.data_file.referenced_data_file = None
1749            }),
1750            (&["data_file", "content_offset"], |e| {
1751                e.data_file.content_offset = None
1752            }),
1753            (&["data_file", "content_size_in_bytes"], |e| {
1754                e.data_file.content_size_in_bytes = None
1755            }),
1756        ];
1757        let required: [&[&str]; 7] = [
1758            &["status"],
1759            &["data_file"],
1760            &["data_file", "file_path"],
1761            &["data_file", "file_format"],
1762            &["data_file", "partition"],
1763            &["data_file", "record_count"],
1764            &["data_file", "file_size_in_bytes"],
1765        ];
1766        assert_reads_without_each_field(
1767            &metadata,
1768            &full_schema,
1769            &full_value,
1770            &full_entry,
1771            &optional,
1772            &required,
1773        );
1774    }
1775
1776    #[test]
1777    fn test_parse_v1_manifest_without_each_field() {
1778        let metadata = writer_schema_test_metadata(FormatVersion::V1);
1779        let partition_type = metadata
1780            .partition_spec
1781            .partition_type(&metadata.schema)
1782            .unwrap();
1783        let full_schema =
1784            serde_json::to_value(manifest_schema_v1(&partition_type).unwrap()).unwrap();
1785        let mut full_entry = expected_entry(Struct::from_iter([
1786            Some(Literal::long(5)),
1787            Some(Literal::string("a")),
1788            Some(Literal::double(2.5)),
1789        ]));
1790        // V1 has no sequence numbers, and every V1 entry reads with 0.
1791        full_entry.sequence_number = Some(0);
1792        full_entry.file_sequence_number = Some(0);
1793        let data_file = &mut full_entry.data_file;
1794        data_file.column_sizes = HashMap::from([(1, 40)]);
1795        data_file.value_counts = HashMap::from([(1, 10)]);
1796        data_file.null_value_counts = HashMap::from([(1, 1)]);
1797        data_file.nan_value_counts = HashMap::from([(3, 2)]);
1798        data_file.lower_bounds = HashMap::from([(1, Datum::long(1))]);
1799        data_file.upper_bounds = HashMap::from([(1, Datum::long(9))]);
1800        data_file.key_metadata = Some(vec![1, 2]);
1801        data_file.split_offsets = Some(vec![4]);
1802        data_file.sort_order_id = Some(0);
1803        let full_value = to_value(
1804            _serde::ManifestEntryV1::try_from(
1805                full_entry.clone(),
1806                &Type::Struct(partition_type.clone()),
1807            )
1808            .unwrap(),
1809        )
1810        .unwrap();
1811
1812        let optional: [(&[&str], ResetField); 10] = [
1813            // Required in V1, but deprecated and not read.
1814            (&["data_file", "block_size_in_bytes"], |_| {}),
1815            (&["data_file", "column_sizes"], |e| {
1816                e.data_file.column_sizes.clear()
1817            }),
1818            (&["data_file", "value_counts"], |e| {
1819                e.data_file.value_counts.clear()
1820            }),
1821            (&["data_file", "null_value_counts"], |e| {
1822                e.data_file.null_value_counts.clear()
1823            }),
1824            (&["data_file", "nan_value_counts"], |e| {
1825                e.data_file.nan_value_counts.clear()
1826            }),
1827            (&["data_file", "lower_bounds"], |e| {
1828                e.data_file.lower_bounds.clear()
1829            }),
1830            (&["data_file", "upper_bounds"], |e| {
1831                e.data_file.upper_bounds.clear()
1832            }),
1833            (&["data_file", "key_metadata"], |e| {
1834                e.data_file.key_metadata = None
1835            }),
1836            (&["data_file", "split_offsets"], |e| {
1837                e.data_file.split_offsets = None
1838            }),
1839            (&["data_file", "sort_order_id"], |e| {
1840                e.data_file.sort_order_id = None
1841            }),
1842        ];
1843        let required: [&[&str]; 8] = [
1844            &["status"],
1845            &["snapshot_id"],
1846            &["data_file"],
1847            &["data_file", "file_path"],
1848            &["data_file", "file_format"],
1849            &["data_file", "partition"],
1850            &["data_file", "record_count"],
1851            &["data_file", "file_size_in_bytes"],
1852        ];
1853        assert_reads_without_each_field(
1854            &metadata,
1855            &full_schema,
1856            &full_value,
1857            &full_entry,
1858            &optional,
1859            &required,
1860        );
1861    }
1862
1863    #[test]
1864    fn test_parse_manifest_matches_partition_fields_by_name() {
1865        // The writer orders the partition fields differently from the spec, omits
1866        // `name`, and adds a field the spec doesn't have.
1867        let metadata = writer_schema_test_metadata(FormatVersion::V2);
1868        let writer_schema = v2_writer_schema(serde_json::json!([
1869            {"name": "score", "type": ["null", "double"], "default": null, "field-id": 1002},
1870            {"name": "extra", "type": ["null", "int"], "default": null, "field-id": 1003},
1871            {"name": "id", "type": ["null", "long"], "default": null, "field-id": 1000},
1872        ]));
1873        let bs = write_with_writer_schema(&metadata, &writer_schema, vec![v2_entry(
1874            serde_json::json!({"score": 2.5, "extra": 9, "id": 5}),
1875        )]);
1876
1877        let manifest = Manifest::parse_avro(&bs).unwrap();
1878
1879        let partition =
1880            Struct::from_iter([Some(Literal::long(5)), None, Some(Literal::double(2.5))]);
1881        assert_eq!(
1882            manifest,
1883            Manifest::new(metadata, vec![expected_entry(partition)])
1884        );
1885    }
1886
1887    #[test]
1888    fn test_parse_manifest_promotes_writer_types() {
1889        // Avro promotes int to long and float to double.
1890        let metadata = writer_schema_test_metadata(FormatVersion::V2);
1891        let mut writer_schema = v2_writer_schema(serde_json::json!([
1892            {"name": "id", "type": ["null", "int"], "default": null, "field-id": 1000},
1893            {"name": "name", "type": ["null", "string"], "default": null, "field-id": 1001},
1894            {"name": "score", "type": ["null", "float"], "default": null, "field-id": 1002},
1895        ]));
1896        for field in ["record_count", "file_size_in_bytes"] {
1897            writer_schema_field(&mut writer_schema, &["data_file", field])["type"] =
1898                serde_json::json!("int");
1899        }
1900        let bs = write_with_writer_schema(&metadata, &writer_schema, vec![v2_entry(
1901            serde_json::json!({"id": 5, "name": "a", "score": 2.5}),
1902        )]);
1903
1904        let manifest = Manifest::parse_avro(&bs).unwrap();
1905
1906        let partition = Struct::from_iter([
1907            Some(Literal::long(5)),
1908            Some(Literal::string("a")),
1909            Some(Literal::double(2.5)),
1910        ]);
1911        assert_eq!(
1912            manifest,
1913            Manifest::new(metadata, vec![expected_entry(partition)])
1914        );
1915    }
1916
1917    #[test]
1918    fn test_parse_manifest_reads_required_writer_fields_as_optional() {
1919        // Fields that are optional in the reader schema are written without a
1920        // union.
1921        let metadata = writer_schema_test_metadata(FormatVersion::V2);
1922        let mut writer_schema = v2_writer_schema(serde_json::json!([
1923            {"name": "id", "type": "long", "field-id": 1000},
1924            {"name": "name", "type": "string", "field-id": 1001},
1925            {"name": "score", "type": "double", "field-id": 1002},
1926        ]));
1927        *writer_schema_field(&mut writer_schema, &["snapshot_id"]) =
1928            serde_json::json!({"name": "snapshot_id", "type": "long", "field-id": 1});
1929        writer_schema_field(&mut writer_schema, &["data_file"])["type"]["fields"]
1930            .as_array_mut()
1931            .unwrap()
1932            .push(serde_json::json!({"name": "sort_order_id", "type": "int", "field-id": 140}));
1933        let mut entry = v2_entry(serde_json::json!({"id": 5, "name": "a", "score": 2.5}));
1934        entry["data_file"]["sort_order_id"] = serde_json::json!(3);
1935        let bs = write_with_writer_schema(&metadata, &writer_schema, vec![entry]);
1936
1937        let manifest = Manifest::parse_avro(&bs).unwrap();
1938
1939        let mut expected = expected_entry(Struct::from_iter([
1940            Some(Literal::long(5)),
1941            Some(Literal::string("a")),
1942            Some(Literal::double(2.5)),
1943        ]));
1944        expected.data_file.sort_order_id = Some(3);
1945        assert_eq!(manifest, Manifest::new(metadata, vec![expected]));
1946    }
1947
1948    #[test]
1949    fn test_parse_manifest_reads_union_writer_fields_as_required() {
1950        // A field that is required in the reader schema is written as a union with
1951        // null.
1952        let metadata = writer_schema_test_metadata(FormatVersion::V2);
1953        let mut writer_schema = v2_writer_schema(serde_json::json!([
1954            {"name": "id", "type": ["null", "long"], "default": null, "field-id": 1000},
1955            {"name": "name", "type": ["null", "string"], "default": null, "field-id": 1001},
1956            {"name": "score", "type": ["null", "double"], "default": null, "field-id": 1002},
1957        ]));
1958        writer_schema_field(&mut writer_schema, &["data_file", "record_count"])["type"] =
1959            serde_json::json!(["null", "long"]);
1960        let partition = serde_json::json!({"id": 5, "name": "a", "score": 2.5});
1961        let bs =
1962            write_with_writer_schema(&metadata, &writer_schema, vec![v2_entry(partition.clone())]);
1963
1964        let manifest = Manifest::parse_avro(&bs).unwrap();
1965
1966        let expected = expected_entry(Struct::from_iter([
1967            Some(Literal::long(5)),
1968            Some(Literal::string("a")),
1969            Some(Literal::double(2.5)),
1970        ]));
1971        assert_eq!(manifest, Manifest::new(metadata.clone(), vec![expected]));
1972
1973        let mut entry = v2_entry(partition);
1974        entry["data_file"]["record_count"] = Value::Null;
1975        let bs = write_with_writer_schema(&metadata, &writer_schema, vec![entry]);
1976
1977        let err = Manifest::parse_avro(&bs).unwrap_err();
1978
1979        assert_eq!(err.kind(), ErrorKind::DataInvalid);
1980    }
1981
1982    #[test]
1983    fn test_parse_manifest_with_fixed_uuid_partition() {
1984        // The spec stores a uuid in Avro as a 16-byte fixed with logical type uuid.
1985        let schema = Arc::new(
1986            Schema::builder()
1987                .with_fields(vec![
1988                    NestedField::optional(1, "u", Type::Primitive(PrimitiveType::Uuid)).into(),
1989                ])
1990                .build()
1991                .unwrap(),
1992        );
1993        let partition_spec = PartitionSpec::builder(schema.clone())
1994            .with_spec_id(0)
1995            .add_partition_field("u", "u", Transform::Identity)
1996            .unwrap()
1997            .build()
1998            .unwrap();
1999        let metadata = ManifestMetadata {
2000            schema_id: 0,
2001            schema,
2002            partition_spec,
2003            content: ManifestContentType::Data,
2004            format_version: FormatVersion::V2,
2005        };
2006        let writer_schema = v2_writer_schema(serde_json::json!([
2007            {"name": "u", "default": null, "field-id": 1000, "type": ["null", {
2008                "type": "fixed", "name": "uuid_fixed", "size": 16, "logicalType": "uuid",
2009            }]},
2010        ]));
2011        let uuid = uuid::Uuid::from_u128(0xf79c3e09_677c_4bbd_a479_3f349cb785e7);
2012        // `Value::resolve` doesn't convert a JSON string to a fixed uuid, so the
2013        // partition value is set as an Avro value.
2014        let mut entry = AvroValue::try_from(v2_entry(serde_json::json!({}))).unwrap();
2015        let AvroValue::Map(fields) = &mut entry else {
2016            unreachable!("a JSON object converts to an Avro map");
2017        };
2018        let Some(AvroValue::Map(data_file)) = fields.get_mut("data_file") else {
2019            unreachable!("a JSON object converts to an Avro map");
2020        };
2021        data_file.insert(
2022            "partition".to_string(),
2023            AvroValue::Map(HashMap::from([("u".to_string(), AvroValue::Uuid(uuid))])),
2024        );
2025        let bs = write_avro_values_with_writer_schema(&metadata, &writer_schema, vec![entry]);
2026
2027        let manifest = Manifest::parse_avro(&bs).unwrap();
2028
2029        let expected = expected_entry(Struct::from_iter([Some(Literal::uuid(uuid))]));
2030        assert_eq!(manifest, Manifest::new(metadata, vec![expected]));
2031    }
2032
2033    #[test]
2034    fn test_parse_manifest_ignores_record_names_and_unknown_fields() {
2035        // Record names differ from the ones Java writes, and the writer has fields
2036        // the reader schema doesn't.
2037        let metadata = writer_schema_test_metadata(FormatVersion::V2);
2038        let mut writer_schema = v2_writer_schema(serde_json::json!([
2039            {"name": "id", "type": ["null", "long"], "default": null, "field-id": 1000},
2040            {"name": "name", "type": ["null", "string"], "default": null, "field-id": 1001},
2041            {"name": "score", "type": ["null", "double"], "default": null, "field-id": 1002},
2042        ]));
2043        writer_schema["name"] = serde_json::json!("entry");
2044        writer_schema["fields"]
2045            .as_array_mut()
2046            .unwrap()
2047            .push(serde_json::json!({"name": "unknown", "type": "string"}));
2048        let data_file = &mut writer_schema_field(&mut writer_schema, &["data_file"])["type"];
2049        data_file["name"] = serde_json::json!("file");
2050        data_file["fields"][3]["type"]["name"] = serde_json::json!("part");
2051        let data_file_fields = data_file["fields"].as_array_mut().unwrap();
2052        data_file_fields.push(serde_json::json!({
2053            "name": "column_sizes",
2054            "field-id": 108,
2055            "default": null,
2056            "type": ["null", {
2057                "type": "array",
2058                "logicalType": "map",
2059                "items": {
2060                    "type": "record",
2061                    "name": "sizes",
2062                    "fields": [
2063                        {"name": "key", "type": "int", "field-id": 117},
2064                        {"name": "value", "type": "long", "field-id": 118},
2065                    ],
2066                },
2067            }],
2068        }));
2069        data_file_fields.push(
2070            serde_json::json!({"name": "block_size_in_bytes", "type": "long", "field-id": 105}),
2071        );
2072        let mut entry = v2_entry(serde_json::json!({"id": 5, "name": "a", "score": 2.5}));
2073        entry["unknown"] = serde_json::json!("ignored");
2074        entry["data_file"]["column_sizes"] = serde_json::json!([{"key": 1, "value": 40}]);
2075        entry["data_file"]["block_size_in_bytes"] = serde_json::json!(64);
2076        let bs = write_with_writer_schema(&metadata, &writer_schema, vec![entry]);
2077
2078        let manifest = Manifest::parse_avro(&bs).unwrap();
2079
2080        let mut expected = expected_entry(Struct::from_iter([
2081            Some(Literal::long(5)),
2082            Some(Literal::string("a")),
2083            Some(Literal::double(2.5)),
2084        ]));
2085        expected.data_file.column_sizes = HashMap::from([(1, 40)]);
2086        assert_eq!(manifest, Manifest::new(metadata, vec![expected]));
2087    }
2088
2089    #[test]
2090    fn test_parse_manifest_equality_ids_written_as_long() {
2091        // PyIceberg wrote `equality_ids` as `array<long>` before
2092        // apache/iceberg-python#3842.
2093        let metadata = writer_schema_test_metadata(FormatVersion::V2);
2094        let mut writer_schema = v2_writer_schema(serde_json::json!([
2095            {"name": "id", "type": ["null", "long"], "default": null, "field-id": 1000},
2096            {"name": "name", "type": ["null", "string"], "default": null, "field-id": 1001},
2097            {"name": "score", "type": ["null", "double"], "default": null, "field-id": 1002},
2098        ]));
2099        writer_schema_field(&mut writer_schema, &["data_file"])["type"]["fields"]
2100            .as_array_mut()
2101            .unwrap()
2102            .push(serde_json::json!({
2103                "name": "equality_ids",
2104                "field-id": 135,
2105                "default": null,
2106                "type": ["null", {"type": "array", "items": "long", "element-id": 136}],
2107            }));
2108        let mut entry = v2_entry(serde_json::json!({"id": 5, "name": "a", "score": 2.5}));
2109        entry["data_file"]["equality_ids"] = serde_json::json!([1, 2]);
2110        let bs = write_with_writer_schema(&metadata, &writer_schema, vec![entry]);
2111
2112        let manifest = Manifest::parse_avro(&bs).unwrap();
2113
2114        let mut expected = expected_entry(Struct::from_iter([
2115            Some(Literal::long(5)),
2116            Some(Literal::string("a")),
2117            Some(Literal::double(2.5)),
2118        ]));
2119        expected.data_file.equality_ids = Some(vec![1, 2]);
2120        assert_eq!(manifest, Manifest::new(metadata, vec![expected]));
2121    }
2122
2123    #[test]
2124    fn test_parse_manifest_rejects_equality_id_above_int_range() {
2125        let metadata = writer_schema_test_metadata(FormatVersion::V2);
2126        let mut writer_schema = v2_writer_schema(serde_json::json!([
2127            {"name": "id", "type": ["null", "long"], "default": null, "field-id": 1000},
2128            {"name": "name", "type": ["null", "string"], "default": null, "field-id": 1001},
2129            {"name": "score", "type": ["null", "double"], "default": null, "field-id": 1002},
2130        ]));
2131        writer_schema_field(&mut writer_schema, &["data_file"])["type"]["fields"]
2132            .as_array_mut()
2133            .unwrap()
2134            .push(serde_json::json!({
2135                "name": "equality_ids",
2136                "field-id": 135,
2137                "default": null,
2138                "type": ["null", {"type": "array", "items": "long", "element-id": 136}],
2139            }));
2140        let mut entry = v2_entry(serde_json::json!({"id": 5, "name": "a", "score": 2.5}));
2141        entry["data_file"]["equality_ids"] = serde_json::json!([i64::from(i32::MAX) + 1]);
2142        let bs = write_with_writer_schema(&metadata, &writer_schema, vec![entry]);
2143
2144        let err = Manifest::parse_avro(&bs).unwrap_err();
2145
2146        assert_eq!(err.kind(), ErrorKind::DataInvalid);
2147    }
2148
2149    #[test]
2150    fn test_parse_manifest_written_by_pyiceberg() {
2151        let bs = fs::read(format!(
2152            "{}/testdata/manifests/pyiceberg-v2-data.avro",
2153            env!("CARGO_MANIFEST_DIR")
2154        ))
2155        .unwrap();
2156
2157        let manifest = Manifest::parse_avro(&bs).unwrap();
2158
2159        let schema = Arc::new(
2160            Schema::builder()
2161                .with_fields(vec![
2162                    NestedField::optional(1, "id", Type::Primitive(PrimitiveType::Long)).into(),
2163                    NestedField::optional(2, "category", Type::Primitive(PrimitiveType::String))
2164                        .into(),
2165                    NestedField::optional(3, "score", Type::Primitive(PrimitiveType::Double))
2166                        .into(),
2167                ])
2168                .build()
2169                .unwrap(),
2170        );
2171        let partition_spec = PartitionSpec::builder(schema.clone())
2172            .with_spec_id(0)
2173            .add_partition_field("category", "category", Transform::Identity)
2174            .unwrap()
2175            .build()
2176            .unwrap();
2177        let metadata = ManifestMetadata {
2178            schema_id: 0,
2179            schema,
2180            partition_spec,
2181            content: ManifestContentType::Data,
2182            format_version: FormatVersion::V2,
2183        };
2184        let entries = (0..2)
2185            .map(|i: i64| {
2186                let category = format!("c{i}");
2187                ManifestEntry {
2188                    status: ManifestStatus::Added,
2189                    snapshot_id: Some(7),
2190                    sequence_number: Some(1),
2191                    file_sequence_number: Some(1),
2192                    data_file: DataFile {
2193                        content: DataContentType::Data,
2194                        file_path: format!(
2195                            "s3://bucket/table/data/category={category}/0000{i}.parquet"
2196                        ),
2197                        file_format: DataFileFormat::Parquet,
2198                        partition: Struct::from_iter([Some(Literal::string(&category))]),
2199                        record_count: 100 + i as u64,
2200                        file_size_in_bytes: 1000 + i as u64,
2201                        column_sizes: HashMap::from([
2202                            (1, 10 + i as u64),
2203                            (2, 20 + i as u64),
2204                            (3, 30 + i as u64),
2205                        ]),
2206                        value_counts: HashMap::from([
2207                            (1, 100 + i as u64),
2208                            (2, 100 + i as u64),
2209                            (3, 100 + i as u64),
2210                        ]),
2211                        null_value_counts: HashMap::from([(1, 0), (2, i as u64), (3, 1)]),
2212                        nan_value_counts: HashMap::from([(3, i as u64)]),
2213                        lower_bounds: HashMap::from([
2214                            (1, Datum::long(i)),
2215                            (2, Datum::string(&category)),
2216                        ]),
2217                        upper_bounds: HashMap::from([
2218                            (1, Datum::long(i + 50)),
2219                            (2, Datum::string(&category)),
2220                        ]),
2221                        key_metadata: None,
2222                        split_offsets: Some(vec![4]),
2223                        equality_ids: None,
2224                        sort_order_id: Some(0),
2225                        partition_spec_id: 0,
2226                        first_row_id: None,
2227                        referenced_data_file: None,
2228                        content_offset: None,
2229                        content_size_in_bytes: None,
2230                    },
2231                }
2232            })
2233            .collect();
2234        assert_eq!(manifest, Manifest::new(metadata, entries));
2235    }
2236
2237    #[test]
2238    fn test_parse_manifest_with_repeated_named_type_definitions() {
2239        let bs = fs::read(format!(
2240            "{}/testdata/manifests/repeated-decimal-type-definitions.avro",
2241            env!("CARGO_MANIFEST_DIR")
2242        ))
2243        .unwrap();
2244
2245        let manifest = Manifest::parse_avro(&bs).unwrap();
2246
2247        assert_eq!(
2248            *manifest.entries()[0].data_file().partition(),
2249            Struct::from_iter([Some(Literal::decimal(12345)), Some(Literal::decimal(-678))])
2250        );
2251    }
2252
2253    #[test]
2254    fn test_parse_manifest_with_repeated_named_type_definitions_reports_original_error() {
2255        let bs = fs::read(format!(
2256            "{}/testdata/manifests/repeated-decimal-type-definitions.avro",
2257            env!("CARGO_MANIFEST_DIR")
2258        ))
2259        .unwrap();
2260        // Cut the header's sync marker in half, so the rewritten header parses
2261        // its schema but fails to read the marker. The file ends with the marker.
2262        let marker = &bs[bs.len() - 16..];
2263        let header_marker = bs.windows(16).position(|w| w == marker).unwrap();
2264        let truncated = &bs[..header_marker + 8];
2265        let log_dir = TempDir::new().unwrap();
2266        let log_path = log_dir.path().join("log");
2267        let subscriber = tracing_subscriber::fmt()
2268            .with_max_level(Level::DEBUG)
2269            .with_writer(Arc::new(fs::File::create(&log_path).unwrap()))
2270            .finish();
2271
2272        let err = with_default(subscriber, || Manifest::parse_avro(truncated)).unwrap_err();
2273
2274        assert_eq!(err.kind(), ErrorKind::DataInvalid);
2275        let message = err.to_string();
2276        assert!(
2277            message.contains("Two named schema defined for same fullname"),
2278            "{message}"
2279        );
2280        assert!(message.contains("Failed to read marker bytes"), "{message}");
2281        // The fallback logs only manifests that it reads.
2282        let log = fs::read_to_string(&log_path).unwrap();
2283        assert!(!log.contains("more than once"), "{log}");
2284    }
2285}