1mod _serde;
19
20mod data_file;
21pub use data_file::*;
22mod entry;
23pub use entry::*;
24mod metadata;
25pub use metadata::*;
26mod reader;
27pub use reader::*;
28mod writer;
29use std::sync::Arc;
30use std::sync::atomic::{AtomicBool, Ordering};
31
32use apache_avro::Reader as AvroReader;
33use apache_avro::error::Details;
34pub use writer::*;
35
36use super::{
37 Datum, FormatVersion, ManifestContentType, PartitionSpec, PrimitiveType, Schema, Struct, Type,
38 UNASSIGNED_SEQUENCE_NUMBER,
39};
40use crate::avro::{Resolved, define_named_types_once};
41use crate::error::{Error, Result, invalid_data};
42
43static WARNED_REPEATED_DEFINITIONS: AtomicBool = AtomicBool::new(false);
46
47#[derive(Debug, PartialEq, Eq, Clone)]
49pub struct Manifest {
50 metadata: ManifestMetadata,
51 entries: Vec<ManifestEntryRef>,
52}
53
54impl Manifest {
55 pub(crate) fn try_from_avro_bytes(
58 bs: &[u8],
59 location: Option<&str>,
60 ) -> Result<(ManifestMetadata, Vec<ManifestEntry>)> {
61 let rewritten;
62 let reader = match AvroReader::new(bs) {
63 Ok(reader) => reader,
64 Err(e) if matches!(e.details(), Details::AmbiguousSchemaDefinition(_)) => {
68 let (bs, repeated) = match define_named_types_once(bs) {
69 Ok(Some(rewrite)) => rewrite,
70 Ok(None) => return Err(e.into()),
71 Err(rewrite_error) => {
72 return Err(Error::from(e).with_context(
73 "error defining each named type once",
74 rewrite_error.to_string(),
75 ));
76 }
77 };
78 rewritten = bs;
79 let reader = AvroReader::new(rewritten.as_slice()).map_err(|retry_error| {
80 Error::from(e).with_context(
81 "error after defining each named type once",
82 retry_error.to_string(),
83 )
84 })?;
85 let location = location.unwrap_or("<unknown location>");
86 if WARNED_REPEATED_DEFINITIONS.swap(true, Ordering::Relaxed) {
87 tracing::debug!(
88 "Manifest {location} defines Avro named types {repeated:?} more than once."
89 );
90 } else {
91 tracing::warn!(
92 "Manifest {location} defines Avro named types {repeated:?} more than once, \
93 which the Avro specification doesn't allow. Reading it with each repeated \
94 definition replaced by a reference to the first. Later manifests like \
95 this are logged at debug level."
96 );
97 }
98 reader
99 }
100 Err(e) => return Err(e.into()),
101 };
102
103 let meta = reader.user_metadata();
105 let metadata = ManifestMetadata::parse(meta)?;
106
107 let partition_type = metadata.partition_spec.partition_type(&metadata.schema)?;
109 let partition_struct_type = Type::Struct(partition_type.clone());
113
114 let entries = match metadata.format_version {
115 FormatVersion::V1 => reader
116 .into_deser_iter::<Resolved<_serde::ManifestEntryV1>>()
117 .map(|entry| {
118 entry?.0.try_into(
119 metadata.partition_spec.spec_id(),
120 &partition_struct_type,
121 &metadata.schema,
122 )
123 })
124 .collect::<Result<Vec<_>>>()?,
125 FormatVersion::V2 => reader
126 .into_deser_iter::<Resolved<_serde::ManifestEntryV2>>()
127 .map(|entry| {
128 entry?.0.try_into(
129 metadata.partition_spec.spec_id(),
130 &partition_struct_type,
131 &metadata.schema,
132 )
133 })
134 .collect::<Result<Vec<_>>>()?,
135 FormatVersion::V3 => reader
136 .into_deser_iter::<Resolved<_serde::ManifestEntryV3>>()
137 .map(|entry| {
138 entry?.0.try_into(
139 metadata.partition_spec.spec_id(),
140 &partition_struct_type,
141 &metadata.schema,
142 )
143 })
144 .collect::<Result<Vec<_>>>()?,
145 };
146
147 Ok((metadata, entries))
148 }
149
150 pub fn parse_avro(bs: &[u8]) -> Result<Self> {
152 let (metadata, entries) = Self::try_from_avro_bytes(bs, None)?;
153 Ok(Self::new(metadata, entries))
154 }
155
156 pub fn entries(&self) -> &[ManifestEntryRef] {
158 &self.entries
159 }
160
161 pub fn metadata(&self) -> &ManifestMetadata {
163 &self.metadata
164 }
165
166 pub fn into_parts(self) -> (Vec<ManifestEntryRef>, ManifestMetadata) {
168 let Self { entries, metadata } = self;
169 (entries, metadata)
170 }
171
172 pub fn new(metadata: ManifestMetadata, entries: Vec<ManifestEntry>) -> Self {
174 Self {
175 metadata,
176 entries: entries.into_iter().map(Arc::new).collect(),
177 }
178 }
179}
180
181pub fn serialize_data_file_to_json(
183 data_file: DataFile,
184 partition_type: &super::StructType,
185 format_version: FormatVersion,
186) -> Result<String> {
187 let partition_struct_type = Type::Struct(partition_type.clone());
188 let serde = _serde::DataFileSerde::try_from(data_file, &partition_struct_type, format_version)?;
189 serde_json::to_string(&serde)
190 .map_err(|e| invalid_data!("Failed to serialize DataFile to JSON!").with_source(e))
191}
192
193pub fn deserialize_data_file_from_json(
195 json: &str,
196 partition_spec_id: i32,
197 partition_type: &super::StructType,
198 schema: &Schema,
199) -> Result<DataFile> {
200 let serde = serde_json::from_str::<_serde::DataFileSerde>(json)
201 .map_err(|e| invalid_data!("Failed to deserialize JSON to DataFile!").with_source(e))?;
202
203 let partition_struct_type = Type::Struct(partition_type.clone());
204 serde.try_into(partition_spec_id, &partition_struct_type, schema)
205}
206
207#[cfg(test)]
208mod tests {
209 use std::collections::HashMap;
210 use std::fs;
211 use std::sync::Arc;
212
213 use apache_avro::types::Value as AvroValue;
214 use apache_avro::{Codec, Writer, to_value};
215 use serde_json::{Value, to_vec};
216 use tempfile::TempDir;
217 use tracing::Level;
218 use tracing::subscriber::with_default;
219
220 use super::*;
221 use crate::ErrorKind;
222 use crate::io::FileIO;
223 use crate::spec::{Literal, NestedField, PrimitiveType, Struct, Transform, Type};
224
225 #[tokio::test]
226 async fn test_parse_manifest_v2_unpartition() {
227 let schema = Arc::new(
228 Schema::builder()
229 .with_fields(vec![
230 Arc::new(NestedField::optional(
232 1,
233 "id",
234 Type::Primitive(PrimitiveType::Long),
235 )),
236 Arc::new(NestedField::optional(
237 2,
238 "v_int",
239 Type::Primitive(PrimitiveType::Int),
240 )),
241 Arc::new(NestedField::optional(
242 3,
243 "v_long",
244 Type::Primitive(PrimitiveType::Long),
245 )),
246 Arc::new(NestedField::optional(
247 4,
248 "v_float",
249 Type::Primitive(PrimitiveType::Float),
250 )),
251 Arc::new(NestedField::optional(
252 5,
253 "v_double",
254 Type::Primitive(PrimitiveType::Double),
255 )),
256 Arc::new(NestedField::optional(
257 6,
258 "v_varchar",
259 Type::Primitive(PrimitiveType::String),
260 )),
261 Arc::new(NestedField::optional(
262 7,
263 "v_bool",
264 Type::Primitive(PrimitiveType::Boolean),
265 )),
266 Arc::new(NestedField::optional(
267 8,
268 "v_date",
269 Type::Primitive(PrimitiveType::Date),
270 )),
271 Arc::new(NestedField::optional(
272 9,
273 "v_timestamp",
274 Type::Primitive(PrimitiveType::Timestamptz),
275 )),
276 Arc::new(NestedField::optional(
277 10,
278 "v_decimal",
279 Type::Primitive(PrimitiveType::Decimal {
280 precision: 36,
281 scale: 10,
282 }),
283 )),
284 Arc::new(NestedField::optional(
285 11,
286 "v_ts_ntz",
287 Type::Primitive(PrimitiveType::Timestamp),
288 )),
289 Arc::new(NestedField::optional(
290 12,
291 "v_ts_ns_ntz",
292 Type::Primitive(PrimitiveType::TimestampNs),
293 )),
294 ])
295 .build()
296 .unwrap(),
297 );
298 let metadata = ManifestMetadata {
299 schema_id: 0,
300 schema: schema.clone(),
301 partition_spec: PartitionSpec::builder(schema)
302 .with_spec_id(0)
303 .build()
304 .unwrap(),
305 content: ManifestContentType::Data,
306 format_version: FormatVersion::V2,
307 };
308 let mut entries = vec![
309 ManifestEntry {
310 status: ManifestStatus::Added,
311 snapshot_id: None,
312 sequence_number: None,
313 file_sequence_number: None,
314 data_file: DataFile {content:DataContentType::Data,file_path:"s3a://icebergdata/demo/s1/t1/data/00000-0-ba56fbfa-f2ff-40c9-bb27-565ad6dc2be8-00000.parquet".to_string(),file_format:DataFileFormat::Parquet,partition:Struct::empty(),record_count:1,file_size_in_bytes:5442,column_sizes:HashMap::from([(0,73),(6,34),(2,73),(7,61),(3,61),(5,62),(9,79),(10,73),(1,61),(4,73),(8,73)]),value_counts:HashMap::from([(4,1),(5,1),(2,1),(0,1),(3,1),(6,1),(8,1),(1,1),(10,1),(7,1),(9,1)]),null_value_counts:HashMap::from([(1,0),(6,0),(2,0),(8,0),(0,0),(3,0),(5,0),(9,0),(7,0),(4,0),(10,0)]),nan_value_counts:HashMap::new(),lower_bounds:HashMap::new(),upper_bounds:HashMap::new(),key_metadata:None,split_offsets:Some(vec![4]),equality_ids:Some(Vec::new()),sort_order_id:None, partition_spec_id: 0,first_row_id: None,referenced_data_file: None,content_offset: None,content_size_in_bytes: None }
315 }
316 ];
317
318 let tmp_dir = TempDir::new().unwrap();
320 let path = tmp_dir.path().join("test_manifest.avro");
321 let io = FileIO::new_with_fs();
322 let output_file = io.new_output(path.to_str().unwrap()).unwrap();
323 let mut writer = ManifestWriterBuilder::new(
324 output_file,
325 Some(1),
326 metadata.schema.clone(),
327 metadata.partition_spec.clone(),
328 )
329 .build_v2_data();
330 for entry in &entries {
331 writer.add_entry(entry.clone()).unwrap();
332 }
333 writer.write_manifest_file().await.unwrap();
334
335 let actual_manifest =
337 Manifest::parse_avro(fs::read(path).expect("read_file must succeed").as_slice())
338 .unwrap();
339 entries[0].snapshot_id = Some(1);
341 assert_eq!(actual_manifest, Manifest::new(metadata, entries));
342 }
343
344 #[test]
345 fn test_parse_snappy_manifest_v2() {
346 let schema = Arc::new(
347 Schema::builder()
348 .with_fields(vec![Arc::new(NestedField::optional(
349 1,
350 "id",
351 Type::Primitive(PrimitiveType::Long),
352 ))])
353 .build()
354 .unwrap(),
355 );
356 let partition_spec = PartitionSpec::builder(schema.clone())
357 .with_spec_id(0)
358 .build()
359 .unwrap();
360
361 for (manifest_content, file_content, file_path) in [
362 (
363 ManifestContentType::Data,
364 DataContentType::Data,
365 "s3://bucket/table/data/data.parquet",
366 ),
367 (
368 ManifestContentType::Deletes,
369 DataContentType::PositionDeletes,
370 "s3://bucket/table/data/delete.parquet",
371 ),
372 ] {
373 let metadata = ManifestMetadata {
374 schema_id: 0,
375 schema: schema.clone(),
376 partition_spec: partition_spec.clone(),
377 content: manifest_content,
378 format_version: FormatVersion::V2,
379 };
380 let entry = ManifestEntry {
381 status: ManifestStatus::Added,
382 snapshot_id: Some(1),
383 sequence_number: None,
384 file_sequence_number: None,
385 data_file: DataFile {
386 content: file_content,
387 file_path: file_path.to_string(),
388 file_format: DataFileFormat::Parquet,
389 partition: Struct::empty(),
390 record_count: 1,
391 file_size_in_bytes: 1024,
392 column_sizes: HashMap::new(),
393 value_counts: HashMap::new(),
394 null_value_counts: HashMap::new(),
395 nan_value_counts: HashMap::new(),
396 lower_bounds: HashMap::new(),
397 upper_bounds: HashMap::new(),
398 key_metadata: None,
399 split_offsets: None,
400 equality_ids: None,
401 sort_order_id: None,
402 partition_spec_id: 0,
403 first_row_id: None,
404 referenced_data_file: None,
405 content_offset: None,
406 content_size_in_bytes: None,
407 },
408 };
409
410 let partition_type = metadata
411 .partition_spec
412 .partition_type(&metadata.schema)
413 .unwrap();
414 let avro_schema = manifest_schema_v2(&partition_type).unwrap();
415 let mut writer = Writer::with_codec(&avro_schema, Vec::new(), Codec::Snappy).unwrap();
416 writer
417 .add_user_metadata("schema".to_string(), to_vec(&metadata.schema).unwrap())
418 .unwrap();
419 writer
420 .add_user_metadata(
421 "schema-id".to_string(),
422 metadata.schema.schema_id().to_string(),
423 )
424 .unwrap();
425 writer
426 .add_user_metadata(
427 "partition-spec".to_string(),
428 to_vec(&metadata.partition_spec.fields()).unwrap(),
429 )
430 .unwrap();
431 writer
432 .add_user_metadata(
433 "partition-spec-id".to_string(),
434 metadata.partition_spec.spec_id().to_string(),
435 )
436 .unwrap();
437 writer
438 .add_user_metadata(
439 "format-version".to_string(),
440 (metadata.format_version as u8).to_string(),
441 )
442 .unwrap();
443 writer
444 .add_user_metadata("content".to_string(), metadata.content.to_string())
445 .unwrap();
446 let value = to_value(
447 _serde::ManifestEntryV2::try_from(
448 entry.clone(),
449 &Type::Struct(partition_type.clone()),
450 )
451 .unwrap(),
452 )
453 .unwrap()
454 .resolve(&avro_schema)
455 .unwrap();
456 writer.append_value(value).unwrap();
457 let bs = writer.into_inner().unwrap();
458
459 let parsed_manifest = Manifest::parse_avro(&bs).unwrap();
460
461 assert_eq!(parsed_manifest, Manifest::new(metadata, vec![entry]));
462 }
463 }
464
465 #[tokio::test]
466 async fn test_parse_manifest_v2_partition() {
467 let schema = Arc::new(
468 Schema::builder()
469 .with_fields(vec![
470 Arc::new(NestedField::optional(
471 1,
472 "id",
473 Type::Primitive(PrimitiveType::Long),
474 )),
475 Arc::new(NestedField::optional(
476 2,
477 "v_int",
478 Type::Primitive(PrimitiveType::Int),
479 )),
480 Arc::new(NestedField::optional(
481 3,
482 "v_long",
483 Type::Primitive(PrimitiveType::Long),
484 )),
485 Arc::new(NestedField::optional(
486 4,
487 "v_float",
488 Type::Primitive(PrimitiveType::Float),
489 )),
490 Arc::new(NestedField::optional(
491 5,
492 "v_double",
493 Type::Primitive(PrimitiveType::Double),
494 )),
495 Arc::new(NestedField::optional(
496 6,
497 "v_varchar",
498 Type::Primitive(PrimitiveType::String),
499 )),
500 Arc::new(NestedField::optional(
501 7,
502 "v_bool",
503 Type::Primitive(PrimitiveType::Boolean),
504 )),
505 Arc::new(NestedField::optional(
506 8,
507 "v_date",
508 Type::Primitive(PrimitiveType::Date),
509 )),
510 Arc::new(NestedField::optional(
511 9,
512 "v_timestamp",
513 Type::Primitive(PrimitiveType::Timestamptz),
514 )),
515 Arc::new(NestedField::optional(
516 10,
517 "v_decimal",
518 Type::Primitive(PrimitiveType::Decimal {
519 precision: 36,
520 scale: 10,
521 }),
522 )),
523 Arc::new(NestedField::optional(
524 11,
525 "v_ts_ntz",
526 Type::Primitive(PrimitiveType::Timestamp),
527 )),
528 Arc::new(NestedField::optional(
529 12,
530 "v_ts_ns_ntz",
531 Type::Primitive(PrimitiveType::TimestampNs),
532 )),
533 ])
534 .build()
535 .unwrap(),
536 );
537 let metadata = ManifestMetadata {
538 schema_id: 0,
539 schema: schema.clone(),
540 partition_spec: PartitionSpec::builder(schema)
541 .with_spec_id(0)
542 .add_partition_field("v_int", "v_int", Transform::Identity)
543 .unwrap()
544 .add_partition_field("v_long", "v_long", Transform::Identity)
545 .unwrap()
546 .build()
547 .unwrap(),
548 content: ManifestContentType::Data,
549 format_version: FormatVersion::V2,
550 };
551 let mut entries = vec![ManifestEntry {
552 status: ManifestStatus::Added,
553 snapshot_id: None,
554 sequence_number: None,
555 file_sequence_number: None,
556 data_file: DataFile {
557 content: DataContentType::Data,
558 file_format: DataFileFormat::Parquet,
559 file_path: "s3a://icebergdata/demo/s1/t1/data/00000-0-378b56f5-5c52-4102-a2c2-f05f8a7cbe4a-00000.parquet".to_string(),
560 partition: Struct::from_iter(
561 vec![
562 Some(Literal::int(1)),
563 Some(Literal::long(1000)),
564 ]
565 .into_iter()
566 ),
567 record_count: 1,
568 file_size_in_bytes: 5442,
569 column_sizes: HashMap::from([
570 (0, 73),
571 (6, 34),
572 (2, 73),
573 (7, 61),
574 (3, 61),
575 (5, 62),
576 (9, 79),
577 (10, 73),
578 (1, 61),
579 (4, 73),
580 (8, 73)
581 ]),
582 value_counts: HashMap::from([
583 (4, 1),
584 (5, 1),
585 (2, 1),
586 (0, 1),
587 (3, 1),
588 (6, 1),
589 (8, 1),
590 (1, 1),
591 (10, 1),
592 (7, 1),
593 (9, 1)
594 ]),
595 null_value_counts: HashMap::from([
596 (1, 0),
597 (6, 0),
598 (2, 0),
599 (8, 0),
600 (0, 0),
601 (3, 0),
602 (5, 0),
603 (9, 0),
604 (7, 0),
605 (4, 0),
606 (10, 0)
607 ]),
608 nan_value_counts: HashMap::new(),
609 lower_bounds: HashMap::new(),
610 upper_bounds: HashMap::new(),
611 key_metadata: None,
612 split_offsets: Some(vec![4]),
613 equality_ids: Some(Vec::new()),
614 sort_order_id: None,
615 partition_spec_id: 0,
616 first_row_id: None,
617 referenced_data_file: None,
618 content_offset: None,
619 content_size_in_bytes: None,
620 },
621 }];
622
623 let tmp_dir = TempDir::new().unwrap();
625 let path = tmp_dir.path().join("test_manifest.avro");
626 let io = FileIO::new_with_fs();
627 let output_file = io.new_output(path.to_str().unwrap()).unwrap();
628 let mut writer = ManifestWriterBuilder::new(
629 output_file,
630 Some(2),
631 metadata.schema.clone(),
632 metadata.partition_spec.clone(),
633 )
634 .build_v2_data();
635 for entry in &entries {
636 writer.add_entry(entry.clone()).unwrap();
637 }
638 let manifest_file = writer.write_manifest_file().await.unwrap();
639 assert_eq!(manifest_file.sequence_number, UNASSIGNED_SEQUENCE_NUMBER);
640 assert_eq!(
641 manifest_file.min_sequence_number,
642 UNASSIGNED_SEQUENCE_NUMBER
643 );
644
645 let actual_manifest =
647 Manifest::parse_avro(fs::read(path).expect("read_file must succeed").as_slice())
648 .unwrap();
649 entries[0].snapshot_id = Some(2);
651 assert_eq!(actual_manifest, Manifest::new(metadata, entries));
652 }
653
654 #[tokio::test]
655 async fn test_parse_manifest_v1_unpartition() {
656 let schema = Arc::new(
657 Schema::builder()
658 .with_schema_id(1)
659 .with_fields(vec![
660 Arc::new(NestedField::optional(
661 1,
662 "id",
663 Type::Primitive(PrimitiveType::Int),
664 )),
665 Arc::new(NestedField::optional(
666 2,
667 "data",
668 Type::Primitive(PrimitiveType::String),
669 )),
670 Arc::new(NestedField::optional(
671 3,
672 "comment",
673 Type::Primitive(PrimitiveType::String),
674 )),
675 ])
676 .build()
677 .unwrap(),
678 );
679 let metadata = ManifestMetadata {
680 schema_id: 1,
681 schema: schema.clone(),
682 partition_spec: PartitionSpec::builder(schema)
683 .with_spec_id(0)
684 .build()
685 .unwrap(),
686 content: ManifestContentType::Data,
687 format_version: FormatVersion::V1,
688 };
689 let mut entries = vec![ManifestEntry {
690 status: ManifestStatus::Added,
691 snapshot_id: Some(0),
692 sequence_number: Some(0),
693 file_sequence_number: Some(0),
694 data_file: DataFile {
695 content: DataContentType::Data,
696 file_path: "s3://testbucket/iceberg_data/iceberg_ctl/iceberg_db/iceberg_tbl/data/00000-7-45268d71-54eb-476c-b42c-942d880c04a1-00001.parquet".to_string(),
697 file_format: DataFileFormat::Parquet,
698 partition: Struct::empty(),
699 record_count: 1,
700 file_size_in_bytes: 875,
701 column_sizes: HashMap::from([(1,47),(2,48),(3,52)]),
702 value_counts: HashMap::from([(1,1),(2,1),(3,1)]),
703 null_value_counts: HashMap::from([(1,0),(2,0),(3,0)]),
704 nan_value_counts: HashMap::new(),
705 lower_bounds: HashMap::from([(1,Datum::int(1)),(2,Datum::string("a")),(3,Datum::string("AC/DC"))]),
706 upper_bounds: HashMap::from([(1,Datum::int(1)),(2,Datum::string("a")),(3,Datum::string("AC/DC"))]),
707 key_metadata: None,
708 split_offsets: Some(vec![4]),
709 equality_ids: None,
710 sort_order_id: Some(0),
711 partition_spec_id: 0,
712 first_row_id: None,
713 referenced_data_file: None,
714 content_offset: None,
715 content_size_in_bytes: None,
716 }
717 }];
718
719 let tmp_dir = TempDir::new().unwrap();
721 let path = tmp_dir.path().join("test_manifest.avro");
722 let io = FileIO::new_with_fs();
723 let output_file = io.new_output(path.to_str().unwrap()).unwrap();
724 let mut writer = ManifestWriterBuilder::new(
725 output_file,
726 Some(3),
727 metadata.schema.clone(),
728 metadata.partition_spec.clone(),
729 )
730 .build_v1();
731 for entry in &entries {
732 writer.add_entry(entry.clone()).unwrap();
733 }
734 writer.write_manifest_file().await.unwrap();
735
736 let actual_manifest =
738 Manifest::parse_avro(fs::read(path).expect("read_file must succeed").as_slice())
739 .unwrap();
740 entries[0].snapshot_id = Some(3);
742 assert_eq!(actual_manifest, Manifest::new(metadata, entries));
743 }
744
745 #[tokio::test]
746 async fn test_parse_manifest_v1_partition() {
747 let schema = Arc::new(
748 Schema::builder()
749 .with_fields(vec![
750 Arc::new(NestedField::optional(
751 1,
752 "id",
753 Type::Primitive(PrimitiveType::Long),
754 )),
755 Arc::new(NestedField::optional(
756 2,
757 "data",
758 Type::Primitive(PrimitiveType::String),
759 )),
760 Arc::new(NestedField::optional(
761 3,
762 "category",
763 Type::Primitive(PrimitiveType::String),
764 )),
765 ])
766 .build()
767 .unwrap(),
768 );
769 let metadata = ManifestMetadata {
770 schema_id: 0,
771 schema: schema.clone(),
772 partition_spec: PartitionSpec::builder(schema)
773 .add_partition_field("category", "category", Transform::Identity)
774 .unwrap()
775 .build()
776 .unwrap(),
777 content: ManifestContentType::Data,
778 format_version: FormatVersion::V1,
779 };
780 let mut entries = vec![
781 ManifestEntry {
782 status: ManifestStatus::Added,
783 snapshot_id: Some(0),
784 sequence_number: Some(0),
785 file_sequence_number: Some(0),
786 data_file: DataFile {
787 content: DataContentType::Data,
788 file_path: "s3://testbucket/prod/db/sample/data/category=x/00010-1-d5c93668-1e52-41ac-92a6-bba590cbf249-00001.parquet".to_string(),
789 file_format: DataFileFormat::Parquet,
790 partition: Struct::from_iter(
791 vec![
792 Some(
793 Literal::string("x"),
794 ),
795 ]
796 .into_iter()
797 ),
798 record_count: 1,
799 file_size_in_bytes: 874,
800 column_sizes: HashMap::from([(1, 46), (2, 48), (3, 48)]),
801 value_counts: HashMap::from([(1, 1), (2, 1), (3, 1)]),
802 null_value_counts: HashMap::from([(1, 0), (2, 0), (3, 0)]),
803 nan_value_counts: HashMap::new(),
804 lower_bounds: HashMap::from([
805 (1, Datum::long(1)),
806 (2, Datum::string("a")),
807 (3, Datum::string("x"))
808 ]),
809 upper_bounds: HashMap::from([
810 (1, Datum::long(1)),
811 (2, Datum::string("a")),
812 (3, Datum::string("x"))
813 ]),
814 key_metadata: None,
815 split_offsets: Some(vec![4]),
816 equality_ids: None,
817 sort_order_id: Some(0),
818 partition_spec_id: 0,
819 first_row_id: None,
820 referenced_data_file: None,
821 content_offset: None,
822 content_size_in_bytes: None,
823 },
824 }
825 ];
826
827 let tmp_dir = TempDir::new().unwrap();
829 let path = tmp_dir.path().join("test_manifest.avro");
830 let io = FileIO::new_with_fs();
831 let output_file = io.new_output(path.to_str().unwrap()).unwrap();
832 let mut writer = ManifestWriterBuilder::new(
833 output_file,
834 Some(2),
835 metadata.schema.clone(),
836 metadata.partition_spec.clone(),
837 )
838 .build_v1();
839 for entry in &entries {
840 writer.add_entry(entry.clone()).unwrap();
841 }
842 let manifest_file = writer.write_manifest_file().await.unwrap();
843 let partitions = manifest_file.partitions.unwrap();
844 assert_eq!(partitions.len(), 1);
845 assert_eq!(
846 partitions[0].clone().lower_bound.unwrap(),
847 Datum::string("x").to_bytes().unwrap()
848 );
849 assert_eq!(
850 partitions[0].clone().upper_bound.unwrap(),
851 Datum::string("x").to_bytes().unwrap()
852 );
853
854 let actual_manifest =
856 Manifest::parse_avro(fs::read(path).expect("read_file must succeed").as_slice())
857 .unwrap();
858 entries[0].snapshot_id = Some(2);
860 assert_eq!(actual_manifest, Manifest::new(metadata, entries));
861 }
862
863 #[tokio::test]
864 async fn test_parse_manifest_with_schema_evolution() {
865 let schema = Arc::new(
866 Schema::builder()
867 .with_fields(vec![
868 Arc::new(NestedField::optional(
869 1,
870 "id",
871 Type::Primitive(PrimitiveType::Long),
872 )),
873 Arc::new(NestedField::optional(
874 2,
875 "v_int",
876 Type::Primitive(PrimitiveType::Int),
877 )),
878 ])
879 .build()
880 .unwrap(),
881 );
882 let metadata = ManifestMetadata {
883 schema_id: 0,
884 schema: schema.clone(),
885 partition_spec: PartitionSpec::builder(schema)
886 .with_spec_id(0)
887 .build()
888 .unwrap(),
889 content: ManifestContentType::Data,
890 format_version: FormatVersion::V2,
891 };
892 let entries = vec![ManifestEntry {
893 status: ManifestStatus::Added,
894 snapshot_id: None,
895 sequence_number: None,
896 file_sequence_number: None,
897 data_file: DataFile {
898 content: DataContentType::Data,
899 file_format: DataFileFormat::Parquet,
900 file_path: "s3a://icebergdata/demo/s1/t1/data/00000-0-378b56f5-5c52-4102-a2c2-f05f8a7cbe4a-00000.parquet".to_string(),
901 partition: Struct::empty(),
902 record_count: 1,
903 file_size_in_bytes: 5442,
904 column_sizes: HashMap::from([
905 (1, 61),
906 (2, 73),
907 (3, 61),
908 ]),
909 value_counts: HashMap::default(),
910 null_value_counts: HashMap::default(),
911 nan_value_counts: HashMap::new(),
912 lower_bounds: HashMap::from([
913 (1, Datum::long(1)),
914 (2, Datum::int(2)),
915 (3, Datum::string("x"))
916 ]),
917 upper_bounds: HashMap::from([
918 (1, Datum::long(1)),
919 (2, Datum::int(2)),
920 (3, Datum::string("x"))
921 ]),
922 key_metadata: None,
923 split_offsets: Some(vec![4]),
924 equality_ids: None,
925 sort_order_id: None,
926 partition_spec_id: 0,
927 first_row_id: None,
928 referenced_data_file: None,
929 content_offset: None,
930 content_size_in_bytes: None,
931 },
932 }];
933
934 let tmp_dir = TempDir::new().unwrap();
936 let path = tmp_dir.path().join("test_manifest.avro");
937 let io = FileIO::new_with_fs();
938 let output_file = io.new_output(path.to_str().unwrap()).unwrap();
939 let mut writer = ManifestWriterBuilder::new(
940 output_file,
941 Some(2),
942 metadata.schema.clone(),
943 metadata.partition_spec.clone(),
944 )
945 .build_v2_data();
946 for entry in &entries {
947 writer.add_entry(entry.clone()).unwrap();
948 }
949 writer.write_manifest_file().await.unwrap();
950
951 let actual_manifest =
953 Manifest::parse_avro(fs::read(path).expect("read_file must succeed").as_slice())
954 .unwrap();
955
956 let schema = Arc::new(
960 Schema::builder()
961 .with_fields(vec![
962 Arc::new(NestedField::optional(
963 1,
964 "id",
965 Type::Primitive(PrimitiveType::Long),
966 )),
967 Arc::new(NestedField::optional(
968 2,
969 "v_int",
970 Type::Primitive(PrimitiveType::Int),
971 )),
972 ])
973 .build()
974 .unwrap(),
975 );
976 let expected_manifest = Manifest {
977 metadata: ManifestMetadata {
978 schema_id: 0,
979 schema: schema.clone(),
980 partition_spec: PartitionSpec::builder(schema).with_spec_id(0).build().unwrap(),
981 content: ManifestContentType::Data,
982 format_version: FormatVersion::V2,
983 },
984 entries: vec![Arc::new(ManifestEntry {
985 status: ManifestStatus::Added,
986 snapshot_id: Some(2),
987 sequence_number: None,
988 file_sequence_number: None,
989 data_file: DataFile {
990 content: DataContentType::Data,
991 file_format: DataFileFormat::Parquet,
992 file_path: "s3a://icebergdata/demo/s1/t1/data/00000-0-378b56f5-5c52-4102-a2c2-f05f8a7cbe4a-00000.parquet".to_string(),
993 partition: Struct::empty(),
994 record_count: 1,
995 file_size_in_bytes: 5442,
996 column_sizes: HashMap::from([
997 (1, 61),
998 (2, 73),
999 (3, 61),
1000 ]),
1001 value_counts: HashMap::default(),
1002 null_value_counts: HashMap::default(),
1003 nan_value_counts: HashMap::new(),
1004 lower_bounds: HashMap::from([
1005 (1, Datum::long(1)),
1006 (2, Datum::int(2)),
1007 ]),
1008 upper_bounds: HashMap::from([
1009 (1, Datum::long(1)),
1010 (2, Datum::int(2)),
1011 ]),
1012 key_metadata: None,
1013 split_offsets: Some(vec![4]),
1014 equality_ids: None,
1015 sort_order_id: None,
1016 partition_spec_id: 0,
1017 first_row_id: None,
1018 referenced_data_file: None,
1019 content_offset: None,
1020 content_size_in_bytes: None,
1021 },
1022 })],
1023 };
1024
1025 assert_eq!(actual_manifest, expected_manifest);
1026 }
1027
1028 #[tokio::test]
1029 async fn test_manifest_summary() {
1030 let schema = Arc::new(
1031 Schema::builder()
1032 .with_fields(vec![
1033 Arc::new(NestedField::optional(
1034 1,
1035 "time",
1036 Type::Primitive(PrimitiveType::Date),
1037 )),
1038 Arc::new(NestedField::optional(
1039 2,
1040 "v_float",
1041 Type::Primitive(PrimitiveType::Float),
1042 )),
1043 Arc::new(NestedField::optional(
1044 3,
1045 "v_double",
1046 Type::Primitive(PrimitiveType::Double),
1047 )),
1048 ])
1049 .build()
1050 .unwrap(),
1051 );
1052 let partition_spec = PartitionSpec::builder(schema.clone())
1053 .with_spec_id(0)
1054 .add_partition_field("time", "year_of_time", Transform::Year)
1055 .unwrap()
1056 .add_partition_field("v_float", "f", Transform::Identity)
1057 .unwrap()
1058 .add_partition_field("v_double", "d", Transform::Identity)
1059 .unwrap()
1060 .build()
1061 .unwrap();
1062 let metadata = ManifestMetadata {
1063 schema_id: 0,
1064 schema,
1065 partition_spec,
1066 content: ManifestContentType::Data,
1067 format_version: FormatVersion::V2,
1068 };
1069 let entries = vec![
1070 ManifestEntry {
1071 status: ManifestStatus::Added,
1072 snapshot_id: None,
1073 sequence_number: None,
1074 file_sequence_number: None,
1075 data_file: DataFile {
1076 content: DataContentType::Data,
1077 file_path: "s3a://icebergdata/demo/s1/t1/data/00000-0-ba56fbfa-f2ff-40c9-bb27-565ad6dc2be8-00000.parquet".to_string(),
1078 file_format: DataFileFormat::Parquet,
1079 partition: Struct::from_iter(
1080 vec![
1081 Some(Literal::int(2021)),
1082 Some(Literal::float(1.0_f32)),
1083 Some(Literal::double(2.0)),
1084 ]
1085 ),
1086 record_count: 1,
1087 file_size_in_bytes: 5442,
1088 column_sizes: HashMap::from([(0,73),(6,34),(2,73),(7,61),(3,61),(5,62),(9,79),(10,73),(1,61),(4,73),(8,73)]),
1089 value_counts: HashMap::from([(4,1),(5,1),(2,1),(0,1),(3,1),(6,1),(8,1),(1,1),(10,1),(7,1),(9,1)]),
1090 null_value_counts: HashMap::from([(1,0),(6,0),(2,0),(8,0),(0,0),(3,0),(5,0),(9,0),(7,0),(4,0),(10,0)]),
1091 nan_value_counts: HashMap::new(),
1092 lower_bounds: HashMap::new(),
1093 upper_bounds: HashMap::new(),
1094 key_metadata: None,
1095 split_offsets: Some(vec![4]),
1096 equality_ids: None,
1097 sort_order_id: None,
1098 partition_spec_id: 0,
1099 first_row_id: None,
1100 referenced_data_file: None,
1101 content_offset: None,
1102 content_size_in_bytes: None,
1103 }
1104 },
1105 ManifestEntry {
1106 status: ManifestStatus::Added,
1107 snapshot_id: None,
1108 sequence_number: None,
1109 file_sequence_number: None,
1110 data_file: DataFile {
1111 content: DataContentType::Data,
1112 file_path: "s3a://icebergdata/demo/s1/t1/data/00000-0-ba56fbfa-f2ff-40c9-bb27-565ad6dc2be8-00000.parquet".to_string(),
1113 file_format: DataFileFormat::Parquet,
1114 partition: Struct::from_iter(
1115 vec![
1116 Some(Literal::int(1111)),
1117 Some(Literal::float(15.5_f32)),
1118 Some(Literal::double(25.5)),
1119 ]
1120 ),
1121 record_count: 1,
1122 file_size_in_bytes: 5442,
1123 column_sizes: HashMap::from([(0,73),(6,34),(2,73),(7,61),(3,61),(5,62),(9,79),(10,73),(1,61),(4,73),(8,73)]),
1124 value_counts: HashMap::from([(4,1),(5,1),(2,1),(0,1),(3,1),(6,1),(8,1),(1,1),(10,1),(7,1),(9,1)]),
1125 null_value_counts: HashMap::from([(1,0),(6,0),(2,0),(8,0),(0,0),(3,0),(5,0),(9,0),(7,0),(4,0),(10,0)]),
1126 nan_value_counts: HashMap::new(),
1127 lower_bounds: HashMap::new(),
1128 upper_bounds: HashMap::new(),
1129 key_metadata: None,
1130 split_offsets: Some(vec![4]),
1131 equality_ids: None,
1132 sort_order_id: None,
1133 partition_spec_id: 0,
1134 first_row_id: None,
1135 referenced_data_file: None,
1136 content_offset: None,
1137 content_size_in_bytes: None,
1138 }
1139 },
1140 ManifestEntry {
1141 status: ManifestStatus::Added,
1142 snapshot_id: None,
1143 sequence_number: None,
1144 file_sequence_number: None,
1145 data_file: DataFile {
1146 content: DataContentType::Data,
1147 file_path: "s3a://icebergdata/demo/s1/t1/data/00000-0-ba56fbfa-f2ff-40c9-bb27-565ad6dc2be8-00000.parquet".to_string(),
1148 file_format: DataFileFormat::Parquet,
1149 partition: Struct::from_iter(
1150 vec![
1151 Some(Literal::int(1211)),
1152 Some(Literal::float(f32::NAN)),
1153 Some(Literal::double(1.0)),
1154 ]
1155 ),
1156 record_count: 1,
1157 file_size_in_bytes: 5442,
1158 column_sizes: HashMap::from([(0,73),(6,34),(2,73),(7,61),(3,61),(5,62),(9,79),(10,73),(1,61),(4,73),(8,73)]),
1159 value_counts: HashMap::from([(4,1),(5,1),(2,1),(0,1),(3,1),(6,1),(8,1),(1,1),(10,1),(7,1),(9,1)]),
1160 null_value_counts: HashMap::from([(1,0),(6,0),(2,0),(8,0),(0,0),(3,0),(5,0),(9,0),(7,0),(4,0),(10,0)]),
1161 nan_value_counts: HashMap::new(),
1162 lower_bounds: HashMap::new(),
1163 upper_bounds: HashMap::new(),
1164 key_metadata: None,
1165 split_offsets: Some(vec![4]),
1166 equality_ids: None,
1167 sort_order_id: None,
1168 partition_spec_id: 0,
1169 first_row_id: None,
1170 referenced_data_file: None,
1171 content_offset: None,
1172 content_size_in_bytes: None,
1173 }
1174 },
1175 ManifestEntry {
1176 status: ManifestStatus::Added,
1177 snapshot_id: None,
1178 sequence_number: None,
1179 file_sequence_number: None,
1180 data_file: DataFile {
1181 content: DataContentType::Data,
1182 file_path: "s3a://icebergdata/demo/s1/t1/data/00000-0-ba56fbfa-f2ff-40c9-bb27-565ad6dc2be8-00000.parquet".to_string(),
1183 file_format: DataFileFormat::Parquet,
1184 partition: Struct::from_iter(
1185 vec![
1186 Some(Literal::int(1111)),
1187 None,
1188 Some(Literal::double(11.0)),
1189 ]
1190 ),
1191 record_count: 1,
1192 file_size_in_bytes: 5442,
1193 column_sizes: HashMap::from([(0,73),(6,34),(2,73),(7,61),(3,61),(5,62),(9,79),(10,73),(1,61),(4,73),(8,73)]),
1194 value_counts: HashMap::from([(4,1),(5,1),(2,1),(0,1),(3,1),(6,1),(8,1),(1,1),(10,1),(7,1),(9,1)]),
1195 null_value_counts: HashMap::from([(1,0),(6,0),(2,0),(8,0),(0,0),(3,0),(5,0),(9,0),(7,0),(4,0),(10,0)]),
1196 nan_value_counts: HashMap::new(),
1197 lower_bounds: HashMap::new(),
1198 upper_bounds: HashMap::new(),
1199 key_metadata: None,
1200 split_offsets: Some(vec![4]),
1201 equality_ids: None,
1202 sort_order_id: None,
1203 partition_spec_id: 0,
1204 first_row_id: None,
1205 referenced_data_file: None,
1206 content_offset: None,
1207 content_size_in_bytes: None,
1208 }
1209 },
1210 ];
1211
1212 let tmp_dir = TempDir::new().unwrap();
1214 let path = tmp_dir.path().join("test_manifest.avro");
1215 let io = FileIO::new_with_fs();
1216 let output_file = io.new_output(path.to_str().unwrap()).unwrap();
1217 let mut writer = ManifestWriterBuilder::new(
1218 output_file,
1219 Some(1),
1220 metadata.schema.clone(),
1221 metadata.partition_spec.clone(),
1222 )
1223 .build_v2_data();
1224 for entry in &entries {
1225 writer.add_entry(entry.clone()).unwrap();
1226 }
1227 let res = writer.write_manifest_file().await.unwrap();
1228
1229 let partitions = res.partitions.unwrap();
1230
1231 assert_eq!(partitions.len(), 3);
1232 assert_eq!(
1233 partitions[0].clone().lower_bound.unwrap(),
1234 Datum::int(1111).to_bytes().unwrap()
1235 );
1236 assert_eq!(
1237 partitions[0].clone().upper_bound.unwrap(),
1238 Datum::int(2021).to_bytes().unwrap()
1239 );
1240 assert!(!partitions[0].clone().contains_null);
1241 assert_eq!(partitions[0].clone().contains_nan, Some(false));
1242
1243 assert_eq!(
1244 partitions[1].clone().lower_bound.unwrap(),
1245 Datum::float(1.0_f32).to_bytes().unwrap()
1246 );
1247 assert_eq!(
1248 partitions[1].clone().upper_bound.unwrap(),
1249 Datum::float(15.5_f32).to_bytes().unwrap()
1250 );
1251 assert!(partitions[1].clone().contains_null);
1252 assert_eq!(partitions[1].clone().contains_nan, Some(true));
1253
1254 assert_eq!(
1255 partitions[2].clone().lower_bound.unwrap(),
1256 Datum::double(1.0).to_bytes().unwrap()
1257 );
1258 assert_eq!(
1259 partitions[2].clone().upper_bound.unwrap(),
1260 Datum::double(25.5).to_bytes().unwrap()
1261 );
1262 assert!(!partitions[2].clone().contains_null);
1263 assert_eq!(partitions[2].clone().contains_nan, Some(false));
1264 }
1265
1266 #[test]
1267 fn test_data_file_serialization() {
1268 let schema = Schema::builder()
1270 .with_schema_id(1)
1271 .with_identifier_field_ids(vec![1])
1272 .with_fields(vec![
1273 NestedField::required(1, "id", Type::Primitive(PrimitiveType::Long)).into(),
1274 NestedField::required(2, "name", Type::Primitive(PrimitiveType::String)).into(),
1275 ])
1276 .build()
1277 .unwrap();
1278
1279 let partition_spec = PartitionSpec::builder(schema.clone())
1281 .with_spec_id(1)
1282 .add_partition_field("id", "id_partition", Transform::Identity)
1283 .unwrap()
1284 .build()
1285 .unwrap();
1286
1287 let partition_type = partition_spec.partition_type(&schema).unwrap();
1289
1290 let data_files = vec![
1292 DataFileBuilder::default()
1293 .content(DataContentType::Data)
1294 .file_format(DataFileFormat::Parquet)
1295 .file_path("path/to/file1.parquet".to_string())
1296 .file_size_in_bytes(1024)
1297 .record_count(100)
1298 .partition_spec_id(1)
1299 .partition(Struct::empty())
1300 .column_sizes(HashMap::from([(1, 512), (2, 1024)]))
1301 .value_counts(HashMap::from([(1, 100), (2, 500)]))
1302 .null_value_counts(HashMap::from([(1, 0), (2, 1)]))
1303 .build()
1304 .unwrap(),
1305 DataFileBuilder::default()
1306 .content(DataContentType::Data)
1307 .file_format(DataFileFormat::Parquet)
1308 .file_path("path/to/file2.parquet".to_string())
1309 .file_size_in_bytes(2048)
1310 .record_count(200)
1311 .partition_spec_id(1)
1312 .partition(Struct::empty())
1313 .column_sizes(HashMap::from([(1, 1024), (2, 2048)]))
1314 .value_counts(HashMap::from([(1, 200), (2, 600)]))
1315 .null_value_counts(HashMap::from([(1, 10), (2, 999)]))
1316 .build()
1317 .unwrap(),
1318 ];
1319
1320 let serialized_files = data_files
1322 .clone()
1323 .into_iter()
1324 .map(|f| serialize_data_file_to_json(f, &partition_type, FormatVersion::V2).unwrap())
1325 .collect::<Vec<String>>();
1326
1327 assert_eq!(serialized_files.len(), 2);
1329 let pretty_json1: Value = serde_json::from_str(serialized_files.first().unwrap()).unwrap();
1330 let pretty_json2: Value = serde_json::from_str(serialized_files.get(1).unwrap()).unwrap();
1331 let expected_serialized_file1 = serde_json::json!({
1332 "content": 0,
1333 "file_path": "path/to/file1.parquet",
1334 "file_format": "PARQUET",
1335 "partition": {},
1336 "record_count": 100,
1337 "file_size_in_bytes": 1024,
1338 "column_sizes": [
1339 { "key": 1, "value": 512 },
1340 { "key": 2, "value": 1024 }
1341 ],
1342 "value_counts": [
1343 { "key": 1, "value": 100 },
1344 { "key": 2, "value": 500 }
1345 ],
1346 "null_value_counts": [
1347 { "key": 1, "value": 0 },
1348 { "key": 2, "value": 1 }
1349 ],
1350 "nan_value_counts": [],
1351 "lower_bounds": [],
1352 "upper_bounds": [],
1353 "key_metadata": null,
1354 "split_offsets": null,
1355 "equality_ids": null,
1356 "sort_order_id": null,
1357 "first_row_id": null,
1358 "referenced_data_file": null,
1359 "content_offset": null,
1360 "content_size_in_bytes": null
1361 });
1362 let expected_serialized_file2 = serde_json::json!({
1363 "content": 0,
1364 "file_path": "path/to/file2.parquet",
1365 "file_format": "PARQUET",
1366 "partition": {},
1367 "record_count": 200,
1368 "file_size_in_bytes": 2048,
1369 "column_sizes": [
1370 { "key": 1, "value": 1024 },
1371 { "key": 2, "value": 2048 }
1372 ],
1373 "value_counts": [
1374 { "key": 1, "value": 200 },
1375 { "key": 2, "value": 600 }
1376 ],
1377 "null_value_counts": [
1378 { "key": 1, "value": 10 },
1379 { "key": 2, "value": 999 }
1380 ],
1381 "nan_value_counts": [],
1382 "lower_bounds": [],
1383 "upper_bounds": [],
1384 "key_metadata": null,
1385 "split_offsets": null,
1386 "equality_ids": null,
1387 "sort_order_id": null,
1388 "first_row_id": null,
1389 "referenced_data_file": null,
1390 "content_offset": null,
1391 "content_size_in_bytes": null
1392 });
1393 assert_eq!(pretty_json1, expected_serialized_file1);
1394 assert_eq!(pretty_json2, expected_serialized_file2);
1395
1396 let deserialized_files: Vec<DataFile> = serialized_files
1398 .into_iter()
1399 .map(|json| {
1400 deserialize_data_file_from_json(
1401 &json,
1402 partition_spec.spec_id(),
1403 &partition_type,
1404 &schema,
1405 )
1406 .unwrap()
1407 })
1408 .collect();
1409
1410 assert_eq!(deserialized_files.len(), 2);
1412 let deserialized_data_file1 = deserialized_files.first().unwrap();
1413 let deserialized_data_file2 = deserialized_files.get(1).unwrap();
1414 let original_data_file1 = data_files.first().unwrap();
1415 let original_data_file2 = data_files.get(1).unwrap();
1416
1417 assert_eq!(deserialized_data_file1, original_data_file1);
1418 assert_eq!(deserialized_data_file2, original_data_file2);
1419 }
1420
1421 fn writer_schema_test_metadata(format_version: FormatVersion) -> ManifestMetadata {
1424 let schema = Arc::new(
1425 Schema::builder()
1426 .with_fields(vec![
1427 NestedField::optional(1, "id", Type::Primitive(PrimitiveType::Long)).into(),
1428 NestedField::optional(2, "name", Type::Primitive(PrimitiveType::String)).into(),
1429 NestedField::optional(3, "score", Type::Primitive(PrimitiveType::Double))
1430 .into(),
1431 ])
1432 .build()
1433 .unwrap(),
1434 );
1435 let partition_spec = PartitionSpec::builder(schema.clone())
1436 .with_spec_id(0)
1437 .add_partition_field("id", "id", Transform::Identity)
1438 .unwrap()
1439 .add_partition_field("name", "name", Transform::Identity)
1440 .unwrap()
1441 .add_partition_field("score", "score", Transform::Identity)
1442 .unwrap()
1443 .build()
1444 .unwrap();
1445 ManifestMetadata {
1446 schema_id: 0,
1447 schema,
1448 partition_spec,
1449 content: ManifestContentType::Data,
1450 format_version,
1451 }
1452 }
1453
1454 fn v2_writer_schema(partition_fields: Value) -> Value {
1457 serde_json::json!({
1458 "type": "record",
1459 "name": "manifest_entry",
1460 "fields": [
1461 {"name": "status", "type": "int", "field-id": 0},
1462 {"name": "snapshot_id", "type": ["null", "long"], "default": null, "field-id": 1},
1463 {"name": "sequence_number", "type": ["null", "long"], "default": null, "field-id": 3},
1464 {"name": "file_sequence_number", "type": ["null", "long"], "default": null, "field-id": 4},
1465 {"name": "data_file", "field-id": 2, "type": {
1466 "type": "record",
1467 "name": "r2",
1468 "fields": [
1469 {"name": "content", "type": "int", "field-id": 134},
1470 {"name": "file_path", "type": "string", "field-id": 100},
1471 {"name": "file_format", "type": "string", "field-id": 101},
1472 {"name": "partition", "field-id": 102, "type": {
1473 "type": "record",
1474 "name": "r102",
1475 "fields": partition_fields,
1476 }},
1477 {"name": "record_count", "type": "long", "field-id": 103},
1478 {"name": "file_size_in_bytes", "type": "long", "field-id": 104},
1479 ],
1480 }},
1481 ],
1482 })
1483 }
1484
1485 fn writer_schema_field<'a>(record: &'a mut Value, path: &[&str]) -> &'a mut Value {
1488 let (name, rest) = path.split_first().unwrap();
1489 let field = record["fields"]
1490 .as_array_mut()
1491 .unwrap()
1492 .iter_mut()
1493 .find(|field| field["name"] == *name)
1494 .unwrap();
1495 if rest.is_empty() {
1496 field
1497 } else {
1498 writer_schema_field(&mut field["type"], rest)
1499 }
1500 }
1501
1502 fn v2_entry(partition: Value) -> Value {
1503 serde_json::json!({
1504 "status": 1,
1505 "snapshot_id": 7,
1506 "sequence_number": null,
1507 "file_sequence_number": null,
1508 "data_file": {
1509 "content": 0,
1510 "file_path": "s3://bucket/table/data/a.parquet",
1511 "file_format": "PARQUET",
1512 "partition": partition,
1513 "record_count": 10,
1514 "file_size_in_bytes": 100,
1515 },
1516 })
1517 }
1518
1519 fn expected_entry(partition: Struct) -> ManifestEntry {
1520 ManifestEntry {
1521 status: ManifestStatus::Added,
1522 snapshot_id: Some(7),
1523 sequence_number: None,
1524 file_sequence_number: None,
1525 data_file: DataFile {
1526 content: DataContentType::Data,
1527 file_path: "s3://bucket/table/data/a.parquet".to_string(),
1528 file_format: DataFileFormat::Parquet,
1529 partition,
1530 record_count: 10,
1531 file_size_in_bytes: 100,
1532 column_sizes: HashMap::new(),
1533 value_counts: HashMap::new(),
1534 null_value_counts: HashMap::new(),
1535 nan_value_counts: HashMap::new(),
1536 lower_bounds: HashMap::new(),
1537 upper_bounds: HashMap::new(),
1538 key_metadata: None,
1539 split_offsets: None,
1540 equality_ids: None,
1541 sort_order_id: None,
1542 partition_spec_id: 0,
1543 first_row_id: None,
1544 referenced_data_file: None,
1545 content_offset: None,
1546 content_size_in_bytes: None,
1547 },
1548 }
1549 }
1550
1551 fn write_with_writer_schema(
1554 metadata: &ManifestMetadata,
1555 writer_schema: &Value,
1556 entries: Vec<Value>,
1557 ) -> Vec<u8> {
1558 write_avro_values_with_writer_schema(
1559 metadata,
1560 writer_schema,
1561 entries
1562 .into_iter()
1563 .map(|entry| AvroValue::try_from(entry).unwrap())
1564 .collect(),
1565 )
1566 }
1567
1568 fn write_avro_values_with_writer_schema(
1570 metadata: &ManifestMetadata,
1571 writer_schema: &Value,
1572 entries: Vec<AvroValue>,
1573 ) -> Vec<u8> {
1574 let avro_schema = apache_avro::Schema::parse(writer_schema).unwrap();
1575 let mut writer = Writer::new(&avro_schema, Vec::new()).unwrap();
1576 for (key, value) in [
1577 ("schema", to_vec(&metadata.schema).unwrap()),
1578 ("schema-id", metadata.schema_id.to_string().into_bytes()),
1579 (
1580 "partition-spec",
1581 to_vec(&metadata.partition_spec.fields()).unwrap(),
1582 ),
1583 (
1584 "partition-spec-id",
1585 metadata.partition_spec.spec_id().to_string().into_bytes(),
1586 ),
1587 (
1588 "format-version",
1589 (metadata.format_version as u8).to_string().into_bytes(),
1590 ),
1591 ("content", metadata.content.to_string().into_bytes()),
1592 ] {
1593 writer.add_user_metadata(key.to_string(), value).unwrap();
1594 }
1595 for entry in entries {
1596 writer
1597 .append_value(entry.resolve(&avro_schema).unwrap())
1598 .unwrap();
1599 }
1600 writer.into_inner().unwrap()
1601 }
1602
1603 fn remove_writer_field(schema: &mut Value, entry: &mut AvroValue, path: &[&str]) {
1606 let (name, rest) = path.split_first().unwrap();
1607 let AvroValue::Record(values) = entry else {
1608 unreachable!("the entry is a record");
1609 };
1610 let fields = schema["fields"].as_array_mut().unwrap();
1611 if rest.is_empty() {
1612 fields.retain(|field| field["name"] != *name);
1613 values.retain(|(value_name, _)| value_name != name);
1614 } else {
1615 let field = fields.iter_mut().find(|field| field["name"] == *name);
1616 let value = values.iter_mut().find(|(value_name, _)| value_name == name);
1617 remove_writer_field(&mut field.unwrap()["type"], &mut value.unwrap().1, rest);
1618 }
1619 }
1620
1621 type ResetField = fn(&mut ManifestEntry);
1622
1623 fn assert_reads_without_each_field(
1627 metadata: &ManifestMetadata,
1628 full_schema: &Value,
1629 full_value: &AvroValue,
1630 full_entry: &ManifestEntry,
1631 optional: &[(&[&str], ResetField)],
1632 required: &[&[&str]],
1633 ) {
1634 for (path, reset) in optional {
1635 let (mut schema, mut value) = (full_schema.clone(), full_value.clone());
1636 remove_writer_field(&mut schema, &mut value, path);
1637 let bs = write_avro_values_with_writer_schema(metadata, &schema, vec![value]);
1638
1639 let manifest = Manifest::parse_avro(&bs).unwrap();
1640
1641 let mut expected = full_entry.clone();
1642 reset(&mut expected);
1643 assert_eq!(
1644 manifest,
1645 Manifest::new(metadata.clone(), vec![expected]),
1646 "{path:?}"
1647 );
1648 }
1649
1650 for path in required {
1651 let (mut schema, mut value) = (full_schema.clone(), full_value.clone());
1652 remove_writer_field(&mut schema, &mut value, path);
1653 let bs = write_avro_values_with_writer_schema(metadata, &schema, vec![value]);
1654
1655 let err = Manifest::parse_avro(&bs).unwrap_err();
1656
1657 assert!(
1658 err.to_string().contains(path.last().unwrap()),
1659 "{path:?}: {err}"
1660 );
1661 }
1662 }
1663
1664 #[test]
1665 fn test_parse_manifest_v3_without_each_field() {
1666 let metadata = writer_schema_test_metadata(FormatVersion::V3);
1669 let partition_type = metadata
1670 .partition_spec
1671 .partition_type(&metadata.schema)
1672 .unwrap();
1673 let full_schema =
1674 serde_json::to_value(manifest_schema_v3(&partition_type).unwrap()).unwrap();
1675 let mut full_entry = expected_entry(Struct::from_iter([
1676 Some(Literal::long(5)),
1677 Some(Literal::string("a")),
1678 Some(Literal::double(2.5)),
1679 ]));
1680 full_entry.sequence_number = Some(3);
1681 full_entry.file_sequence_number = Some(4);
1682 let data_file = &mut full_entry.data_file;
1683 data_file.content = DataContentType::PositionDeletes;
1684 data_file.column_sizes = HashMap::from([(1, 40)]);
1685 data_file.value_counts = HashMap::from([(1, 10)]);
1686 data_file.null_value_counts = HashMap::from([(1, 1)]);
1687 data_file.nan_value_counts = HashMap::from([(3, 2)]);
1688 data_file.lower_bounds = HashMap::from([(1, Datum::long(1))]);
1689 data_file.upper_bounds = HashMap::from([(1, Datum::long(9))]);
1690 data_file.key_metadata = Some(vec![1, 2]);
1691 data_file.split_offsets = Some(vec![4]);
1692 data_file.equality_ids = Some(vec![1]);
1693 data_file.sort_order_id = Some(0);
1694 data_file.first_row_id = Some(100);
1695 data_file.referenced_data_file = Some("s3://bucket/table/data/b.parquet".to_string());
1696 data_file.content_offset = Some(4);
1697 data_file.content_size_in_bytes = Some(8);
1698 let full_value = to_value(
1699 _serde::ManifestEntryV3::try_from(
1700 full_entry.clone(),
1701 &Type::Struct(partition_type.clone()),
1702 )
1703 .unwrap(),
1704 )
1705 .unwrap();
1706
1707 let optional: [(&[&str], ResetField); 18] = [
1708 (&["snapshot_id"], |e| e.snapshot_id = None),
1709 (&["sequence_number"], |e| e.sequence_number = None),
1710 (&["file_sequence_number"], |e| e.file_sequence_number = None),
1711 (&["data_file", "content"], |e| {
1712 e.data_file.content = DataContentType::Data
1713 }),
1714 (&["data_file", "column_sizes"], |e| {
1715 e.data_file.column_sizes.clear()
1716 }),
1717 (&["data_file", "value_counts"], |e| {
1718 e.data_file.value_counts.clear()
1719 }),
1720 (&["data_file", "null_value_counts"], |e| {
1721 e.data_file.null_value_counts.clear()
1722 }),
1723 (&["data_file", "nan_value_counts"], |e| {
1724 e.data_file.nan_value_counts.clear()
1725 }),
1726 (&["data_file", "lower_bounds"], |e| {
1727 e.data_file.lower_bounds.clear()
1728 }),
1729 (&["data_file", "upper_bounds"], |e| {
1730 e.data_file.upper_bounds.clear()
1731 }),
1732 (&["data_file", "key_metadata"], |e| {
1733 e.data_file.key_metadata = None
1734 }),
1735 (&["data_file", "split_offsets"], |e| {
1736 e.data_file.split_offsets = None
1737 }),
1738 (&["data_file", "equality_ids"], |e| {
1739 e.data_file.equality_ids = None
1740 }),
1741 (&["data_file", "sort_order_id"], |e| {
1742 e.data_file.sort_order_id = None
1743 }),
1744 (&["data_file", "first_row_id"], |e| {
1745 e.data_file.first_row_id = None
1746 }),
1747 (&["data_file", "referenced_data_file"], |e| {
1748 e.data_file.referenced_data_file = None
1749 }),
1750 (&["data_file", "content_offset"], |e| {
1751 e.data_file.content_offset = None
1752 }),
1753 (&["data_file", "content_size_in_bytes"], |e| {
1754 e.data_file.content_size_in_bytes = None
1755 }),
1756 ];
1757 let required: [&[&str]; 7] = [
1758 &["status"],
1759 &["data_file"],
1760 &["data_file", "file_path"],
1761 &["data_file", "file_format"],
1762 &["data_file", "partition"],
1763 &["data_file", "record_count"],
1764 &["data_file", "file_size_in_bytes"],
1765 ];
1766 assert_reads_without_each_field(
1767 &metadata,
1768 &full_schema,
1769 &full_value,
1770 &full_entry,
1771 &optional,
1772 &required,
1773 );
1774 }
1775
1776 #[test]
1777 fn test_parse_v1_manifest_without_each_field() {
1778 let metadata = writer_schema_test_metadata(FormatVersion::V1);
1779 let partition_type = metadata
1780 .partition_spec
1781 .partition_type(&metadata.schema)
1782 .unwrap();
1783 let full_schema =
1784 serde_json::to_value(manifest_schema_v1(&partition_type).unwrap()).unwrap();
1785 let mut full_entry = expected_entry(Struct::from_iter([
1786 Some(Literal::long(5)),
1787 Some(Literal::string("a")),
1788 Some(Literal::double(2.5)),
1789 ]));
1790 full_entry.sequence_number = Some(0);
1792 full_entry.file_sequence_number = Some(0);
1793 let data_file = &mut full_entry.data_file;
1794 data_file.column_sizes = HashMap::from([(1, 40)]);
1795 data_file.value_counts = HashMap::from([(1, 10)]);
1796 data_file.null_value_counts = HashMap::from([(1, 1)]);
1797 data_file.nan_value_counts = HashMap::from([(3, 2)]);
1798 data_file.lower_bounds = HashMap::from([(1, Datum::long(1))]);
1799 data_file.upper_bounds = HashMap::from([(1, Datum::long(9))]);
1800 data_file.key_metadata = Some(vec![1, 2]);
1801 data_file.split_offsets = Some(vec![4]);
1802 data_file.sort_order_id = Some(0);
1803 let full_value = to_value(
1804 _serde::ManifestEntryV1::try_from(
1805 full_entry.clone(),
1806 &Type::Struct(partition_type.clone()),
1807 )
1808 .unwrap(),
1809 )
1810 .unwrap();
1811
1812 let optional: [(&[&str], ResetField); 10] = [
1813 (&["data_file", "block_size_in_bytes"], |_| {}),
1815 (&["data_file", "column_sizes"], |e| {
1816 e.data_file.column_sizes.clear()
1817 }),
1818 (&["data_file", "value_counts"], |e| {
1819 e.data_file.value_counts.clear()
1820 }),
1821 (&["data_file", "null_value_counts"], |e| {
1822 e.data_file.null_value_counts.clear()
1823 }),
1824 (&["data_file", "nan_value_counts"], |e| {
1825 e.data_file.nan_value_counts.clear()
1826 }),
1827 (&["data_file", "lower_bounds"], |e| {
1828 e.data_file.lower_bounds.clear()
1829 }),
1830 (&["data_file", "upper_bounds"], |e| {
1831 e.data_file.upper_bounds.clear()
1832 }),
1833 (&["data_file", "key_metadata"], |e| {
1834 e.data_file.key_metadata = None
1835 }),
1836 (&["data_file", "split_offsets"], |e| {
1837 e.data_file.split_offsets = None
1838 }),
1839 (&["data_file", "sort_order_id"], |e| {
1840 e.data_file.sort_order_id = None
1841 }),
1842 ];
1843 let required: [&[&str]; 8] = [
1844 &["status"],
1845 &["snapshot_id"],
1846 &["data_file"],
1847 &["data_file", "file_path"],
1848 &["data_file", "file_format"],
1849 &["data_file", "partition"],
1850 &["data_file", "record_count"],
1851 &["data_file", "file_size_in_bytes"],
1852 ];
1853 assert_reads_without_each_field(
1854 &metadata,
1855 &full_schema,
1856 &full_value,
1857 &full_entry,
1858 &optional,
1859 &required,
1860 );
1861 }
1862
1863 #[test]
1864 fn test_parse_manifest_matches_partition_fields_by_name() {
1865 let metadata = writer_schema_test_metadata(FormatVersion::V2);
1868 let writer_schema = v2_writer_schema(serde_json::json!([
1869 {"name": "score", "type": ["null", "double"], "default": null, "field-id": 1002},
1870 {"name": "extra", "type": ["null", "int"], "default": null, "field-id": 1003},
1871 {"name": "id", "type": ["null", "long"], "default": null, "field-id": 1000},
1872 ]));
1873 let bs = write_with_writer_schema(&metadata, &writer_schema, vec![v2_entry(
1874 serde_json::json!({"score": 2.5, "extra": 9, "id": 5}),
1875 )]);
1876
1877 let manifest = Manifest::parse_avro(&bs).unwrap();
1878
1879 let partition =
1880 Struct::from_iter([Some(Literal::long(5)), None, Some(Literal::double(2.5))]);
1881 assert_eq!(
1882 manifest,
1883 Manifest::new(metadata, vec![expected_entry(partition)])
1884 );
1885 }
1886
1887 #[test]
1888 fn test_parse_manifest_promotes_writer_types() {
1889 let metadata = writer_schema_test_metadata(FormatVersion::V2);
1891 let mut writer_schema = v2_writer_schema(serde_json::json!([
1892 {"name": "id", "type": ["null", "int"], "default": null, "field-id": 1000},
1893 {"name": "name", "type": ["null", "string"], "default": null, "field-id": 1001},
1894 {"name": "score", "type": ["null", "float"], "default": null, "field-id": 1002},
1895 ]));
1896 for field in ["record_count", "file_size_in_bytes"] {
1897 writer_schema_field(&mut writer_schema, &["data_file", field])["type"] =
1898 serde_json::json!("int");
1899 }
1900 let bs = write_with_writer_schema(&metadata, &writer_schema, vec![v2_entry(
1901 serde_json::json!({"id": 5, "name": "a", "score": 2.5}),
1902 )]);
1903
1904 let manifest = Manifest::parse_avro(&bs).unwrap();
1905
1906 let partition = Struct::from_iter([
1907 Some(Literal::long(5)),
1908 Some(Literal::string("a")),
1909 Some(Literal::double(2.5)),
1910 ]);
1911 assert_eq!(
1912 manifest,
1913 Manifest::new(metadata, vec![expected_entry(partition)])
1914 );
1915 }
1916
1917 #[test]
1918 fn test_parse_manifest_reads_required_writer_fields_as_optional() {
1919 let metadata = writer_schema_test_metadata(FormatVersion::V2);
1922 let mut writer_schema = v2_writer_schema(serde_json::json!([
1923 {"name": "id", "type": "long", "field-id": 1000},
1924 {"name": "name", "type": "string", "field-id": 1001},
1925 {"name": "score", "type": "double", "field-id": 1002},
1926 ]));
1927 *writer_schema_field(&mut writer_schema, &["snapshot_id"]) =
1928 serde_json::json!({"name": "snapshot_id", "type": "long", "field-id": 1});
1929 writer_schema_field(&mut writer_schema, &["data_file"])["type"]["fields"]
1930 .as_array_mut()
1931 .unwrap()
1932 .push(serde_json::json!({"name": "sort_order_id", "type": "int", "field-id": 140}));
1933 let mut entry = v2_entry(serde_json::json!({"id": 5, "name": "a", "score": 2.5}));
1934 entry["data_file"]["sort_order_id"] = serde_json::json!(3);
1935 let bs = write_with_writer_schema(&metadata, &writer_schema, vec![entry]);
1936
1937 let manifest = Manifest::parse_avro(&bs).unwrap();
1938
1939 let mut expected = expected_entry(Struct::from_iter([
1940 Some(Literal::long(5)),
1941 Some(Literal::string("a")),
1942 Some(Literal::double(2.5)),
1943 ]));
1944 expected.data_file.sort_order_id = Some(3);
1945 assert_eq!(manifest, Manifest::new(metadata, vec![expected]));
1946 }
1947
1948 #[test]
1949 fn test_parse_manifest_reads_union_writer_fields_as_required() {
1950 let metadata = writer_schema_test_metadata(FormatVersion::V2);
1953 let mut writer_schema = v2_writer_schema(serde_json::json!([
1954 {"name": "id", "type": ["null", "long"], "default": null, "field-id": 1000},
1955 {"name": "name", "type": ["null", "string"], "default": null, "field-id": 1001},
1956 {"name": "score", "type": ["null", "double"], "default": null, "field-id": 1002},
1957 ]));
1958 writer_schema_field(&mut writer_schema, &["data_file", "record_count"])["type"] =
1959 serde_json::json!(["null", "long"]);
1960 let partition = serde_json::json!({"id": 5, "name": "a", "score": 2.5});
1961 let bs =
1962 write_with_writer_schema(&metadata, &writer_schema, vec![v2_entry(partition.clone())]);
1963
1964 let manifest = Manifest::parse_avro(&bs).unwrap();
1965
1966 let expected = expected_entry(Struct::from_iter([
1967 Some(Literal::long(5)),
1968 Some(Literal::string("a")),
1969 Some(Literal::double(2.5)),
1970 ]));
1971 assert_eq!(manifest, Manifest::new(metadata.clone(), vec![expected]));
1972
1973 let mut entry = v2_entry(partition);
1974 entry["data_file"]["record_count"] = Value::Null;
1975 let bs = write_with_writer_schema(&metadata, &writer_schema, vec![entry]);
1976
1977 let err = Manifest::parse_avro(&bs).unwrap_err();
1978
1979 assert_eq!(err.kind(), ErrorKind::DataInvalid);
1980 }
1981
1982 #[test]
1983 fn test_parse_manifest_with_fixed_uuid_partition() {
1984 let schema = Arc::new(
1986 Schema::builder()
1987 .with_fields(vec![
1988 NestedField::optional(1, "u", Type::Primitive(PrimitiveType::Uuid)).into(),
1989 ])
1990 .build()
1991 .unwrap(),
1992 );
1993 let partition_spec = PartitionSpec::builder(schema.clone())
1994 .with_spec_id(0)
1995 .add_partition_field("u", "u", Transform::Identity)
1996 .unwrap()
1997 .build()
1998 .unwrap();
1999 let metadata = ManifestMetadata {
2000 schema_id: 0,
2001 schema,
2002 partition_spec,
2003 content: ManifestContentType::Data,
2004 format_version: FormatVersion::V2,
2005 };
2006 let writer_schema = v2_writer_schema(serde_json::json!([
2007 {"name": "u", "default": null, "field-id": 1000, "type": ["null", {
2008 "type": "fixed", "name": "uuid_fixed", "size": 16, "logicalType": "uuid",
2009 }]},
2010 ]));
2011 let uuid = uuid::Uuid::from_u128(0xf79c3e09_677c_4bbd_a479_3f349cb785e7);
2012 let mut entry = AvroValue::try_from(v2_entry(serde_json::json!({}))).unwrap();
2015 let AvroValue::Map(fields) = &mut entry else {
2016 unreachable!("a JSON object converts to an Avro map");
2017 };
2018 let Some(AvroValue::Map(data_file)) = fields.get_mut("data_file") else {
2019 unreachable!("a JSON object converts to an Avro map");
2020 };
2021 data_file.insert(
2022 "partition".to_string(),
2023 AvroValue::Map(HashMap::from([("u".to_string(), AvroValue::Uuid(uuid))])),
2024 );
2025 let bs = write_avro_values_with_writer_schema(&metadata, &writer_schema, vec![entry]);
2026
2027 let manifest = Manifest::parse_avro(&bs).unwrap();
2028
2029 let expected = expected_entry(Struct::from_iter([Some(Literal::uuid(uuid))]));
2030 assert_eq!(manifest, Manifest::new(metadata, vec![expected]));
2031 }
2032
2033 #[test]
2034 fn test_parse_manifest_ignores_record_names_and_unknown_fields() {
2035 let metadata = writer_schema_test_metadata(FormatVersion::V2);
2038 let mut writer_schema = v2_writer_schema(serde_json::json!([
2039 {"name": "id", "type": ["null", "long"], "default": null, "field-id": 1000},
2040 {"name": "name", "type": ["null", "string"], "default": null, "field-id": 1001},
2041 {"name": "score", "type": ["null", "double"], "default": null, "field-id": 1002},
2042 ]));
2043 writer_schema["name"] = serde_json::json!("entry");
2044 writer_schema["fields"]
2045 .as_array_mut()
2046 .unwrap()
2047 .push(serde_json::json!({"name": "unknown", "type": "string"}));
2048 let data_file = &mut writer_schema_field(&mut writer_schema, &["data_file"])["type"];
2049 data_file["name"] = serde_json::json!("file");
2050 data_file["fields"][3]["type"]["name"] = serde_json::json!("part");
2051 let data_file_fields = data_file["fields"].as_array_mut().unwrap();
2052 data_file_fields.push(serde_json::json!({
2053 "name": "column_sizes",
2054 "field-id": 108,
2055 "default": null,
2056 "type": ["null", {
2057 "type": "array",
2058 "logicalType": "map",
2059 "items": {
2060 "type": "record",
2061 "name": "sizes",
2062 "fields": [
2063 {"name": "key", "type": "int", "field-id": 117},
2064 {"name": "value", "type": "long", "field-id": 118},
2065 ],
2066 },
2067 }],
2068 }));
2069 data_file_fields.push(
2070 serde_json::json!({"name": "block_size_in_bytes", "type": "long", "field-id": 105}),
2071 );
2072 let mut entry = v2_entry(serde_json::json!({"id": 5, "name": "a", "score": 2.5}));
2073 entry["unknown"] = serde_json::json!("ignored");
2074 entry["data_file"]["column_sizes"] = serde_json::json!([{"key": 1, "value": 40}]);
2075 entry["data_file"]["block_size_in_bytes"] = serde_json::json!(64);
2076 let bs = write_with_writer_schema(&metadata, &writer_schema, vec![entry]);
2077
2078 let manifest = Manifest::parse_avro(&bs).unwrap();
2079
2080 let mut expected = expected_entry(Struct::from_iter([
2081 Some(Literal::long(5)),
2082 Some(Literal::string("a")),
2083 Some(Literal::double(2.5)),
2084 ]));
2085 expected.data_file.column_sizes = HashMap::from([(1, 40)]);
2086 assert_eq!(manifest, Manifest::new(metadata, vec![expected]));
2087 }
2088
2089 #[test]
2090 fn test_parse_manifest_equality_ids_written_as_long() {
2091 let metadata = writer_schema_test_metadata(FormatVersion::V2);
2094 let mut writer_schema = v2_writer_schema(serde_json::json!([
2095 {"name": "id", "type": ["null", "long"], "default": null, "field-id": 1000},
2096 {"name": "name", "type": ["null", "string"], "default": null, "field-id": 1001},
2097 {"name": "score", "type": ["null", "double"], "default": null, "field-id": 1002},
2098 ]));
2099 writer_schema_field(&mut writer_schema, &["data_file"])["type"]["fields"]
2100 .as_array_mut()
2101 .unwrap()
2102 .push(serde_json::json!({
2103 "name": "equality_ids",
2104 "field-id": 135,
2105 "default": null,
2106 "type": ["null", {"type": "array", "items": "long", "element-id": 136}],
2107 }));
2108 let mut entry = v2_entry(serde_json::json!({"id": 5, "name": "a", "score": 2.5}));
2109 entry["data_file"]["equality_ids"] = serde_json::json!([1, 2]);
2110 let bs = write_with_writer_schema(&metadata, &writer_schema, vec![entry]);
2111
2112 let manifest = Manifest::parse_avro(&bs).unwrap();
2113
2114 let mut expected = expected_entry(Struct::from_iter([
2115 Some(Literal::long(5)),
2116 Some(Literal::string("a")),
2117 Some(Literal::double(2.5)),
2118 ]));
2119 expected.data_file.equality_ids = Some(vec![1, 2]);
2120 assert_eq!(manifest, Manifest::new(metadata, vec![expected]));
2121 }
2122
2123 #[test]
2124 fn test_parse_manifest_rejects_equality_id_above_int_range() {
2125 let metadata = writer_schema_test_metadata(FormatVersion::V2);
2126 let mut writer_schema = v2_writer_schema(serde_json::json!([
2127 {"name": "id", "type": ["null", "long"], "default": null, "field-id": 1000},
2128 {"name": "name", "type": ["null", "string"], "default": null, "field-id": 1001},
2129 {"name": "score", "type": ["null", "double"], "default": null, "field-id": 1002},
2130 ]));
2131 writer_schema_field(&mut writer_schema, &["data_file"])["type"]["fields"]
2132 .as_array_mut()
2133 .unwrap()
2134 .push(serde_json::json!({
2135 "name": "equality_ids",
2136 "field-id": 135,
2137 "default": null,
2138 "type": ["null", {"type": "array", "items": "long", "element-id": 136}],
2139 }));
2140 let mut entry = v2_entry(serde_json::json!({"id": 5, "name": "a", "score": 2.5}));
2141 entry["data_file"]["equality_ids"] = serde_json::json!([i64::from(i32::MAX) + 1]);
2142 let bs = write_with_writer_schema(&metadata, &writer_schema, vec![entry]);
2143
2144 let err = Manifest::parse_avro(&bs).unwrap_err();
2145
2146 assert_eq!(err.kind(), ErrorKind::DataInvalid);
2147 }
2148
2149 #[test]
2150 fn test_parse_manifest_written_by_pyiceberg() {
2151 let bs = fs::read(format!(
2152 "{}/testdata/manifests/pyiceberg-v2-data.avro",
2153 env!("CARGO_MANIFEST_DIR")
2154 ))
2155 .unwrap();
2156
2157 let manifest = Manifest::parse_avro(&bs).unwrap();
2158
2159 let schema = Arc::new(
2160 Schema::builder()
2161 .with_fields(vec![
2162 NestedField::optional(1, "id", Type::Primitive(PrimitiveType::Long)).into(),
2163 NestedField::optional(2, "category", Type::Primitive(PrimitiveType::String))
2164 .into(),
2165 NestedField::optional(3, "score", Type::Primitive(PrimitiveType::Double))
2166 .into(),
2167 ])
2168 .build()
2169 .unwrap(),
2170 );
2171 let partition_spec = PartitionSpec::builder(schema.clone())
2172 .with_spec_id(0)
2173 .add_partition_field("category", "category", Transform::Identity)
2174 .unwrap()
2175 .build()
2176 .unwrap();
2177 let metadata = ManifestMetadata {
2178 schema_id: 0,
2179 schema,
2180 partition_spec,
2181 content: ManifestContentType::Data,
2182 format_version: FormatVersion::V2,
2183 };
2184 let entries = (0..2)
2185 .map(|i: i64| {
2186 let category = format!("c{i}");
2187 ManifestEntry {
2188 status: ManifestStatus::Added,
2189 snapshot_id: Some(7),
2190 sequence_number: Some(1),
2191 file_sequence_number: Some(1),
2192 data_file: DataFile {
2193 content: DataContentType::Data,
2194 file_path: format!(
2195 "s3://bucket/table/data/category={category}/0000{i}.parquet"
2196 ),
2197 file_format: DataFileFormat::Parquet,
2198 partition: Struct::from_iter([Some(Literal::string(&category))]),
2199 record_count: 100 + i as u64,
2200 file_size_in_bytes: 1000 + i as u64,
2201 column_sizes: HashMap::from([
2202 (1, 10 + i as u64),
2203 (2, 20 + i as u64),
2204 (3, 30 + i as u64),
2205 ]),
2206 value_counts: HashMap::from([
2207 (1, 100 + i as u64),
2208 (2, 100 + i as u64),
2209 (3, 100 + i as u64),
2210 ]),
2211 null_value_counts: HashMap::from([(1, 0), (2, i as u64), (3, 1)]),
2212 nan_value_counts: HashMap::from([(3, i as u64)]),
2213 lower_bounds: HashMap::from([
2214 (1, Datum::long(i)),
2215 (2, Datum::string(&category)),
2216 ]),
2217 upper_bounds: HashMap::from([
2218 (1, Datum::long(i + 50)),
2219 (2, Datum::string(&category)),
2220 ]),
2221 key_metadata: None,
2222 split_offsets: Some(vec![4]),
2223 equality_ids: None,
2224 sort_order_id: Some(0),
2225 partition_spec_id: 0,
2226 first_row_id: None,
2227 referenced_data_file: None,
2228 content_offset: None,
2229 content_size_in_bytes: None,
2230 },
2231 }
2232 })
2233 .collect();
2234 assert_eq!(manifest, Manifest::new(metadata, entries));
2235 }
2236
2237 #[test]
2238 fn test_parse_manifest_with_repeated_named_type_definitions() {
2239 let bs = fs::read(format!(
2240 "{}/testdata/manifests/repeated-decimal-type-definitions.avro",
2241 env!("CARGO_MANIFEST_DIR")
2242 ))
2243 .unwrap();
2244
2245 let manifest = Manifest::parse_avro(&bs).unwrap();
2246
2247 assert_eq!(
2248 *manifest.entries()[0].data_file().partition(),
2249 Struct::from_iter([Some(Literal::decimal(12345)), Some(Literal::decimal(-678))])
2250 );
2251 }
2252
2253 #[test]
2254 fn test_parse_manifest_with_repeated_named_type_definitions_reports_original_error() {
2255 let bs = fs::read(format!(
2256 "{}/testdata/manifests/repeated-decimal-type-definitions.avro",
2257 env!("CARGO_MANIFEST_DIR")
2258 ))
2259 .unwrap();
2260 let marker = &bs[bs.len() - 16..];
2263 let header_marker = bs.windows(16).position(|w| w == marker).unwrap();
2264 let truncated = &bs[..header_marker + 8];
2265 let log_dir = TempDir::new().unwrap();
2266 let log_path = log_dir.path().join("log");
2267 let subscriber = tracing_subscriber::fmt()
2268 .with_max_level(Level::DEBUG)
2269 .with_writer(Arc::new(fs::File::create(&log_path).unwrap()))
2270 .finish();
2271
2272 let err = with_default(subscriber, || Manifest::parse_avro(truncated)).unwrap_err();
2273
2274 assert_eq!(err.kind(), ErrorKind::DataInvalid);
2275 let message = err.to_string();
2276 assert!(
2277 message.contains("Two named schema defined for same fullname"),
2278 "{message}"
2279 );
2280 assert!(message.contains("Failed to read marker bytes"), "{message}");
2281 let log = fs::read_to_string(&log_path).unwrap();
2283 assert!(!log.contains("more than once"), "{log}");
2284 }
2285}