1use std::cmp::min;
19
20use apache_avro::to_value;
21use bytes::Bytes;
22use itertools::Itertools;
23use serde_json::to_vec;
24
25use super::{
26 Datum, FormatVersion, ManifestContentType, PartitionSpec, PrimitiveType,
27 UNASSIGNED_SEQUENCE_NUMBER,
28};
29use crate::avro::new_writer;
30use crate::encryption::EncryptedOutputFile;
31use crate::error::{Result, invalid_data};
32use crate::io::{FileMetadata, FileWrite, OutputFile};
33use crate::spec::manifest::_serde::{ManifestEntryV1, ManifestEntryV2, ManifestEntryV3};
34use crate::spec::manifest::{manifest_schema_v1, manifest_schema_v2, manifest_schema_v3};
35use crate::spec::{
36 DataContentType, DataFile, FieldSummary, ManifestEntry, ManifestFile, ManifestMetadata,
37 ManifestStatus, PrimitiveLiteral, SchemaRef, StructType, Type,
38};
39
40const UNASSIGNED_SNAPSHOT_ID: i64 = -1;
43
44pub(crate) enum ManifestOutput {
46 Plain(OutputFile),
47 Encrypted(EncryptedOutputFile),
48}
49
50impl ManifestOutput {
51 async fn writer(&self) -> Result<Box<dyn FileWrite>> {
52 match self {
53 Self::Plain(output) => output.writer().await,
54 Self::Encrypted(output) => output.writer().await,
55 }
56 }
57
58 fn encoded_key_metadata(&self, file_metadata: &FileMetadata) -> Result<Option<Vec<u8>>> {
59 match self {
60 Self::Plain(_) => Ok(None),
61 Self::Encrypted(output) => Ok(Some(
62 output
63 .key_metadata_with_saved_file_metadata(file_metadata)
64 .encode()?
65 .into_vec(),
66 )),
67 }
68 }
69}
70
71pub struct ManifestWriterBuilder {
73 output: ManifestOutput,
74 location: String,
75 snapshot_id: Option<i64>,
76 schema: SchemaRef,
77 partition_spec: PartitionSpec,
78}
79
80impl ManifestWriterBuilder {
81 pub fn new(
83 output: OutputFile,
84 snapshot_id: Option<i64>,
85 schema: SchemaRef,
86 partition_spec: PartitionSpec,
87 ) -> Self {
88 let location = output.location().to_owned();
89 Self {
90 output: ManifestOutput::Plain(output),
91 location,
92 snapshot_id,
93 schema,
94 partition_spec,
95 }
96 }
97
98 pub fn new_from_encrypted(
102 encrypted_output: EncryptedOutputFile,
103 snapshot_id: Option<i64>,
104 schema: SchemaRef,
105 partition_spec: PartitionSpec,
106 ) -> Result<Self> {
107 let location = encrypted_output.location().to_owned();
108 Ok(Self {
109 output: ManifestOutput::Encrypted(encrypted_output),
110 location,
111 snapshot_id,
112 schema,
113 partition_spec,
114 })
115 }
116
117 pub fn build_v1(self) -> ManifestWriter {
119 let metadata = ManifestMetadata::builder()
120 .schema_id(self.schema.schema_id())
121 .schema(self.schema)
122 .partition_spec(self.partition_spec)
123 .format_version(FormatVersion::V1)
124 .content(ManifestContentType::Data)
125 .build();
126 ManifestWriter::new(self.output, self.location, self.snapshot_id, metadata, None)
127 }
128
129 pub fn build_v2_data(self) -> ManifestWriter {
131 let metadata = ManifestMetadata::builder()
132 .schema_id(self.schema.schema_id())
133 .schema(self.schema)
134 .partition_spec(self.partition_spec)
135 .format_version(FormatVersion::V2)
136 .content(ManifestContentType::Data)
137 .build();
138
139 ManifestWriter::new(self.output, self.location, self.snapshot_id, metadata, None)
140 }
141
142 pub fn build_v2_deletes(self) -> ManifestWriter {
144 let metadata = ManifestMetadata::builder()
145 .schema_id(self.schema.schema_id())
146 .schema(self.schema)
147 .partition_spec(self.partition_spec)
148 .format_version(FormatVersion::V2)
149 .content(ManifestContentType::Deletes)
150 .build();
151 ManifestWriter::new(self.output, self.location, self.snapshot_id, metadata, None)
152 }
153
154 pub fn build_v3_data(self) -> ManifestWriter {
156 let metadata = ManifestMetadata::builder()
157 .schema_id(self.schema.schema_id())
158 .schema(self.schema)
159 .partition_spec(self.partition_spec)
160 .format_version(FormatVersion::V3)
161 .content(ManifestContentType::Data)
162 .build();
163 ManifestWriter::new(
164 self.output,
165 self.location,
166 self.snapshot_id,
167 metadata,
168 None,
171 )
172 }
173
174 pub fn build_v3_deletes(self) -> ManifestWriter {
176 let metadata = ManifestMetadata::builder()
177 .schema_id(self.schema.schema_id())
178 .schema(self.schema)
179 .partition_spec(self.partition_spec)
180 .format_version(FormatVersion::V3)
181 .content(ManifestContentType::Deletes)
182 .build();
183 ManifestWriter::new(self.output, self.location, self.snapshot_id, metadata, None)
184 }
185}
186
187pub struct ManifestWriter {
189 output: ManifestOutput,
190 location: String,
191
192 snapshot_id: Option<i64>,
193
194 added_files: u32,
195 added_rows: u64,
196 existing_files: u32,
197 existing_rows: u64,
198 deleted_files: u32,
199 deleted_rows: u64,
200 first_row_id: Option<u64>,
201
202 min_seq_num: Option<i64>,
203
204 manifest_entries: Vec<ManifestEntry>,
205
206 metadata: ManifestMetadata,
207}
208
209impl ManifestWriter {
210 pub(crate) fn new(
212 output: ManifestOutput,
213 location: String,
214 snapshot_id: Option<i64>,
215 metadata: ManifestMetadata,
216 first_row_id: Option<u64>,
217 ) -> Self {
218 Self {
219 output,
220 location,
221 snapshot_id,
222 added_files: 0,
223 added_rows: 0,
224 existing_files: 0,
225 existing_rows: 0,
226 deleted_files: 0,
227 deleted_rows: 0,
228 first_row_id,
229 min_seq_num: None,
230 manifest_entries: Vec::new(),
231 metadata,
232 }
233 }
234
235 fn construct_partition_summaries(
236 &mut self,
237 partition_type: &StructType,
238 ) -> Result<Vec<FieldSummary>> {
239 let mut field_stats: Vec<_> = partition_type
240 .fields()
241 .iter()
242 .map(|f| PartitionFieldStats::new(f.field_type.as_primitive_type().unwrap().clone()))
243 .collect();
244 for partition in self.manifest_entries.iter().map(|e| &e.data_file.partition) {
245 for (literal, stat) in partition.iter().zip_eq(field_stats.iter_mut()) {
246 let primitive_literal = literal.map(|v| v.as_primitive_literal().unwrap());
247 stat.update(primitive_literal)?;
248 }
249 }
250 Ok(field_stats.into_iter().map(|stat| stat.finish()).collect())
251 }
252
253 fn check_data_file(&self, data_file: &DataFile) -> Result<()> {
254 match self.metadata.content {
255 ManifestContentType::Data => {
256 if data_file.content != DataContentType::Data {
257 return Err(invalid_data!(
258 "Date file at path {} with manifest content type `data`, should have DataContentType `Data`, but has `{:?}`",
259 data_file.file_path(),
260 data_file.content
261 ));
262 }
263 }
264 ManifestContentType::Deletes => {
265 if data_file.content != DataContentType::EqualityDeletes
266 && data_file.content != DataContentType::PositionDeletes
267 {
268 return Err(invalid_data!(
269 "Date file at path {} with manifest content type `deletes`, should have DataContentType `Data`, but has `{:?}`",
270 data_file.file_path(),
271 data_file.content
272 ));
273 }
274 }
275 }
276 Ok(())
277 }
278
279 pub(crate) fn add_entry(&mut self, mut entry: ManifestEntry) -> Result<()> {
285 self.check_data_file(&entry.data_file)?;
286 if entry.sequence_number().is_some_and(|n| n >= 0) {
287 entry.status = ManifestStatus::Added;
288 entry.snapshot_id = self.snapshot_id;
289 entry.file_sequence_number = None;
290 } else {
291 entry.status = ManifestStatus::Added;
292 entry.snapshot_id = self.snapshot_id;
293 entry.sequence_number = None;
294 entry.file_sequence_number = None;
295 };
296 self.add_entry_inner(entry)?;
297 Ok(())
298 }
299
300 pub fn add_file(&mut self, data_file: DataFile, sequence_number: i64) -> Result<()> {
304 self.check_data_file(&data_file)?;
305 let entry = ManifestEntry {
306 status: ManifestStatus::Added,
307 snapshot_id: self.snapshot_id,
308 sequence_number: (sequence_number >= 0).then_some(sequence_number),
309 file_sequence_number: None,
310 data_file,
311 };
312 self.add_entry_inner(entry)?;
313 Ok(())
314 }
315
316 #[allow(dead_code)]
323 pub(crate) fn add_delete_entry(&mut self, mut entry: ManifestEntry) -> Result<()> {
324 self.check_data_file(&entry.data_file)?;
325 entry.status = ManifestStatus::Deleted;
326 entry.snapshot_id = self.snapshot_id;
327 self.add_entry_inner(entry)?;
328 Ok(())
329 }
330
331 pub fn add_delete_file(
335 &mut self,
336 data_file: DataFile,
337 sequence_number: i64,
338 file_sequence_number: Option<i64>,
339 ) -> Result<()> {
340 self.check_data_file(&data_file)?;
341 let entry = ManifestEntry {
342 status: ManifestStatus::Deleted,
343 snapshot_id: self.snapshot_id,
344 sequence_number: Some(sequence_number),
345 file_sequence_number,
346 data_file,
347 };
348 self.add_entry_inner(entry)?;
349 Ok(())
350 }
351
352 #[allow(dead_code)]
358 pub(crate) fn add_existing_entry(&mut self, mut entry: ManifestEntry) -> Result<()> {
359 self.check_data_file(&entry.data_file)?;
360 entry.status = ManifestStatus::Existing;
361 self.add_entry_inner(entry)?;
362 Ok(())
363 }
364
365 pub fn add_existing_file(
368 &mut self,
369 data_file: DataFile,
370 snapshot_id: i64,
371 sequence_number: i64,
372 file_sequence_number: Option<i64>,
373 ) -> Result<()> {
374 self.check_data_file(&data_file)?;
375 let entry = ManifestEntry {
376 status: ManifestStatus::Existing,
377 snapshot_id: Some(snapshot_id),
378 sequence_number: Some(sequence_number),
379 file_sequence_number,
380 data_file,
381 };
382 self.add_entry_inner(entry)?;
383 Ok(())
384 }
385
386 fn add_entry_inner(&mut self, entry: ManifestEntry) -> Result<()> {
387 if (entry.status == ManifestStatus::Deleted || entry.status == ManifestStatus::Existing)
389 && (entry.sequence_number.is_none() || entry.file_sequence_number.is_none())
390 {
391 return Err(invalid_data!(
392 "Manifest entry with status Existing or Deleted should have sequence number"
393 ));
394 }
395
396 match entry.status {
398 ManifestStatus::Added => {
399 self.added_files += 1;
400 self.added_rows += entry.data_file.record_count;
401 }
402 ManifestStatus::Deleted => {
403 self.deleted_files += 1;
404 self.deleted_rows += entry.data_file.record_count;
405 }
406 ManifestStatus::Existing => {
407 self.existing_files += 1;
408 self.existing_rows += entry.data_file.record_count;
409 }
410 }
411 if entry.is_alive()
412 && let Some(seq_num) = entry.sequence_number
413 {
414 self.min_seq_num = Some(self.min_seq_num.map_or(seq_num, |v| min(v, seq_num)));
415 }
416 self.manifest_entries.push(entry);
417 Ok(())
418 }
419
420 pub async fn write_manifest_file(mut self) -> Result<ManifestFile> {
422 let partition_type = self
424 .metadata
425 .partition_spec
426 .partition_type(&self.metadata.schema)?;
427 let partition_struct_type = Type::Struct(partition_type.clone());
430 let table_schema = &self.metadata.schema;
431 let avro_schema = match self.metadata.format_version {
432 FormatVersion::V1 => manifest_schema_v1(&partition_type)?,
433 FormatVersion::V2 => manifest_schema_v2(&partition_type)?,
434 FormatVersion::V3 => manifest_schema_v3(&partition_type)?,
435 };
436 let mut avro_writer = new_writer(&avro_schema, Vec::new())?;
437 avro_writer.add_user_metadata(
438 "schema".to_string(),
439 to_vec(table_schema)
440 .map_err(|err| invalid_data!("Fail to serialize table schema").with_source(err))?,
441 )?;
442 avro_writer.add_user_metadata(
443 "schema-id".to_string(),
444 table_schema.schema_id().to_string(),
445 )?;
446 avro_writer.add_user_metadata(
447 "partition-spec".to_string(),
448 to_vec(&self.metadata.partition_spec.fields()).map_err(|err| {
449 invalid_data!("Fail to serialize partition spec").with_source(err)
450 })?,
451 )?;
452 avro_writer.add_user_metadata(
453 "partition-spec-id".to_string(),
454 self.metadata.partition_spec.spec_id().to_string(),
455 )?;
456 avro_writer.add_user_metadata(
457 "format-version".to_string(),
458 (self.metadata.format_version as u8).to_string(),
459 )?;
460 match self.metadata.format_version {
461 FormatVersion::V1 => {}
462 FormatVersion::V2 | FormatVersion::V3 => {
463 avro_writer
464 .add_user_metadata("content".to_string(), self.metadata.content.to_string())?;
465 }
466 }
467
468 let partition_summary = self.construct_partition_summaries(&partition_type)?;
469 for entry in std::mem::take(&mut self.manifest_entries) {
471 let value = match self.metadata.format_version {
472 FormatVersion::V1 => {
473 to_value(ManifestEntryV1::try_from(entry, &partition_struct_type)?)?
474 .resolve(&avro_schema)?
475 }
476 FormatVersion::V2 => {
477 to_value(ManifestEntryV2::try_from(entry, &partition_struct_type)?)?
478 .resolve(&avro_schema)?
479 }
480 FormatVersion::V3 => {
481 to_value(ManifestEntryV3::try_from(entry, &partition_struct_type)?)?
482 .resolve(&avro_schema)?
483 }
484 };
485
486 avro_writer.append_value(value)?;
487 }
488
489 let content = avro_writer.into_inner()?;
490 let mut writer = self.output.writer().await?;
491 writer.write(Bytes::from(content)).await?;
492 let file_metadata = writer.close().await?;
493 let key_metadata = self.output.encoded_key_metadata(&file_metadata)?;
494
495 Ok(ManifestFile {
496 manifest_path: self.location,
497 manifest_length: file_metadata.size.try_into()?,
499 partition_spec_id: self.metadata.partition_spec.spec_id(),
500 content: self.metadata.content,
501 sequence_number: UNASSIGNED_SEQUENCE_NUMBER,
504 min_sequence_number: self.min_seq_num.unwrap_or(UNASSIGNED_SEQUENCE_NUMBER),
505 added_snapshot_id: self.snapshot_id.unwrap_or(UNASSIGNED_SNAPSHOT_ID),
506 added_files_count: Some(self.added_files),
507 existing_files_count: Some(self.existing_files),
508 deleted_files_count: Some(self.deleted_files),
509 added_rows_count: Some(self.added_rows),
510 existing_rows_count: Some(self.existing_rows),
511 deleted_rows_count: Some(self.deleted_rows),
512 partitions: Some(partition_summary),
513 key_metadata,
514 first_row_id: self.first_row_id,
515 })
516 }
517}
518
519struct PartitionFieldStats {
520 partition_type: PrimitiveType,
521
522 contains_null: bool,
523 contains_nan: Option<bool>,
524 lower_bound: Option<Datum>,
525 upper_bound: Option<Datum>,
526}
527
528impl PartitionFieldStats {
529 pub(crate) fn new(partition_type: PrimitiveType) -> Self {
530 Self {
531 partition_type,
532 contains_null: false,
533 contains_nan: Some(false),
534 upper_bound: None,
535 lower_bound: None,
536 }
537 }
538
539 pub(crate) fn update(&mut self, value: Option<PrimitiveLiteral>) -> Result<()> {
540 let Some(value) = value else {
541 self.contains_null = true;
542 return Ok(());
543 };
544 if !self.partition_type.compatible(&value) {
545 return Err(invalid_data!("value is not compatible with type"));
546 }
547 let value = Datum::new(self.partition_type.clone(), value);
548
549 if value.is_nan() {
550 self.contains_nan = Some(true);
551 return Ok(());
552 }
553
554 self.lower_bound = Some(self.lower_bound.take().map_or(value.clone(), |original| {
555 if value < original {
556 value.clone()
557 } else {
558 original
559 }
560 }));
561 self.upper_bound = Some(self.upper_bound.take().map_or(value.clone(), |original| {
562 if value > original { value } else { original }
563 }));
564
565 Ok(())
566 }
567
568 pub(crate) fn finish(self) -> FieldSummary {
569 FieldSummary {
570 contains_null: self.contains_null,
571 contains_nan: self.contains_nan,
572 upper_bound: self.upper_bound.map(|v| v.to_bytes().unwrap()),
573 lower_bound: self.lower_bound.map(|v| v.to_bytes().unwrap()),
574 }
575 }
576}
577
578#[cfg(test)]
579mod tests {
580 use std::collections::HashMap;
581 use std::fs;
582 use std::sync::Arc;
583
584 use apache_avro::Reader as AvroReader;
585 use apache_avro::schema::Schema as AvroSchema;
586 use tempfile::TempDir;
587
588 use super::*;
589 use crate::avro::{assert_deflate_codec, define_named_types_repeatedly};
590 use crate::io::FileIO;
591 use crate::spec::{
592 DataContentType, DataFileBuilder, DataFileFormat, Literal, Manifest, NestedField,
593 PrimitiveType, Schema, Struct, Transform, Type,
594 };
595
596 #[tokio::test]
597 async fn test_add_delete_existing() {
598 let schema = Arc::new(
599 Schema::builder()
600 .with_fields(vec![
601 Arc::new(NestedField::optional(
602 1,
603 "id",
604 Type::Primitive(PrimitiveType::Int),
605 )),
606 Arc::new(NestedField::optional(
607 2,
608 "name",
609 Type::Primitive(PrimitiveType::String),
610 )),
611 ])
612 .build()
613 .unwrap(),
614 );
615 let metadata = ManifestMetadata {
616 schema_id: 0,
617 schema: schema.clone(),
618 partition_spec: PartitionSpec::builder(schema)
619 .with_spec_id(0)
620 .build()
621 .unwrap(),
622 content: ManifestContentType::Data,
623 format_version: FormatVersion::V2,
624 };
625 let mut entries = vec![
626 ManifestEntry {
627 status: ManifestStatus::Added,
628 snapshot_id: None,
629 sequence_number: Some(1),
630 file_sequence_number: Some(1),
631 data_file: DataFile {
632 content: DataContentType::Data,
633 file_path: "s3a://icebergdata/demo/s1/t1/data/00000-0-ba56fbfa-f2ff-40c9-bb27-565ad6dc2be8-00000.parquet".to_string(),
634 file_format: DataFileFormat::Parquet,
635 partition: Struct::empty(),
636 record_count: 1,
637 file_size_in_bytes: 5442,
638 column_sizes: HashMap::from([(1, 61), (2, 73)]),
639 value_counts: HashMap::from([(1, 1), (2, 1)]),
640 null_value_counts: HashMap::from([(1, 0), (2, 0)]),
641 nan_value_counts: HashMap::new(),
642 lower_bounds: HashMap::new(),
643 upper_bounds: HashMap::new(),
644 key_metadata: Some(Vec::new()),
645 split_offsets: Some(vec![4]),
646 equality_ids: None,
647 sort_order_id: None,
648 partition_spec_id: 0,
649 first_row_id: None,
650 referenced_data_file: None,
651 content_offset: None,
652 content_size_in_bytes: None,
653 },
654 },
655 ManifestEntry {
656 status: ManifestStatus::Deleted,
657 snapshot_id: Some(1),
658 sequence_number: Some(1),
659 file_sequence_number: Some(1),
660 data_file: DataFile {
661 content: DataContentType::Data,
662 file_path: "s3a://icebergdata/demo/s1/t1/data/00000-0-ba56fbfa-f2ff-40c9-bb27-565ad6dc2be8-00000.parquet".to_string(),
663 file_format: DataFileFormat::Parquet,
664 partition: Struct::empty(),
665 record_count: 1,
666 file_size_in_bytes: 5442,
667 column_sizes: HashMap::from([(1, 61), (2, 73)]),
668 value_counts: HashMap::from([(1, 1), (2, 1)]),
669 null_value_counts: HashMap::from([(1, 0), (2, 0)]),
670 nan_value_counts: HashMap::new(),
671 lower_bounds: HashMap::new(),
672 upper_bounds: HashMap::new(),
673 key_metadata: Some(Vec::new()),
674 split_offsets: Some(vec![4]),
675 equality_ids: None,
676 sort_order_id: None,
677 partition_spec_id: 0,
678 first_row_id: None,
679 referenced_data_file: None,
680 content_offset: None,
681 content_size_in_bytes: None,
682 },
683 },
684 ManifestEntry {
685 status: ManifestStatus::Existing,
686 snapshot_id: Some(1),
687 sequence_number: Some(1),
688 file_sequence_number: Some(1),
689 data_file: DataFile {
690 content: DataContentType::Data,
691 file_path: "s3a://icebergdata/demo/s1/t1/data/00000-0-ba56fbfa-f2ff-40c9-bb27-565ad6dc2be8-00000.parquet".to_string(),
692 file_format: DataFileFormat::Parquet,
693 partition: Struct::empty(),
694 record_count: 1,
695 file_size_in_bytes: 5442,
696 column_sizes: HashMap::from([(1, 61), (2, 73)]),
697 value_counts: HashMap::from([(1, 1), (2, 1)]),
698 null_value_counts: HashMap::from([(1, 0), (2, 0)]),
699 nan_value_counts: HashMap::new(),
700 lower_bounds: HashMap::new(),
701 upper_bounds: HashMap::new(),
702 key_metadata: Some(Vec::new()),
703 split_offsets: Some(vec![4]),
704 equality_ids: None,
705 sort_order_id: None,
706 partition_spec_id: 0,
707 first_row_id: None,
708 referenced_data_file: None,
709 content_offset: None,
710 content_size_in_bytes: None,
711 },
712 },
713 ];
714
715 let tmp_dir = TempDir::new().unwrap();
717 let path = tmp_dir.path().join("test_manifest.avro");
718 let io = FileIO::new_with_fs();
719 let output_file = io.new_output(path.to_str().unwrap()).unwrap();
720 let mut writer = ManifestWriterBuilder::new(
721 output_file,
722 Some(3),
723 metadata.schema.clone(),
724 metadata.partition_spec.clone(),
725 )
726 .build_v2_data();
727 writer.add_entry(entries[0].clone()).unwrap();
728 writer.add_delete_entry(entries[1].clone()).unwrap();
729 writer.add_existing_entry(entries[2].clone()).unwrap();
730 writer.write_manifest_file().await.unwrap();
731
732 let bytes = fs::read(path).expect("read_file must succeed");
734 assert_deflate_codec(&bytes);
735 let actual_manifest = Manifest::parse_avro(&bytes).unwrap();
736
737 entries[0].snapshot_id = Some(3);
739 entries[1].snapshot_id = Some(3);
740 entries[0].file_sequence_number = None;
742 assert_eq!(actual_manifest, Manifest::new(metadata, entries));
743 }
744
745 #[tokio::test]
746 async fn test_v3_delete_manifest_delete_file_roundtrip() {
747 let schema = Arc::new(
748 Schema::builder()
749 .with_fields(vec![
750 Arc::new(NestedField::optional(
751 1,
752 "id",
753 Type::Primitive(PrimitiveType::Long),
754 )),
755 Arc::new(NestedField::optional(
756 2,
757 "data",
758 Type::Primitive(PrimitiveType::String),
759 )),
760 ])
761 .build()
762 .unwrap(),
763 );
764
765 let partition_spec = PartitionSpec::builder(schema.clone())
766 .with_spec_id(0)
767 .build()
768 .unwrap();
769
770 let delete_entry = ManifestEntry {
772 status: ManifestStatus::Added,
773 snapshot_id: None,
774 sequence_number: None,
775 file_sequence_number: None,
776 data_file: DataFile {
777 content: DataContentType::PositionDeletes,
778 file_path: "s3://bucket/table/data/delete-00000.parquet".to_string(),
779 file_format: DataFileFormat::Parquet,
780 partition: Struct::empty(),
781 record_count: 10,
782 file_size_in_bytes: 1024,
783 column_sizes: HashMap::new(),
784 value_counts: HashMap::new(),
785 null_value_counts: HashMap::new(),
786 nan_value_counts: HashMap::new(),
787 lower_bounds: HashMap::new(),
788 upper_bounds: HashMap::new(),
789 key_metadata: None,
790 split_offsets: None,
791 equality_ids: None,
792 sort_order_id: None,
793 partition_spec_id: 0,
794 first_row_id: None,
795 referenced_data_file: None,
796 content_offset: None,
797 content_size_in_bytes: None,
798 },
799 };
800
801 let tmp_dir = TempDir::new().unwrap();
803 let path = tmp_dir.path().join("v3_delete_manifest.avro");
804 let io = FileIO::new_with_fs();
805 let output_file = io.new_output(path.to_str().unwrap()).unwrap();
806
807 let mut writer = ManifestWriterBuilder::new(
808 output_file,
809 Some(1),
810 schema.clone(),
811 partition_spec.clone(),
812 )
813 .build_v3_deletes();
814
815 writer.add_entry(delete_entry).unwrap();
816 let manifest_file = writer.write_manifest_file().await.unwrap();
817
818 assert_eq!(manifest_file.content, ManifestContentType::Deletes);
820
821 let bytes = fs::read(&path).expect("read_file must succeed");
823 assert_deflate_codec(&bytes);
824 assert_eq!(manifest_file.manifest_length, bytes.len() as i64);
825 let actual_manifest = Manifest::parse_avro(&bytes).unwrap();
826
827 assert_eq!(
829 actual_manifest.metadata().content,
830 ManifestContentType::Deletes,
831 );
832 }
833
834 async fn roundtrip_two_partition_fields_of_type(
837 field_type: Type,
838 values: [Literal; 2],
839 ) -> Vec<u8> {
840 let schema = Arc::new(
841 Schema::builder()
842 .with_fields(vec![
843 NestedField::optional(1, "a", field_type.clone()).into(),
844 NestedField::optional(2, "b", field_type).into(),
845 ])
846 .build()
847 .unwrap(),
848 );
849 let partition_spec = PartitionSpec::builder(schema.clone())
850 .with_spec_id(0)
851 .add_partition_field("a", "a", Transform::Identity)
852 .unwrap()
853 .add_partition_field("b", "b", Transform::Identity)
854 .unwrap()
855 .build()
856 .unwrap();
857 let partition = Struct::from_iter(values.map(Some));
858 let tmp_dir = TempDir::new().unwrap();
859 let path = tmp_dir.path().join("manifest.avro");
860 let output_file = FileIO::new_with_fs()
861 .new_output(path.to_str().unwrap())
862 .unwrap();
863 let mut writer = ManifestWriterBuilder::new(output_file, Some(1), schema, partition_spec)
864 .build_v2_data();
865 writer
866 .add_file(
867 DataFileBuilder::default()
868 .content(DataContentType::Data)
869 .file_path("s3://bucket/table/data/a.parquet".to_string())
870 .file_format(DataFileFormat::Parquet)
871 .partition(partition.clone())
872 .record_count(1)
873 .file_size_in_bytes(1)
874 .partition_spec_id(0)
875 .build()
876 .unwrap(),
877 1,
878 )
879 .unwrap();
880 writer.write_manifest_file().await.unwrap();
881 let bs = fs::read(&path).unwrap();
882
883 for bs in [bs.clone(), define_named_types_repeatedly(&bs)] {
886 let manifest = Manifest::parse_avro(&bs).unwrap();
887
888 assert_eq!(*manifest.entries()[0].data_file().partition(), partition);
889 }
890 bs
891 }
892
893 #[tokio::test]
894 async fn test_write_manifest_header_marks_int_keyed_maps() {
895 let bs = roundtrip_two_partition_fields_of_type(Type::Primitive(PrimitiveType::Int), [
900 Literal::int(1),
901 Literal::int(2),
902 ])
903 .await;
904
905 let file = String::from_utf8_lossy(&bs);
906 assert_eq!(file.matches(r#""logicalType":"map""#).count(), 6);
909 }
910
911 #[tokio::test]
912 async fn test_write_manifest_with_repeated_decimal_partition_type() {
913 roundtrip_two_partition_fields_of_type(
914 Type::Primitive(PrimitiveType::Decimal {
915 precision: 10,
916 scale: 2,
917 }),
918 [Literal::decimal(12345), Literal::decimal(-678)],
919 )
920 .await;
921 }
922
923 #[tokio::test]
924 async fn test_write_manifest_with_repeated_fixed_partition_type() {
925 roundtrip_two_partition_fields_of_type(Type::Primitive(PrimitiveType::Fixed(4)), [
926 Literal::fixed(vec![1, 2, 3, 4]),
927 Literal::fixed(vec![5, 6, 7, 8]),
928 ])
929 .await;
930 }
931
932 async fn written_data_file_field_names(
935 build: fn(ManifestWriterBuilder) -> ManifestWriter,
936 ) -> Vec<String> {
937 let schema = Arc::new(
938 Schema::builder()
939 .with_fields(vec![
940 NestedField::optional(1, "id", Type::Primitive(PrimitiveType::Int)).into(),
941 ])
942 .build()
943 .unwrap(),
944 );
945 let partition_spec = PartitionSpec::builder(schema.clone())
946 .with_spec_id(0)
947 .build()
948 .unwrap();
949 let tmp_dir = TempDir::new().unwrap();
950 let path = tmp_dir.path().join("manifest.avro");
951 let output_file = FileIO::new_with_fs()
952 .new_output(path.to_str().unwrap())
953 .unwrap();
954 let mut writer = build(ManifestWriterBuilder::new(
955 output_file,
956 Some(1),
957 schema,
958 partition_spec,
959 ));
960 writer
961 .add_file(
962 DataFileBuilder::default()
963 .content(DataContentType::Data)
964 .file_path("s3://bucket/table/data/a.parquet".to_string())
965 .file_format(DataFileFormat::Parquet)
966 .partition(Struct::empty())
967 .record_count(1)
968 .file_size_in_bytes(1)
969 .partition_spec_id(0)
970 .build()
971 .unwrap(),
972 1,
973 )
974 .unwrap();
975 writer.write_manifest_file().await.unwrap();
976 let bs = fs::read(&path).unwrap();
977
978 let reader = AvroReader::new(&bs[..]).unwrap();
979 let AvroSchema::Record(entry) = reader.writer_schema() else {
980 panic!("manifest_entry is not a record");
981 };
982 let data_file_schema = &entry.fields[entry.lookup["data_file"]].schema;
983 let AvroSchema::Record(data_file) = data_file_schema else {
984 panic!("data_file is not a record");
985 };
986 data_file.fields.iter().map(|f| f.name.clone()).collect()
987 }
988
989 #[tokio::test]
990 async fn test_v1_manifest_data_file_schema() {
991 let field_names = written_data_file_field_names(ManifestWriterBuilder::build_v1).await;
992 assert_eq!(field_names, vec![
993 "file_path",
994 "file_format",
995 "partition",
996 "record_count",
997 "file_size_in_bytes",
998 "block_size_in_bytes",
999 "column_sizes",
1000 "value_counts",
1001 "null_value_counts",
1002 "nan_value_counts",
1003 "lower_bounds",
1004 "upper_bounds",
1005 "key_metadata",
1006 "split_offsets",
1007 "sort_order_id",
1008 ]);
1009 }
1010
1011 #[tokio::test]
1012 async fn test_v2_manifest_data_file_schema() {
1013 let field_names = written_data_file_field_names(ManifestWriterBuilder::build_v2_data).await;
1014 assert_eq!(field_names, vec![
1015 "content",
1016 "file_path",
1017 "file_format",
1018 "partition",
1019 "record_count",
1020 "file_size_in_bytes",
1021 "column_sizes",
1022 "value_counts",
1023 "null_value_counts",
1024 "nan_value_counts",
1025 "lower_bounds",
1026 "upper_bounds",
1027 "key_metadata",
1028 "split_offsets",
1029 "equality_ids",
1030 "sort_order_id",
1031 "referenced_data_file",
1032 ]);
1033 }
1034
1035 #[tokio::test]
1036 async fn test_v3_manifest_data_file_schema() {
1037 let field_names = written_data_file_field_names(ManifestWriterBuilder::build_v3_data).await;
1038 assert_eq!(field_names, vec![
1039 "content",
1040 "file_path",
1041 "file_format",
1042 "partition",
1043 "record_count",
1044 "file_size_in_bytes",
1045 "column_sizes",
1046 "value_counts",
1047 "null_value_counts",
1048 "nan_value_counts",
1049 "lower_bounds",
1050 "upper_bounds",
1051 "key_metadata",
1052 "split_offsets",
1053 "equality_ids",
1054 "sort_order_id",
1055 "first_row_id",
1056 "referenced_data_file",
1057 "content_offset",
1058 "content_size_in_bytes",
1059 ]);
1060 }
1061}