1use std::sync::Arc;
19
20use apache_avro::Schema as AvroSchema;
21use once_cell::sync::Lazy;
22use typed_builder::TypedBuilder;
23
24use crate::Error;
25use crate::avro::schema_to_avro_schema;
26use crate::error::{Result, invalid_data};
27use crate::spec::{
28 DataContentType, DataFile, INITIAL_SEQUENCE_NUMBER, ListType, Literal, ManifestFile, MapType,
29 NestedField, NestedFieldRef, PrimitiveLiteral, PrimitiveType, Schema, StructType, Type,
30};
31
32pub type ManifestEntryRef = Arc<ManifestEntry>;
34
35#[derive(Debug, PartialEq, Eq, Clone, TypedBuilder)]
39pub struct ManifestEntry {
40 pub status: ManifestStatus,
44 #[builder(default, setter(strip_option(fallback = snapshot_id_opt)))]
49 pub snapshot_id: Option<i64>,
50 #[builder(default, setter(strip_option(fallback = sequence_number_opt)))]
55 pub sequence_number: Option<i64>,
56 #[builder(default, setter(strip_option(fallback = file_sequence_number_opt)))]
61 pub file_sequence_number: Option<i64>,
62 pub data_file: DataFile,
66}
67
68impl ManifestEntry {
69 pub fn is_alive(&self) -> bool {
71 matches!(
72 self.status,
73 ManifestStatus::Added | ManifestStatus::Existing
74 )
75 }
76
77 pub fn status(&self) -> ManifestStatus {
79 self.status
80 }
81
82 #[inline]
84 pub fn content_type(&self) -> DataContentType {
85 self.data_file.content
86 }
87
88 #[inline]
90 pub fn file_format(&self) -> DataFileFormat {
91 self.data_file.file_format
92 }
93
94 #[inline]
96 pub fn file_path(&self) -> &str {
97 &self.data_file.file_path
98 }
99
100 #[inline]
102 pub fn record_count(&self) -> u64 {
103 self.data_file.record_count
104 }
105
106 pub(crate) fn inherit_data(&mut self, snapshot_entry: &ManifestFile) {
108 if self.snapshot_id.is_none() {
109 self.snapshot_id = Some(snapshot_entry.added_snapshot_id);
110 }
111
112 if self.sequence_number.is_none()
113 && (self.status == ManifestStatus::Added
114 || snapshot_entry.sequence_number == INITIAL_SEQUENCE_NUMBER)
115 {
116 self.sequence_number = Some(snapshot_entry.sequence_number);
117 }
118
119 if self.file_sequence_number.is_none()
120 && (self.status == ManifestStatus::Added
121 || snapshot_entry.sequence_number == INITIAL_SEQUENCE_NUMBER)
122 {
123 self.file_sequence_number = Some(snapshot_entry.sequence_number);
124 }
125 }
126
127 #[inline]
129 pub fn snapshot_id(&self) -> Option<i64> {
130 self.snapshot_id
131 }
132
133 #[inline]
135 pub fn sequence_number(&self) -> Option<i64> {
136 self.sequence_number
137 }
138
139 #[inline]
141 pub fn file_size_in_bytes(&self) -> u64 {
142 self.data_file.file_size_in_bytes
143 }
144
145 #[inline]
147 pub fn data_file(&self) -> &DataFile {
148 &self.data_file
149 }
150}
151
152#[derive(Debug, PartialEq, Eq, Clone, Copy)]
154pub enum ManifestStatus {
155 Existing = 0,
157 Added = 1,
159 Deleted = 2,
163}
164
165impl TryFrom<i32> for ManifestStatus {
166 type Error = Error;
167
168 fn try_from(v: i32) -> Result<ManifestStatus> {
169 match v {
170 0 => Ok(ManifestStatus::Existing),
171 1 => Ok(ManifestStatus::Added),
172 2 => Ok(ManifestStatus::Deleted),
173 _ => Err(invalid_data!("manifest status {v} is invalid")),
174 }
175 }
176}
177
178use super::DataFileFormat;
179
180pub(crate) const DATA_FILE_FIELD_ID: i32 = 2;
182pub(crate) const SEQUENCE_NUMBER_FIELD_ID: i32 = 3;
184pub(crate) const FILE_SEQUENCE_NUMBER_FIELD_ID: i32 = 4;
186pub(crate) const DATA_FILE_PARTITION_FIELD_ID: i32 = 102;
188pub(crate) const DATA_FILE_CONTENT_FIELD_ID: i32 = 134;
190
191static STATUS: Lazy<NestedFieldRef> = {
192 Lazy::new(|| {
193 Arc::new(NestedField::required(
194 0,
195 "status",
196 Type::Primitive(PrimitiveType::Int),
197 ))
198 })
199};
200
201static SNAPSHOT_ID_V1: Lazy<NestedFieldRef> = {
202 Lazy::new(|| {
203 Arc::new(NestedField::required(
204 1,
205 "snapshot_id",
206 Type::Primitive(PrimitiveType::Long),
207 ))
208 })
209};
210
211static SNAPSHOT_ID_V2: Lazy<NestedFieldRef> = {
212 Lazy::new(|| {
213 Arc::new(NestedField::optional(
214 1,
215 "snapshot_id",
216 Type::Primitive(PrimitiveType::Long),
217 ))
218 })
219};
220
221static SEQUENCE_NUMBER: Lazy<NestedFieldRef> = {
222 Lazy::new(|| {
223 Arc::new(NestedField::optional(
224 SEQUENCE_NUMBER_FIELD_ID,
225 "sequence_number",
226 Type::Primitive(PrimitiveType::Long),
227 ))
228 })
229};
230
231static FILE_SEQUENCE_NUMBER: Lazy<NestedFieldRef> = {
232 Lazy::new(|| {
233 Arc::new(NestedField::optional(
234 FILE_SEQUENCE_NUMBER_FIELD_ID,
235 "file_sequence_number",
236 Type::Primitive(PrimitiveType::Long),
237 ))
238 })
239};
240
241static CONTENT: Lazy<NestedFieldRef> = {
242 Lazy::new(|| {
243 Arc::new(
244 NestedField::required(
245 DATA_FILE_CONTENT_FIELD_ID,
246 "content",
247 Type::Primitive(PrimitiveType::Int),
248 )
249 .with_initial_default(Literal::Primitive(PrimitiveLiteral::Int(0))),
251 )
252 })
253};
254
255static FILE_PATH: Lazy<NestedFieldRef> = {
256 Lazy::new(|| {
257 Arc::new(NestedField::required(
258 100,
259 "file_path",
260 Type::Primitive(PrimitiveType::String),
261 ))
262 })
263};
264
265static FILE_FORMAT: Lazy<NestedFieldRef> = {
266 Lazy::new(|| {
267 Arc::new(NestedField::required(
268 101,
269 "file_format",
270 Type::Primitive(PrimitiveType::String),
271 ))
272 })
273};
274
275static RECORD_COUNT: Lazy<NestedFieldRef> = {
276 Lazy::new(|| {
277 Arc::new(NestedField::required(
278 103,
279 "record_count",
280 Type::Primitive(PrimitiveType::Long),
281 ))
282 })
283};
284
285static FILE_SIZE_IN_BYTES: Lazy<NestedFieldRef> = {
286 Lazy::new(|| {
287 Arc::new(NestedField::required(
288 104,
289 "file_size_in_bytes",
290 Type::Primitive(PrimitiveType::Long),
291 ))
292 })
293};
294
295static BLOCK_SIZE_IN_BYTES: Lazy<NestedFieldRef> = {
297 Lazy::new(|| {
298 Arc::new(NestedField::required(
299 105,
300 "block_size_in_bytes",
301 Type::Primitive(PrimitiveType::Long),
302 ))
303 })
304};
305
306static COLUMN_SIZES: Lazy<NestedFieldRef> = {
307 Lazy::new(|| {
308 Arc::new(NestedField::optional(
309 108,
310 "column_sizes",
311 Type::Map(MapType {
312 key_field: Arc::new(NestedField::required(
313 117,
314 "key",
315 Type::Primitive(PrimitiveType::Int),
316 )),
317 value_field: Arc::new(NestedField::required(
318 118,
319 "value",
320 Type::Primitive(PrimitiveType::Long),
321 )),
322 }),
323 ))
324 })
325};
326
327static VALUE_COUNTS: Lazy<NestedFieldRef> = {
328 Lazy::new(|| {
329 Arc::new(NestedField::optional(
330 109,
331 "value_counts",
332 Type::Map(MapType {
333 key_field: Arc::new(NestedField::required(
334 119,
335 "key",
336 Type::Primitive(PrimitiveType::Int),
337 )),
338 value_field: Arc::new(NestedField::required(
339 120,
340 "value",
341 Type::Primitive(PrimitiveType::Long),
342 )),
343 }),
344 ))
345 })
346};
347
348static NULL_VALUE_COUNTS: Lazy<NestedFieldRef> = {
349 Lazy::new(|| {
350 Arc::new(NestedField::optional(
351 110,
352 "null_value_counts",
353 Type::Map(MapType {
354 key_field: Arc::new(NestedField::required(
355 121,
356 "key",
357 Type::Primitive(PrimitiveType::Int),
358 )),
359 value_field: Arc::new(NestedField::required(
360 122,
361 "value",
362 Type::Primitive(PrimitiveType::Long),
363 )),
364 }),
365 ))
366 })
367};
368
369static NAN_VALUE_COUNTS: Lazy<NestedFieldRef> = {
370 Lazy::new(|| {
371 Arc::new(NestedField::optional(
372 137,
373 "nan_value_counts",
374 Type::Map(MapType {
375 key_field: Arc::new(NestedField::required(
376 138,
377 "key",
378 Type::Primitive(PrimitiveType::Int),
379 )),
380 value_field: Arc::new(NestedField::required(
381 139,
382 "value",
383 Type::Primitive(PrimitiveType::Long),
384 )),
385 }),
386 ))
387 })
388};
389
390static LOWER_BOUNDS: Lazy<NestedFieldRef> = {
391 Lazy::new(|| {
392 Arc::new(NestedField::optional(
393 125,
394 "lower_bounds",
395 Type::Map(MapType {
396 key_field: Arc::new(NestedField::required(
397 126,
398 "key",
399 Type::Primitive(PrimitiveType::Int),
400 )),
401 value_field: Arc::new(NestedField::required(
402 127,
403 "value",
404 Type::Primitive(PrimitiveType::Binary),
405 )),
406 }),
407 ))
408 })
409};
410
411static UPPER_BOUNDS: Lazy<NestedFieldRef> = {
412 Lazy::new(|| {
413 Arc::new(NestedField::optional(
414 128,
415 "upper_bounds",
416 Type::Map(MapType {
417 key_field: Arc::new(NestedField::required(
418 129,
419 "key",
420 Type::Primitive(PrimitiveType::Int),
421 )),
422 value_field: Arc::new(NestedField::required(
423 130,
424 "value",
425 Type::Primitive(PrimitiveType::Binary),
426 )),
427 }),
428 ))
429 })
430};
431
432static KEY_METADATA: Lazy<NestedFieldRef> = {
433 Lazy::new(|| {
434 Arc::new(NestedField::optional(
435 131,
436 "key_metadata",
437 Type::Primitive(PrimitiveType::Binary),
438 ))
439 })
440};
441
442static SPLIT_OFFSETS: Lazy<NestedFieldRef> = {
443 Lazy::new(|| {
444 Arc::new(NestedField::optional(
445 132,
446 "split_offsets",
447 Type::List(ListType {
448 element_field: Arc::new(NestedField::required(
449 133,
450 "element",
451 Type::Primitive(PrimitiveType::Long),
452 )),
453 }),
454 ))
455 })
456};
457
458static EQUALITY_IDS: Lazy<NestedFieldRef> = {
459 Lazy::new(|| {
460 Arc::new(NestedField::optional(
461 135,
462 "equality_ids",
463 Type::List(ListType {
464 element_field: Arc::new(NestedField::required(
465 136,
466 "element",
467 Type::Primitive(PrimitiveType::Int),
468 )),
469 }),
470 ))
471 })
472};
473
474static SORT_ORDER_ID: Lazy<NestedFieldRef> = {
475 Lazy::new(|| {
476 Arc::new(NestedField::optional(
477 140,
478 "sort_order_id",
479 Type::Primitive(PrimitiveType::Int),
480 ))
481 })
482};
483
484static FIRST_ROW_ID: Lazy<NestedFieldRef> = {
485 Lazy::new(|| {
486 Arc::new(NestedField::optional(
487 142,
488 "first_row_id",
489 Type::Primitive(PrimitiveType::Long),
490 ))
491 })
492};
493
494static REFERENCE_DATA_FILE: Lazy<NestedFieldRef> = {
495 Lazy::new(|| {
496 Arc::new(NestedField::optional(
497 143,
498 "referenced_data_file",
499 Type::Primitive(PrimitiveType::String),
500 ))
501 })
502};
503
504static CONTENT_OFFSET: Lazy<NestedFieldRef> = {
505 Lazy::new(|| {
506 Arc::new(NestedField::optional(
507 144,
508 "content_offset",
509 Type::Primitive(PrimitiveType::Long),
510 ))
511 })
512};
513
514static CONTENT_SIZE_IN_BYTES: Lazy<NestedFieldRef> = {
515 Lazy::new(|| {
516 Arc::new(NestedField::optional(
517 145,
518 "content_size_in_bytes",
519 Type::Primitive(PrimitiveType::Long),
520 ))
521 })
522};
523
524fn data_file_fields_v3(partition_type: &StructType) -> Vec<NestedFieldRef> {
525 vec![
526 CONTENT.clone(),
527 FILE_PATH.clone(),
528 FILE_FORMAT.clone(),
529 Arc::new(NestedField::required(
530 DATA_FILE_PARTITION_FIELD_ID,
531 "partition",
532 Type::Struct(partition_type.clone()),
533 )),
534 RECORD_COUNT.clone(),
535 FILE_SIZE_IN_BYTES.clone(),
536 COLUMN_SIZES.clone(),
537 VALUE_COUNTS.clone(),
538 NULL_VALUE_COUNTS.clone(),
539 NAN_VALUE_COUNTS.clone(),
540 LOWER_BOUNDS.clone(),
541 UPPER_BOUNDS.clone(),
542 KEY_METADATA.clone(),
543 SPLIT_OFFSETS.clone(),
544 EQUALITY_IDS.clone(),
545 SORT_ORDER_ID.clone(),
546 FIRST_ROW_ID.clone(),
547 REFERENCE_DATA_FILE.clone(),
548 CONTENT_OFFSET.clone(),
549 CONTENT_SIZE_IN_BYTES.clone(),
550 ]
551}
552
553pub(crate) fn manifest_entry_fields(partition_type: &StructType) -> Vec<NestedFieldRef> {
558 vec![
559 STATUS.clone(),
560 SNAPSHOT_ID_V2.clone(),
561 SEQUENCE_NUMBER.clone(),
562 FILE_SEQUENCE_NUMBER.clone(),
563 Arc::new(NestedField::required(
564 DATA_FILE_FIELD_ID,
565 "data_file",
566 Type::Struct(StructType::new(data_file_fields_v3(partition_type))),
567 )),
568 ]
569}
570
571pub(super) fn data_file_schema_v3(partition_type: &StructType) -> Result<AvroSchema> {
572 let schema = Schema::builder()
573 .with_fields(data_file_fields_v3(partition_type))
574 .build()?;
575 schema_to_avro_schema("data_file", &schema)
576}
577
578fn data_file_fields_v2(partition_type: &StructType) -> Vec<NestedFieldRef> {
579 vec![
580 CONTENT.clone(),
581 FILE_PATH.clone(),
582 FILE_FORMAT.clone(),
583 Arc::new(NestedField::required(
584 DATA_FILE_PARTITION_FIELD_ID,
585 "partition",
586 Type::Struct(partition_type.clone()),
587 )),
588 RECORD_COUNT.clone(),
589 FILE_SIZE_IN_BYTES.clone(),
590 COLUMN_SIZES.clone(),
591 VALUE_COUNTS.clone(),
592 NULL_VALUE_COUNTS.clone(),
593 NAN_VALUE_COUNTS.clone(),
594 LOWER_BOUNDS.clone(),
595 UPPER_BOUNDS.clone(),
596 KEY_METADATA.clone(),
597 SPLIT_OFFSETS.clone(),
598 EQUALITY_IDS.clone(),
599 SORT_ORDER_ID.clone(),
600 REFERENCE_DATA_FILE.clone(),
601 ]
602}
603
604pub(super) fn data_file_schema_v2(partition_type: &StructType) -> Result<AvroSchema> {
605 let schema = Schema::builder()
606 .with_fields(data_file_fields_v2(partition_type))
607 .build()?;
608 schema_to_avro_schema("data_file", &schema)
609}
610
611pub(super) fn manifest_schema_v3(partition_type: &StructType) -> Result<AvroSchema> {
612 manifest_schema_v2_v3(data_file_fields_v3(partition_type))
613}
614
615pub(super) fn manifest_schema_v2(partition_type: &StructType) -> Result<AvroSchema> {
616 manifest_schema_v2_v3(data_file_fields_v2(partition_type))
617}
618
619fn manifest_schema_v2_v3(data_file_fields: Vec<NestedFieldRef>) -> Result<AvroSchema> {
622 let fields = vec![
623 STATUS.clone(),
624 SNAPSHOT_ID_V2.clone(),
625 SEQUENCE_NUMBER.clone(),
626 FILE_SEQUENCE_NUMBER.clone(),
627 Arc::new(NestedField::required(
628 DATA_FILE_FIELD_ID,
629 "data_file",
630 Type::Struct(StructType::new(data_file_fields)),
631 )),
632 ];
633 let schema = Schema::builder().with_fields(fields).build()?;
634 schema_to_avro_schema("manifest_entry", &schema)
635}
636
637fn data_file_fields_v1(partition_type: &StructType) -> Vec<NestedFieldRef> {
638 vec![
639 FILE_PATH.clone(),
640 FILE_FORMAT.clone(),
641 Arc::new(NestedField::required(
642 DATA_FILE_PARTITION_FIELD_ID,
643 "partition",
644 Type::Struct(partition_type.clone()),
645 )),
646 RECORD_COUNT.clone(),
647 FILE_SIZE_IN_BYTES.clone(),
648 BLOCK_SIZE_IN_BYTES.clone(),
649 COLUMN_SIZES.clone(),
650 VALUE_COUNTS.clone(),
651 NULL_VALUE_COUNTS.clone(),
652 NAN_VALUE_COUNTS.clone(),
653 LOWER_BOUNDS.clone(),
654 UPPER_BOUNDS.clone(),
655 KEY_METADATA.clone(),
656 SPLIT_OFFSETS.clone(),
657 SORT_ORDER_ID.clone(),
658 ]
659}
660
661pub(super) fn data_file_schema_v1(partition_type: &StructType) -> Result<AvroSchema> {
662 let schema = Schema::builder()
663 .with_fields(data_file_fields_v1(partition_type))
664 .build()?;
665 schema_to_avro_schema("data_file", &schema)
666}
667
668pub(super) fn manifest_schema_v1(partition_type: &StructType) -> Result<AvroSchema> {
669 let fields = vec![
670 STATUS.clone(),
671 SNAPSHOT_ID_V1.clone(),
672 Arc::new(NestedField::required(
673 DATA_FILE_FIELD_ID,
674 "data_file",
675 Type::Struct(StructType::new(data_file_fields_v1(partition_type))),
676 )),
677 ];
678 let schema = Schema::builder().with_fields(fields).build()?;
679 schema_to_avro_schema("manifest_entry", &schema)
680}