iceberg/spec/manifest_list/
manifest_file.rs

1// Licensed to the Apache Software Foundation (ASF) under one
2// or more contributor license agreements.  See the NOTICE file
3// distributed with this work for additional information
4// regarding copyright ownership.  The ASF licenses this file
5// to you under the Apache License, Version 2.0 (the
6// "License"); you may not use this file except in compliance
7// with the License.  You may obtain a copy of the License at
8//
9//   http://www.apache.org/licenses/LICENSE-2.0
10//
11// Unless required by applicable law or agreed to in writing,
12// software distributed under the License is distributed on an
13// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14// KIND, either express or implied.  See the License for the
15// specific language governing permissions and limitations
16// under the License.
17
18use std::str::FromStr;
19
20use serde_derive::{Deserialize, Serialize};
21
22use super::ByteBuf;
23use crate::error::Result;
24use crate::{Error, ErrorKind};
25
26/// Entry in a manifest list.
27#[derive(Debug, PartialEq, Clone, Eq, Hash)]
28pub struct ManifestFile {
29    /// field: 500
30    ///
31    /// Location of the manifest file
32    pub manifest_path: String,
33    /// field: 501
34    ///
35    /// Length of the manifest file in bytes
36    pub manifest_length: i64,
37    /// field: 502
38    ///
39    /// ID of a partition spec used to write the manifest; must be listed
40    /// in table metadata partition-specs
41    pub partition_spec_id: i32,
42    /// field: 517
43    ///
44    /// The type of files tracked by the manifest, either data or delete
45    /// files; 0 for all v1 manifests
46    pub content: ManifestContentType,
47    /// field: 515
48    ///
49    /// The sequence number when the manifest was added to the table; use 0
50    /// when reading v1 manifest lists
51    pub sequence_number: i64,
52    /// field: 516
53    ///
54    /// The minimum data sequence number of all live data or delete files in
55    /// the manifest; use 0 when reading v1 manifest lists
56    pub min_sequence_number: i64,
57    /// field: 503
58    ///
59    /// ID of the snapshot where the manifest file was added
60    pub added_snapshot_id: i64,
61    /// field: 504
62    ///
63    /// Number of entries in the manifest that have status ADDED, when null
64    /// this is assumed to be non-zero
65    pub added_files_count: Option<u32>,
66    /// field: 505
67    ///
68    /// Number of entries in the manifest that have status EXISTING (0),
69    /// when null this is assumed to be non-zero
70    pub existing_files_count: Option<u32>,
71    /// field: 506
72    ///
73    /// Number of entries in the manifest that have status DELETED (2),
74    /// when null this is assumed to be non-zero
75    pub deleted_files_count: Option<u32>,
76    /// field: 512
77    ///
78    /// Number of rows in all of files in the manifest that have status
79    /// ADDED, when null this is assumed to be non-zero
80    pub added_rows_count: Option<u64>,
81    /// field: 513
82    ///
83    /// Number of rows in all of files in the manifest that have status
84    /// EXISTING, when null this is assumed to be non-zero
85    pub existing_rows_count: Option<u64>,
86    /// field: 514
87    ///
88    /// Number of rows in all of files in the manifest that have status
89    /// DELETED, when null this is assumed to be non-zero
90    pub deleted_rows_count: Option<u64>,
91    /// field: 507
92    /// element_field: 508
93    ///
94    /// A list of field summaries for each partition field in the spec. Each
95    /// field in the list corresponds to a field in the manifest file’s
96    /// partition spec.
97    pub partitions: Option<Vec<FieldSummary>>,
98    /// field: 519
99    ///
100    /// Implementation-specific key metadata for encryption
101    pub key_metadata: Option<Vec<u8>>,
102    /// field 520
103    ///
104    /// The starting _row_id to assign to rows added by ADDED data files
105    pub first_row_id: Option<u64>,
106}
107
108impl ManifestFile {
109    /// Checks if the manifest file has any added files.
110    pub fn has_added_files(&self) -> bool {
111        self.added_files_count.map(|c| c > 0).unwrap_or(true)
112    }
113
114    /// Checks whether this manifest contains entries with DELETED status.
115    pub fn has_deleted_files(&self) -> bool {
116        self.deleted_files_count.map(|c| c > 0).unwrap_or(true)
117    }
118
119    /// Checks if the manifest file has any existed files.
120    pub fn has_existing_files(&self) -> bool {
121        self.existing_files_count.map(|c| c > 0).unwrap_or(true)
122    }
123}
124
125/// The type of files tracked by the manifest, either data or delete files; Data(0) for all v1 manifests
126#[derive(Debug, PartialEq, Clone, Copy, Eq, Hash, Default)]
127pub enum ManifestContentType {
128    /// The manifest content is data.
129    #[default]
130    Data = 0,
131    /// The manifest content is deletes.
132    Deletes = 1,
133}
134
135impl FromStr for ManifestContentType {
136    type Err = Error;
137
138    fn from_str(s: &str) -> Result<Self> {
139        match s {
140            "data" => Ok(ManifestContentType::Data),
141            "deletes" => Ok(ManifestContentType::Deletes),
142            _ => Err(Error::new(
143                ErrorKind::DataInvalid,
144                format!("Invalid manifest content type: {s}"),
145            )),
146        }
147    }
148}
149
150impl std::fmt::Display for ManifestContentType {
151    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
152        match self {
153            ManifestContentType::Data => write!(f, "data"),
154            ManifestContentType::Deletes => write!(f, "deletes"),
155        }
156    }
157}
158
159impl TryFrom<i32> for ManifestContentType {
160    type Error = Error;
161
162    fn try_from(value: i32) -> std::result::Result<Self, Self::Error> {
163        match value {
164            0 => Ok(ManifestContentType::Data),
165            1 => Ok(ManifestContentType::Deletes),
166            _ => Err(Error::new(
167                ErrorKind::DataInvalid,
168                format!("Invalid manifest content type. Expected 0 or 1, got {value}"),
169            )),
170        }
171    }
172}
173
174/// Field summary for partition field in the spec.
175///
176/// Each field in the list corresponds to a field in the manifest file’s partition spec.
177#[derive(Debug, Serialize, Deserialize, PartialEq, Eq, Clone, Default, Hash)]
178pub struct FieldSummary {
179    /// field: 509
180    ///
181    /// Whether the manifest contains at least one partition with a null
182    /// value for the field
183    pub contains_null: bool,
184    /// field: 518
185    /// Whether the manifest contains at least one partition with a NaN
186    /// value for the field
187    pub contains_nan: Option<bool>,
188    /// field: 510
189    /// The minimum value for the field in the manifests
190    /// partitions.
191    pub lower_bound: Option<ByteBuf>,
192    /// field: 511
193    /// The maximum value for the field in the manifests
194    /// partitions.
195    pub upper_bound: Option<ByteBuf>,
196}
197
198#[cfg(test)]
199mod test {
200    use std::sync::Arc;
201
202    use super::{ManifestContentType, ManifestFile};
203    use crate::ErrorKind;
204    use crate::encryption::{EncryptedOutputFile, StandardKeyMetadata};
205    use crate::io::FileIO;
206    use crate::spec::{
207        DataContentType, DataFileBuilder, DataFileFormat, ManifestEntry, ManifestReader,
208        ManifestStatus, ManifestWriterBuilder, NestedField, PartitionSpec, PrimitiveType, Schema,
209        SchemaRef, Type,
210    };
211
212    #[test]
213    fn test_manifest_content_type_default() {
214        assert_eq!(ManifestContentType::default(), ManifestContentType::Data);
215    }
216
217    #[test]
218    fn test_manifest_content_type_default_value() {
219        assert_eq!(ManifestContentType::default() as i32, 0);
220    }
221
222    /// A single-field schema used by the manifest-writing test helpers.
223    fn test_schema() -> SchemaRef {
224        Arc::new(
225            Schema::builder()
226                .with_fields(vec![Arc::new(NestedField::optional(
227                    1,
228                    "id",
229                    Type::Primitive(PrimitiveType::Long),
230                ))])
231                .build()
232                .unwrap(),
233        )
234    }
235
236    /// Writes a single-entry v3 data manifest to `io` at `path`, without
237    /// encryption, and returns the resulting [`ManifestFile`].
238    async fn write_manifest(io: &FileIO, path: &str) -> ManifestFile {
239        let schema = test_schema();
240        let partition_spec = PartitionSpec::builder(schema.clone())
241            .with_spec_id(0)
242            .build()
243            .unwrap();
244
245        let output_file = io.new_output(path).unwrap();
246        let mut writer = ManifestWriterBuilder::new(output_file, Some(1), schema, partition_spec)
247            .build_v3_data();
248
249        writer
250            .add_entry(data_entry(ManifestStatus::Added, 100, None))
251            .unwrap();
252
253        writer.write_manifest_file().await.unwrap()
254    }
255
256    /// Writes a single-entry v3 data manifest to `io` at `path`, encrypting it
257    /// with `key_metadata`, and returns the resulting [`ManifestFile`].
258    async fn write_encrypted_manifest(
259        io: &FileIO,
260        path: &str,
261        key_metadata: StandardKeyMetadata,
262    ) -> ManifestFile {
263        let schema = test_schema();
264        let partition_spec = PartitionSpec::builder(schema.clone())
265            .with_spec_id(0)
266            .build()
267            .unwrap();
268
269        let output_file = io.new_output(path).unwrap();
270        let encrypted_output = EncryptedOutputFile::new(output_file, key_metadata);
271
272        let mut writer = ManifestWriterBuilder::new_from_encrypted(
273            encrypted_output,
274            Some(1),
275            schema,
276            partition_spec,
277        )
278        .expect("Expected a valid writer")
279        .build_v3_data();
280
281        writer
282            .add_entry(data_entry(ManifestStatus::Added, 100, None))
283            .unwrap();
284
285        writer.write_manifest_file().await.unwrap()
286    }
287
288    #[tokio::test]
289    async fn test_load_manifest_decrypts_when_key_metadata_present() {
290        let key_metadata = StandardKeyMetadata::try_new(b"0123456789abcdef")
291            .unwrap()
292            .with_aad_prefix(b"test-aad-prefix!");
293        let encoded_key_metadata = key_metadata.encode().unwrap().to_vec();
294
295        let io = FileIO::new_with_memory();
296        let path = "memory:///test/encrypted_manifest.avro";
297        let manifest_file = write_encrypted_manifest(&io, path, key_metadata).await;
298        assert_eq!(manifest_file.key_metadata, Some(encoded_key_metadata));
299
300        let manifest = ManifestReader::new(io).read(&manifest_file).await.unwrap();
301        assert_eq!(manifest.entries().len(), 1);
302        assert_eq!(
303            manifest.entries()[0].file_path(),
304            "s3://bucket/table/data/00000.parquet"
305        );
306        assert_eq!(manifest.entries()[0].data_file.record_count, 100);
307    }
308
309    #[tokio::test]
310    async fn test_load_manifest_fails_with_wrong_key() {
311        let key_metadata = StandardKeyMetadata::try_new(b"0123456789abcdef")
312            .unwrap()
313            .with_aad_prefix(b"test-aad-prefix!");
314
315        let io = FileIO::new_with_memory();
316        let path = "memory:///test/wrong_key_manifest.avro";
317        let mut manifest_file = write_encrypted_manifest(&io, path, key_metadata).await;
318
319        // Point the manifest file at key metadata carrying a different DEK (but
320        // the same AAD prefix). The bytes on disk were encrypted with the
321        // original key, so GCM authentication must fail rather than silently
322        // returning garbage.
323        let wrong_key_metadata = StandardKeyMetadata::try_new(b"fedcba9876543210")
324            .unwrap()
325            .with_aad_prefix(b"test-aad-prefix!");
326        manifest_file.key_metadata = Some(wrong_key_metadata.encode().unwrap().to_vec());
327
328        let err = ManifestReader::new(io)
329            .read(&manifest_file)
330            .await
331            .expect_err("read must fail when decrypting with the wrong key");
332        assert_eq!(err.kind(), ErrorKind::Unexpected);
333    }
334
335    #[tokio::test]
336    async fn test_load_manifest_fails_with_wrong_aad() {
337        let key_metadata = StandardKeyMetadata::try_new(b"0123456789abcdef")
338            .unwrap()
339            .with_aad_prefix(b"test-aad-prefix!");
340
341        let io = FileIO::new_with_memory();
342        let path = "memory:///test/wrong_aad_manifest.avro";
343        let mut manifest_file = write_encrypted_manifest(&io, path, key_metadata).await;
344
345        // Point the manifest file at key metadata carrying the correct DEK but a
346        // different AAD prefix. The per-block AAD is `aad_prefix || block_index`,
347        // so GCM authentication must fail even though the key is right.
348        let wrong_aad_metadata = StandardKeyMetadata::try_new(b"0123456789abcdef")
349            .unwrap()
350            .with_aad_prefix(b"wrong-aad-prefix");
351        manifest_file.key_metadata = Some(wrong_aad_metadata.encode().unwrap().to_vec());
352
353        let err = ManifestReader::new(io)
354            .read(&manifest_file)
355            .await
356            .expect_err("read must fail when decrypting with the wrong AAD prefix");
357        assert_eq!(err.kind(), ErrorKind::Unexpected);
358    }
359
360    /// Builds a data-file manifest entry with the given status, record count,
361    /// and pre-existing `first_row_id`.
362    fn data_entry(
363        status: ManifestStatus,
364        record_count: u64,
365        first_row_id: Option<i64>,
366    ) -> ManifestEntry {
367        let data_file = DataFileBuilder::default()
368            .content(DataContentType::Data)
369            .file_path("s3://bucket/table/data/00000.parquet".to_string())
370            .file_format(DataFileFormat::Parquet)
371            .file_size_in_bytes(4096)
372            .record_count(record_count)
373            .first_row_id(first_row_id)
374            .build()
375            .unwrap();
376
377        ManifestEntry::builder()
378            .status(status)
379            .data_file(data_file)
380            .build()
381    }
382
383    #[tokio::test]
384    async fn test_load_manifest_reads_written_entries() {
385        let io = FileIO::new_with_memory();
386        let path = "memory:///test/plaintext_manifest.avro";
387        let manifest_file = write_manifest(&io, path).await;
388        assert_eq!(manifest_file.key_metadata, None);
389
390        let manifest = ManifestReader::new(io).read(&manifest_file).await.unwrap();
391        assert_eq!(manifest.entries().len(), 1);
392        assert_eq!(
393            manifest.entries()[0].file_path(),
394            "s3://bucket/table/data/00000.parquet"
395        );
396        assert_eq!(manifest.entries()[0].data_file.record_count, 100);
397    }
398
399    /// End-to-end: writing a v3 data manifest, stamping a manifest-level
400    /// `first_row_id`, and loading it must assign inherited `first_row_id`s to
401    /// the entries. This exercises the wiring in [`ManifestReader`] and the
402    /// write/read round-trip that leaves per-file `first_row_id` as `None`.
403    #[tokio::test]
404    async fn test_load_manifest_assigns_first_row_ids() {
405        let io = FileIO::new_with_memory();
406        let path = "memory:///test/first_row_id_manifest.avro";
407        let mut manifest_file = write_manifest(&io, path).await;
408
409        // Stamp a manifest-level first_row_id, as the manifest-list writer would.
410        manifest_file.first_row_id = Some(1000);
411
412        let manifest = ManifestReader::new(io).read(&manifest_file).await.unwrap();
413        assert_eq!(manifest.entries().len(), 1);
414        assert_eq!(manifest.entries()[0].data_file().first_row_id(), Some(1000));
415    }
416}
iceberg/spec/manifest_list/manifest_file.rs

iceberg/spec/manifest_list/
manifest_file.rs
