iceberg/spec/manifest_list/
manifest_file.rs

1// Licensed to the Apache Software Foundation (ASF) under one
2// or more contributor license agreements.  See the NOTICE file
3// distributed with this work for additional information
4// regarding copyright ownership.  The ASF licenses this file
5// to you under the Apache License, Version 2.0 (the
6// "License"); you may not use this file except in compliance
7// with the License.  You may obtain a copy of the License at
8//
9//   http://www.apache.org/licenses/LICENSE-2.0
10//
11// Unless required by applicable law or agreed to in writing,
12// software distributed under the License is distributed on an
13// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14// KIND, either express or implied.  See the License for the
15// specific language governing permissions and limitations
16// under the License.
17
18use std::str::FromStr;
19
20use serde_derive::{Deserialize, Serialize};
21
22use super::ByteBuf;
23use crate::Error;
24use crate::error::{Result, invalid_data};
25
26/// Entry in a manifest list.
27#[derive(Debug, PartialEq, Clone, Eq, Hash)]
28pub struct ManifestFile {
29    /// field: 500
30    ///
31    /// Location of the manifest file
32    pub manifest_path: String,
33    /// field: 501
34    ///
35    /// Length of the manifest file in bytes
36    pub manifest_length: i64,
37    /// field: 502
38    ///
39    /// ID of a partition spec used to write the manifest; must be listed
40    /// in table metadata partition-specs
41    pub partition_spec_id: i32,
42    /// field: 517
43    ///
44    /// The type of files tracked by the manifest, either data or delete
45    /// files; 0 for all v1 manifests
46    pub content: ManifestContentType,
47    /// field: 515
48    ///
49    /// The sequence number when the manifest was added to the table; use 0
50    /// when reading v1 manifest lists
51    pub sequence_number: i64,
52    /// field: 516
53    ///
54    /// The minimum data sequence number of all live data or delete files in
55    /// the manifest; use 0 when reading v1 manifest lists
56    pub min_sequence_number: i64,
57    /// field: 503
58    ///
59    /// ID of the snapshot where the manifest file was added
60    pub added_snapshot_id: i64,
61    /// field: 504
62    ///
63    /// Number of entries in the manifest that have status ADDED, when null
64    /// this is assumed to be non-zero
65    pub added_files_count: Option<u32>,
66    /// field: 505
67    ///
68    /// Number of entries in the manifest that have status EXISTING (0),
69    /// when null this is assumed to be non-zero
70    pub existing_files_count: Option<u32>,
71    /// field: 506
72    ///
73    /// Number of entries in the manifest that have status DELETED (2),
74    /// when null this is assumed to be non-zero
75    pub deleted_files_count: Option<u32>,
76    /// field: 512
77    ///
78    /// Number of rows in all of files in the manifest that have status
79    /// ADDED, when null this is assumed to be non-zero
80    pub added_rows_count: Option<u64>,
81    /// field: 513
82    ///
83    /// Number of rows in all of files in the manifest that have status
84    /// EXISTING, when null this is assumed to be non-zero
85    pub existing_rows_count: Option<u64>,
86    /// field: 514
87    ///
88    /// Number of rows in all of files in the manifest that have status
89    /// DELETED, when null this is assumed to be non-zero
90    pub deleted_rows_count: Option<u64>,
91    /// field: 507
92    /// element_field: 508
93    ///
94    /// A list of field summaries for each partition field in the spec. Each
95    /// field in the list corresponds to a field in the manifest file’s
96    /// partition spec.
97    pub partitions: Option<Vec<FieldSummary>>,
98    /// field: 519
99    ///
100    /// Implementation-specific key metadata for encryption
101    pub key_metadata: Option<Vec<u8>>,
102    /// field 520
103    ///
104    /// The starting _row_id to assign to rows added by ADDED data files
105    pub first_row_id: Option<u64>,
106}
107
108impl ManifestFile {
109    /// Checks if the manifest file has any added files.
110    pub fn has_added_files(&self) -> bool {
111        self.added_files_count.map(|c| c > 0).unwrap_or(true)
112    }
113
114    /// Checks whether this manifest contains entries with DELETED status.
115    pub fn has_deleted_files(&self) -> bool {
116        self.deleted_files_count.map(|c| c > 0).unwrap_or(true)
117    }
118
119    /// Checks if the manifest file has any existed files.
120    pub fn has_existing_files(&self) -> bool {
121        self.existing_files_count.map(|c| c > 0).unwrap_or(true)
122    }
123}
124
125/// The type of files tracked by the manifest, either data or delete files; Data(0) for all v1 manifests
126#[derive(Debug, PartialEq, Clone, Copy, Eq, Hash, Default)]
127pub enum ManifestContentType {
128    /// The manifest content is data.
129    #[default]
130    Data = 0,
131    /// The manifest content is deletes.
132    Deletes = 1,
133}
134
135impl FromStr for ManifestContentType {
136    type Err = Error;
137
138    fn from_str(s: &str) -> Result<Self> {
139        match s {
140            "data" => Ok(ManifestContentType::Data),
141            "deletes" => Ok(ManifestContentType::Deletes),
142            _ => Err(invalid_data!("Invalid manifest content type: {s}")),
143        }
144    }
145}
146
147impl std::fmt::Display for ManifestContentType {
148    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
149        match self {
150            ManifestContentType::Data => write!(f, "data"),
151            ManifestContentType::Deletes => write!(f, "deletes"),
152        }
153    }
154}
155
156impl TryFrom<i32> for ManifestContentType {
157    type Error = Error;
158
159    fn try_from(value: i32) -> std::result::Result<Self, Self::Error> {
160        match value {
161            0 => Ok(ManifestContentType::Data),
162            1 => Ok(ManifestContentType::Deletes),
163            _ => Err(invalid_data!(
164                "Invalid manifest content type. Expected 0 or 1, got {value}"
165            )),
166        }
167    }
168}
169
170/// Field summary for partition field in the spec.
171///
172/// Each field in the list corresponds to a field in the manifest file’s partition spec.
173#[derive(Debug, Serialize, Deserialize, PartialEq, Eq, Clone, Default, Hash)]
174pub struct FieldSummary {
175    /// field: 509
176    ///
177    /// Whether the manifest contains at least one partition with a null
178    /// value for the field
179    pub contains_null: bool,
180    /// field: 518
181    /// Whether the manifest contains at least one partition with a NaN
182    /// value for the field
183    pub contains_nan: Option<bool>,
184    /// field: 510
185    /// The minimum value for the field in the manifests
186    /// partitions.
187    pub lower_bound: Option<ByteBuf>,
188    /// field: 511
189    /// The maximum value for the field in the manifests
190    /// partitions.
191    pub upper_bound: Option<ByteBuf>,
192}
193
194#[cfg(test)]
195mod test {
196    use std::sync::Arc;
197
198    use super::{ManifestContentType, ManifestFile};
199    use crate::ErrorKind;
200    use crate::encryption::{EncryptedOutputFile, StandardKeyMetadata};
201    use crate::io::FileIO;
202    use crate::spec::{
203        DataContentType, DataFileBuilder, DataFileFormat, ManifestEntry, ManifestReader,
204        ManifestStatus, ManifestWriterBuilder, NestedField, PartitionSpec, PrimitiveType, Schema,
205        SchemaRef, Type,
206    };
207
208    #[test]
209    fn test_manifest_content_type_default() {
210        assert_eq!(ManifestContentType::default(), ManifestContentType::Data);
211    }
212
213    #[test]
214    fn test_manifest_content_type_default_value() {
215        assert_eq!(ManifestContentType::default() as i32, 0);
216    }
217
218    /// A single-field schema used by the manifest-writing test helpers.
219    fn test_schema() -> SchemaRef {
220        Arc::new(
221            Schema::builder()
222                .with_fields(vec![Arc::new(NestedField::optional(
223                    1,
224                    "id",
225                    Type::Primitive(PrimitiveType::Long),
226                ))])
227                .build()
228                .unwrap(),
229        )
230    }
231
232    /// Writes a single-entry v3 data manifest to `io` at `path`, without
233    /// encryption, and returns the resulting [`ManifestFile`].
234    async fn write_manifest(io: &FileIO, path: &str) -> ManifestFile {
235        let schema = test_schema();
236        let partition_spec = PartitionSpec::builder(schema.clone())
237            .with_spec_id(0)
238            .build()
239            .unwrap();
240
241        let output_file = io.new_output(path).unwrap();
242        let mut writer = ManifestWriterBuilder::new(output_file, Some(1), schema, partition_spec)
243            .build_v3_data();
244
245        writer
246            .add_entry(data_entry(ManifestStatus::Added, 100, None))
247            .unwrap();
248
249        writer.write_manifest_file().await.unwrap()
250    }
251
252    /// Writes a single-entry v3 data manifest to `io` at `path`, encrypting it
253    /// with `key_metadata`, and returns the resulting [`ManifestFile`].
254    async fn write_encrypted_manifest(
255        io: &FileIO,
256        path: &str,
257        key_metadata: StandardKeyMetadata,
258    ) -> ManifestFile {
259        let schema = test_schema();
260        let partition_spec = PartitionSpec::builder(schema.clone())
261            .with_spec_id(0)
262            .build()
263            .unwrap();
264
265        let output_file = io.new_output(path).unwrap();
266        let encrypted_output = EncryptedOutputFile::new(output_file, key_metadata);
267
268        let mut writer = ManifestWriterBuilder::new_from_encrypted(
269            encrypted_output,
270            Some(1),
271            schema,
272            partition_spec,
273        )
274        .expect("Expected a valid writer")
275        .build_v3_data();
276
277        writer
278            .add_entry(data_entry(ManifestStatus::Added, 100, None))
279            .unwrap();
280
281        writer.write_manifest_file().await.unwrap()
282    }
283
284    #[tokio::test]
285    async fn test_load_manifest_decrypts_when_key_metadata_present() {
286        let key_metadata = StandardKeyMetadata::try_new(b"0123456789abcdef")
287            .unwrap()
288            .with_aad_prefix(b"test-aad-prefix!");
289        let io = FileIO::new_with_memory();
290        let path = "memory:///test/encrypted_manifest.avro";
291        let manifest_file = write_encrypted_manifest(&io, path, key_metadata.clone()).await;
292        let size = io.new_input(path).unwrap().metadata().await.unwrap().size;
293        assert_eq!(manifest_file.manifest_length, size as i64);
294        assert_eq!(
295            StandardKeyMetadata::decode(manifest_file.key_metadata.as_ref().unwrap()).unwrap(),
296            key_metadata.with_file_length(size)
297        );
298
299        let manifest = ManifestReader::new(io).read(&manifest_file).await.unwrap();
300        assert_eq!(manifest.entries().len(), 1);
301        assert_eq!(
302            manifest.entries()[0].file_path(),
303            "s3://bucket/table/data/00000.parquet"
304        );
305        assert_eq!(manifest.entries()[0].data_file.record_count, 100);
306    }
307
308    #[tokio::test]
309    async fn test_load_manifest_fails_with_wrong_key() {
310        let key_metadata = StandardKeyMetadata::try_new(b"0123456789abcdef")
311            .unwrap()
312            .with_aad_prefix(b"test-aad-prefix!");
313
314        let io = FileIO::new_with_memory();
315        let path = "memory:///test/wrong_key_manifest.avro";
316        let mut manifest_file = write_encrypted_manifest(&io, path, key_metadata).await;
317
318        // Point the manifest file at key metadata carrying a different DEK (but
319        // the same AAD prefix). The bytes on disk were encrypted with the
320        // original key, so GCM authentication must fail rather than silently
321        // returning garbage.
322        let wrong_key_metadata = StandardKeyMetadata::try_new(b"fedcba9876543210")
323            .unwrap()
324            .with_aad_prefix(b"test-aad-prefix!")
325            .with_file_length(manifest_file.manifest_length as u64);
326        manifest_file.key_metadata = Some(wrong_key_metadata.encode().unwrap().to_vec());
327
328        let err = ManifestReader::new(io)
329            .read(&manifest_file)
330            .await
331            .expect_err("read must fail when decrypting with the wrong key");
332        assert_eq!(err.kind(), ErrorKind::Unexpected);
333    }
334
335    #[tokio::test]
336    async fn test_load_manifest_fails_with_wrong_aad() {
337        let key_metadata = StandardKeyMetadata::try_new(b"0123456789abcdef")
338            .unwrap()
339            .with_aad_prefix(b"test-aad-prefix!");
340
341        let io = FileIO::new_with_memory();
342        let path = "memory:///test/wrong_aad_manifest.avro";
343        let mut manifest_file = write_encrypted_manifest(&io, path, key_metadata).await;
344
345        // Point the manifest file at key metadata carrying the correct DEK but a
346        // different AAD prefix. The per-block AAD is `aad_prefix || block_index`,
347        // so GCM authentication must fail even though the key is right.
348        let wrong_aad_metadata = StandardKeyMetadata::try_new(b"0123456789abcdef")
349            .unwrap()
350            .with_aad_prefix(b"wrong-aad-prefix")
351            .with_file_length(manifest_file.manifest_length as u64);
352        manifest_file.key_metadata = Some(wrong_aad_metadata.encode().unwrap().to_vec());
353
354        let err = ManifestReader::new(io)
355            .read(&manifest_file)
356            .await
357            .expect_err("read must fail when decrypting with the wrong AAD prefix");
358        assert_eq!(err.kind(), ErrorKind::Unexpected);
359    }
360
361    /// Builds a data-file manifest entry with the given status, record count,
362    /// and pre-existing `first_row_id`.
363    fn data_entry(
364        status: ManifestStatus,
365        record_count: u64,
366        first_row_id: Option<i64>,
367    ) -> ManifestEntry {
368        let data_file = DataFileBuilder::default()
369            .content(DataContentType::Data)
370            .file_path("s3://bucket/table/data/00000.parquet".to_string())
371            .file_format(DataFileFormat::Parquet)
372            .file_size_in_bytes(4096)
373            .record_count(record_count)
374            .first_row_id(first_row_id)
375            .build()
376            .unwrap();
377
378        ManifestEntry::builder()
379            .status(status)
380            .data_file(data_file)
381            .build()
382    }
383
384    #[tokio::test]
385    async fn test_load_manifest_reads_written_entries() {
386        let io = FileIO::new_with_memory();
387        let path = "memory:///test/plaintext_manifest.avro";
388        let manifest_file = write_manifest(&io, path).await;
389        assert_eq!(manifest_file.key_metadata, None);
390
391        let manifest = ManifestReader::new(io).read(&manifest_file).await.unwrap();
392        assert_eq!(manifest.entries().len(), 1);
393        assert_eq!(
394            manifest.entries()[0].file_path(),
395            "s3://bucket/table/data/00000.parquet"
396        );
397        assert_eq!(manifest.entries()[0].data_file.record_count, 100);
398    }
399
400    /// End-to-end: writing a v3 data manifest, stamping a manifest-level
401    /// `first_row_id`, and loading it must assign inherited `first_row_id`s to
402    /// the entries. This exercises the wiring in [`ManifestReader`] and the
403    /// write/read round-trip that leaves per-file `first_row_id` as `None`.
404    #[tokio::test]
405    async fn test_load_manifest_assigns_first_row_ids() {
406        let io = FileIO::new_with_memory();
407        let path = "memory:///test/first_row_id_manifest.avro";
408        let mut manifest_file = write_manifest(&io, path).await;
409
410        // Stamp a manifest-level first_row_id, as the manifest-list writer would.
411        manifest_file.first_row_id = Some(1000);
412
413        let manifest = ManifestReader::new(io).read(&manifest_file).await.unwrap();
414        assert_eq!(manifest.entries().len(), 1);
415        assert_eq!(manifest.entries()[0].data_file().first_row_id(), Some(1000));
416    }
417}
iceberg/spec/manifest_list/manifest_file.rs

iceberg/spec/manifest_list/
manifest_file.rs
