Skip to main content

iceberg/spec/manifest_list/
manifest_file.rs

1// Licensed to the Apache Software Foundation (ASF) under one
2// or more contributor license agreements.  See the NOTICE file
3// distributed with this work for additional information
4// regarding copyright ownership.  The ASF licenses this file
5// to you under the Apache License, Version 2.0 (the
6// "License"); you may not use this file except in compliance
7// with the License.  You may obtain a copy of the License at
8//
9//   http://www.apache.org/licenses/LICENSE-2.0
10//
11// Unless required by applicable law or agreed to in writing,
12// software distributed under the License is distributed on an
13// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14// KIND, either express or implied.  See the License for the
15// specific language governing permissions and limitations
16// under the License.
17
18use std::str::FromStr;
19
20use serde_derive::{Deserialize, Serialize};
21
22use super::ByteBuf;
23use crate::Error;
24use crate::error::{Result, invalid_data};
25
26/// Entry in a manifest list.
27#[derive(Debug, PartialEq, Clone, Eq, Hash)]
28pub struct ManifestFile {
29    /// field: 500
30    ///
31    /// Location of the manifest file
32    pub manifest_path: String,
33    /// field: 501
34    ///
35    /// Length of the manifest file in bytes
36    pub manifest_length: i64,
37    /// field: 502
38    ///
39    /// ID of a partition spec used to write the manifest; must be listed
40    /// in table metadata partition-specs
41    pub partition_spec_id: i32,
42    /// field: 517
43    ///
44    /// The type of files tracked by the manifest, either data or delete
45    /// files; 0 for all v1 manifests
46    pub content: ManifestContentType,
47    /// field: 515
48    ///
49    /// The sequence number when the manifest was added to the table; use 0
50    /// when reading v1 manifest lists
51    pub sequence_number: i64,
52    /// field: 516
53    ///
54    /// The minimum data sequence number of all live data or delete files in
55    /// the manifest; use 0 when reading v1 manifest lists
56    pub min_sequence_number: i64,
57    /// field: 503
58    ///
59    /// ID of the snapshot where the manifest file was added
60    pub added_snapshot_id: i64,
61    /// field: 504
62    ///
63    /// Number of entries in the manifest that have status ADDED, when null
64    /// this is assumed to be non-zero
65    pub added_files_count: Option<u32>,
66    /// field: 505
67    ///
68    /// Number of entries in the manifest that have status EXISTING (0),
69    /// when null this is assumed to be non-zero
70    pub existing_files_count: Option<u32>,
71    /// field: 506
72    ///
73    /// Number of entries in the manifest that have status DELETED (2),
74    /// when null this is assumed to be non-zero
75    pub deleted_files_count: Option<u32>,
76    /// field: 512
77    ///
78    /// Number of rows in all of files in the manifest that have status
79    /// ADDED, when null this is assumed to be non-zero
80    pub added_rows_count: Option<u64>,
81    /// field: 513
82    ///
83    /// Number of rows in all of files in the manifest that have status
84    /// EXISTING, when null this is assumed to be non-zero
85    pub existing_rows_count: Option<u64>,
86    /// field: 514
87    ///
88    /// Number of rows in all of files in the manifest that have status
89    /// DELETED, when null this is assumed to be non-zero
90    pub deleted_rows_count: Option<u64>,
91    /// field: 507
92    /// element_field: 508
93    ///
94    /// A list of field summaries for each partition field in the spec. Each
95    /// field in the list corresponds to a field in the manifest file’s
96    /// partition spec.
97    pub partitions: Option<Vec<FieldSummary>>,
98    /// field: 519
99    ///
100    /// Implementation-specific key metadata for encryption
101    pub key_metadata: Option<Vec<u8>>,
102    /// field 520
103    ///
104    /// The starting _row_id to assign to rows added by ADDED data files
105    pub first_row_id: Option<u64>,
106}
107
108impl ManifestFile {
109    /// Checks if the manifest file has any added files.
110    pub fn has_added_files(&self) -> bool {
111        self.added_files_count.map(|c| c > 0).unwrap_or(true)
112    }
113
114    /// Checks whether this manifest contains entries with DELETED status.
115    pub fn has_deleted_files(&self) -> bool {
116        self.deleted_files_count.map(|c| c > 0).unwrap_or(true)
117    }
118
119    /// Checks if the manifest file has any existed files.
120    pub fn has_existing_files(&self) -> bool {
121        self.existing_files_count.map(|c| c > 0).unwrap_or(true)
122    }
123}
124
125/// The type of files tracked by the manifest, either data or delete files; Data(0) for all v1 manifests
126#[derive(Debug, PartialEq, Clone, Copy, Eq, Hash, Default)]
127pub enum ManifestContentType {
128    /// The manifest content is data.
129    #[default]
130    Data = 0,
131    /// The manifest content is deletes.
132    Deletes = 1,
133}
134
135impl FromStr for ManifestContentType {
136    type Err = Error;
137
138    fn from_str(s: &str) -> Result<Self> {
139        match s {
140            "data" => Ok(ManifestContentType::Data),
141            "deletes" => Ok(ManifestContentType::Deletes),
142            _ => Err(invalid_data!("Invalid manifest content type: {s}")),
143        }
144    }
145}
146
147impl std::fmt::Display for ManifestContentType {
148    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
149        match self {
150            ManifestContentType::Data => write!(f, "data"),
151            ManifestContentType::Deletes => write!(f, "deletes"),
152        }
153    }
154}
155
156impl TryFrom<i32> for ManifestContentType {
157    type Error = Error;
158
159    fn try_from(value: i32) -> std::result::Result<Self, Self::Error> {
160        match value {
161            0 => Ok(ManifestContentType::Data),
162            1 => Ok(ManifestContentType::Deletes),
163            _ => Err(invalid_data!(
164                "Invalid manifest content type. Expected 0 or 1, got {value}"
165            )),
166        }
167    }
168}
169
170/// Field summary for partition field in the spec.
171///
172/// Each field in the list corresponds to a field in the manifest file’s partition spec.
173#[derive(Debug, Serialize, Deserialize, PartialEq, Eq, Clone, Default, Hash)]
174pub struct FieldSummary {
175    /// field: 509
176    ///
177    /// Whether the manifest contains at least one partition with a null
178    /// value for the field
179    pub contains_null: bool,
180    /// field: 518
181    /// Whether the manifest contains at least one partition with a NaN
182    /// value for the field
183    pub contains_nan: Option<bool>,
184    /// field: 510
185    /// The minimum value for the field in the manifests
186    /// partitions.
187    pub lower_bound: Option<ByteBuf>,
188    /// field: 511
189    /// The maximum value for the field in the manifests
190    /// partitions.
191    pub upper_bound: Option<ByteBuf>,
192}
193
194#[cfg(test)]
195mod test {
196    use std::sync::Arc;
197
198    use super::{ManifestContentType, ManifestFile};
199    use crate::ErrorKind;
200    use crate::encryption::{EncryptedOutputFile, StandardKeyMetadata};
201    use crate::io::FileIO;
202    use crate::spec::{
203        DataContentType, DataFileBuilder, DataFileFormat, ManifestEntry, ManifestReader,
204        ManifestStatus, ManifestWriterBuilder, NestedField, PartitionSpec, PrimitiveType, Schema,
205        SchemaRef, Type,
206    };
207    use crate::test_utils::test_runtime;
208
209    #[test]
210    fn test_manifest_content_type_default() {
211        assert_eq!(ManifestContentType::default(), ManifestContentType::Data);
212    }
213
214    #[test]
215    fn test_manifest_content_type_default_value() {
216        assert_eq!(ManifestContentType::default() as i32, 0);
217    }
218
219    /// A single-field schema used by the manifest-writing test helpers.
220    fn test_schema() -> SchemaRef {
221        Arc::new(
222            Schema::builder()
223                .with_fields(vec![Arc::new(NestedField::optional(
224                    1,
225                    "id",
226                    Type::Primitive(PrimitiveType::Long),
227                ))])
228                .build()
229                .unwrap(),
230        )
231    }
232
233    /// Writes a single-entry v3 data manifest to `io` at `path`, without
234    /// encryption, and returns the resulting [`ManifestFile`].
235    async fn write_manifest(io: &FileIO, path: &str) -> ManifestFile {
236        let schema = test_schema();
237        let partition_spec = PartitionSpec::builder(schema.clone())
238            .with_spec_id(0)
239            .build()
240            .unwrap();
241
242        let output_file = io.new_output(path).unwrap();
243        let mut writer = ManifestWriterBuilder::new(output_file, Some(1), schema, partition_spec)
244            .build_v3_data();
245
246        writer
247            .add_entry(data_entry(ManifestStatus::Added, 100, None))
248            .unwrap();
249
250        writer.write_manifest_file().await.unwrap()
251    }
252
253    /// Writes a single-entry v3 data manifest to `io` at `path`, encrypting it
254    /// with `key_metadata`, and returns the resulting [`ManifestFile`].
255    async fn write_encrypted_manifest(
256        io: &FileIO,
257        path: &str,
258        key_metadata: StandardKeyMetadata,
259    ) -> ManifestFile {
260        let schema = test_schema();
261        let partition_spec = PartitionSpec::builder(schema.clone())
262            .with_spec_id(0)
263            .build()
264            .unwrap();
265
266        let output_file = io.new_output(path).unwrap();
267        let encrypted_output = EncryptedOutputFile::new(output_file, key_metadata);
268
269        let mut writer = ManifestWriterBuilder::new_from_encrypted(
270            encrypted_output,
271            Some(1),
272            schema,
273            partition_spec,
274        )
275        .expect("Expected a valid writer")
276        .build_v3_data();
277
278        writer
279            .add_entry(data_entry(ManifestStatus::Added, 100, None))
280            .unwrap();
281
282        writer.write_manifest_file().await.unwrap()
283    }
284
285    #[tokio::test]
286    async fn test_load_manifest_decrypts_when_key_metadata_present() {
287        let key_metadata = StandardKeyMetadata::try_new(b"0123456789abcdef")
288            .unwrap()
289            .with_aad_prefix(b"test-aad-prefix!");
290        let io = FileIO::new_with_memory();
291        let path = "memory:///test/encrypted_manifest.avro";
292        let manifest_file = write_encrypted_manifest(&io, path, key_metadata.clone()).await;
293        let size = io.new_input(path).unwrap().metadata().await.unwrap().size;
294        assert_eq!(manifest_file.manifest_length, size as i64);
295        assert_eq!(
296            StandardKeyMetadata::decode(manifest_file.key_metadata.as_ref().unwrap()).unwrap(),
297            key_metadata.with_file_length(size)
298        );
299
300        let manifest = ManifestReader::new(io, test_runtime())
301            .read(&manifest_file)
302            .await
303            .unwrap();
304        assert_eq!(manifest.entries().len(), 1);
305        assert_eq!(
306            manifest.entries()[0].file_path(),
307            "s3://bucket/table/data/00000.parquet"
308        );
309        assert_eq!(manifest.entries()[0].data_file.record_count, 100);
310    }
311
312    #[tokio::test]
313    async fn test_load_manifest_fails_with_wrong_key() {
314        let key_metadata = StandardKeyMetadata::try_new(b"0123456789abcdef")
315            .unwrap()
316            .with_aad_prefix(b"test-aad-prefix!");
317
318        let io = FileIO::new_with_memory();
319        let path = "memory:///test/wrong_key_manifest.avro";
320        let mut manifest_file = write_encrypted_manifest(&io, path, key_metadata).await;
321
322        // Point the manifest file at key metadata carrying a different DEK (but
323        // the same AAD prefix). The bytes on disk were encrypted with the
324        // original key, so GCM authentication must fail rather than silently
325        // returning garbage.
326        let wrong_key_metadata = StandardKeyMetadata::try_new(b"fedcba9876543210")
327            .unwrap()
328            .with_aad_prefix(b"test-aad-prefix!")
329            .with_file_length(manifest_file.manifest_length as u64);
330        manifest_file.key_metadata = Some(wrong_key_metadata.encode().unwrap().to_vec());
331
332        let err = ManifestReader::new(io, test_runtime())
333            .read(&manifest_file)
334            .await
335            .expect_err("read must fail when decrypting with the wrong key");
336        assert_eq!(err.kind(), ErrorKind::Unexpected);
337    }
338
339    #[tokio::test]
340    async fn test_load_manifest_fails_with_wrong_aad() {
341        let key_metadata = StandardKeyMetadata::try_new(b"0123456789abcdef")
342            .unwrap()
343            .with_aad_prefix(b"test-aad-prefix!");
344
345        let io = FileIO::new_with_memory();
346        let path = "memory:///test/wrong_aad_manifest.avro";
347        let mut manifest_file = write_encrypted_manifest(&io, path, key_metadata).await;
348
349        // Point the manifest file at key metadata carrying the correct DEK but a
350        // different AAD prefix. The per-block AAD is `aad_prefix || block_index`,
351        // so GCM authentication must fail even though the key is right.
352        let wrong_aad_metadata = StandardKeyMetadata::try_new(b"0123456789abcdef")
353            .unwrap()
354            .with_aad_prefix(b"wrong-aad-prefix")
355            .with_file_length(manifest_file.manifest_length as u64);
356        manifest_file.key_metadata = Some(wrong_aad_metadata.encode().unwrap().to_vec());
357
358        let err = ManifestReader::new(io, test_runtime())
359            .read(&manifest_file)
360            .await
361            .expect_err("read must fail when decrypting with the wrong AAD prefix");
362        assert_eq!(err.kind(), ErrorKind::Unexpected);
363    }
364
365    /// Builds a data-file manifest entry with the given status, record count,
366    /// and pre-existing `first_row_id`.
367    fn data_entry(
368        status: ManifestStatus,
369        record_count: u64,
370        first_row_id: Option<i64>,
371    ) -> ManifestEntry {
372        let data_file = DataFileBuilder::default()
373            .content(DataContentType::Data)
374            .file_path("s3://bucket/table/data/00000.parquet".to_string())
375            .file_format(DataFileFormat::Parquet)
376            .file_size_in_bytes(4096)
377            .record_count(record_count)
378            .first_row_id(first_row_id)
379            .build()
380            .unwrap();
381
382        ManifestEntry::builder()
383            .status(status)
384            .data_file(data_file)
385            .build()
386    }
387
388    #[tokio::test]
389    async fn test_load_manifest_reads_written_entries() {
390        let io = FileIO::new_with_memory();
391        let path = "memory:///test/plaintext_manifest.avro";
392        let manifest_file = write_manifest(&io, path).await;
393        assert_eq!(manifest_file.key_metadata, None);
394
395        let manifest = ManifestReader::new(io, test_runtime())
396            .read(&manifest_file)
397            .await
398            .unwrap();
399        assert_eq!(manifest.entries().len(), 1);
400        assert_eq!(
401            manifest.entries()[0].file_path(),
402            "s3://bucket/table/data/00000.parquet"
403        );
404        assert_eq!(manifest.entries()[0].data_file.record_count, 100);
405    }
406
407    /// End-to-end: writing a v3 data manifest, stamping a manifest-level
408    /// `first_row_id`, and loading it must assign inherited `first_row_id`s to
409    /// the entries. This exercises the wiring in [`ManifestReader`] and the
410    /// write/read round-trip that leaves per-file `first_row_id` as `None`.
411    #[tokio::test]
412    async fn test_load_manifest_assigns_first_row_ids() {
413        let io = FileIO::new_with_memory();
414        let path = "memory:///test/first_row_id_manifest.avro";
415        let mut manifest_file = write_manifest(&io, path).await;
416
417        // Stamp a manifest-level first_row_id, as the manifest-list writer would.
418        manifest_file.first_row_id = Some(1000);
419
420        let manifest = ManifestReader::new(io, test_runtime())
421            .read(&manifest_file)
422            .await
423            .unwrap();
424        assert_eq!(manifest.entries().len(), 1);
425        assert_eq!(manifest.entries()[0].data_file().first_row_id(), Some(1000));
426    }
427}