Skip to main content

iceberg/spec/manifest/
reader.rs

1// Licensed to the Apache Software Foundation (ASF) under one
2// or more contributor license agreements.  See the NOTICE file
3// distributed with this work for additional information
4// regarding copyright ownership.  The ASF licenses this file
5// to you under the Apache License, Version 2.0 (the
6// "License"); you may not use this file except in compliance
7// with the License.  You may obtain a copy of the License at
8//
9//   http://www.apache.org/licenses/LICENSE-2.0
10//
11// Unless required by applicable law or agreed to in writing,
12// software distributed under the License is distributed on an
13// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14// KIND, either express or implied.  See the License for the
15// specific language governing permissions and limitations
16// under the License.
17
18use super::Manifest;
19use crate::encryption::{EncryptedInputFile, StandardKeyMetadata};
20use crate::error::{Result, invalid_data};
21use crate::io::FileIO;
22use crate::runtime::Runtime;
23use crate::spec::{ManifestContentType, ManifestEntry, ManifestFile};
24
25/// Reads a manifest file referenced by a manifest list entry, transparently
26/// decrypting it when the entry records key metadata.
27pub struct ManifestReader {
28    file_io: FileIO,
29    /// Manifests are parsed on this runtime's cpu handle, so the Avro decoding
30    /// of many manifests runs in parallel instead of on the calling task.
31    runtime: Runtime,
32}
33
34impl ManifestReader {
35    /// Create a manifest reader that parses manifests on `runtime`.
36    pub(crate) fn new(file_io: FileIO, runtime: Runtime) -> Self {
37        Self { file_io, runtime }
38    }
39
40    /// Read, decrypt, parse and return the manifest described by
41    /// `manifest_file`.
42    ///
43    /// The file is fetched on the calling task and parsed on the reader's cpu
44    /// runtime. A parse that panics, or a cpu runtime that has shut down, returns
45    /// an [`ErrorKind::Unexpected`](crate::ErrorKind::Unexpected) error naming the
46    /// manifest.
47    pub async fn read(self, manifest_file: &ManifestFile) -> Result<Manifest> {
48        let input_file = self.file_io.new_input(&manifest_file.manifest_path)?;
49        let key_metadata = manifest_file
50            .key_metadata
51            .as_deref()
52            .map(StandardKeyMetadata::decode)
53            .transpose()?;
54        let bytes = match key_metadata {
55            Some(key_metadata) => {
56                EncryptedInputFile::new(input_file, key_metadata)
57                    .read()
58                    .await?
59            }
60            None => input_file.read().await?,
61        };
62
63        let location = manifest_file.manifest_path.clone();
64        let (metadata, mut entries) = self
65            .runtime
66            .cpu()
67            .spawn(async move { Manifest::try_from_avro_bytes(&bytes, Some(&location)) })
68            .await
69            // A `JoinError` (the parse panicked, or the cpu runtime shut down)
70            // names no manifest; a parse error already does.
71            .map_err(|err| err.with_context("manifest", &manifest_file.manifest_path))??;
72
73        for entry in &mut entries {
74            entry.inherit_data(manifest_file);
75        }
76
77        self.assign_first_row_ids(manifest_file, &mut entries)?;
78
79        Ok(Manifest::new(metadata, entries))
80    }
81
82    /// Assigns `first_row_id` to the live data-file entries, following the
83    /// row-lineage inheritance rules in
84    /// <https://github.com/apache/iceberg/blob/main/format/spec.md#first-row-id-inheritance>.
85    ///
86    /// With a manifest-level `first_row_id`, each live entry lacking one is
87    /// assigned the running id, which then advances by that entry's record
88    /// count; entries that already carry a `first_row_id` keep it and do not
89    /// advance the counter. Without a manifest-level `first_row_id`, any
90    /// inherited per-entry value is cleared so callers never observe a stale id.
91    fn assign_first_row_ids(
92        &self,
93        manifest_file: &ManifestFile,
94        entries: &mut [ManifestEntry],
95    ) -> Result<()> {
96        // A `first_row_id` is only valid on data manifests. Delete files always
97        // have a null `first_row_id`, so there is nothing to assign or clear; a
98        // stray value on a delete manifest is a spec violation by the writer,
99        // which we surface without failing the read.
100        if manifest_file.content != ManifestContentType::Data {
101            if let Some(manifest_first_row_id) = manifest_file.first_row_id {
102                tracing::warn!(
103                    "Ignoring first_row_id {manifest_first_row_id} on delete manifest {}",
104                    manifest_file.manifest_path
105                );
106            }
107
108            return Ok(());
109        }
110
111        let Some(manifest_first_row_id) = manifest_file.first_row_id else {
112            // A data manifest with no manifest-level `first_row_id` predates row
113            // lineage; clear any per-entry value inherited from an earlier read.
114            for entry in entries {
115                entry.data_file.first_row_id = None;
116            }
117
118            return Ok(());
119        };
120
121        let mut next_row_id = i64::try_from(manifest_first_row_id).map_err(|_| {
122            invalid_data!("Invalid first_row_id: {manifest_first_row_id} (exceeds i64::MAX)")
123        })?;
124
125        for entry in entries {
126            if !entry.is_alive() {
127                continue;
128            }
129
130            if entry.data_file.first_row_id.is_none() {
131                let file_first_row_id = next_row_id;
132                entry.data_file.first_row_id = Some(file_first_row_id);
133                let record_count = entry.data_file.record_count;
134                next_row_id = file_first_row_id.checked_add_unsigned(record_count).ok_or_else(|| {
135                    invalid_data!("Row ID overflow assigning first_row_id in {}. File first_row_id: {file_first_row_id}, record count: {record_count}",
136                            manifest_file.manifest_path)
137                })?;
138            }
139        }
140
141        Ok(())
142    }
143}
144
145#[cfg(test)]
146mod tests {
147    use std::collections::HashMap;
148    use std::sync::Arc;
149
150    use super::*;
151    use crate::ErrorKind;
152    use crate::encryption::{EncryptedOutputFile, StandardKeyMetadata};
153    use crate::io::FileIO;
154    use crate::spec::{
155        DataContentType, DataFile, DataFileBuilder, DataFileFormat, ManifestEntry, ManifestStatus,
156        ManifestWriterBuilder, NestedField, PartitionSpec, PrimitiveType, Schema, Struct, Type,
157    };
158    use crate::test_utils::test_runtime;
159
160    #[tokio::test]
161    async fn test_read_plaintext_manifest_inherits_entries() {
162        let schema = test_schema();
163        let partition_spec = PartitionSpec::builder(schema.clone())
164            .with_spec_id(0)
165            .build()
166            .unwrap();
167
168        let io = FileIO::new_with_memory();
169        let path = "memory:///table/metadata/plain.avro";
170        let mut writer = ManifestWriterBuilder::new(
171            io.new_output(path).unwrap(),
172            Some(1),
173            schema.clone(),
174            partition_spec,
175        )
176        .build_v2_data();
177        writer.add_entry(test_entry()).unwrap();
178        // Writing the manifest yields the manifest list entry describing it.
179        let manifest_file = writer.write_manifest_file().await.unwrap();
180
181        let manifest = ManifestReader::new(io, test_runtime())
182            .read(&manifest_file)
183            .await
184            .unwrap();
185        assert_eq!(manifest.entries().len(), 1);
186        assert_eq!(
187            manifest.entries()[0].data_file().file_path(),
188            "memory:///table/data/00000.parquet"
189        );
190        // Entries must inherit values from the manifest list entry.
191        assert_eq!(
192            manifest.entries()[0].sequence_number(),
193            Some(manifest_file.sequence_number)
194        );
195        assert_eq!(
196            manifest.entries()[0].snapshot_id(),
197            Some(manifest_file.added_snapshot_id)
198        );
199    }
200
201    /// Per spec, inheritance does not apply to v1: a v1 manifest's entries read
202    /// with sequence number 0 even when the manifest list entry referencing them
203    /// carries a nonzero one.
204    #[tokio::test]
205    async fn test_read_v1_manifest_defaults_sequence_number_to_zero() {
206        let schema = test_schema();
207        let partition_spec = PartitionSpec::builder(schema.clone())
208            .with_spec_id(0)
209            .build()
210            .unwrap();
211
212        let io = FileIO::new_with_memory();
213        let path = "memory:///table/metadata/v1.avro";
214        let mut writer = ManifestWriterBuilder::new(
215            io.new_output(path).unwrap(),
216            Some(1),
217            schema.clone(),
218            partition_spec,
219        )
220        .build_v1();
221        writer.add_entry(test_entry()).unwrap();
222        let mut manifest_file = writer.write_manifest_file().await.unwrap();
223        manifest_file.sequence_number = 36;
224
225        let manifest = ManifestReader::new(io, test_runtime())
226            .read(&manifest_file)
227            .await
228            .unwrap();
229        assert_eq!(manifest.entries()[0].sequence_number(), Some(0));
230        assert_eq!(manifest.entries()[0].file_sequence_number, Some(0));
231    }
232
233    /// A parse spawned on a shut-down cpu runtime fails with a `JoinError`,
234    /// which names no file; the reader must add the manifest path itself.
235    #[tokio::test]
236    async fn test_read_names_manifest_when_parse_is_cancelled() {
237        let schema = test_schema();
238        let partition_spec = PartitionSpec::builder(schema.clone())
239            .with_spec_id(0)
240            .build()
241            .unwrap();
242
243        let io = FileIO::new_with_memory();
244        let path = "memory:///table/metadata/cancelled.avro";
245        let mut writer = ManifestWriterBuilder::new(
246            io.new_output(path).unwrap(),
247            Some(1),
248            schema.clone(),
249            partition_spec,
250        )
251        .build_v2_data();
252        writer.add_entry(test_entry()).unwrap();
253        let manifest_file = writer.write_manifest_file().await.unwrap();
254
255        let cpu_rt = tokio::runtime::Builder::new_multi_thread().build().unwrap();
256        let runtime = Runtime::new(&cpu_rt);
257        cpu_rt.shutdown_background();
258
259        let err = ManifestReader::new(io, runtime)
260            .read(&manifest_file)
261            .await
262            .expect_err("a cancelled parse must fail the read");
263        assert_eq!(err.kind(), ErrorKind::Unexpected);
264        assert!(err.to_string().contains(path), "{err}");
265    }
266
267    #[tokio::test]
268    async fn test_read_encrypted_manifest_roundtrip() {
269        let schema = test_schema();
270        let partition_spec = PartitionSpec::builder(schema.clone())
271            .with_spec_id(0)
272            .build()
273            .unwrap();
274
275        let io = FileIO::new_with_memory();
276        let path = "memory:///table/metadata/encrypted.avro";
277        let encrypted_output =
278            EncryptedOutputFile::new(io.new_output(path).unwrap(), key_metadata());
279
280        let mut writer = ManifestWriterBuilder::new_from_encrypted(
281            encrypted_output,
282            Some(1),
283            schema.clone(),
284            partition_spec,
285        )
286        .unwrap()
287        .build_v3_data();
288        writer.add_entry(test_entry()).unwrap();
289        // The returned manifest list entry records the key metadata.
290        let manifest_file = writer.write_manifest_file().await.unwrap();
291        assert!(manifest_file.key_metadata.is_some());
292
293        // Reading with the recorded key metadata must recover the entry.
294        let manifest = ManifestReader::new(io.clone(), test_runtime())
295            .read(&manifest_file)
296            .await
297            .unwrap();
298        assert_eq!(manifest.entries().len(), 1);
299        assert_eq!(
300            manifest.entries()[0].data_file().file_path(),
301            "memory:///table/data/00000.parquet"
302        );
303
304        // Without the key metadata the encrypted bytes must not read as plaintext.
305        let mut plaintext_entry = manifest_file.clone();
306        plaintext_entry.key_metadata = None;
307        assert!(
308            ManifestReader::new(io, test_runtime())
309                .read(&plaintext_entry)
310                .await
311                .is_err(),
312            "encrypted manifest must not parse as plaintext"
313        );
314    }
315
316    fn test_schema() -> Arc<Schema> {
317        Arc::new(
318            Schema::builder()
319                .with_fields(vec![Arc::new(NestedField::optional(
320                    1,
321                    "id",
322                    Type::Primitive(PrimitiveType::Long),
323                ))])
324                .build()
325                .unwrap(),
326        )
327    }
328
329    fn test_entry() -> ManifestEntry {
330        ManifestEntry {
331            status: ManifestStatus::Added,
332            snapshot_id: None,
333            sequence_number: None,
334            file_sequence_number: None,
335            data_file: DataFile {
336                content: DataContentType::Data,
337                file_path: "memory:///table/data/00000.parquet".to_string(),
338                file_format: DataFileFormat::Parquet,
339                partition: Struct::empty(),
340                record_count: 1,
341                file_size_in_bytes: 4096,
342                column_sizes: HashMap::new(),
343                value_counts: HashMap::new(),
344                null_value_counts: HashMap::new(),
345                nan_value_counts: HashMap::new(),
346                lower_bounds: HashMap::new(),
347                upper_bounds: HashMap::new(),
348                key_metadata: None,
349                split_offsets: None,
350                equality_ids: None,
351                sort_order_id: None,
352                partition_spec_id: 0,
353                first_row_id: None,
354                referenced_data_file: None,
355                content_offset: None,
356                content_size_in_bytes: None,
357            },
358        }
359    }
360
361    fn key_metadata() -> StandardKeyMetadata {
362        StandardKeyMetadata::try_new(b"0123456789abcdef").unwrap()
363    }
364
365    /// A reader with an unused in-memory `FileIO`, for exercising the pure
366    /// row-id assignment logic without touching storage.
367    fn test_reader() -> ManifestReader {
368        ManifestReader::new(FileIO::new_with_memory(), test_runtime())
369    }
370
371    /// Builds a data-file manifest entry with the given status, record count,
372    /// and pre-existing `first_row_id`.
373    fn data_entry(
374        status: ManifestStatus,
375        record_count: u64,
376        first_row_id: Option<i64>,
377    ) -> ManifestEntry {
378        let data_file = DataFileBuilder::default()
379            .content(DataContentType::Data)
380            .file_path("s3://bucket/table/data/00000.parquet".to_string())
381            .file_format(DataFileFormat::Parquet)
382            .file_size_in_bytes(4096)
383            .record_count(record_count)
384            .first_row_id(first_row_id)
385            .build()
386            .unwrap();
387
388        ManifestEntry::builder()
389            .status(status)
390            .data_file(data_file)
391            .build()
392    }
393
394    /// Builds a manifest file with the given content type and manifest-level
395    /// `first_row_id`. Other fields are irrelevant to row-id assignment.
396    fn manifest_file(content: ManifestContentType, first_row_id: Option<u64>) -> ManifestFile {
397        ManifestFile {
398            manifest_path: "memory:///m.avro".to_string(),
399            manifest_length: 0,
400            partition_spec_id: 0,
401            content,
402            sequence_number: 0,
403            min_sequence_number: 0,
404            added_snapshot_id: 0,
405            added_files_count: None,
406            existing_files_count: None,
407            deleted_files_count: None,
408            added_rows_count: None,
409            existing_rows_count: None,
410            deleted_rows_count: None,
411            partitions: None,
412            key_metadata: None,
413            first_row_id,
414        }
415    }
416
417    #[test]
418    fn test_assign_first_row_ids_interleaved() {
419        let manifest = manifest_file(ManifestContentType::Data, Some(10));
420        let mut entries = vec![
421            data_entry(ManifestStatus::Added, 3, None),
422            // A pre-assigned entry between two assigned ones: it keeps its id and
423            // must not advance the running counter.
424            data_entry(ManifestStatus::Added, 5, Some(100)),
425            // A deleted entry with a pre-set id: it is skipped, so the id is
426            // preserved verbatim and does not advance the counter.
427            data_entry(ManifestStatus::Deleted, 7, Some(999)),
428            data_entry(ManifestStatus::Existing, 2, None),
429        ];
430
431        test_reader()
432            .assign_first_row_ids(&manifest, &mut entries)
433            .unwrap();
434
435        assert_eq!(entries[0].data_file.first_row_id, Some(10));
436        assert_eq!(entries[1].data_file.first_row_id, Some(100));
437        assert_eq!(entries[2].data_file.first_row_id, Some(999));
438        // 10 + 3 = 13; the preserved and deleted entries in between do not move it.
439        assert_eq!(entries[3].data_file.first_row_id, Some(13));
440    }
441
442    #[test]
443    fn test_assign_first_row_ids_clears_without_manifest_first_row_id() {
444        // A data manifest with no manifest-level first_row_id predates row
445        // lineage: any per-entry value inherited from an earlier read is cleared
446        // so callers never observe a stale id.
447        let manifest = manifest_file(ManifestContentType::Data, None);
448        let mut entries = vec![
449            data_entry(ManifestStatus::Added, 3, None),
450            data_entry(ManifestStatus::Existing, 5, Some(100)),
451        ];
452
453        test_reader()
454            .assign_first_row_ids(&manifest, &mut entries)
455            .unwrap();
456
457        assert_eq!(entries[0].data_file.first_row_id, None);
458        assert_eq!(entries[1].data_file.first_row_id, None);
459    }
460
461    #[test]
462    fn test_assign_first_row_ids_ignores_delete_manifest() {
463        // A stray first_row_id on a delete manifest is a writer-side spec
464        // violation; the read ignores it rather than failing, and does not
465        // assign ids to the entries.
466        let manifest = manifest_file(ManifestContentType::Deletes, Some(10));
467        let mut entries = vec![data_entry(ManifestStatus::Added, 3, None)];
468
469        test_reader()
470            .assign_first_row_ids(&manifest, &mut entries)
471            .unwrap();
472
473        assert_eq!(entries[0].data_file.first_row_id, None);
474    }
475
476    #[test]
477    fn test_assign_first_row_ids_rejects_oversized_manifest_first_row_id() {
478        // A manifest-level first_row_id above i64::MAX cannot be represented as the
479        // signed running counter and must be rejected.
480        let manifest = manifest_file(ManifestContentType::Data, Some(i64::MAX as u64 + 1));
481        let mut entries = vec![data_entry(ManifestStatus::Added, 3, None)];
482
483        let err = test_reader()
484            .assign_first_row_ids(&manifest, &mut entries)
485            .expect_err("an oversized manifest first_row_id must be rejected");
486        assert_eq!(err.kind(), ErrorKind::DataInvalid);
487        assert!(err.message().contains("Invalid first_row_id"));
488    }
489
490    #[test]
491    fn test_assign_first_row_ids_rejects_counter_overflow() {
492        // Advancing the running counter past i64::MAX must be rejected rather than
493        // wrapping to a negative value that would corrupt subsequent assignments.
494        let manifest = manifest_file(ManifestContentType::Data, Some(i64::MAX as u64));
495        let mut entries = vec![data_entry(ManifestStatus::Added, 1, None)];
496
497        let err = test_reader()
498            .assign_first_row_ids(&manifest, &mut entries)
499            .expect_err("counter overflow past i64::MAX must be rejected");
500        assert_eq!(err.kind(), ErrorKind::DataInvalid);
501        assert!(err.message().contains("Row ID overflow"));
502    }
503}