Skip to main content

iceberg/spec/manifest_list/
mod.rs

1// Licensed to the Apache Software Foundation (ASF) under one
2// or more contributor license agreements.  See the NOTICE file
3// distributed with this work for additional information
4// regarding copyright ownership.  The ASF licenses this file
5// to you under the Apache License, Version 2.0 (the
6// "License"); you may not use this file except in compliance
7// with the License.  You may obtain a copy of the License at
8//
9//   http://www.apache.org/licenses/LICENSE-2.0
10//
11// Unless required by applicable law or agreed to in writing,
12// software distributed under the License is distributed on an
13// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14// KIND, either express or implied.  See the License for the
15// specific language governing permissions and limitations
16// under the License.
17
18//! ManifestList for Iceberg.
19
20mod _const_schema;
21pub(super) mod _serde;
22mod manifest_file;
23mod reader;
24mod writer;
25
26use apache_avro::types::Value;
27use apache_avro::{Reader, from_value};
28pub use manifest_file::*;
29pub use reader::*;
30pub use serde_bytes::ByteBuf;
31pub use writer::*;
32
33use self::_const_schema::MANIFEST_LIST_AVRO_SCHEMA_V1;
34pub(crate) use self::_const_schema::{
35    MANIFEST_CONTENT_FIELD_ID, MANIFEST_SEQUENCE_NUMBER_FIELD_ID, MIN_SEQUENCE_NUMBER_FIELD_ID,
36    manifest_file_fields,
37};
38use super::FormatVersion;
39use crate::error::Result;
40
41/// Placeholder for sequence number. The field with this value must be replaced with the actual sequence number before it write.
42pub const UNASSIGNED_SEQUENCE_NUMBER: i64 = -1;
43
44/// Snapshots are embedded in table metadata, but the list of manifests for a
45/// snapshot are stored in a separate manifest list file.
46///
47/// A new manifest list is written for each attempt to commit a snapshot
48/// because the list of manifests always changes to produce a new snapshot.
49/// When a manifest list is written, the (optimistic) sequence number of the
50/// snapshot is written for all new manifest files tracked by the list.
51///
52/// A manifest list includes summary metadata that can be used to avoid
53/// scanning all of the manifests in a snapshot when planning a table scan.
54/// This includes the number of added, existing, and deleted files, and a
55/// summary of values for each field of the partition spec used to write the
56/// manifest.
57#[derive(Debug, Clone, PartialEq)]
58pub struct ManifestList {
59    /// Entries in a manifest list.
60    entries: Vec<ManifestFile>,
61}
62
63impl ManifestList {
64    /// Parse manifest list from bytes.
65    pub fn parse_with_version(bs: &[u8], version: FormatVersion) -> Result<ManifestList> {
66        match version {
67            FormatVersion::V1 => {
68                let reader = Reader::builder(bs)
69                    .reader_schema(&MANIFEST_LIST_AVRO_SCHEMA_V1)
70                    .build()?;
71                let values = Value::Array(reader.collect::<std::result::Result<Vec<Value>, _>>()?);
72                from_value::<_serde::ManifestListV1>(&values)?.try_into()
73            }
74            FormatVersion::V2 => {
75                let reader = Reader::new(bs)?;
76                let values = Value::Array(reader.collect::<std::result::Result<Vec<Value>, _>>()?);
77                from_value::<_serde::ManifestListV2>(&values)?.try_into()
78            }
79            FormatVersion::V3 => {
80                let reader = Reader::new(bs)?;
81                let values = Value::Array(reader.collect::<std::result::Result<Vec<Value>, _>>()?);
82                from_value::<_serde::ManifestListV3>(&values)?.try_into()
83            }
84        }
85    }
86
87    /// Get the entries in the manifest list.
88    pub fn entries(&self) -> &[ManifestFile] {
89        &self.entries
90    }
91
92    /// Take ownership of the entries in the manifest list, consuming it
93    pub fn consume_entries(self) -> impl IntoIterator<Item = ManifestFile> {
94        Box::new(self.entries.into_iter())
95    }
96}
97
98#[cfg(test)]
99mod test {
100    use std::fs;
101
102    use apache_avro::{Codec, Writer};
103    use tempfile::TempDir;
104
105    use super::_const_schema::MANIFEST_LIST_AVRO_SCHEMA_V2;
106    use super::_serde::ManifestFileV2;
107    use super::*;
108    use crate::io::FileIO;
109    use crate::spec::{Datum, FieldSummary, ManifestContentType, ManifestFile};
110
111    #[tokio::test]
112    async fn test_parse_manifest_list_v1() {
113        let manifest_list = ManifestList {
114            entries: vec![
115                ManifestFile {
116                    manifest_path: "/opt/bitnami/spark/warehouse/db/table/metadata/10d28031-9739-484c-92db-cdf2975cead4-m0.avro".to_string(),
117                    manifest_length: 5806,
118                    partition_spec_id: 0,
119                    content: ManifestContentType::Data,
120                    sequence_number: 0,
121                    min_sequence_number: 0,
122                    added_snapshot_id: 1646658105718557341,
123                    added_files_count: Some(3),
124                    existing_files_count: Some(0),
125                    deleted_files_count: Some(0),
126                    added_rows_count: Some(3),
127                    existing_rows_count: Some(0),
128                    deleted_rows_count: Some(0),
129                    partitions: Some(vec![]),
130                    key_metadata: None,
131                    first_row_id: None,
132                }
133            ]
134        };
135
136        let file_io = FileIO::new_with_fs();
137
138        let tmp_dir = TempDir::new().unwrap();
139        let file_name = "simple_manifest_list_v1.avro";
140        let full_path = format!("{}/{}", tmp_dir.path().to_str().unwrap(), file_name);
141
142        let mut writer = ManifestListWriter::v1(
143            file_io
144                .new_output(full_path.clone())
145                .unwrap()
146                .writer()
147                .await
148                .unwrap(),
149            1646658105718557341,
150            Some(1646658105718557341),
151        );
152
153        writer
154            .add_manifests(manifest_list.entries.clone().into_iter())
155            .unwrap();
156        writer.close().await.unwrap();
157
158        let bs = fs::read(full_path).expect("read_file must succeed");
159
160        let parsed_manifest_list =
161            ManifestList::parse_with_version(&bs, FormatVersion::V1).unwrap();
162
163        assert_eq!(manifest_list, parsed_manifest_list);
164    }
165
166    #[tokio::test]
167    async fn test_parse_manifest_list_v2() {
168        let manifest_list = ManifestList {
169            entries: vec![
170                ManifestFile {
171                    manifest_path: "s3a://icebergdata/demo/s1/t1/metadata/05ffe08b-810f-49b3-a8f4-e88fc99b254a-m0.avro".to_string(),
172                    manifest_length: 6926,
173                    partition_spec_id: 1,
174                    content: ManifestContentType::Data,
175                    sequence_number: 1,
176                    min_sequence_number: 1,
177                    added_snapshot_id: 377075049360453639,
178                    added_files_count: Some(1),
179                    existing_files_count: Some(0),
180                    deleted_files_count: Some(0),
181                    added_rows_count: Some(3),
182                    existing_rows_count: Some(0),
183                    deleted_rows_count: Some(0),
184                    partitions: Some(
185                        vec![FieldSummary { contains_null: false, contains_nan: Some(false), lower_bound: Some(Datum::long(1).to_bytes().unwrap()), upper_bound: Some(Datum::long(1).to_bytes().unwrap())}]
186                    ),
187                    key_metadata: None,
188                    first_row_id: None,
189                },
190                ManifestFile {
191                    manifest_path: "s3a://icebergdata/demo/s1/t1/metadata/05ffe08b-810f-49b3-a8f4-e88fc99b254a-m1.avro".to_string(),
192                    manifest_length: 6926,
193                    partition_spec_id: 2,
194                    content: ManifestContentType::Data,
195                    sequence_number: 1,
196                    min_sequence_number: 1,
197                    added_snapshot_id: 377075049360453639,
198                    added_files_count: Some(1),
199                    existing_files_count: Some(0),
200                    deleted_files_count: Some(0),
201                    added_rows_count: Some(3),
202                    existing_rows_count: Some(0),
203                    deleted_rows_count: Some(0),
204                    partitions: Some(
205                        vec![FieldSummary { contains_null: false, contains_nan: Some(false), lower_bound: Some(Datum::float(1.1_f32).to_bytes().unwrap()), upper_bound: Some(Datum::float(2.1_f32).to_bytes().unwrap())}]
206                    ),
207                    key_metadata: None,
208                    first_row_id: None,
209                }
210            ]
211        };
212
213        let file_io = FileIO::new_with_fs();
214
215        let tmp_dir = TempDir::new().unwrap();
216        let file_name = "simple_manifest_list_v1.avro";
217        let full_path = format!("{}/{}", tmp_dir.path().to_str().unwrap(), file_name);
218
219        let mut writer = ManifestListWriter::v2(
220            file_io
221                .new_output(full_path.clone())
222                .unwrap()
223                .writer()
224                .await
225                .unwrap(),
226            1646658105718557341,
227            Some(1646658105718557341),
228            1,
229        );
230
231        writer
232            .add_manifests(manifest_list.entries.clone().into_iter())
233            .unwrap();
234        writer.close().await.unwrap();
235
236        let bs = fs::read(full_path).expect("read_file must succeed");
237
238        let parsed_manifest_list =
239            ManifestList::parse_with_version(&bs, FormatVersion::V2).unwrap();
240
241        assert_eq!(manifest_list, parsed_manifest_list);
242    }
243
244    #[test]
245    fn test_parse_snappy_manifest_list_v2() {
246        let manifest_list = ManifestList {
247            entries: vec![ManifestFile {
248                manifest_path: "s3a://icebergdata/demo/s1/t1/metadata/snappy-m0.avro".to_string(),
249                manifest_length: 6926,
250                partition_spec_id: 1,
251                content: ManifestContentType::Data,
252                sequence_number: 1,
253                min_sequence_number: 1,
254                added_snapshot_id: 377075049360453639,
255                added_files_count: Some(1),
256                existing_files_count: Some(0),
257                deleted_files_count: Some(0),
258                added_rows_count: Some(3),
259                existing_rows_count: Some(0),
260                deleted_rows_count: Some(0),
261                partitions: Some(vec![FieldSummary {
262                    contains_null: false,
263                    contains_nan: Some(false),
264                    lower_bound: Some(Datum::long(1).to_bytes().unwrap()),
265                    upper_bound: Some(Datum::long(1).to_bytes().unwrap()),
266                }]),
267                key_metadata: None,
268                first_row_id: None,
269            }],
270        };
271
272        let manifest_entry: ManifestFileV2 = manifest_list.entries[0].clone().try_into().unwrap();
273        let mut writer =
274            Writer::with_codec(&MANIFEST_LIST_AVRO_SCHEMA_V2, Vec::new(), Codec::Snappy).unwrap();
275        writer.append_ser(manifest_entry).unwrap();
276        let bs = writer.into_inner().unwrap();
277
278        let parsed_manifest_list =
279            ManifestList::parse_with_version(&bs, FormatVersion::V2).unwrap();
280
281        assert_eq!(manifest_list, parsed_manifest_list);
282    }
283
284    #[tokio::test]
285    async fn test_parse_manifest_list_v3() {
286        let manifest_list = ManifestList {
287            entries: vec![
288                ManifestFile {
289                    manifest_path: "s3a://icebergdata/demo/s1/t1/metadata/05ffe08b-810f-49b3-a8f4-e88fc99b254a-m0.avro".to_string(),
290                    manifest_length: 6926,
291                    partition_spec_id: 1,
292                    content: ManifestContentType::Data,
293                    sequence_number: 1,
294                    min_sequence_number: 1,
295                    added_snapshot_id: 377075049360453639,
296                    added_files_count: Some(1),
297                    existing_files_count: Some(0),
298                    deleted_files_count: Some(0),
299                    added_rows_count: Some(3),
300                    existing_rows_count: Some(0),
301                    deleted_rows_count: Some(0),
302                    partitions: Some(
303                        vec![FieldSummary { contains_null: false, contains_nan: Some(false), lower_bound: Some(Datum::long(1).to_bytes().unwrap()), upper_bound: Some(Datum::long(1).to_bytes().unwrap())}]
304                    ),
305                    key_metadata: None,
306                    first_row_id: Some(10),
307                },
308                ManifestFile {
309                    manifest_path: "s3a://icebergdata/demo/s1/t1/metadata/05ffe08b-810f-49b3-a8f4-e88fc99b254a-m1.avro".to_string(),
310                    manifest_length: 6926,
311                    partition_spec_id: 2,
312                    content: ManifestContentType::Data,
313                    sequence_number: 1,
314                    min_sequence_number: 1,
315                    added_snapshot_id: 377075049360453639,
316                    added_files_count: Some(1),
317                    existing_files_count: Some(0),
318                    deleted_files_count: Some(0),
319                    added_rows_count: Some(3),
320                    existing_rows_count: Some(0),
321                    deleted_rows_count: Some(0),
322                    partitions: Some(
323                        vec![FieldSummary { contains_null: false, contains_nan: Some(false), lower_bound: Some(Datum::float(1.1_f32).to_bytes().unwrap()), upper_bound: Some(Datum::float(2.1_f32).to_bytes().unwrap())}]
324                    ),
325                    key_metadata: None,
326                    first_row_id: Some(13),
327                }
328            ]
329        };
330
331        let file_io = FileIO::new_with_fs();
332
333        let tmp_dir = TempDir::new().unwrap();
334        let file_name = "simple_manifest_list_v3.avro";
335        let full_path = format!("{}/{}", tmp_dir.path().to_str().unwrap(), file_name);
336
337        let mut writer = ManifestListWriter::v3(
338            file_io
339                .new_output(full_path.clone())
340                .unwrap()
341                .writer()
342                .await
343                .unwrap(),
344            377075049360453639,
345            Some(377075049360453639),
346            1,
347            Some(10),
348        );
349
350        writer
351            .add_manifests(manifest_list.entries.clone().into_iter())
352            .unwrap();
353        writer.close().await.unwrap();
354
355        let bs = fs::read(full_path).expect("read_file must succeed");
356
357        let parsed_manifest_list =
358            ManifestList::parse_with_version(&bs, FormatVersion::V3).unwrap();
359
360        assert_eq!(manifest_list, parsed_manifest_list);
361    }
362}