iceberg/spec/manifest_list/
mod.rs1mod _const_schema;
21pub(super) mod _serde;
22mod manifest_file;
23mod reader;
24mod writer;
25
26use apache_avro::types::Value;
27use apache_avro::{Reader, from_value};
28pub use manifest_file::*;
29pub use reader::*;
30pub use serde_bytes::ByteBuf;
31pub use writer::*;
32
33use self::_const_schema::MANIFEST_LIST_AVRO_SCHEMA_V1;
34pub(crate) use self::_const_schema::{
35 MANIFEST_CONTENT_FIELD_ID, MANIFEST_SEQUENCE_NUMBER_FIELD_ID, MIN_SEQUENCE_NUMBER_FIELD_ID,
36 manifest_file_fields,
37};
38use super::FormatVersion;
39use crate::error::Result;
40
41pub const UNASSIGNED_SEQUENCE_NUMBER: i64 = -1;
43
44#[derive(Debug, Clone, PartialEq)]
58pub struct ManifestList {
59 entries: Vec<ManifestFile>,
61}
62
63impl ManifestList {
64 pub fn parse_with_version(bs: &[u8], version: FormatVersion) -> Result<ManifestList> {
66 match version {
67 FormatVersion::V1 => {
68 let reader = Reader::builder(bs)
69 .reader_schema(&MANIFEST_LIST_AVRO_SCHEMA_V1)
70 .build()?;
71 let values = Value::Array(reader.collect::<std::result::Result<Vec<Value>, _>>()?);
72 from_value::<_serde::ManifestListV1>(&values)?.try_into()
73 }
74 FormatVersion::V2 => {
75 let reader = Reader::new(bs)?;
76 let values = Value::Array(reader.collect::<std::result::Result<Vec<Value>, _>>()?);
77 from_value::<_serde::ManifestListV2>(&values)?.try_into()
78 }
79 FormatVersion::V3 => {
80 let reader = Reader::new(bs)?;
81 let values = Value::Array(reader.collect::<std::result::Result<Vec<Value>, _>>()?);
82 from_value::<_serde::ManifestListV3>(&values)?.try_into()
83 }
84 }
85 }
86
87 pub fn entries(&self) -> &[ManifestFile] {
89 &self.entries
90 }
91
92 pub fn consume_entries(self) -> impl IntoIterator<Item = ManifestFile> {
94 Box::new(self.entries.into_iter())
95 }
96}
97
98#[cfg(test)]
99mod test {
100 use std::fs;
101
102 use apache_avro::{Codec, Writer};
103 use tempfile::TempDir;
104
105 use super::_const_schema::MANIFEST_LIST_AVRO_SCHEMA_V2;
106 use super::_serde::ManifestFileV2;
107 use super::*;
108 use crate::io::FileIO;
109 use crate::spec::{Datum, FieldSummary, ManifestContentType, ManifestFile};
110
111 #[tokio::test]
112 async fn test_parse_manifest_list_v1() {
113 let manifest_list = ManifestList {
114 entries: vec![
115 ManifestFile {
116 manifest_path: "/opt/bitnami/spark/warehouse/db/table/metadata/10d28031-9739-484c-92db-cdf2975cead4-m0.avro".to_string(),
117 manifest_length: 5806,
118 partition_spec_id: 0,
119 content: ManifestContentType::Data,
120 sequence_number: 0,
121 min_sequence_number: 0,
122 added_snapshot_id: 1646658105718557341,
123 added_files_count: Some(3),
124 existing_files_count: Some(0),
125 deleted_files_count: Some(0),
126 added_rows_count: Some(3),
127 existing_rows_count: Some(0),
128 deleted_rows_count: Some(0),
129 partitions: Some(vec![]),
130 key_metadata: None,
131 first_row_id: None,
132 }
133 ]
134 };
135
136 let file_io = FileIO::new_with_fs();
137
138 let tmp_dir = TempDir::new().unwrap();
139 let file_name = "simple_manifest_list_v1.avro";
140 let full_path = format!("{}/{}", tmp_dir.path().to_str().unwrap(), file_name);
141
142 let mut writer = ManifestListWriter::v1(
143 file_io
144 .new_output(full_path.clone())
145 .unwrap()
146 .writer()
147 .await
148 .unwrap(),
149 1646658105718557341,
150 Some(1646658105718557341),
151 );
152
153 writer
154 .add_manifests(manifest_list.entries.clone().into_iter())
155 .unwrap();
156 writer.close().await.unwrap();
157
158 let bs = fs::read(full_path).expect("read_file must succeed");
159
160 let parsed_manifest_list =
161 ManifestList::parse_with_version(&bs, FormatVersion::V1).unwrap();
162
163 assert_eq!(manifest_list, parsed_manifest_list);
164 }
165
166 #[tokio::test]
167 async fn test_parse_manifest_list_v2() {
168 let manifest_list = ManifestList {
169 entries: vec![
170 ManifestFile {
171 manifest_path: "s3a://icebergdata/demo/s1/t1/metadata/05ffe08b-810f-49b3-a8f4-e88fc99b254a-m0.avro".to_string(),
172 manifest_length: 6926,
173 partition_spec_id: 1,
174 content: ManifestContentType::Data,
175 sequence_number: 1,
176 min_sequence_number: 1,
177 added_snapshot_id: 377075049360453639,
178 added_files_count: Some(1),
179 existing_files_count: Some(0),
180 deleted_files_count: Some(0),
181 added_rows_count: Some(3),
182 existing_rows_count: Some(0),
183 deleted_rows_count: Some(0),
184 partitions: Some(
185 vec![FieldSummary { contains_null: false, contains_nan: Some(false), lower_bound: Some(Datum::long(1).to_bytes().unwrap()), upper_bound: Some(Datum::long(1).to_bytes().unwrap())}]
186 ),
187 key_metadata: None,
188 first_row_id: None,
189 },
190 ManifestFile {
191 manifest_path: "s3a://icebergdata/demo/s1/t1/metadata/05ffe08b-810f-49b3-a8f4-e88fc99b254a-m1.avro".to_string(),
192 manifest_length: 6926,
193 partition_spec_id: 2,
194 content: ManifestContentType::Data,
195 sequence_number: 1,
196 min_sequence_number: 1,
197 added_snapshot_id: 377075049360453639,
198 added_files_count: Some(1),
199 existing_files_count: Some(0),
200 deleted_files_count: Some(0),
201 added_rows_count: Some(3),
202 existing_rows_count: Some(0),
203 deleted_rows_count: Some(0),
204 partitions: Some(
205 vec![FieldSummary { contains_null: false, contains_nan: Some(false), lower_bound: Some(Datum::float(1.1_f32).to_bytes().unwrap()), upper_bound: Some(Datum::float(2.1_f32).to_bytes().unwrap())}]
206 ),
207 key_metadata: None,
208 first_row_id: None,
209 }
210 ]
211 };
212
213 let file_io = FileIO::new_with_fs();
214
215 let tmp_dir = TempDir::new().unwrap();
216 let file_name = "simple_manifest_list_v1.avro";
217 let full_path = format!("{}/{}", tmp_dir.path().to_str().unwrap(), file_name);
218
219 let mut writer = ManifestListWriter::v2(
220 file_io
221 .new_output(full_path.clone())
222 .unwrap()
223 .writer()
224 .await
225 .unwrap(),
226 1646658105718557341,
227 Some(1646658105718557341),
228 1,
229 );
230
231 writer
232 .add_manifests(manifest_list.entries.clone().into_iter())
233 .unwrap();
234 writer.close().await.unwrap();
235
236 let bs = fs::read(full_path).expect("read_file must succeed");
237
238 let parsed_manifest_list =
239 ManifestList::parse_with_version(&bs, FormatVersion::V2).unwrap();
240
241 assert_eq!(manifest_list, parsed_manifest_list);
242 }
243
244 #[test]
245 fn test_parse_snappy_manifest_list_v2() {
246 let manifest_list = ManifestList {
247 entries: vec![ManifestFile {
248 manifest_path: "s3a://icebergdata/demo/s1/t1/metadata/snappy-m0.avro".to_string(),
249 manifest_length: 6926,
250 partition_spec_id: 1,
251 content: ManifestContentType::Data,
252 sequence_number: 1,
253 min_sequence_number: 1,
254 added_snapshot_id: 377075049360453639,
255 added_files_count: Some(1),
256 existing_files_count: Some(0),
257 deleted_files_count: Some(0),
258 added_rows_count: Some(3),
259 existing_rows_count: Some(0),
260 deleted_rows_count: Some(0),
261 partitions: Some(vec![FieldSummary {
262 contains_null: false,
263 contains_nan: Some(false),
264 lower_bound: Some(Datum::long(1).to_bytes().unwrap()),
265 upper_bound: Some(Datum::long(1).to_bytes().unwrap()),
266 }]),
267 key_metadata: None,
268 first_row_id: None,
269 }],
270 };
271
272 let manifest_entry: ManifestFileV2 = manifest_list.entries[0].clone().try_into().unwrap();
273 let mut writer =
274 Writer::with_codec(&MANIFEST_LIST_AVRO_SCHEMA_V2, Vec::new(), Codec::Snappy).unwrap();
275 writer.append_ser(manifest_entry).unwrap();
276 let bs = writer.into_inner().unwrap();
277
278 let parsed_manifest_list =
279 ManifestList::parse_with_version(&bs, FormatVersion::V2).unwrap();
280
281 assert_eq!(manifest_list, parsed_manifest_list);
282 }
283
284 #[tokio::test]
285 async fn test_parse_manifest_list_v3() {
286 let manifest_list = ManifestList {
287 entries: vec![
288 ManifestFile {
289 manifest_path: "s3a://icebergdata/demo/s1/t1/metadata/05ffe08b-810f-49b3-a8f4-e88fc99b254a-m0.avro".to_string(),
290 manifest_length: 6926,
291 partition_spec_id: 1,
292 content: ManifestContentType::Data,
293 sequence_number: 1,
294 min_sequence_number: 1,
295 added_snapshot_id: 377075049360453639,
296 added_files_count: Some(1),
297 existing_files_count: Some(0),
298 deleted_files_count: Some(0),
299 added_rows_count: Some(3),
300 existing_rows_count: Some(0),
301 deleted_rows_count: Some(0),
302 partitions: Some(
303 vec![FieldSummary { contains_null: false, contains_nan: Some(false), lower_bound: Some(Datum::long(1).to_bytes().unwrap()), upper_bound: Some(Datum::long(1).to_bytes().unwrap())}]
304 ),
305 key_metadata: None,
306 first_row_id: Some(10),
307 },
308 ManifestFile {
309 manifest_path: "s3a://icebergdata/demo/s1/t1/metadata/05ffe08b-810f-49b3-a8f4-e88fc99b254a-m1.avro".to_string(),
310 manifest_length: 6926,
311 partition_spec_id: 2,
312 content: ManifestContentType::Data,
313 sequence_number: 1,
314 min_sequence_number: 1,
315 added_snapshot_id: 377075049360453639,
316 added_files_count: Some(1),
317 existing_files_count: Some(0),
318 deleted_files_count: Some(0),
319 added_rows_count: Some(3),
320 existing_rows_count: Some(0),
321 deleted_rows_count: Some(0),
322 partitions: Some(
323 vec![FieldSummary { contains_null: false, contains_nan: Some(false), lower_bound: Some(Datum::float(1.1_f32).to_bytes().unwrap()), upper_bound: Some(Datum::float(2.1_f32).to_bytes().unwrap())}]
324 ),
325 key_metadata: None,
326 first_row_id: Some(13),
327 }
328 ]
329 };
330
331 let file_io = FileIO::new_with_fs();
332
333 let tmp_dir = TempDir::new().unwrap();
334 let file_name = "simple_manifest_list_v3.avro";
335 let full_path = format!("{}/{}", tmp_dir.path().to_str().unwrap(), file_name);
336
337 let mut writer = ManifestListWriter::v3(
338 file_io
339 .new_output(full_path.clone())
340 .unwrap()
341 .writer()
342 .await
343 .unwrap(),
344 377075049360453639,
345 Some(377075049360453639),
346 1,
347 Some(10),
348 );
349
350 writer
351 .add_manifests(manifest_list.entries.clone().into_iter())
352 .unwrap();
353 writer.close().await.unwrap();
354
355 let bs = fs::read(full_path).expect("read_file must succeed");
356
357 let parsed_manifest_list =
358 ManifestList::parse_with_version(&bs, FormatVersion::V3).unwrap();
359
360 assert_eq!(manifest_list, parsed_manifest_list);
361 }
362}