iceberg/spec/manifest_list/
manifest_file.rs1use std::str::FromStr;
19
20use serde_derive::{Deserialize, Serialize};
21
22use super::ByteBuf;
23use crate::Error;
24use crate::error::{Result, invalid_data};
25
26#[derive(Debug, PartialEq, Clone, Eq, Hash)]
28pub struct ManifestFile {
29 pub manifest_path: String,
33 pub manifest_length: i64,
37 pub partition_spec_id: i32,
42 pub content: ManifestContentType,
47 pub sequence_number: i64,
52 pub min_sequence_number: i64,
57 pub added_snapshot_id: i64,
61 pub added_files_count: Option<u32>,
66 pub existing_files_count: Option<u32>,
71 pub deleted_files_count: Option<u32>,
76 pub added_rows_count: Option<u64>,
81 pub existing_rows_count: Option<u64>,
86 pub deleted_rows_count: Option<u64>,
91 pub partitions: Option<Vec<FieldSummary>>,
98 pub key_metadata: Option<Vec<u8>>,
102 pub first_row_id: Option<u64>,
106}
107
108impl ManifestFile {
109 pub fn has_added_files(&self) -> bool {
111 self.added_files_count.map(|c| c > 0).unwrap_or(true)
112 }
113
114 pub fn has_deleted_files(&self) -> bool {
116 self.deleted_files_count.map(|c| c > 0).unwrap_or(true)
117 }
118
119 pub fn has_existing_files(&self) -> bool {
121 self.existing_files_count.map(|c| c > 0).unwrap_or(true)
122 }
123}
124
125#[derive(Debug, PartialEq, Clone, Copy, Eq, Hash, Default)]
127pub enum ManifestContentType {
128 #[default]
130 Data = 0,
131 Deletes = 1,
133}
134
135impl FromStr for ManifestContentType {
136 type Err = Error;
137
138 fn from_str(s: &str) -> Result<Self> {
139 match s {
140 "data" => Ok(ManifestContentType::Data),
141 "deletes" => Ok(ManifestContentType::Deletes),
142 _ => Err(invalid_data!("Invalid manifest content type: {s}")),
143 }
144 }
145}
146
147impl std::fmt::Display for ManifestContentType {
148 fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
149 match self {
150 ManifestContentType::Data => write!(f, "data"),
151 ManifestContentType::Deletes => write!(f, "deletes"),
152 }
153 }
154}
155
156impl TryFrom<i32> for ManifestContentType {
157 type Error = Error;
158
159 fn try_from(value: i32) -> std::result::Result<Self, Self::Error> {
160 match value {
161 0 => Ok(ManifestContentType::Data),
162 1 => Ok(ManifestContentType::Deletes),
163 _ => Err(invalid_data!(
164 "Invalid manifest content type. Expected 0 or 1, got {value}"
165 )),
166 }
167 }
168}
169
170#[derive(Debug, Serialize, Deserialize, PartialEq, Eq, Clone, Default, Hash)]
174pub struct FieldSummary {
175 pub contains_null: bool,
180 pub contains_nan: Option<bool>,
184 pub lower_bound: Option<ByteBuf>,
188 pub upper_bound: Option<ByteBuf>,
192}
193
194#[cfg(test)]
195mod test {
196 use std::sync::Arc;
197
198 use super::{ManifestContentType, ManifestFile};
199 use crate::ErrorKind;
200 use crate::encryption::{EncryptedOutputFile, StandardKeyMetadata};
201 use crate::io::FileIO;
202 use crate::spec::{
203 DataContentType, DataFileBuilder, DataFileFormat, ManifestEntry, ManifestReader,
204 ManifestStatus, ManifestWriterBuilder, NestedField, PartitionSpec, PrimitiveType, Schema,
205 SchemaRef, Type,
206 };
207 use crate::test_utils::test_runtime;
208
209 #[test]
210 fn test_manifest_content_type_default() {
211 assert_eq!(ManifestContentType::default(), ManifestContentType::Data);
212 }
213
214 #[test]
215 fn test_manifest_content_type_default_value() {
216 assert_eq!(ManifestContentType::default() as i32, 0);
217 }
218
219 fn test_schema() -> SchemaRef {
221 Arc::new(
222 Schema::builder()
223 .with_fields(vec![Arc::new(NestedField::optional(
224 1,
225 "id",
226 Type::Primitive(PrimitiveType::Long),
227 ))])
228 .build()
229 .unwrap(),
230 )
231 }
232
233 async fn write_manifest(io: &FileIO, path: &str) -> ManifestFile {
236 let schema = test_schema();
237 let partition_spec = PartitionSpec::builder(schema.clone())
238 .with_spec_id(0)
239 .build()
240 .unwrap();
241
242 let output_file = io.new_output(path).unwrap();
243 let mut writer = ManifestWriterBuilder::new(output_file, Some(1), schema, partition_spec)
244 .build_v3_data();
245
246 writer
247 .add_entry(data_entry(ManifestStatus::Added, 100, None))
248 .unwrap();
249
250 writer.write_manifest_file().await.unwrap()
251 }
252
253 async fn write_encrypted_manifest(
256 io: &FileIO,
257 path: &str,
258 key_metadata: StandardKeyMetadata,
259 ) -> ManifestFile {
260 let schema = test_schema();
261 let partition_spec = PartitionSpec::builder(schema.clone())
262 .with_spec_id(0)
263 .build()
264 .unwrap();
265
266 let output_file = io.new_output(path).unwrap();
267 let encrypted_output = EncryptedOutputFile::new(output_file, key_metadata);
268
269 let mut writer = ManifestWriterBuilder::new_from_encrypted(
270 encrypted_output,
271 Some(1),
272 schema,
273 partition_spec,
274 )
275 .expect("Expected a valid writer")
276 .build_v3_data();
277
278 writer
279 .add_entry(data_entry(ManifestStatus::Added, 100, None))
280 .unwrap();
281
282 writer.write_manifest_file().await.unwrap()
283 }
284
285 #[tokio::test]
286 async fn test_load_manifest_decrypts_when_key_metadata_present() {
287 let key_metadata = StandardKeyMetadata::try_new(b"0123456789abcdef")
288 .unwrap()
289 .with_aad_prefix(b"test-aad-prefix!");
290 let io = FileIO::new_with_memory();
291 let path = "memory:///test/encrypted_manifest.avro";
292 let manifest_file = write_encrypted_manifest(&io, path, key_metadata.clone()).await;
293 let size = io.new_input(path).unwrap().metadata().await.unwrap().size;
294 assert_eq!(manifest_file.manifest_length, size as i64);
295 assert_eq!(
296 StandardKeyMetadata::decode(manifest_file.key_metadata.as_ref().unwrap()).unwrap(),
297 key_metadata.with_file_length(size)
298 );
299
300 let manifest = ManifestReader::new(io, test_runtime())
301 .read(&manifest_file)
302 .await
303 .unwrap();
304 assert_eq!(manifest.entries().len(), 1);
305 assert_eq!(
306 manifest.entries()[0].file_path(),
307 "s3://bucket/table/data/00000.parquet"
308 );
309 assert_eq!(manifest.entries()[0].data_file.record_count, 100);
310 }
311
312 #[tokio::test]
313 async fn test_load_manifest_fails_with_wrong_key() {
314 let key_metadata = StandardKeyMetadata::try_new(b"0123456789abcdef")
315 .unwrap()
316 .with_aad_prefix(b"test-aad-prefix!");
317
318 let io = FileIO::new_with_memory();
319 let path = "memory:///test/wrong_key_manifest.avro";
320 let mut manifest_file = write_encrypted_manifest(&io, path, key_metadata).await;
321
322 let wrong_key_metadata = StandardKeyMetadata::try_new(b"fedcba9876543210")
327 .unwrap()
328 .with_aad_prefix(b"test-aad-prefix!")
329 .with_file_length(manifest_file.manifest_length as u64);
330 manifest_file.key_metadata = Some(wrong_key_metadata.encode().unwrap().to_vec());
331
332 let err = ManifestReader::new(io, test_runtime())
333 .read(&manifest_file)
334 .await
335 .expect_err("read must fail when decrypting with the wrong key");
336 assert_eq!(err.kind(), ErrorKind::Unexpected);
337 }
338
339 #[tokio::test]
340 async fn test_load_manifest_fails_with_wrong_aad() {
341 let key_metadata = StandardKeyMetadata::try_new(b"0123456789abcdef")
342 .unwrap()
343 .with_aad_prefix(b"test-aad-prefix!");
344
345 let io = FileIO::new_with_memory();
346 let path = "memory:///test/wrong_aad_manifest.avro";
347 let mut manifest_file = write_encrypted_manifest(&io, path, key_metadata).await;
348
349 let wrong_aad_metadata = StandardKeyMetadata::try_new(b"0123456789abcdef")
353 .unwrap()
354 .with_aad_prefix(b"wrong-aad-prefix")
355 .with_file_length(manifest_file.manifest_length as u64);
356 manifest_file.key_metadata = Some(wrong_aad_metadata.encode().unwrap().to_vec());
357
358 let err = ManifestReader::new(io, test_runtime())
359 .read(&manifest_file)
360 .await
361 .expect_err("read must fail when decrypting with the wrong AAD prefix");
362 assert_eq!(err.kind(), ErrorKind::Unexpected);
363 }
364
365 fn data_entry(
368 status: ManifestStatus,
369 record_count: u64,
370 first_row_id: Option<i64>,
371 ) -> ManifestEntry {
372 let data_file = DataFileBuilder::default()
373 .content(DataContentType::Data)
374 .file_path("s3://bucket/table/data/00000.parquet".to_string())
375 .file_format(DataFileFormat::Parquet)
376 .file_size_in_bytes(4096)
377 .record_count(record_count)
378 .first_row_id(first_row_id)
379 .build()
380 .unwrap();
381
382 ManifestEntry::builder()
383 .status(status)
384 .data_file(data_file)
385 .build()
386 }
387
388 #[tokio::test]
389 async fn test_load_manifest_reads_written_entries() {
390 let io = FileIO::new_with_memory();
391 let path = "memory:///test/plaintext_manifest.avro";
392 let manifest_file = write_manifest(&io, path).await;
393 assert_eq!(manifest_file.key_metadata, None);
394
395 let manifest = ManifestReader::new(io, test_runtime())
396 .read(&manifest_file)
397 .await
398 .unwrap();
399 assert_eq!(manifest.entries().len(), 1);
400 assert_eq!(
401 manifest.entries()[0].file_path(),
402 "s3://bucket/table/data/00000.parquet"
403 );
404 assert_eq!(manifest.entries()[0].data_file.record_count, 100);
405 }
406
407 #[tokio::test]
412 async fn test_load_manifest_assigns_first_row_ids() {
413 let io = FileIO::new_with_memory();
414 let path = "memory:///test/first_row_id_manifest.avro";
415 let mut manifest_file = write_manifest(&io, path).await;
416
417 manifest_file.first_row_id = Some(1000);
419
420 let manifest = ManifestReader::new(io, test_runtime())
421 .read(&manifest_file)
422 .await
423 .unwrap();
424 assert_eq!(manifest.entries().len(), 1);
425 assert_eq!(manifest.entries()[0].data_file().first_row_id(), Some(1000));
426 }
427}