1use super::Manifest;
19use crate::encryption::{EncryptedInputFile, StandardKeyMetadata};
20use crate::error::{Result, invalid_data};
21use crate::io::FileIO;
22use crate::runtime::Runtime;
23use crate::spec::{ManifestContentType, ManifestEntry, ManifestFile};
24
25pub struct ManifestReader {
28 file_io: FileIO,
29 runtime: Runtime,
32}
33
34impl ManifestReader {
35 pub(crate) fn new(file_io: FileIO, runtime: Runtime) -> Self {
37 Self { file_io, runtime }
38 }
39
40 pub async fn read(self, manifest_file: &ManifestFile) -> Result<Manifest> {
48 let input_file = self.file_io.new_input(&manifest_file.manifest_path)?;
49 let key_metadata = manifest_file
50 .key_metadata
51 .as_deref()
52 .map(StandardKeyMetadata::decode)
53 .transpose()?;
54 let bytes = match key_metadata {
55 Some(key_metadata) => {
56 EncryptedInputFile::new(input_file, key_metadata)
57 .read()
58 .await?
59 }
60 None => input_file.read().await?,
61 };
62
63 let location = manifest_file.manifest_path.clone();
64 let (metadata, mut entries) = self
65 .runtime
66 .cpu()
67 .spawn(async move { Manifest::try_from_avro_bytes(&bytes, Some(&location)) })
68 .await
69 .map_err(|err| err.with_context("manifest", &manifest_file.manifest_path))??;
72
73 for entry in &mut entries {
74 entry.inherit_data(manifest_file);
75 }
76
77 self.assign_first_row_ids(manifest_file, &mut entries)?;
78
79 Ok(Manifest::new(metadata, entries))
80 }
81
82 fn assign_first_row_ids(
92 &self,
93 manifest_file: &ManifestFile,
94 entries: &mut [ManifestEntry],
95 ) -> Result<()> {
96 if manifest_file.content != ManifestContentType::Data {
101 if let Some(manifest_first_row_id) = manifest_file.first_row_id {
102 tracing::warn!(
103 "Ignoring first_row_id {manifest_first_row_id} on delete manifest {}",
104 manifest_file.manifest_path
105 );
106 }
107
108 return Ok(());
109 }
110
111 let Some(manifest_first_row_id) = manifest_file.first_row_id else {
112 for entry in entries {
115 entry.data_file.first_row_id = None;
116 }
117
118 return Ok(());
119 };
120
121 let mut next_row_id = i64::try_from(manifest_first_row_id).map_err(|_| {
122 invalid_data!("Invalid first_row_id: {manifest_first_row_id} (exceeds i64::MAX)")
123 })?;
124
125 for entry in entries {
126 if !entry.is_alive() {
127 continue;
128 }
129
130 if entry.data_file.first_row_id.is_none() {
131 let file_first_row_id = next_row_id;
132 entry.data_file.first_row_id = Some(file_first_row_id);
133 let record_count = entry.data_file.record_count;
134 next_row_id = file_first_row_id.checked_add_unsigned(record_count).ok_or_else(|| {
135 invalid_data!("Row ID overflow assigning first_row_id in {}. File first_row_id: {file_first_row_id}, record count: {record_count}",
136 manifest_file.manifest_path)
137 })?;
138 }
139 }
140
141 Ok(())
142 }
143}
144
145#[cfg(test)]
146mod tests {
147 use std::collections::HashMap;
148 use std::sync::Arc;
149
150 use super::*;
151 use crate::ErrorKind;
152 use crate::encryption::{EncryptedOutputFile, StandardKeyMetadata};
153 use crate::io::FileIO;
154 use crate::spec::{
155 DataContentType, DataFile, DataFileBuilder, DataFileFormat, ManifestEntry, ManifestStatus,
156 ManifestWriterBuilder, NestedField, PartitionSpec, PrimitiveType, Schema, Struct, Type,
157 };
158 use crate::test_utils::test_runtime;
159
160 #[tokio::test]
161 async fn test_read_plaintext_manifest_inherits_entries() {
162 let schema = test_schema();
163 let partition_spec = PartitionSpec::builder(schema.clone())
164 .with_spec_id(0)
165 .build()
166 .unwrap();
167
168 let io = FileIO::new_with_memory();
169 let path = "memory:///table/metadata/plain.avro";
170 let mut writer = ManifestWriterBuilder::new(
171 io.new_output(path).unwrap(),
172 Some(1),
173 schema.clone(),
174 partition_spec,
175 )
176 .build_v2_data();
177 writer.add_entry(test_entry()).unwrap();
178 let manifest_file = writer.write_manifest_file().await.unwrap();
180
181 let manifest = ManifestReader::new(io, test_runtime())
182 .read(&manifest_file)
183 .await
184 .unwrap();
185 assert_eq!(manifest.entries().len(), 1);
186 assert_eq!(
187 manifest.entries()[0].data_file().file_path(),
188 "memory:///table/data/00000.parquet"
189 );
190 assert_eq!(
192 manifest.entries()[0].sequence_number(),
193 Some(manifest_file.sequence_number)
194 );
195 assert_eq!(
196 manifest.entries()[0].snapshot_id(),
197 Some(manifest_file.added_snapshot_id)
198 );
199 }
200
201 #[tokio::test]
205 async fn test_read_v1_manifest_defaults_sequence_number_to_zero() {
206 let schema = test_schema();
207 let partition_spec = PartitionSpec::builder(schema.clone())
208 .with_spec_id(0)
209 .build()
210 .unwrap();
211
212 let io = FileIO::new_with_memory();
213 let path = "memory:///table/metadata/v1.avro";
214 let mut writer = ManifestWriterBuilder::new(
215 io.new_output(path).unwrap(),
216 Some(1),
217 schema.clone(),
218 partition_spec,
219 )
220 .build_v1();
221 writer.add_entry(test_entry()).unwrap();
222 let mut manifest_file = writer.write_manifest_file().await.unwrap();
223 manifest_file.sequence_number = 36;
224
225 let manifest = ManifestReader::new(io, test_runtime())
226 .read(&manifest_file)
227 .await
228 .unwrap();
229 assert_eq!(manifest.entries()[0].sequence_number(), Some(0));
230 assert_eq!(manifest.entries()[0].file_sequence_number, Some(0));
231 }
232
233 #[tokio::test]
236 async fn test_read_names_manifest_when_parse_is_cancelled() {
237 let schema = test_schema();
238 let partition_spec = PartitionSpec::builder(schema.clone())
239 .with_spec_id(0)
240 .build()
241 .unwrap();
242
243 let io = FileIO::new_with_memory();
244 let path = "memory:///table/metadata/cancelled.avro";
245 let mut writer = ManifestWriterBuilder::new(
246 io.new_output(path).unwrap(),
247 Some(1),
248 schema.clone(),
249 partition_spec,
250 )
251 .build_v2_data();
252 writer.add_entry(test_entry()).unwrap();
253 let manifest_file = writer.write_manifest_file().await.unwrap();
254
255 let cpu_rt = tokio::runtime::Builder::new_multi_thread().build().unwrap();
256 let runtime = Runtime::new(&cpu_rt);
257 cpu_rt.shutdown_background();
258
259 let err = ManifestReader::new(io, runtime)
260 .read(&manifest_file)
261 .await
262 .expect_err("a cancelled parse must fail the read");
263 assert_eq!(err.kind(), ErrorKind::Unexpected);
264 assert!(err.to_string().contains(path), "{err}");
265 }
266
267 #[tokio::test]
268 async fn test_read_encrypted_manifest_roundtrip() {
269 let schema = test_schema();
270 let partition_spec = PartitionSpec::builder(schema.clone())
271 .with_spec_id(0)
272 .build()
273 .unwrap();
274
275 let io = FileIO::new_with_memory();
276 let path = "memory:///table/metadata/encrypted.avro";
277 let encrypted_output =
278 EncryptedOutputFile::new(io.new_output(path).unwrap(), key_metadata());
279
280 let mut writer = ManifestWriterBuilder::new_from_encrypted(
281 encrypted_output,
282 Some(1),
283 schema.clone(),
284 partition_spec,
285 )
286 .unwrap()
287 .build_v3_data();
288 writer.add_entry(test_entry()).unwrap();
289 let manifest_file = writer.write_manifest_file().await.unwrap();
291 assert!(manifest_file.key_metadata.is_some());
292
293 let manifest = ManifestReader::new(io.clone(), test_runtime())
295 .read(&manifest_file)
296 .await
297 .unwrap();
298 assert_eq!(manifest.entries().len(), 1);
299 assert_eq!(
300 manifest.entries()[0].data_file().file_path(),
301 "memory:///table/data/00000.parquet"
302 );
303
304 let mut plaintext_entry = manifest_file.clone();
306 plaintext_entry.key_metadata = None;
307 assert!(
308 ManifestReader::new(io, test_runtime())
309 .read(&plaintext_entry)
310 .await
311 .is_err(),
312 "encrypted manifest must not parse as plaintext"
313 );
314 }
315
316 fn test_schema() -> Arc<Schema> {
317 Arc::new(
318 Schema::builder()
319 .with_fields(vec![Arc::new(NestedField::optional(
320 1,
321 "id",
322 Type::Primitive(PrimitiveType::Long),
323 ))])
324 .build()
325 .unwrap(),
326 )
327 }
328
329 fn test_entry() -> ManifestEntry {
330 ManifestEntry {
331 status: ManifestStatus::Added,
332 snapshot_id: None,
333 sequence_number: None,
334 file_sequence_number: None,
335 data_file: DataFile {
336 content: DataContentType::Data,
337 file_path: "memory:///table/data/00000.parquet".to_string(),
338 file_format: DataFileFormat::Parquet,
339 partition: Struct::empty(),
340 record_count: 1,
341 file_size_in_bytes: 4096,
342 column_sizes: HashMap::new(),
343 value_counts: HashMap::new(),
344 null_value_counts: HashMap::new(),
345 nan_value_counts: HashMap::new(),
346 lower_bounds: HashMap::new(),
347 upper_bounds: HashMap::new(),
348 key_metadata: None,
349 split_offsets: None,
350 equality_ids: None,
351 sort_order_id: None,
352 partition_spec_id: 0,
353 first_row_id: None,
354 referenced_data_file: None,
355 content_offset: None,
356 content_size_in_bytes: None,
357 },
358 }
359 }
360
361 fn key_metadata() -> StandardKeyMetadata {
362 StandardKeyMetadata::try_new(b"0123456789abcdef").unwrap()
363 }
364
365 fn test_reader() -> ManifestReader {
368 ManifestReader::new(FileIO::new_with_memory(), test_runtime())
369 }
370
371 fn data_entry(
374 status: ManifestStatus,
375 record_count: u64,
376 first_row_id: Option<i64>,
377 ) -> ManifestEntry {
378 let data_file = DataFileBuilder::default()
379 .content(DataContentType::Data)
380 .file_path("s3://bucket/table/data/00000.parquet".to_string())
381 .file_format(DataFileFormat::Parquet)
382 .file_size_in_bytes(4096)
383 .record_count(record_count)
384 .first_row_id(first_row_id)
385 .build()
386 .unwrap();
387
388 ManifestEntry::builder()
389 .status(status)
390 .data_file(data_file)
391 .build()
392 }
393
394 fn manifest_file(content: ManifestContentType, first_row_id: Option<u64>) -> ManifestFile {
397 ManifestFile {
398 manifest_path: "memory:///m.avro".to_string(),
399 manifest_length: 0,
400 partition_spec_id: 0,
401 content,
402 sequence_number: 0,
403 min_sequence_number: 0,
404 added_snapshot_id: 0,
405 added_files_count: None,
406 existing_files_count: None,
407 deleted_files_count: None,
408 added_rows_count: None,
409 existing_rows_count: None,
410 deleted_rows_count: None,
411 partitions: None,
412 key_metadata: None,
413 first_row_id,
414 }
415 }
416
417 #[test]
418 fn test_assign_first_row_ids_interleaved() {
419 let manifest = manifest_file(ManifestContentType::Data, Some(10));
420 let mut entries = vec![
421 data_entry(ManifestStatus::Added, 3, None),
422 data_entry(ManifestStatus::Added, 5, Some(100)),
425 data_entry(ManifestStatus::Deleted, 7, Some(999)),
428 data_entry(ManifestStatus::Existing, 2, None),
429 ];
430
431 test_reader()
432 .assign_first_row_ids(&manifest, &mut entries)
433 .unwrap();
434
435 assert_eq!(entries[0].data_file.first_row_id, Some(10));
436 assert_eq!(entries[1].data_file.first_row_id, Some(100));
437 assert_eq!(entries[2].data_file.first_row_id, Some(999));
438 assert_eq!(entries[3].data_file.first_row_id, Some(13));
440 }
441
442 #[test]
443 fn test_assign_first_row_ids_clears_without_manifest_first_row_id() {
444 let manifest = manifest_file(ManifestContentType::Data, None);
448 let mut entries = vec![
449 data_entry(ManifestStatus::Added, 3, None),
450 data_entry(ManifestStatus::Existing, 5, Some(100)),
451 ];
452
453 test_reader()
454 .assign_first_row_ids(&manifest, &mut entries)
455 .unwrap();
456
457 assert_eq!(entries[0].data_file.first_row_id, None);
458 assert_eq!(entries[1].data_file.first_row_id, None);
459 }
460
461 #[test]
462 fn test_assign_first_row_ids_ignores_delete_manifest() {
463 let manifest = manifest_file(ManifestContentType::Deletes, Some(10));
467 let mut entries = vec![data_entry(ManifestStatus::Added, 3, None)];
468
469 test_reader()
470 .assign_first_row_ids(&manifest, &mut entries)
471 .unwrap();
472
473 assert_eq!(entries[0].data_file.first_row_id, None);
474 }
475
476 #[test]
477 fn test_assign_first_row_ids_rejects_oversized_manifest_first_row_id() {
478 let manifest = manifest_file(ManifestContentType::Data, Some(i64::MAX as u64 + 1));
481 let mut entries = vec![data_entry(ManifestStatus::Added, 3, None)];
482
483 let err = test_reader()
484 .assign_first_row_ids(&manifest, &mut entries)
485 .expect_err("an oversized manifest first_row_id must be rejected");
486 assert_eq!(err.kind(), ErrorKind::DataInvalid);
487 assert!(err.message().contains("Invalid first_row_id"));
488 }
489
490 #[test]
491 fn test_assign_first_row_ids_rejects_counter_overflow() {
492 let manifest = manifest_file(ManifestContentType::Data, Some(i64::MAX as u64));
495 let mut entries = vec![data_entry(ManifestStatus::Added, 1, None)];
496
497 let err = test_reader()
498 .assign_first_row_ids(&manifest, &mut entries)
499 .expect_err("counter overflow past i64::MAX must be rejected");
500 assert_eq!(err.kind(), ErrorKind::DataInvalid);
501 assert!(err.message().contains("Row ID overflow"));
502 }
503}