iceberg/puffin/
reader.rs

1// Licensed to the Apache Software Foundation (ASF) under one
2// or more contributor license agreements.  See the NOTICE file
3// distributed with this work for additional information
4// regarding copyright ownership.  The ASF licenses this file
5// to you under the Apache License, Version 2.0 (the
6// "License"); you may not use this file except in compliance
7// with the License.  You may obtain a copy of the License at
8//
9//   http://www.apache.org/licenses/LICENSE-2.0
10//
11// Unless required by applicable law or agreed to in writing,
12// software distributed under the License is distributed on an
13// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14// KIND, either express or implied.  See the License for the
15// specific language governing permissions and limitations
16// under the License.
17
18use tokio::sync::OnceCell;
19
20use super::validate_puffin_compression;
21use crate::Result;
22use crate::encryption::EncryptedInputFile;
23use crate::io::{FileRead, InputFile};
24use crate::puffin::blob::Blob;
25use crate::puffin::metadata::{BlobMetadata, FileMetadata};
26
27/// Puffin reader
28pub struct PuffinReader {
29    file_read: Box<dyn FileRead>,
30    file_length: u64,
31    file_metadata: OnceCell<FileMetadata>,
32}
33
34impl PuffinReader {
35    /// Returns a new Puffin reader for an unencrypted file.
36    pub async fn new(input_file: InputFile) -> Result<Self> {
37        let file_length = input_file.metadata().await?.size;
38        let file_read = input_file.reader().await?;
39        Ok(Self::from_parts(file_read, file_length))
40    }
41
42    /// Returns a new Puffin reader from an [`EncryptedInputFile`].
43    ///
44    /// Use this when reading Puffin files with transparent decryption. The
45    /// reader operates over plaintext offsets and length, so all blob and
46    /// footer positions match those written to the unencrypted file.
47    pub async fn new_from_encrypted(encrypted_input: EncryptedInputFile) -> Result<Self> {
48        let file_length = encrypted_input.metadata().await?.size;
49        let file_read = encrypted_input.reader().await?;
50        Ok(Self::from_parts(file_read, file_length))
51    }
52
53    fn from_parts(file_read: Box<dyn FileRead>, file_length: u64) -> Self {
54        Self {
55            file_read,
56            file_length,
57            file_metadata: OnceCell::new(),
58        }
59    }
60
61    /// Returns file metadata
62    pub async fn file_metadata(&self) -> Result<&FileMetadata> {
63        self.file_metadata
64            .get_or_try_init(|| FileMetadata::read(self.file_read.as_ref(), self.file_length))
65            .await
66    }
67
68    /// Returns blob
69    pub async fn blob(&self, blob_metadata: &BlobMetadata) -> Result<Blob> {
70        validate_puffin_compression(blob_metadata.compression_codec)?;
71
72        let start = blob_metadata.offset;
73        let end = start + blob_metadata.length;
74        let bytes = self.file_read.read(start..end).await?;
75        let data = blob_metadata.compression_codec.decompress(bytes.to_vec())?;
76
77        Ok(Blob {
78            r#type: blob_metadata.r#type.clone(),
79            fields: blob_metadata.fields.clone(),
80            snapshot_id: blob_metadata.snapshot_id,
81            sequence_number: blob_metadata.sequence_number,
82            data,
83            properties: blob_metadata.properties.clone(),
84        })
85    }
86}
87
88#[cfg(test)]
89mod tests {
90    use std::collections::HashMap;
91
92    use crate::ErrorKind;
93    use crate::compression::CompressionCodec;
94    use crate::puffin::metadata::BlobMetadata;
95    use crate::puffin::reader::PuffinReader;
96    use crate::puffin::test_utils::{
97        blob_0, blob_1, java_uncompressed_metric_input_file,
98        java_zstd_compressed_metric_input_file, uncompressed_metric_file_metadata,
99        zstd_compressed_metric_file_metadata,
100    };
101
102    #[tokio::test]
103    async fn test_puffin_reader_uncompressed_metric_data() {
104        let input_file = java_uncompressed_metric_input_file();
105        let puffin_reader = PuffinReader::new(input_file).await.unwrap();
106
107        let file_metadata = puffin_reader.file_metadata().await.unwrap().clone();
108        assert_eq!(file_metadata, uncompressed_metric_file_metadata());
109
110        assert_eq!(
111            puffin_reader
112                .blob(file_metadata.blobs.first().unwrap())
113                .await
114                .unwrap(),
115            blob_0()
116        );
117
118        assert_eq!(
119            puffin_reader
120                .blob(file_metadata.blobs.get(1).unwrap())
121                .await
122                .unwrap(),
123            blob_1(),
124        )
125    }
126
127    #[tokio::test]
128    async fn test_puffin_reader_zstd_compressed_metric_data() {
129        let input_file = java_zstd_compressed_metric_input_file();
130        let puffin_reader = PuffinReader::new(input_file).await.unwrap();
131
132        let file_metadata = puffin_reader.file_metadata().await.unwrap().clone();
133        assert_eq!(file_metadata, zstd_compressed_metric_file_metadata());
134
135        assert_eq!(
136            puffin_reader
137                .blob(file_metadata.blobs.first().unwrap())
138                .await
139                .unwrap(),
140            blob_0()
141        );
142
143        assert_eq!(
144            puffin_reader
145                .blob(file_metadata.blobs.get(1).unwrap())
146                .await
147                .unwrap(),
148            blob_1(),
149        )
150    }
151
152    #[tokio::test]
153    async fn test_gzip_compression_rejected_on_blob_access() {
154        // Use a real puffin file
155        let input_file = java_uncompressed_metric_input_file();
156        let reader = PuffinReader::new(input_file).await.unwrap();
157
158        // Create a BlobMetadata with Gzip compression
159        let gzip_blob_metadata = BlobMetadata {
160            r#type: "test-type".to_string(),
161            fields: vec![1],
162            snapshot_id: 1,
163            sequence_number: 1,
164            offset: 4,
165            length: 10,
166            compression_codec: CompressionCodec::gzip_default(),
167            properties: HashMap::new(),
168        };
169
170        // Attempting to access the blob should fail
171        let result = reader.blob(&gzip_blob_metadata).await;
172        assert!(result.is_err());
173        let err = result.unwrap_err();
174        assert_eq!(err.kind(), ErrorKind::DataInvalid);
175        assert!(err.to_string().contains("gzip"));
176        assert!(
177            err.to_string()
178                .contains("is not supported for Puffin files")
179        );
180    }
181}
iceberg/puffin/reader.rs

iceberg/puffin/
reader.rs
