iceberg/spec/manifest/
metadata.rs

1// Licensed to the Apache Software Foundation (ASF) under one
2// or more contributor license agreements.  See the NOTICE file
3// distributed with this work for additional information
4// regarding copyright ownership.  The ASF licenses this file
5// to you under the Apache License, Version 2.0 (the
6// "License"); you may not use this file except in compliance
7// with the License.  You may obtain a copy of the License at
8//
9//   http://www.apache.org/licenses/LICENSE-2.0
10//
11// Unless required by applicable law or agreed to in writing,
12// software distributed under the License is distributed on an
13// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14// KIND, either express or implied.  See the License for the
15// specific language governing permissions and limitations
16// under the License.
17
18use std::collections::HashMap;
19use std::sync::Arc;
20
21use typed_builder::TypedBuilder;
22
23use super::{FormatVersion, ManifestContentType, PartitionSpec, Schema};
24use crate::error::{Result, invalid_data};
25use crate::spec::{PartitionField, SchemaId, SchemaRef};
26
27/// Meta data of a manifest that is stored in the key-value metadata of the Avro file
28#[derive(Debug, PartialEq, Clone, Eq, TypedBuilder)]
29pub struct ManifestMetadata {
30    /// The table schema at the time the manifest
31    /// was written
32    pub schema: SchemaRef,
33    /// ID of the schema used to write the manifest as a string
34    pub schema_id: SchemaId,
35    /// The partition spec used to write the manifest
36    pub partition_spec: PartitionSpec,
37    /// Table format version number of the manifest as a string
38    pub format_version: FormatVersion,
39    /// Type of content files tracked by the manifest: “data” or “deletes”
40    pub content: ManifestContentType,
41}
42
43impl ManifestMetadata {
44    /// Parse from metadata in avro file.
45    pub fn parse(meta: &HashMap<String, Vec<u8>>) -> Result<Self> {
46        let schema = Arc::new({
47            let bs = meta.get("schema").ok_or_else(|| {
48                invalid_data!("schema is required in manifest metadata but not found")
49            })?;
50            serde_json::from_slice::<Schema>(bs).map_err(|err| {
51                invalid_data!("Fail to parse schema in manifest metadata").with_source(err)
52            })?
53        });
54        let schema_id: i32 = meta
55            .get("schema-id")
56            .map(|bs| {
57                String::from_utf8_lossy(bs).parse().map_err(|err| {
58                    invalid_data!("Fail to parse schema id in manifest metadata").with_source(err)
59                })
60            })
61            .transpose()?
62            .unwrap_or(0);
63        let partition_spec = {
64            let fields = {
65                let bs = meta.get("partition-spec").ok_or_else(|| {
66                    invalid_data!("partition-spec is required in manifest metadata but not found")
67                })?;
68                serde_json::from_slice::<Vec<PartitionField>>(bs).map_err(|err| {
69                    invalid_data!("Fail to parse partition spec in manifest metadata")
70                        .with_source(err)
71                })?
72            };
73            let spec_id = meta
74                .get("partition-spec-id")
75                .map(|bs| {
76                    String::from_utf8_lossy(bs).parse().map_err(|err| {
77                        invalid_data!("Fail to parse partition spec id in manifest metadata")
78                            .with_source(err)
79                    })
80                })
81                .transpose()?
82                .unwrap_or(0);
83            PartitionSpec::builder(schema.clone())
84                .with_spec_id(spec_id)
85                .add_unbound_fields(fields.into_iter().map(|f| f.into_unbound()))?
86                .build()?
87        };
88        let format_version = if let Some(bs) = meta.get("format-version") {
89            serde_json::from_slice::<FormatVersion>(bs).map_err(|err| {
90                invalid_data!("Fail to parse format version in manifest metadata").with_source(err)
91            })?
92        } else {
93            FormatVersion::V1
94        };
95        let content = if let Some(v) = meta.get("content") {
96            let v = String::from_utf8_lossy(v);
97            v.parse()?
98        } else {
99            ManifestContentType::Data
100        };
101        Ok(ManifestMetadata {
102            schema,
103            schema_id,
104            partition_spec,
105            format_version,
106            content,
107        })
108    }
109
110    /// Get the schema of table at the time manifest was written
111    pub fn schema(&self) -> &SchemaRef {
112        &self.schema
113    }
114
115    /// Get the ID of schema used to write the manifest
116    pub fn schema_id(&self) -> SchemaId {
117        self.schema_id
118    }
119
120    /// Get the partition spec used to write manifest
121    pub fn partition_spec(&self) -> &PartitionSpec {
122        &self.partition_spec
123    }
124
125    /// Get the table format version
126    pub fn format_version(&self) -> &FormatVersion {
127        &self.format_version
128    }
129
130    /// Get the type of content files tracked by manifest
131    pub fn content(&self) -> &ManifestContentType {
132        &self.content
133    }
134}
iceberg/spec/manifest/metadata.rs

iceberg/spec/manifest/
metadata.rs
