iceberg/spec/
datatypes.rs

1// Licensed to the Apache Software Foundation (ASF) under one
2// or more contributor license agreements.  See the NOTICE file
3// distributed with this work for additional information
4// regarding copyright ownership.  The ASF licenses this file
5// to you under the Apache License, Version 2.0 (the
6// "License"); you may not use this file except in compliance
7// with the License.  You may obtain a copy of the License at
8//
9//   http://www.apache.org/licenses/LICENSE-2.0
10//
11// Unless required by applicable law or agreed to in writing,
12// software distributed under the License is distributed on an
13// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14// KIND, either express or implied.  See the License for the
15// specific language governing permissions and limitations
16// under the License.
17
18/*!
19 * Data Types
20 */
21use std::collections::HashMap;
22use std::convert::identity;
23use std::fmt;
24use std::ops::Index;
25use std::sync::{Arc, OnceLock};
26
27use ::serde::de::{MapAccess, Visitor};
28use serde::de::{Error, IntoDeserializer};
29use serde::{Deserialize, Deserializer, Serialize, Serializer};
30use serde_json::Value as JsonValue;
31
32use super::values::Literal;
33use crate::ensure_data_valid;
34use crate::error::Result;
35use crate::spec::datatypes::_decimal::{MAX_PRECISION, REQUIRED_LENGTH};
36use crate::spec::{FormatVersion, PrimitiveLiteral};
37
38/// Field name for list type.
39pub const LIST_FIELD_NAME: &str = "element";
40/// Field name for map type's key.
41pub const MAP_KEY_FIELD_NAME: &str = "key";
42/// Field name for map type's value.
43pub const MAP_VALUE_FIELD_NAME: &str = "value";
44
45pub(crate) const MAX_DECIMAL_BYTES: u32 = 24;
46pub(crate) const MAX_DECIMAL_PRECISION: u32 = 38;
47
48mod _decimal {
49    use once_cell::sync::Lazy;
50
51    use crate::spec::{MAX_DECIMAL_BYTES, MAX_DECIMAL_PRECISION};
52
53    // Max precision of bytes, starts from 1
54    pub(super) static MAX_PRECISION: Lazy<[u32; MAX_DECIMAL_BYTES as usize]> = Lazy::new(|| {
55        let mut ret: [u32; 24] = [0; 24];
56        for (i, prec) in ret.iter_mut().enumerate() {
57            *prec = 2f64.powi((8 * (i + 1) - 1) as i32).log10().floor() as u32;
58        }
59
60        ret
61    });
62
63    //  Required bytes of precision, starts from 1
64    pub(super) static REQUIRED_LENGTH: Lazy<[u32; MAX_DECIMAL_PRECISION as usize]> =
65        Lazy::new(|| {
66            let mut ret: [u32; MAX_DECIMAL_PRECISION as usize] =
67                [0; MAX_DECIMAL_PRECISION as usize];
68
69            for (i, required_len) in ret.iter_mut().enumerate() {
70                for j in 0..MAX_PRECISION.len() {
71                    if MAX_PRECISION[j] >= ((i + 1) as u32) {
72                        *required_len = (j + 1) as u32;
73                        break;
74                    }
75                }
76            }
77
78            ret
79        });
80}
81
82#[derive(Debug, PartialEq, Eq, Clone)]
83/// All data types are either primitives or nested types, which are maps, lists, or structs.
84pub enum Type {
85    /// Primitive types
86    Primitive(PrimitiveType),
87    /// Struct type
88    Struct(StructType),
89    /// List type.
90    List(ListType),
91    /// Map type
92    Map(MapType),
93    /// Variant Type
94    Variant(VariantType),
95}
96
97impl fmt::Display for Type {
98    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
99        match self {
100            Type::Primitive(primitive) => write!(f, "{primitive}"),
101            Type::Struct(s) => write!(f, "{s}"),
102            Type::List(_) => write!(f, "list"),
103            Type::Map(_) => write!(f, "map"),
104            Type::Variant(_) => write!(f, "variant"),
105        }
106    }
107}
108
109impl Type {
110    /// Whether the type is primitive type.
111    #[inline(always)]
112    pub fn is_primitive(&self) -> bool {
113        matches!(self, Type::Primitive(_))
114    }
115
116    /// Whether the type is struct type.
117    #[inline(always)]
118    pub fn is_struct(&self) -> bool {
119        matches!(self, Type::Struct(_))
120    }
121
122    /// Whether the type is nested type.
123    #[inline(always)]
124    pub fn is_nested(&self) -> bool {
125        matches!(self, Type::Struct(_) | Type::List(_) | Type::Map(_))
126    }
127
128    /// Whether the type is variant type.
129    #[inline(always)]
130    pub fn is_variant(&self) -> bool {
131        matches!(self, Type::Variant(_))
132    }
133
134    /// Minimum [`FormatVersion`] required to support this type, **without** taking
135    /// nested field types into account.
136    ///
137    /// `TimestampNs` / `TimestamptzNs` / `Variant` require [`FormatVersion::V3`]; every
138    /// other type is valid from [`FormatVersion::V1`]. Mirrors Java's
139    /// `Schema.MIN_FORMAT_VERSIONS` (a shallow lookup keyed by type id), so it
140    /// intentionally does not recurse: callers needing the floor for a whole schema
141    /// iterate its flattened fields (see [`Schema::calc_min_compatible_format`]).
142    ///
143    /// [`Schema::calc_min_compatible_format`]: crate::spec::Schema::calc_min_compatible_format
144    pub(crate) fn min_format_version(&self) -> FormatVersion {
145        match self {
146            Type::Primitive(PrimitiveType::TimestampNs | PrimitiveType::TimestamptzNs)
147            | Type::Variant(_) => FormatVersion::V3,
148            _ => FormatVersion::V1,
149        }
150    }
151
152    /// Convert Type to reference of PrimitiveType
153    pub fn as_primitive_type(&self) -> Option<&PrimitiveType> {
154        if let Type::Primitive(primitive_type) = self {
155            Some(primitive_type)
156        } else {
157            None
158        }
159    }
160
161    /// Convert Type to StructType
162    pub fn to_struct_type(self) -> Option<StructType> {
163        if let Type::Struct(struct_type) = self {
164            Some(struct_type)
165        } else {
166            None
167        }
168    }
169
170    /// Return max precision for decimal given [`num_bytes`] bytes.
171    #[inline(always)]
172    pub fn decimal_max_precision(num_bytes: u32) -> Result<u32> {
173        ensure_data_valid!(
174            num_bytes > 0 && num_bytes <= MAX_DECIMAL_BYTES,
175            "Decimal length larger than {MAX_DECIMAL_BYTES} is not supported: {num_bytes}",
176        );
177        Ok(MAX_PRECISION[num_bytes as usize - 1])
178    }
179
180    /// Returns minimum bytes required for decimal with [`precision`].
181    #[inline(always)]
182    pub fn decimal_required_bytes(precision: u32) -> Result<u32> {
183        ensure_data_valid!(
184            precision > 0 && precision <= MAX_DECIMAL_PRECISION,
185            "Decimals with precision larger than {MAX_DECIMAL_PRECISION} are not supported: {precision}",
186        );
187        Ok(REQUIRED_LENGTH[precision as usize - 1])
188    }
189
190    /// Creates  decimal type.
191    #[inline(always)]
192    pub fn decimal(precision: u32, scale: u32) -> Result<Self> {
193        ensure_data_valid!(
194            precision > 0 && precision <= MAX_DECIMAL_PRECISION,
195            "Decimals with precision larger than {MAX_DECIMAL_PRECISION} are not supported: {precision}",
196        );
197        Ok(Type::Primitive(PrimitiveType::Decimal { precision, scale }))
198    }
199
200    /// Check if it's float or double type.
201    #[inline(always)]
202    pub fn is_floating_type(&self) -> bool {
203        matches!(
204            self,
205            Type::Primitive(PrimitiveType::Float) | Type::Primitive(PrimitiveType::Double)
206        )
207    }
208}
209
210impl From<PrimitiveType> for Type {
211    fn from(value: PrimitiveType) -> Self {
212        Self::Primitive(value)
213    }
214}
215
216impl From<StructType> for Type {
217    fn from(value: StructType) -> Self {
218        Type::Struct(value)
219    }
220}
221
222impl From<ListType> for Type {
223    fn from(value: ListType) -> Self {
224        Type::List(value)
225    }
226}
227
228impl From<MapType> for Type {
229    fn from(value: MapType) -> Self {
230        Type::Map(value)
231    }
232}
233
234/// Primitive data types
235#[derive(Debug, Serialize, Deserialize, PartialEq, Eq, Clone, Hash)]
236#[serde(rename_all = "lowercase", remote = "Self")]
237pub enum PrimitiveType {
238    /// True or False
239    Boolean,
240    /// 32-bit signed integer
241    Int,
242    /// 64-bit signed integer
243    Long,
244    /// 32-bit IEEE 754 floating point.
245    Float,
246    /// 64-bit IEEE 754 floating point.
247    Double,
248    /// Fixed point decimal
249    Decimal {
250        /// Precision, must be 38 or less
251        precision: u32,
252        /// Scale
253        scale: u32,
254    },
255    /// Calendar date without timezone or time.
256    Date,
257    /// Time of day in microsecond precision, without date or timezone.
258    Time,
259    /// Timestamp in microsecond precision, without timezone
260    Timestamp,
261    /// Timestamp in microsecond precision, with timezone
262    Timestamptz,
263    /// Timestamp in nanosecond precision, without timezone
264    #[serde(rename = "timestamp_ns")]
265    TimestampNs,
266    /// Timestamp in nanosecond precision with timezone
267    #[serde(rename = "timestamptz_ns")]
268    TimestamptzNs,
269    /// Arbitrary-length character sequences encoded in utf-8
270    String,
271    /// Universally Unique Identifiers, should use 16-byte fixed
272    Uuid,
273    /// Fixed length byte array
274    Fixed(u64),
275    /// Arbitrary-length byte array.
276    Binary,
277}
278
279impl PrimitiveType {
280    /// Check whether literal is compatible with the type.
281    pub fn compatible(&self, literal: &PrimitiveLiteral) -> bool {
282        matches!(
283            (self, literal),
284            (PrimitiveType::Boolean, PrimitiveLiteral::Boolean(_))
285                | (PrimitiveType::Int, PrimitiveLiteral::Int(_))
286                | (PrimitiveType::Long, PrimitiveLiteral::Long(_))
287                | (PrimitiveType::Float, PrimitiveLiteral::Float(_))
288                | (PrimitiveType::Double, PrimitiveLiteral::Double(_))
289                | (PrimitiveType::Decimal { .. }, PrimitiveLiteral::Int128(_))
290                | (PrimitiveType::Date, PrimitiveLiteral::Int(_))
291                | (PrimitiveType::Time, PrimitiveLiteral::Long(_))
292                | (PrimitiveType::Timestamp, PrimitiveLiteral::Long(_))
293                | (PrimitiveType::Timestamptz, PrimitiveLiteral::Long(_))
294                | (PrimitiveType::TimestampNs, PrimitiveLiteral::Long(_))
295                | (PrimitiveType::TimestamptzNs, PrimitiveLiteral::Long(_))
296                | (PrimitiveType::String, PrimitiveLiteral::String(_))
297                | (PrimitiveType::Uuid, PrimitiveLiteral::UInt128(_))
298                | (PrimitiveType::Fixed(_), PrimitiveLiteral::Binary(_))
299                | (PrimitiveType::Binary, PrimitiveLiteral::Binary(_))
300        )
301    }
302}
303
304impl Serialize for Type {
305    fn serialize<S>(&self, serializer: S) -> std::result::Result<S::Ok, S::Error>
306    where S: Serializer {
307        let type_serde = _serde::SerdeType::from(self);
308        type_serde.serialize(serializer)
309    }
310}
311
312impl<'de> Deserialize<'de> for Type {
313    fn deserialize<D>(deserializer: D) -> std::result::Result<Self, D::Error>
314    where D: Deserializer<'de> {
315        let type_serde = _serde::SerdeType::deserialize(deserializer)?;
316        Ok(Type::from(type_serde))
317    }
318}
319
320impl<'de> Deserialize<'de> for PrimitiveType {
321    fn deserialize<D>(deserializer: D) -> std::result::Result<Self, D::Error>
322    where D: Deserializer<'de> {
323        let s = String::deserialize(deserializer)?;
324        if s.starts_with("decimal") {
325            deserialize_decimal(s.into_deserializer())
326        } else if s.starts_with("fixed") {
327            deserialize_fixed(s.into_deserializer())
328        } else {
329            PrimitiveType::deserialize(s.into_deserializer())
330        }
331    }
332}
333
334impl Serialize for PrimitiveType {
335    fn serialize<S>(&self, serializer: S) -> std::result::Result<S::Ok, S::Error>
336    where S: Serializer {
337        match self {
338            PrimitiveType::Decimal { precision, scale } => {
339                serialize_decimal(precision, scale, serializer)
340            }
341            PrimitiveType::Fixed(l) => serialize_fixed(l, serializer),
342            _ => PrimitiveType::serialize(self, serializer),
343        }
344    }
345}
346
347fn deserialize_decimal<'de, D>(deserializer: D) -> std::result::Result<PrimitiveType, D::Error>
348where D: Deserializer<'de> {
349    let s = String::deserialize(deserializer)?;
350    let (precision, scale) = s
351        .trim_start_matches(r"decimal(")
352        .trim_end_matches(')')
353        .split_once(',')
354        .ok_or_else(|| D::Error::custom("Decimal requires precision and scale: {s}"))?;
355
356    Ok(PrimitiveType::Decimal {
357        precision: precision.trim().parse().map_err(D::Error::custom)?,
358        scale: scale.trim().parse().map_err(D::Error::custom)?,
359    })
360}
361
362fn serialize_decimal<S>(
363    precision: &u32,
364    scale: &u32,
365    serializer: S,
366) -> std::result::Result<S::Ok, S::Error>
367where
368    S: Serializer,
369{
370    serializer.serialize_str(&format!("decimal({precision}, {scale})"))
371}
372
373fn deserialize_fixed<'de, D>(deserializer: D) -> std::result::Result<PrimitiveType, D::Error>
374where D: Deserializer<'de> {
375    let fixed = String::deserialize(deserializer)?
376        .trim_start_matches(r"fixed[")
377        .trim_end_matches(']')
378        .to_owned();
379
380    fixed
381        .parse()
382        .map(PrimitiveType::Fixed)
383        .map_err(D::Error::custom)
384}
385
386fn serialize_fixed<S>(value: &u64, serializer: S) -> std::result::Result<S::Ok, S::Error>
387where S: Serializer {
388    serializer.serialize_str(&format!("fixed[{value}]"))
389}
390
391impl fmt::Display for PrimitiveType {
392    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
393        match self {
394            PrimitiveType::Boolean => write!(f, "boolean"),
395            PrimitiveType::Int => write!(f, "int"),
396            PrimitiveType::Long => write!(f, "long"),
397            PrimitiveType::Float => write!(f, "float"),
398            PrimitiveType::Double => write!(f, "double"),
399            PrimitiveType::Decimal { precision, scale } => {
400                write!(f, "decimal({precision}, {scale})")
401            }
402            PrimitiveType::Date => write!(f, "date"),
403            PrimitiveType::Time => write!(f, "time"),
404            PrimitiveType::Timestamp => write!(f, "timestamp"),
405            PrimitiveType::Timestamptz => write!(f, "timestamptz"),
406            PrimitiveType::TimestampNs => write!(f, "timestamp_ns"),
407            PrimitiveType::TimestamptzNs => write!(f, "timestamptz_ns"),
408            PrimitiveType::String => write!(f, "string"),
409            PrimitiveType::Uuid => write!(f, "uuid"),
410            PrimitiveType::Fixed(size) => write!(f, "fixed({size})"),
411            PrimitiveType::Binary => write!(f, "binary"),
412        }
413    }
414}
415
416/// DataType for a specific struct
417#[derive(Debug, Serialize, Clone, Default)]
418#[serde(rename = "struct", tag = "type")]
419pub struct StructType {
420    /// Struct fields
421    fields: Vec<NestedFieldRef>,
422    /// Lookup for index by field id
423    #[serde(skip_serializing)]
424    id_lookup: OnceLock<HashMap<i32, usize>>,
425    #[serde(skip_serializing)]
426    name_lookup: OnceLock<HashMap<String, usize>>,
427}
428
429impl<'de> Deserialize<'de> for StructType {
430    fn deserialize<D>(deserializer: D) -> std::result::Result<Self, D::Error>
431    where D: Deserializer<'de> {
432        #[derive(Deserialize)]
433        #[serde(field_identifier, rename_all = "lowercase")]
434        enum Field {
435            Type,
436            Fields,
437        }
438
439        struct StructTypeVisitor;
440
441        impl<'de> Visitor<'de> for StructTypeVisitor {
442            type Value = StructType;
443
444            fn expecting(&self, formatter: &mut fmt::Formatter) -> fmt::Result {
445                formatter.write_str("struct")
446            }
447
448            fn visit_map<V>(self, mut map: V) -> std::result::Result<StructType, V::Error>
449            where V: MapAccess<'de> {
450                let mut fields = None;
451                while let Some(key) = map.next_key()? {
452                    match key {
453                        Field::Type => {
454                            let type_val: String = map.next_value()?;
455                            if type_val != "struct" {
456                                return Err(Error::custom(format!(
457                                    "expected type 'struct', got '{type_val}'"
458                                )));
459                            }
460                        }
461                        Field::Fields => {
462                            if fields.is_some() {
463                                return Err(Error::duplicate_field("fields"));
464                            }
465                            fields = Some(map.next_value()?);
466                        }
467                    }
468                }
469                let fields: Vec<NestedFieldRef> =
470                    fields.ok_or_else(|| Error::missing_field("fields"))?;
471
472                Ok(StructType::new(fields))
473            }
474        }
475
476        const FIELDS: &[&str] = &["type", "fields"];
477        deserializer.deserialize_struct("struct", FIELDS, StructTypeVisitor)
478    }
479}
480
481impl StructType {
482    /// Creates a struct type with the given fields.
483    pub fn new(fields: Vec<NestedFieldRef>) -> Self {
484        Self {
485            fields,
486            id_lookup: OnceLock::new(),
487            name_lookup: OnceLock::new(),
488        }
489    }
490
491    /// Get struct field with certain id
492    pub fn field_by_id(&self, id: i32) -> Option<&NestedFieldRef> {
493        self.field_id_to_index(id).map(|idx| &self.fields[idx])
494    }
495
496    fn field_id_to_index(&self, field_id: i32) -> Option<usize> {
497        self.id_lookup
498            .get_or_init(|| {
499                HashMap::from_iter(self.fields.iter().enumerate().map(|(i, x)| (x.id, i)))
500            })
501            .get(&field_id)
502            .copied()
503    }
504
505    /// Get struct field with certain field name
506    pub fn field_by_name(&self, name: &str) -> Option<&NestedFieldRef> {
507        self.field_name_to_index(name).map(|idx| &self.fields[idx])
508    }
509
510    fn field_name_to_index(&self, name: &str) -> Option<usize> {
511        self.name_lookup
512            .get_or_init(|| {
513                HashMap::from_iter(
514                    self.fields
515                        .iter()
516                        .enumerate()
517                        .map(|(i, x)| (x.name.clone(), i)),
518                )
519            })
520            .get(name)
521            .copied()
522    }
523
524    /// Get fields.
525    pub fn fields(&self) -> &[NestedFieldRef] {
526        &self.fields
527    }
528}
529
530impl PartialEq for StructType {
531    fn eq(&self, other: &Self) -> bool {
532        self.fields == other.fields
533    }
534}
535
536impl Eq for StructType {}
537
538impl Index<usize> for StructType {
539    type Output = NestedField;
540
541    fn index(&self, index: usize) -> &Self::Output {
542        &self.fields[index]
543    }
544}
545
546impl fmt::Display for StructType {
547    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
548        write!(f, "struct<")?;
549        for field in &self.fields {
550            write!(f, "{}", field.field_type)?;
551        }
552        write!(f, ">")
553    }
554}
555
556#[derive(Debug, PartialEq, Serialize, Deserialize, Eq, Clone)]
557#[serde(from = "SerdeNestedField", into = "SerdeNestedField")]
558/// A struct is a tuple of typed values. Each field in the tuple is named and has an integer id that is unique in the table schema.
559/// Each field can be either optional or required, meaning that values can (or cannot) be null. Fields may be any type.
560/// Fields may have an optional comment or doc string. Fields can have default values.
561pub struct NestedField {
562    /// Id unique in table schema
563    pub id: i32,
564    /// Field Name
565    pub name: String,
566    /// Optional or required
567    pub required: bool,
568    /// Datatype
569    pub field_type: Box<Type>,
570    /// Fields may have an optional comment or doc string.
571    pub doc: Option<String>,
572    /// Used to populate the field’s value for all records that were written before the field was added to the schema
573    pub initial_default: Option<Literal>,
574    /// Used to populate the field’s value for any records written after the field was added to the schema, if the writer does not supply the field’s value
575    pub write_default: Option<Literal>,
576}
577
578#[derive(Debug, Serialize, Deserialize, PartialEq, Eq, Clone)]
579#[serde(rename_all = "kebab-case")]
580struct SerdeNestedField {
581    pub id: i32,
582    pub name: String,
583    pub required: bool,
584    #[serde(rename = "type")]
585    pub field_type: Box<Type>,
586    #[serde(skip_serializing_if = "Option::is_none")]
587    pub doc: Option<String>,
588    #[serde(skip_serializing_if = "Option::is_none")]
589    pub initial_default: Option<JsonValue>,
590    #[serde(skip_serializing_if = "Option::is_none")]
591    pub write_default: Option<JsonValue>,
592}
593
594impl From<SerdeNestedField> for NestedField {
595    fn from(value: SerdeNestedField) -> Self {
596        NestedField {
597            id: value.id,
598            name: value.name,
599            required: value.required,
600            initial_default: value.initial_default.and_then(|x| {
601                Literal::try_from_json(x, &value.field_type)
602                    .ok()
603                    .and_then(identity)
604            }),
605            write_default: value.write_default.and_then(|x| {
606                Literal::try_from_json(x, &value.field_type)
607                    .ok()
608                    .and_then(identity)
609            }),
610            field_type: value.field_type,
611            doc: value.doc,
612        }
613    }
614}
615
616impl From<NestedField> for SerdeNestedField {
617    fn from(value: NestedField) -> Self {
618        let initial_default = value.initial_default.map(|x| x.try_into_json(&value.field_type).expect("We should have checked this in NestedField::with_initial_default, it can't be converted to json value"));
619        let write_default = value.write_default.map(|x| x.try_into_json(&value.field_type).expect("We should have checked this in NestedField::with_write_default, it can't be converted to json value"));
620        SerdeNestedField {
621            id: value.id,
622            name: value.name,
623            required: value.required,
624            field_type: value.field_type,
625            doc: value.doc,
626            initial_default,
627            write_default,
628        }
629    }
630}
631
632/// Reference to nested field.
633pub type NestedFieldRef = Arc<NestedField>;
634
635impl NestedField {
636    /// Construct a new field.
637    pub fn new(id: i32, name: impl ToString, field_type: Type, required: bool) -> Self {
638        Self {
639            id,
640            name: name.to_string(),
641            required,
642            field_type: Box::new(field_type),
643            doc: None,
644            initial_default: None,
645            write_default: None,
646        }
647    }
648
649    /// Construct a required field.
650    pub fn required(id: i32, name: impl ToString, field_type: Type) -> Self {
651        Self::new(id, name, field_type, true)
652    }
653
654    /// Construct an optional field.
655    pub fn optional(id: i32, name: impl ToString, field_type: Type) -> Self {
656        Self::new(id, name, field_type, false)
657    }
658
659    /// Construct list type's element field.
660    pub fn list_element(id: i32, field_type: Type, required: bool) -> Self {
661        Self::new(id, LIST_FIELD_NAME, field_type, required)
662    }
663
664    /// Construct map type's key field.
665    pub fn map_key_element(id: i32, field_type: Type) -> Self {
666        Self::required(id, MAP_KEY_FIELD_NAME, field_type)
667    }
668
669    /// Construct map type's value field.
670    pub fn map_value_element(id: i32, field_type: Type, required: bool) -> Self {
671        Self::new(id, MAP_VALUE_FIELD_NAME, field_type, required)
672    }
673
674    /// Set the field's doc.
675    pub fn with_doc(mut self, doc: impl ToString) -> Self {
676        self.doc = Some(doc.to_string());
677        self
678    }
679
680    /// Set the field's initial default value.
681    pub fn with_initial_default(mut self, value: Literal) -> Self {
682        self.initial_default = Some(value);
683        self
684    }
685
686    /// Set the field's initial default value.
687    pub fn with_write_default(mut self, value: Literal) -> Self {
688        self.write_default = Some(value);
689        self
690    }
691
692    /// Set the id of the field.
693    pub(crate) fn with_id(mut self, id: i32) -> Self {
694        self.id = id;
695        self
696    }
697}
698
699impl fmt::Display for NestedField {
700    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
701        write!(f, "{}: ", self.id)?;
702        write!(f, "{}: ", self.name)?;
703        if self.required {
704            write!(f, "required ")?;
705        } else {
706            write!(f, "optional ")?;
707        }
708        write!(f, "{} ", self.field_type)?;
709        if let Some(doc) = &self.doc {
710            write!(f, "{doc}")?;
711        }
712        Ok(())
713    }
714}
715
716#[derive(Debug, PartialEq, Eq, Clone)]
717/// A list is a collection of values with some element type. The element field has an integer id that is unique in the table schema.
718/// Elements can be either optional or required. Element types may be any type.
719pub struct ListType {
720    /// Element field of list type.
721    pub element_field: NestedFieldRef,
722}
723
724impl ListType {
725    /// Construct a list type with the given element field.
726    pub fn new(element_field: NestedFieldRef) -> Self {
727        Self { element_field }
728    }
729}
730
731/// Module for type serialization/deserialization.
732pub(super) mod _serde {
733    use std::borrow::Cow;
734
735    use serde_derive::{Deserialize, Serialize};
736
737    use crate::spec::datatypes::Type::Map;
738    use crate::spec::datatypes::{
739        ListType, MapType, NestedField, NestedFieldRef, PrimitiveType, StructType, Type,
740        VariantType,
741    };
742
743    /// List type for serialization and deserialization
744    #[derive(Serialize, Deserialize)]
745    #[serde(untagged)]
746    pub(super) enum SerdeType<'a> {
747        #[serde(rename_all = "kebab-case")]
748        List {
749            r#type: String,
750            element_id: i32,
751            element_required: bool,
752            element: Cow<'a, Type>,
753        },
754        Struct {
755            r#type: String,
756            fields: Cow<'a, [NestedFieldRef]>,
757        },
758        #[serde(rename_all = "kebab-case")]
759        Map {
760            r#type: String,
761            key_id: i32,
762            key: Cow<'a, Type>,
763            value_id: i32,
764            value_required: bool,
765            value: Cow<'a, Type>,
766        },
767        Primitive(PrimitiveType),
768        Variant(VariantType),
769    }
770
771    impl From<SerdeType<'_>> for Type {
772        fn from(value: SerdeType) -> Self {
773            match value {
774                SerdeType::List {
775                    r#type: _,
776                    element_id,
777                    element_required,
778                    element,
779                } => Self::List(ListType {
780                    element_field: NestedField::list_element(
781                        element_id,
782                        element.into_owned(),
783                        element_required,
784                    )
785                    .into(),
786                }),
787                SerdeType::Map {
788                    r#type: _,
789                    key_id,
790                    key,
791                    value_id,
792                    value_required,
793                    value,
794                } => Map(MapType {
795                    key_field: NestedField::map_key_element(key_id, key.into_owned()).into(),
796                    value_field: NestedField::map_value_element(
797                        value_id,
798                        value.into_owned(),
799                        value_required,
800                    )
801                    .into(),
802                }),
803                SerdeType::Struct { r#type: _, fields } => {
804                    Self::Struct(StructType::new(fields.into_owned()))
805                }
806                SerdeType::Primitive(p) => Self::Primitive(p),
807                SerdeType::Variant(v) => Self::Variant(v),
808            }
809        }
810    }
811
812    impl<'a> From<&'a Type> for SerdeType<'a> {
813        fn from(value: &'a Type) -> Self {
814            match value {
815                Type::List(list) => SerdeType::List {
816                    r#type: "list".to_string(),
817                    element_id: list.element_field.id,
818                    element_required: list.element_field.required,
819                    element: Cow::Borrowed(&list.element_field.field_type),
820                },
821                Map(map) => SerdeType::Map {
822                    r#type: "map".to_string(),
823                    key_id: map.key_field.id,
824                    key: Cow::Borrowed(&map.key_field.field_type),
825                    value_id: map.value_field.id,
826                    value_required: map.value_field.required,
827                    value: Cow::Borrowed(&map.value_field.field_type),
828                },
829                Type::Struct(s) => SerdeType::Struct {
830                    r#type: "struct".to_string(),
831                    fields: Cow::Borrowed(&s.fields),
832                },
833                Type::Primitive(p) => SerdeType::Primitive(p.clone()),
834                Type::Variant(v) => SerdeType::Variant(*v),
835            }
836        }
837    }
838}
839
840#[derive(Debug, PartialEq, Eq, Clone)]
841/// A map is a collection of key-value pairs with a key type and a value type.
842/// Both the key field and value field each have an integer id that is unique in the table schema.
843/// Map keys are required and map values can be either optional or required.
844/// Both map keys and map values may be any type, including nested types.
845pub struct MapType {
846    /// Field for key.
847    pub key_field: NestedFieldRef,
848    /// Field for value.
849    pub value_field: NestedFieldRef,
850}
851
852impl MapType {
853    /// Construct a map type with the given key and value fields.
854    pub fn new(key_field: NestedFieldRef, value_field: NestedFieldRef) -> Self {
855        Self {
856            key_field,
857            value_field,
858        }
859    }
860
861    /// Construct an optional map type with the given key and value fields.
862    pub fn optional(key_id: i32, key_type: Type, value_id: i32, value_type: Type) -> Self {
863        Self {
864            key_field: NestedField::map_key_element(key_id, key_type).into(),
865            value_field: NestedField::map_value_element(value_id, value_type, false).into(),
866        }
867    }
868
869    /// Construct a required map type with the given key and value fields.
870    pub fn required(key_id: i32, key_type: Type, value_id: i32, value_type: Type) -> Self {
871        Self {
872            key_field: NestedField::map_key_element(key_id, key_type).into(),
873            value_field: NestedField::map_value_element(value_id, value_type, true).into(),
874        }
875    }
876}
877
878/// Variant type - can hold semi-structured data of any type.
879/// This is an Iceberg V3 feature.
880#[derive(Debug, PartialEq, Eq, Clone, Copy, Hash)]
881pub struct VariantType;
882
883impl fmt::Display for VariantType {
884    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
885        write!(f, "variant")
886    }
887}
888
889impl From<VariantType> for Type {
890    fn from(_: VariantType) -> Self {
891        Type::Variant(VariantType)
892    }
893}
894
895impl Serialize for VariantType {
896    fn serialize<S>(&self, serializer: S) -> std::result::Result<S::Ok, S::Error>
897    where S: Serializer {
898        serializer.serialize_str("variant")
899    }
900}
901
902impl<'de> Deserialize<'de> for VariantType {
903    fn deserialize<D>(deserializer: D) -> std::result::Result<Self, D::Error>
904    where D: Deserializer<'de> {
905        let s = String::deserialize(deserializer)?;
906        if s == "variant" {
907            Ok(VariantType)
908        } else {
909            Err(D::Error::custom(format!("expected 'variant', got '{s}'")))
910        }
911    }
912}
913
914#[cfg(test)]
915mod tests {
916    use pretty_assertions::assert_eq;
917    use uuid::Uuid;
918
919    use super::*;
920    use crate::spec::values::PrimitiveLiteral;
921
922    fn check_type_serde(json: &str, expected_type: Type) {
923        let desered_type: Type = serde_json::from_str(json).unwrap();
924        assert_eq!(desered_type, expected_type);
925
926        let sered_json = serde_json::to_string(&expected_type).unwrap();
927        let parsed_json_value = serde_json::from_str::<serde_json::Value>(&sered_json).unwrap();
928        let raw_json_value = serde_json::from_str::<serde_json::Value>(json).unwrap();
929
930        assert_eq!(parsed_json_value, raw_json_value);
931    }
932
933    #[test]
934    fn primitive_type_serde() {
935        let record = r#"
936    {
937        "type": "struct",
938        "fields": [
939            {"id": 1, "name": "bool_field", "required": true, "type": "boolean"},
940            {"id": 2, "name": "int_field", "required": true, "type": "int"},
941            {"id": 3, "name": "long_field", "required": true, "type": "long"},
942            {"id": 4, "name": "float_field", "required": true, "type": "float"},
943            {"id": 5, "name": "double_field", "required": true, "type": "double"},
944            {"id": 6, "name": "decimal_field", "required": true, "type": "decimal(9, 2)"},
945            {"id": 7, "name": "date_field", "required": true, "type": "date"},
946            {"id": 8, "name": "time_field", "required": true, "type": "time"},
947            {"id": 9, "name": "timestamp_field", "required": true, "type": "timestamp"},
948            {"id": 10, "name": "timestamptz_field", "required": true, "type": "timestamptz"},
949            {"id": 11, "name": "timestamp_ns_field", "required": true, "type": "timestamp_ns"},
950            {"id": 12, "name": "timestamptz_ns_field", "required": true, "type": "timestamptz_ns"},
951            {"id": 13, "name": "uuid_field", "required": true, "type": "uuid"},
952            {"id": 14, "name": "fixed_field", "required": true, "type": "fixed[10]"},
953            {"id": 15, "name": "binary_field", "required": true, "type": "binary"},
954            {"id": 16, "name": "string_field", "required": true, "type": "string"}
955        ]
956    }
957    "#;
958
959        check_type_serde(
960            record,
961            Type::Struct(StructType {
962                fields: vec![
963                    NestedField::required(1, "bool_field", Type::Primitive(PrimitiveType::Boolean))
964                        .into(),
965                    NestedField::required(2, "int_field", Type::Primitive(PrimitiveType::Int))
966                        .into(),
967                    NestedField::required(3, "long_field", Type::Primitive(PrimitiveType::Long))
968                        .into(),
969                    NestedField::required(4, "float_field", Type::Primitive(PrimitiveType::Float))
970                        .into(),
971                    NestedField::required(
972                        5,
973                        "double_field",
974                        Type::Primitive(PrimitiveType::Double),
975                    )
976                    .into(),
977                    NestedField::required(
978                        6,
979                        "decimal_field",
980                        Type::Primitive(PrimitiveType::Decimal {
981                            precision: 9,
982                            scale: 2,
983                        }),
984                    )
985                    .into(),
986                    NestedField::required(7, "date_field", Type::Primitive(PrimitiveType::Date))
987                        .into(),
988                    NestedField::required(8, "time_field", Type::Primitive(PrimitiveType::Time))
989                        .into(),
990                    NestedField::required(
991                        9,
992                        "timestamp_field",
993                        Type::Primitive(PrimitiveType::Timestamp),
994                    )
995                    .into(),
996                    NestedField::required(
997                        10,
998                        "timestamptz_field",
999                        Type::Primitive(PrimitiveType::Timestamptz),
1000                    )
1001                    .into(),
1002                    NestedField::required(
1003                        11,
1004                        "timestamp_ns_field",
1005                        Type::Primitive(PrimitiveType::TimestampNs),
1006                    )
1007                    .into(),
1008                    NestedField::required(
1009                        12,
1010                        "timestamptz_ns_field",
1011                        Type::Primitive(PrimitiveType::TimestamptzNs),
1012                    )
1013                    .into(),
1014                    NestedField::required(13, "uuid_field", Type::Primitive(PrimitiveType::Uuid))
1015                        .into(),
1016                    NestedField::required(
1017                        14,
1018                        "fixed_field",
1019                        Type::Primitive(PrimitiveType::Fixed(10)),
1020                    )
1021                    .into(),
1022                    NestedField::required(
1023                        15,
1024                        "binary_field",
1025                        Type::Primitive(PrimitiveType::Binary),
1026                    )
1027                    .into(),
1028                    NestedField::required(
1029                        16,
1030                        "string_field",
1031                        Type::Primitive(PrimitiveType::String),
1032                    )
1033                    .into(),
1034                ],
1035                id_lookup: OnceLock::default(),
1036                name_lookup: OnceLock::default(),
1037            }),
1038        )
1039    }
1040
1041    #[test]
1042    fn struct_type() {
1043        let record = r#"
1044        {
1045            "type": "struct",
1046            "fields": [
1047                {
1048                    "id": 1,
1049                    "name": "id",
1050                    "required": true,
1051                    "type": "uuid",
1052                    "initial-default": "0db3e2a8-9d1d-42b9-aa7b-74ebe558dceb",
1053                    "write-default": "ec5911be-b0a7-458c-8438-c9a3e53cffae"
1054                }, {
1055                    "id": 2,
1056                    "name": "data",
1057                    "required": false,
1058                    "type": "int"
1059                }
1060            ]
1061        }
1062        "#;
1063
1064        check_type_serde(
1065            record,
1066            Type::Struct(StructType {
1067                fields: vec![
1068                    NestedField::required(1, "id", Type::Primitive(PrimitiveType::Uuid))
1069                        .with_initial_default(Literal::Primitive(PrimitiveLiteral::UInt128(
1070                            Uuid::parse_str("0db3e2a8-9d1d-42b9-aa7b-74ebe558dceb")
1071                                .unwrap()
1072                                .as_u128(),
1073                        )))
1074                        .with_write_default(Literal::Primitive(PrimitiveLiteral::UInt128(
1075                            Uuid::parse_str("ec5911be-b0a7-458c-8438-c9a3e53cffae")
1076                                .unwrap()
1077                                .as_u128(),
1078                        )))
1079                        .into(),
1080                    NestedField::optional(2, "data", Type::Primitive(PrimitiveType::Int)).into(),
1081                ],
1082                id_lookup: HashMap::from([(1, 0), (2, 1)]).into(),
1083                name_lookup: HashMap::from([("id".to_string(), 0), ("data".to_string(), 1)]).into(),
1084            }),
1085        )
1086    }
1087
1088    #[test]
1089    fn test_deeply_nested_struct() {
1090        let record = r#"
1091{
1092  "type": "struct",
1093  "fields": [
1094    {
1095      "id": 1,
1096      "name": "id",
1097      "required": true,
1098      "type": "uuid",
1099      "initial-default": "0db3e2a8-9d1d-42b9-aa7b-74ebe558dceb",
1100      "write-default": "ec5911be-b0a7-458c-8438-c9a3e53cffae"
1101    },
1102    {
1103      "id": 2,
1104      "name": "data",
1105      "required": false,
1106      "type": "int"
1107    },
1108    {
1109      "id": 3,
1110      "name": "address",
1111      "required": true,
1112      "type": {
1113        "type": "struct",
1114        "fields": [
1115          {
1116            "id": 4,
1117            "name": "street",
1118            "required": true,
1119            "type": "string"
1120          },
1121          {
1122            "id": 5,
1123            "name": "province",
1124            "required": false,
1125            "type": "string"
1126          },
1127          {
1128            "id": 6,
1129            "name": "zip",
1130            "required": true,
1131            "type": "int"
1132          }
1133        ]
1134      }
1135    }
1136  ]
1137}
1138"#;
1139
1140        let struct_type = Type::Struct(StructType::new(vec![
1141            NestedField::required(1, "id", Type::Primitive(PrimitiveType::Uuid))
1142                .with_initial_default(Literal::Primitive(PrimitiveLiteral::UInt128(
1143                    Uuid::parse_str("0db3e2a8-9d1d-42b9-aa7b-74ebe558dceb")
1144                        .unwrap()
1145                        .as_u128(),
1146                )))
1147                .with_write_default(Literal::Primitive(PrimitiveLiteral::UInt128(
1148                    Uuid::parse_str("ec5911be-b0a7-458c-8438-c9a3e53cffae")
1149                        .unwrap()
1150                        .as_u128(),
1151                )))
1152                .into(),
1153            NestedField::optional(2, "data", Type::Primitive(PrimitiveType::Int)).into(),
1154            NestedField::required(
1155                3,
1156                "address",
1157                Type::Struct(StructType::new(vec![
1158                    NestedField::required(4, "street", Type::Primitive(PrimitiveType::String))
1159                        .into(),
1160                    NestedField::optional(5, "province", Type::Primitive(PrimitiveType::String))
1161                        .into(),
1162                    NestedField::required(6, "zip", Type::Primitive(PrimitiveType::Int)).into(),
1163                ])),
1164            )
1165            .into(),
1166        ]));
1167
1168        check_type_serde(record, struct_type)
1169    }
1170
1171    #[test]
1172    fn list() {
1173        let record = r#"
1174        {
1175            "type": "list",
1176            "element-id": 3,
1177            "element-required": true,
1178            "element": "string"
1179        }
1180        "#;
1181
1182        check_type_serde(
1183            record,
1184            Type::List(ListType {
1185                element_field: NestedField::list_element(
1186                    3,
1187                    Type::Primitive(PrimitiveType::String),
1188                    true,
1189                )
1190                .into(),
1191            }),
1192        );
1193    }
1194
1195    #[test]
1196    fn map() {
1197        let record = r#"
1198        {
1199            "type": "map",
1200            "key-id": 4,
1201            "key": "string",
1202            "value-id": 5,
1203            "value-required": false,
1204            "value": "double"
1205        }
1206        "#;
1207
1208        check_type_serde(
1209            record,
1210            Type::Map(MapType {
1211                key_field: NestedField::map_key_element(4, Type::Primitive(PrimitiveType::String))
1212                    .into(),
1213                value_field: NestedField::map_value_element(
1214                    5,
1215                    Type::Primitive(PrimitiveType::Double),
1216                    false,
1217                )
1218                .into(),
1219            }),
1220        );
1221
1222        check_type_serde(
1223            record,
1224            Type::Map(MapType::optional(
1225                4,
1226                Type::Primitive(PrimitiveType::String),
1227                5,
1228                Type::Primitive(PrimitiveType::Double),
1229            )),
1230        );
1231    }
1232
1233    #[test]
1234    fn map_int() {
1235        let record = r#"
1236        {
1237            "type": "map",
1238            "key-id": 4,
1239            "key": "int",
1240            "value-id": 5,
1241            "value-required": false,
1242            "value": "string"
1243        }
1244        "#;
1245
1246        check_type_serde(
1247            record,
1248            Type::Map(MapType {
1249                key_field: NestedField::map_key_element(4, Type::Primitive(PrimitiveType::Int))
1250                    .into(),
1251                value_field: NestedField::map_value_element(
1252                    5,
1253                    Type::Primitive(PrimitiveType::String),
1254                    false,
1255                )
1256                .into(),
1257            }),
1258        );
1259
1260        check_type_serde(
1261            record,
1262            Type::Map(MapType::optional(
1263                4,
1264                Type::Primitive(PrimitiveType::Int),
1265                5,
1266                Type::Primitive(PrimitiveType::String),
1267            )),
1268        );
1269    }
1270
1271    #[test]
1272    fn map_required_int() {
1273        let record = r#"
1274        {
1275            "type": "map",
1276            "key-id": 4,
1277            "key": "int",
1278            "value-id": 5,
1279            "value-required": true,
1280            "value": "string"
1281        }
1282        "#;
1283
1284        check_type_serde(
1285            record,
1286            Type::Map(MapType::required(
1287                4,
1288                Type::Primitive(PrimitiveType::Int),
1289                5,
1290                Type::Primitive(PrimitiveType::String),
1291            )),
1292        );
1293    }
1294
1295    #[test]
1296    fn test_decimal_precision() {
1297        let expected_max_precision = [
1298            2, 4, 6, 9, 11, 14, 16, 18, 21, 23, 26, 28, 31, 33, 35, 38, 40, 43, 45, 47, 50, 52, 55,
1299            57,
1300        ];
1301        for (i, max_precision) in expected_max_precision.iter().enumerate() {
1302            assert_eq!(
1303                *max_precision,
1304                Type::decimal_max_precision(i as u32 + 1).unwrap(),
1305                "Failed calculate max precision for {i}"
1306            );
1307        }
1308
1309        assert_eq!(5, Type::decimal_required_bytes(10).unwrap());
1310        assert_eq!(16, Type::decimal_required_bytes(38).unwrap());
1311    }
1312
1313    #[test]
1314    fn test_primitive_type_compatible() {
1315        let pairs = vec![
1316            (PrimitiveType::Boolean, PrimitiveLiteral::Boolean(true)),
1317            (PrimitiveType::Int, PrimitiveLiteral::Int(1)),
1318            (PrimitiveType::Long, PrimitiveLiteral::Long(1)),
1319            (PrimitiveType::Float, PrimitiveLiteral::Float(1.0.into())),
1320            (PrimitiveType::Double, PrimitiveLiteral::Double(1.0.into())),
1321            (
1322                PrimitiveType::Decimal {
1323                    precision: 9,
1324                    scale: 2,
1325                },
1326                PrimitiveLiteral::Int128(1),
1327            ),
1328            (PrimitiveType::Date, PrimitiveLiteral::Int(1)),
1329            (PrimitiveType::Time, PrimitiveLiteral::Long(1)),
1330            (PrimitiveType::Timestamptz, PrimitiveLiteral::Long(1)),
1331            (PrimitiveType::Timestamp, PrimitiveLiteral::Long(1)),
1332            (PrimitiveType::TimestamptzNs, PrimitiveLiteral::Long(1)),
1333            (PrimitiveType::TimestampNs, PrimitiveLiteral::Long(1)),
1334            (
1335                PrimitiveType::Uuid,
1336                PrimitiveLiteral::UInt128(Uuid::new_v4().as_u128()),
1337            ),
1338            (PrimitiveType::Fixed(8), PrimitiveLiteral::Binary(vec![1])),
1339            (PrimitiveType::Binary, PrimitiveLiteral::Binary(vec![1])),
1340        ];
1341        for (ty, literal) in pairs {
1342            assert!(ty.compatible(&literal));
1343        }
1344    }
1345
1346    #[test]
1347    fn variant_type_serde() {
1348        let json = r#"{"id": 1, "name": "v", "required": true, "type": "variant"}"#;
1349        let field: NestedField = serde_json::from_str(json).unwrap();
1350        assert_eq!(*field.field_type, Type::Variant(VariantType));
1351
1352        let serialized = serde_json::to_string(&field).unwrap();
1353        let roundtrip: NestedField = serde_json::from_str(&serialized).unwrap();
1354        assert_eq!(field, roundtrip);
1355    }
1356
1357    #[test]
1358    fn struct_type_with_type_field() {
1359        // Test that StructType properly deserializes JSON with "type":"struct" field
1360        // This was previously broken because the deserializer wasn't consuming the type field value
1361        let json = r#"
1362        {
1363            "type": "struct",
1364            "fields": [
1365                {"id": 1, "name": "field1", "required": true, "type": "string"}
1366            ]
1367        }
1368        "#;
1369
1370        let struct_type: StructType = serde_json::from_str(json)
1371            .expect("Should successfully deserialize StructType with type field");
1372
1373        assert_eq!(struct_type.fields().len(), 1);
1374        assert_eq!(struct_type.fields()[0].name, "field1");
1375    }
1376
1377    #[test]
1378    fn struct_type_rejects_wrong_type() {
1379        // Test that StructType validation rejects incorrect type field values
1380        let json = r#"
1381        {
1382            "type": "list",
1383            "fields": [
1384                {"id": 1, "name": "field1", "required": true, "type": "string"}
1385            ]
1386        }
1387        "#;
1388
1389        let result = serde_json::from_str::<StructType>(json);
1390        assert!(
1391            result.is_err(),
1392            "Should reject StructType with wrong type field"
1393        );
1394        assert!(
1395            result
1396                .unwrap_err()
1397                .to_string()
1398                .contains("expected type 'struct'")
1399        );
1400    }
1401}
iceberg/spec/datatypes.rs

iceberg/spec/
datatypes.rs
