iceberg/spec/
datatypes.rs

1// Licensed to the Apache Software Foundation (ASF) under one
2// or more contributor license agreements.  See the NOTICE file
3// distributed with this work for additional information
4// regarding copyright ownership.  The ASF licenses this file
5// to you under the Apache License, Version 2.0 (the
6// "License"); you may not use this file except in compliance
7// with the License.  You may obtain a copy of the License at
8//
9//   http://www.apache.org/licenses/LICENSE-2.0
10//
11// Unless required by applicable law or agreed to in writing,
12// software distributed under the License is distributed on an
13// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14// KIND, either express or implied.  See the License for the
15// specific language governing permissions and limitations
16// under the License.
17
18/*!
19 * Data Types
20 */
21use std::collections::HashMap;
22use std::fmt;
23use std::ops::Index;
24use std::sync::{Arc, OnceLock};
25
26use ::serde::de::{MapAccess, Visitor};
27use serde::de::{Error, IntoDeserializer};
28use serde::{Deserialize, Deserializer, Serialize, Serializer};
29use serde_json::Value as JsonValue;
30
31use super::values::Literal;
32use crate::ensure_data_valid;
33use crate::error::Result;
34use crate::spec::datatypes::_decimal::{MAX_PRECISION, REQUIRED_LENGTH};
35use crate::spec::{FormatVersion, PrimitiveLiteral};
36
37/// Field name for list type.
38pub const LIST_FIELD_NAME: &str = "element";
39/// Field name for map type's key.
40pub const MAP_KEY_FIELD_NAME: &str = "key";
41/// Field name for map type's value.
42pub const MAP_VALUE_FIELD_NAME: &str = "value";
43
44pub(crate) const MAX_DECIMAL_BYTES: u32 = 24;
45pub(crate) const MAX_DECIMAL_PRECISION: u32 = 38;
46
47mod _decimal {
48    use once_cell::sync::Lazy;
49
50    use crate::spec::{MAX_DECIMAL_BYTES, MAX_DECIMAL_PRECISION};
51
52    // Max precision of bytes, starts from 1
53    pub(super) static MAX_PRECISION: Lazy<[u32; MAX_DECIMAL_BYTES as usize]> = Lazy::new(|| {
54        let mut ret: [u32; 24] = [0; 24];
55        for (i, prec) in ret.iter_mut().enumerate() {
56            *prec = 2f64.powi((8 * (i + 1) - 1) as i32).log10().floor() as u32;
57        }
58
59        ret
60    });
61
62    //  Required bytes of precision, starts from 1
63    pub(super) static REQUIRED_LENGTH: Lazy<[u32; MAX_DECIMAL_PRECISION as usize]> =
64        Lazy::new(|| {
65            let mut ret: [u32; MAX_DECIMAL_PRECISION as usize] =
66                [0; MAX_DECIMAL_PRECISION as usize];
67
68            for (i, required_len) in ret.iter_mut().enumerate() {
69                for j in 0..MAX_PRECISION.len() {
70                    if MAX_PRECISION[j] >= ((i + 1) as u32) {
71                        *required_len = (j + 1) as u32;
72                        break;
73                    }
74                }
75            }
76
77            ret
78        });
79}
80
81#[derive(Debug, PartialEq, Eq, Clone)]
82/// All data types are either primitives or nested types, which are maps, lists, or structs.
83pub enum Type {
84    /// Primitive types
85    Primitive(PrimitiveType),
86    /// Struct type
87    Struct(StructType),
88    /// List type.
89    List(ListType),
90    /// Map type
91    Map(MapType),
92    /// Variant Type
93    Variant(VariantType),
94}
95
96impl fmt::Display for Type {
97    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
98        match self {
99            Type::Primitive(primitive) => write!(f, "{primitive}"),
100            Type::Struct(s) => write!(f, "{s}"),
101            Type::List(_) => write!(f, "list"),
102            Type::Map(_) => write!(f, "map"),
103            Type::Variant(_) => write!(f, "variant"),
104        }
105    }
106}
107
108impl Type {
109    /// Whether the type is primitive type.
110    #[inline(always)]
111    pub fn is_primitive(&self) -> bool {
112        matches!(self, Type::Primitive(_))
113    }
114
115    /// Whether the type is struct type.
116    #[inline(always)]
117    pub fn is_struct(&self) -> bool {
118        matches!(self, Type::Struct(_))
119    }
120
121    /// Whether the type is nested type.
122    #[inline(always)]
123    pub fn is_nested(&self) -> bool {
124        matches!(self, Type::Struct(_) | Type::List(_) | Type::Map(_))
125    }
126
127    /// Whether the type is variant type.
128    #[inline(always)]
129    pub fn is_variant(&self) -> bool {
130        matches!(self, Type::Variant(_))
131    }
132
133    /// Minimum [`FormatVersion`] required to support this type, **without** taking
134    /// nested field types into account.
135    ///
136    /// `TimestampNs` / `TimestamptzNs` / `Variant` require [`FormatVersion::V3`]; every
137    /// other type is valid from [`FormatVersion::V1`]. Mirrors Java's
138    /// `Schema.MIN_FORMAT_VERSIONS` (a shallow lookup keyed by type id), so it
139    /// intentionally does not recurse: callers needing the floor for a whole schema
140    /// iterate its flattened fields (see [`Schema::calc_min_compatible_format`]).
141    ///
142    /// [`Schema::calc_min_compatible_format`]: crate::spec::Schema::calc_min_compatible_format
143    pub(crate) fn min_format_version(&self) -> FormatVersion {
144        match self {
145            Type::Primitive(PrimitiveType::TimestampNs | PrimitiveType::TimestamptzNs)
146            | Type::Variant(_) => FormatVersion::V3,
147            _ => FormatVersion::V1,
148        }
149    }
150
151    /// Convert Type to reference of PrimitiveType
152    pub fn as_primitive_type(&self) -> Option<&PrimitiveType> {
153        if let Type::Primitive(primitive_type) = self {
154            Some(primitive_type)
155        } else {
156            None
157        }
158    }
159
160    /// Convert Type to StructType
161    pub fn to_struct_type(self) -> Option<StructType> {
162        if let Type::Struct(struct_type) = self {
163            Some(struct_type)
164        } else {
165            None
166        }
167    }
168
169    /// Return max precision for decimal given `num_bytes` bytes.
170    #[inline(always)]
171    pub fn decimal_max_precision(num_bytes: u32) -> Result<u32> {
172        ensure_data_valid!(
173            num_bytes > 0 && num_bytes <= MAX_DECIMAL_BYTES,
174            "Decimal length larger than {MAX_DECIMAL_BYTES} is not supported: {num_bytes}",
175        );
176        Ok(MAX_PRECISION[num_bytes as usize - 1])
177    }
178
179    /// Returns minimum bytes required for decimal with `precision`.
180    #[inline(always)]
181    pub fn decimal_required_bytes(precision: u32) -> Result<u32> {
182        ensure_data_valid!(
183            precision > 0 && precision <= MAX_DECIMAL_PRECISION,
184            "Decimals with precision larger than {MAX_DECIMAL_PRECISION} are not supported: {precision}",
185        );
186        Ok(REQUIRED_LENGTH[precision as usize - 1])
187    }
188
189    /// Creates  decimal type.
190    #[inline(always)]
191    pub fn decimal(precision: u32, scale: u32) -> Result<Self> {
192        ensure_data_valid!(
193            precision > 0 && precision <= MAX_DECIMAL_PRECISION,
194            "Decimals with precision larger than {MAX_DECIMAL_PRECISION} are not supported: {precision}",
195        );
196        Ok(Type::Primitive(PrimitiveType::Decimal { precision, scale }))
197    }
198
199    /// Check if it's float or double type.
200    #[inline(always)]
201    pub fn is_floating_type(&self) -> bool {
202        matches!(
203            self,
204            Type::Primitive(PrimitiveType::Float) | Type::Primitive(PrimitiveType::Double)
205        )
206    }
207}
208
209impl From<PrimitiveType> for Type {
210    fn from(value: PrimitiveType) -> Self {
211        Self::Primitive(value)
212    }
213}
214
215impl From<StructType> for Type {
216    fn from(value: StructType) -> Self {
217        Type::Struct(value)
218    }
219}
220
221impl From<ListType> for Type {
222    fn from(value: ListType) -> Self {
223        Type::List(value)
224    }
225}
226
227impl From<MapType> for Type {
228    fn from(value: MapType) -> Self {
229        Type::Map(value)
230    }
231}
232
233/// Primitive data types
234#[derive(Debug, Serialize, Deserialize, PartialEq, Eq, Clone, Hash)]
235#[serde(rename_all = "lowercase", remote = "Self")]
236pub enum PrimitiveType {
237    /// True or False
238    Boolean,
239    /// 32-bit signed integer
240    Int,
241    /// 64-bit signed integer
242    Long,
243    /// 32-bit IEEE 754 floating point.
244    Float,
245    /// 64-bit IEEE 754 floating point.
246    Double,
247    /// Fixed point decimal
248    Decimal {
249        /// Precision, must be 38 or less
250        precision: u32,
251        /// Scale
252        scale: u32,
253    },
254    /// Calendar date without timezone or time.
255    Date,
256    /// Time of day in microsecond precision, without date or timezone.
257    Time,
258    /// Timestamp in microsecond precision, without timezone
259    Timestamp,
260    /// Timestamp in microsecond precision, with timezone
261    Timestamptz,
262    /// Timestamp in nanosecond precision, without timezone
263    #[serde(rename = "timestamp_ns")]
264    TimestampNs,
265    /// Timestamp in nanosecond precision with timezone
266    #[serde(rename = "timestamptz_ns")]
267    TimestamptzNs,
268    /// Arbitrary-length character sequences encoded in utf-8
269    String,
270    /// Universally Unique Identifiers, should use 16-byte fixed
271    Uuid,
272    /// Fixed length byte array
273    Fixed(u64),
274    /// Arbitrary-length byte array.
275    Binary,
276}
277
278impl PrimitiveType {
279    /// Check whether literal is compatible with the type.
280    pub fn compatible(&self, literal: &PrimitiveLiteral) -> bool {
281        matches!(
282            (self, literal),
283            (PrimitiveType::Boolean, PrimitiveLiteral::Boolean(_))
284                | (PrimitiveType::Int, PrimitiveLiteral::Int(_))
285                | (PrimitiveType::Long, PrimitiveLiteral::Long(_))
286                | (PrimitiveType::Float, PrimitiveLiteral::Float(_))
287                | (PrimitiveType::Double, PrimitiveLiteral::Double(_))
288                | (PrimitiveType::Decimal { .. }, PrimitiveLiteral::Int128(_))
289                | (PrimitiveType::Date, PrimitiveLiteral::Int(_))
290                | (PrimitiveType::Time, PrimitiveLiteral::Long(_))
291                | (PrimitiveType::Timestamp, PrimitiveLiteral::Long(_))
292                | (PrimitiveType::Timestamptz, PrimitiveLiteral::Long(_))
293                | (PrimitiveType::TimestampNs, PrimitiveLiteral::Long(_))
294                | (PrimitiveType::TimestamptzNs, PrimitiveLiteral::Long(_))
295                | (PrimitiveType::String, PrimitiveLiteral::String(_))
296                | (PrimitiveType::Uuid, PrimitiveLiteral::UInt128(_))
297                | (PrimitiveType::Fixed(_), PrimitiveLiteral::Binary(_))
298                | (PrimitiveType::Binary, PrimitiveLiteral::Binary(_))
299        )
300    }
301}
302
303impl Serialize for Type {
304    fn serialize<S>(&self, serializer: S) -> std::result::Result<S::Ok, S::Error>
305    where S: Serializer {
306        let type_serde = _serde::SerdeType::from(self);
307        type_serde.serialize(serializer)
308    }
309}
310
311impl<'de> Deserialize<'de> for Type {
312    fn deserialize<D>(deserializer: D) -> std::result::Result<Self, D::Error>
313    where D: Deserializer<'de> {
314        let type_serde = _serde::SerdeType::deserialize(deserializer)?;
315        Ok(Type::from(type_serde))
316    }
317}
318
319impl<'de> Deserialize<'de> for PrimitiveType {
320    fn deserialize<D>(deserializer: D) -> std::result::Result<Self, D::Error>
321    where D: Deserializer<'de> {
322        let s = String::deserialize(deserializer)?;
323        if s.starts_with("decimal") {
324            deserialize_decimal(s.into_deserializer())
325        } else if s.starts_with("fixed") {
326            deserialize_fixed(s.into_deserializer())
327        } else {
328            PrimitiveType::deserialize(s.into_deserializer())
329        }
330    }
331}
332
333impl Serialize for PrimitiveType {
334    fn serialize<S>(&self, serializer: S) -> std::result::Result<S::Ok, S::Error>
335    where S: Serializer {
336        match self {
337            PrimitiveType::Decimal { precision, scale } => {
338                serialize_decimal(precision, scale, serializer)
339            }
340            PrimitiveType::Fixed(l) => serialize_fixed(l, serializer),
341            _ => PrimitiveType::serialize(self, serializer),
342        }
343    }
344}
345
346fn deserialize_decimal<'de, D>(deserializer: D) -> std::result::Result<PrimitiveType, D::Error>
347where D: Deserializer<'de> {
348    let s = String::deserialize(deserializer)?;
349    let (precision, scale) = s
350        .trim_start_matches(r"decimal(")
351        .trim_end_matches(')')
352        .split_once(',')
353        .ok_or_else(|| D::Error::custom("Decimal requires precision and scale: {s}"))?;
354
355    Ok(PrimitiveType::Decimal {
356        precision: precision.trim().parse().map_err(D::Error::custom)?,
357        scale: scale.trim().parse().map_err(D::Error::custom)?,
358    })
359}
360
361fn serialize_decimal<S>(
362    precision: &u32,
363    scale: &u32,
364    serializer: S,
365) -> std::result::Result<S::Ok, S::Error>
366where
367    S: Serializer,
368{
369    serializer.serialize_str(&format!("decimal({precision}, {scale})"))
370}
371
372fn deserialize_fixed<'de, D>(deserializer: D) -> std::result::Result<PrimitiveType, D::Error>
373where D: Deserializer<'de> {
374    let fixed = String::deserialize(deserializer)?
375        .trim_start_matches(r"fixed[")
376        .trim_end_matches(']')
377        .to_owned();
378
379    fixed
380        .parse()
381        .map(PrimitiveType::Fixed)
382        .map_err(D::Error::custom)
383}
384
385fn serialize_fixed<S>(value: &u64, serializer: S) -> std::result::Result<S::Ok, S::Error>
386where S: Serializer {
387    serializer.serialize_str(&format!("fixed[{value}]"))
388}
389
390impl fmt::Display for PrimitiveType {
391    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
392        match self {
393            PrimitiveType::Boolean => write!(f, "boolean"),
394            PrimitiveType::Int => write!(f, "int"),
395            PrimitiveType::Long => write!(f, "long"),
396            PrimitiveType::Float => write!(f, "float"),
397            PrimitiveType::Double => write!(f, "double"),
398            PrimitiveType::Decimal { precision, scale } => {
399                write!(f, "decimal({precision}, {scale})")
400            }
401            PrimitiveType::Date => write!(f, "date"),
402            PrimitiveType::Time => write!(f, "time"),
403            PrimitiveType::Timestamp => write!(f, "timestamp"),
404            PrimitiveType::Timestamptz => write!(f, "timestamptz"),
405            PrimitiveType::TimestampNs => write!(f, "timestamp_ns"),
406            PrimitiveType::TimestamptzNs => write!(f, "timestamptz_ns"),
407            PrimitiveType::String => write!(f, "string"),
408            PrimitiveType::Uuid => write!(f, "uuid"),
409            PrimitiveType::Fixed(size) => write!(f, "fixed({size})"),
410            PrimitiveType::Binary => write!(f, "binary"),
411        }
412    }
413}
414
415/// DataType for a specific struct
416#[derive(Debug, Serialize, Clone, Default)]
417#[serde(rename = "struct", tag = "type")]
418pub struct StructType {
419    /// Struct fields
420    fields: Vec<NestedFieldRef>,
421    /// Lookup for index by field id
422    #[serde(skip_serializing)]
423    id_lookup: OnceLock<HashMap<i32, usize>>,
424    #[serde(skip_serializing)]
425    name_lookup: OnceLock<HashMap<String, usize>>,
426}
427
428impl<'de> Deserialize<'de> for StructType {
429    fn deserialize<D>(deserializer: D) -> std::result::Result<Self, D::Error>
430    where D: Deserializer<'de> {
431        #[derive(Deserialize)]
432        #[serde(field_identifier, rename_all = "lowercase")]
433        enum Field {
434            Type,
435            Fields,
436        }
437
438        struct StructTypeVisitor;
439
440        impl<'de> Visitor<'de> for StructTypeVisitor {
441            type Value = StructType;
442
443            fn expecting(&self, formatter: &mut fmt::Formatter) -> fmt::Result {
444                formatter.write_str("struct")
445            }
446
447            fn visit_map<V>(self, mut map: V) -> std::result::Result<StructType, V::Error>
448            where V: MapAccess<'de> {
449                let mut fields = None;
450                while let Some(key) = map.next_key()? {
451                    match key {
452                        Field::Type => {
453                            let type_val: String = map.next_value()?;
454                            if type_val != "struct" {
455                                return Err(Error::custom(format!(
456                                    "expected type 'struct', got '{type_val}'"
457                                )));
458                            }
459                        }
460                        Field::Fields => {
461                            if fields.is_some() {
462                                return Err(Error::duplicate_field("fields"));
463                            }
464                            fields = Some(map.next_value()?);
465                        }
466                    }
467                }
468                let fields: Vec<NestedFieldRef> =
469                    fields.ok_or_else(|| Error::missing_field("fields"))?;
470
471                Ok(StructType::new(fields))
472            }
473        }
474
475        const FIELDS: &[&str] = &["type", "fields"];
476        deserializer.deserialize_struct("struct", FIELDS, StructTypeVisitor)
477    }
478}
479
480impl StructType {
481    /// Creates a struct type with the given fields.
482    pub fn new(fields: Vec<NestedFieldRef>) -> Self {
483        Self {
484            fields,
485            id_lookup: OnceLock::new(),
486            name_lookup: OnceLock::new(),
487        }
488    }
489
490    /// Get struct field with certain id
491    pub fn field_by_id(&self, id: i32) -> Option<&NestedFieldRef> {
492        self.field_id_to_index(id).map(|idx| &self.fields[idx])
493    }
494
495    fn field_id_to_index(&self, field_id: i32) -> Option<usize> {
496        self.id_lookup
497            .get_or_init(|| {
498                HashMap::from_iter(self.fields.iter().enumerate().map(|(i, x)| (x.id, i)))
499            })
500            .get(&field_id)
501            .copied()
502    }
503
504    /// Get struct field with certain field name
505    pub fn field_by_name(&self, name: &str) -> Option<&NestedFieldRef> {
506        self.field_name_to_index(name).map(|idx| &self.fields[idx])
507    }
508
509    fn field_name_to_index(&self, name: &str) -> Option<usize> {
510        self.name_lookup
511            .get_or_init(|| {
512                HashMap::from_iter(
513                    self.fields
514                        .iter()
515                        .enumerate()
516                        .map(|(i, x)| (x.name.clone(), i)),
517                )
518            })
519            .get(name)
520            .copied()
521    }
522
523    /// Get fields.
524    pub fn fields(&self) -> &[NestedFieldRef] {
525        &self.fields
526    }
527}
528
529impl PartialEq for StructType {
530    fn eq(&self, other: &Self) -> bool {
531        self.fields == other.fields
532    }
533}
534
535impl Eq for StructType {}
536
537impl Index<usize> for StructType {
538    type Output = NestedField;
539
540    fn index(&self, index: usize) -> &Self::Output {
541        &self.fields[index]
542    }
543}
544
545impl fmt::Display for StructType {
546    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
547        write!(f, "struct<")?;
548        for field in &self.fields {
549            write!(f, "{}", field.field_type)?;
550        }
551        write!(f, ">")
552    }
553}
554
555#[derive(Debug, PartialEq, Serialize, Deserialize, Eq, Clone)]
556#[serde(try_from = "SerdeNestedField", into = "SerdeNestedField")]
557/// A struct is a tuple of typed values. Each field in the tuple is named and has an integer id that is unique in the table schema.
558/// Each field can be either optional or required, meaning that values can (or cannot) be null. Fields may be any type.
559/// Fields may have an optional comment or doc string. Fields can have default values.
560pub struct NestedField {
561    /// Id unique in table schema
562    pub id: i32,
563    /// Field Name
564    pub name: String,
565    /// Optional or required
566    pub required: bool,
567    /// Datatype
568    pub field_type: Box<Type>,
569    /// Fields may have an optional comment or doc string.
570    pub doc: Option<String>,
571    /// Used to populate the field’s value for all records that were written before the field was added to the schema
572    pub initial_default: Option<Literal>,
573    /// Used to populate the field’s value for any records written after the field was added to the schema, if the writer does not supply the field’s value
574    pub write_default: Option<Literal>,
575}
576
577#[derive(Debug, Serialize, Deserialize, PartialEq, Eq, Clone)]
578#[serde(rename_all = "kebab-case")]
579struct SerdeNestedField {
580    pub id: i32,
581    pub name: String,
582    pub required: bool,
583    #[serde(rename = "type")]
584    pub field_type: Box<Type>,
585    #[serde(skip_serializing_if = "Option::is_none")]
586    pub doc: Option<String>,
587    #[serde(skip_serializing_if = "Option::is_none")]
588    pub initial_default: Option<JsonValue>,
589    #[serde(skip_serializing_if = "Option::is_none")]
590    pub write_default: Option<JsonValue>,
591}
592
593impl TryFrom<SerdeNestedField> for NestedField {
594    type Error = crate::Error;
595
596    fn try_from(value: SerdeNestedField) -> Result<Self> {
597        let initial_default = value
598            .initial_default
599            .map(|x| Literal::try_from_json(x, &value.field_type))
600            .transpose()?
601            .flatten();
602        let write_default = value
603            .write_default
604            .map(|x| Literal::try_from_json(x, &value.field_type))
605            .transpose()?
606            .flatten();
607
608        Ok(NestedField {
609            id: value.id,
610            name: value.name,
611            required: value.required,
612            initial_default,
613            write_default,
614            field_type: value.field_type,
615            doc: value.doc,
616        })
617    }
618}
619
620impl From<NestedField> for SerdeNestedField {
621    fn from(value: NestedField) -> Self {
622        let initial_default = value.initial_default.map(|x| x.try_into_json(&value.field_type).expect("We should have checked this in NestedField::with_initial_default, it can't be converted to json value"));
623        let write_default = value.write_default.map(|x| x.try_into_json(&value.field_type).expect("We should have checked this in NestedField::with_write_default, it can't be converted to json value"));
624        SerdeNestedField {
625            id: value.id,
626            name: value.name,
627            required: value.required,
628            field_type: value.field_type,
629            doc: value.doc,
630            initial_default,
631            write_default,
632        }
633    }
634}
635
636/// Reference to nested field.
637pub type NestedFieldRef = Arc<NestedField>;
638
639impl NestedField {
640    /// Construct a new field.
641    pub fn new(id: i32, name: impl ToString, field_type: Type, required: bool) -> Self {
642        Self {
643            id,
644            name: name.to_string(),
645            required,
646            field_type: Box::new(field_type),
647            doc: None,
648            initial_default: None,
649            write_default: None,
650        }
651    }
652
653    /// Construct a required field.
654    pub fn required(id: i32, name: impl ToString, field_type: Type) -> Self {
655        Self::new(id, name, field_type, true)
656    }
657
658    /// Construct an optional field.
659    pub fn optional(id: i32, name: impl ToString, field_type: Type) -> Self {
660        Self::new(id, name, field_type, false)
661    }
662
663    /// Construct list type's element field.
664    pub fn list_element(id: i32, field_type: Type, required: bool) -> Self {
665        Self::new(id, LIST_FIELD_NAME, field_type, required)
666    }
667
668    /// Construct map type's key field.
669    pub fn map_key_element(id: i32, field_type: Type) -> Self {
670        Self::required(id, MAP_KEY_FIELD_NAME, field_type)
671    }
672
673    /// Construct map type's value field.
674    pub fn map_value_element(id: i32, field_type: Type, required: bool) -> Self {
675        Self::new(id, MAP_VALUE_FIELD_NAME, field_type, required)
676    }
677
678    /// Set the field's doc.
679    pub fn with_doc(mut self, doc: impl ToString) -> Self {
680        self.doc = Some(doc.to_string());
681        self
682    }
683
684    /// Set the field's initial default value.
685    pub fn with_initial_default(mut self, value: Literal) -> Self {
686        self.initial_default = Some(value);
687        self
688    }
689
690    /// Set the field's initial default value.
691    pub fn with_write_default(mut self, value: Literal) -> Self {
692        self.write_default = Some(value);
693        self
694    }
695
696    /// Set the id of the field.
697    pub(crate) fn with_id(mut self, id: i32) -> Self {
698        self.id = id;
699        self
700    }
701}
702
703impl fmt::Display for NestedField {
704    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
705        write!(f, "{}: ", self.id)?;
706        write!(f, "{}: ", self.name)?;
707        if self.required {
708            write!(f, "required ")?;
709        } else {
710            write!(f, "optional ")?;
711        }
712        write!(f, "{} ", self.field_type)?;
713        if let Some(doc) = &self.doc {
714            write!(f, "{doc}")?;
715        }
716        Ok(())
717    }
718}
719
720#[derive(Debug, PartialEq, Eq, Clone)]
721/// A list is a collection of values with some element type. The element field has an integer id that is unique in the table schema.
722/// Elements can be either optional or required. Element types may be any type.
723pub struct ListType {
724    /// Element field of list type.
725    pub element_field: NestedFieldRef,
726}
727
728impl ListType {
729    /// Construct a list type with the given element field.
730    pub fn new(element_field: NestedFieldRef) -> Self {
731        Self { element_field }
732    }
733}
734
735/// Module for type serialization/deserialization.
736pub(super) mod _serde {
737    use std::borrow::Cow;
738
739    use serde_derive::{Deserialize, Serialize};
740
741    use crate::spec::datatypes::Type::Map;
742    use crate::spec::datatypes::{
743        ListType, MapType, NestedField, NestedFieldRef, PrimitiveType, StructType, Type,
744        VariantType,
745    };
746
747    /// List type for serialization and deserialization
748    #[derive(Serialize, Deserialize)]
749    #[serde(untagged)]
750    pub(super) enum SerdeType<'a> {
751        #[serde(rename_all = "kebab-case")]
752        List {
753            r#type: String,
754            element_id: i32,
755            element_required: bool,
756            element: Cow<'a, Type>,
757        },
758        Struct {
759            r#type: String,
760            fields: Cow<'a, [NestedFieldRef]>,
761        },
762        #[serde(rename_all = "kebab-case")]
763        Map {
764            r#type: String,
765            key_id: i32,
766            key: Cow<'a, Type>,
767            value_id: i32,
768            value_required: bool,
769            value: Cow<'a, Type>,
770        },
771        Primitive(PrimitiveType),
772        Variant(VariantType),
773    }
774
775    impl From<SerdeType<'_>> for Type {
776        fn from(value: SerdeType) -> Self {
777            match value {
778                SerdeType::List {
779                    r#type: _,
780                    element_id,
781                    element_required,
782                    element,
783                } => Self::List(ListType {
784                    element_field: NestedField::list_element(
785                        element_id,
786                        element.into_owned(),
787                        element_required,
788                    )
789                    .into(),
790                }),
791                SerdeType::Map {
792                    r#type: _,
793                    key_id,
794                    key,
795                    value_id,
796                    value_required,
797                    value,
798                } => Map(MapType {
799                    key_field: NestedField::map_key_element(key_id, key.into_owned()).into(),
800                    value_field: NestedField::map_value_element(
801                        value_id,
802                        value.into_owned(),
803                        value_required,
804                    )
805                    .into(),
806                }),
807                SerdeType::Struct { r#type: _, fields } => {
808                    Self::Struct(StructType::new(fields.into_owned()))
809                }
810                SerdeType::Primitive(p) => Self::Primitive(p),
811                SerdeType::Variant(v) => Self::Variant(v),
812            }
813        }
814    }
815
816    impl<'a> From<&'a Type> for SerdeType<'a> {
817        fn from(value: &'a Type) -> Self {
818            match value {
819                Type::List(list) => SerdeType::List {
820                    r#type: "list".to_string(),
821                    element_id: list.element_field.id,
822                    element_required: list.element_field.required,
823                    element: Cow::Borrowed(&list.element_field.field_type),
824                },
825                Map(map) => SerdeType::Map {
826                    r#type: "map".to_string(),
827                    key_id: map.key_field.id,
828                    key: Cow::Borrowed(&map.key_field.field_type),
829                    value_id: map.value_field.id,
830                    value_required: map.value_field.required,
831                    value: Cow::Borrowed(&map.value_field.field_type),
832                },
833                Type::Struct(s) => SerdeType::Struct {
834                    r#type: "struct".to_string(),
835                    fields: Cow::Borrowed(&s.fields),
836                },
837                Type::Primitive(p) => SerdeType::Primitive(p.clone()),
838                Type::Variant(v) => SerdeType::Variant(*v),
839            }
840        }
841    }
842}
843
844#[derive(Debug, PartialEq, Eq, Clone)]
845/// A map is a collection of key-value pairs with a key type and a value type.
846/// Both the key field and value field each have an integer id that is unique in the table schema.
847/// Map keys are required and map values can be either optional or required.
848/// Both map keys and map values may be any type, including nested types.
849pub struct MapType {
850    /// Field for key.
851    pub key_field: NestedFieldRef,
852    /// Field for value.
853    pub value_field: NestedFieldRef,
854}
855
856impl MapType {
857    /// Construct a map type with the given key and value fields.
858    pub fn new(key_field: NestedFieldRef, value_field: NestedFieldRef) -> Self {
859        Self {
860            key_field,
861            value_field,
862        }
863    }
864
865    /// Construct an optional map type with the given key and value fields.
866    pub fn optional(key_id: i32, key_type: Type, value_id: i32, value_type: Type) -> Self {
867        Self {
868            key_field: NestedField::map_key_element(key_id, key_type).into(),
869            value_field: NestedField::map_value_element(value_id, value_type, false).into(),
870        }
871    }
872
873    /// Construct a required map type with the given key and value fields.
874    pub fn required(key_id: i32, key_type: Type, value_id: i32, value_type: Type) -> Self {
875        Self {
876            key_field: NestedField::map_key_element(key_id, key_type).into(),
877            value_field: NestedField::map_value_element(value_id, value_type, true).into(),
878        }
879    }
880}
881
882/// Variant type - can hold semi-structured data of any type.
883/// This is an Iceberg V3 feature.
884#[derive(Debug, PartialEq, Eq, Clone, Copy, Hash)]
885pub struct VariantType;
886
887impl fmt::Display for VariantType {
888    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
889        write!(f, "variant")
890    }
891}
892
893impl From<VariantType> for Type {
894    fn from(_: VariantType) -> Self {
895        Type::Variant(VariantType)
896    }
897}
898
899impl Serialize for VariantType {
900    fn serialize<S>(&self, serializer: S) -> std::result::Result<S::Ok, S::Error>
901    where S: Serializer {
902        serializer.serialize_str("variant")
903    }
904}
905
906impl<'de> Deserialize<'de> for VariantType {
907    fn deserialize<D>(deserializer: D) -> std::result::Result<Self, D::Error>
908    where D: Deserializer<'de> {
909        let s = String::deserialize(deserializer)?;
910        if s == "variant" {
911            Ok(VariantType)
912        } else {
913            Err(D::Error::custom(format!("expected 'variant', got '{s}'")))
914        }
915    }
916}
917
918#[cfg(test)]
919mod tests {
920    use pretty_assertions::assert_eq;
921    use uuid::Uuid;
922
923    use super::*;
924    use crate::spec::values::PrimitiveLiteral;
925
926    fn check_type_serde(json: &str, expected_type: Type) {
927        let desered_type: Type = serde_json::from_str(json).unwrap();
928        assert_eq!(desered_type, expected_type);
929
930        let sered_json = serde_json::to_string(&expected_type).unwrap();
931        let parsed_json_value = serde_json::from_str::<serde_json::Value>(&sered_json).unwrap();
932        let raw_json_value = serde_json::from_str::<serde_json::Value>(json).unwrap();
933
934        assert_eq!(parsed_json_value, raw_json_value);
935    }
936
937    #[test]
938    fn primitive_type_serde() {
939        let record = r#"
940    {
941        "type": "struct",
942        "fields": [
943            {"id": 1, "name": "bool_field", "required": true, "type": "boolean"},
944            {"id": 2, "name": "int_field", "required": true, "type": "int"},
945            {"id": 3, "name": "long_field", "required": true, "type": "long"},
946            {"id": 4, "name": "float_field", "required": true, "type": "float"},
947            {"id": 5, "name": "double_field", "required": true, "type": "double"},
948            {"id": 6, "name": "decimal_field", "required": true, "type": "decimal(9, 2)"},
949            {"id": 7, "name": "date_field", "required": true, "type": "date"},
950            {"id": 8, "name": "time_field", "required": true, "type": "time"},
951            {"id": 9, "name": "timestamp_field", "required": true, "type": "timestamp"},
952            {"id": 10, "name": "timestamptz_field", "required": true, "type": "timestamptz"},
953            {"id": 11, "name": "timestamp_ns_field", "required": true, "type": "timestamp_ns"},
954            {"id": 12, "name": "timestamptz_ns_field", "required": true, "type": "timestamptz_ns"},
955            {"id": 13, "name": "uuid_field", "required": true, "type": "uuid"},
956            {"id": 14, "name": "fixed_field", "required": true, "type": "fixed[10]"},
957            {"id": 15, "name": "binary_field", "required": true, "type": "binary"},
958            {"id": 16, "name": "string_field", "required": true, "type": "string"}
959        ]
960    }
961    "#;
962
963        check_type_serde(
964            record,
965            Type::Struct(StructType {
966                fields: vec![
967                    NestedField::required(1, "bool_field", Type::Primitive(PrimitiveType::Boolean))
968                        .into(),
969                    NestedField::required(2, "int_field", Type::Primitive(PrimitiveType::Int))
970                        .into(),
971                    NestedField::required(3, "long_field", Type::Primitive(PrimitiveType::Long))
972                        .into(),
973                    NestedField::required(4, "float_field", Type::Primitive(PrimitiveType::Float))
974                        .into(),
975                    NestedField::required(
976                        5,
977                        "double_field",
978                        Type::Primitive(PrimitiveType::Double),
979                    )
980                    .into(),
981                    NestedField::required(
982                        6,
983                        "decimal_field",
984                        Type::Primitive(PrimitiveType::Decimal {
985                            precision: 9,
986                            scale: 2,
987                        }),
988                    )
989                    .into(),
990                    NestedField::required(7, "date_field", Type::Primitive(PrimitiveType::Date))
991                        .into(),
992                    NestedField::required(8, "time_field", Type::Primitive(PrimitiveType::Time))
993                        .into(),
994                    NestedField::required(
995                        9,
996                        "timestamp_field",
997                        Type::Primitive(PrimitiveType::Timestamp),
998                    )
999                    .into(),
1000                    NestedField::required(
1001                        10,
1002                        "timestamptz_field",
1003                        Type::Primitive(PrimitiveType::Timestamptz),
1004                    )
1005                    .into(),
1006                    NestedField::required(
1007                        11,
1008                        "timestamp_ns_field",
1009                        Type::Primitive(PrimitiveType::TimestampNs),
1010                    )
1011                    .into(),
1012                    NestedField::required(
1013                        12,
1014                        "timestamptz_ns_field",
1015                        Type::Primitive(PrimitiveType::TimestamptzNs),
1016                    )
1017                    .into(),
1018                    NestedField::required(13, "uuid_field", Type::Primitive(PrimitiveType::Uuid))
1019                        .into(),
1020                    NestedField::required(
1021                        14,
1022                        "fixed_field",
1023                        Type::Primitive(PrimitiveType::Fixed(10)),
1024                    )
1025                    .into(),
1026                    NestedField::required(
1027                        15,
1028                        "binary_field",
1029                        Type::Primitive(PrimitiveType::Binary),
1030                    )
1031                    .into(),
1032                    NestedField::required(
1033                        16,
1034                        "string_field",
1035                        Type::Primitive(PrimitiveType::String),
1036                    )
1037                    .into(),
1038                ],
1039                id_lookup: OnceLock::default(),
1040                name_lookup: OnceLock::default(),
1041            }),
1042        )
1043    }
1044
1045    #[test]
1046    fn struct_type() {
1047        let record = r#"
1048        {
1049            "type": "struct",
1050            "fields": [
1051                {
1052                    "id": 1,
1053                    "name": "id",
1054                    "required": true,
1055                    "type": "uuid",
1056                    "initial-default": "0db3e2a8-9d1d-42b9-aa7b-74ebe558dceb",
1057                    "write-default": "ec5911be-b0a7-458c-8438-c9a3e53cffae"
1058                }, {
1059                    "id": 2,
1060                    "name": "data",
1061                    "required": false,
1062                    "type": "int"
1063                }
1064            ]
1065        }
1066        "#;
1067
1068        check_type_serde(
1069            record,
1070            Type::Struct(StructType {
1071                fields: vec![
1072                    NestedField::required(1, "id", Type::Primitive(PrimitiveType::Uuid))
1073                        .with_initial_default(Literal::Primitive(PrimitiveLiteral::UInt128(
1074                            Uuid::parse_str("0db3e2a8-9d1d-42b9-aa7b-74ebe558dceb")
1075                                .unwrap()
1076                                .as_u128(),
1077                        )))
1078                        .with_write_default(Literal::Primitive(PrimitiveLiteral::UInt128(
1079                            Uuid::parse_str("ec5911be-b0a7-458c-8438-c9a3e53cffae")
1080                                .unwrap()
1081                                .as_u128(),
1082                        )))
1083                        .into(),
1084                    NestedField::optional(2, "data", Type::Primitive(PrimitiveType::Int)).into(),
1085                ],
1086                id_lookup: HashMap::from([(1, 0), (2, 1)]).into(),
1087                name_lookup: HashMap::from([("id".to_string(), 0), ("data".to_string(), 1)]).into(),
1088            }),
1089        )
1090    }
1091
1092    #[test]
1093    fn test_deeply_nested_struct() {
1094        let record = r#"
1095{
1096  "type": "struct",
1097  "fields": [
1098    {
1099      "id": 1,
1100      "name": "id",
1101      "required": true,
1102      "type": "uuid",
1103      "initial-default": "0db3e2a8-9d1d-42b9-aa7b-74ebe558dceb",
1104      "write-default": "ec5911be-b0a7-458c-8438-c9a3e53cffae"
1105    },
1106    {
1107      "id": 2,
1108      "name": "data",
1109      "required": false,
1110      "type": "int"
1111    },
1112    {
1113      "id": 3,
1114      "name": "address",
1115      "required": true,
1116      "type": {
1117        "type": "struct",
1118        "fields": [
1119          {
1120            "id": 4,
1121            "name": "street",
1122            "required": true,
1123            "type": "string"
1124          },
1125          {
1126            "id": 5,
1127            "name": "province",
1128            "required": false,
1129            "type": "string"
1130          },
1131          {
1132            "id": 6,
1133            "name": "zip",
1134            "required": true,
1135            "type": "int"
1136          }
1137        ]
1138      }
1139    }
1140  ]
1141}
1142"#;
1143
1144        let struct_type = Type::Struct(StructType::new(vec![
1145            NestedField::required(1, "id", Type::Primitive(PrimitiveType::Uuid))
1146                .with_initial_default(Literal::Primitive(PrimitiveLiteral::UInt128(
1147                    Uuid::parse_str("0db3e2a8-9d1d-42b9-aa7b-74ebe558dceb")
1148                        .unwrap()
1149                        .as_u128(),
1150                )))
1151                .with_write_default(Literal::Primitive(PrimitiveLiteral::UInt128(
1152                    Uuid::parse_str("ec5911be-b0a7-458c-8438-c9a3e53cffae")
1153                        .unwrap()
1154                        .as_u128(),
1155                )))
1156                .into(),
1157            NestedField::optional(2, "data", Type::Primitive(PrimitiveType::Int)).into(),
1158            NestedField::required(
1159                3,
1160                "address",
1161                Type::Struct(StructType::new(vec![
1162                    NestedField::required(4, "street", Type::Primitive(PrimitiveType::String))
1163                        .into(),
1164                    NestedField::optional(5, "province", Type::Primitive(PrimitiveType::String))
1165                        .into(),
1166                    NestedField::required(6, "zip", Type::Primitive(PrimitiveType::Int)).into(),
1167                ])),
1168            )
1169            .into(),
1170        ]));
1171
1172        check_type_serde(record, struct_type)
1173    }
1174
1175    #[test]
1176    fn list() {
1177        let record = r#"
1178        {
1179            "type": "list",
1180            "element-id": 3,
1181            "element-required": true,
1182            "element": "string"
1183        }
1184        "#;
1185
1186        check_type_serde(
1187            record,
1188            Type::List(ListType {
1189                element_field: NestedField::list_element(
1190                    3,
1191                    Type::Primitive(PrimitiveType::String),
1192                    true,
1193                )
1194                .into(),
1195            }),
1196        );
1197    }
1198
1199    #[test]
1200    fn map() {
1201        let record = r#"
1202        {
1203            "type": "map",
1204            "key-id": 4,
1205            "key": "string",
1206            "value-id": 5,
1207            "value-required": false,
1208            "value": "double"
1209        }
1210        "#;
1211
1212        check_type_serde(
1213            record,
1214            Type::Map(MapType {
1215                key_field: NestedField::map_key_element(4, Type::Primitive(PrimitiveType::String))
1216                    .into(),
1217                value_field: NestedField::map_value_element(
1218                    5,
1219                    Type::Primitive(PrimitiveType::Double),
1220                    false,
1221                )
1222                .into(),
1223            }),
1224        );
1225
1226        check_type_serde(
1227            record,
1228            Type::Map(MapType::optional(
1229                4,
1230                Type::Primitive(PrimitiveType::String),
1231                5,
1232                Type::Primitive(PrimitiveType::Double),
1233            )),
1234        );
1235    }
1236
1237    #[test]
1238    fn map_int() {
1239        let record = r#"
1240        {
1241            "type": "map",
1242            "key-id": 4,
1243            "key": "int",
1244            "value-id": 5,
1245            "value-required": false,
1246            "value": "string"
1247        }
1248        "#;
1249
1250        check_type_serde(
1251            record,
1252            Type::Map(MapType {
1253                key_field: NestedField::map_key_element(4, Type::Primitive(PrimitiveType::Int))
1254                    .into(),
1255                value_field: NestedField::map_value_element(
1256                    5,
1257                    Type::Primitive(PrimitiveType::String),
1258                    false,
1259                )
1260                .into(),
1261            }),
1262        );
1263
1264        check_type_serde(
1265            record,
1266            Type::Map(MapType::optional(
1267                4,
1268                Type::Primitive(PrimitiveType::Int),
1269                5,
1270                Type::Primitive(PrimitiveType::String),
1271            )),
1272        );
1273    }
1274
1275    #[test]
1276    fn map_required_int() {
1277        let record = r#"
1278        {
1279            "type": "map",
1280            "key-id": 4,
1281            "key": "int",
1282            "value-id": 5,
1283            "value-required": true,
1284            "value": "string"
1285        }
1286        "#;
1287
1288        check_type_serde(
1289            record,
1290            Type::Map(MapType::required(
1291                4,
1292                Type::Primitive(PrimitiveType::Int),
1293                5,
1294                Type::Primitive(PrimitiveType::String),
1295            )),
1296        );
1297    }
1298
1299    #[test]
1300    fn test_decimal_precision() {
1301        let expected_max_precision = [
1302            2, 4, 6, 9, 11, 14, 16, 18, 21, 23, 26, 28, 31, 33, 35, 38, 40, 43, 45, 47, 50, 52, 55,
1303            57,
1304        ];
1305        for (i, max_precision) in expected_max_precision.iter().enumerate() {
1306            assert_eq!(
1307                *max_precision,
1308                Type::decimal_max_precision(i as u32 + 1).unwrap(),
1309                "Failed calculate max precision for {i}"
1310            );
1311        }
1312
1313        assert_eq!(5, Type::decimal_required_bytes(10).unwrap());
1314        assert_eq!(16, Type::decimal_required_bytes(38).unwrap());
1315    }
1316
1317    #[test]
1318    fn test_primitive_type_compatible() {
1319        let pairs = vec![
1320            (PrimitiveType::Boolean, PrimitiveLiteral::Boolean(true)),
1321            (PrimitiveType::Int, PrimitiveLiteral::Int(1)),
1322            (PrimitiveType::Long, PrimitiveLiteral::Long(1)),
1323            (PrimitiveType::Float, PrimitiveLiteral::Float(1.0.into())),
1324            (PrimitiveType::Double, PrimitiveLiteral::Double(1.0.into())),
1325            (
1326                PrimitiveType::Decimal {
1327                    precision: 9,
1328                    scale: 2,
1329                },
1330                PrimitiveLiteral::Int128(1),
1331            ),
1332            (PrimitiveType::Date, PrimitiveLiteral::Int(1)),
1333            (PrimitiveType::Time, PrimitiveLiteral::Long(1)),
1334            (PrimitiveType::Timestamptz, PrimitiveLiteral::Long(1)),
1335            (PrimitiveType::Timestamp, PrimitiveLiteral::Long(1)),
1336            (PrimitiveType::TimestamptzNs, PrimitiveLiteral::Long(1)),
1337            (PrimitiveType::TimestampNs, PrimitiveLiteral::Long(1)),
1338            (
1339                PrimitiveType::Uuid,
1340                PrimitiveLiteral::UInt128(Uuid::new_v4().as_u128()),
1341            ),
1342            (PrimitiveType::Fixed(8), PrimitiveLiteral::Binary(vec![1])),
1343            (PrimitiveType::Binary, PrimitiveLiteral::Binary(vec![1])),
1344        ];
1345        for (ty, literal) in pairs {
1346            assert!(ty.compatible(&literal));
1347        }
1348    }
1349
1350    #[test]
1351    fn variant_type_serde() {
1352        let json = r#"{"id": 1, "name": "v", "required": true, "type": "variant"}"#;
1353        let field: NestedField = serde_json::from_str(json).unwrap();
1354        assert_eq!(*field.field_type, Type::Variant(VariantType));
1355
1356        let serialized = serde_json::to_string(&field).unwrap();
1357        let roundtrip: NestedField = serde_json::from_str(&serialized).unwrap();
1358        assert_eq!(field, roundtrip);
1359    }
1360
1361    #[test]
1362    fn nested_field_rejects_invalid_map_defaults() {
1363        for default_name in ["initial-default", "write-default"] {
1364            let json = format!(
1365                r#"{{
1366                    "id": 1,
1367                    "name": "properties",
1368                    "required": false,
1369                    "type": {{
1370                        "type": "map",
1371                        "key-id": 2,
1372                        "key": "string",
1373                        "value-id": 3,
1374                        "value-required": false,
1375                        "value": "int"
1376                    }},
1377                    "{default_name}": {{"keys": ["a", "b"], "values": [1]}}
1378                }}"#
1379            );
1380
1381            let error = serde_json::from_str::<NestedField>(&json).unwrap_err();
1382            assert!(error.to_string().contains("must have the same length"));
1383        }
1384    }
1385
1386    #[test]
1387    fn struct_type_with_type_field() {
1388        // Test that StructType properly deserializes JSON with "type":"struct" field
1389        // This was previously broken because the deserializer wasn't consuming the type field value
1390        let json = r#"
1391        {
1392            "type": "struct",
1393            "fields": [
1394                {"id": 1, "name": "field1", "required": true, "type": "string"}
1395            ]
1396        }
1397        "#;
1398
1399        let struct_type: StructType = serde_json::from_str(json)
1400            .expect("Should successfully deserialize StructType with type field");
1401
1402        assert_eq!(struct_type.fields().len(), 1);
1403        assert_eq!(struct_type.fields()[0].name, "field1");
1404    }
1405
1406    #[test]
1407    fn struct_type_rejects_wrong_type() {
1408        // Test that StructType validation rejects incorrect type field values
1409        let json = r#"
1410        {
1411            "type": "list",
1412            "fields": [
1413                {"id": 1, "name": "field1", "required": true, "type": "string"}
1414            ]
1415        }
1416        "#;
1417
1418        let result = serde_json::from_str::<StructType>(json);
1419        assert!(
1420            result.is_err(),
1421            "Should reject StructType with wrong type field"
1422        );
1423        assert!(
1424            result
1425                .unwrap_err()
1426                .to_string()
1427                .contains("expected type 'struct'")
1428        );
1429    }
1430}
iceberg/spec/datatypes.rs

iceberg/spec/
datatypes.rs
