iceberg/spec/
datatypes.rs

1// Licensed to the Apache Software Foundation (ASF) under one
2// or more contributor license agreements.  See the NOTICE file
3// distributed with this work for additional information
4// regarding copyright ownership.  The ASF licenses this file
5// to you under the Apache License, Version 2.0 (the
6// "License"); you may not use this file except in compliance
7// with the License.  You may obtain a copy of the License at
8//
9//   http://www.apache.org/licenses/LICENSE-2.0
10//
11// Unless required by applicable law or agreed to in writing,
12// software distributed under the License is distributed on an
13// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14// KIND, either express or implied.  See the License for the
15// specific language governing permissions and limitations
16// under the License.
17
18/*!
19 * Data Types
20 */
21use std::collections::HashMap;
22use std::fmt;
23use std::ops::Index;
24use std::sync::{Arc, OnceLock};
25
26use ::serde::de::{MapAccess, Visitor};
27use serde::de::{Error, IntoDeserializer};
28use serde::{Deserialize, Deserializer, Serialize, Serializer};
29use serde_json::Value as JsonValue;
30
31use super::values::Literal;
32use crate::ensure_data_valid;
33use crate::error::Result;
34use crate::spec::datatypes::_decimal::{MAX_PRECISION, REQUIRED_LENGTH};
35use crate::spec::{FormatVersion, PrimitiveLiteral};
36
37/// Field name for list type.
38pub const LIST_FIELD_NAME: &str = "element";
39/// Field name for map type's key.
40pub const MAP_KEY_FIELD_NAME: &str = "key";
41/// Field name for map type's value.
42pub const MAP_VALUE_FIELD_NAME: &str = "value";
43
44pub(crate) const MAX_DECIMAL_BYTES: u32 = 24;
45pub(crate) const MAX_DECIMAL_PRECISION: u32 = 38;
46
47mod _decimal {
48    use once_cell::sync::Lazy;
49
50    use crate::spec::{MAX_DECIMAL_BYTES, MAX_DECIMAL_PRECISION};
51
52    // Max precision of bytes, starts from 1
53    pub(super) static MAX_PRECISION: Lazy<[u32; MAX_DECIMAL_BYTES as usize]> = Lazy::new(|| {
54        let mut ret: [u32; 24] = [0; 24];
55        for (i, prec) in ret.iter_mut().enumerate() {
56            *prec = 2f64.powi((8 * (i + 1) - 1) as i32).log10().floor() as u32;
57        }
58
59        ret
60    });
61
62    //  Required bytes of precision, starts from 1
63    pub(super) static REQUIRED_LENGTH: Lazy<[u32; MAX_DECIMAL_PRECISION as usize]> =
64        Lazy::new(|| {
65            let mut ret: [u32; MAX_DECIMAL_PRECISION as usize] =
66                [0; MAX_DECIMAL_PRECISION as usize];
67
68            for (i, required_len) in ret.iter_mut().enumerate() {
69                for j in 0..MAX_PRECISION.len() {
70                    if MAX_PRECISION[j] >= ((i + 1) as u32) {
71                        *required_len = (j + 1) as u32;
72                        break;
73                    }
74                }
75            }
76
77            ret
78        });
79}
80
81#[derive(Debug, PartialEq, Eq, Clone)]
82/// All data types are either primitives or nested types, which are maps, lists, or structs.
83pub enum Type {
84    /// Primitive types
85    Primitive(PrimitiveType),
86    /// Struct type
87    Struct(StructType),
88    /// List type.
89    List(ListType),
90    /// Map type
91    Map(MapType),
92    /// Variant Type
93    Variant(VariantType),
94}
95
96impl fmt::Display for Type {
97    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
98        match self {
99            Type::Primitive(primitive) => write!(f, "{primitive}"),
100            Type::Struct(s) => write!(f, "{s}"),
101            Type::List(_) => write!(f, "list"),
102            Type::Map(_) => write!(f, "map"),
103            Type::Variant(_) => write!(f, "variant"),
104        }
105    }
106}
107
108impl Type {
109    /// Whether the type is primitive type.
110    #[inline(always)]
111    pub fn is_primitive(&self) -> bool {
112        matches!(self, Type::Primitive(_))
113    }
114
115    /// Whether the type is struct type.
116    #[inline(always)]
117    pub fn is_struct(&self) -> bool {
118        matches!(self, Type::Struct(_))
119    }
120
121    /// Whether the type is nested type.
122    #[inline(always)]
123    pub fn is_nested(&self) -> bool {
124        matches!(self, Type::Struct(_) | Type::List(_) | Type::Map(_))
125    }
126
127    /// Whether the type is variant type.
128    #[inline(always)]
129    pub fn is_variant(&self) -> bool {
130        matches!(self, Type::Variant(_))
131    }
132
133    /// Minimum [`FormatVersion`] required to support this type, **without** taking
134    /// nested field types into account.
135    ///
136    /// `Unknown` / `TimestampNs` / `TimestamptzNs` / `Variant` require
137    /// [`FormatVersion::V3`]; every other type is valid from [`FormatVersion::V1`]. Mirrors Java's
138    /// `Schema.MIN_FORMAT_VERSIONS` (a shallow lookup keyed by type id), so it
139    /// intentionally does not recurse: callers needing the floor for a whole schema
140    /// iterate its flattened fields (see [`Schema::calc_min_compatible_format`]).
141    ///
142    /// [`Schema::calc_min_compatible_format`]: crate::spec::Schema::calc_min_compatible_format
143    pub(crate) fn min_format_version(&self) -> FormatVersion {
144        match self {
145            Type::Primitive(
146                PrimitiveType::Unknown | PrimitiveType::TimestampNs | PrimitiveType::TimestamptzNs,
147            )
148            | Type::Variant(_) => FormatVersion::V3,
149            _ => FormatVersion::V1,
150        }
151    }
152
153    /// Convert Type to reference of PrimitiveType
154    pub fn as_primitive_type(&self) -> Option<&PrimitiveType> {
155        if let Type::Primitive(primitive_type) = self {
156            Some(primitive_type)
157        } else {
158            None
159        }
160    }
161
162    /// Convert Type to StructType
163    pub fn to_struct_type(self) -> Option<StructType> {
164        if let Type::Struct(struct_type) = self {
165            Some(struct_type)
166        } else {
167            None
168        }
169    }
170
171    /// Return max precision for decimal given `num_bytes` bytes.
172    #[inline(always)]
173    pub fn decimal_max_precision(num_bytes: u32) -> Result<u32> {
174        ensure_data_valid!(
175            num_bytes > 0 && num_bytes <= MAX_DECIMAL_BYTES,
176            "Decimal length larger than {MAX_DECIMAL_BYTES} is not supported: {num_bytes}",
177        );
178        Ok(MAX_PRECISION[num_bytes as usize - 1])
179    }
180
181    /// Returns minimum bytes required for decimal with `precision`.
182    #[inline(always)]
183    pub fn decimal_required_bytes(precision: u32) -> Result<u32> {
184        ensure_data_valid!(
185            precision > 0 && precision <= MAX_DECIMAL_PRECISION,
186            "Decimals with precision larger than {MAX_DECIMAL_PRECISION} are not supported: {precision}",
187        );
188        Ok(REQUIRED_LENGTH[precision as usize - 1])
189    }
190
191    /// Creates  decimal type.
192    #[inline(always)]
193    pub fn decimal(precision: u32, scale: u32) -> Result<Self> {
194        validate_decimal_precision(precision)?;
195        Ok(Type::Primitive(PrimitiveType::Decimal { precision, scale }))
196    }
197
198    /// Check if it's float or double type.
199    #[inline(always)]
200    pub fn is_floating_type(&self) -> bool {
201        matches!(
202            self,
203            Type::Primitive(PrimitiveType::Float) | Type::Primitive(PrimitiveType::Double)
204        )
205    }
206}
207
208impl From<PrimitiveType> for Type {
209    fn from(value: PrimitiveType) -> Self {
210        Self::Primitive(value)
211    }
212}
213
214impl From<StructType> for Type {
215    fn from(value: StructType) -> Self {
216        Type::Struct(value)
217    }
218}
219
220impl From<ListType> for Type {
221    fn from(value: ListType) -> Self {
222        Type::List(value)
223    }
224}
225
226impl From<MapType> for Type {
227    fn from(value: MapType) -> Self {
228        Type::Map(value)
229    }
230}
231
232/// Primitive data types
233#[derive(Debug, Serialize, Deserialize, PartialEq, Eq, Clone, Hash)]
234#[serde(rename_all = "lowercase", remote = "Self")]
235pub enum PrimitiveType {
236    /// True or False
237    Boolean,
238    /// 32-bit signed integer
239    Int,
240    /// 64-bit signed integer
241    Long,
242    /// 32-bit IEEE 754 floating point.
243    Float,
244    /// 64-bit IEEE 754 floating point.
245    Double,
246    /// Fixed point decimal
247    Decimal {
248        /// Precision, must be 38 or less
249        precision: u32,
250        /// Scale
251        scale: u32,
252    },
253    /// Calendar date without timezone or time.
254    Date,
255    /// Time of day in microsecond precision, without date or timezone.
256    Time,
257    /// Timestamp in microsecond precision, without timezone
258    Timestamp,
259    /// Timestamp in microsecond precision, with timezone
260    Timestamptz,
261    /// Timestamp in nanosecond precision, without timezone
262    #[serde(rename = "timestamp_ns")]
263    TimestampNs,
264    /// Timestamp in nanosecond precision with timezone
265    #[serde(rename = "timestamptz_ns")]
266    TimestamptzNs,
267    /// Arbitrary-length character sequences encoded in utf-8
268    String,
269    /// Universally Unique Identifiers, should use 16-byte fixed
270    Uuid,
271    /// Fixed length byte array
272    Fixed(u64),
273    /// Arbitrary-length byte array.
274    Binary,
275    /// Default / null column type used when a more specific type is not known.
276    Unknown,
277}
278
279impl PrimitiveType {
280    /// Check whether literal is compatible with the type.
281    pub fn compatible(&self, literal: &PrimitiveLiteral) -> bool {
282        matches!(
283            (self, literal),
284            (PrimitiveType::Boolean, PrimitiveLiteral::Boolean(_))
285                | (PrimitiveType::Int, PrimitiveLiteral::Int(_))
286                | (PrimitiveType::Long, PrimitiveLiteral::Long(_))
287                | (PrimitiveType::Float, PrimitiveLiteral::Float(_))
288                | (PrimitiveType::Double, PrimitiveLiteral::Double(_))
289                | (PrimitiveType::Decimal { .. }, PrimitiveLiteral::Int128(_))
290                | (PrimitiveType::Date, PrimitiveLiteral::Int(_))
291                | (PrimitiveType::Time, PrimitiveLiteral::Long(_))
292                | (PrimitiveType::Timestamp, PrimitiveLiteral::Long(_))
293                | (PrimitiveType::Timestamptz, PrimitiveLiteral::Long(_))
294                | (PrimitiveType::TimestampNs, PrimitiveLiteral::Long(_))
295                | (PrimitiveType::TimestamptzNs, PrimitiveLiteral::Long(_))
296                | (PrimitiveType::String, PrimitiveLiteral::String(_))
297                | (PrimitiveType::Uuid, PrimitiveLiteral::UInt128(_))
298                | (PrimitiveType::Fixed(_), PrimitiveLiteral::Binary(_))
299                | (PrimitiveType::Binary, PrimitiveLiteral::Binary(_))
300        )
301    }
302}
303
304impl Serialize for Type {
305    fn serialize<S>(&self, serializer: S) -> std::result::Result<S::Ok, S::Error>
306    where S: Serializer {
307        let type_serde = _serde::SerdeType::from(self);
308        type_serde.serialize(serializer)
309    }
310}
311
312impl<'de> Deserialize<'de> for Type {
313    fn deserialize<D>(deserializer: D) -> std::result::Result<Self, D::Error>
314    where D: Deserializer<'de> {
315        let type_serde = _serde::SerdeType::deserialize(deserializer)?;
316        Ok(Type::from(type_serde))
317    }
318}
319
320impl<'de> Deserialize<'de> for PrimitiveType {
321    fn deserialize<D>(deserializer: D) -> std::result::Result<Self, D::Error>
322    where D: Deserializer<'de> {
323        let s = String::deserialize(deserializer)?;
324        if s.starts_with("decimal") {
325            deserialize_decimal(s.into_deserializer())
326        } else if s.starts_with("fixed") {
327            deserialize_fixed(s.into_deserializer())
328        } else {
329            PrimitiveType::deserialize(s.into_deserializer())
330        }
331    }
332}
333
334impl Serialize for PrimitiveType {
335    fn serialize<S>(&self, serializer: S) -> std::result::Result<S::Ok, S::Error>
336    where S: Serializer {
337        match self {
338            PrimitiveType::Decimal { precision, scale } => {
339                serialize_decimal(precision, scale, serializer)
340            }
341            PrimitiveType::Fixed(l) => serialize_fixed(l, serializer),
342            _ => PrimitiveType::serialize(self, serializer),
343        }
344    }
345}
346
347fn validate_decimal_precision(precision: u32) -> Result<()> {
348    // Leave scale unconstrained for Java parity; the spec leaves scale > precision open.
349    ensure_data_valid!(precision > 0, "Decimal precision must be greater than zero",);
350    ensure_data_valid!(
351        precision <= MAX_DECIMAL_PRECISION,
352        "Decimals with precision larger than {MAX_DECIMAL_PRECISION} are not supported: {precision}",
353    );
354    Ok(())
355}
356
357fn deserialize_decimal<'de, D>(deserializer: D) -> std::result::Result<PrimitiveType, D::Error>
358where D: Deserializer<'de> {
359    let s = String::deserialize(deserializer)?;
360    let malformed = || D::Error::custom(format!("Invalid decimal type: {s}"));
361
362    let (precision, scale) = s
363        .strip_prefix("decimal(")
364        .and_then(|inner| inner.strip_suffix(')'))
365        .ok_or_else(malformed)?
366        .split_once(',')
367        .ok_or_else(|| D::Error::custom(format!("Decimal requires precision and scale: {s}")))?;
368
369    let (precision, scale) = (precision.trim(), scale.trim());
370    if [precision, scale]
371        .iter()
372        .any(|token| token.is_empty() || !token.bytes().all(|byte| byte.is_ascii_digit()))
373    {
374        return Err(malformed());
375    }
376    let precision: u32 = precision.parse().map_err(D::Error::custom)?;
377    let scale: u32 = scale.parse().map_err(D::Error::custom)?;
378    validate_decimal_precision(precision).map_err(D::Error::custom)?;
379
380    Ok(PrimitiveType::Decimal { precision, scale })
381}
382
383fn serialize_decimal<S>(
384    precision: &u32,
385    scale: &u32,
386    serializer: S,
387) -> std::result::Result<S::Ok, S::Error>
388where
389    S: Serializer,
390{
391    serializer.serialize_str(&format!("decimal({precision}, {scale})"))
392}
393
394fn deserialize_fixed<'de, D>(deserializer: D) -> std::result::Result<PrimitiveType, D::Error>
395where D: Deserializer<'de> {
396    let s = String::deserialize(deserializer)?;
397
398    let length = s
399        .strip_prefix("fixed[")
400        .and_then(|inner| inner.strip_suffix(']'))
401        .map(str::trim)
402        .filter(|length| !length.is_empty() && length.bytes().all(|byte| byte.is_ascii_digit()))
403        .ok_or_else(|| D::Error::custom(format!("Invalid fixed type: {s}")))?;
404
405    length
406        .parse()
407        .map(PrimitiveType::Fixed)
408        .map_err(D::Error::custom)
409}
410
411fn serialize_fixed<S>(value: &u64, serializer: S) -> std::result::Result<S::Ok, S::Error>
412where S: Serializer {
413    serializer.serialize_str(&format!("fixed[{value}]"))
414}
415
416impl fmt::Display for PrimitiveType {
417    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
418        match self {
419            PrimitiveType::Unknown => write!(f, "unknown"),
420            PrimitiveType::Boolean => write!(f, "boolean"),
421            PrimitiveType::Int => write!(f, "int"),
422            PrimitiveType::Long => write!(f, "long"),
423            PrimitiveType::Float => write!(f, "float"),
424            PrimitiveType::Double => write!(f, "double"),
425            PrimitiveType::Decimal { precision, scale } => {
426                write!(f, "decimal({precision}, {scale})")
427            }
428            PrimitiveType::Date => write!(f, "date"),
429            PrimitiveType::Time => write!(f, "time"),
430            PrimitiveType::Timestamp => write!(f, "timestamp"),
431            PrimitiveType::Timestamptz => write!(f, "timestamptz"),
432            PrimitiveType::TimestampNs => write!(f, "timestamp_ns"),
433            PrimitiveType::TimestamptzNs => write!(f, "timestamptz_ns"),
434            PrimitiveType::String => write!(f, "string"),
435            PrimitiveType::Uuid => write!(f, "uuid"),
436            PrimitiveType::Fixed(size) => write!(f, "fixed({size})"),
437            PrimitiveType::Binary => write!(f, "binary"),
438        }
439    }
440}
441
442/// DataType for a specific struct
443#[derive(Debug, Serialize, Clone, Default)]
444#[serde(rename = "struct", tag = "type")]
445pub struct StructType {
446    /// Struct fields
447    fields: Vec<NestedFieldRef>,
448    /// Lookup for index by field id
449    #[serde(skip_serializing)]
450    id_lookup: OnceLock<HashMap<i32, usize>>,
451    #[serde(skip_serializing)]
452    name_lookup: OnceLock<HashMap<String, usize>>,
453}
454
455impl<'de> Deserialize<'de> for StructType {
456    fn deserialize<D>(deserializer: D) -> std::result::Result<Self, D::Error>
457    where D: Deserializer<'de> {
458        #[derive(Deserialize)]
459        #[serde(field_identifier, rename_all = "lowercase")]
460        enum Field {
461            Type,
462            Fields,
463        }
464
465        struct StructTypeVisitor;
466
467        impl<'de> Visitor<'de> for StructTypeVisitor {
468            type Value = StructType;
469
470            fn expecting(&self, formatter: &mut fmt::Formatter) -> fmt::Result {
471                formatter.write_str("struct")
472            }
473
474            fn visit_map<V>(self, mut map: V) -> std::result::Result<StructType, V::Error>
475            where V: MapAccess<'de> {
476                let mut fields = None;
477                while let Some(key) = map.next_key()? {
478                    match key {
479                        Field::Type => {
480                            let type_val: String = map.next_value()?;
481                            if type_val != "struct" {
482                                return Err(Error::custom(format!(
483                                    "expected type 'struct', got '{type_val}'"
484                                )));
485                            }
486                        }
487                        Field::Fields => {
488                            if fields.is_some() {
489                                return Err(Error::duplicate_field("fields"));
490                            }
491                            fields = Some(map.next_value()?);
492                        }
493                    }
494                }
495                let fields: Vec<NestedFieldRef> =
496                    fields.ok_or_else(|| Error::missing_field("fields"))?;
497
498                Ok(StructType::new(fields))
499            }
500        }
501
502        const FIELDS: &[&str] = &["type", "fields"];
503        deserializer.deserialize_struct("struct", FIELDS, StructTypeVisitor)
504    }
505}
506
507impl StructType {
508    /// Creates a struct type with the given fields.
509    pub fn new(fields: Vec<NestedFieldRef>) -> Self {
510        Self {
511            fields,
512            id_lookup: OnceLock::new(),
513            name_lookup: OnceLock::new(),
514        }
515    }
516
517    /// Get struct field with certain id
518    pub fn field_by_id(&self, id: i32) -> Option<&NestedFieldRef> {
519        self.field_id_to_index(id).map(|idx| &self.fields[idx])
520    }
521
522    fn field_id_to_index(&self, field_id: i32) -> Option<usize> {
523        self.id_lookup
524            .get_or_init(|| {
525                HashMap::from_iter(self.fields.iter().enumerate().map(|(i, x)| (x.id, i)))
526            })
527            .get(&field_id)
528            .copied()
529    }
530
531    /// Get struct field with certain field name
532    pub fn field_by_name(&self, name: &str) -> Option<&NestedFieldRef> {
533        self.field_name_to_index(name).map(|idx| &self.fields[idx])
534    }
535
536    fn field_name_to_index(&self, name: &str) -> Option<usize> {
537        self.name_lookup
538            .get_or_init(|| {
539                HashMap::from_iter(
540                    self.fields
541                        .iter()
542                        .enumerate()
543                        .map(|(i, x)| (x.name.clone(), i)),
544                )
545            })
546            .get(name)
547            .copied()
548    }
549
550    /// Get fields.
551    pub fn fields(&self) -> &[NestedFieldRef] {
552        &self.fields
553    }
554}
555
556impl PartialEq for StructType {
557    fn eq(&self, other: &Self) -> bool {
558        self.fields == other.fields
559    }
560}
561
562impl Eq for StructType {}
563
564impl Index<usize> for StructType {
565    type Output = NestedField;
566
567    fn index(&self, index: usize) -> &Self::Output {
568        &self.fields[index]
569    }
570}
571
572impl fmt::Display for StructType {
573    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
574        write!(f, "struct<")?;
575        for field in &self.fields {
576            write!(f, "{}", field.field_type)?;
577        }
578        write!(f, ">")
579    }
580}
581
582#[derive(Debug, PartialEq, Serialize, Deserialize, Eq, Clone)]
583#[serde(try_from = "SerdeNestedField", into = "SerdeNestedField")]
584/// A struct is a tuple of typed values. Each field in the tuple is named and has an integer id that is unique in the table schema.
585/// Each field can be either optional or required, meaning that values can (or cannot) be null. Fields may be any type.
586/// Fields may have an optional comment or doc string. Fields can have default values.
587pub struct NestedField {
588    /// Id unique in table schema
589    pub id: i32,
590    /// Field Name
591    pub name: String,
592    /// Optional or required
593    pub required: bool,
594    /// Datatype
595    pub field_type: Box<Type>,
596    /// Fields may have an optional comment or doc string.
597    pub doc: Option<String>,
598    /// Used to populate the field’s value for all records that were written before the field was added to the schema
599    pub initial_default: Option<Literal>,
600    /// Used to populate the field’s value for any records written after the field was added to the schema, if the writer does not supply the field’s value
601    pub write_default: Option<Literal>,
602}
603
604#[derive(Debug, Serialize, Deserialize, PartialEq, Eq, Clone)]
605#[serde(rename_all = "kebab-case")]
606struct SerdeNestedField {
607    pub id: i32,
608    pub name: String,
609    pub required: bool,
610    #[serde(rename = "type")]
611    pub field_type: Box<Type>,
612    #[serde(skip_serializing_if = "Option::is_none")]
613    pub doc: Option<String>,
614    #[serde(skip_serializing_if = "Option::is_none")]
615    pub initial_default: Option<JsonValue>,
616    #[serde(skip_serializing_if = "Option::is_none")]
617    pub write_default: Option<JsonValue>,
618}
619
620impl TryFrom<SerdeNestedField> for NestedField {
621    type Error = crate::Error;
622
623    fn try_from(value: SerdeNestedField) -> Result<Self> {
624        let initial_default = value
625            .initial_default
626            .map(|default| Literal::try_from_json(default, &value.field_type))
627            .transpose()?
628            .flatten();
629        let write_default = value
630            .write_default
631            .map(|default| Literal::try_from_json(default, &value.field_type))
632            .transpose()?
633            .flatten();
634
635        Ok(NestedField {
636            id: value.id,
637            name: value.name,
638            required: value.required,
639            initial_default,
640            write_default,
641            field_type: value.field_type,
642            doc: value.doc,
643        })
644    }
645}
646
647impl From<NestedField> for SerdeNestedField {
648    fn from(value: NestedField) -> Self {
649        let initial_default = value.initial_default.map(|x| x.try_into_json(&value.field_type).expect("We should have checked this in NestedField::with_initial_default, it can't be converted to json value"));
650        let write_default = value.write_default.map(|x| x.try_into_json(&value.field_type).expect("We should have checked this in NestedField::with_write_default, it can't be converted to json value"));
651        SerdeNestedField {
652            id: value.id,
653            name: value.name,
654            required: value.required,
655            field_type: value.field_type,
656            doc: value.doc,
657            initial_default,
658            write_default,
659        }
660    }
661}
662
663/// Reference to nested field.
664pub type NestedFieldRef = Arc<NestedField>;
665
666impl NestedField {
667    /// Construct a new field.
668    pub fn new(id: i32, name: impl ToString, field_type: Type, required: bool) -> Self {
669        Self {
670            id,
671            name: name.to_string(),
672            required,
673            field_type: Box::new(field_type),
674            doc: None,
675            initial_default: None,
676            write_default: None,
677        }
678    }
679
680    /// Construct a required field.
681    pub fn required(id: i32, name: impl ToString, field_type: Type) -> Self {
682        Self::new(id, name, field_type, true)
683    }
684
685    /// Construct an optional field.
686    pub fn optional(id: i32, name: impl ToString, field_type: Type) -> Self {
687        Self::new(id, name, field_type, false)
688    }
689
690    /// Construct list type's element field.
691    pub fn list_element(id: i32, field_type: Type, required: bool) -> Self {
692        Self::new(id, LIST_FIELD_NAME, field_type, required)
693    }
694
695    /// Construct map type's key field.
696    pub fn map_key_element(id: i32, field_type: Type) -> Self {
697        Self::required(id, MAP_KEY_FIELD_NAME, field_type)
698    }
699
700    /// Construct map type's value field.
701    pub fn map_value_element(id: i32, field_type: Type, required: bool) -> Self {
702        Self::new(id, MAP_VALUE_FIELD_NAME, field_type, required)
703    }
704
705    /// Set the field's doc.
706    pub fn with_doc(mut self, doc: impl ToString) -> Self {
707        self.doc = Some(doc.to_string());
708        self
709    }
710
711    /// Set the field's initial default value.
712    pub fn with_initial_default(mut self, value: Literal) -> Self {
713        self.initial_default = Some(value);
714        self
715    }
716
717    /// Set the field's initial default value.
718    pub fn with_write_default(mut self, value: Literal) -> Self {
719        self.write_default = Some(value);
720        self
721    }
722
723    /// Set the id of the field.
724    pub(crate) fn with_id(mut self, id: i32) -> Self {
725        self.id = id;
726        self
727    }
728}
729
730impl fmt::Display for NestedField {
731    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
732        write!(f, "{}: ", self.id)?;
733        write!(f, "{}: ", self.name)?;
734        if self.required {
735            write!(f, "required ")?;
736        } else {
737            write!(f, "optional ")?;
738        }
739        write!(f, "{} ", self.field_type)?;
740        if let Some(doc) = &self.doc {
741            write!(f, "{doc}")?;
742        }
743        Ok(())
744    }
745}
746
747#[derive(Debug, PartialEq, Eq, Clone)]
748/// A list is a collection of values with some element type. The element field has an integer id that is unique in the table schema.
749/// Elements can be either optional or required. Element types may be any type.
750pub struct ListType {
751    /// Element field of list type.
752    pub element_field: NestedFieldRef,
753}
754
755impl ListType {
756    /// Construct a list type with the given element field.
757    pub fn new(element_field: NestedFieldRef) -> Self {
758        Self { element_field }
759    }
760}
761
762/// Module for type serialization/deserialization.
763pub(super) mod _serde {
764    use std::borrow::Cow;
765
766    use serde_derive::{Deserialize, Serialize};
767
768    use crate::spec::datatypes::Type::Map;
769    use crate::spec::datatypes::{
770        ListType, MapType, NestedField, NestedFieldRef, PrimitiveType, StructType, Type,
771        VariantType,
772    };
773
774    /// List type for serialization and deserialization
775    #[derive(Serialize, Deserialize)]
776    #[serde(untagged)]
777    pub(super) enum SerdeType<'a> {
778        #[serde(rename_all = "kebab-case")]
779        List {
780            r#type: String,
781            element_id: i32,
782            element_required: bool,
783            element: Cow<'a, Type>,
784        },
785        Struct {
786            r#type: String,
787            fields: Cow<'a, [NestedFieldRef]>,
788        },
789        #[serde(rename_all = "kebab-case")]
790        Map {
791            r#type: String,
792            key_id: i32,
793            key: Cow<'a, Type>,
794            value_id: i32,
795            value_required: bool,
796            value: Cow<'a, Type>,
797        },
798        Primitive(PrimitiveType),
799        Variant(VariantType),
800    }
801
802    impl From<SerdeType<'_>> for Type {
803        fn from(value: SerdeType) -> Self {
804            match value {
805                SerdeType::List {
806                    r#type: _,
807                    element_id,
808                    element_required,
809                    element,
810                } => Self::List(ListType {
811                    element_field: NestedField::list_element(
812                        element_id,
813                        element.into_owned(),
814                        element_required,
815                    )
816                    .into(),
817                }),
818                SerdeType::Map {
819                    r#type: _,
820                    key_id,
821                    key,
822                    value_id,
823                    value_required,
824                    value,
825                } => Map(MapType {
826                    key_field: NestedField::map_key_element(key_id, key.into_owned()).into(),
827                    value_field: NestedField::map_value_element(
828                        value_id,
829                        value.into_owned(),
830                        value_required,
831                    )
832                    .into(),
833                }),
834                SerdeType::Struct { r#type: _, fields } => {
835                    Self::Struct(StructType::new(fields.into_owned()))
836                }
837                SerdeType::Primitive(p) => Self::Primitive(p),
838                SerdeType::Variant(v) => Self::Variant(v),
839            }
840        }
841    }
842
843    impl<'a> From<&'a Type> for SerdeType<'a> {
844        fn from(value: &'a Type) -> Self {
845            match value {
846                Type::List(list) => SerdeType::List {
847                    r#type: "list".to_string(),
848                    element_id: list.element_field.id,
849                    element_required: list.element_field.required,
850                    element: Cow::Borrowed(&list.element_field.field_type),
851                },
852                Map(map) => SerdeType::Map {
853                    r#type: "map".to_string(),
854                    key_id: map.key_field.id,
855                    key: Cow::Borrowed(&map.key_field.field_type),
856                    value_id: map.value_field.id,
857                    value_required: map.value_field.required,
858                    value: Cow::Borrowed(&map.value_field.field_type),
859                },
860                Type::Struct(s) => SerdeType::Struct {
861                    r#type: "struct".to_string(),
862                    fields: Cow::Borrowed(&s.fields),
863                },
864                Type::Primitive(p) => SerdeType::Primitive(p.clone()),
865                Type::Variant(v) => SerdeType::Variant(*v),
866            }
867        }
868    }
869}
870
871#[derive(Debug, PartialEq, Eq, Clone)]
872/// A map is a collection of key-value pairs with a key type and a value type.
873/// Both the key field and value field each have an integer id that is unique in the table schema.
874/// Map keys are required and map values can be either optional or required.
875/// Both map keys and map values may be any type, including nested types.
876pub struct MapType {
877    /// Field for key.
878    pub key_field: NestedFieldRef,
879    /// Field for value.
880    pub value_field: NestedFieldRef,
881}
882
883impl MapType {
884    /// Construct a map type with the given key and value fields.
885    pub fn new(key_field: NestedFieldRef, value_field: NestedFieldRef) -> Self {
886        Self {
887            key_field,
888            value_field,
889        }
890    }
891
892    /// Construct an optional map type with the given key and value fields.
893    pub fn optional(key_id: i32, key_type: Type, value_id: i32, value_type: Type) -> Self {
894        Self {
895            key_field: NestedField::map_key_element(key_id, key_type).into(),
896            value_field: NestedField::map_value_element(value_id, value_type, false).into(),
897        }
898    }
899
900    /// Construct a required map type with the given key and value fields.
901    pub fn required(key_id: i32, key_type: Type, value_id: i32, value_type: Type) -> Self {
902        Self {
903            key_field: NestedField::map_key_element(key_id, key_type).into(),
904            value_field: NestedField::map_value_element(value_id, value_type, true).into(),
905        }
906    }
907}
908
909/// Variant type - can hold semi-structured data of any type.
910/// This is an Iceberg V3 feature.
911#[derive(Debug, PartialEq, Eq, Clone, Copy, Hash)]
912pub struct VariantType;
913
914impl fmt::Display for VariantType {
915    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
916        write!(f, "variant")
917    }
918}
919
920impl From<VariantType> for Type {
921    fn from(_: VariantType) -> Self {
922        Type::Variant(VariantType)
923    }
924}
925
926impl Serialize for VariantType {
927    fn serialize<S>(&self, serializer: S) -> std::result::Result<S::Ok, S::Error>
928    where S: Serializer {
929        serializer.serialize_str("variant")
930    }
931}
932
933impl<'de> Deserialize<'de> for VariantType {
934    fn deserialize<D>(deserializer: D) -> std::result::Result<Self, D::Error>
935    where D: Deserializer<'de> {
936        let s = String::deserialize(deserializer)?;
937        if s == "variant" {
938            Ok(VariantType)
939        } else {
940            Err(D::Error::custom(format!("expected 'variant', got '{s}'")))
941        }
942    }
943}
944
945#[cfg(test)]
946mod tests {
947    use pretty_assertions::assert_eq;
948    use uuid::Uuid;
949
950    use super::*;
951    use crate::spec::values::PrimitiveLiteral;
952
953    fn check_type_serde(json: &str, expected_type: Type) {
954        let desered_type: Type = serde_json::from_str(json).unwrap();
955        assert_eq!(desered_type, expected_type);
956
957        let sered_json = serde_json::to_string(&expected_type).unwrap();
958        let parsed_json_value = serde_json::from_str::<serde_json::Value>(&sered_json).unwrap();
959        let raw_json_value = serde_json::from_str::<serde_json::Value>(json).unwrap();
960
961        assert_eq!(parsed_json_value, raw_json_value);
962    }
963
964    #[test]
965    fn primitive_type_serde() {
966        let record = r#"
967    {
968        "type": "struct",
969        "fields": [
970            {"id": 17, "name": "unknown_field", "required": false, "type": "unknown"},
971            {"id": 1, "name": "bool_field", "required": true, "type": "boolean"},
972            {"id": 2, "name": "int_field", "required": true, "type": "int"},
973            {"id": 3, "name": "long_field", "required": true, "type": "long"},
974            {"id": 4, "name": "float_field", "required": true, "type": "float"},
975            {"id": 5, "name": "double_field", "required": true, "type": "double"},
976            {"id": 6, "name": "decimal_field", "required": true, "type": "decimal(9, 2)"},
977            {"id": 7, "name": "date_field", "required": true, "type": "date"},
978            {"id": 8, "name": "time_field", "required": true, "type": "time"},
979            {"id": 9, "name": "timestamp_field", "required": true, "type": "timestamp"},
980            {"id": 10, "name": "timestamptz_field", "required": true, "type": "timestamptz"},
981            {"id": 11, "name": "timestamp_ns_field", "required": true, "type": "timestamp_ns"},
982            {"id": 12, "name": "timestamptz_ns_field", "required": true, "type": "timestamptz_ns"},
983            {"id": 13, "name": "uuid_field", "required": true, "type": "uuid"},
984            {"id": 14, "name": "fixed_field", "required": true, "type": "fixed[10]"},
985            {"id": 15, "name": "binary_field", "required": true, "type": "binary"},
986            {"id": 16, "name": "string_field", "required": true, "type": "string"}
987        ]
988    }
989    "#;
990
991        check_type_serde(
992            record,
993            Type::Struct(StructType {
994                fields: vec![
995                    NestedField::optional(
996                        17,
997                        "unknown_field",
998                        Type::Primitive(PrimitiveType::Unknown),
999                    )
1000                    .into(),
1001                    NestedField::required(1, "bool_field", Type::Primitive(PrimitiveType::Boolean))
1002                        .into(),
1003                    NestedField::required(2, "int_field", Type::Primitive(PrimitiveType::Int))
1004                        .into(),
1005                    NestedField::required(3, "long_field", Type::Primitive(PrimitiveType::Long))
1006                        .into(),
1007                    NestedField::required(4, "float_field", Type::Primitive(PrimitiveType::Float))
1008                        .into(),
1009                    NestedField::required(
1010                        5,
1011                        "double_field",
1012                        Type::Primitive(PrimitiveType::Double),
1013                    )
1014                    .into(),
1015                    NestedField::required(
1016                        6,
1017                        "decimal_field",
1018                        Type::Primitive(PrimitiveType::Decimal {
1019                            precision: 9,
1020                            scale: 2,
1021                        }),
1022                    )
1023                    .into(),
1024                    NestedField::required(7, "date_field", Type::Primitive(PrimitiveType::Date))
1025                        .into(),
1026                    NestedField::required(8, "time_field", Type::Primitive(PrimitiveType::Time))
1027                        .into(),
1028                    NestedField::required(
1029                        9,
1030                        "timestamp_field",
1031                        Type::Primitive(PrimitiveType::Timestamp),
1032                    )
1033                    .into(),
1034                    NestedField::required(
1035                        10,
1036                        "timestamptz_field",
1037                        Type::Primitive(PrimitiveType::Timestamptz),
1038                    )
1039                    .into(),
1040                    NestedField::required(
1041                        11,
1042                        "timestamp_ns_field",
1043                        Type::Primitive(PrimitiveType::TimestampNs),
1044                    )
1045                    .into(),
1046                    NestedField::required(
1047                        12,
1048                        "timestamptz_ns_field",
1049                        Type::Primitive(PrimitiveType::TimestamptzNs),
1050                    )
1051                    .into(),
1052                    NestedField::required(13, "uuid_field", Type::Primitive(PrimitiveType::Uuid))
1053                        .into(),
1054                    NestedField::required(
1055                        14,
1056                        "fixed_field",
1057                        Type::Primitive(PrimitiveType::Fixed(10)),
1058                    )
1059                    .into(),
1060                    NestedField::required(
1061                        15,
1062                        "binary_field",
1063                        Type::Primitive(PrimitiveType::Binary),
1064                    )
1065                    .into(),
1066                    NestedField::required(
1067                        16,
1068                        "string_field",
1069                        Type::Primitive(PrimitiveType::String),
1070                    )
1071                    .into(),
1072                ],
1073                id_lookup: OnceLock::default(),
1074                name_lookup: OnceLock::default(),
1075            }),
1076        )
1077    }
1078
1079    #[test]
1080    fn struct_type() {
1081        let record = r#"
1082        {
1083            "type": "struct",
1084            "fields": [
1085                {
1086                    "id": 1,
1087                    "name": "id",
1088                    "required": true,
1089                    "type": "uuid",
1090                    "initial-default": "0db3e2a8-9d1d-42b9-aa7b-74ebe558dceb",
1091                    "write-default": "ec5911be-b0a7-458c-8438-c9a3e53cffae"
1092                }, {
1093                    "id": 2,
1094                    "name": "data",
1095                    "required": false,
1096                    "type": "int"
1097                }
1098            ]
1099        }
1100        "#;
1101
1102        check_type_serde(
1103            record,
1104            Type::Struct(StructType {
1105                fields: vec![
1106                    NestedField::required(1, "id", Type::Primitive(PrimitiveType::Uuid))
1107                        .with_initial_default(Literal::Primitive(PrimitiveLiteral::UInt128(
1108                            Uuid::parse_str("0db3e2a8-9d1d-42b9-aa7b-74ebe558dceb")
1109                                .unwrap()
1110                                .as_u128(),
1111                        )))
1112                        .with_write_default(Literal::Primitive(PrimitiveLiteral::UInt128(
1113                            Uuid::parse_str("ec5911be-b0a7-458c-8438-c9a3e53cffae")
1114                                .unwrap()
1115                                .as_u128(),
1116                        )))
1117                        .into(),
1118                    NestedField::optional(2, "data", Type::Primitive(PrimitiveType::Int)).into(),
1119                ],
1120                id_lookup: HashMap::from([(1, 0), (2, 1)]).into(),
1121                name_lookup: HashMap::from([("id".to_string(), 0), ("data".to_string(), 1)]).into(),
1122            }),
1123        )
1124    }
1125
1126    #[test]
1127    fn test_deeply_nested_struct() {
1128        let record = r#"
1129{
1130  "type": "struct",
1131  "fields": [
1132    {
1133      "id": 1,
1134      "name": "id",
1135      "required": true,
1136      "type": "uuid",
1137      "initial-default": "0db3e2a8-9d1d-42b9-aa7b-74ebe558dceb",
1138      "write-default": "ec5911be-b0a7-458c-8438-c9a3e53cffae"
1139    },
1140    {
1141      "id": 2,
1142      "name": "data",
1143      "required": false,
1144      "type": "int"
1145    },
1146    {
1147      "id": 3,
1148      "name": "address",
1149      "required": true,
1150      "type": {
1151        "type": "struct",
1152        "fields": [
1153          {
1154            "id": 4,
1155            "name": "street",
1156            "required": true,
1157            "type": "string"
1158          },
1159          {
1160            "id": 5,
1161            "name": "province",
1162            "required": false,
1163            "type": "string"
1164          },
1165          {
1166            "id": 6,
1167            "name": "zip",
1168            "required": true,
1169            "type": "int"
1170          }
1171        ]
1172      }
1173    }
1174  ]
1175}
1176"#;
1177
1178        let struct_type = Type::Struct(StructType::new(vec![
1179            NestedField::required(1, "id", Type::Primitive(PrimitiveType::Uuid))
1180                .with_initial_default(Literal::Primitive(PrimitiveLiteral::UInt128(
1181                    Uuid::parse_str("0db3e2a8-9d1d-42b9-aa7b-74ebe558dceb")
1182                        .unwrap()
1183                        .as_u128(),
1184                )))
1185                .with_write_default(Literal::Primitive(PrimitiveLiteral::UInt128(
1186                    Uuid::parse_str("ec5911be-b0a7-458c-8438-c9a3e53cffae")
1187                        .unwrap()
1188                        .as_u128(),
1189                )))
1190                .into(),
1191            NestedField::optional(2, "data", Type::Primitive(PrimitiveType::Int)).into(),
1192            NestedField::required(
1193                3,
1194                "address",
1195                Type::Struct(StructType::new(vec![
1196                    NestedField::required(4, "street", Type::Primitive(PrimitiveType::String))
1197                        .into(),
1198                    NestedField::optional(5, "province", Type::Primitive(PrimitiveType::String))
1199                        .into(),
1200                    NestedField::required(6, "zip", Type::Primitive(PrimitiveType::Int)).into(),
1201                ])),
1202            )
1203            .into(),
1204        ]));
1205
1206        check_type_serde(record, struct_type)
1207    }
1208
1209    #[test]
1210    fn list() {
1211        let record = r#"
1212        {
1213            "type": "list",
1214            "element-id": 3,
1215            "element-required": true,
1216            "element": "string"
1217        }
1218        "#;
1219
1220        check_type_serde(
1221            record,
1222            Type::List(ListType {
1223                element_field: NestedField::list_element(
1224                    3,
1225                    Type::Primitive(PrimitiveType::String),
1226                    true,
1227                )
1228                .into(),
1229            }),
1230        );
1231    }
1232
1233    #[test]
1234    fn map() {
1235        let record = r#"
1236        {
1237            "type": "map",
1238            "key-id": 4,
1239            "key": "string",
1240            "value-id": 5,
1241            "value-required": false,
1242            "value": "double"
1243        }
1244        "#;
1245
1246        check_type_serde(
1247            record,
1248            Type::Map(MapType {
1249                key_field: NestedField::map_key_element(4, Type::Primitive(PrimitiveType::String))
1250                    .into(),
1251                value_field: NestedField::map_value_element(
1252                    5,
1253                    Type::Primitive(PrimitiveType::Double),
1254                    false,
1255                )
1256                .into(),
1257            }),
1258        );
1259
1260        check_type_serde(
1261            record,
1262            Type::Map(MapType::optional(
1263                4,
1264                Type::Primitive(PrimitiveType::String),
1265                5,
1266                Type::Primitive(PrimitiveType::Double),
1267            )),
1268        );
1269    }
1270
1271    #[test]
1272    fn map_int() {
1273        let record = r#"
1274        {
1275            "type": "map",
1276            "key-id": 4,
1277            "key": "int",
1278            "value-id": 5,
1279            "value-required": false,
1280            "value": "string"
1281        }
1282        "#;
1283
1284        check_type_serde(
1285            record,
1286            Type::Map(MapType {
1287                key_field: NestedField::map_key_element(4, Type::Primitive(PrimitiveType::Int))
1288                    .into(),
1289                value_field: NestedField::map_value_element(
1290                    5,
1291                    Type::Primitive(PrimitiveType::String),
1292                    false,
1293                )
1294                .into(),
1295            }),
1296        );
1297
1298        check_type_serde(
1299            record,
1300            Type::Map(MapType::optional(
1301                4,
1302                Type::Primitive(PrimitiveType::Int),
1303                5,
1304                Type::Primitive(PrimitiveType::String),
1305            )),
1306        );
1307    }
1308
1309    #[test]
1310    fn map_required_int() {
1311        let record = r#"
1312        {
1313            "type": "map",
1314            "key-id": 4,
1315            "key": "int",
1316            "value-id": 5,
1317            "value-required": true,
1318            "value": "string"
1319        }
1320        "#;
1321
1322        check_type_serde(
1323            record,
1324            Type::Map(MapType::required(
1325                4,
1326                Type::Primitive(PrimitiveType::Int),
1327                5,
1328                Type::Primitive(PrimitiveType::String),
1329            )),
1330        );
1331    }
1332
1333    #[test]
1334    fn test_decimal_precision() {
1335        let expected_max_precision = [
1336            2, 4, 6, 9, 11, 14, 16, 18, 21, 23, 26, 28, 31, 33, 35, 38, 40, 43, 45, 47, 50, 52, 55,
1337            57,
1338        ];
1339        for (i, max_precision) in expected_max_precision.iter().enumerate() {
1340            assert_eq!(
1341                *max_precision,
1342                Type::decimal_max_precision(i as u32 + 1).unwrap(),
1343                "Failed calculate max precision for {i}"
1344            );
1345        }
1346
1347        assert_eq!(5, Type::decimal_required_bytes(10).unwrap());
1348        assert_eq!(16, Type::decimal_required_bytes(38).unwrap());
1349    }
1350
1351    #[test]
1352    fn test_reject_malformed_decimal_and_fixed_type_strings() {
1353        for invalid in [
1354            r#""decimal(50, 2)""#,
1355            r#""decimal(0, 0)""#,
1356            r#""decimal(decimal(5, 2)))""#,
1357            r#""decimal(decimal(5, 2)""#,
1358            r#""decimal(5, 2""#,
1359            r#""decimal(5, 2)))))""#,
1360            r#""decimal(5, 2, 3)""#,
1361            r#""decimal(-5, 2)""#,
1362            r#""decimal(5, -2)""#,
1363            r#""decimal(-5, -2)""#,
1364            r#""decimal(-2, -5)""#,
1365            r#""decimal((5, 2))""#,
1366            r#""decimal[5, 2]""#,
1367            r#""decimal()""#,
1368            r#""fixed[fixed[16]]]""#,
1369            r#""fixed[16""#,
1370            r#""fixed[16]]]""#,
1371            r#""fixed[[16]]""#,
1372            r#""fixed[[16]""#,
1373            r#""fixed(16)""#,
1374            r#""fixed[]""#,
1375        ] {
1376            assert!(
1377                serde_json::from_str::<Type>(invalid).is_err(),
1378                "expected {invalid} to be rejected"
1379            );
1380        }
1381    }
1382
1383    #[test]
1384    fn test_reject_leading_plus_in_type_strings() {
1385        for invalid in [
1386            r#""decimal(+5, 2)""#,
1387            r#""decimal(5, +2)""#,
1388            r#""decimal(+5, +2)""#,
1389            r#""fixed[+16]""#,
1390        ] {
1391            assert!(serde_json::from_str::<Type>(invalid).is_err(), "{invalid}");
1392            assert!(
1393                serde_json::from_str::<PrimitiveType>(invalid).is_err(),
1394                "{invalid}"
1395            );
1396        }
1397    }
1398
1399    #[test]
1400    fn test_decimal_constructor_json_roundtrip() {
1401        for (precision, scale) in [(1, 0), (5, 5), (5, 8), (38, 38)] {
1402            let decimal = Type::decimal(precision, scale).unwrap();
1403            let serialized = serde_json::to_string(&decimal).unwrap();
1404            let reparsed: Type = serde_json::from_str(&serialized).unwrap();
1405            assert_eq!(reparsed, decimal);
1406            let primitive: PrimitiveType = serde_json::from_str(&serialized).unwrap();
1407            assert_eq!(Type::Primitive(primitive), decimal);
1408        }
1409        for precision in [0, 39] {
1410            assert!(Type::decimal(precision, 0).is_err());
1411            let json = format!(r#""decimal({precision}, 0)""#);
1412            assert!(serde_json::from_str::<Type>(&json).is_err());
1413            assert!(serde_json::from_str::<PrimitiveType>(&json).is_err());
1414        }
1415    }
1416
1417    #[test]
1418    fn test_empty_decimal_and_fixed_tokens() {
1419        for invalid in [
1420            r#""decimal(, 2)""#,
1421            r#""decimal(5,)""#,
1422            r#""decimal( , )""#,
1423            r#""fixed[]""#,
1424            r#""fixed[ ]""#,
1425        ] {
1426            assert!(serde_json::from_str::<Type>(invalid).is_err(), "{invalid}");
1427            assert!(
1428                serde_json::from_str::<PrimitiveType>(invalid).is_err(),
1429                "{invalid}"
1430            );
1431        }
1432    }
1433
1434    #[test]
1435    fn test_decimal_and_fixed_overflow_errors() {
1436        for invalid in [
1437            r#""decimal(9999999999, 2)""#,
1438            r#""decimal(5, 9999999999)""#,
1439            r#""fixed[18446744073709551616]""#,
1440        ] {
1441            assert!(serde_json::from_str::<Type>(invalid).is_err(), "{invalid}");
1442            let error = serde_json::from_str::<PrimitiveType>(invalid).unwrap_err();
1443            assert!(
1444                error
1445                    .to_string()
1446                    .contains("number too large to fit in target type"),
1447                "unexpected error for {invalid}: {error}"
1448            );
1449        }
1450    }
1451
1452    #[test]
1453    fn test_decimal_zero_precision_error() {
1454        let error = serde_json::from_str::<PrimitiveType>(r#""decimal(0, 0)""#).unwrap_err();
1455        assert!(
1456            error
1457                .to_string()
1458                .contains("Decimal precision must be greater than zero"),
1459            "unexpected error: {error}"
1460        );
1461    }
1462
1463    #[test]
1464    fn test_accept_valid_decimal_and_fixed_type_strings() {
1465        for (json, canonical, expected) in [
1466            (
1467                r#""decimal(9, 2)""#,
1468                r#""decimal(9, 2)""#,
1469                Type::Primitive(PrimitiveType::Decimal {
1470                    precision: 9,
1471                    scale: 2,
1472                }),
1473            ),
1474            (
1475                r#""decimal(38, 10)""#,
1476                r#""decimal(38, 10)""#,
1477                Type::Primitive(PrimitiveType::Decimal {
1478                    precision: 38,
1479                    scale: 10,
1480                }),
1481            ),
1482            (
1483                r#""decimal(5,2)""#,
1484                r#""decimal(5, 2)""#,
1485                Type::Primitive(PrimitiveType::Decimal {
1486                    precision: 5,
1487                    scale: 2,
1488                }),
1489            ),
1490            (
1491                r#""decimal(5, 0)""#,
1492                r#""decimal(5, 0)""#,
1493                Type::Primitive(PrimitiveType::Decimal {
1494                    precision: 5,
1495                    scale: 0,
1496                }),
1497            ),
1498            (
1499                r#""decimal(5, 5)""#,
1500                r#""decimal(5, 5)""#,
1501                Type::Primitive(PrimitiveType::Decimal {
1502                    precision: 5,
1503                    scale: 5,
1504                }),
1505            ),
1506            (
1507                r#""fixed[16]""#,
1508                r#""fixed[16]""#,
1509                Type::Primitive(PrimitiveType::Fixed(16)),
1510            ),
1511        ] {
1512            check_type_serde_roundtrip_value(json, canonical, expected);
1513        }
1514    }
1515
1516    fn check_type_serde_roundtrip_value(json: &str, canonical: &str, expected_type: Type) {
1517        let parsed: Type = serde_json::from_str(json).unwrap();
1518        assert_eq!(parsed, expected_type);
1519        let serialized = serde_json::to_string(&parsed).unwrap();
1520        assert_eq!(serialized, canonical);
1521        let primitive: PrimitiveType = serde_json::from_str(json).unwrap();
1522        assert_eq!(serde_json::to_string(&primitive).unwrap(), canonical);
1523        let reparsed: Type = serde_json::from_str(&serialized).unwrap();
1524        assert_eq!(reparsed, expected_type);
1525    }
1526
1527    #[test]
1528    fn test_primitive_type_compatible() {
1529        let pairs = vec![
1530            (PrimitiveType::Boolean, PrimitiveLiteral::Boolean(true)),
1531            (PrimitiveType::Int, PrimitiveLiteral::Int(1)),
1532            (PrimitiveType::Long, PrimitiveLiteral::Long(1)),
1533            (PrimitiveType::Float, PrimitiveLiteral::Float(1.0.into())),
1534            (PrimitiveType::Double, PrimitiveLiteral::Double(1.0.into())),
1535            (
1536                PrimitiveType::Decimal {
1537                    precision: 9,
1538                    scale: 2,
1539                },
1540                PrimitiveLiteral::Int128(1),
1541            ),
1542            (PrimitiveType::Date, PrimitiveLiteral::Int(1)),
1543            (PrimitiveType::Time, PrimitiveLiteral::Long(1)),
1544            (PrimitiveType::Timestamptz, PrimitiveLiteral::Long(1)),
1545            (PrimitiveType::Timestamp, PrimitiveLiteral::Long(1)),
1546            (PrimitiveType::TimestamptzNs, PrimitiveLiteral::Long(1)),
1547            (PrimitiveType::TimestampNs, PrimitiveLiteral::Long(1)),
1548            (
1549                PrimitiveType::Uuid,
1550                PrimitiveLiteral::UInt128(Uuid::new_v4().as_u128()),
1551            ),
1552            (PrimitiveType::Fixed(8), PrimitiveLiteral::Binary(vec![1])),
1553            (PrimitiveType::Binary, PrimitiveLiteral::Binary(vec![1])),
1554        ];
1555        for (ty, literal) in pairs {
1556            assert!(ty.compatible(&literal));
1557        }
1558
1559        assert!(!PrimitiveType::Unknown.compatible(&PrimitiveLiteral::Int(1)));
1560    }
1561
1562    #[test]
1563    fn variant_type_serde() {
1564        let json = r#"{"id": 1, "name": "v", "required": true, "type": "variant"}"#;
1565        let field: NestedField = serde_json::from_str(json).unwrap();
1566        assert_eq!(*field.field_type, Type::Variant(VariantType));
1567
1568        let serialized = serde_json::to_string(&field).unwrap();
1569        let roundtrip: NestedField = serde_json::from_str(&serialized).unwrap();
1570        assert_eq!(field, roundtrip);
1571    }
1572
1573    #[test]
1574    fn nested_field_rejects_invalid_map_defaults() {
1575        for default_name in ["initial-default", "write-default"] {
1576            let json = format!(
1577                r#"{{
1578                    "id": 1,
1579                    "name": "properties",
1580                    "required": false,
1581                    "type": {{
1582                        "type": "map",
1583                        "key-id": 2,
1584                        "key": "string",
1585                        "value-id": 3,
1586                        "value-required": false,
1587                        "value": "int"
1588                    }},
1589                    "{default_name}": {{"keys": ["a", "b"], "values": [1]}}
1590                }}"#
1591            );
1592
1593            let error = serde_json::from_str::<NestedField>(&json).unwrap_err();
1594            assert!(error.to_string().contains("must have the same length"));
1595        }
1596    }
1597
1598    #[test]
1599    fn struct_type_with_type_field() {
1600        // Test that StructType properly deserializes JSON with "type":"struct" field
1601        // This was previously broken because the deserializer wasn't consuming the type field value
1602        let json = r#"
1603        {
1604            "type": "struct",
1605            "fields": [
1606                {"id": 1, "name": "field1", "required": true, "type": "string"}
1607            ]
1608        }
1609        "#;
1610
1611        let struct_type: StructType = serde_json::from_str(json)
1612            .expect("Should successfully deserialize StructType with type field");
1613
1614        assert_eq!(struct_type.fields().len(), 1);
1615        assert_eq!(struct_type.fields()[0].name, "field1");
1616    }
1617
1618    #[test]
1619    fn struct_type_rejects_wrong_type() {
1620        // Test that StructType validation rejects incorrect type field values
1621        let json = r#"
1622        {
1623            "type": "list",
1624            "fields": [
1625                {"id": 1, "name": "field1", "required": true, "type": "string"}
1626            ]
1627        }
1628        "#;
1629
1630        let result = serde_json::from_str::<StructType>(json);
1631        assert!(
1632            result.is_err(),
1633            "Should reject StructType with wrong type field"
1634        );
1635        assert!(
1636            result
1637                .unwrap_err()
1638                .to_string()
1639                .contains("expected type 'struct'")
1640        );
1641    }
1642}
iceberg/spec/datatypes.rs

iceberg/spec/
datatypes.rs
