iceberg/spec/
datatypes.rs

1// Licensed to the Apache Software Foundation (ASF) under one
2// or more contributor license agreements.  See the NOTICE file
3// distributed with this work for additional information
4// regarding copyright ownership.  The ASF licenses this file
5// to you under the Apache License, Version 2.0 (the
6// "License"); you may not use this file except in compliance
7// with the License.  You may obtain a copy of the License at
8//
9//   http://www.apache.org/licenses/LICENSE-2.0
10//
11// Unless required by applicable law or agreed to in writing,
12// software distributed under the License is distributed on an
13// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14// KIND, either express or implied.  See the License for the
15// specific language governing permissions and limitations
16// under the License.
17
18/*!
19 * Data Types
20 */
21use std::collections::HashMap;
22use std::fmt;
23use std::ops::Index;
24use std::sync::{Arc, OnceLock};
25
26use ::serde::de::{MapAccess, Visitor};
27use serde::de::{Error, IntoDeserializer};
28use serde::{Deserialize, Deserializer, Serialize, Serializer};
29use serde_json::Value as JsonValue;
30
31use super::geospatial::{GeographyType, GeometryType, parse_geography, parse_geometry};
32use super::values::Literal;
33use crate::ensure_data_valid;
34use crate::error::Result;
35use crate::spec::datatypes::_decimal::{MAX_PRECISION, REQUIRED_LENGTH};
36use crate::spec::{FormatVersion, PrimitiveLiteral};
37
38/// Field name for list type.
39pub const LIST_FIELD_NAME: &str = "element";
40/// Field name for map type's key.
41pub const MAP_KEY_FIELD_NAME: &str = "key";
42/// Field name for map type's value.
43pub const MAP_VALUE_FIELD_NAME: &str = "value";
44
45pub(crate) const MAX_DECIMAL_BYTES: u32 = 24;
46pub(crate) const MAX_DECIMAL_PRECISION: u32 = 38;
47
48mod _decimal {
49    use once_cell::sync::Lazy;
50
51    use crate::spec::{MAX_DECIMAL_BYTES, MAX_DECIMAL_PRECISION};
52
53    // Max precision of bytes, starts from 1
54    pub(super) static MAX_PRECISION: Lazy<[u32; MAX_DECIMAL_BYTES as usize]> = Lazy::new(|| {
55        let mut ret: [u32; 24] = [0; 24];
56        for (i, prec) in ret.iter_mut().enumerate() {
57            *prec = 2f64.powi((8 * (i + 1) - 1) as i32).log10().floor() as u32;
58        }
59
60        ret
61    });
62
63    //  Required bytes of precision, starts from 1
64    pub(super) static REQUIRED_LENGTH: Lazy<[u32; MAX_DECIMAL_PRECISION as usize]> =
65        Lazy::new(|| {
66            let mut ret: [u32; MAX_DECIMAL_PRECISION as usize] =
67                [0; MAX_DECIMAL_PRECISION as usize];
68
69            for (i, required_len) in ret.iter_mut().enumerate() {
70                for j in 0..MAX_PRECISION.len() {
71                    if MAX_PRECISION[j] >= ((i + 1) as u32) {
72                        *required_len = (j + 1) as u32;
73                        break;
74                    }
75                }
76            }
77
78            ret
79        });
80}
81
82#[derive(Debug, PartialEq, Eq, Clone)]
83/// All data types are either primitives or nested types, which are maps, lists, or structs.
84pub enum Type {
85    /// Primitive types
86    Primitive(PrimitiveType),
87    /// Struct type
88    Struct(StructType),
89    /// List type.
90    List(ListType),
91    /// Map type
92    Map(MapType),
93    /// Variant Type
94    Variant(VariantType),
95}
96
97impl fmt::Display for Type {
98    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
99        match self {
100            Type::Primitive(primitive) => write!(f, "{primitive}"),
101            Type::Struct(s) => write!(f, "{s}"),
102            Type::List(_) => write!(f, "list"),
103            Type::Map(_) => write!(f, "map"),
104            Type::Variant(_) => write!(f, "variant"),
105        }
106    }
107}
108
109impl Type {
110    /// Whether the type is primitive type.
111    #[inline(always)]
112    pub fn is_primitive(&self) -> bool {
113        matches!(self, Type::Primitive(_))
114    }
115
116    /// Whether the type is struct type.
117    #[inline(always)]
118    pub fn is_struct(&self) -> bool {
119        matches!(self, Type::Struct(_))
120    }
121
122    /// Whether the type is nested type.
123    #[inline(always)]
124    pub fn is_nested(&self) -> bool {
125        matches!(self, Type::Struct(_) | Type::List(_) | Type::Map(_))
126    }
127
128    /// Whether the type is variant type.
129    #[inline(always)]
130    pub fn is_variant(&self) -> bool {
131        matches!(self, Type::Variant(_))
132    }
133
134    /// Minimum [`FormatVersion`] required to support this type, **without** taking
135    /// nested field types into account.
136    ///
137    /// `Unknown` / `TimestampNs` / `TimestamptzNs` / `Geometry` / `Geography` / `Variant` require
138    /// [`FormatVersion::V3`]; every other type is valid from [`FormatVersion::V1`]. Mirrors Java's
139    /// `Schema.MIN_FORMAT_VERSIONS` (a shallow lookup keyed by type id), so it
140    /// intentionally does not recurse: callers needing the floor for a whole schema
141    /// iterate its flattened fields (see [`Schema::calc_min_compatible_format`]).
142    ///
143    /// [`Schema::calc_min_compatible_format`]: crate::spec::Schema::calc_min_compatible_format
144    pub(crate) fn min_format_version(&self) -> FormatVersion {
145        match self {
146            Type::Primitive(
147                PrimitiveType::Unknown
148                | PrimitiveType::TimestampNs
149                | PrimitiveType::TimestamptzNs
150                | PrimitiveType::Geometry(_)
151                | PrimitiveType::Geography(_),
152            )
153            | Type::Variant(_) => FormatVersion::V3,
154            _ => FormatVersion::V1,
155        }
156    }
157
158    /// Convert Type to reference of PrimitiveType
159    pub fn as_primitive_type(&self) -> Option<&PrimitiveType> {
160        if let Type::Primitive(primitive_type) = self {
161            Some(primitive_type)
162        } else {
163            None
164        }
165    }
166
167    /// Convert Type to StructType
168    pub fn to_struct_type(self) -> Option<StructType> {
169        if let Type::Struct(struct_type) = self {
170            Some(struct_type)
171        } else {
172            None
173        }
174    }
175
176    /// Return max precision for decimal given `num_bytes` bytes.
177    #[inline(always)]
178    pub fn decimal_max_precision(num_bytes: u32) -> Result<u32> {
179        ensure_data_valid!(
180            num_bytes > 0 && num_bytes <= MAX_DECIMAL_BYTES,
181            "Decimal length larger than {MAX_DECIMAL_BYTES} is not supported: {num_bytes}",
182        );
183        Ok(MAX_PRECISION[num_bytes as usize - 1])
184    }
185
186    /// Returns minimum bytes required for decimal with `precision`.
187    #[inline(always)]
188    pub fn decimal_required_bytes(precision: u32) -> Result<u32> {
189        ensure_data_valid!(
190            precision > 0 && precision <= MAX_DECIMAL_PRECISION,
191            "Decimals with precision larger than {MAX_DECIMAL_PRECISION} are not supported: {precision}",
192        );
193        Ok(REQUIRED_LENGTH[precision as usize - 1])
194    }
195
196    /// Creates  decimal type.
197    #[inline(always)]
198    pub fn decimal(precision: u32, scale: u32) -> Result<Self> {
199        validate_decimal_precision(precision)?;
200        Ok(Type::Primitive(PrimitiveType::Decimal { precision, scale }))
201    }
202
203    /// Check if it's float or double type.
204    #[inline(always)]
205    pub fn is_floating_type(&self) -> bool {
206        matches!(
207            self,
208            Type::Primitive(PrimitiveType::Float) | Type::Primitive(PrimitiveType::Double)
209        )
210    }
211}
212
213impl From<PrimitiveType> for Type {
214    fn from(value: PrimitiveType) -> Self {
215        Self::Primitive(value)
216    }
217}
218
219impl From<StructType> for Type {
220    fn from(value: StructType) -> Self {
221        Type::Struct(value)
222    }
223}
224
225impl From<ListType> for Type {
226    fn from(value: ListType) -> Self {
227        Type::List(value)
228    }
229}
230
231impl From<MapType> for Type {
232    fn from(value: MapType) -> Self {
233        Type::Map(value)
234    }
235}
236
237/// Primitive data types
238#[derive(Debug, Serialize, Deserialize, PartialEq, Eq, Clone, Hash)]
239#[serde(rename_all = "lowercase", remote = "Self")]
240pub enum PrimitiveType {
241    /// True or False
242    Boolean,
243    /// 32-bit signed integer
244    Int,
245    /// 64-bit signed integer
246    Long,
247    /// 32-bit IEEE 754 floating point.
248    Float,
249    /// 64-bit IEEE 754 floating point.
250    Double,
251    /// Fixed point decimal
252    Decimal {
253        /// Precision, must be 38 or less
254        precision: u32,
255        /// Scale
256        scale: u32,
257    },
258    /// Calendar date without timezone or time.
259    Date,
260    /// Time of day in microsecond precision, without date or timezone.
261    Time,
262    /// Timestamp in microsecond precision, without timezone
263    Timestamp,
264    /// Timestamp in microsecond precision, with timezone
265    Timestamptz,
266    /// Timestamp in nanosecond precision, without timezone
267    #[serde(rename = "timestamp_ns")]
268    TimestampNs,
269    /// Timestamp in nanosecond precision with timezone
270    #[serde(rename = "timestamptz_ns")]
271    TimestamptzNs,
272    /// Arbitrary-length character sequences encoded in utf-8
273    String,
274    /// Universally Unique Identifiers, should use 16-byte fixed
275    Uuid,
276    /// Fixed length byte array
277    Fixed(u64),
278    /// Arbitrary-length byte array.
279    Binary,
280    /// Default / null column type used when a more specific type is not known.
281    Unknown,
282    /// Geometry values encoded as well-known binary.
283    Geometry(GeometryType),
284    /// Geography values encoded as well-known binary.
285    Geography(GeographyType),
286}
287
288impl PrimitiveType {
289    /// Check whether literal is compatible with the type.
290    pub fn compatible(&self, literal: &PrimitiveLiteral) -> bool {
291        matches!(
292            (self, literal),
293            (PrimitiveType::Boolean, PrimitiveLiteral::Boolean(_))
294                | (PrimitiveType::Int, PrimitiveLiteral::Int(_))
295                | (PrimitiveType::Long, PrimitiveLiteral::Long(_))
296                | (PrimitiveType::Float, PrimitiveLiteral::Float(_))
297                | (PrimitiveType::Double, PrimitiveLiteral::Double(_))
298                | (PrimitiveType::Decimal { .. }, PrimitiveLiteral::Int128(_))
299                | (PrimitiveType::Date, PrimitiveLiteral::Int(_))
300                | (PrimitiveType::Time, PrimitiveLiteral::Long(_))
301                | (PrimitiveType::Timestamp, PrimitiveLiteral::Long(_))
302                | (PrimitiveType::Timestamptz, PrimitiveLiteral::Long(_))
303                | (PrimitiveType::TimestampNs, PrimitiveLiteral::Long(_))
304                | (PrimitiveType::TimestamptzNs, PrimitiveLiteral::Long(_))
305                | (PrimitiveType::String, PrimitiveLiteral::String(_))
306                | (PrimitiveType::Uuid, PrimitiveLiteral::UInt128(_))
307                | (PrimitiveType::Fixed(_), PrimitiveLiteral::Binary(_))
308                | (PrimitiveType::Binary, PrimitiveLiteral::Binary(_))
309                | (PrimitiveType::Geometry(_), PrimitiveLiteral::Binary(_))
310                | (PrimitiveType::Geography(_), PrimitiveLiteral::Binary(_))
311        )
312    }
313}
314
315impl Serialize for Type {
316    fn serialize<S>(&self, serializer: S) -> std::result::Result<S::Ok, S::Error>
317    where S: Serializer {
318        let type_serde = _serde::SerdeType::from(self);
319        type_serde.serialize(serializer)
320    }
321}
322
323impl<'de> Deserialize<'de> for Type {
324    fn deserialize<D>(deserializer: D) -> std::result::Result<Self, D::Error>
325    where D: Deserializer<'de> {
326        let type_serde = _serde::SerdeType::deserialize(deserializer)?;
327        Ok(Type::from(type_serde))
328    }
329}
330
331impl<'de> Deserialize<'de> for PrimitiveType {
332    fn deserialize<D>(deserializer: D) -> std::result::Result<Self, D::Error>
333    where D: Deserializer<'de> {
334        let s = String::deserialize(deserializer)?;
335        if s.starts_with("decimal") {
336            deserialize_decimal(s.into_deserializer())
337        } else if s.starts_with("fixed") {
338            deserialize_fixed(s.into_deserializer())
339        } else if s
340            .get(..8)
341            .is_some_and(|prefix| prefix.eq_ignore_ascii_case("geometry"))
342        {
343            parse_geometry(&s).map_err(D::Error::custom)
344        } else if s
345            .get(..9)
346            .is_some_and(|prefix| prefix.eq_ignore_ascii_case("geography"))
347        {
348            parse_geography(&s).map_err(D::Error::custom)
349        } else {
350            PrimitiveType::deserialize(s.into_deserializer())
351        }
352    }
353}
354
355impl Serialize for PrimitiveType {
356    fn serialize<S>(&self, serializer: S) -> std::result::Result<S::Ok, S::Error>
357    where S: Serializer {
358        match self {
359            PrimitiveType::Decimal { precision, scale } => {
360                serialize_decimal(precision, scale, serializer)
361            }
362            PrimitiveType::Fixed(l) => serialize_fixed(l, serializer),
363            PrimitiveType::Geometry(_) | PrimitiveType::Geography(_) => {
364                serializer.serialize_str(&self.to_string())
365            }
366            _ => PrimitiveType::serialize(self, serializer),
367        }
368    }
369}
370
371fn validate_decimal_precision(precision: u32) -> Result<()> {
372    // Leave scale unconstrained for Java parity; the spec leaves scale > precision open.
373    ensure_data_valid!(precision > 0, "Decimal precision must be greater than zero",);
374    ensure_data_valid!(
375        precision <= MAX_DECIMAL_PRECISION,
376        "Decimals with precision larger than {MAX_DECIMAL_PRECISION} are not supported: {precision}",
377    );
378    Ok(())
379}
380
381fn deserialize_decimal<'de, D>(deserializer: D) -> std::result::Result<PrimitiveType, D::Error>
382where D: Deserializer<'de> {
383    let s = String::deserialize(deserializer)?;
384    let malformed = || D::Error::custom(format!("Invalid decimal type: {s}"));
385
386    let (precision, scale) = s
387        .strip_prefix("decimal(")
388        .and_then(|inner| inner.strip_suffix(')'))
389        .ok_or_else(malformed)?
390        .split_once(',')
391        .ok_or_else(|| D::Error::custom(format!("Decimal requires precision and scale: {s}")))?;
392
393    let (precision, scale) = (precision.trim(), scale.trim());
394    if [precision, scale]
395        .iter()
396        .any(|token| token.is_empty() || !token.bytes().all(|byte| byte.is_ascii_digit()))
397    {
398        return Err(malformed());
399    }
400    let precision: u32 = precision.parse().map_err(D::Error::custom)?;
401    let scale: u32 = scale.parse().map_err(D::Error::custom)?;
402    validate_decimal_precision(precision).map_err(D::Error::custom)?;
403
404    Ok(PrimitiveType::Decimal { precision, scale })
405}
406
407fn serialize_decimal<S>(
408    precision: &u32,
409    scale: &u32,
410    serializer: S,
411) -> std::result::Result<S::Ok, S::Error>
412where
413    S: Serializer,
414{
415    serializer.serialize_str(&format!("decimal({precision}, {scale})"))
416}
417
418fn deserialize_fixed<'de, D>(deserializer: D) -> std::result::Result<PrimitiveType, D::Error>
419where D: Deserializer<'de> {
420    let s = String::deserialize(deserializer)?;
421
422    let length = s
423        .strip_prefix("fixed[")
424        .and_then(|inner| inner.strip_suffix(']'))
425        .map(str::trim)
426        .filter(|length| !length.is_empty() && length.bytes().all(|byte| byte.is_ascii_digit()))
427        .ok_or_else(|| D::Error::custom(format!("Invalid fixed type: {s}")))?;
428
429    length
430        .parse()
431        .map(PrimitiveType::Fixed)
432        .map_err(D::Error::custom)
433}
434
435fn serialize_fixed<S>(value: &u64, serializer: S) -> std::result::Result<S::Ok, S::Error>
436where S: Serializer {
437    serializer.serialize_str(&format!("fixed[{value}]"))
438}
439
440impl fmt::Display for PrimitiveType {
441    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
442        match self {
443            PrimitiveType::Unknown => write!(f, "unknown"),
444            PrimitiveType::Boolean => write!(f, "boolean"),
445            PrimitiveType::Int => write!(f, "int"),
446            PrimitiveType::Long => write!(f, "long"),
447            PrimitiveType::Float => write!(f, "float"),
448            PrimitiveType::Double => write!(f, "double"),
449            PrimitiveType::Decimal { precision, scale } => {
450                write!(f, "decimal({precision}, {scale})")
451            }
452            PrimitiveType::Date => write!(f, "date"),
453            PrimitiveType::Time => write!(f, "time"),
454            PrimitiveType::Timestamp => write!(f, "timestamp"),
455            PrimitiveType::Timestamptz => write!(f, "timestamptz"),
456            PrimitiveType::TimestampNs => write!(f, "timestamp_ns"),
457            PrimitiveType::TimestamptzNs => write!(f, "timestamptz_ns"),
458            PrimitiveType::String => write!(f, "string"),
459            PrimitiveType::Uuid => write!(f, "uuid"),
460            PrimitiveType::Fixed(size) => write!(f, "fixed({size})"),
461            PrimitiveType::Binary => write!(f, "binary"),
462            PrimitiveType::Geometry(geometry) => write!(f, "geometry({})", geometry.crs()),
463            PrimitiveType::Geography(geography) => write!(
464                f,
465                "geography({}, {})",
466                geography.crs(),
467                geography.algorithm().as_str()
468            ),
469        }
470    }
471}
472
473/// DataType for a specific struct
474#[derive(Debug, Serialize, Clone, Default)]
475#[serde(rename = "struct", tag = "type")]
476pub struct StructType {
477    /// Struct fields
478    fields: Vec<NestedFieldRef>,
479    /// Lookup for index by field id
480    #[serde(skip_serializing)]
481    id_lookup: OnceLock<HashMap<i32, usize>>,
482    #[serde(skip_serializing)]
483    name_lookup: OnceLock<HashMap<String, usize>>,
484}
485
486impl<'de> Deserialize<'de> for StructType {
487    fn deserialize<D>(deserializer: D) -> std::result::Result<Self, D::Error>
488    where D: Deserializer<'de> {
489        #[derive(Deserialize)]
490        #[serde(field_identifier, rename_all = "lowercase")]
491        enum Field {
492            Type,
493            Fields,
494        }
495
496        struct StructTypeVisitor;
497
498        impl<'de> Visitor<'de> for StructTypeVisitor {
499            type Value = StructType;
500
501            fn expecting(&self, formatter: &mut fmt::Formatter) -> fmt::Result {
502                formatter.write_str("struct")
503            }
504
505            fn visit_map<V>(self, mut map: V) -> std::result::Result<StructType, V::Error>
506            where V: MapAccess<'de> {
507                let mut fields = None;
508                while let Some(key) = map.next_key()? {
509                    match key {
510                        Field::Type => {
511                            let type_val: String = map.next_value()?;
512                            if type_val != "struct" {
513                                return Err(Error::custom(format!(
514                                    "expected type 'struct', got '{type_val}'"
515                                )));
516                            }
517                        }
518                        Field::Fields => {
519                            if fields.is_some() {
520                                return Err(Error::duplicate_field("fields"));
521                            }
522                            fields = Some(map.next_value()?);
523                        }
524                    }
525                }
526                let fields: Vec<NestedFieldRef> =
527                    fields.ok_or_else(|| Error::missing_field("fields"))?;
528
529                Ok(StructType::new(fields))
530            }
531        }
532
533        const FIELDS: &[&str] = &["type", "fields"];
534        deserializer.deserialize_struct("struct", FIELDS, StructTypeVisitor)
535    }
536}
537
538impl StructType {
539    /// Creates a struct type with the given fields.
540    pub fn new(fields: Vec<NestedFieldRef>) -> Self {
541        Self {
542            fields,
543            id_lookup: OnceLock::new(),
544            name_lookup: OnceLock::new(),
545        }
546    }
547
548    /// Get struct field with certain id
549    pub fn field_by_id(&self, id: i32) -> Option<&NestedFieldRef> {
550        self.field_id_to_index(id).map(|idx| &self.fields[idx])
551    }
552
553    fn field_id_to_index(&self, field_id: i32) -> Option<usize> {
554        self.id_lookup
555            .get_or_init(|| {
556                HashMap::from_iter(self.fields.iter().enumerate().map(|(i, x)| (x.id, i)))
557            })
558            .get(&field_id)
559            .copied()
560    }
561
562    /// Get struct field with certain field name
563    pub fn field_by_name(&self, name: &str) -> Option<&NestedFieldRef> {
564        self.field_name_to_index(name).map(|idx| &self.fields[idx])
565    }
566
567    fn field_name_to_index(&self, name: &str) -> Option<usize> {
568        self.name_lookup
569            .get_or_init(|| {
570                HashMap::from_iter(
571                    self.fields
572                        .iter()
573                        .enumerate()
574                        .map(|(i, x)| (x.name.clone(), i)),
575                )
576            })
577            .get(name)
578            .copied()
579    }
580
581    /// Get fields.
582    pub fn fields(&self) -> &[NestedFieldRef] {
583        &self.fields
584    }
585}
586
587impl PartialEq for StructType {
588    fn eq(&self, other: &Self) -> bool {
589        self.fields == other.fields
590    }
591}
592
593impl Eq for StructType {}
594
595impl Index<usize> for StructType {
596    type Output = NestedField;
597
598    fn index(&self, index: usize) -> &Self::Output {
599        &self.fields[index]
600    }
601}
602
603impl fmt::Display for StructType {
604    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
605        write!(f, "struct<")?;
606        for field in &self.fields {
607            write!(f, "{}", field.field_type)?;
608        }
609        write!(f, ">")
610    }
611}
612
613#[derive(Debug, PartialEq, Serialize, Deserialize, Eq, Clone)]
614#[serde(try_from = "SerdeNestedField", into = "SerdeNestedField")]
615/// A struct is a tuple of typed values. Each field in the tuple is named and has an integer id that is unique in the table schema.
616/// Each field can be either optional or required, meaning that values can (or cannot) be null. Fields may be any type.
617/// Fields may have an optional comment or doc string. Fields can have default values.
618pub struct NestedField {
619    /// Id unique in table schema
620    pub id: i32,
621    /// Field Name
622    pub name: String,
623    /// Optional or required
624    pub required: bool,
625    /// Datatype
626    pub field_type: Box<Type>,
627    /// Fields may have an optional comment or doc string.
628    pub doc: Option<String>,
629    /// Used to populate the field’s value for all records that were written before the field was added to the schema
630    pub initial_default: Option<Literal>,
631    /// Used to populate the field’s value for any records written after the field was added to the schema, if the writer does not supply the field’s value
632    pub write_default: Option<Literal>,
633}
634
635#[derive(Debug, Serialize, Deserialize, PartialEq, Eq, Clone)]
636#[serde(rename_all = "kebab-case")]
637struct SerdeNestedField {
638    pub id: i32,
639    pub name: String,
640    pub required: bool,
641    #[serde(rename = "type")]
642    pub field_type: Box<Type>,
643    #[serde(skip_serializing_if = "Option::is_none")]
644    pub doc: Option<String>,
645    #[serde(skip_serializing_if = "Option::is_none")]
646    pub initial_default: Option<JsonValue>,
647    #[serde(skip_serializing_if = "Option::is_none")]
648    pub write_default: Option<JsonValue>,
649}
650
651impl TryFrom<SerdeNestedField> for NestedField {
652    type Error = crate::Error;
653
654    fn try_from(value: SerdeNestedField) -> Result<Self> {
655        let initial_default = value
656            .initial_default
657            .map(|default| Literal::try_from_json(default, &value.field_type))
658            .transpose()?
659            .flatten();
660        let write_default = value
661            .write_default
662            .map(|default| Literal::try_from_json(default, &value.field_type))
663            .transpose()?
664            .flatten();
665
666        Ok(NestedField {
667            id: value.id,
668            name: value.name,
669            required: value.required,
670            initial_default,
671            write_default,
672            field_type: value.field_type,
673            doc: value.doc,
674        })
675    }
676}
677
678impl From<NestedField> for SerdeNestedField {
679    fn from(value: NestedField) -> Self {
680        let initial_default = value.initial_default.map(|x| x.try_into_json(&value.field_type).expect("We should have checked this in NestedField::with_initial_default, it can't be converted to json value"));
681        let write_default = value.write_default.map(|x| x.try_into_json(&value.field_type).expect("We should have checked this in NestedField::with_write_default, it can't be converted to json value"));
682        SerdeNestedField {
683            id: value.id,
684            name: value.name,
685            required: value.required,
686            field_type: value.field_type,
687            doc: value.doc,
688            initial_default,
689            write_default,
690        }
691    }
692}
693
694/// Reference to nested field.
695pub type NestedFieldRef = Arc<NestedField>;
696
697impl NestedField {
698    /// Construct a new field.
699    pub fn new(id: i32, name: impl ToString, field_type: impl Into<Type>, required: bool) -> Self {
700        Self {
701            id,
702            name: name.to_string(),
703            required,
704            field_type: Box::new(field_type.into()),
705            doc: None,
706            initial_default: None,
707            write_default: None,
708        }
709    }
710
711    /// Construct a required field.
712    pub fn required(id: i32, name: impl ToString, field_type: impl Into<Type>) -> Self {
713        Self::new(id, name, field_type, true)
714    }
715
716    /// Construct an optional field.
717    pub fn optional(id: i32, name: impl ToString, field_type: impl Into<Type>) -> Self {
718        Self::new(id, name, field_type, false)
719    }
720
721    /// Construct list type's element field.
722    pub fn list_element(id: i32, field_type: impl Into<Type>, required: bool) -> Self {
723        Self::new(id, LIST_FIELD_NAME, field_type, required)
724    }
725
726    /// Construct map type's key field.
727    pub fn map_key_element(id: i32, field_type: impl Into<Type>) -> Self {
728        Self::required(id, MAP_KEY_FIELD_NAME, field_type)
729    }
730
731    /// Construct map type's value field.
732    pub fn map_value_element(id: i32, field_type: impl Into<Type>, required: bool) -> Self {
733        Self::new(id, MAP_VALUE_FIELD_NAME, field_type, required)
734    }
735
736    /// Set the field's doc.
737    pub fn with_doc(mut self, doc: impl ToString) -> Self {
738        self.doc = Some(doc.to_string());
739        self
740    }
741
742    /// Set the field's initial default value.
743    pub fn with_initial_default(mut self, value: Literal) -> Self {
744        self.initial_default = Some(value);
745        self
746    }
747
748    /// Set the field's initial default value.
749    pub fn with_write_default(mut self, value: Literal) -> Self {
750        self.write_default = Some(value);
751        self
752    }
753
754    /// Set the id of the field.
755    pub(crate) fn with_id(mut self, id: i32) -> Self {
756        self.id = id;
757        self
758    }
759}
760
761impl fmt::Display for NestedField {
762    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
763        write!(f, "{}: ", self.id)?;
764        write!(f, "{}: ", self.name)?;
765        if self.required {
766            write!(f, "required ")?;
767        } else {
768            write!(f, "optional ")?;
769        }
770        write!(f, "{} ", self.field_type)?;
771        if let Some(doc) = &self.doc {
772            write!(f, "{doc}")?;
773        }
774        Ok(())
775    }
776}
777
778#[derive(Debug, PartialEq, Eq, Clone)]
779/// A list is a collection of values with some element type. The element field has an integer id that is unique in the table schema.
780/// Elements can be either optional or required. Element types may be any type.
781pub struct ListType {
782    /// Element field of list type.
783    pub element_field: NestedFieldRef,
784}
785
786impl ListType {
787    /// Construct a list type with the given element field.
788    pub fn new(element_field: NestedFieldRef) -> Self {
789        Self { element_field }
790    }
791}
792
793/// Module for type serialization/deserialization.
794pub(super) mod _serde {
795    use std::borrow::Cow;
796
797    use serde_derive::{Deserialize, Serialize};
798
799    use crate::spec::datatypes::Type::Map;
800    use crate::spec::datatypes::{
801        ListType, MapType, NestedField, NestedFieldRef, PrimitiveType, StructType, Type,
802        VariantType,
803    };
804
805    /// List type for serialization and deserialization
806    #[derive(Serialize, Deserialize)]
807    #[serde(untagged)]
808    pub(super) enum SerdeType<'a> {
809        #[serde(rename_all = "kebab-case")]
810        List {
811            r#type: String,
812            element_id: i32,
813            element_required: bool,
814            element: Cow<'a, Type>,
815        },
816        Struct {
817            r#type: String,
818            fields: Cow<'a, [NestedFieldRef]>,
819        },
820        #[serde(rename_all = "kebab-case")]
821        Map {
822            r#type: String,
823            key_id: i32,
824            key: Cow<'a, Type>,
825            value_id: i32,
826            value_required: bool,
827            value: Cow<'a, Type>,
828        },
829        Primitive(PrimitiveType),
830        Variant(VariantType),
831    }
832
833    impl From<SerdeType<'_>> for Type {
834        fn from(value: SerdeType) -> Self {
835            match value {
836                SerdeType::List {
837                    r#type: _,
838                    element_id,
839                    element_required,
840                    element,
841                } => Self::List(ListType {
842                    element_field: NestedField::list_element(
843                        element_id,
844                        element.into_owned(),
845                        element_required,
846                    )
847                    .into(),
848                }),
849                SerdeType::Map {
850                    r#type: _,
851                    key_id,
852                    key,
853                    value_id,
854                    value_required,
855                    value,
856                } => Map(MapType {
857                    key_field: NestedField::map_key_element(key_id, key.into_owned()).into(),
858                    value_field: NestedField::map_value_element(
859                        value_id,
860                        value.into_owned(),
861                        value_required,
862                    )
863                    .into(),
864                }),
865                SerdeType::Struct { r#type: _, fields } => {
866                    Self::Struct(StructType::new(fields.into_owned()))
867                }
868                SerdeType::Primitive(p) => Self::Primitive(p),
869                SerdeType::Variant(v) => Self::Variant(v),
870            }
871        }
872    }
873
874    impl<'a> From<&'a Type> for SerdeType<'a> {
875        fn from(value: &'a Type) -> Self {
876            match value {
877                Type::List(list) => SerdeType::List {
878                    r#type: "list".to_string(),
879                    element_id: list.element_field.id,
880                    element_required: list.element_field.required,
881                    element: Cow::Borrowed(&list.element_field.field_type),
882                },
883                Map(map) => SerdeType::Map {
884                    r#type: "map".to_string(),
885                    key_id: map.key_field.id,
886                    key: Cow::Borrowed(&map.key_field.field_type),
887                    value_id: map.value_field.id,
888                    value_required: map.value_field.required,
889                    value: Cow::Borrowed(&map.value_field.field_type),
890                },
891                Type::Struct(s) => SerdeType::Struct {
892                    r#type: "struct".to_string(),
893                    fields: Cow::Borrowed(&s.fields),
894                },
895                Type::Primitive(p) => SerdeType::Primitive(p.clone()),
896                Type::Variant(v) => SerdeType::Variant(*v),
897            }
898        }
899    }
900}
901
902#[derive(Debug, PartialEq, Eq, Clone)]
903/// A map is a collection of key-value pairs with a key type and a value type.
904/// Both the key field and value field each have an integer id that is unique in the table schema.
905/// Map keys are required and map values can be either optional or required.
906/// Both map keys and map values may be any type, including nested types.
907pub struct MapType {
908    /// Field for key.
909    pub key_field: NestedFieldRef,
910    /// Field for value.
911    pub value_field: NestedFieldRef,
912}
913
914impl MapType {
915    /// Construct a map type with the given key and value fields.
916    pub fn new(key_field: NestedFieldRef, value_field: NestedFieldRef) -> Self {
917        Self {
918            key_field,
919            value_field,
920        }
921    }
922
923    /// Construct an optional map type with the given key and value fields.
924    pub fn optional(
925        key_id: i32,
926        key_type: impl Into<Type>,
927        value_id: i32,
928        value_type: impl Into<Type>,
929    ) -> Self {
930        Self {
931            key_field: NestedField::map_key_element(key_id, key_type).into(),
932            value_field: NestedField::map_value_element(value_id, value_type, false).into(),
933        }
934    }
935
936    /// Construct a required map type with the given key and value fields.
937    pub fn required(
938        key_id: i32,
939        key_type: impl Into<Type>,
940        value_id: i32,
941        value_type: impl Into<Type>,
942    ) -> Self {
943        Self {
944            key_field: NestedField::map_key_element(key_id, key_type).into(),
945            value_field: NestedField::map_value_element(value_id, value_type, true).into(),
946        }
947    }
948}
949
950/// Variant type - can hold semi-structured data of any type.
951/// This is an Iceberg V3 feature.
952#[derive(Debug, PartialEq, Eq, Clone, Copy, Hash)]
953pub struct VariantType;
954
955impl fmt::Display for VariantType {
956    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
957        write!(f, "variant")
958    }
959}
960
961impl From<VariantType> for Type {
962    fn from(_: VariantType) -> Self {
963        Type::Variant(VariantType)
964    }
965}
966
967impl Serialize for VariantType {
968    fn serialize<S>(&self, serializer: S) -> std::result::Result<S::Ok, S::Error>
969    where S: Serializer {
970        serializer.serialize_str("variant")
971    }
972}
973
974impl<'de> Deserialize<'de> for VariantType {
975    fn deserialize<D>(deserializer: D) -> std::result::Result<Self, D::Error>
976    where D: Deserializer<'de> {
977        let s = String::deserialize(deserializer)?;
978        if s == "variant" {
979            Ok(VariantType)
980        } else {
981            Err(D::Error::custom(format!("expected 'variant', got '{s}'")))
982        }
983    }
984}
985
986#[cfg(test)]
987mod tests {
988    use std::hash::{Hash, Hasher};
989
990    use pretty_assertions::assert_eq;
991    use uuid::Uuid;
992
993    use super::*;
994    use crate::spec::geospatial::{DEFAULT_GEOSPATIAL_CRS, EdgeInterpolationAlgorithm};
995    use crate::spec::values::PrimitiveLiteral;
996
997    fn check_type_serde(json: &str, expected_type: Type) {
998        let desered_type: Type = serde_json::from_str(json).unwrap();
999        assert_eq!(desered_type, expected_type);
1000
1001        let sered_json = serde_json::to_string(&expected_type).unwrap();
1002        let parsed_json_value = serde_json::from_str::<serde_json::Value>(&sered_json).unwrap();
1003        let raw_json_value = serde_json::from_str::<serde_json::Value>(json).unwrap();
1004
1005        assert_eq!(parsed_json_value, raw_json_value);
1006    }
1007
1008    #[test]
1009    fn primitive_type_serde() {
1010        let record = r#"
1011    {
1012        "type": "struct",
1013        "fields": [
1014            {"id": 17, "name": "unknown_field", "required": false, "type": "unknown"},
1015            {"id": 1, "name": "bool_field", "required": true, "type": "boolean"},
1016            {"id": 2, "name": "int_field", "required": true, "type": "int"},
1017            {"id": 3, "name": "long_field", "required": true, "type": "long"},
1018            {"id": 4, "name": "float_field", "required": true, "type": "float"},
1019            {"id": 5, "name": "double_field", "required": true, "type": "double"},
1020            {"id": 6, "name": "decimal_field", "required": true, "type": "decimal(9, 2)"},
1021            {"id": 7, "name": "date_field", "required": true, "type": "date"},
1022            {"id": 8, "name": "time_field", "required": true, "type": "time"},
1023            {"id": 9, "name": "timestamp_field", "required": true, "type": "timestamp"},
1024            {"id": 10, "name": "timestamptz_field", "required": true, "type": "timestamptz"},
1025            {"id": 11, "name": "timestamp_ns_field", "required": true, "type": "timestamp_ns"},
1026            {"id": 12, "name": "timestamptz_ns_field", "required": true, "type": "timestamptz_ns"},
1027            {"id": 13, "name": "uuid_field", "required": true, "type": "uuid"},
1028            {"id": 14, "name": "fixed_field", "required": true, "type": "fixed[10]"},
1029            {"id": 15, "name": "binary_field", "required": true, "type": "binary"},
1030            {"id": 16, "name": "string_field", "required": true, "type": "string"}
1031        ]
1032    }
1033    "#;
1034
1035        check_type_serde(
1036            record,
1037            Type::Struct(StructType {
1038                fields: vec![
1039                    NestedField::optional(
1040                        17,
1041                        "unknown_field",
1042                        Type::Primitive(PrimitiveType::Unknown),
1043                    )
1044                    .into(),
1045                    NestedField::required(1, "bool_field", Type::Primitive(PrimitiveType::Boolean))
1046                        .into(),
1047                    NestedField::required(2, "int_field", Type::Primitive(PrimitiveType::Int))
1048                        .into(),
1049                    NestedField::required(3, "long_field", Type::Primitive(PrimitiveType::Long))
1050                        .into(),
1051                    NestedField::required(4, "float_field", Type::Primitive(PrimitiveType::Float))
1052                        .into(),
1053                    NestedField::required(
1054                        5,
1055                        "double_field",
1056                        Type::Primitive(PrimitiveType::Double),
1057                    )
1058                    .into(),
1059                    NestedField::required(
1060                        6,
1061                        "decimal_field",
1062                        Type::Primitive(PrimitiveType::Decimal {
1063                            precision: 9,
1064                            scale: 2,
1065                        }),
1066                    )
1067                    .into(),
1068                    NestedField::required(7, "date_field", Type::Primitive(PrimitiveType::Date))
1069                        .into(),
1070                    NestedField::required(8, "time_field", Type::Primitive(PrimitiveType::Time))
1071                        .into(),
1072                    NestedField::required(
1073                        9,
1074                        "timestamp_field",
1075                        Type::Primitive(PrimitiveType::Timestamp),
1076                    )
1077                    .into(),
1078                    NestedField::required(
1079                        10,
1080                        "timestamptz_field",
1081                        Type::Primitive(PrimitiveType::Timestamptz),
1082                    )
1083                    .into(),
1084                    NestedField::required(
1085                        11,
1086                        "timestamp_ns_field",
1087                        Type::Primitive(PrimitiveType::TimestampNs),
1088                    )
1089                    .into(),
1090                    NestedField::required(
1091                        12,
1092                        "timestamptz_ns_field",
1093                        Type::Primitive(PrimitiveType::TimestamptzNs),
1094                    )
1095                    .into(),
1096                    NestedField::required(13, "uuid_field", Type::Primitive(PrimitiveType::Uuid))
1097                        .into(),
1098                    NestedField::required(
1099                        14,
1100                        "fixed_field",
1101                        Type::Primitive(PrimitiveType::Fixed(10)),
1102                    )
1103                    .into(),
1104                    NestedField::required(
1105                        15,
1106                        "binary_field",
1107                        Type::Primitive(PrimitiveType::Binary),
1108                    )
1109                    .into(),
1110                    NestedField::required(
1111                        16,
1112                        "string_field",
1113                        Type::Primitive(PrimitiveType::String),
1114                    )
1115                    .into(),
1116                ],
1117                id_lookup: OnceLock::default(),
1118                name_lookup: OnceLock::default(),
1119            }),
1120        )
1121    }
1122
1123    #[test]
1124    fn primitive_type_geospatial() {
1125        let cases = vec![
1126            (
1127                r#""geometry""#,
1128                PrimitiveType::Geometry(GeometryType::default()),
1129                "geometry(OGC:CRS84)",
1130            ),
1131            (
1132                r#""GeOmEtRy ( EPSG:3857 )""#,
1133                PrimitiveType::Geometry(GeometryType::new(Some("EPSG:3857".to_string())).unwrap()),
1134                "geometry(EPSG:3857)",
1135            ),
1136            (
1137                r#""GEOGRAPHY""#,
1138                PrimitiveType::Geography(GeographyType::default()),
1139                "geography(OGC:CRS84, spherical)",
1140            ),
1141            (
1142                r#""GeOgRaPhY ( OGC:CRS27 , KARNEY )""#,
1143                PrimitiveType::Geography(
1144                    GeographyType::new(
1145                        Some("OGC:CRS27".to_string()),
1146                        EdgeInterpolationAlgorithm::Karney,
1147                    )
1148                    .unwrap(),
1149                ),
1150                "geography(OGC:CRS27, karney)",
1151            ),
1152        ];
1153
1154        for (json, expected, display) in cases {
1155            let actual: PrimitiveType = serde_json::from_str(json).unwrap();
1156            assert_eq!(
1157                actual, expected,
1158                "parsed primitive type did not match expectation"
1159            );
1160            assert_eq!(actual.to_string(), display, "display impl did not match");
1161            assert_eq!(
1162                serde_json::to_string(&actual).unwrap(),
1163                format!(r#""{display}""#),
1164                "JSON serialization did not match expectation"
1165            );
1166        }
1167
1168        let equivalent_default = GeometryType::new(Some("EPSG:4326".to_string())).unwrap();
1169        assert_eq!(equivalent_default.crs(), "EPSG:4326");
1170        assert_ne!(equivalent_default, GeometryType::default());
1171        let lowercase_default =
1172            GeometryType::new(Some(DEFAULT_GEOSPATIAL_CRS.to_ascii_lowercase())).unwrap();
1173        assert_eq!(lowercase_default, GeometryType::default());
1174        let mut lowercase_hash = std::collections::hash_map::DefaultHasher::new();
1175        lowercase_default.hash(&mut lowercase_hash);
1176        let mut default_hash = std::collections::hash_map::DefaultHasher::new();
1177        GeometryType::default().hash(&mut default_hash);
1178        assert_eq!(lowercase_hash.finish(), default_hash.finish());
1179        assert!(
1180            serde_json::from_str::<PrimitiveType>(r#""geography(OGC:CRS27,unknown)""#).is_err()
1181        );
1182    }
1183
1184    #[test]
1185    fn struct_type() {
1186        let record = r#"
1187        {
1188            "type": "struct",
1189            "fields": [
1190                {
1191                    "id": 1,
1192                    "name": "id",
1193                    "required": true,
1194                    "type": "uuid",
1195                    "initial-default": "0db3e2a8-9d1d-42b9-aa7b-74ebe558dceb",
1196                    "write-default": "ec5911be-b0a7-458c-8438-c9a3e53cffae"
1197                }, {
1198                    "id": 2,
1199                    "name": "data",
1200                    "required": false,
1201                    "type": "int"
1202                }
1203            ]
1204        }
1205        "#;
1206
1207        check_type_serde(
1208            record,
1209            Type::Struct(StructType {
1210                fields: vec![
1211                    NestedField::required(1, "id", Type::Primitive(PrimitiveType::Uuid))
1212                        .with_initial_default(Literal::Primitive(PrimitiveLiteral::UInt128(
1213                            Uuid::parse_str("0db3e2a8-9d1d-42b9-aa7b-74ebe558dceb")
1214                                .unwrap()
1215                                .as_u128(),
1216                        )))
1217                        .with_write_default(Literal::Primitive(PrimitiveLiteral::UInt128(
1218                            Uuid::parse_str("ec5911be-b0a7-458c-8438-c9a3e53cffae")
1219                                .unwrap()
1220                                .as_u128(),
1221                        )))
1222                        .into(),
1223                    NestedField::optional(2, "data", Type::Primitive(PrimitiveType::Int)).into(),
1224                ],
1225                id_lookup: HashMap::from([(1, 0), (2, 1)]).into(),
1226                name_lookup: HashMap::from([("id".to_string(), 0), ("data".to_string(), 1)]).into(),
1227            }),
1228        )
1229    }
1230
1231    #[test]
1232    fn test_deeply_nested_struct() {
1233        let record = r#"
1234{
1235  "type": "struct",
1236  "fields": [
1237    {
1238      "id": 1,
1239      "name": "id",
1240      "required": true,
1241      "type": "uuid",
1242      "initial-default": "0db3e2a8-9d1d-42b9-aa7b-74ebe558dceb",
1243      "write-default": "ec5911be-b0a7-458c-8438-c9a3e53cffae"
1244    },
1245    {
1246      "id": 2,
1247      "name": "data",
1248      "required": false,
1249      "type": "int"
1250    },
1251    {
1252      "id": 3,
1253      "name": "address",
1254      "required": true,
1255      "type": {
1256        "type": "struct",
1257        "fields": [
1258          {
1259            "id": 4,
1260            "name": "street",
1261            "required": true,
1262            "type": "string"
1263          },
1264          {
1265            "id": 5,
1266            "name": "province",
1267            "required": false,
1268            "type": "string"
1269          },
1270          {
1271            "id": 6,
1272            "name": "zip",
1273            "required": true,
1274            "type": "int"
1275          }
1276        ]
1277      }
1278    }
1279  ]
1280}
1281"#;
1282
1283        let struct_type = Type::Struct(StructType::new(vec![
1284            NestedField::required(1, "id", Type::Primitive(PrimitiveType::Uuid))
1285                .with_initial_default(Literal::Primitive(PrimitiveLiteral::UInt128(
1286                    Uuid::parse_str("0db3e2a8-9d1d-42b9-aa7b-74ebe558dceb")
1287                        .unwrap()
1288                        .as_u128(),
1289                )))
1290                .with_write_default(Literal::Primitive(PrimitiveLiteral::UInt128(
1291                    Uuid::parse_str("ec5911be-b0a7-458c-8438-c9a3e53cffae")
1292                        .unwrap()
1293                        .as_u128(),
1294                )))
1295                .into(),
1296            NestedField::optional(2, "data", Type::Primitive(PrimitiveType::Int)).into(),
1297            NestedField::required(
1298                3,
1299                "address",
1300                Type::Struct(StructType::new(vec![
1301                    NestedField::required(4, "street", Type::Primitive(PrimitiveType::String))
1302                        .into(),
1303                    NestedField::optional(5, "province", Type::Primitive(PrimitiveType::String))
1304                        .into(),
1305                    NestedField::required(6, "zip", Type::Primitive(PrimitiveType::Int)).into(),
1306                ])),
1307            )
1308            .into(),
1309        ]));
1310
1311        check_type_serde(record, struct_type)
1312    }
1313
1314    #[test]
1315    fn list() {
1316        let record = r#"
1317        {
1318            "type": "list",
1319            "element-id": 3,
1320            "element-required": true,
1321            "element": "string"
1322        }
1323        "#;
1324
1325        check_type_serde(
1326            record,
1327            Type::List(ListType {
1328                element_field: NestedField::list_element(
1329                    3,
1330                    Type::Primitive(PrimitiveType::String),
1331                    true,
1332                )
1333                .into(),
1334            }),
1335        );
1336    }
1337
1338    #[test]
1339    fn map() {
1340        let record = r#"
1341        {
1342            "type": "map",
1343            "key-id": 4,
1344            "key": "string",
1345            "value-id": 5,
1346            "value-required": false,
1347            "value": "double"
1348        }
1349        "#;
1350
1351        check_type_serde(
1352            record,
1353            Type::Map(MapType {
1354                key_field: NestedField::map_key_element(4, Type::Primitive(PrimitiveType::String))
1355                    .into(),
1356                value_field: NestedField::map_value_element(
1357                    5,
1358                    Type::Primitive(PrimitiveType::Double),
1359                    false,
1360                )
1361                .into(),
1362            }),
1363        );
1364
1365        check_type_serde(
1366            record,
1367            Type::Map(MapType::optional(
1368                4,
1369                Type::Primitive(PrimitiveType::String),
1370                5,
1371                Type::Primitive(PrimitiveType::Double),
1372            )),
1373        );
1374    }
1375
1376    #[test]
1377    fn map_int() {
1378        let record = r#"
1379        {
1380            "type": "map",
1381            "key-id": 4,
1382            "key": "int",
1383            "value-id": 5,
1384            "value-required": false,
1385            "value": "string"
1386        }
1387        "#;
1388
1389        check_type_serde(
1390            record,
1391            Type::Map(MapType {
1392                key_field: NestedField::map_key_element(4, Type::Primitive(PrimitiveType::Int))
1393                    .into(),
1394                value_field: NestedField::map_value_element(
1395                    5,
1396                    Type::Primitive(PrimitiveType::String),
1397                    false,
1398                )
1399                .into(),
1400            }),
1401        );
1402
1403        check_type_serde(
1404            record,
1405            Type::Map(MapType::optional(
1406                4,
1407                Type::Primitive(PrimitiveType::Int),
1408                5,
1409                Type::Primitive(PrimitiveType::String),
1410            )),
1411        );
1412    }
1413
1414    #[test]
1415    fn map_required_int() {
1416        let record = r#"
1417        {
1418            "type": "map",
1419            "key-id": 4,
1420            "key": "int",
1421            "value-id": 5,
1422            "value-required": true,
1423            "value": "string"
1424        }
1425        "#;
1426
1427        check_type_serde(
1428            record,
1429            Type::Map(MapType::required(
1430                4,
1431                Type::Primitive(PrimitiveType::Int),
1432                5,
1433                Type::Primitive(PrimitiveType::String),
1434            )),
1435        );
1436    }
1437
1438    #[test]
1439    fn test_decimal_precision() {
1440        let expected_max_precision = [
1441            2, 4, 6, 9, 11, 14, 16, 18, 21, 23, 26, 28, 31, 33, 35, 38, 40, 43, 45, 47, 50, 52, 55,
1442            57,
1443        ];
1444        for (i, max_precision) in expected_max_precision.iter().enumerate() {
1445            assert_eq!(
1446                *max_precision,
1447                Type::decimal_max_precision(i as u32 + 1).unwrap(),
1448                "Failed calculate max precision for {i}"
1449            );
1450        }
1451
1452        assert_eq!(5, Type::decimal_required_bytes(10).unwrap());
1453        assert_eq!(16, Type::decimal_required_bytes(38).unwrap());
1454    }
1455
1456    #[test]
1457    fn test_reject_malformed_decimal_and_fixed_type_strings() {
1458        for invalid in [
1459            r#""decimal(50, 2)""#,
1460            r#""decimal(0, 0)""#,
1461            r#""decimal(decimal(5, 2)))""#,
1462            r#""decimal(decimal(5, 2)""#,
1463            r#""decimal(5, 2""#,
1464            r#""decimal(5, 2)))))""#,
1465            r#""decimal(5, 2, 3)""#,
1466            r#""decimal(-5, 2)""#,
1467            r#""decimal(5, -2)""#,
1468            r#""decimal(-5, -2)""#,
1469            r#""decimal(-2, -5)""#,
1470            r#""decimal((5, 2))""#,
1471            r#""decimal[5, 2]""#,
1472            r#""decimal()""#,
1473            r#""fixed[fixed[16]]]""#,
1474            r#""fixed[16""#,
1475            r#""fixed[16]]]""#,
1476            r#""fixed[[16]]""#,
1477            r#""fixed[[16]""#,
1478            r#""fixed(16)""#,
1479            r#""fixed[]""#,
1480        ] {
1481            assert!(
1482                serde_json::from_str::<Type>(invalid).is_err(),
1483                "expected {invalid} to be rejected"
1484            );
1485        }
1486    }
1487
1488    #[test]
1489    fn test_reject_leading_plus_in_type_strings() {
1490        for invalid in [
1491            r#""decimal(+5, 2)""#,
1492            r#""decimal(5, +2)""#,
1493            r#""decimal(+5, +2)""#,
1494            r#""fixed[+16]""#,
1495        ] {
1496            assert!(serde_json::from_str::<Type>(invalid).is_err(), "{invalid}");
1497            assert!(
1498                serde_json::from_str::<PrimitiveType>(invalid).is_err(),
1499                "{invalid}"
1500            );
1501        }
1502    }
1503
1504    #[test]
1505    fn test_decimal_constructor_json_roundtrip() {
1506        for (precision, scale) in [(1, 0), (5, 5), (5, 8), (38, 38)] {
1507            let decimal = Type::decimal(precision, scale).unwrap();
1508            let serialized = serde_json::to_string(&decimal).unwrap();
1509            let reparsed: Type = serde_json::from_str(&serialized).unwrap();
1510            assert_eq!(reparsed, decimal);
1511            let primitive: PrimitiveType = serde_json::from_str(&serialized).unwrap();
1512            assert_eq!(Type::Primitive(primitive), decimal);
1513        }
1514        for precision in [0, 39] {
1515            assert!(Type::decimal(precision, 0).is_err());
1516            let json = format!(r#""decimal({precision}, 0)""#);
1517            assert!(serde_json::from_str::<Type>(&json).is_err());
1518            assert!(serde_json::from_str::<PrimitiveType>(&json).is_err());
1519        }
1520    }
1521
1522    #[test]
1523    fn test_empty_decimal_and_fixed_tokens() {
1524        for invalid in [
1525            r#""decimal(, 2)""#,
1526            r#""decimal(5,)""#,
1527            r#""decimal( , )""#,
1528            r#""fixed[]""#,
1529            r#""fixed[ ]""#,
1530        ] {
1531            assert!(serde_json::from_str::<Type>(invalid).is_err(), "{invalid}");
1532            assert!(
1533                serde_json::from_str::<PrimitiveType>(invalid).is_err(),
1534                "{invalid}"
1535            );
1536        }
1537    }
1538
1539    #[test]
1540    fn test_decimal_and_fixed_overflow_errors() {
1541        for invalid in [
1542            r#""decimal(9999999999, 2)""#,
1543            r#""decimal(5, 9999999999)""#,
1544            r#""fixed[18446744073709551616]""#,
1545        ] {
1546            assert!(serde_json::from_str::<Type>(invalid).is_err(), "{invalid}");
1547            let error = serde_json::from_str::<PrimitiveType>(invalid).unwrap_err();
1548            assert!(
1549                error
1550                    .to_string()
1551                    .contains("number too large to fit in target type"),
1552                "unexpected error for {invalid}: {error}"
1553            );
1554        }
1555    }
1556
1557    #[test]
1558    fn test_decimal_zero_precision_error() {
1559        let error = serde_json::from_str::<PrimitiveType>(r#""decimal(0, 0)""#).unwrap_err();
1560        assert!(
1561            error
1562                .to_string()
1563                .contains("Decimal precision must be greater than zero"),
1564            "unexpected error: {error}"
1565        );
1566    }
1567
1568    #[test]
1569    fn test_accept_valid_decimal_and_fixed_type_strings() {
1570        for (json, canonical, expected) in [
1571            (
1572                r#""decimal(9, 2)""#,
1573                r#""decimal(9, 2)""#,
1574                Type::Primitive(PrimitiveType::Decimal {
1575                    precision: 9,
1576                    scale: 2,
1577                }),
1578            ),
1579            (
1580                r#""decimal(38, 10)""#,
1581                r#""decimal(38, 10)""#,
1582                Type::Primitive(PrimitiveType::Decimal {
1583                    precision: 38,
1584                    scale: 10,
1585                }),
1586            ),
1587            (
1588                r#""decimal(5,2)""#,
1589                r#""decimal(5, 2)""#,
1590                Type::Primitive(PrimitiveType::Decimal {
1591                    precision: 5,
1592                    scale: 2,
1593                }),
1594            ),
1595            (
1596                r#""decimal(5, 0)""#,
1597                r#""decimal(5, 0)""#,
1598                Type::Primitive(PrimitiveType::Decimal {
1599                    precision: 5,
1600                    scale: 0,
1601                }),
1602            ),
1603            (
1604                r#""decimal(5, 5)""#,
1605                r#""decimal(5, 5)""#,
1606                Type::Primitive(PrimitiveType::Decimal {
1607                    precision: 5,
1608                    scale: 5,
1609                }),
1610            ),
1611            (
1612                r#""fixed[16]""#,
1613                r#""fixed[16]""#,
1614                Type::Primitive(PrimitiveType::Fixed(16)),
1615            ),
1616        ] {
1617            check_type_serde_roundtrip_value(json, canonical, expected);
1618        }
1619    }
1620
1621    fn check_type_serde_roundtrip_value(json: &str, canonical: &str, expected_type: Type) {
1622        let parsed: Type = serde_json::from_str(json).unwrap();
1623        assert_eq!(parsed, expected_type);
1624        let serialized = serde_json::to_string(&parsed).unwrap();
1625        assert_eq!(serialized, canonical);
1626        let primitive: PrimitiveType = serde_json::from_str(json).unwrap();
1627        assert_eq!(serde_json::to_string(&primitive).unwrap(), canonical);
1628        let reparsed: Type = serde_json::from_str(&serialized).unwrap();
1629        assert_eq!(reparsed, expected_type);
1630    }
1631
1632    #[test]
1633    fn test_primitive_type_compatible() {
1634        let pairs = vec![
1635            (PrimitiveType::Boolean, PrimitiveLiteral::Boolean(true)),
1636            (PrimitiveType::Int, PrimitiveLiteral::Int(1)),
1637            (PrimitiveType::Long, PrimitiveLiteral::Long(1)),
1638            (PrimitiveType::Float, PrimitiveLiteral::Float(1.0.into())),
1639            (PrimitiveType::Double, PrimitiveLiteral::Double(1.0.into())),
1640            (
1641                PrimitiveType::Decimal {
1642                    precision: 9,
1643                    scale: 2,
1644                },
1645                PrimitiveLiteral::Int128(1),
1646            ),
1647            (PrimitiveType::Date, PrimitiveLiteral::Int(1)),
1648            (PrimitiveType::Time, PrimitiveLiteral::Long(1)),
1649            (PrimitiveType::Timestamptz, PrimitiveLiteral::Long(1)),
1650            (PrimitiveType::Timestamp, PrimitiveLiteral::Long(1)),
1651            (PrimitiveType::TimestamptzNs, PrimitiveLiteral::Long(1)),
1652            (PrimitiveType::TimestampNs, PrimitiveLiteral::Long(1)),
1653            (
1654                PrimitiveType::Uuid,
1655                PrimitiveLiteral::UInt128(Uuid::new_v4().as_u128()),
1656            ),
1657            (PrimitiveType::Fixed(8), PrimitiveLiteral::Binary(vec![1])),
1658            (PrimitiveType::Binary, PrimitiveLiteral::Binary(vec![1])),
1659            (
1660                PrimitiveType::Geometry(GeometryType::default()),
1661                PrimitiveLiteral::Binary(vec![1]),
1662            ),
1663            (
1664                PrimitiveType::Geography(GeographyType::default()),
1665                PrimitiveLiteral::Binary(vec![1]),
1666            ),
1667        ];
1668        for (ty, literal) in pairs {
1669            assert!(ty.compatible(&literal));
1670        }
1671
1672        assert!(!PrimitiveType::Unknown.compatible(&PrimitiveLiteral::Int(1)));
1673    }
1674
1675    #[test]
1676    fn variant_type_serde() {
1677        let json = r#"{"id": 1, "name": "v", "required": true, "type": "variant"}"#;
1678        let field: NestedField = serde_json::from_str(json).unwrap();
1679        assert_eq!(*field.field_type, Type::Variant(VariantType));
1680
1681        let serialized = serde_json::to_string(&field).unwrap();
1682        let roundtrip: NestedField = serde_json::from_str(&serialized).unwrap();
1683        assert_eq!(field, roundtrip);
1684    }
1685
1686    #[test]
1687    fn nested_field_rejects_invalid_map_defaults() {
1688        for default_name in ["initial-default", "write-default"] {
1689            let json = format!(
1690                r#"{{
1691                    "id": 1,
1692                    "name": "properties",
1693                    "required": false,
1694                    "type": {{
1695                        "type": "map",
1696                        "key-id": 2,
1697                        "key": "string",
1698                        "value-id": 3,
1699                        "value-required": false,
1700                        "value": "int"
1701                    }},
1702                    "{default_name}": {{"keys": ["a", "b"], "values": [1]}}
1703                }}"#
1704            );
1705
1706            let error = serde_json::from_str::<NestedField>(&json).unwrap_err();
1707            assert!(error.to_string().contains("must have the same length"));
1708        }
1709    }
1710
1711    #[test]
1712    fn struct_type_with_type_field() {
1713        // Test that StructType properly deserializes JSON with "type":"struct" field
1714        // This was previously broken because the deserializer wasn't consuming the type field value
1715        let json = r#"
1716        {
1717            "type": "struct",
1718            "fields": [
1719                {"id": 1, "name": "field1", "required": true, "type": "string"}
1720            ]
1721        }
1722        "#;
1723
1724        let struct_type: StructType = serde_json::from_str(json)
1725            .expect("Should successfully deserialize StructType with type field");
1726
1727        assert_eq!(struct_type.fields().len(), 1);
1728        assert_eq!(struct_type.fields()[0].name, "field1");
1729    }
1730
1731    #[test]
1732    fn struct_type_rejects_wrong_type() {
1733        // Test that StructType validation rejects incorrect type field values
1734        let json = r#"
1735        {
1736            "type": "list",
1737            "fields": [
1738                {"id": 1, "name": "field1", "required": true, "type": "string"}
1739            ]
1740        }
1741        "#;
1742
1743        let result = serde_json::from_str::<StructType>(json);
1744        assert!(
1745            result.is_err(),
1746            "Should reject StructType with wrong type field"
1747        );
1748        assert!(
1749            result
1750                .unwrap_err()
1751                .to_string()
1752                .contains("expected type 'struct'")
1753        );
1754    }
1755}
iceberg/spec/datatypes.rs

iceberg/spec/
datatypes.rs
