iceberg/spec/schema/
mod.rs

1// Licensed to the Apache Software Foundation (ASF) under one
2// or more contributor license agreements.  See the NOTICE file
3// distributed with this work for additional information
4// regarding copyright ownership.  The ASF licenses this file
5// to you under the Apache License, Version 2.0 (the
6// "License"); you may not use this file except in compliance
7// with the License.  You may obtain a copy of the License at
8//
9//   http://www.apache.org/licenses/LICENSE-2.0
10//
11// Unless required by applicable law or agreed to in writing,
12// software distributed under the License is distributed on an
13// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14// KIND, either express or implied.  See the License for the
15// specific language governing permissions and limitations
16// under the License.
17
18//! This module defines schema in iceberg.
19
20use std::collections::{HashMap, HashSet};
21use std::fmt::{Display, Formatter};
22use std::sync::Arc;
23
24mod utils;
25mod visitor;
26pub use self::visitor::*;
27pub(super) mod _serde;
28mod id_reassigner;
29mod index;
30mod prune_columns;
31use bimap::BiHashMap;
32use itertools::{Itertools, zip_eq};
33use serde::{Deserialize, Serialize};
34
35use self::_serde::SchemaEnum;
36use self::id_reassigner::ReassignFieldIds;
37use self::index::{IndexByName, index_by_id, index_parents};
38pub use self::prune_columns::prune_columns;
39use super::NestedField;
40use crate::error::{Result, invalid_data};
41use crate::expr::accessor::StructAccessor;
42use crate::spec::FormatVersion;
43use crate::spec::datatypes::{
44    LIST_FIELD_NAME, ListType, MAP_KEY_FIELD_NAME, MAP_VALUE_FIELD_NAME, MapType, NestedFieldRef,
45    PrimitiveType, StructType, Type,
46};
47use crate::{Error, ErrorKind, ensure_data_valid};
48
49/// Type alias for schema id.
50pub type SchemaId = i32;
51/// Reference to [`Schema`].
52pub type SchemaRef = Arc<Schema>;
53/// Default schema id.
54pub const DEFAULT_SCHEMA_ID: SchemaId = 0;
55/// Delimiter for schema name, which denotes a nested struct.
56pub const SCHEMA_NAME_DELIMITER: &str = ".";
57/// Minimum format version that allows non-null field default values.
58/// Mirrors Java's `Schema.DEFAULT_VALUES_MIN_FORMAT_VERSION`.
59pub(crate) const DEFAULT_VALUES_MIN_FORMAT_VERSION: FormatVersion = FormatVersion::V3;
60
61/// Defines schema in iceberg.
62#[derive(Debug, Serialize, Deserialize, Clone)]
63#[serde(try_from = "SchemaEnum", into = "SchemaEnum")]
64pub struct Schema {
65    r#struct: StructType,
66    schema_id: SchemaId,
67    highest_field_id: i32,
68    identifier_field_ids: HashSet<i32>,
69
70    alias_to_id: BiHashMap<String, i32>,
71    id_to_field: HashMap<i32, NestedFieldRef>,
72
73    name_to_id: HashMap<String, i32>,
74    lowercase_name_to_id: HashMap<String, i32>,
75    id_to_name: HashMap<i32, String>,
76
77    field_id_to_accessor: HashMap<i32, Arc<StructAccessor>>,
78}
79
80impl PartialEq for Schema {
81    fn eq(&self, other: &Self) -> bool {
82        self.r#struct == other.r#struct
83            && self.schema_id == other.schema_id
84            && self.identifier_field_ids == other.identifier_field_ids
85    }
86}
87
88impl Eq for Schema {}
89
90/// Schema builder.
91#[derive(Debug)]
92pub struct SchemaBuilder {
93    schema_id: i32,
94    fields: Vec<NestedFieldRef>,
95    alias_to_id: BiHashMap<String, i32>,
96    identifier_field_ids: HashSet<i32>,
97    reassign_field_ids_from: Option<i32>,
98}
99
100impl SchemaBuilder {
101    /// Add fields to schema builder.
102    pub fn with_fields(mut self, fields: impl IntoIterator<Item = NestedFieldRef>) -> Self {
103        self.fields.extend(fields);
104        self
105    }
106
107    /// Reassign all field-ids (including nested) on build.
108    /// Reassignment starts from the field-id specified in `start_from` (inclusive).
109    ///
110    /// All specified aliases and identifier fields will be updated to the new field-ids.
111    pub(crate) fn with_reassigned_field_ids(mut self, start_from: i32) -> Self {
112        self.reassign_field_ids_from = Some(start_from);
113        self
114    }
115
116    /// Set schema id.
117    pub fn with_schema_id(mut self, schema_id: i32) -> Self {
118        self.schema_id = schema_id;
119        self
120    }
121
122    /// Set identifier field ids.
123    pub fn with_identifier_field_ids(mut self, ids: impl IntoIterator<Item = i32>) -> Self {
124        self.identifier_field_ids.extend(ids);
125        self
126    }
127
128    /// Set alias to filed id mapping.
129    pub fn with_alias(mut self, alias_to_id: BiHashMap<String, i32>) -> Self {
130        self.alias_to_id = alias_to_id;
131        self
132    }
133
134    /// Builds the schema.
135    pub fn build(self) -> Result<Schema> {
136        let field_id_to_accessor = self.build_accessors();
137
138        let r#struct = StructType::new(self.fields);
139        let id_to_field = index_by_id(&r#struct)?;
140
141        Self::validate_identifier_ids(
142            &r#struct,
143            &id_to_field,
144            self.identifier_field_ids.iter().copied(),
145        )?;
146
147        let (name_to_id, id_to_name) = {
148            let mut index = IndexByName::default();
149            visit_struct(&r#struct, &mut index)?;
150            index.indexes()
151        };
152
153        let lowercase_name_to_id = name_to_id
154            .iter()
155            .map(|(k, v)| (k.to_lowercase(), *v))
156            .collect();
157
158        let highest_field_id = id_to_field.keys().max().cloned().unwrap_or(0);
159
160        let mut schema = Schema {
161            r#struct,
162            schema_id: self.schema_id,
163            highest_field_id,
164            identifier_field_ids: self.identifier_field_ids,
165            alias_to_id: self.alias_to_id,
166            id_to_field,
167
168            name_to_id,
169            lowercase_name_to_id,
170            id_to_name,
171
172            field_id_to_accessor,
173        };
174
175        if let Some(start_from) = self.reassign_field_ids_from {
176            let mut id_reassigner = ReassignFieldIds::new(start_from);
177            let new_fields = id_reassigner.reassign_field_ids(schema.r#struct.fields().to_vec())?;
178            let new_identifier_field_ids =
179                id_reassigner.apply_to_identifier_fields(schema.identifier_field_ids)?;
180            let new_alias_to_id = id_reassigner.apply_to_aliases(schema.alias_to_id.clone())?;
181
182            schema = Schema::builder()
183                .with_schema_id(schema.schema_id)
184                .with_fields(new_fields)
185                .with_identifier_field_ids(new_identifier_field_ids)
186                .with_alias(new_alias_to_id)
187                .build()?;
188        }
189
190        Ok(schema)
191    }
192
193    fn build_accessors(&self) -> HashMap<i32, Arc<StructAccessor>> {
194        let mut map = HashMap::new();
195
196        for (pos, field) in self.fields.iter().enumerate() {
197            match field.field_type.as_ref() {
198                Type::Primitive(prim_type) => {
199                    // add an accessor for this field
200                    let accessor = Arc::new(StructAccessor::new(pos, prim_type.clone()));
201                    map.insert(field.id, accessor.clone());
202                }
203
204                Type::Struct(nested) => {
205                    // add accessors for nested fields
206                    for (field_id, accessor) in Self::build_accessors_nested(nested.fields()) {
207                        let new_accessor = Arc::new(StructAccessor::wrap(pos, accessor));
208                        map.insert(field_id, new_accessor.clone());
209                    }
210                }
211                _ => {
212                    // Accessors don't get built for Map or List types
213                }
214            }
215        }
216
217        map
218    }
219
220    fn build_accessors_nested(fields: &[NestedFieldRef]) -> Vec<(i32, Box<StructAccessor>)> {
221        let mut results = vec![];
222        for (pos, field) in fields.iter().enumerate() {
223            match field.field_type.as_ref() {
224                Type::Primitive(prim_type) => {
225                    let accessor = Box::new(StructAccessor::new(pos, prim_type.clone()));
226                    results.push((field.id, accessor));
227                }
228                Type::Struct(nested) => {
229                    let nested_accessors = Self::build_accessors_nested(nested.fields());
230
231                    let wrapped_nested_accessors =
232                        nested_accessors.into_iter().map(|(id, accessor)| {
233                            let new_accessor = Box::new(StructAccessor::wrap(pos, accessor));
234                            (id, new_accessor.clone())
235                        });
236
237                    results.extend(wrapped_nested_accessors);
238                }
239                _ => {
240                    // Accessors don't get built for Map or List types
241                }
242            }
243        }
244
245        results
246    }
247
248    /// According to [the spec](https://iceberg.apache.org/spec/#identifier-fields), the identifier fields
249    /// must meet the following requirements:
250    /// - Float, double, and optional fields cannot be used as identifier fields.
251    /// - Identifier fields may be nested in structs but cannot be nested within maps or lists.
252    /// - A nested field cannot be used as an identifier field if it is nested in an optional struct, to avoid null values in identifiers.
253    fn validate_identifier_ids(
254        r#struct: &StructType,
255        id_to_field: &HashMap<i32, NestedFieldRef>,
256        identifier_field_ids: impl Iterator<Item = i32>,
257    ) -> Result<()> {
258        let id_to_parent = index_parents(r#struct)?;
259        for identifier_field_id in identifier_field_ids {
260            let field = id_to_field.get(&identifier_field_id).ok_or_else(|| {
261                invalid_data!(
262                    "Cannot add identifier field {identifier_field_id}: field does not exist"
263                )
264            })?;
265            ensure_data_valid!(
266                field.required,
267                "Cannot add identifier field: {} is an optional field",
268                field.name
269            );
270            if let Type::Primitive(p) = field.field_type.as_ref() {
271                ensure_data_valid!(
272                    !matches!(p, PrimitiveType::Double | PrimitiveType::Float),
273                    "Cannot add identifier field {}: cannot be a float or double type",
274                    field.name
275                );
276            } else {
277                return Err(invalid_data!(
278                    "Cannot add field {} as an identifier field: not a primitive type field",
279                    field.name
280                ));
281            }
282
283            let mut cur_field_id = identifier_field_id;
284            while let Some(parent) = id_to_parent.get(&cur_field_id) {
285                let parent_field = id_to_field
286                    .get(parent)
287                    .expect("Field id should not disappear.");
288                ensure_data_valid!(
289                    parent_field.field_type.is_struct(),
290                    "Cannot add field {} as an identifier field: must not be nested in {:?}",
291                    field.name,
292                    parent_field
293                );
294                ensure_data_valid!(
295                    parent_field.required,
296                    "Cannot add field {} as an identifier field: must not be nested in an optional field {}",
297                    field.name,
298                    parent_field
299                );
300                cur_field_id = *parent;
301            }
302        }
303
304        Ok(())
305    }
306}
307
308impl Schema {
309    /// Create a schema builder.
310    pub fn builder() -> SchemaBuilder {
311        SchemaBuilder {
312            schema_id: DEFAULT_SCHEMA_ID,
313            fields: vec![],
314            identifier_field_ids: HashSet::default(),
315            alias_to_id: BiHashMap::default(),
316            reassign_field_ids_from: None,
317        }
318    }
319
320    /// Create a new schema builder from a schema.
321    pub fn into_builder(self) -> SchemaBuilder {
322        SchemaBuilder {
323            schema_id: self.schema_id,
324            fields: self.r#struct.fields().to_vec(),
325            alias_to_id: self.alias_to_id,
326            identifier_field_ids: self.identifier_field_ids,
327            reassign_field_ids_from: None,
328        }
329    }
330
331    /// Get field by field id.
332    pub fn field_by_id(&self, field_id: i32) -> Option<&NestedFieldRef> {
333        self.id_to_field.get(&field_id)
334    }
335
336    /// Get field by field name.
337    ///
338    /// Both full name and short name could work here.
339    pub fn field_by_name(&self, field_name: &str) -> Option<&NestedFieldRef> {
340        self.name_to_id
341            .get(field_name)
342            .and_then(|id| self.field_by_id(*id))
343    }
344
345    /// Get field by field name, but in case-insensitive way.
346    ///
347    /// Both full name and short name could work here.
348    pub fn field_by_name_case_insensitive(&self, field_name: &str) -> Option<&NestedFieldRef> {
349        self.lowercase_name_to_id
350            .get(&field_name.to_lowercase())
351            .and_then(|id| self.field_by_id(*id))
352    }
353
354    /// Get field by alias.
355    pub fn field_by_alias(&self, alias: &str) -> Option<&NestedFieldRef> {
356        self.alias_to_id
357            .get_by_left(alias)
358            .and_then(|id| self.field_by_id(*id))
359    }
360
361    /// Returns the highest field ID assigned in this schema.
362    #[inline]
363    pub fn highest_field_id(&self) -> i32 {
364        self.highest_field_id
365    }
366
367    /// Returns the Schema's ID.
368    #[inline]
369    pub fn schema_id(&self) -> SchemaId {
370        self.schema_id
371    }
372
373    /// Returns the type for the struct.
374    #[inline]
375    pub fn as_struct(&self) -> &StructType {
376        &self.r#struct
377    }
378
379    /// Returns the IDs of the fields.
380    #[inline]
381    pub fn identifier_field_ids(&self) -> impl ExactSizeIterator<Item = i32> + '_ {
382        self.identifier_field_ids.iter().copied()
383    }
384
385    /// Get field id by full name.
386    pub fn field_id_by_name(&self, name: &str) -> Option<i32> {
387        self.name_to_id.get(name).copied()
388    }
389
390    /// Get full name by field id.
391    pub fn name_by_field_id(&self, field_id: i32) -> Option<&str> {
392        self.id_to_name.get(&field_id).map(String::as_str)
393    }
394
395    /// Get an accessor for retrieving data in a struct
396    pub fn accessor_by_field_id(&self, field_id: i32) -> Option<Arc<StructAccessor>> {
397        self.field_id_to_accessor.get(&field_id).cloned()
398    }
399
400    /// Check if this schema is identical to another schema semantically - excluding schema id.
401    pub(crate) fn is_same_schema(&self, other: &SchemaRef) -> bool {
402        self.as_struct().eq(other.as_struct())
403            && self.identifier_field_ids().eq(other.identifier_field_ids())
404    }
405
406    /// Change the schema id of this schema.
407    // This is redundant with the `with_schema_id` method on the builder, but useful
408    // as it is infallible in contrast to the builder `build()` method.
409    pub(crate) fn with_schema_id(self, schema_id: SchemaId) -> Self {
410        Self { schema_id, ..self }
411    }
412
413    /// Return A HashMap matching field ids to field names.
414    pub fn field_id_to_name_map(&self) -> &HashMap<i32, String> {
415        &self.id_to_name
416    }
417
418    /// Return a hashmap matching field ids to nested fields.
419    pub fn field_id_to_fields(&self) -> &HashMap<i32, NestedFieldRef> {
420        &self.id_to_field
421    }
422
423    /// Minimum [`FormatVersion`] required to represent all *types* in this schema.
424    ///
425    /// Walks the whole schema — every field, nested ones included — and returns the
426    /// highest per-type floor, so this is O(fields) rather than a cheap field read.
427    /// Types only; for initial-default version floors see [`Schema::check_format_compatibility`].
428    pub fn calc_min_compatible_format(&self) -> FormatVersion {
429        // `id_to_field` is flattened, so the max over all fields covers nested ones too.
430        self.id_to_field
431            .values()
432            .map(|f| f.field_type.min_format_version())
433            .max()
434            .unwrap_or(FormatVersion::V1)
435    }
436
437    /// Returns an error listing every field incompatible with `format_version`.
438    /// Mirrors Java's `Schema.checkCompatibility()`. Two checks per field:
439    ///
440    /// - **Type** — minimum format version required to support that type, without
441    ///   taking nested field types into account.
442    /// - **Initial default** — a non-null `initial_default` backfills pre-existing rows,
443    ///   so it requires `DEFAULT_VALUES_MIN_FORMAT_VERSION`; `write_default` is not
444    ///   checked, as it only affects newly written rows (read identically at any version).
445    pub fn check_format_compatibility(&self, format_version: FormatVersion) -> Result<()> {
446        // (field id, message); sorted by id below for a deterministic error.
447        let mut problems: Vec<(i32, String)> = Vec::new();
448
449        // `id_to_field` is flattened, so checking each field by its own type keeps the
450        // blame on the offending leaf, not its container (mirrors Java's `lazyIdToField`).
451        for field in self.id_to_field.values() {
452            let min_version = field.field_type.min_format_version();
453            if format_version < min_version {
454                // Every id in `id_to_field` is also indexed in `id_to_name`; a miss means
455                // the schema's indexes are inconsistent (a bug), so surface it rather than
456                // guessing an unqualified name.
457                let name = self.name_by_field_id(field.id).ok_or_else(|| {
458                    Error::new(
459                        ErrorKind::Unexpected,
460                        format!(
461                            "Field id {} is missing from the schema's name index",
462                            field.id
463                        ),
464                    )
465                })?;
466                problems.push((field.id, format!(
467                    "Invalid type for {name}: {} is not supported until {min_version} but format version is {format_version}.",
468                    field.field_type,
469                )));
470            }
471
472            if let Some(default) = &field.initial_default
473                && format_version < DEFAULT_VALUES_MIN_FORMAT_VERSION
474            {
475                let name = self.name_by_field_id(field.id).ok_or_else(|| {
476                    Error::new(
477                        ErrorKind::Unexpected,
478                        format!(
479                            "Field id {} is missing from the schema's name index",
480                            field.id
481                        ),
482                    )
483                })?;
484                problems.push((field.id, format!(
485                    "Invalid initial default for {name}: non-null default ({default:?}) is not supported until {DEFAULT_VALUES_MIN_FORMAT_VERSION} but format version is {format_version}."
486                )));
487            }
488        }
489
490        if problems.is_empty() {
491            return Ok(());
492        }
493
494        // Stable sort by id: HashMap order is nondeterministic, and stability keeps a
495        // field's type problem before its default problem (matches Java's TreeMap order).
496        let message = problems
497            .into_iter()
498            .sorted_by_key(|(id, _)| *id)
499            .map(|(_, msg)| msg)
500            .join("\n- ");
501        Err(invalid_data!(
502            "Invalid schema for {format_version}:\n- {message}"
503        ))
504    }
505}
506
507impl Display for Schema {
508    fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result {
509        writeln!(f, "table {{")?;
510        for field in self.as_struct().fields() {
511            writeln!(f, "  {field}")?;
512        }
513        writeln!(f, "}}")
514    }
515}
516
517#[cfg(test)]
518mod tests {
519    use std::collections::HashMap;
520
521    use bimap::BiHashMap;
522
523    use crate::spec::datatypes::Type::{List, Map, Primitive, Struct, Variant};
524    use crate::spec::datatypes::{
525        ListType, MapType, NestedField, NestedFieldRef, PrimitiveType, StructType,
526    };
527    use crate::spec::schema::Schema;
528    use crate::spec::values::Map as MapValue;
529    use crate::spec::{Datum, Literal};
530
531    #[test]
532    fn test_check_format_compatibility() {
533        use crate::spec::{FormatVersion, PrimitiveLiteral, VariantType};
534
535        fn schema_with(fields: Vec<NestedFieldRef>) -> Schema {
536            Schema::builder().with_fields(fields).build().unwrap()
537        }
538
539        // Variant type requires v3.
540        let variant = schema_with(vec![
541            NestedField::optional(1, "v", Variant(VariantType)).into(),
542        ]);
543        assert!(
544            variant
545                .check_format_compatibility(FormatVersion::V2)
546                .is_err()
547        );
548        assert!(
549            variant
550                .check_format_compatibility(FormatVersion::V3)
551                .is_ok()
552        );
553
554        // A non-null initial default requires v3, even for a v1-compatible type.
555        let with_default = schema_with(vec![
556            NestedField::optional(1, "a", Primitive(PrimitiveType::Int))
557                .with_initial_default(Literal::Primitive(PrimitiveLiteral::Int(1)))
558                .into(),
559        ]);
560        let err = with_default
561            .check_format_compatibility(FormatVersion::V2)
562            .unwrap_err();
563        assert!(
564            err.message().contains("Invalid initial default for a"),
565            "{err}"
566        );
567        assert!(
568            with_default
569                .check_format_compatibility(FormatVersion::V3)
570                .is_ok()
571        );
572
573        // No default is fine at any version (an absent/null default never trips).
574        let no_default = schema_with(vec![
575            NestedField::optional(1, "a", Primitive(PrimitiveType::Int)).into(),
576        ]);
577        assert!(
578            no_default
579                .check_format_compatibility(FormatVersion::V1)
580                .is_ok()
581        );
582
583        // Recursion: a default on a field nested inside a struct is caught.
584        let nested = schema_with(vec![
585            NestedField::required(
586                1,
587                "s",
588                Struct(StructType::new(vec![
589                    NestedField::optional(2, "inner", Primitive(PrimitiveType::Long))
590                        .with_initial_default(Literal::Primitive(PrimitiveLiteral::Long(7)))
591                        .into(),
592                ])),
593            )
594            .into(),
595        ]);
596        let err = nested
597            .check_format_compatibility(FormatVersion::V2)
598            .unwrap_err();
599        assert!(err.message().contains("inner"), "{err}");
600
601        // A v3-only type nested in a container blames the leaf, never the container.
602        let nested_variant = schema_with(vec![
603            NestedField::required(
604                1,
605                "container",
606                Struct(StructType::new(vec![
607                    NestedField::optional(2, "v", Variant(VariantType)).into(),
608                ])),
609            )
610            .into(),
611        ]);
612        let err = nested_variant
613            .check_format_compatibility(FormatVersion::V2)
614            .unwrap_err();
615        assert!(err.message().contains("container.v"), "{err}");
616        assert!(
617            !err.message().contains("Invalid type for container:"),
618            "container must not be blamed: {err}"
619        );
620    }
621
622    #[test]
623    fn test_calc_min_compatible_format() {
624        use crate::spec::{FormatVersion, VariantType};
625
626        fn schema_with(fields: Vec<NestedFieldRef>) -> Schema {
627            Schema::builder().with_fields(fields).build().unwrap()
628        }
629
630        // All v1-compatible types → V1.
631        let v1 = schema_with(vec![
632            NestedField::required(1, "a", Primitive(PrimitiveType::Int)).into(),
633            NestedField::optional(2, "b", Primitive(PrimitiveType::String)).into(),
634        ]);
635        assert_eq!(v1.calc_min_compatible_format(), FormatVersion::V1);
636
637        // A top-level variant → V3.
638        let variant = schema_with(vec![
639            NestedField::optional(1, "v", Variant(VariantType)).into(),
640        ]);
641        assert_eq!(variant.calc_min_compatible_format(), FormatVersion::V3);
642
643        // Unknown is a v3-only primitive type.
644        let unknown = schema_with(vec![
645            NestedField::optional(1, "u", Primitive(PrimitiveType::Unknown)).into(),
646        ]);
647        assert_eq!(unknown.calc_min_compatible_format(), FormatVersion::V3);
648        assert!(
649            unknown
650                .check_format_compatibility(FormatVersion::V2)
651                .is_err()
652        );
653
654        // A v3-only type nested inside a list inside a struct → V3 (flattened fields).
655        let nested = schema_with(vec![
656            NestedField::required(
657                1,
658                "s",
659                Struct(StructType::new(vec![
660                    NestedField::optional(
661                        2,
662                        "l",
663                        List(ListType::new(
664                            NestedField::required(
665                                3,
666                                "element",
667                                Primitive(PrimitiveType::TimestampNs),
668                            )
669                            .into(),
670                        )),
671                    )
672                    .into(),
673                ])),
674            )
675            .into(),
676        ]);
677        assert_eq!(nested.calc_min_compatible_format(), FormatVersion::V3);
678    }
679
680    #[test]
681    fn test_check_format_compatibility_message_order() {
682        use crate::spec::{FormatVersion, PrimitiveLiteral, VariantType};
683
684        // Fields declared out of id order; field 2 fails both checks (ts_ns type
685        // AND a non-null initial default). The error must list problems by field id,
686        // with the type problem before the initial-default problem for field 2.
687        let schema = Schema::builder()
688            .with_fields(vec![
689                NestedField::optional(3, "c", Variant(VariantType)).into(),
690                NestedField::optional(2, "b", Primitive(PrimitiveType::TimestampNs))
691                    .with_initial_default(Literal::Primitive(PrimitiveLiteral::Long(0)))
692                    .into(),
693                NestedField::required(1, "a", Primitive(PrimitiveType::Int)).into(),
694            ])
695            .build()
696            .unwrap();
697
698        let message = schema
699            .check_format_compatibility(FormatVersion::V2)
700            .unwrap_err()
701            .message()
702            .to_string();
703
704        let lines: Vec<&str> = message.lines().skip(1).collect();
705        assert_eq!(
706            lines,
707            vec![
708                "- Invalid type for b: timestamp_ns is not supported until v3 but format version is v2.",
709                "- Invalid initial default for b: non-null default (Primitive(Long(0))) is not supported until v3 but format version is v2.",
710                "- Invalid type for c: variant is not supported until v3 but format version is v2.",
711            ],
712            "{message}"
713        );
714    }
715
716    #[test]
717    fn test_construct_schema() {
718        let field1: NestedFieldRef =
719            NestedField::required(1, "f1", Primitive(PrimitiveType::Boolean)).into();
720        let field2: NestedFieldRef =
721            NestedField::optional(2, "f2", Primitive(PrimitiveType::Int)).into();
722
723        let schema = Schema::builder()
724            .with_fields(vec![field1.clone()])
725            .with_fields(vec![field2.clone()])
726            .with_schema_id(3)
727            .build()
728            .unwrap();
729
730        assert_eq!(3, schema.schema_id());
731        assert_eq!(2, schema.highest_field_id());
732        assert_eq!(Some(&field1), schema.field_by_id(1));
733        assert_eq!(Some(&field2), schema.field_by_id(2));
734        assert_eq!(None, schema.field_by_id(3));
735    }
736
737    pub fn table_schema_simple<'a>() -> (Schema, &'a str) {
738        let schema = Schema::builder()
739            .with_schema_id(1)
740            .with_identifier_field_ids(vec![2])
741            .with_fields(vec![
742                NestedField::optional(1, "foo", Primitive(PrimitiveType::String)).into(),
743                NestedField::required(2, "bar", Primitive(PrimitiveType::Int)).into(),
744                NestedField::optional(3, "baz", Primitive(PrimitiveType::Boolean)).into(),
745            ])
746            .build()
747            .unwrap();
748        let record = r#"{
749            "type":"struct",
750            "schema-id":1,
751            "fields":[
752                {
753                    "id":1,
754                    "name":"foo",
755                    "required":false,
756                    "type":"string"
757                },
758                {
759                    "id":2,
760                    "name":"bar",
761                    "required":true,
762                    "type":"int"
763                },
764                {
765                    "id":3,
766                    "name":"baz",
767                    "required":false,
768                    "type":"boolean"
769                }
770            ],
771            "identifier-field-ids":[2]
772        }"#;
773        (schema, record)
774    }
775
776    pub fn table_schema_nested() -> Schema {
777        Schema::builder()
778            .with_schema_id(1)
779            .with_identifier_field_ids(vec![2])
780            .with_fields(vec![
781                NestedField::optional(1, "foo", Primitive(PrimitiveType::String)).into(),
782                NestedField::required(2, "bar", Primitive(PrimitiveType::Int)).into(),
783                NestedField::optional(3, "baz", Primitive(PrimitiveType::Boolean)).into(),
784                NestedField::required(
785                    4,
786                    "qux",
787                    List(ListType {
788                        element_field: NestedField::list_element(
789                            5,
790                            Primitive(PrimitiveType::String),
791                            true,
792                        )
793                        .into(),
794                    }),
795                )
796                .into(),
797                NestedField::required(
798                    6,
799                    "quux",
800                    Map(MapType {
801                        key_field: NestedField::map_key_element(
802                            7,
803                            Primitive(PrimitiveType::String),
804                        )
805                        .into(),
806                        value_field: NestedField::map_value_element(
807                            8,
808                            Map(MapType {
809                                key_field: NestedField::map_key_element(
810                                    9,
811                                    Primitive(PrimitiveType::String),
812                                )
813                                .into(),
814                                value_field: NestedField::map_value_element(
815                                    10,
816                                    Primitive(PrimitiveType::Int),
817                                    true,
818                                )
819                                .into(),
820                            }),
821                            true,
822                        )
823                        .into(),
824                    }),
825                )
826                .into(),
827                NestedField::required(
828                    11,
829                    "location",
830                    List(ListType {
831                        element_field: NestedField::list_element(
832                            12,
833                            Struct(StructType::new(vec![
834                                NestedField::optional(
835                                    13,
836                                    "latitude",
837                                    Primitive(PrimitiveType::Float),
838                                )
839                                .into(),
840                                NestedField::optional(
841                                    14,
842                                    "longitude",
843                                    Primitive(PrimitiveType::Float),
844                                )
845                                .into(),
846                            ])),
847                            true,
848                        )
849                        .into(),
850                    }),
851                )
852                .into(),
853                NestedField::optional(
854                    15,
855                    "person",
856                    Struct(StructType::new(vec![
857                        NestedField::optional(16, "name", Primitive(PrimitiveType::String)).into(),
858                        NestedField::required(17, "age", Primitive(PrimitiveType::Int)).into(),
859                    ])),
860                )
861                .into(),
862            ])
863            .build()
864            .unwrap()
865    }
866
867    #[test]
868    fn test_schema_display() {
869        let expected_str = "
870table {
871  1: foo: optional string\x20
872  2: bar: required int\x20
873  3: baz: optional boolean\x20
874}
875";
876
877        assert_eq!(expected_str, format!("\n{}", table_schema_simple().0));
878    }
879
880    #[test]
881    fn test_schema_build_failed_on_duplicate_names() {
882        let ret = Schema::builder()
883            .with_schema_id(1)
884            .with_identifier_field_ids(vec![1])
885            .with_fields(vec![
886                NestedField::required(1, "foo", Primitive(PrimitiveType::String)).into(),
887                NestedField::required(2, "bar", Primitive(PrimitiveType::Int)).into(),
888                NestedField::optional(3, "baz", Primitive(PrimitiveType::Boolean)).into(),
889                NestedField::optional(4, "baz", Primitive(PrimitiveType::Boolean)).into(),
890            ])
891            .build();
892
893        assert!(
894            ret.unwrap_err()
895                .message()
896                .contains("Invalid schema: multiple fields for name baz")
897        );
898    }
899
900    #[test]
901    fn test_schema_into_builder() {
902        let original_schema = table_schema_nested();
903        let builder = original_schema.clone().into_builder();
904        let schema = builder.build().unwrap();
905
906        assert_eq!(original_schema, schema);
907    }
908
909    #[test]
910    fn test_schema_index_by_name() {
911        let expected_name_to_id = HashMap::from(
912            [
913                ("foo", 1),
914                ("bar", 2),
915                ("baz", 3),
916                ("qux", 4),
917                ("qux.element", 5),
918                ("quux", 6),
919                ("quux.key", 7),
920                ("quux.value", 8),
921                ("quux.value.key", 9),
922                ("quux.value.value", 10),
923                ("location", 11),
924                ("location.element", 12),
925                ("location.element.latitude", 13),
926                ("location.element.longitude", 14),
927                ("location.latitude", 13),
928                ("location.longitude", 14),
929                ("person", 15),
930                ("person.name", 16),
931                ("person.age", 17),
932            ]
933            .map(|e| (e.0.to_string(), e.1)),
934        );
935
936        let schema = table_schema_nested();
937        assert_eq!(&expected_name_to_id, &schema.name_to_id);
938    }
939
940    #[test]
941    fn test_schema_index_by_name_case_insensitive() {
942        let expected_name_to_id = HashMap::from(
943            [
944                ("fOo", 1),
945                ("Bar", 2),
946                ("BAz", 3),
947                ("quX", 4),
948                ("quX.ELEment", 5),
949                ("qUUx", 6),
950                ("QUUX.KEY", 7),
951                ("QUUX.Value", 8),
952                ("qUUX.VALUE.Key", 9),
953                ("qUux.VaLue.Value", 10),
954                ("lOCAtION", 11),
955                ("LOCAtioN.ELeMENt", 12),
956                ("LoCATion.element.LATitude", 13),
957                ("locatION.ElemeNT.LONgitude", 14),
958                ("LOCAtiON.LATITUDE", 13),
959                ("LOCATION.LONGITUDE", 14),
960                ("PERSon", 15),
961                ("PERSON.Name", 16),
962                ("peRSON.AGe", 17),
963            ]
964            .map(|e| (e.0.to_string(), e.1)),
965        );
966
967        let schema = table_schema_nested();
968        for (name, id) in expected_name_to_id {
969            assert_eq!(
970                Some(id),
971                schema.field_by_name_case_insensitive(&name).map(|f| f.id)
972            );
973        }
974    }
975
976    #[test]
977    fn test_schema_find_column_name() {
978        let expected_column_name = HashMap::from([
979            (1, "foo"),
980            (2, "bar"),
981            (3, "baz"),
982            (4, "qux"),
983            (5, "qux.element"),
984            (6, "quux"),
985            (7, "quux.key"),
986            (8, "quux.value"),
987            (9, "quux.value.key"),
988            (10, "quux.value.value"),
989            (11, "location"),
990            (12, "location.element"),
991            (13, "location.element.latitude"),
992            (14, "location.element.longitude"),
993        ]);
994
995        let schema = table_schema_nested();
996        for (id, name) in expected_column_name {
997            assert_eq!(
998                Some(name),
999                schema.name_by_field_id(id),
1000                "Column name for field id {id} not match."
1001            );
1002        }
1003    }
1004
1005    #[test]
1006    fn test_schema_find_column_name_not_found() {
1007        let schema = table_schema_nested();
1008
1009        assert!(schema.name_by_field_id(99).is_none());
1010    }
1011
1012    #[test]
1013    fn test_schema_find_column_name_by_id_simple() {
1014        let expected_id_to_name = HashMap::from([(1, "foo"), (2, "bar"), (3, "baz")]);
1015
1016        let schema = table_schema_simple().0;
1017
1018        for (id, name) in expected_id_to_name {
1019            assert_eq!(
1020                Some(name),
1021                schema.name_by_field_id(id),
1022                "Column name for field id {id} not match."
1023            );
1024        }
1025    }
1026
1027    #[test]
1028    fn test_schema_find_simple() {
1029        let schema = table_schema_simple().0;
1030
1031        assert_eq!(
1032            Some(schema.r#struct.fields()[0].clone()),
1033            schema.field_by_id(1).cloned()
1034        );
1035        assert_eq!(
1036            Some(schema.r#struct.fields()[1].clone()),
1037            schema.field_by_id(2).cloned()
1038        );
1039        assert_eq!(
1040            Some(schema.r#struct.fields()[2].clone()),
1041            schema.field_by_id(3).cloned()
1042        );
1043
1044        assert!(schema.field_by_id(4).is_none());
1045        assert!(schema.field_by_name("non exist").is_none());
1046    }
1047
1048    #[test]
1049    fn test_schema_find_nested() {
1050        let expected_id_to_field: HashMap<i32, NestedField> = HashMap::from([
1051            (
1052                1,
1053                NestedField::optional(1, "foo", Primitive(PrimitiveType::String)),
1054            ),
1055            (
1056                2,
1057                NestedField::required(2, "bar", Primitive(PrimitiveType::Int)),
1058            ),
1059            (
1060                3,
1061                NestedField::optional(3, "baz", Primitive(PrimitiveType::Boolean)),
1062            ),
1063            (
1064                4,
1065                NestedField::required(
1066                    4,
1067                    "qux",
1068                    List(ListType {
1069                        element_field: NestedField::list_element(
1070                            5,
1071                            Primitive(PrimitiveType::String),
1072                            true,
1073                        )
1074                        .into(),
1075                    }),
1076                ),
1077            ),
1078            (
1079                5,
1080                NestedField::required(5, "element", Primitive(PrimitiveType::String)),
1081            ),
1082            (
1083                6,
1084                NestedField::required(
1085                    6,
1086                    "quux",
1087                    Map(MapType {
1088                        key_field: NestedField::map_key_element(
1089                            7,
1090                            Primitive(PrimitiveType::String),
1091                        )
1092                        .into(),
1093                        value_field: NestedField::map_value_element(
1094                            8,
1095                            Map(MapType {
1096                                key_field: NestedField::map_key_element(
1097                                    9,
1098                                    Primitive(PrimitiveType::String),
1099                                )
1100                                .into(),
1101                                value_field: NestedField::map_value_element(
1102                                    10,
1103                                    Primitive(PrimitiveType::Int),
1104                                    true,
1105                                )
1106                                .into(),
1107                            }),
1108                            true,
1109                        )
1110                        .into(),
1111                    }),
1112                ),
1113            ),
1114            (
1115                7,
1116                NestedField::required(7, "key", Primitive(PrimitiveType::String)),
1117            ),
1118            (
1119                8,
1120                NestedField::required(
1121                    8,
1122                    "value",
1123                    Map(MapType {
1124                        key_field: NestedField::map_key_element(
1125                            9,
1126                            Primitive(PrimitiveType::String),
1127                        )
1128                        .into(),
1129                        value_field: NestedField::map_value_element(
1130                            10,
1131                            Primitive(PrimitiveType::Int),
1132                            true,
1133                        )
1134                        .into(),
1135                    }),
1136                ),
1137            ),
1138            (
1139                9,
1140                NestedField::required(9, "key", Primitive(PrimitiveType::String)),
1141            ),
1142            (
1143                10,
1144                NestedField::required(10, "value", Primitive(PrimitiveType::Int)),
1145            ),
1146            (
1147                11,
1148                NestedField::required(
1149                    11,
1150                    "location",
1151                    List(ListType {
1152                        element_field: NestedField::list_element(
1153                            12,
1154                            Struct(StructType::new(vec![
1155                                NestedField::optional(
1156                                    13,
1157                                    "latitude",
1158                                    Primitive(PrimitiveType::Float),
1159                                )
1160                                .into(),
1161                                NestedField::optional(
1162                                    14,
1163                                    "longitude",
1164                                    Primitive(PrimitiveType::Float),
1165                                )
1166                                .into(),
1167                            ])),
1168                            true,
1169                        )
1170                        .into(),
1171                    }),
1172                ),
1173            ),
1174            (
1175                12,
1176                NestedField::list_element(
1177                    12,
1178                    Struct(StructType::new(vec![
1179                        NestedField::optional(13, "latitude", Primitive(PrimitiveType::Float))
1180                            .into(),
1181                        NestedField::optional(14, "longitude", Primitive(PrimitiveType::Float))
1182                            .into(),
1183                    ])),
1184                    true,
1185                ),
1186            ),
1187            (
1188                13,
1189                NestedField::optional(13, "latitude", Primitive(PrimitiveType::Float)),
1190            ),
1191            (
1192                14,
1193                NestedField::optional(14, "longitude", Primitive(PrimitiveType::Float)),
1194            ),
1195            (
1196                15,
1197                NestedField::optional(
1198                    15,
1199                    "person",
1200                    Struct(StructType::new(vec![
1201                        NestedField::optional(16, "name", Primitive(PrimitiveType::String)).into(),
1202                        NestedField::required(17, "age", Primitive(PrimitiveType::Int)).into(),
1203                    ])),
1204                ),
1205            ),
1206            (
1207                16,
1208                NestedField::optional(16, "name", Primitive(PrimitiveType::String)),
1209            ),
1210            (
1211                17,
1212                NestedField::required(17, "age", Primitive(PrimitiveType::Int)),
1213            ),
1214        ]);
1215
1216        let schema = table_schema_nested();
1217        for (id, field) in expected_id_to_field {
1218            assert_eq!(
1219                Some(&field),
1220                schema.field_by_id(id).map(|f| f.as_ref()),
1221                "Field for {id} not match."
1222            );
1223        }
1224    }
1225
1226    #[test]
1227    fn test_build_accessors() {
1228        let schema = table_schema_nested();
1229
1230        let test_struct = crate::spec::Struct::from_iter(vec![
1231            Some(Literal::string("foo value")),
1232            Some(Literal::int(1002)),
1233            Some(Literal::bool(true)),
1234            Some(Literal::List(vec![
1235                Some(Literal::string("qux item 1")),
1236                Some(Literal::string("qux item 2")),
1237            ])),
1238            Some(Literal::Map(MapValue::from([(
1239                Literal::string("quux key 1"),
1240                Some(Literal::Map(MapValue::from([(
1241                    Literal::string("quux nested key 1"),
1242                    Some(Literal::int(1000)),
1243                )]))),
1244            )]))),
1245            Some(Literal::List(vec![Some(Literal::Struct(
1246                crate::spec::Struct::from_iter(vec![
1247                    Some(Literal::float(52.509_09_f32)),
1248                    Some(Literal::float(-1.885_249_f32)),
1249                ]),
1250            ))])),
1251            Some(Literal::Struct(crate::spec::Struct::from_iter(vec![
1252                Some(Literal::string("Testy McTest")),
1253                Some(Literal::int(33)),
1254            ]))),
1255        ]);
1256
1257        assert_eq!(
1258            schema
1259                .accessor_by_field_id(1)
1260                .unwrap()
1261                .get(&test_struct)
1262                .unwrap(),
1263            Some(Datum::string("foo value"))
1264        );
1265        assert_eq!(
1266            schema
1267                .accessor_by_field_id(2)
1268                .unwrap()
1269                .get(&test_struct)
1270                .unwrap(),
1271            Some(Datum::int(1002))
1272        );
1273        assert_eq!(
1274            schema
1275                .accessor_by_field_id(3)
1276                .unwrap()
1277                .get(&test_struct)
1278                .unwrap(),
1279            Some(Datum::bool(true))
1280        );
1281        assert_eq!(
1282            schema
1283                .accessor_by_field_id(16)
1284                .unwrap()
1285                .get(&test_struct)
1286                .unwrap(),
1287            Some(Datum::string("Testy McTest"))
1288        );
1289        assert_eq!(
1290            schema
1291                .accessor_by_field_id(17)
1292                .unwrap()
1293                .get(&test_struct)
1294                .unwrap(),
1295            Some(Datum::int(33))
1296        );
1297    }
1298
1299    #[test]
1300    fn test_highest_field_id() {
1301        let schema = table_schema_nested();
1302        assert_eq!(17, schema.highest_field_id());
1303
1304        let schema = table_schema_simple().0;
1305        assert_eq!(3, schema.highest_field_id());
1306    }
1307
1308    #[test]
1309    fn test_highest_field_id_no_fields() {
1310        let schema = Schema::builder().with_schema_id(1).build().unwrap();
1311        assert_eq!(0, schema.highest_field_id());
1312    }
1313
1314    #[test]
1315    fn test_field_ids_must_be_unique() {
1316        let reassigned_schema = Schema::builder()
1317            .with_schema_id(1)
1318            .with_identifier_field_ids(vec![5])
1319            .with_alias(BiHashMap::from_iter(vec![("bar_alias".to_string(), 3)]))
1320            .with_fields(vec![
1321                NestedField::required(5, "foo", Primitive(PrimitiveType::String)).into(),
1322                NestedField::optional(3, "bar", Primitive(PrimitiveType::Int)).into(),
1323                NestedField::optional(3, "baz", Primitive(PrimitiveType::Boolean)).into(),
1324            ])
1325            .build()
1326            .unwrap_err();
1327
1328        assert!(reassigned_schema.message().contains("'field.id' 3"));
1329    }
1330
1331    #[test]
1332    fn test_reassign_ids_empty_schema() {
1333        let schema = Schema::builder().with_schema_id(1).build().unwrap();
1334        let reassigned_schema = schema
1335            .clone()
1336            .into_builder()
1337            .with_reassigned_field_ids(0)
1338            .build()
1339            .unwrap();
1340
1341        assert_eq!(schema, reassigned_schema);
1342        assert_eq!(schema.highest_field_id(), 0);
1343    }
1344
1345    #[test]
1346    fn test_identifier_field_ids() {
1347        // field in map
1348        assert!(
1349            Schema::builder()
1350                .with_schema_id(1)
1351                .with_identifier_field_ids(vec![2])
1352                .with_fields(vec![
1353                    NestedField::required(
1354                        1,
1355                        "Map",
1356                        Map(MapType::new(
1357                            NestedField::map_key_element(2, Primitive(PrimitiveType::String))
1358                                .into(),
1359                            NestedField::map_value_element(
1360                                3,
1361                                Primitive(PrimitiveType::Boolean),
1362                                true,
1363                            )
1364                            .into(),
1365                        )),
1366                    )
1367                    .into()
1368                ])
1369                .build()
1370                .is_err()
1371        );
1372        assert!(
1373            Schema::builder()
1374                .with_schema_id(1)
1375                .with_identifier_field_ids(vec![3])
1376                .with_fields(vec![
1377                    NestedField::required(
1378                        1,
1379                        "Map",
1380                        Map(MapType::new(
1381                            NestedField::map_key_element(2, Primitive(PrimitiveType::String))
1382                                .into(),
1383                            NestedField::map_value_element(
1384                                3,
1385                                Primitive(PrimitiveType::Boolean),
1386                                true,
1387                            )
1388                            .into(),
1389                        )),
1390                    )
1391                    .into()
1392                ])
1393                .build()
1394                .is_err()
1395        );
1396
1397        // field in list
1398        assert!(
1399            Schema::builder()
1400                .with_schema_id(1)
1401                .with_identifier_field_ids(vec![2])
1402                .with_fields(vec![
1403                    NestedField::required(
1404                        1,
1405                        "List",
1406                        List(ListType::new(
1407                            NestedField::list_element(2, Primitive(PrimitiveType::String), true)
1408                                .into(),
1409                        )),
1410                    )
1411                    .into()
1412                ])
1413                .build()
1414                .is_err()
1415        );
1416
1417        // field in optional struct
1418        assert!(
1419            Schema::builder()
1420                .with_schema_id(1)
1421                .with_identifier_field_ids(vec![2])
1422                .with_fields(vec![
1423                    NestedField::optional(
1424                        1,
1425                        "Struct",
1426                        Struct(StructType::new(vec![
1427                            NestedField::required(2, "name", Primitive(PrimitiveType::String))
1428                                .into(),
1429                            NestedField::optional(3, "age", Primitive(PrimitiveType::Int)).into(),
1430                        ])),
1431                    )
1432                    .into()
1433                ])
1434                .build()
1435                .is_err()
1436        );
1437
1438        // float and double
1439        assert!(
1440            Schema::builder()
1441                .with_schema_id(1)
1442                .with_identifier_field_ids(vec![1])
1443                .with_fields(vec![
1444                    NestedField::required(1, "Float", Primitive(PrimitiveType::Float),).into()
1445                ])
1446                .build()
1447                .is_err()
1448        );
1449        assert!(
1450            Schema::builder()
1451                .with_schema_id(1)
1452                .with_identifier_field_ids(vec![1])
1453                .with_fields(vec![
1454                    NestedField::required(1, "Double", Primitive(PrimitiveType::Double),).into()
1455                ])
1456                .build()
1457                .is_err()
1458        );
1459
1460        // optional field
1461        assert!(
1462            Schema::builder()
1463                .with_schema_id(1)
1464                .with_identifier_field_ids(vec![1])
1465                .with_fields(vec![
1466                    NestedField::required(1, "Required", Primitive(PrimitiveType::String),).into()
1467                ])
1468                .build()
1469                .is_ok()
1470        );
1471        assert!(
1472            Schema::builder()
1473                .with_schema_id(1)
1474                .with_identifier_field_ids(vec![1])
1475                .with_fields(vec![
1476                    NestedField::optional(1, "Optional", Primitive(PrimitiveType::String),).into()
1477                ])
1478                .build()
1479                .is_err()
1480        );
1481    }
1482
1483    #[test]
1484    fn test_unknown_type_deserialization_rejects_non_null_default() {
1485        let field_json = serde_json::json!({
1486            "id": 1,
1487            "name": "empty",
1488            "required": false,
1489            "type": "unknown",
1490            "initial-default": 1
1491        });
1492
1493        let error = serde_json::from_value::<NestedField>(field_json.clone()).unwrap_err();
1494        assert!(
1495            error
1496                .to_string()
1497                .contains("Unknown type only supports null default values"),
1498            "unexpected error: {error}"
1499        );
1500
1501        let schema_json = serde_json::json!({
1502            "type": "struct",
1503            "schema-id": 1,
1504            "fields": [field_json]
1505        });
1506        assert!(serde_json::from_value::<Schema>(schema_json).is_err());
1507    }
1508
1509    #[test]
1510    fn test_unknown_type_deserialization_accepts_null_defaults() {
1511        let schema_json = serde_json::json!({
1512            "type": "struct",
1513            "schema-id": 1,
1514            "fields": [
1515                {
1516                    "id": 1,
1517                    "name": "empty",
1518                    "required": false,
1519                    "type": "unknown",
1520                    "initial-default": null,
1521                    "write-default": null
1522                }
1523            ]
1524        });
1525
1526        serde_json::from_value::<Schema>(schema_json).unwrap();
1527    }
1528
1529    #[test]
1530    fn test_unknown_type_accepts_null_container_defaults() {
1531        let cases = [
1532            (
1533                "struct",
1534                Struct(StructType::new(vec![
1535                    NestedField::optional(2, "empty", Primitive(PrimitiveType::Unknown)).into(),
1536                ])),
1537                Literal::Struct(crate::spec::Struct::from_iter([None])),
1538            ),
1539            (
1540                "list",
1541                List(ListType::new(
1542                    NestedField::list_element(2, Primitive(PrimitiveType::Unknown), false).into(),
1543                )),
1544                Literal::List(vec![None]),
1545            ),
1546            (
1547                "map",
1548                Map(MapType::optional(
1549                    2,
1550                    Primitive(PrimitiveType::String),
1551                    3,
1552                    Primitive(PrimitiveType::Unknown),
1553                )),
1554                Literal::Map(MapValue::from([(Literal::string("key"), None)])),
1555            ),
1556        ];
1557
1558        for (name, field_type, default) in cases {
1559            let schema = Schema::builder()
1560                .with_schema_id(1)
1561                .with_fields(vec![
1562                    NestedField::optional(1, name, field_type)
1563                        .with_write_default(default)
1564                        .into(),
1565                ])
1566                .build()
1567                .unwrap();
1568            serde_json::to_value(schema).unwrap();
1569        }
1570    }
1571
1572    #[test]
1573    fn test_unknown_type_deserialization_rejects_non_null_container_defaults() {
1574        let cases = [
1575            (
1576                "struct",
1577                serde_json::json!({
1578                    "type": "struct",
1579                    "fields": [{
1580                        "id": 2,
1581                        "name": "empty",
1582                        "required": false,
1583                        "type": "unknown"
1584                    }]
1585                }),
1586                serde_json::json!({"2": 1}),
1587            ),
1588            (
1589                "list",
1590                serde_json::json!({
1591                    "type": "list",
1592                    "element-id": 2,
1593                    "element-required": false,
1594                    "element": "unknown"
1595                }),
1596                serde_json::json!([1]),
1597            ),
1598            (
1599                "map",
1600                serde_json::json!({
1601                    "type": "map",
1602                    "key-id": 2,
1603                    "key": "string",
1604                    "value-id": 3,
1605                    "value-required": false,
1606                    "value": "unknown"
1607                }),
1608                serde_json::json!({"keys": ["key"], "values": [1]}),
1609            ),
1610        ];
1611
1612        for (name, field_type, default) in cases {
1613            let schema_json = serde_json::json!({
1614                "type": "struct",
1615                "schema-id": 1,
1616                "fields": [{
1617                    "id": 1,
1618                    "name": name,
1619                    "required": false,
1620                    "type": field_type,
1621                    "initial-default": default
1622                }]
1623            });
1624
1625            assert!(
1626                serde_json::from_value::<Schema>(schema_json).is_err(),
1627                "non-null unknown default in {name} should be rejected"
1628            );
1629        }
1630    }
1631}
iceberg/spec/schema/mod.rs

iceberg/spec/schema/
mod.rs
