Skip to main content

iceberg/spec/schema/
mod.rs

1// Licensed to the Apache Software Foundation (ASF) under one
2// or more contributor license agreements.  See the NOTICE file
3// distributed with this work for additional information
4// regarding copyright ownership.  The ASF licenses this file
5// to you under the Apache License, Version 2.0 (the
6// "License"); you may not use this file except in compliance
7// with the License.  You may obtain a copy of the License at
8//
9//   http://www.apache.org/licenses/LICENSE-2.0
10//
11// Unless required by applicable law or agreed to in writing,
12// software distributed under the License is distributed on an
13// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14// KIND, either express or implied.  See the License for the
15// specific language governing permissions and limitations
16// under the License.
17
18//! This module defines schema in iceberg.
19
20use std::collections::{HashMap, HashSet};
21use std::fmt::{Display, Formatter};
22use std::sync::Arc;
23
24mod utils;
25mod visitor;
26pub use self::visitor::*;
27pub(super) mod _serde;
28mod assign_fresh_ids;
29mod id_reassigner;
30mod index;
31mod prune_columns;
32use bimap::BiHashMap;
33use itertools::{Itertools, zip_eq};
34use serde::{Deserialize, Serialize};
35
36use self::_serde::SchemaEnum;
37pub(crate) use self::assign_fresh_ids::assign_fresh_ids;
38use self::id_reassigner::ReassignFieldIds;
39use self::index::{IndexByName, index_by_id, index_parents};
40pub use self::prune_columns::prune_columns;
41use super::NestedField;
42use crate::error::{Result, invalid_data};
43use crate::expr::accessor::StructAccessor;
44use crate::spec::FormatVersion;
45use crate::spec::datatypes::{
46    LIST_FIELD_NAME, ListType, MAP_KEY_FIELD_NAME, MAP_VALUE_FIELD_NAME, MapType, NestedFieldRef,
47    PrimitiveType, StructType, Type,
48};
49use crate::{Error, ErrorKind, ensure_data_valid};
50
51/// Type alias for schema id.
52pub type SchemaId = i32;
53/// Reference to [`Schema`].
54pub type SchemaRef = Arc<Schema>;
55/// Default schema id.
56pub const DEFAULT_SCHEMA_ID: SchemaId = 0;
57/// Delimiter for schema name, which denotes a nested struct.
58pub const SCHEMA_NAME_DELIMITER: &str = ".";
59/// Minimum format version that allows non-null field default values.
60/// Mirrors Java's `Schema.DEFAULT_VALUES_MIN_FORMAT_VERSION`.
61pub(crate) const DEFAULT_VALUES_MIN_FORMAT_VERSION: FormatVersion = FormatVersion::V3;
62
63/// Defines schema in iceberg.
64#[derive(Debug, Serialize, Deserialize, Clone)]
65#[serde(try_from = "SchemaEnum", into = "SchemaEnum")]
66pub struct Schema {
67    r#struct: StructType,
68    schema_id: SchemaId,
69    highest_field_id: i32,
70    identifier_field_ids: HashSet<i32>,
71
72    alias_to_id: BiHashMap<String, i32>,
73    id_to_field: HashMap<i32, NestedFieldRef>,
74
75    name_to_id: HashMap<String, i32>,
76    // `None` marks a lower-cased name that resolves to more than one field.
77    lowercase_name_to_id: HashMap<String, Option<i32>>,
78    id_to_name: HashMap<i32, String>,
79
80    field_id_to_accessor: HashMap<i32, Arc<StructAccessor>>,
81}
82
83impl PartialEq for Schema {
84    fn eq(&self, other: &Self) -> bool {
85        self.r#struct == other.r#struct
86            && self.schema_id == other.schema_id
87            && self.identifier_field_ids == other.identifier_field_ids
88    }
89}
90
91impl Eq for Schema {}
92
93/// Schema builder.
94#[derive(Debug)]
95pub struct SchemaBuilder {
96    schema_id: i32,
97    fields: Vec<NestedFieldRef>,
98    alias_to_id: BiHashMap<String, i32>,
99    identifier_field_ids: HashSet<i32>,
100    reassign_field_ids_from: Option<i32>,
101}
102
103impl SchemaBuilder {
104    /// Add fields to schema builder.
105    pub fn with_fields(mut self, fields: impl IntoIterator<Item = NestedFieldRef>) -> Self {
106        self.fields.extend(fields);
107        self
108    }
109
110    /// Reassign all field-ids (including nested) on build.
111    /// Reassignment starts from the field-id specified in `start_from` (inclusive).
112    ///
113    /// All specified aliases and identifier fields will be updated to the new field-ids.
114    pub(crate) fn with_reassigned_field_ids(mut self, start_from: i32) -> Self {
115        self.reassign_field_ids_from = Some(start_from);
116        self
117    }
118
119    /// Set schema id.
120    pub fn with_schema_id(mut self, schema_id: i32) -> Self {
121        self.schema_id = schema_id;
122        self
123    }
124
125    /// Set identifier field ids.
126    pub fn with_identifier_field_ids(mut self, ids: impl IntoIterator<Item = i32>) -> Self {
127        self.identifier_field_ids.extend(ids);
128        self
129    }
130
131    /// Set alias to filed id mapping.
132    pub fn with_alias(mut self, alias_to_id: BiHashMap<String, i32>) -> Self {
133        self.alias_to_id = alias_to_id;
134        self
135    }
136
137    /// Builds the schema.
138    pub fn build(self) -> Result<Schema> {
139        let field_id_to_accessor = self.build_accessors();
140
141        let r#struct = StructType::new(self.fields);
142        let id_to_field = index_by_id(&r#struct)?;
143
144        Self::validate_identifier_ids(
145            &r#struct,
146            &id_to_field,
147            self.identifier_field_ids.iter().copied(),
148        )?;
149
150        let (name_to_id, id_to_name) = {
151            let mut index = IndexByName::default();
152            visit_struct(&r#struct, &mut index)?;
153            index.indexes()
154        };
155
156        let mut lowercase_name_to_id = HashMap::new();
157        for (name, field_id) in &name_to_id {
158            lowercase_name_to_id
159                .entry(name.to_lowercase())
160                .and_modify(|existing| *existing = None)
161                .or_insert(Some(*field_id));
162        }
163
164        let highest_field_id = id_to_field.keys().max().cloned().unwrap_or(0);
165
166        let mut schema = Schema {
167            r#struct,
168            schema_id: self.schema_id,
169            highest_field_id,
170            identifier_field_ids: self.identifier_field_ids,
171            alias_to_id: self.alias_to_id,
172            id_to_field,
173
174            name_to_id,
175            lowercase_name_to_id,
176            id_to_name,
177
178            field_id_to_accessor,
179        };
180
181        if let Some(start_from) = self.reassign_field_ids_from {
182            let mut id_reassigner = ReassignFieldIds::new(start_from);
183            let new_fields = id_reassigner.reassign_field_ids(schema.r#struct.fields().to_vec())?;
184            let new_identifier_field_ids =
185                id_reassigner.apply_to_identifier_fields(schema.identifier_field_ids)?;
186            let new_alias_to_id = id_reassigner.apply_to_aliases(schema.alias_to_id.clone())?;
187
188            schema = Schema::builder()
189                .with_schema_id(schema.schema_id)
190                .with_fields(new_fields)
191                .with_identifier_field_ids(new_identifier_field_ids)
192                .with_alias(new_alias_to_id)
193                .build()?;
194        }
195
196        Ok(schema)
197    }
198
199    fn build_accessors(&self) -> HashMap<i32, Arc<StructAccessor>> {
200        let mut map = HashMap::new();
201
202        for (pos, field) in self.fields.iter().enumerate() {
203            match field.field_type.as_ref() {
204                Type::Primitive(prim_type) => {
205                    // add an accessor for this field
206                    let accessor = Arc::new(StructAccessor::new(pos, prim_type.clone()));
207                    map.insert(field.id, accessor.clone());
208                }
209
210                Type::Struct(nested) => {
211                    // add accessors for nested fields
212                    for (field_id, accessor) in Self::build_accessors_nested(nested.fields()) {
213                        let new_accessor = Arc::new(StructAccessor::wrap(pos, accessor));
214                        map.insert(field_id, new_accessor.clone());
215                    }
216                }
217                _ => {
218                    // Accessors don't get built for Map or List types
219                }
220            }
221        }
222
223        map
224    }
225
226    fn build_accessors_nested(fields: &[NestedFieldRef]) -> Vec<(i32, Box<StructAccessor>)> {
227        let mut results = vec![];
228        for (pos, field) in fields.iter().enumerate() {
229            match field.field_type.as_ref() {
230                Type::Primitive(prim_type) => {
231                    let accessor = Box::new(StructAccessor::new(pos, prim_type.clone()));
232                    results.push((field.id, accessor));
233                }
234                Type::Struct(nested) => {
235                    let nested_accessors = Self::build_accessors_nested(nested.fields());
236
237                    let wrapped_nested_accessors =
238                        nested_accessors.into_iter().map(|(id, accessor)| {
239                            let new_accessor = Box::new(StructAccessor::wrap(pos, accessor));
240                            (id, new_accessor.clone())
241                        });
242
243                    results.extend(wrapped_nested_accessors);
244                }
245                _ => {
246                    // Accessors don't get built for Map or List types
247                }
248            }
249        }
250
251        results
252    }
253
254    /// According to [the spec](https://iceberg.apache.org/spec/#identifier-fields), the identifier fields
255    /// must meet the following requirements:
256    /// - Float, double, and optional fields cannot be used as identifier fields.
257    /// - Identifier fields may be nested in structs but cannot be nested within maps or lists.
258    /// - A nested field cannot be used as an identifier field if it is nested in an optional struct, to avoid null values in identifiers.
259    fn validate_identifier_ids(
260        r#struct: &StructType,
261        id_to_field: &HashMap<i32, NestedFieldRef>,
262        identifier_field_ids: impl Iterator<Item = i32>,
263    ) -> Result<()> {
264        let id_to_parent = index_parents(r#struct)?;
265        for identifier_field_id in identifier_field_ids {
266            let field = id_to_field.get(&identifier_field_id).ok_or_else(|| {
267                invalid_data!(
268                    "Cannot add identifier field {identifier_field_id}: field does not exist"
269                )
270            })?;
271            ensure_data_valid!(
272                field.required,
273                "Cannot add identifier field: {} is an optional field",
274                field.name
275            );
276            if let Type::Primitive(p) = field.field_type.as_ref() {
277                ensure_data_valid!(
278                    !matches!(p, PrimitiveType::Double | PrimitiveType::Float),
279                    "Cannot add identifier field {}: cannot be a float or double type",
280                    field.name
281                );
282            } else {
283                return Err(invalid_data!(
284                    "Cannot add field {} as an identifier field: not a primitive type field",
285                    field.name
286                ));
287            }
288
289            let mut cur_field_id = identifier_field_id;
290            while let Some(parent) = id_to_parent.get(&cur_field_id) {
291                let parent_field = id_to_field
292                    .get(parent)
293                    .expect("Field id should not disappear.");
294                ensure_data_valid!(
295                    parent_field.field_type.is_struct(),
296                    "Cannot add field {} as an identifier field: must not be nested in {:?}",
297                    field.name,
298                    parent_field
299                );
300                ensure_data_valid!(
301                    parent_field.required,
302                    "Cannot add field {} as an identifier field: must not be nested in an optional field {}",
303                    field.name,
304                    parent_field
305                );
306                cur_field_id = *parent;
307            }
308        }
309
310        Ok(())
311    }
312}
313
314impl Schema {
315    /// Create a schema builder.
316    pub fn builder() -> SchemaBuilder {
317        SchemaBuilder {
318            schema_id: DEFAULT_SCHEMA_ID,
319            fields: vec![],
320            identifier_field_ids: HashSet::default(),
321            alias_to_id: BiHashMap::default(),
322            reassign_field_ids_from: None,
323        }
324    }
325
326    /// Create a new schema builder from a schema.
327    pub fn into_builder(self) -> SchemaBuilder {
328        SchemaBuilder {
329            schema_id: self.schema_id,
330            fields: self.r#struct.fields().to_vec(),
331            alias_to_id: self.alias_to_id,
332            identifier_field_ids: self.identifier_field_ids,
333            reassign_field_ids_from: None,
334        }
335    }
336
337    /// Get field by field id.
338    pub fn field_by_id(&self, field_id: i32) -> Option<&NestedFieldRef> {
339        self.id_to_field.get(&field_id)
340    }
341
342    /// Get field by field name.
343    ///
344    /// Both full name and short name could work here.
345    pub fn field_by_name(&self, field_name: &str) -> Option<&NestedFieldRef> {
346        self.name_to_id
347            .get(field_name)
348            .and_then(|id| self.field_by_id(*id))
349    }
350
351    /// Get field by field name in a case-insensitive way.
352    ///
353    /// Both full name and short name could work here.
354    /// Returns `None` if the name is missing or matches multiple fields case-insensitively.
355    /// Use [`Self::field_by_name_case_insensitive_checked`] to distinguish those cases.
356    pub fn field_by_name_case_insensitive(&self, field_name: &str) -> Option<&NestedFieldRef> {
357        self.lowercase_name_to_id
358            .get(&field_name.to_lowercase())
359            .copied()
360            .flatten()
361            .and_then(|id| self.field_by_id(id))
362    }
363
364    /// Get field by field name in a case-insensitive way, reporting ambiguous names.
365    ///
366    /// Both full name and short name could work here. Returns `Ok(None)` if the name is
367    /// missing, or an error if it matches multiple fields case-insensitively. A collision
368    /// on another name does not prevent this name from resolving. This differs from Iceberg
369    /// Java, which rejects all case-insensitive lookups after a collision, and PyIceberg,
370    /// which silently keeps one of the colliding fields.
371    pub fn field_by_name_case_insensitive_checked(
372        &self,
373        field_name: &str,
374    ) -> Result<Option<&NestedFieldRef>> {
375        let lowercase_name = field_name.to_lowercase();
376        match self.lowercase_name_to_id.get(&lowercase_name) {
377            Some(Some(id)) => Ok(self.field_by_id(*id)),
378            Some(None) => {
379                let mut colliders: Vec<&str> = self
380                    .name_to_id
381                    .keys()
382                    .filter(|name| name.to_lowercase() == lowercase_name)
383                    .map(String::as_str)
384                    .collect();
385                colliders.sort_unstable();
386                Err(invalid_data!(
387                    "Multiple fields match {field_name} case-insensitively: {colliders:?}"
388                ))
389            }
390            None => Ok(None),
391        }
392    }
393
394    /// Get field by alias.
395    pub fn field_by_alias(&self, alias: &str) -> Option<&NestedFieldRef> {
396        self.alias_to_id
397            .get_by_left(alias)
398            .and_then(|id| self.field_by_id(*id))
399    }
400
401    /// Returns the highest field ID assigned in this schema.
402    #[inline]
403    pub fn highest_field_id(&self) -> i32 {
404        self.highest_field_id
405    }
406
407    /// Returns the Schema's ID.
408    #[inline]
409    pub fn schema_id(&self) -> SchemaId {
410        self.schema_id
411    }
412
413    /// Returns the type for the struct.
414    #[inline]
415    pub fn as_struct(&self) -> &StructType {
416        &self.r#struct
417    }
418
419    /// Returns the IDs of the fields.
420    #[inline]
421    pub fn identifier_field_ids(&self) -> impl ExactSizeIterator<Item = i32> + '_ {
422        self.identifier_field_ids.iter().copied()
423    }
424
425    /// Get field id by full name.
426    pub fn field_id_by_name(&self, name: &str) -> Option<i32> {
427        self.name_to_id.get(name).copied()
428    }
429
430    /// Get full name by field id.
431    pub fn name_by_field_id(&self, field_id: i32) -> Option<&str> {
432        self.id_to_name.get(&field_id).map(String::as_str)
433    }
434
435    /// Get an accessor for retrieving data in a struct
436    pub fn accessor_by_field_id(&self, field_id: i32) -> Option<Arc<StructAccessor>> {
437        self.field_id_to_accessor.get(&field_id).cloned()
438    }
439
440    /// Check if this schema is identical to another schema semantically - excluding schema id.
441    pub(crate) fn is_same_schema(&self, other: &SchemaRef) -> bool {
442        self.as_struct().eq(other.as_struct())
443            && self.identifier_field_ids().eq(other.identifier_field_ids())
444    }
445
446    /// Change the schema id of this schema.
447    // This is redundant with the `with_schema_id` method on the builder, but useful
448    // as it is infallible in contrast to the builder `build()` method.
449    pub(crate) fn with_schema_id(self, schema_id: SchemaId) -> Self {
450        Self { schema_id, ..self }
451    }
452
453    /// Return A HashMap matching field ids to field names.
454    pub fn field_id_to_name_map(&self) -> &HashMap<i32, String> {
455        &self.id_to_name
456    }
457
458    /// Return a hashmap matching field ids to nested fields.
459    pub fn field_id_to_fields(&self) -> &HashMap<i32, NestedFieldRef> {
460        &self.id_to_field
461    }
462
463    /// Minimum [`FormatVersion`] required to represent all *types* in this schema.
464    ///
465    /// Walks the whole schema — every field, nested ones included — and returns the
466    /// highest per-type floor, so this is O(fields) rather than a cheap field read.
467    /// Types only; for initial-default version floors see [`Schema::check_format_compatibility`].
468    pub fn calc_min_compatible_format(&self) -> FormatVersion {
469        // `id_to_field` is flattened, so the max over all fields covers nested ones too.
470        self.id_to_field
471            .values()
472            .map(|f| f.field_type.min_format_version())
473            .max()
474            .unwrap_or(FormatVersion::V1)
475    }
476
477    /// Returns an error listing every field incompatible with `format_version`.
478    /// Mirrors Java's `Schema.checkCompatibility()`. Two checks per field:
479    ///
480    /// - **Type** — minimum format version required to support that type, without
481    ///   taking nested field types into account.
482    /// - **Initial default** — a non-null `initial_default` backfills pre-existing rows,
483    ///   so it requires `DEFAULT_VALUES_MIN_FORMAT_VERSION`; `write_default` is not
484    ///   checked, as it only affects newly written rows (read identically at any version).
485    pub fn check_format_compatibility(&self, format_version: FormatVersion) -> Result<()> {
486        // (field id, message); sorted by id below for a deterministic error.
487        let mut problems: Vec<(i32, String)> = Vec::new();
488
489        // `id_to_field` is flattened, so checking each field by its own type keeps the
490        // blame on the offending leaf, not its container (mirrors Java's `lazyIdToField`).
491        for field in self.id_to_field.values() {
492            let min_version = field.field_type.min_format_version();
493            if format_version < min_version {
494                // Every id in `id_to_field` is also indexed in `id_to_name`; a miss means
495                // the schema's indexes are inconsistent (a bug), so surface it rather than
496                // guessing an unqualified name.
497                let name = self.name_by_field_id(field.id).ok_or_else(|| {
498                    Error::new(
499                        ErrorKind::Unexpected,
500                        format!(
501                            "Field id {} is missing from the schema's name index",
502                            field.id
503                        ),
504                    )
505                })?;
506                problems.push((field.id, format!(
507                    "Invalid type for {name}: {} is not supported until {min_version} but format version is {format_version}.",
508                    field.field_type,
509                )));
510            }
511
512            if let Some(default) = &field.initial_default
513                && format_version < DEFAULT_VALUES_MIN_FORMAT_VERSION
514            {
515                let name = self.name_by_field_id(field.id).ok_or_else(|| {
516                    Error::new(
517                        ErrorKind::Unexpected,
518                        format!(
519                            "Field id {} is missing from the schema's name index",
520                            field.id
521                        ),
522                    )
523                })?;
524                problems.push((field.id, format!(
525                    "Invalid initial default for {name}: non-null default ({default:?}) is not supported until {DEFAULT_VALUES_MIN_FORMAT_VERSION} but format version is {format_version}."
526                )));
527            }
528        }
529
530        if problems.is_empty() {
531            return Ok(());
532        }
533
534        // Stable sort by id: HashMap order is nondeterministic, and stability keeps a
535        // field's type problem before its default problem (matches Java's TreeMap order).
536        let message = problems
537            .into_iter()
538            .sorted_by_key(|(id, _)| *id)
539            .map(|(_, msg)| msg)
540            .join("\n- ");
541        Err(invalid_data!(
542            "Invalid schema for {format_version}:\n- {message}"
543        ))
544    }
545}
546
547impl Display for Schema {
548    fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result {
549        writeln!(f, "table {{")?;
550        for field in self.as_struct().fields() {
551            writeln!(f, "  {field}")?;
552        }
553        writeln!(f, "}}")
554    }
555}
556
557#[cfg(test)]
558mod tests {
559    use std::collections::HashMap;
560
561    use bimap::BiHashMap;
562
563    use crate::spec::datatypes::Type::{List, Map, Primitive, Struct, Variant};
564    use crate::spec::datatypes::{
565        ListType, MapType, NestedField, NestedFieldRef, PrimitiveType, StructType,
566    };
567    use crate::spec::schema::Schema;
568    use crate::spec::values::Map as MapValue;
569    use crate::spec::{Datum, Literal};
570
571    #[test]
572    fn test_check_format_compatibility() {
573        use crate::spec::{FormatVersion, PrimitiveLiteral, VariantType};
574
575        fn schema_with(fields: Vec<NestedFieldRef>) -> Schema {
576            Schema::builder().with_fields(fields).build().unwrap()
577        }
578
579        // Variant type requires v3.
580        let variant = schema_with(vec![
581            NestedField::optional(1, "v", Variant(VariantType)).into(),
582        ]);
583        assert!(
584            variant
585                .check_format_compatibility(FormatVersion::V2)
586                .is_err()
587        );
588        assert!(
589            variant
590                .check_format_compatibility(FormatVersion::V3)
591                .is_ok()
592        );
593
594        // A non-null initial default requires v3, even for a v1-compatible type.
595        let with_default = schema_with(vec![
596            NestedField::optional(1, "a", Primitive(PrimitiveType::Int))
597                .with_initial_default(Literal::Primitive(PrimitiveLiteral::Int(1)))
598                .into(),
599        ]);
600        let err = with_default
601            .check_format_compatibility(FormatVersion::V2)
602            .unwrap_err();
603        assert!(
604            err.message().contains("Invalid initial default for a"),
605            "{err}"
606        );
607        assert!(
608            with_default
609                .check_format_compatibility(FormatVersion::V3)
610                .is_ok()
611        );
612
613        // No default is fine at any version (an absent/null default never trips).
614        let no_default = schema_with(vec![
615            NestedField::optional(1, "a", Primitive(PrimitiveType::Int)).into(),
616        ]);
617        assert!(
618            no_default
619                .check_format_compatibility(FormatVersion::V1)
620                .is_ok()
621        );
622
623        // Recursion: a default on a field nested inside a struct is caught.
624        let nested = schema_with(vec![
625            NestedField::required(
626                1,
627                "s",
628                Struct(StructType::new(vec![
629                    NestedField::optional(2, "inner", Primitive(PrimitiveType::Long))
630                        .with_initial_default(Literal::Primitive(PrimitiveLiteral::Long(7)))
631                        .into(),
632                ])),
633            )
634            .into(),
635        ]);
636        let err = nested
637            .check_format_compatibility(FormatVersion::V2)
638            .unwrap_err();
639        assert!(err.message().contains("inner"), "{err}");
640
641        // A v3-only type nested in a container blames the leaf, never the container.
642        let nested_variant = schema_with(vec![
643            NestedField::required(
644                1,
645                "container",
646                Struct(StructType::new(vec![
647                    NestedField::optional(2, "v", Variant(VariantType)).into(),
648                ])),
649            )
650            .into(),
651        ]);
652        let err = nested_variant
653            .check_format_compatibility(FormatVersion::V2)
654            .unwrap_err();
655        assert!(err.message().contains("container.v"), "{err}");
656        assert!(
657            !err.message().contains("Invalid type for container:"),
658            "container must not be blamed: {err}"
659        );
660    }
661
662    #[test]
663    fn test_calc_min_compatible_format() {
664        use crate::spec::{FormatVersion, GeometryType, VariantType};
665
666        fn schema_with(fields: Vec<NestedFieldRef>) -> Schema {
667            Schema::builder().with_fields(fields).build().unwrap()
668        }
669
670        // All v1-compatible types → V1.
671        let v1 = schema_with(vec![
672            NestedField::required(1, "a", Primitive(PrimitiveType::Int)).into(),
673            NestedField::optional(2, "b", Primitive(PrimitiveType::String)).into(),
674        ]);
675        assert_eq!(v1.calc_min_compatible_format(), FormatVersion::V1);
676
677        // A top-level variant → V3.
678        let variant = schema_with(vec![
679            NestedField::optional(1, "v", Variant(VariantType)).into(),
680        ]);
681        assert_eq!(variant.calc_min_compatible_format(), FormatVersion::V3);
682
683        // Unknown is a v3-only primitive type.
684        let unknown = schema_with(vec![
685            NestedField::optional(1, "u", Primitive(PrimitiveType::Unknown)).into(),
686        ]);
687        assert_eq!(unknown.calc_min_compatible_format(), FormatVersion::V3);
688        assert!(
689            unknown
690                .check_format_compatibility(FormatVersion::V2)
691                .is_err()
692        );
693
694        // Geospatial types are v3-only.
695        let geometry = schema_with(vec![
696            NestedField::optional(
697                1,
698                "geom",
699                Primitive(PrimitiveType::Geometry(GeometryType::default())),
700            )
701            .into(),
702        ]);
703        assert_eq!(geometry.calc_min_compatible_format(), FormatVersion::V3);
704        assert!(
705            geometry
706                .check_format_compatibility(FormatVersion::V2)
707                .is_err()
708        );
709
710        // A v3-only type nested inside a list inside a struct → V3 (flattened fields).
711        let nested = schema_with(vec![
712            NestedField::required(
713                1,
714                "s",
715                Struct(StructType::new(vec![
716                    NestedField::optional(
717                        2,
718                        "l",
719                        List(ListType::new(
720                            NestedField::required(
721                                3,
722                                "element",
723                                Primitive(PrimitiveType::TimestampNs),
724                            )
725                            .into(),
726                        )),
727                    )
728                    .into(),
729                ])),
730            )
731            .into(),
732        ]);
733        assert_eq!(nested.calc_min_compatible_format(), FormatVersion::V3);
734    }
735
736    #[test]
737    fn test_check_format_compatibility_message_order() {
738        use crate::spec::{FormatVersion, PrimitiveLiteral, VariantType};
739
740        // Fields declared out of id order; field 2 fails both checks (ts_ns type
741        // AND a non-null initial default). The error must list problems by field id,
742        // with the type problem before the initial-default problem for field 2.
743        let schema = Schema::builder()
744            .with_fields(vec![
745                NestedField::optional(3, "c", Variant(VariantType)).into(),
746                NestedField::optional(2, "b", Primitive(PrimitiveType::TimestampNs))
747                    .with_initial_default(Literal::Primitive(PrimitiveLiteral::Long(0)))
748                    .into(),
749                NestedField::required(1, "a", Primitive(PrimitiveType::Int)).into(),
750            ])
751            .build()
752            .unwrap();
753
754        let message = schema
755            .check_format_compatibility(FormatVersion::V2)
756            .unwrap_err()
757            .message()
758            .to_string();
759
760        let lines: Vec<&str> = message.lines().skip(1).collect();
761        assert_eq!(
762            lines,
763            vec![
764                "- Invalid type for b: timestamp_ns is not supported until v3 but format version is v2.",
765                "- Invalid initial default for b: non-null default (Primitive(Long(0))) is not supported until v3 but format version is v2.",
766                "- Invalid type for c: variant is not supported until v3 but format version is v2.",
767            ],
768            "{message}"
769        );
770    }
771
772    #[test]
773    fn test_construct_schema() {
774        let field1: NestedFieldRef =
775            NestedField::required(1, "f1", Primitive(PrimitiveType::Boolean)).into();
776        let field2: NestedFieldRef =
777            NestedField::optional(2, "f2", Primitive(PrimitiveType::Int)).into();
778
779        let schema = Schema::builder()
780            .with_fields(vec![field1.clone()])
781            .with_fields(vec![field2.clone()])
782            .with_schema_id(3)
783            .build()
784            .unwrap();
785
786        assert_eq!(3, schema.schema_id());
787        assert_eq!(2, schema.highest_field_id());
788        assert_eq!(Some(&field1), schema.field_by_id(1));
789        assert_eq!(Some(&field2), schema.field_by_id(2));
790        assert_eq!(None, schema.field_by_id(3));
791    }
792
793    pub fn table_schema_simple<'a>() -> (Schema, &'a str) {
794        let schema = Schema::builder()
795            .with_schema_id(1)
796            .with_identifier_field_ids(vec![2])
797            .with_fields(vec![
798                NestedField::optional(1, "foo", Primitive(PrimitiveType::String)).into(),
799                NestedField::required(2, "bar", Primitive(PrimitiveType::Int)).into(),
800                NestedField::optional(3, "baz", Primitive(PrimitiveType::Boolean)).into(),
801            ])
802            .build()
803            .unwrap();
804        let record = r#"{
805            "type":"struct",
806            "schema-id":1,
807            "fields":[
808                {
809                    "id":1,
810                    "name":"foo",
811                    "required":false,
812                    "type":"string"
813                },
814                {
815                    "id":2,
816                    "name":"bar",
817                    "required":true,
818                    "type":"int"
819                },
820                {
821                    "id":3,
822                    "name":"baz",
823                    "required":false,
824                    "type":"boolean"
825                }
826            ],
827            "identifier-field-ids":[2]
828        }"#;
829        (schema, record)
830    }
831
832    pub fn table_schema_nested() -> Schema {
833        Schema::builder()
834            .with_schema_id(1)
835            .with_identifier_field_ids(vec![2])
836            .with_fields(vec![
837                NestedField::optional(1, "foo", Primitive(PrimitiveType::String)).into(),
838                NestedField::required(2, "bar", Primitive(PrimitiveType::Int)).into(),
839                NestedField::optional(3, "baz", Primitive(PrimitiveType::Boolean)).into(),
840                NestedField::required(
841                    4,
842                    "qux",
843                    List(ListType {
844                        element_field: NestedField::list_element(
845                            5,
846                            Primitive(PrimitiveType::String),
847                            true,
848                        )
849                        .into(),
850                    }),
851                )
852                .into(),
853                NestedField::required(
854                    6,
855                    "quux",
856                    Map(MapType {
857                        key_field: NestedField::map_key_element(
858                            7,
859                            Primitive(PrimitiveType::String),
860                        )
861                        .into(),
862                        value_field: NestedField::map_value_element(
863                            8,
864                            Map(MapType {
865                                key_field: NestedField::map_key_element(
866                                    9,
867                                    Primitive(PrimitiveType::String),
868                                )
869                                .into(),
870                                value_field: NestedField::map_value_element(
871                                    10,
872                                    Primitive(PrimitiveType::Int),
873                                    true,
874                                )
875                                .into(),
876                            }),
877                            true,
878                        )
879                        .into(),
880                    }),
881                )
882                .into(),
883                NestedField::required(
884                    11,
885                    "location",
886                    List(ListType {
887                        element_field: NestedField::list_element(
888                            12,
889                            Struct(StructType::new(vec![
890                                NestedField::optional(
891                                    13,
892                                    "latitude",
893                                    Primitive(PrimitiveType::Float),
894                                )
895                                .into(),
896                                NestedField::optional(
897                                    14,
898                                    "longitude",
899                                    Primitive(PrimitiveType::Float),
900                                )
901                                .into(),
902                            ])),
903                            true,
904                        )
905                        .into(),
906                    }),
907                )
908                .into(),
909                NestedField::optional(
910                    15,
911                    "person",
912                    Struct(StructType::new(vec![
913                        NestedField::optional(16, "name", Primitive(PrimitiveType::String)).into(),
914                        NestedField::required(17, "age", Primitive(PrimitiveType::Int)).into(),
915                    ])),
916                )
917                .into(),
918            ])
919            .build()
920            .unwrap()
921    }
922
923    #[test]
924    fn test_schema_display() {
925        let expected_str = "
926table {
927  1: foo: optional string\x20
928  2: bar: required int\x20
929  3: baz: optional boolean\x20
930}
931";
932
933        assert_eq!(expected_str, format!("\n{}", table_schema_simple().0));
934    }
935
936    #[test]
937    fn test_schema_build_failed_on_duplicate_names() {
938        let ret = Schema::builder()
939            .with_schema_id(1)
940            .with_identifier_field_ids(vec![1])
941            .with_fields(vec![
942                NestedField::required(1, "foo", Primitive(PrimitiveType::String)).into(),
943                NestedField::required(2, "bar", Primitive(PrimitiveType::Int)).into(),
944                NestedField::optional(3, "baz", Primitive(PrimitiveType::Boolean)).into(),
945                NestedField::optional(4, "baz", Primitive(PrimitiveType::Boolean)).into(),
946            ])
947            .build();
948
949        assert!(
950            ret.unwrap_err()
951                .message()
952                .contains("Invalid schema: multiple fields for name baz")
953        );
954    }
955
956    #[test]
957    fn test_schema_into_builder() {
958        let original_schema = table_schema_nested();
959        let builder = original_schema.clone().into_builder();
960        let schema = builder.build().unwrap();
961
962        assert_eq!(original_schema, schema);
963    }
964
965    #[test]
966    fn test_schema_index_by_name() {
967        let expected_name_to_id = HashMap::from(
968            [
969                ("foo", 1),
970                ("bar", 2),
971                ("baz", 3),
972                ("qux", 4),
973                ("qux.element", 5),
974                ("quux", 6),
975                ("quux.key", 7),
976                ("quux.value", 8),
977                ("quux.value.key", 9),
978                ("quux.value.value", 10),
979                ("location", 11),
980                ("location.element", 12),
981                ("location.element.latitude", 13),
982                ("location.element.longitude", 14),
983                ("location.latitude", 13),
984                ("location.longitude", 14),
985                ("person", 15),
986                ("person.name", 16),
987                ("person.age", 17),
988            ]
989            .map(|e| (e.0.to_string(), e.1)),
990        );
991
992        let schema = table_schema_nested();
993        assert_eq!(&expected_name_to_id, &schema.name_to_id);
994    }
995
996    #[test]
997    fn test_schema_index_by_name_case_insensitive() {
998        let expected_name_to_id = HashMap::from(
999            [
1000                ("fOo", 1),
1001                ("Bar", 2),
1002                ("BAz", 3),
1003                ("quX", 4),
1004                ("quX.ELEment", 5),
1005                ("qUUx", 6),
1006                ("QUUX.KEY", 7),
1007                ("QUUX.Value", 8),
1008                ("qUUX.VALUE.Key", 9),
1009                ("qUux.VaLue.Value", 10),
1010                ("lOCAtION", 11),
1011                ("LOCAtioN.ELeMENt", 12),
1012                ("LoCATion.element.LATitude", 13),
1013                ("locatION.ElemeNT.LONgitude", 14),
1014                ("LOCAtiON.LATITUDE", 13),
1015                ("LOCATION.LONGITUDE", 14),
1016                ("PERSon", 15),
1017                ("PERSON.Name", 16),
1018                ("peRSON.AGe", 17),
1019            ]
1020            .map(|e| (e.0.to_string(), e.1)),
1021        );
1022
1023        let schema = table_schema_nested();
1024        for (name, id) in expected_name_to_id {
1025            assert_eq!(
1026                Some(id),
1027                schema.field_by_name_case_insensitive(&name).map(|f| f.id)
1028            );
1029        }
1030    }
1031
1032    #[test]
1033    fn test_schema_field_by_name_case_insensitive_checked() {
1034        let schema = Schema::builder()
1035            .with_fields(vec![
1036                NestedField::optional(1, "id", Primitive(PrimitiveType::Int)).into(),
1037                NestedField::optional(2, "ID", Primitive(PrimitiveType::Int)).into(),
1038                NestedField::optional(3, "data", Primitive(PrimitiveType::String)).into(),
1039            ])
1040            .build()
1041            .expect("case-insensitive name ambiguity is valid in a schema");
1042
1043        assert_eq!(
1044            schema
1045                .field_by_name_case_insensitive_checked("DATA")
1046                .unwrap()
1047                .map(|field| field.id),
1048            Some(3)
1049        );
1050        assert!(
1051            schema
1052                .field_by_name_case_insensitive_checked("missing")
1053                .unwrap()
1054                .is_none()
1055        );
1056
1057        let error = schema
1058            .field_by_name_case_insensitive_checked("Id")
1059            .unwrap_err();
1060        assert_eq!(error.kind(), crate::ErrorKind::DataInvalid);
1061        assert!(error.message().contains("case-insensitively"), "{error}");
1062        assert!(error.message().contains("[\"ID\", \"id\"]"), "{error}");
1063        assert!(schema.field_by_name_case_insensitive("Id").is_none());
1064    }
1065
1066    #[test]
1067    fn test_schema_find_column_name() {
1068        let expected_column_name = HashMap::from([
1069            (1, "foo"),
1070            (2, "bar"),
1071            (3, "baz"),
1072            (4, "qux"),
1073            (5, "qux.element"),
1074            (6, "quux"),
1075            (7, "quux.key"),
1076            (8, "quux.value"),
1077            (9, "quux.value.key"),
1078            (10, "quux.value.value"),
1079            (11, "location"),
1080            (12, "location.element"),
1081            (13, "location.element.latitude"),
1082            (14, "location.element.longitude"),
1083        ]);
1084
1085        let schema = table_schema_nested();
1086        for (id, name) in expected_column_name {
1087            assert_eq!(
1088                Some(name),
1089                schema.name_by_field_id(id),
1090                "Column name for field id {id} not match."
1091            );
1092        }
1093    }
1094
1095    #[test]
1096    fn test_schema_find_column_name_not_found() {
1097        let schema = table_schema_nested();
1098
1099        assert!(schema.name_by_field_id(99).is_none());
1100    }
1101
1102    #[test]
1103    fn test_schema_find_column_name_by_id_simple() {
1104        let expected_id_to_name = HashMap::from([(1, "foo"), (2, "bar"), (3, "baz")]);
1105
1106        let schema = table_schema_simple().0;
1107
1108        for (id, name) in expected_id_to_name {
1109            assert_eq!(
1110                Some(name),
1111                schema.name_by_field_id(id),
1112                "Column name for field id {id} not match."
1113            );
1114        }
1115    }
1116
1117    #[test]
1118    fn test_schema_find_simple() {
1119        let schema = table_schema_simple().0;
1120
1121        assert_eq!(
1122            Some(schema.r#struct.fields()[0].clone()),
1123            schema.field_by_id(1).cloned()
1124        );
1125        assert_eq!(
1126            Some(schema.r#struct.fields()[1].clone()),
1127            schema.field_by_id(2).cloned()
1128        );
1129        assert_eq!(
1130            Some(schema.r#struct.fields()[2].clone()),
1131            schema.field_by_id(3).cloned()
1132        );
1133
1134        assert!(schema.field_by_id(4).is_none());
1135        assert!(schema.field_by_name("non exist").is_none());
1136    }
1137
1138    #[test]
1139    fn test_schema_find_nested() {
1140        let expected_id_to_field: HashMap<i32, NestedField> = HashMap::from([
1141            (
1142                1,
1143                NestedField::optional(1, "foo", Primitive(PrimitiveType::String)),
1144            ),
1145            (
1146                2,
1147                NestedField::required(2, "bar", Primitive(PrimitiveType::Int)),
1148            ),
1149            (
1150                3,
1151                NestedField::optional(3, "baz", Primitive(PrimitiveType::Boolean)),
1152            ),
1153            (
1154                4,
1155                NestedField::required(
1156                    4,
1157                    "qux",
1158                    List(ListType {
1159                        element_field: NestedField::list_element(
1160                            5,
1161                            Primitive(PrimitiveType::String),
1162                            true,
1163                        )
1164                        .into(),
1165                    }),
1166                ),
1167            ),
1168            (
1169                5,
1170                NestedField::required(5, "element", Primitive(PrimitiveType::String)),
1171            ),
1172            (
1173                6,
1174                NestedField::required(
1175                    6,
1176                    "quux",
1177                    Map(MapType {
1178                        key_field: NestedField::map_key_element(
1179                            7,
1180                            Primitive(PrimitiveType::String),
1181                        )
1182                        .into(),
1183                        value_field: NestedField::map_value_element(
1184                            8,
1185                            Map(MapType {
1186                                key_field: NestedField::map_key_element(
1187                                    9,
1188                                    Primitive(PrimitiveType::String),
1189                                )
1190                                .into(),
1191                                value_field: NestedField::map_value_element(
1192                                    10,
1193                                    Primitive(PrimitiveType::Int),
1194                                    true,
1195                                )
1196                                .into(),
1197                            }),
1198                            true,
1199                        )
1200                        .into(),
1201                    }),
1202                ),
1203            ),
1204            (
1205                7,
1206                NestedField::required(7, "key", Primitive(PrimitiveType::String)),
1207            ),
1208            (
1209                8,
1210                NestedField::required(
1211                    8,
1212                    "value",
1213                    Map(MapType {
1214                        key_field: NestedField::map_key_element(
1215                            9,
1216                            Primitive(PrimitiveType::String),
1217                        )
1218                        .into(),
1219                        value_field: NestedField::map_value_element(
1220                            10,
1221                            Primitive(PrimitiveType::Int),
1222                            true,
1223                        )
1224                        .into(),
1225                    }),
1226                ),
1227            ),
1228            (
1229                9,
1230                NestedField::required(9, "key", Primitive(PrimitiveType::String)),
1231            ),
1232            (
1233                10,
1234                NestedField::required(10, "value", Primitive(PrimitiveType::Int)),
1235            ),
1236            (
1237                11,
1238                NestedField::required(
1239                    11,
1240                    "location",
1241                    List(ListType {
1242                        element_field: NestedField::list_element(
1243                            12,
1244                            Struct(StructType::new(vec![
1245                                NestedField::optional(
1246                                    13,
1247                                    "latitude",
1248                                    Primitive(PrimitiveType::Float),
1249                                )
1250                                .into(),
1251                                NestedField::optional(
1252                                    14,
1253                                    "longitude",
1254                                    Primitive(PrimitiveType::Float),
1255                                )
1256                                .into(),
1257                            ])),
1258                            true,
1259                        )
1260                        .into(),
1261                    }),
1262                ),
1263            ),
1264            (
1265                12,
1266                NestedField::list_element(
1267                    12,
1268                    Struct(StructType::new(vec![
1269                        NestedField::optional(13, "latitude", Primitive(PrimitiveType::Float))
1270                            .into(),
1271                        NestedField::optional(14, "longitude", Primitive(PrimitiveType::Float))
1272                            .into(),
1273                    ])),
1274                    true,
1275                ),
1276            ),
1277            (
1278                13,
1279                NestedField::optional(13, "latitude", Primitive(PrimitiveType::Float)),
1280            ),
1281            (
1282                14,
1283                NestedField::optional(14, "longitude", Primitive(PrimitiveType::Float)),
1284            ),
1285            (
1286                15,
1287                NestedField::optional(
1288                    15,
1289                    "person",
1290                    Struct(StructType::new(vec![
1291                        NestedField::optional(16, "name", Primitive(PrimitiveType::String)).into(),
1292                        NestedField::required(17, "age", Primitive(PrimitiveType::Int)).into(),
1293                    ])),
1294                ),
1295            ),
1296            (
1297                16,
1298                NestedField::optional(16, "name", Primitive(PrimitiveType::String)),
1299            ),
1300            (
1301                17,
1302                NestedField::required(17, "age", Primitive(PrimitiveType::Int)),
1303            ),
1304        ]);
1305
1306        let schema = table_schema_nested();
1307        for (id, field) in expected_id_to_field {
1308            assert_eq!(
1309                Some(&field),
1310                schema.field_by_id(id).map(|f| f.as_ref()),
1311                "Field for {id} not match."
1312            );
1313        }
1314    }
1315
1316    #[test]
1317    fn test_build_accessors() {
1318        let schema = table_schema_nested();
1319
1320        let test_struct = crate::spec::Struct::from_iter(vec![
1321            Some(Literal::string("foo value")),
1322            Some(Literal::int(1002)),
1323            Some(Literal::bool(true)),
1324            Some(Literal::List(vec![
1325                Some(Literal::string("qux item 1")),
1326                Some(Literal::string("qux item 2")),
1327            ])),
1328            Some(Literal::Map(MapValue::from([(
1329                Literal::string("quux key 1"),
1330                Some(Literal::Map(MapValue::from([(
1331                    Literal::string("quux nested key 1"),
1332                    Some(Literal::int(1000)),
1333                )]))),
1334            )]))),
1335            Some(Literal::List(vec![Some(Literal::Struct(
1336                crate::spec::Struct::from_iter(vec![
1337                    Some(Literal::float(52.509_09_f32)),
1338                    Some(Literal::float(-1.885_249_f32)),
1339                ]),
1340            ))])),
1341            Some(Literal::Struct(crate::spec::Struct::from_iter(vec![
1342                Some(Literal::string("Testy McTest")),
1343                Some(Literal::int(33)),
1344            ]))),
1345        ]);
1346
1347        assert_eq!(
1348            schema
1349                .accessor_by_field_id(1)
1350                .unwrap()
1351                .get(&test_struct)
1352                .unwrap(),
1353            Some(Datum::string("foo value"))
1354        );
1355        assert_eq!(
1356            schema
1357                .accessor_by_field_id(2)
1358                .unwrap()
1359                .get(&test_struct)
1360                .unwrap(),
1361            Some(Datum::int(1002))
1362        );
1363        assert_eq!(
1364            schema
1365                .accessor_by_field_id(3)
1366                .unwrap()
1367                .get(&test_struct)
1368                .unwrap(),
1369            Some(Datum::bool(true))
1370        );
1371        assert_eq!(
1372            schema
1373                .accessor_by_field_id(16)
1374                .unwrap()
1375                .get(&test_struct)
1376                .unwrap(),
1377            Some(Datum::string("Testy McTest"))
1378        );
1379        assert_eq!(
1380            schema
1381                .accessor_by_field_id(17)
1382                .unwrap()
1383                .get(&test_struct)
1384                .unwrap(),
1385            Some(Datum::int(33))
1386        );
1387    }
1388
1389    #[test]
1390    fn test_highest_field_id() {
1391        let schema = table_schema_nested();
1392        assert_eq!(17, schema.highest_field_id());
1393
1394        let schema = table_schema_simple().0;
1395        assert_eq!(3, schema.highest_field_id());
1396    }
1397
1398    #[test]
1399    fn test_highest_field_id_no_fields() {
1400        let schema = Schema::builder().with_schema_id(1).build().unwrap();
1401        assert_eq!(0, schema.highest_field_id());
1402    }
1403
1404    #[test]
1405    fn test_field_ids_must_be_unique() {
1406        let reassigned_schema = Schema::builder()
1407            .with_schema_id(1)
1408            .with_identifier_field_ids(vec![5])
1409            .with_alias(BiHashMap::from_iter(vec![("bar_alias".to_string(), 3)]))
1410            .with_fields(vec![
1411                NestedField::required(5, "foo", Primitive(PrimitiveType::String)).into(),
1412                NestedField::optional(3, "bar", Primitive(PrimitiveType::Int)).into(),
1413                NestedField::optional(3, "baz", Primitive(PrimitiveType::Boolean)).into(),
1414            ])
1415            .build()
1416            .unwrap_err();
1417
1418        assert!(reassigned_schema.message().contains("'field.id' 3"));
1419    }
1420
1421    #[test]
1422    fn test_reassign_ids_empty_schema() {
1423        let schema = Schema::builder().with_schema_id(1).build().unwrap();
1424        let reassigned_schema = schema
1425            .clone()
1426            .into_builder()
1427            .with_reassigned_field_ids(0)
1428            .build()
1429            .unwrap();
1430
1431        assert_eq!(schema, reassigned_schema);
1432        assert_eq!(schema.highest_field_id(), 0);
1433    }
1434
1435    #[test]
1436    fn test_identifier_field_ids() {
1437        // field in map
1438        assert!(
1439            Schema::builder()
1440                .with_schema_id(1)
1441                .with_identifier_field_ids(vec![2])
1442                .with_fields(vec![
1443                    NestedField::required(
1444                        1,
1445                        "Map",
1446                        Map(MapType::new(
1447                            NestedField::map_key_element(2, Primitive(PrimitiveType::String))
1448                                .into(),
1449                            NestedField::map_value_element(
1450                                3,
1451                                Primitive(PrimitiveType::Boolean),
1452                                true,
1453                            )
1454                            .into(),
1455                        )),
1456                    )
1457                    .into()
1458                ])
1459                .build()
1460                .is_err()
1461        );
1462        assert!(
1463            Schema::builder()
1464                .with_schema_id(1)
1465                .with_identifier_field_ids(vec![3])
1466                .with_fields(vec![
1467                    NestedField::required(
1468                        1,
1469                        "Map",
1470                        Map(MapType::new(
1471                            NestedField::map_key_element(2, Primitive(PrimitiveType::String))
1472                                .into(),
1473                            NestedField::map_value_element(
1474                                3,
1475                                Primitive(PrimitiveType::Boolean),
1476                                true,
1477                            )
1478                            .into(),
1479                        )),
1480                    )
1481                    .into()
1482                ])
1483                .build()
1484                .is_err()
1485        );
1486
1487        // field in list
1488        assert!(
1489            Schema::builder()
1490                .with_schema_id(1)
1491                .with_identifier_field_ids(vec![2])
1492                .with_fields(vec![
1493                    NestedField::required(
1494                        1,
1495                        "List",
1496                        List(ListType::new(
1497                            NestedField::list_element(2, Primitive(PrimitiveType::String), true)
1498                                .into(),
1499                        )),
1500                    )
1501                    .into()
1502                ])
1503                .build()
1504                .is_err()
1505        );
1506
1507        // field in optional struct
1508        assert!(
1509            Schema::builder()
1510                .with_schema_id(1)
1511                .with_identifier_field_ids(vec![2])
1512                .with_fields(vec![
1513                    NestedField::optional(
1514                        1,
1515                        "Struct",
1516                        Struct(StructType::new(vec![
1517                            NestedField::required(2, "name", Primitive(PrimitiveType::String))
1518                                .into(),
1519                            NestedField::optional(3, "age", Primitive(PrimitiveType::Int)).into(),
1520                        ])),
1521                    )
1522                    .into()
1523                ])
1524                .build()
1525                .is_err()
1526        );
1527
1528        // float and double
1529        assert!(
1530            Schema::builder()
1531                .with_schema_id(1)
1532                .with_identifier_field_ids(vec![1])
1533                .with_fields(vec![
1534                    NestedField::required(1, "Float", Primitive(PrimitiveType::Float),).into()
1535                ])
1536                .build()
1537                .is_err()
1538        );
1539        assert!(
1540            Schema::builder()
1541                .with_schema_id(1)
1542                .with_identifier_field_ids(vec![1])
1543                .with_fields(vec![
1544                    NestedField::required(1, "Double", Primitive(PrimitiveType::Double),).into()
1545                ])
1546                .build()
1547                .is_err()
1548        );
1549
1550        // optional field
1551        assert!(
1552            Schema::builder()
1553                .with_schema_id(1)
1554                .with_identifier_field_ids(vec![1])
1555                .with_fields(vec![
1556                    NestedField::required(1, "Required", Primitive(PrimitiveType::String),).into()
1557                ])
1558                .build()
1559                .is_ok()
1560        );
1561        assert!(
1562            Schema::builder()
1563                .with_schema_id(1)
1564                .with_identifier_field_ids(vec![1])
1565                .with_fields(vec![
1566                    NestedField::optional(1, "Optional", Primitive(PrimitiveType::String),).into()
1567                ])
1568                .build()
1569                .is_err()
1570        );
1571    }
1572
1573    #[test]
1574    fn test_unknown_type_deserialization_rejects_non_null_default() {
1575        let field_json = serde_json::json!({
1576            "id": 1,
1577            "name": "empty",
1578            "required": false,
1579            "type": "unknown",
1580            "initial-default": 1
1581        });
1582
1583        let error = serde_json::from_value::<NestedField>(field_json.clone()).unwrap_err();
1584        assert!(
1585            error
1586                .to_string()
1587                .contains("Unknown type only supports null default values"),
1588            "unexpected error: {error}"
1589        );
1590
1591        let schema_json = serde_json::json!({
1592            "type": "struct",
1593            "schema-id": 1,
1594            "fields": [field_json]
1595        });
1596        assert!(serde_json::from_value::<Schema>(schema_json).is_err());
1597    }
1598
1599    #[test]
1600    fn test_unknown_type_deserialization_accepts_null_defaults() {
1601        let schema_json = serde_json::json!({
1602            "type": "struct",
1603            "schema-id": 1,
1604            "fields": [
1605                {
1606                    "id": 1,
1607                    "name": "empty",
1608                    "required": false,
1609                    "type": "unknown",
1610                    "initial-default": null,
1611                    "write-default": null
1612                }
1613            ]
1614        });
1615
1616        serde_json::from_value::<Schema>(schema_json).unwrap();
1617    }
1618
1619    #[test]
1620    fn test_unknown_type_accepts_null_container_defaults() {
1621        let cases = [
1622            (
1623                "struct",
1624                Struct(StructType::new(vec![
1625                    NestedField::optional(2, "empty", Primitive(PrimitiveType::Unknown)).into(),
1626                ])),
1627                Literal::Struct(crate::spec::Struct::from_iter([None])),
1628            ),
1629            (
1630                "list",
1631                List(ListType::new(
1632                    NestedField::list_element(2, Primitive(PrimitiveType::Unknown), false).into(),
1633                )),
1634                Literal::List(vec![None]),
1635            ),
1636            (
1637                "map",
1638                Map(MapType::optional(
1639                    2,
1640                    Primitive(PrimitiveType::String),
1641                    3,
1642                    Primitive(PrimitiveType::Unknown),
1643                )),
1644                Literal::Map(MapValue::from([(Literal::string("key"), None)])),
1645            ),
1646        ];
1647
1648        for (name, field_type, default) in cases {
1649            let schema = Schema::builder()
1650                .with_schema_id(1)
1651                .with_fields(vec![
1652                    NestedField::optional(1, name, field_type)
1653                        .with_write_default(default)
1654                        .into(),
1655                ])
1656                .build()
1657                .unwrap();
1658            serde_json::to_value(schema).unwrap();
1659        }
1660    }
1661
1662    #[test]
1663    fn test_unknown_type_deserialization_rejects_non_null_container_defaults() {
1664        let cases = [
1665            (
1666                "struct",
1667                serde_json::json!({
1668                    "type": "struct",
1669                    "fields": [{
1670                        "id": 2,
1671                        "name": "empty",
1672                        "required": false,
1673                        "type": "unknown"
1674                    }]
1675                }),
1676                serde_json::json!({"2": 1}),
1677            ),
1678            (
1679                "list",
1680                serde_json::json!({
1681                    "type": "list",
1682                    "element-id": 2,
1683                    "element-required": false,
1684                    "element": "unknown"
1685                }),
1686                serde_json::json!([1]),
1687            ),
1688            (
1689                "map",
1690                serde_json::json!({
1691                    "type": "map",
1692                    "key-id": 2,
1693                    "key": "string",
1694                    "value-id": 3,
1695                    "value-required": false,
1696                    "value": "unknown"
1697                }),
1698                serde_json::json!({"keys": ["key"], "values": [1]}),
1699            ),
1700        ];
1701
1702        for (name, field_type, default) in cases {
1703            let schema_json = serde_json::json!({
1704                "type": "struct",
1705                "schema-id": 1,
1706                "fields": [{
1707                    "id": 1,
1708                    "name": name,
1709                    "required": false,
1710                    "type": field_type,
1711                    "initial-default": default
1712                }]
1713            });
1714
1715            assert!(
1716                serde_json::from_value::<Schema>(schema_json).is_err(),
1717                "non-null unknown default in {name} should be rejected"
1718            );
1719        }
1720    }
1721}