iceberg/spec/schema/
mod.rs

1// Licensed to the Apache Software Foundation (ASF) under one
2// or more contributor license agreements.  See the NOTICE file
3// distributed with this work for additional information
4// regarding copyright ownership.  The ASF licenses this file
5// to you under the Apache License, Version 2.0 (the
6// "License"); you may not use this file except in compliance
7// with the License.  You may obtain a copy of the License at
8//
9//   http://www.apache.org/licenses/LICENSE-2.0
10//
11// Unless required by applicable law or agreed to in writing,
12// software distributed under the License is distributed on an
13// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14// KIND, either express or implied.  See the License for the
15// specific language governing permissions and limitations
16// under the License.
17
18//! This module defines schema in iceberg.
19
20use std::collections::{HashMap, HashSet};
21use std::fmt::{Display, Formatter};
22use std::sync::Arc;
23
24mod utils;
25mod visitor;
26pub use self::visitor::*;
27pub(super) mod _serde;
28// TODO: Use for table replacement in https://github.com/apache/iceberg-rust/issues/3056.
29#[allow(dead_code)]
30pub(crate) mod assign_fresh_ids;
31mod id_reassigner;
32mod index;
33mod prune_columns;
34use bimap::BiHashMap;
35use itertools::{Itertools, zip_eq};
36use serde::{Deserialize, Serialize};
37
38use self::_serde::SchemaEnum;
39use self::id_reassigner::ReassignFieldIds;
40use self::index::{IndexByName, index_by_id, index_parents};
41pub use self::prune_columns::prune_columns;
42use super::NestedField;
43use crate::error::{Result, invalid_data};
44use crate::expr::accessor::StructAccessor;
45use crate::spec::FormatVersion;
46use crate::spec::datatypes::{
47    LIST_FIELD_NAME, ListType, MAP_KEY_FIELD_NAME, MAP_VALUE_FIELD_NAME, MapType, NestedFieldRef,
48    PrimitiveType, StructType, Type,
49};
50use crate::{Error, ErrorKind, ensure_data_valid};
51
52/// Type alias for schema id.
53pub type SchemaId = i32;
54/// Reference to [`Schema`].
55pub type SchemaRef = Arc<Schema>;
56/// Default schema id.
57pub const DEFAULT_SCHEMA_ID: SchemaId = 0;
58/// Delimiter for schema name, which denotes a nested struct.
59pub const SCHEMA_NAME_DELIMITER: &str = ".";
60/// Minimum format version that allows non-null field default values.
61/// Mirrors Java's `Schema.DEFAULT_VALUES_MIN_FORMAT_VERSION`.
62pub(crate) const DEFAULT_VALUES_MIN_FORMAT_VERSION: FormatVersion = FormatVersion::V3;
63
64/// Defines schema in iceberg.
65#[derive(Debug, Serialize, Deserialize, Clone)]
66#[serde(try_from = "SchemaEnum", into = "SchemaEnum")]
67pub struct Schema {
68    r#struct: StructType,
69    schema_id: SchemaId,
70    highest_field_id: i32,
71    identifier_field_ids: HashSet<i32>,
72
73    alias_to_id: BiHashMap<String, i32>,
74    id_to_field: HashMap<i32, NestedFieldRef>,
75
76    name_to_id: HashMap<String, i32>,
77    // `None` marks a lower-cased name that resolves to more than one field.
78    lowercase_name_to_id: HashMap<String, Option<i32>>,
79    id_to_name: HashMap<i32, String>,
80
81    field_id_to_accessor: HashMap<i32, Arc<StructAccessor>>,
82}
83
84impl PartialEq for Schema {
85    fn eq(&self, other: &Self) -> bool {
86        self.r#struct == other.r#struct
87            && self.schema_id == other.schema_id
88            && self.identifier_field_ids == other.identifier_field_ids
89    }
90}
91
92impl Eq for Schema {}
93
94/// Schema builder.
95#[derive(Debug)]
96pub struct SchemaBuilder {
97    schema_id: i32,
98    fields: Vec<NestedFieldRef>,
99    alias_to_id: BiHashMap<String, i32>,
100    identifier_field_ids: HashSet<i32>,
101    reassign_field_ids_from: Option<i32>,
102}
103
104impl SchemaBuilder {
105    /// Add fields to schema builder.
106    pub fn with_fields(mut self, fields: impl IntoIterator<Item = NestedFieldRef>) -> Self {
107        self.fields.extend(fields);
108        self
109    }
110
111    /// Reassign all field-ids (including nested) on build.
112    /// Reassignment starts from the field-id specified in `start_from` (inclusive).
113    ///
114    /// All specified aliases and identifier fields will be updated to the new field-ids.
115    pub(crate) fn with_reassigned_field_ids(mut self, start_from: i32) -> Self {
116        self.reassign_field_ids_from = Some(start_from);
117        self
118    }
119
120    /// Set schema id.
121    pub fn with_schema_id(mut self, schema_id: i32) -> Self {
122        self.schema_id = schema_id;
123        self
124    }
125
126    /// Set identifier field ids.
127    pub fn with_identifier_field_ids(mut self, ids: impl IntoIterator<Item = i32>) -> Self {
128        self.identifier_field_ids.extend(ids);
129        self
130    }
131
132    /// Set alias to filed id mapping.
133    pub fn with_alias(mut self, alias_to_id: BiHashMap<String, i32>) -> Self {
134        self.alias_to_id = alias_to_id;
135        self
136    }
137
138    /// Builds the schema.
139    pub fn build(self) -> Result<Schema> {
140        let field_id_to_accessor = self.build_accessors();
141
142        let r#struct = StructType::new(self.fields);
143        let id_to_field = index_by_id(&r#struct)?;
144
145        Self::validate_identifier_ids(
146            &r#struct,
147            &id_to_field,
148            self.identifier_field_ids.iter().copied(),
149        )?;
150
151        let (name_to_id, id_to_name) = {
152            let mut index = IndexByName::default();
153            visit_struct(&r#struct, &mut index)?;
154            index.indexes()
155        };
156
157        let mut lowercase_name_to_id = HashMap::new();
158        for (name, field_id) in &name_to_id {
159            lowercase_name_to_id
160                .entry(name.to_lowercase())
161                .and_modify(|existing| *existing = None)
162                .or_insert(Some(*field_id));
163        }
164
165        let highest_field_id = id_to_field.keys().max().cloned().unwrap_or(0);
166
167        let mut schema = Schema {
168            r#struct,
169            schema_id: self.schema_id,
170            highest_field_id,
171            identifier_field_ids: self.identifier_field_ids,
172            alias_to_id: self.alias_to_id,
173            id_to_field,
174
175            name_to_id,
176            lowercase_name_to_id,
177            id_to_name,
178
179            field_id_to_accessor,
180        };
181
182        if let Some(start_from) = self.reassign_field_ids_from {
183            let mut id_reassigner = ReassignFieldIds::new(start_from);
184            let new_fields = id_reassigner.reassign_field_ids(schema.r#struct.fields().to_vec())?;
185            let new_identifier_field_ids =
186                id_reassigner.apply_to_identifier_fields(schema.identifier_field_ids)?;
187            let new_alias_to_id = id_reassigner.apply_to_aliases(schema.alias_to_id.clone())?;
188
189            schema = Schema::builder()
190                .with_schema_id(schema.schema_id)
191                .with_fields(new_fields)
192                .with_identifier_field_ids(new_identifier_field_ids)
193                .with_alias(new_alias_to_id)
194                .build()?;
195        }
196
197        Ok(schema)
198    }
199
200    fn build_accessors(&self) -> HashMap<i32, Arc<StructAccessor>> {
201        let mut map = HashMap::new();
202
203        for (pos, field) in self.fields.iter().enumerate() {
204            match field.field_type.as_ref() {
205                Type::Primitive(prim_type) => {
206                    // add an accessor for this field
207                    let accessor = Arc::new(StructAccessor::new(pos, prim_type.clone()));
208                    map.insert(field.id, accessor.clone());
209                }
210
211                Type::Struct(nested) => {
212                    // add accessors for nested fields
213                    for (field_id, accessor) in Self::build_accessors_nested(nested.fields()) {
214                        let new_accessor = Arc::new(StructAccessor::wrap(pos, accessor));
215                        map.insert(field_id, new_accessor.clone());
216                    }
217                }
218                _ => {
219                    // Accessors don't get built for Map or List types
220                }
221            }
222        }
223
224        map
225    }
226
227    fn build_accessors_nested(fields: &[NestedFieldRef]) -> Vec<(i32, Box<StructAccessor>)> {
228        let mut results = vec![];
229        for (pos, field) in fields.iter().enumerate() {
230            match field.field_type.as_ref() {
231                Type::Primitive(prim_type) => {
232                    let accessor = Box::new(StructAccessor::new(pos, prim_type.clone()));
233                    results.push((field.id, accessor));
234                }
235                Type::Struct(nested) => {
236                    let nested_accessors = Self::build_accessors_nested(nested.fields());
237
238                    let wrapped_nested_accessors =
239                        nested_accessors.into_iter().map(|(id, accessor)| {
240                            let new_accessor = Box::new(StructAccessor::wrap(pos, accessor));
241                            (id, new_accessor.clone())
242                        });
243
244                    results.extend(wrapped_nested_accessors);
245                }
246                _ => {
247                    // Accessors don't get built for Map or List types
248                }
249            }
250        }
251
252        results
253    }
254
255    /// According to [the spec](https://iceberg.apache.org/spec/#identifier-fields), the identifier fields
256    /// must meet the following requirements:
257    /// - Float, double, and optional fields cannot be used as identifier fields.
258    /// - Identifier fields may be nested in structs but cannot be nested within maps or lists.
259    /// - A nested field cannot be used as an identifier field if it is nested in an optional struct, to avoid null values in identifiers.
260    fn validate_identifier_ids(
261        r#struct: &StructType,
262        id_to_field: &HashMap<i32, NestedFieldRef>,
263        identifier_field_ids: impl Iterator<Item = i32>,
264    ) -> Result<()> {
265        let id_to_parent = index_parents(r#struct)?;
266        for identifier_field_id in identifier_field_ids {
267            let field = id_to_field.get(&identifier_field_id).ok_or_else(|| {
268                invalid_data!(
269                    "Cannot add identifier field {identifier_field_id}: field does not exist"
270                )
271            })?;
272            ensure_data_valid!(
273                field.required,
274                "Cannot add identifier field: {} is an optional field",
275                field.name
276            );
277            if let Type::Primitive(p) = field.field_type.as_ref() {
278                ensure_data_valid!(
279                    !matches!(p, PrimitiveType::Double | PrimitiveType::Float),
280                    "Cannot add identifier field {}: cannot be a float or double type",
281                    field.name
282                );
283            } else {
284                return Err(invalid_data!(
285                    "Cannot add field {} as an identifier field: not a primitive type field",
286                    field.name
287                ));
288            }
289
290            let mut cur_field_id = identifier_field_id;
291            while let Some(parent) = id_to_parent.get(&cur_field_id) {
292                let parent_field = id_to_field
293                    .get(parent)
294                    .expect("Field id should not disappear.");
295                ensure_data_valid!(
296                    parent_field.field_type.is_struct(),
297                    "Cannot add field {} as an identifier field: must not be nested in {:?}",
298                    field.name,
299                    parent_field
300                );
301                ensure_data_valid!(
302                    parent_field.required,
303                    "Cannot add field {} as an identifier field: must not be nested in an optional field {}",
304                    field.name,
305                    parent_field
306                );
307                cur_field_id = *parent;
308            }
309        }
310
311        Ok(())
312    }
313}
314
315impl Schema {
316    /// Create a schema builder.
317    pub fn builder() -> SchemaBuilder {
318        SchemaBuilder {
319            schema_id: DEFAULT_SCHEMA_ID,
320            fields: vec![],
321            identifier_field_ids: HashSet::default(),
322            alias_to_id: BiHashMap::default(),
323            reassign_field_ids_from: None,
324        }
325    }
326
327    /// Create a new schema builder from a schema.
328    pub fn into_builder(self) -> SchemaBuilder {
329        SchemaBuilder {
330            schema_id: self.schema_id,
331            fields: self.r#struct.fields().to_vec(),
332            alias_to_id: self.alias_to_id,
333            identifier_field_ids: self.identifier_field_ids,
334            reassign_field_ids_from: None,
335        }
336    }
337
338    /// Get field by field id.
339    pub fn field_by_id(&self, field_id: i32) -> Option<&NestedFieldRef> {
340        self.id_to_field.get(&field_id)
341    }
342
343    /// Get field by field name.
344    ///
345    /// Both full name and short name could work here.
346    pub fn field_by_name(&self, field_name: &str) -> Option<&NestedFieldRef> {
347        self.name_to_id
348            .get(field_name)
349            .and_then(|id| self.field_by_id(*id))
350    }
351
352    /// Get field by field name in a case-insensitive way.
353    ///
354    /// Both full name and short name could work here.
355    /// Returns `None` if the name is missing or matches multiple fields case-insensitively.
356    /// Use [`Self::field_by_name_case_insensitive_checked`] to distinguish those cases.
357    pub fn field_by_name_case_insensitive(&self, field_name: &str) -> Option<&NestedFieldRef> {
358        self.lowercase_name_to_id
359            .get(&field_name.to_lowercase())
360            .copied()
361            .flatten()
362            .and_then(|id| self.field_by_id(id))
363    }
364
365    /// Get field by field name in a case-insensitive way, reporting ambiguous names.
366    ///
367    /// Both full name and short name could work here. Returns `Ok(None)` if the name is
368    /// missing, or an error if it matches multiple fields case-insensitively. A collision
369    /// on another name does not prevent this name from resolving. This differs from Iceberg
370    /// Java, which rejects all case-insensitive lookups after a collision, and PyIceberg,
371    /// which silently keeps one of the colliding fields.
372    pub fn field_by_name_case_insensitive_checked(
373        &self,
374        field_name: &str,
375    ) -> Result<Option<&NestedFieldRef>> {
376        let lowercase_name = field_name.to_lowercase();
377        match self.lowercase_name_to_id.get(&lowercase_name) {
378            Some(Some(id)) => Ok(self.field_by_id(*id)),
379            Some(None) => {
380                let mut colliders: Vec<&str> = self
381                    .name_to_id
382                    .keys()
383                    .filter(|name| name.to_lowercase() == lowercase_name)
384                    .map(String::as_str)
385                    .collect();
386                colliders.sort_unstable();
387                Err(invalid_data!(
388                    "Multiple fields match {field_name} case-insensitively: {colliders:?}"
389                ))
390            }
391            None => Ok(None),
392        }
393    }
394
395    /// Get field by alias.
396    pub fn field_by_alias(&self, alias: &str) -> Option<&NestedFieldRef> {
397        self.alias_to_id
398            .get_by_left(alias)
399            .and_then(|id| self.field_by_id(*id))
400    }
401
402    /// Returns the highest field ID assigned in this schema.
403    #[inline]
404    pub fn highest_field_id(&self) -> i32 {
405        self.highest_field_id
406    }
407
408    /// Returns the Schema's ID.
409    #[inline]
410    pub fn schema_id(&self) -> SchemaId {
411        self.schema_id
412    }
413
414    /// Returns the type for the struct.
415    #[inline]
416    pub fn as_struct(&self) -> &StructType {
417        &self.r#struct
418    }
419
420    /// Returns the IDs of the fields.
421    #[inline]
422    pub fn identifier_field_ids(&self) -> impl ExactSizeIterator<Item = i32> + '_ {
423        self.identifier_field_ids.iter().copied()
424    }
425
426    /// Get field id by full name.
427    pub fn field_id_by_name(&self, name: &str) -> Option<i32> {
428        self.name_to_id.get(name).copied()
429    }
430
431    /// Get full name by field id.
432    pub fn name_by_field_id(&self, field_id: i32) -> Option<&str> {
433        self.id_to_name.get(&field_id).map(String::as_str)
434    }
435
436    /// Get an accessor for retrieving data in a struct
437    pub fn accessor_by_field_id(&self, field_id: i32) -> Option<Arc<StructAccessor>> {
438        self.field_id_to_accessor.get(&field_id).cloned()
439    }
440
441    /// Check if this schema is identical to another schema semantically - excluding schema id.
442    pub(crate) fn is_same_schema(&self, other: &SchemaRef) -> bool {
443        self.as_struct().eq(other.as_struct())
444            && self.identifier_field_ids().eq(other.identifier_field_ids())
445    }
446
447    /// Change the schema id of this schema.
448    // This is redundant with the `with_schema_id` method on the builder, but useful
449    // as it is infallible in contrast to the builder `build()` method.
450    pub(crate) fn with_schema_id(self, schema_id: SchemaId) -> Self {
451        Self { schema_id, ..self }
452    }
453
454    /// Return A HashMap matching field ids to field names.
455    pub fn field_id_to_name_map(&self) -> &HashMap<i32, String> {
456        &self.id_to_name
457    }
458
459    /// Return a hashmap matching field ids to nested fields.
460    pub fn field_id_to_fields(&self) -> &HashMap<i32, NestedFieldRef> {
461        &self.id_to_field
462    }
463
464    /// Minimum [`FormatVersion`] required to represent all *types* in this schema.
465    ///
466    /// Walks the whole schema — every field, nested ones included — and returns the
467    /// highest per-type floor, so this is O(fields) rather than a cheap field read.
468    /// Types only; for initial-default version floors see [`Schema::check_format_compatibility`].
469    pub fn calc_min_compatible_format(&self) -> FormatVersion {
470        // `id_to_field` is flattened, so the max over all fields covers nested ones too.
471        self.id_to_field
472            .values()
473            .map(|f| f.field_type.min_format_version())
474            .max()
475            .unwrap_or(FormatVersion::V1)
476    }
477
478    /// Returns an error listing every field incompatible with `format_version`.
479    /// Mirrors Java's `Schema.checkCompatibility()`. Two checks per field:
480    ///
481    /// - **Type** — minimum format version required to support that type, without
482    ///   taking nested field types into account.
483    /// - **Initial default** — a non-null `initial_default` backfills pre-existing rows,
484    ///   so it requires `DEFAULT_VALUES_MIN_FORMAT_VERSION`; `write_default` is not
485    ///   checked, as it only affects newly written rows (read identically at any version).
486    pub fn check_format_compatibility(&self, format_version: FormatVersion) -> Result<()> {
487        // (field id, message); sorted by id below for a deterministic error.
488        let mut problems: Vec<(i32, String)> = Vec::new();
489
490        // `id_to_field` is flattened, so checking each field by its own type keeps the
491        // blame on the offending leaf, not its container (mirrors Java's `lazyIdToField`).
492        for field in self.id_to_field.values() {
493            let min_version = field.field_type.min_format_version();
494            if format_version < min_version {
495                // Every id in `id_to_field` is also indexed in `id_to_name`; a miss means
496                // the schema's indexes are inconsistent (a bug), so surface it rather than
497                // guessing an unqualified name.
498                let name = self.name_by_field_id(field.id).ok_or_else(|| {
499                    Error::new(
500                        ErrorKind::Unexpected,
501                        format!(
502                            "Field id {} is missing from the schema's name index",
503                            field.id
504                        ),
505                    )
506                })?;
507                problems.push((field.id, format!(
508                    "Invalid type for {name}: {} is not supported until {min_version} but format version is {format_version}.",
509                    field.field_type,
510                )));
511            }
512
513            if let Some(default) = &field.initial_default
514                && format_version < DEFAULT_VALUES_MIN_FORMAT_VERSION
515            {
516                let name = self.name_by_field_id(field.id).ok_or_else(|| {
517                    Error::new(
518                        ErrorKind::Unexpected,
519                        format!(
520                            "Field id {} is missing from the schema's name index",
521                            field.id
522                        ),
523                    )
524                })?;
525                problems.push((field.id, format!(
526                    "Invalid initial default for {name}: non-null default ({default:?}) is not supported until {DEFAULT_VALUES_MIN_FORMAT_VERSION} but format version is {format_version}."
527                )));
528            }
529        }
530
531        if problems.is_empty() {
532            return Ok(());
533        }
534
535        // Stable sort by id: HashMap order is nondeterministic, and stability keeps a
536        // field's type problem before its default problem (matches Java's TreeMap order).
537        let message = problems
538            .into_iter()
539            .sorted_by_key(|(id, _)| *id)
540            .map(|(_, msg)| msg)
541            .join("\n- ");
542        Err(invalid_data!(
543            "Invalid schema for {format_version}:\n- {message}"
544        ))
545    }
546}
547
548impl Display for Schema {
549    fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result {
550        writeln!(f, "table {{")?;
551        for field in self.as_struct().fields() {
552            writeln!(f, "  {field}")?;
553        }
554        writeln!(f, "}}")
555    }
556}
557
558#[cfg(test)]
559mod tests {
560    use std::collections::HashMap;
561
562    use bimap::BiHashMap;
563
564    use crate::spec::datatypes::Type::{List, Map, Primitive, Struct, Variant};
565    use crate::spec::datatypes::{
566        ListType, MapType, NestedField, NestedFieldRef, PrimitiveType, StructType,
567    };
568    use crate::spec::schema::Schema;
569    use crate::spec::values::Map as MapValue;
570    use crate::spec::{Datum, Literal};
571
572    #[test]
573    fn test_check_format_compatibility() {
574        use crate::spec::{FormatVersion, PrimitiveLiteral, VariantType};
575
576        fn schema_with(fields: Vec<NestedFieldRef>) -> Schema {
577            Schema::builder().with_fields(fields).build().unwrap()
578        }
579
580        // Variant type requires v3.
581        let variant = schema_with(vec![
582            NestedField::optional(1, "v", Variant(VariantType)).into(),
583        ]);
584        assert!(
585            variant
586                .check_format_compatibility(FormatVersion::V2)
587                .is_err()
588        );
589        assert!(
590            variant
591                .check_format_compatibility(FormatVersion::V3)
592                .is_ok()
593        );
594
595        // A non-null initial default requires v3, even for a v1-compatible type.
596        let with_default = schema_with(vec![
597            NestedField::optional(1, "a", Primitive(PrimitiveType::Int))
598                .with_initial_default(Literal::Primitive(PrimitiveLiteral::Int(1)))
599                .into(),
600        ]);
601        let err = with_default
602            .check_format_compatibility(FormatVersion::V2)
603            .unwrap_err();
604        assert!(
605            err.message().contains("Invalid initial default for a"),
606            "{err}"
607        );
608        assert!(
609            with_default
610                .check_format_compatibility(FormatVersion::V3)
611                .is_ok()
612        );
613
614        // No default is fine at any version (an absent/null default never trips).
615        let no_default = schema_with(vec![
616            NestedField::optional(1, "a", Primitive(PrimitiveType::Int)).into(),
617        ]);
618        assert!(
619            no_default
620                .check_format_compatibility(FormatVersion::V1)
621                .is_ok()
622        );
623
624        // Recursion: a default on a field nested inside a struct is caught.
625        let nested = schema_with(vec![
626            NestedField::required(
627                1,
628                "s",
629                Struct(StructType::new(vec![
630                    NestedField::optional(2, "inner", Primitive(PrimitiveType::Long))
631                        .with_initial_default(Literal::Primitive(PrimitiveLiteral::Long(7)))
632                        .into(),
633                ])),
634            )
635            .into(),
636        ]);
637        let err = nested
638            .check_format_compatibility(FormatVersion::V2)
639            .unwrap_err();
640        assert!(err.message().contains("inner"), "{err}");
641
642        // A v3-only type nested in a container blames the leaf, never the container.
643        let nested_variant = schema_with(vec![
644            NestedField::required(
645                1,
646                "container",
647                Struct(StructType::new(vec![
648                    NestedField::optional(2, "v", Variant(VariantType)).into(),
649                ])),
650            )
651            .into(),
652        ]);
653        let err = nested_variant
654            .check_format_compatibility(FormatVersion::V2)
655            .unwrap_err();
656        assert!(err.message().contains("container.v"), "{err}");
657        assert!(
658            !err.message().contains("Invalid type for container:"),
659            "container must not be blamed: {err}"
660        );
661    }
662
663    #[test]
664    fn test_calc_min_compatible_format() {
665        use crate::spec::{FormatVersion, GeometryType, VariantType};
666
667        fn schema_with(fields: Vec<NestedFieldRef>) -> Schema {
668            Schema::builder().with_fields(fields).build().unwrap()
669        }
670
671        // All v1-compatible types → V1.
672        let v1 = schema_with(vec![
673            NestedField::required(1, "a", Primitive(PrimitiveType::Int)).into(),
674            NestedField::optional(2, "b", Primitive(PrimitiveType::String)).into(),
675        ]);
676        assert_eq!(v1.calc_min_compatible_format(), FormatVersion::V1);
677
678        // A top-level variant → V3.
679        let variant = schema_with(vec![
680            NestedField::optional(1, "v", Variant(VariantType)).into(),
681        ]);
682        assert_eq!(variant.calc_min_compatible_format(), FormatVersion::V3);
683
684        // Unknown is a v3-only primitive type.
685        let unknown = schema_with(vec![
686            NestedField::optional(1, "u", Primitive(PrimitiveType::Unknown)).into(),
687        ]);
688        assert_eq!(unknown.calc_min_compatible_format(), FormatVersion::V3);
689        assert!(
690            unknown
691                .check_format_compatibility(FormatVersion::V2)
692                .is_err()
693        );
694
695        // Geospatial types are v3-only.
696        let geometry = schema_with(vec![
697            NestedField::optional(
698                1,
699                "geom",
700                Primitive(PrimitiveType::Geometry(GeometryType::default())),
701            )
702            .into(),
703        ]);
704        assert_eq!(geometry.calc_min_compatible_format(), FormatVersion::V3);
705        assert!(
706            geometry
707                .check_format_compatibility(FormatVersion::V2)
708                .is_err()
709        );
710
711        // A v3-only type nested inside a list inside a struct → V3 (flattened fields).
712        let nested = schema_with(vec![
713            NestedField::required(
714                1,
715                "s",
716                Struct(StructType::new(vec![
717                    NestedField::optional(
718                        2,
719                        "l",
720                        List(ListType::new(
721                            NestedField::required(
722                                3,
723                                "element",
724                                Primitive(PrimitiveType::TimestampNs),
725                            )
726                            .into(),
727                        )),
728                    )
729                    .into(),
730                ])),
731            )
732            .into(),
733        ]);
734        assert_eq!(nested.calc_min_compatible_format(), FormatVersion::V3);
735    }
736
737    #[test]
738    fn test_check_format_compatibility_message_order() {
739        use crate::spec::{FormatVersion, PrimitiveLiteral, VariantType};
740
741        // Fields declared out of id order; field 2 fails both checks (ts_ns type
742        // AND a non-null initial default). The error must list problems by field id,
743        // with the type problem before the initial-default problem for field 2.
744        let schema = Schema::builder()
745            .with_fields(vec![
746                NestedField::optional(3, "c", Variant(VariantType)).into(),
747                NestedField::optional(2, "b", Primitive(PrimitiveType::TimestampNs))
748                    .with_initial_default(Literal::Primitive(PrimitiveLiteral::Long(0)))
749                    .into(),
750                NestedField::required(1, "a", Primitive(PrimitiveType::Int)).into(),
751            ])
752            .build()
753            .unwrap();
754
755        let message = schema
756            .check_format_compatibility(FormatVersion::V2)
757            .unwrap_err()
758            .message()
759            .to_string();
760
761        let lines: Vec<&str> = message.lines().skip(1).collect();
762        assert_eq!(
763            lines,
764            vec![
765                "- Invalid type for b: timestamp_ns is not supported until v3 but format version is v2.",
766                "- Invalid initial default for b: non-null default (Primitive(Long(0))) is not supported until v3 but format version is v2.",
767                "- Invalid type for c: variant is not supported until v3 but format version is v2.",
768            ],
769            "{message}"
770        );
771    }
772
773    #[test]
774    fn test_construct_schema() {
775        let field1: NestedFieldRef =
776            NestedField::required(1, "f1", Primitive(PrimitiveType::Boolean)).into();
777        let field2: NestedFieldRef =
778            NestedField::optional(2, "f2", Primitive(PrimitiveType::Int)).into();
779
780        let schema = Schema::builder()
781            .with_fields(vec![field1.clone()])
782            .with_fields(vec![field2.clone()])
783            .with_schema_id(3)
784            .build()
785            .unwrap();
786
787        assert_eq!(3, schema.schema_id());
788        assert_eq!(2, schema.highest_field_id());
789        assert_eq!(Some(&field1), schema.field_by_id(1));
790        assert_eq!(Some(&field2), schema.field_by_id(2));
791        assert_eq!(None, schema.field_by_id(3));
792    }
793
794    pub fn table_schema_simple<'a>() -> (Schema, &'a str) {
795        let schema = Schema::builder()
796            .with_schema_id(1)
797            .with_identifier_field_ids(vec![2])
798            .with_fields(vec![
799                NestedField::optional(1, "foo", Primitive(PrimitiveType::String)).into(),
800                NestedField::required(2, "bar", Primitive(PrimitiveType::Int)).into(),
801                NestedField::optional(3, "baz", Primitive(PrimitiveType::Boolean)).into(),
802            ])
803            .build()
804            .unwrap();
805        let record = r#"{
806            "type":"struct",
807            "schema-id":1,
808            "fields":[
809                {
810                    "id":1,
811                    "name":"foo",
812                    "required":false,
813                    "type":"string"
814                },
815                {
816                    "id":2,
817                    "name":"bar",
818                    "required":true,
819                    "type":"int"
820                },
821                {
822                    "id":3,
823                    "name":"baz",
824                    "required":false,
825                    "type":"boolean"
826                }
827            ],
828            "identifier-field-ids":[2]
829        }"#;
830        (schema, record)
831    }
832
833    pub fn table_schema_nested() -> Schema {
834        Schema::builder()
835            .with_schema_id(1)
836            .with_identifier_field_ids(vec![2])
837            .with_fields(vec![
838                NestedField::optional(1, "foo", Primitive(PrimitiveType::String)).into(),
839                NestedField::required(2, "bar", Primitive(PrimitiveType::Int)).into(),
840                NestedField::optional(3, "baz", Primitive(PrimitiveType::Boolean)).into(),
841                NestedField::required(
842                    4,
843                    "qux",
844                    List(ListType {
845                        element_field: NestedField::list_element(
846                            5,
847                            Primitive(PrimitiveType::String),
848                            true,
849                        )
850                        .into(),
851                    }),
852                )
853                .into(),
854                NestedField::required(
855                    6,
856                    "quux",
857                    Map(MapType {
858                        key_field: NestedField::map_key_element(
859                            7,
860                            Primitive(PrimitiveType::String),
861                        )
862                        .into(),
863                        value_field: NestedField::map_value_element(
864                            8,
865                            Map(MapType {
866                                key_field: NestedField::map_key_element(
867                                    9,
868                                    Primitive(PrimitiveType::String),
869                                )
870                                .into(),
871                                value_field: NestedField::map_value_element(
872                                    10,
873                                    Primitive(PrimitiveType::Int),
874                                    true,
875                                )
876                                .into(),
877                            }),
878                            true,
879                        )
880                        .into(),
881                    }),
882                )
883                .into(),
884                NestedField::required(
885                    11,
886                    "location",
887                    List(ListType {
888                        element_field: NestedField::list_element(
889                            12,
890                            Struct(StructType::new(vec![
891                                NestedField::optional(
892                                    13,
893                                    "latitude",
894                                    Primitive(PrimitiveType::Float),
895                                )
896                                .into(),
897                                NestedField::optional(
898                                    14,
899                                    "longitude",
900                                    Primitive(PrimitiveType::Float),
901                                )
902                                .into(),
903                            ])),
904                            true,
905                        )
906                        .into(),
907                    }),
908                )
909                .into(),
910                NestedField::optional(
911                    15,
912                    "person",
913                    Struct(StructType::new(vec![
914                        NestedField::optional(16, "name", Primitive(PrimitiveType::String)).into(),
915                        NestedField::required(17, "age", Primitive(PrimitiveType::Int)).into(),
916                    ])),
917                )
918                .into(),
919            ])
920            .build()
921            .unwrap()
922    }
923
924    #[test]
925    fn test_schema_display() {
926        let expected_str = "
927table {
928  1: foo: optional string\x20
929  2: bar: required int\x20
930  3: baz: optional boolean\x20
931}
932";
933
934        assert_eq!(expected_str, format!("\n{}", table_schema_simple().0));
935    }
936
937    #[test]
938    fn test_schema_build_failed_on_duplicate_names() {
939        let ret = Schema::builder()
940            .with_schema_id(1)
941            .with_identifier_field_ids(vec![1])
942            .with_fields(vec![
943                NestedField::required(1, "foo", Primitive(PrimitiveType::String)).into(),
944                NestedField::required(2, "bar", Primitive(PrimitiveType::Int)).into(),
945                NestedField::optional(3, "baz", Primitive(PrimitiveType::Boolean)).into(),
946                NestedField::optional(4, "baz", Primitive(PrimitiveType::Boolean)).into(),
947            ])
948            .build();
949
950        assert!(
951            ret.unwrap_err()
952                .message()
953                .contains("Invalid schema: multiple fields for name baz")
954        );
955    }
956
957    #[test]
958    fn test_schema_into_builder() {
959        let original_schema = table_schema_nested();
960        let builder = original_schema.clone().into_builder();
961        let schema = builder.build().unwrap();
962
963        assert_eq!(original_schema, schema);
964    }
965
966    #[test]
967    fn test_schema_index_by_name() {
968        let expected_name_to_id = HashMap::from(
969            [
970                ("foo", 1),
971                ("bar", 2),
972                ("baz", 3),
973                ("qux", 4),
974                ("qux.element", 5),
975                ("quux", 6),
976                ("quux.key", 7),
977                ("quux.value", 8),
978                ("quux.value.key", 9),
979                ("quux.value.value", 10),
980                ("location", 11),
981                ("location.element", 12),
982                ("location.element.latitude", 13),
983                ("location.element.longitude", 14),
984                ("location.latitude", 13),
985                ("location.longitude", 14),
986                ("person", 15),
987                ("person.name", 16),
988                ("person.age", 17),
989            ]
990            .map(|e| (e.0.to_string(), e.1)),
991        );
992
993        let schema = table_schema_nested();
994        assert_eq!(&expected_name_to_id, &schema.name_to_id);
995    }
996
997    #[test]
998    fn test_schema_index_by_name_case_insensitive() {
999        let expected_name_to_id = HashMap::from(
1000            [
1001                ("fOo", 1),
1002                ("Bar", 2),
1003                ("BAz", 3),
1004                ("quX", 4),
1005                ("quX.ELEment", 5),
1006                ("qUUx", 6),
1007                ("QUUX.KEY", 7),
1008                ("QUUX.Value", 8),
1009                ("qUUX.VALUE.Key", 9),
1010                ("qUux.VaLue.Value", 10),
1011                ("lOCAtION", 11),
1012                ("LOCAtioN.ELeMENt", 12),
1013                ("LoCATion.element.LATitude", 13),
1014                ("locatION.ElemeNT.LONgitude", 14),
1015                ("LOCAtiON.LATITUDE", 13),
1016                ("LOCATION.LONGITUDE", 14),
1017                ("PERSon", 15),
1018                ("PERSON.Name", 16),
1019                ("peRSON.AGe", 17),
1020            ]
1021            .map(|e| (e.0.to_string(), e.1)),
1022        );
1023
1024        let schema = table_schema_nested();
1025        for (name, id) in expected_name_to_id {
1026            assert_eq!(
1027                Some(id),
1028                schema.field_by_name_case_insensitive(&name).map(|f| f.id)
1029            );
1030        }
1031    }
1032
1033    #[test]
1034    fn test_schema_field_by_name_case_insensitive_checked() {
1035        let schema = Schema::builder()
1036            .with_fields(vec![
1037                NestedField::optional(1, "id", Primitive(PrimitiveType::Int)).into(),
1038                NestedField::optional(2, "ID", Primitive(PrimitiveType::Int)).into(),
1039                NestedField::optional(3, "data", Primitive(PrimitiveType::String)).into(),
1040            ])
1041            .build()
1042            .expect("case-insensitive name ambiguity is valid in a schema");
1043
1044        assert_eq!(
1045            schema
1046                .field_by_name_case_insensitive_checked("DATA")
1047                .unwrap()
1048                .map(|field| field.id),
1049            Some(3)
1050        );
1051        assert!(
1052            schema
1053                .field_by_name_case_insensitive_checked("missing")
1054                .unwrap()
1055                .is_none()
1056        );
1057
1058        let error = schema
1059            .field_by_name_case_insensitive_checked("Id")
1060            .unwrap_err();
1061        assert_eq!(error.kind(), crate::ErrorKind::DataInvalid);
1062        assert!(error.message().contains("case-insensitively"), "{error}");
1063        assert!(error.message().contains("[\"ID\", \"id\"]"), "{error}");
1064        assert!(schema.field_by_name_case_insensitive("Id").is_none());
1065    }
1066
1067    #[test]
1068    fn test_schema_find_column_name() {
1069        let expected_column_name = HashMap::from([
1070            (1, "foo"),
1071            (2, "bar"),
1072            (3, "baz"),
1073            (4, "qux"),
1074            (5, "qux.element"),
1075            (6, "quux"),
1076            (7, "quux.key"),
1077            (8, "quux.value"),
1078            (9, "quux.value.key"),
1079            (10, "quux.value.value"),
1080            (11, "location"),
1081            (12, "location.element"),
1082            (13, "location.element.latitude"),
1083            (14, "location.element.longitude"),
1084        ]);
1085
1086        let schema = table_schema_nested();
1087        for (id, name) in expected_column_name {
1088            assert_eq!(
1089                Some(name),
1090                schema.name_by_field_id(id),
1091                "Column name for field id {id} not match."
1092            );
1093        }
1094    }
1095
1096    #[test]
1097    fn test_schema_find_column_name_not_found() {
1098        let schema = table_schema_nested();
1099
1100        assert!(schema.name_by_field_id(99).is_none());
1101    }
1102
1103    #[test]
1104    fn test_schema_find_column_name_by_id_simple() {
1105        let expected_id_to_name = HashMap::from([(1, "foo"), (2, "bar"), (3, "baz")]);
1106
1107        let schema = table_schema_simple().0;
1108
1109        for (id, name) in expected_id_to_name {
1110            assert_eq!(
1111                Some(name),
1112                schema.name_by_field_id(id),
1113                "Column name for field id {id} not match."
1114            );
1115        }
1116    }
1117
1118    #[test]
1119    fn test_schema_find_simple() {
1120        let schema = table_schema_simple().0;
1121
1122        assert_eq!(
1123            Some(schema.r#struct.fields()[0].clone()),
1124            schema.field_by_id(1).cloned()
1125        );
1126        assert_eq!(
1127            Some(schema.r#struct.fields()[1].clone()),
1128            schema.field_by_id(2).cloned()
1129        );
1130        assert_eq!(
1131            Some(schema.r#struct.fields()[2].clone()),
1132            schema.field_by_id(3).cloned()
1133        );
1134
1135        assert!(schema.field_by_id(4).is_none());
1136        assert!(schema.field_by_name("non exist").is_none());
1137    }
1138
1139    #[test]
1140    fn test_schema_find_nested() {
1141        let expected_id_to_field: HashMap<i32, NestedField> = HashMap::from([
1142            (
1143                1,
1144                NestedField::optional(1, "foo", Primitive(PrimitiveType::String)),
1145            ),
1146            (
1147                2,
1148                NestedField::required(2, "bar", Primitive(PrimitiveType::Int)),
1149            ),
1150            (
1151                3,
1152                NestedField::optional(3, "baz", Primitive(PrimitiveType::Boolean)),
1153            ),
1154            (
1155                4,
1156                NestedField::required(
1157                    4,
1158                    "qux",
1159                    List(ListType {
1160                        element_field: NestedField::list_element(
1161                            5,
1162                            Primitive(PrimitiveType::String),
1163                            true,
1164                        )
1165                        .into(),
1166                    }),
1167                ),
1168            ),
1169            (
1170                5,
1171                NestedField::required(5, "element", Primitive(PrimitiveType::String)),
1172            ),
1173            (
1174                6,
1175                NestedField::required(
1176                    6,
1177                    "quux",
1178                    Map(MapType {
1179                        key_field: NestedField::map_key_element(
1180                            7,
1181                            Primitive(PrimitiveType::String),
1182                        )
1183                        .into(),
1184                        value_field: NestedField::map_value_element(
1185                            8,
1186                            Map(MapType {
1187                                key_field: NestedField::map_key_element(
1188                                    9,
1189                                    Primitive(PrimitiveType::String),
1190                                )
1191                                .into(),
1192                                value_field: NestedField::map_value_element(
1193                                    10,
1194                                    Primitive(PrimitiveType::Int),
1195                                    true,
1196                                )
1197                                .into(),
1198                            }),
1199                            true,
1200                        )
1201                        .into(),
1202                    }),
1203                ),
1204            ),
1205            (
1206                7,
1207                NestedField::required(7, "key", Primitive(PrimitiveType::String)),
1208            ),
1209            (
1210                8,
1211                NestedField::required(
1212                    8,
1213                    "value",
1214                    Map(MapType {
1215                        key_field: NestedField::map_key_element(
1216                            9,
1217                            Primitive(PrimitiveType::String),
1218                        )
1219                        .into(),
1220                        value_field: NestedField::map_value_element(
1221                            10,
1222                            Primitive(PrimitiveType::Int),
1223                            true,
1224                        )
1225                        .into(),
1226                    }),
1227                ),
1228            ),
1229            (
1230                9,
1231                NestedField::required(9, "key", Primitive(PrimitiveType::String)),
1232            ),
1233            (
1234                10,
1235                NestedField::required(10, "value", Primitive(PrimitiveType::Int)),
1236            ),
1237            (
1238                11,
1239                NestedField::required(
1240                    11,
1241                    "location",
1242                    List(ListType {
1243                        element_field: NestedField::list_element(
1244                            12,
1245                            Struct(StructType::new(vec![
1246                                NestedField::optional(
1247                                    13,
1248                                    "latitude",
1249                                    Primitive(PrimitiveType::Float),
1250                                )
1251                                .into(),
1252                                NestedField::optional(
1253                                    14,
1254                                    "longitude",
1255                                    Primitive(PrimitiveType::Float),
1256                                )
1257                                .into(),
1258                            ])),
1259                            true,
1260                        )
1261                        .into(),
1262                    }),
1263                ),
1264            ),
1265            (
1266                12,
1267                NestedField::list_element(
1268                    12,
1269                    Struct(StructType::new(vec![
1270                        NestedField::optional(13, "latitude", Primitive(PrimitiveType::Float))
1271                            .into(),
1272                        NestedField::optional(14, "longitude", Primitive(PrimitiveType::Float))
1273                            .into(),
1274                    ])),
1275                    true,
1276                ),
1277            ),
1278            (
1279                13,
1280                NestedField::optional(13, "latitude", Primitive(PrimitiveType::Float)),
1281            ),
1282            (
1283                14,
1284                NestedField::optional(14, "longitude", Primitive(PrimitiveType::Float)),
1285            ),
1286            (
1287                15,
1288                NestedField::optional(
1289                    15,
1290                    "person",
1291                    Struct(StructType::new(vec![
1292                        NestedField::optional(16, "name", Primitive(PrimitiveType::String)).into(),
1293                        NestedField::required(17, "age", Primitive(PrimitiveType::Int)).into(),
1294                    ])),
1295                ),
1296            ),
1297            (
1298                16,
1299                NestedField::optional(16, "name", Primitive(PrimitiveType::String)),
1300            ),
1301            (
1302                17,
1303                NestedField::required(17, "age", Primitive(PrimitiveType::Int)),
1304            ),
1305        ]);
1306
1307        let schema = table_schema_nested();
1308        for (id, field) in expected_id_to_field {
1309            assert_eq!(
1310                Some(&field),
1311                schema.field_by_id(id).map(|f| f.as_ref()),
1312                "Field for {id} not match."
1313            );
1314        }
1315    }
1316
1317    #[test]
1318    fn test_build_accessors() {
1319        let schema = table_schema_nested();
1320
1321        let test_struct = crate::spec::Struct::from_iter(vec![
1322            Some(Literal::string("foo value")),
1323            Some(Literal::int(1002)),
1324            Some(Literal::bool(true)),
1325            Some(Literal::List(vec![
1326                Some(Literal::string("qux item 1")),
1327                Some(Literal::string("qux item 2")),
1328            ])),
1329            Some(Literal::Map(MapValue::from([(
1330                Literal::string("quux key 1"),
1331                Some(Literal::Map(MapValue::from([(
1332                    Literal::string("quux nested key 1"),
1333                    Some(Literal::int(1000)),
1334                )]))),
1335            )]))),
1336            Some(Literal::List(vec![Some(Literal::Struct(
1337                crate::spec::Struct::from_iter(vec![
1338                    Some(Literal::float(52.509_09_f32)),
1339                    Some(Literal::float(-1.885_249_f32)),
1340                ]),
1341            ))])),
1342            Some(Literal::Struct(crate::spec::Struct::from_iter(vec![
1343                Some(Literal::string("Testy McTest")),
1344                Some(Literal::int(33)),
1345            ]))),
1346        ]);
1347
1348        assert_eq!(
1349            schema
1350                .accessor_by_field_id(1)
1351                .unwrap()
1352                .get(&test_struct)
1353                .unwrap(),
1354            Some(Datum::string("foo value"))
1355        );
1356        assert_eq!(
1357            schema
1358                .accessor_by_field_id(2)
1359                .unwrap()
1360                .get(&test_struct)
1361                .unwrap(),
1362            Some(Datum::int(1002))
1363        );
1364        assert_eq!(
1365            schema
1366                .accessor_by_field_id(3)
1367                .unwrap()
1368                .get(&test_struct)
1369                .unwrap(),
1370            Some(Datum::bool(true))
1371        );
1372        assert_eq!(
1373            schema
1374                .accessor_by_field_id(16)
1375                .unwrap()
1376                .get(&test_struct)
1377                .unwrap(),
1378            Some(Datum::string("Testy McTest"))
1379        );
1380        assert_eq!(
1381            schema
1382                .accessor_by_field_id(17)
1383                .unwrap()
1384                .get(&test_struct)
1385                .unwrap(),
1386            Some(Datum::int(33))
1387        );
1388    }
1389
1390    #[test]
1391    fn test_highest_field_id() {
1392        let schema = table_schema_nested();
1393        assert_eq!(17, schema.highest_field_id());
1394
1395        let schema = table_schema_simple().0;
1396        assert_eq!(3, schema.highest_field_id());
1397    }
1398
1399    #[test]
1400    fn test_highest_field_id_no_fields() {
1401        let schema = Schema::builder().with_schema_id(1).build().unwrap();
1402        assert_eq!(0, schema.highest_field_id());
1403    }
1404
1405    #[test]
1406    fn test_field_ids_must_be_unique() {
1407        let reassigned_schema = Schema::builder()
1408            .with_schema_id(1)
1409            .with_identifier_field_ids(vec![5])
1410            .with_alias(BiHashMap::from_iter(vec![("bar_alias".to_string(), 3)]))
1411            .with_fields(vec![
1412                NestedField::required(5, "foo", Primitive(PrimitiveType::String)).into(),
1413                NestedField::optional(3, "bar", Primitive(PrimitiveType::Int)).into(),
1414                NestedField::optional(3, "baz", Primitive(PrimitiveType::Boolean)).into(),
1415            ])
1416            .build()
1417            .unwrap_err();
1418
1419        assert!(reassigned_schema.message().contains("'field.id' 3"));
1420    }
1421
1422    #[test]
1423    fn test_reassign_ids_empty_schema() {
1424        let schema = Schema::builder().with_schema_id(1).build().unwrap();
1425        let reassigned_schema = schema
1426            .clone()
1427            .into_builder()
1428            .with_reassigned_field_ids(0)
1429            .build()
1430            .unwrap();
1431
1432        assert_eq!(schema, reassigned_schema);
1433        assert_eq!(schema.highest_field_id(), 0);
1434    }
1435
1436    #[test]
1437    fn test_identifier_field_ids() {
1438        // field in map
1439        assert!(
1440            Schema::builder()
1441                .with_schema_id(1)
1442                .with_identifier_field_ids(vec![2])
1443                .with_fields(vec![
1444                    NestedField::required(
1445                        1,
1446                        "Map",
1447                        Map(MapType::new(
1448                            NestedField::map_key_element(2, Primitive(PrimitiveType::String))
1449                                .into(),
1450                            NestedField::map_value_element(
1451                                3,
1452                                Primitive(PrimitiveType::Boolean),
1453                                true,
1454                            )
1455                            .into(),
1456                        )),
1457                    )
1458                    .into()
1459                ])
1460                .build()
1461                .is_err()
1462        );
1463        assert!(
1464            Schema::builder()
1465                .with_schema_id(1)
1466                .with_identifier_field_ids(vec![3])
1467                .with_fields(vec![
1468                    NestedField::required(
1469                        1,
1470                        "Map",
1471                        Map(MapType::new(
1472                            NestedField::map_key_element(2, Primitive(PrimitiveType::String))
1473                                .into(),
1474                            NestedField::map_value_element(
1475                                3,
1476                                Primitive(PrimitiveType::Boolean),
1477                                true,
1478                            )
1479                            .into(),
1480                        )),
1481                    )
1482                    .into()
1483                ])
1484                .build()
1485                .is_err()
1486        );
1487
1488        // field in list
1489        assert!(
1490            Schema::builder()
1491                .with_schema_id(1)
1492                .with_identifier_field_ids(vec![2])
1493                .with_fields(vec![
1494                    NestedField::required(
1495                        1,
1496                        "List",
1497                        List(ListType::new(
1498                            NestedField::list_element(2, Primitive(PrimitiveType::String), true)
1499                                .into(),
1500                        )),
1501                    )
1502                    .into()
1503                ])
1504                .build()
1505                .is_err()
1506        );
1507
1508        // field in optional struct
1509        assert!(
1510            Schema::builder()
1511                .with_schema_id(1)
1512                .with_identifier_field_ids(vec![2])
1513                .with_fields(vec![
1514                    NestedField::optional(
1515                        1,
1516                        "Struct",
1517                        Struct(StructType::new(vec![
1518                            NestedField::required(2, "name", Primitive(PrimitiveType::String))
1519                                .into(),
1520                            NestedField::optional(3, "age", Primitive(PrimitiveType::Int)).into(),
1521                        ])),
1522                    )
1523                    .into()
1524                ])
1525                .build()
1526                .is_err()
1527        );
1528
1529        // float and double
1530        assert!(
1531            Schema::builder()
1532                .with_schema_id(1)
1533                .with_identifier_field_ids(vec![1])
1534                .with_fields(vec![
1535                    NestedField::required(1, "Float", Primitive(PrimitiveType::Float),).into()
1536                ])
1537                .build()
1538                .is_err()
1539        );
1540        assert!(
1541            Schema::builder()
1542                .with_schema_id(1)
1543                .with_identifier_field_ids(vec![1])
1544                .with_fields(vec![
1545                    NestedField::required(1, "Double", Primitive(PrimitiveType::Double),).into()
1546                ])
1547                .build()
1548                .is_err()
1549        );
1550
1551        // optional field
1552        assert!(
1553            Schema::builder()
1554                .with_schema_id(1)
1555                .with_identifier_field_ids(vec![1])
1556                .with_fields(vec![
1557                    NestedField::required(1, "Required", Primitive(PrimitiveType::String),).into()
1558                ])
1559                .build()
1560                .is_ok()
1561        );
1562        assert!(
1563            Schema::builder()
1564                .with_schema_id(1)
1565                .with_identifier_field_ids(vec![1])
1566                .with_fields(vec![
1567                    NestedField::optional(1, "Optional", Primitive(PrimitiveType::String),).into()
1568                ])
1569                .build()
1570                .is_err()
1571        );
1572    }
1573
1574    #[test]
1575    fn test_unknown_type_deserialization_rejects_non_null_default() {
1576        let field_json = serde_json::json!({
1577            "id": 1,
1578            "name": "empty",
1579            "required": false,
1580            "type": "unknown",
1581            "initial-default": 1
1582        });
1583
1584        let error = serde_json::from_value::<NestedField>(field_json.clone()).unwrap_err();
1585        assert!(
1586            error
1587                .to_string()
1588                .contains("Unknown type only supports null default values"),
1589            "unexpected error: {error}"
1590        );
1591
1592        let schema_json = serde_json::json!({
1593            "type": "struct",
1594            "schema-id": 1,
1595            "fields": [field_json]
1596        });
1597        assert!(serde_json::from_value::<Schema>(schema_json).is_err());
1598    }
1599
1600    #[test]
1601    fn test_unknown_type_deserialization_accepts_null_defaults() {
1602        let schema_json = serde_json::json!({
1603            "type": "struct",
1604            "schema-id": 1,
1605            "fields": [
1606                {
1607                    "id": 1,
1608                    "name": "empty",
1609                    "required": false,
1610                    "type": "unknown",
1611                    "initial-default": null,
1612                    "write-default": null
1613                }
1614            ]
1615        });
1616
1617        serde_json::from_value::<Schema>(schema_json).unwrap();
1618    }
1619
1620    #[test]
1621    fn test_unknown_type_accepts_null_container_defaults() {
1622        let cases = [
1623            (
1624                "struct",
1625                Struct(StructType::new(vec![
1626                    NestedField::optional(2, "empty", Primitive(PrimitiveType::Unknown)).into(),
1627                ])),
1628                Literal::Struct(crate::spec::Struct::from_iter([None])),
1629            ),
1630            (
1631                "list",
1632                List(ListType::new(
1633                    NestedField::list_element(2, Primitive(PrimitiveType::Unknown), false).into(),
1634                )),
1635                Literal::List(vec![None]),
1636            ),
1637            (
1638                "map",
1639                Map(MapType::optional(
1640                    2,
1641                    Primitive(PrimitiveType::String),
1642                    3,
1643                    Primitive(PrimitiveType::Unknown),
1644                )),
1645                Literal::Map(MapValue::from([(Literal::string("key"), None)])),
1646            ),
1647        ];
1648
1649        for (name, field_type, default) in cases {
1650            let schema = Schema::builder()
1651                .with_schema_id(1)
1652                .with_fields(vec![
1653                    NestedField::optional(1, name, field_type)
1654                        .with_write_default(default)
1655                        .into(),
1656                ])
1657                .build()
1658                .unwrap();
1659            serde_json::to_value(schema).unwrap();
1660        }
1661    }
1662
1663    #[test]
1664    fn test_unknown_type_deserialization_rejects_non_null_container_defaults() {
1665        let cases = [
1666            (
1667                "struct",
1668                serde_json::json!({
1669                    "type": "struct",
1670                    "fields": [{
1671                        "id": 2,
1672                        "name": "empty",
1673                        "required": false,
1674                        "type": "unknown"
1675                    }]
1676                }),
1677                serde_json::json!({"2": 1}),
1678            ),
1679            (
1680                "list",
1681                serde_json::json!({
1682                    "type": "list",
1683                    "element-id": 2,
1684                    "element-required": false,
1685                    "element": "unknown"
1686                }),
1687                serde_json::json!([1]),
1688            ),
1689            (
1690                "map",
1691                serde_json::json!({
1692                    "type": "map",
1693                    "key-id": 2,
1694                    "key": "string",
1695                    "value-id": 3,
1696                    "value-required": false,
1697                    "value": "unknown"
1698                }),
1699                serde_json::json!({"keys": ["key"], "values": [1]}),
1700            ),
1701        ];
1702
1703        for (name, field_type, default) in cases {
1704            let schema_json = serde_json::json!({
1705                "type": "struct",
1706                "schema-id": 1,
1707                "fields": [{
1708                    "id": 1,
1709                    "name": name,
1710                    "required": false,
1711                    "type": field_type,
1712                    "initial-default": default
1713                }]
1714            });
1715
1716            assert!(
1717                serde_json::from_value::<Schema>(schema_json).is_err(),
1718                "non-null unknown default in {name} should be rejected"
1719            );
1720        }
1721    }
1722}
iceberg/spec/schema/mod.rs

iceberg/spec/schema/
mod.rs
