iceberg/spec/schema/
mod.rs

1// Licensed to the Apache Software Foundation (ASF) under one
2// or more contributor license agreements.  See the NOTICE file
3// distributed with this work for additional information
4// regarding copyright ownership.  The ASF licenses this file
5// to you under the Apache License, Version 2.0 (the
6// "License"); you may not use this file except in compliance
7// with the License.  You may obtain a copy of the License at
8//
9//   http://www.apache.org/licenses/LICENSE-2.0
10//
11// Unless required by applicable law or agreed to in writing,
12// software distributed under the License is distributed on an
13// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14// KIND, either express or implied.  See the License for the
15// specific language governing permissions and limitations
16// under the License.
17
18//! This module defines schema in iceberg.
19
20use std::collections::{HashMap, HashSet};
21use std::fmt::{Display, Formatter};
22use std::sync::Arc;
23
24mod utils;
25mod visitor;
26pub use self::visitor::*;
27pub(super) mod _serde;
28mod id_reassigner;
29mod index;
30mod prune_columns;
31use bimap::BiHashMap;
32use itertools::{Itertools, zip_eq};
33use serde::{Deserialize, Serialize};
34
35use self::_serde::SchemaEnum;
36use self::id_reassigner::ReassignFieldIds;
37use self::index::{IndexByName, index_by_id, index_parents};
38pub use self::prune_columns::prune_columns;
39use super::NestedField;
40use crate::error::Result;
41use crate::expr::accessor::StructAccessor;
42use crate::spec::FormatVersion;
43use crate::spec::datatypes::{
44    LIST_FIELD_NAME, ListType, MAP_KEY_FIELD_NAME, MAP_VALUE_FIELD_NAME, MapType, NestedFieldRef,
45    PrimitiveType, StructType, Type,
46};
47use crate::{Error, ErrorKind, ensure_data_valid};
48
49/// Type alias for schema id.
50pub type SchemaId = i32;
51/// Reference to [`Schema`].
52pub type SchemaRef = Arc<Schema>;
53/// Default schema id.
54pub const DEFAULT_SCHEMA_ID: SchemaId = 0;
55/// Delimiter for schema name, which denotes a nested struct.
56pub const SCHEMA_NAME_DELIMITER: &str = ".";
57/// Minimum format version that allows non-null field default values.
58/// Mirrors Java's `Schema.DEFAULT_VALUES_MIN_FORMAT_VERSION`.
59pub(crate) const DEFAULT_VALUES_MIN_FORMAT_VERSION: FormatVersion = FormatVersion::V3;
60
61/// Defines schema in iceberg.
62#[derive(Debug, Serialize, Deserialize, Clone)]
63#[serde(try_from = "SchemaEnum", into = "SchemaEnum")]
64pub struct Schema {
65    r#struct: StructType,
66    schema_id: SchemaId,
67    highest_field_id: i32,
68    identifier_field_ids: HashSet<i32>,
69
70    alias_to_id: BiHashMap<String, i32>,
71    id_to_field: HashMap<i32, NestedFieldRef>,
72
73    name_to_id: HashMap<String, i32>,
74    lowercase_name_to_id: HashMap<String, i32>,
75    id_to_name: HashMap<i32, String>,
76
77    field_id_to_accessor: HashMap<i32, Arc<StructAccessor>>,
78}
79
80impl PartialEq for Schema {
81    fn eq(&self, other: &Self) -> bool {
82        self.r#struct == other.r#struct
83            && self.schema_id == other.schema_id
84            && self.identifier_field_ids == other.identifier_field_ids
85    }
86}
87
88impl Eq for Schema {}
89
90/// Schema builder.
91#[derive(Debug)]
92pub struct SchemaBuilder {
93    schema_id: i32,
94    fields: Vec<NestedFieldRef>,
95    alias_to_id: BiHashMap<String, i32>,
96    identifier_field_ids: HashSet<i32>,
97    reassign_field_ids_from: Option<i32>,
98}
99
100impl SchemaBuilder {
101    /// Add fields to schema builder.
102    pub fn with_fields(mut self, fields: impl IntoIterator<Item = NestedFieldRef>) -> Self {
103        self.fields.extend(fields);
104        self
105    }
106
107    /// Reassign all field-ids (including nested) on build.
108    /// Reassignment starts from the field-id specified in `start_from` (inclusive).
109    ///
110    /// All specified aliases and identifier fields will be updated to the new field-ids.
111    pub(crate) fn with_reassigned_field_ids(mut self, start_from: i32) -> Self {
112        self.reassign_field_ids_from = Some(start_from);
113        self
114    }
115
116    /// Set schema id.
117    pub fn with_schema_id(mut self, schema_id: i32) -> Self {
118        self.schema_id = schema_id;
119        self
120    }
121
122    /// Set identifier field ids.
123    pub fn with_identifier_field_ids(mut self, ids: impl IntoIterator<Item = i32>) -> Self {
124        self.identifier_field_ids.extend(ids);
125        self
126    }
127
128    /// Set alias to filed id mapping.
129    pub fn with_alias(mut self, alias_to_id: BiHashMap<String, i32>) -> Self {
130        self.alias_to_id = alias_to_id;
131        self
132    }
133
134    /// Builds the schema.
135    pub fn build(self) -> Result<Schema> {
136        let field_id_to_accessor = self.build_accessors();
137
138        let r#struct = StructType::new(self.fields);
139        let id_to_field = index_by_id(&r#struct)?;
140
141        Self::validate_identifier_ids(
142            &r#struct,
143            &id_to_field,
144            self.identifier_field_ids.iter().copied(),
145        )?;
146
147        let (name_to_id, id_to_name) = {
148            let mut index = IndexByName::default();
149            visit_struct(&r#struct, &mut index)?;
150            index.indexes()
151        };
152
153        let lowercase_name_to_id = name_to_id
154            .iter()
155            .map(|(k, v)| (k.to_lowercase(), *v))
156            .collect();
157
158        let highest_field_id = id_to_field.keys().max().cloned().unwrap_or(0);
159
160        let mut schema = Schema {
161            r#struct,
162            schema_id: self.schema_id,
163            highest_field_id,
164            identifier_field_ids: self.identifier_field_ids,
165            alias_to_id: self.alias_to_id,
166            id_to_field,
167
168            name_to_id,
169            lowercase_name_to_id,
170            id_to_name,
171
172            field_id_to_accessor,
173        };
174
175        if let Some(start_from) = self.reassign_field_ids_from {
176            let mut id_reassigner = ReassignFieldIds::new(start_from);
177            let new_fields = id_reassigner.reassign_field_ids(schema.r#struct.fields().to_vec())?;
178            let new_identifier_field_ids =
179                id_reassigner.apply_to_identifier_fields(schema.identifier_field_ids)?;
180            let new_alias_to_id = id_reassigner.apply_to_aliases(schema.alias_to_id.clone())?;
181
182            schema = Schema::builder()
183                .with_schema_id(schema.schema_id)
184                .with_fields(new_fields)
185                .with_identifier_field_ids(new_identifier_field_ids)
186                .with_alias(new_alias_to_id)
187                .build()?;
188        }
189
190        Ok(schema)
191    }
192
193    fn build_accessors(&self) -> HashMap<i32, Arc<StructAccessor>> {
194        let mut map = HashMap::new();
195
196        for (pos, field) in self.fields.iter().enumerate() {
197            match field.field_type.as_ref() {
198                Type::Primitive(prim_type) => {
199                    // add an accessor for this field
200                    let accessor = Arc::new(StructAccessor::new(pos, prim_type.clone()));
201                    map.insert(field.id, accessor.clone());
202                }
203
204                Type::Struct(nested) => {
205                    // add accessors for nested fields
206                    for (field_id, accessor) in Self::build_accessors_nested(nested.fields()) {
207                        let new_accessor = Arc::new(StructAccessor::wrap(pos, accessor));
208                        map.insert(field_id, new_accessor.clone());
209                    }
210                }
211                _ => {
212                    // Accessors don't get built for Map or List types
213                }
214            }
215        }
216
217        map
218    }
219
220    fn build_accessors_nested(fields: &[NestedFieldRef]) -> Vec<(i32, Box<StructAccessor>)> {
221        let mut results = vec![];
222        for (pos, field) in fields.iter().enumerate() {
223            match field.field_type.as_ref() {
224                Type::Primitive(prim_type) => {
225                    let accessor = Box::new(StructAccessor::new(pos, prim_type.clone()));
226                    results.push((field.id, accessor));
227                }
228                Type::Struct(nested) => {
229                    let nested_accessors = Self::build_accessors_nested(nested.fields());
230
231                    let wrapped_nested_accessors =
232                        nested_accessors.into_iter().map(|(id, accessor)| {
233                            let new_accessor = Box::new(StructAccessor::wrap(pos, accessor));
234                            (id, new_accessor.clone())
235                        });
236
237                    results.extend(wrapped_nested_accessors);
238                }
239                _ => {
240                    // Accessors don't get built for Map or List types
241                }
242            }
243        }
244
245        results
246    }
247
248    /// According to [the spec](https://iceberg.apache.org/spec/#identifier-fields), the identifier fields
249    /// must meet the following requirements:
250    /// - Float, double, and optional fields cannot be used as identifier fields.
251    /// - Identifier fields may be nested in structs but cannot be nested within maps or lists.
252    /// - A nested field cannot be used as an identifier field if it is nested in an optional struct, to avoid null values in identifiers.
253    fn validate_identifier_ids(
254        r#struct: &StructType,
255        id_to_field: &HashMap<i32, NestedFieldRef>,
256        identifier_field_ids: impl Iterator<Item = i32>,
257    ) -> Result<()> {
258        let id_to_parent = index_parents(r#struct)?;
259        for identifier_field_id in identifier_field_ids {
260            let field = id_to_field.get(&identifier_field_id).ok_or_else(|| {
261                Error::new(
262                    ErrorKind::DataInvalid,
263                    format!(
264                        "Cannot add identifier field {identifier_field_id}: field does not exist"
265                    ),
266                )
267            })?;
268            ensure_data_valid!(
269                field.required,
270                "Cannot add identifier field: {} is an optional field",
271                field.name
272            );
273            if let Type::Primitive(p) = field.field_type.as_ref() {
274                ensure_data_valid!(
275                    !matches!(p, PrimitiveType::Double | PrimitiveType::Float),
276                    "Cannot add identifier field {}: cannot be a float or double type",
277                    field.name
278                );
279            } else {
280                return Err(Error::new(
281                    ErrorKind::DataInvalid,
282                    format!(
283                        "Cannot add field {} as an identifier field: not a primitive type field",
284                        field.name
285                    ),
286                ));
287            }
288
289            let mut cur_field_id = identifier_field_id;
290            while let Some(parent) = id_to_parent.get(&cur_field_id) {
291                let parent_field = id_to_field
292                    .get(parent)
293                    .expect("Field id should not disappear.");
294                ensure_data_valid!(
295                    parent_field.field_type.is_struct(),
296                    "Cannot add field {} as an identifier field: must not be nested in {:?}",
297                    field.name,
298                    parent_field
299                );
300                ensure_data_valid!(
301                    parent_field.required,
302                    "Cannot add field {} as an identifier field: must not be nested in an optional field {}",
303                    field.name,
304                    parent_field
305                );
306                cur_field_id = *parent;
307            }
308        }
309
310        Ok(())
311    }
312}
313
314impl Schema {
315    /// Create a schema builder.
316    pub fn builder() -> SchemaBuilder {
317        SchemaBuilder {
318            schema_id: DEFAULT_SCHEMA_ID,
319            fields: vec![],
320            identifier_field_ids: HashSet::default(),
321            alias_to_id: BiHashMap::default(),
322            reassign_field_ids_from: None,
323        }
324    }
325
326    /// Create a new schema builder from a schema.
327    pub fn into_builder(self) -> SchemaBuilder {
328        SchemaBuilder {
329            schema_id: self.schema_id,
330            fields: self.r#struct.fields().to_vec(),
331            alias_to_id: self.alias_to_id,
332            identifier_field_ids: self.identifier_field_ids,
333            reassign_field_ids_from: None,
334        }
335    }
336
337    /// Get field by field id.
338    pub fn field_by_id(&self, field_id: i32) -> Option<&NestedFieldRef> {
339        self.id_to_field.get(&field_id)
340    }
341
342    /// Get field by field name.
343    ///
344    /// Both full name and short name could work here.
345    pub fn field_by_name(&self, field_name: &str) -> Option<&NestedFieldRef> {
346        self.name_to_id
347            .get(field_name)
348            .and_then(|id| self.field_by_id(*id))
349    }
350
351    /// Get field by field name, but in case-insensitive way.
352    ///
353    /// Both full name and short name could work here.
354    pub fn field_by_name_case_insensitive(&self, field_name: &str) -> Option<&NestedFieldRef> {
355        self.lowercase_name_to_id
356            .get(&field_name.to_lowercase())
357            .and_then(|id| self.field_by_id(*id))
358    }
359
360    /// Get field by alias.
361    pub fn field_by_alias(&self, alias: &str) -> Option<&NestedFieldRef> {
362        self.alias_to_id
363            .get_by_left(alias)
364            .and_then(|id| self.field_by_id(*id))
365    }
366
367    /// Returns [`highest_field_id`].
368    #[inline]
369    pub fn highest_field_id(&self) -> i32 {
370        self.highest_field_id
371    }
372
373    /// Returns [`schema_id`].
374    #[inline]
375    pub fn schema_id(&self) -> SchemaId {
376        self.schema_id
377    }
378
379    /// Returns [`r#struct`].
380    #[inline]
381    pub fn as_struct(&self) -> &StructType {
382        &self.r#struct
383    }
384
385    /// Returns [`identifier_field_ids`].
386    #[inline]
387    pub fn identifier_field_ids(&self) -> impl ExactSizeIterator<Item = i32> + '_ {
388        self.identifier_field_ids.iter().copied()
389    }
390
391    /// Get field id by full name.
392    pub fn field_id_by_name(&self, name: &str) -> Option<i32> {
393        self.name_to_id.get(name).copied()
394    }
395
396    /// Get full name by field id.
397    pub fn name_by_field_id(&self, field_id: i32) -> Option<&str> {
398        self.id_to_name.get(&field_id).map(String::as_str)
399    }
400
401    /// Get an accessor for retrieving data in a struct
402    pub fn accessor_by_field_id(&self, field_id: i32) -> Option<Arc<StructAccessor>> {
403        self.field_id_to_accessor.get(&field_id).cloned()
404    }
405
406    /// Check if this schema is identical to another schema semantically - excluding schema id.
407    pub(crate) fn is_same_schema(&self, other: &SchemaRef) -> bool {
408        self.as_struct().eq(other.as_struct())
409            && self.identifier_field_ids().eq(other.identifier_field_ids())
410    }
411
412    /// Change the schema id of this schema.
413    // This is redundant with the `with_schema_id` method on the builder, but useful
414    // as it is infallible in contrast to the builder `build()` method.
415    pub(crate) fn with_schema_id(self, schema_id: SchemaId) -> Self {
416        Self { schema_id, ..self }
417    }
418
419    /// Return A HashMap matching field ids to field names.
420    pub fn field_id_to_name_map(&self) -> &HashMap<i32, String> {
421        &self.id_to_name
422    }
423
424    /// Return a hashmap matching field ids to nested fields.
425    pub fn field_id_to_fields(&self) -> &HashMap<i32, NestedFieldRef> {
426        &self.id_to_field
427    }
428
429    /// Minimum [`FormatVersion`] required to represent all *types* in this schema.
430    ///
431    /// Walks the whole schema — every field, nested ones included — and returns the
432    /// highest per-type floor, so this is O(fields) rather than a cheap field read.
433    /// Types only; for initial-default version floors see [`Schema::check_format_compatibility`].
434    pub fn calc_min_compatible_format(&self) -> FormatVersion {
435        // `id_to_field` is flattened, so the max over all fields covers nested ones too.
436        self.id_to_field
437            .values()
438            .map(|f| f.field_type.min_format_version())
439            .max()
440            .unwrap_or(FormatVersion::V1)
441    }
442
443    /// Returns an error listing every field incompatible with `format_version`.
444    /// Mirrors Java's `Schema.checkCompatibility()`. Two checks per field:
445    ///
446    /// - **Type** — minimum format version required to support that type, without
447    ///   taking nested field types into account.
448    /// - **Initial default** — a non-null `initial_default` backfills pre-existing rows,
449    ///   so it requires `DEFAULT_VALUES_MIN_FORMAT_VERSION`; `write_default` is not
450    ///   checked, as it only affects newly written rows (read identically at any version).
451    pub fn check_format_compatibility(&self, format_version: FormatVersion) -> Result<()> {
452        // (field id, message); sorted by id below for a deterministic error.
453        let mut problems: Vec<(i32, String)> = Vec::new();
454
455        // `id_to_field` is flattened, so checking each field by its own type keeps the
456        // blame on the offending leaf, not its container (mirrors Java's `lazyIdToField`).
457        for field in self.id_to_field.values() {
458            let min_version = field.field_type.min_format_version();
459            if format_version < min_version {
460                // Every id in `id_to_field` is also indexed in `id_to_name`; a miss means
461                // the schema's indexes are inconsistent (a bug), so surface it rather than
462                // guessing an unqualified name.
463                let name = self.name_by_field_id(field.id).ok_or_else(|| {
464                    Error::new(
465                        ErrorKind::Unexpected,
466                        format!(
467                            "Field id {} is missing from the schema's name index",
468                            field.id
469                        ),
470                    )
471                })?;
472                problems.push((field.id, format!(
473                    "Invalid type for {name}: {} is not supported until {min_version} but format version is {format_version}.",
474                    field.field_type,
475                )));
476            }
477
478            if let Some(default) = &field.initial_default
479                && format_version < DEFAULT_VALUES_MIN_FORMAT_VERSION
480            {
481                let name = self.name_by_field_id(field.id).ok_or_else(|| {
482                    Error::new(
483                        ErrorKind::Unexpected,
484                        format!(
485                            "Field id {} is missing from the schema's name index",
486                            field.id
487                        ),
488                    )
489                })?;
490                problems.push((field.id, format!(
491                    "Invalid initial default for {name}: non-null default ({default:?}) is not supported until {DEFAULT_VALUES_MIN_FORMAT_VERSION} but format version is {format_version}."
492                )));
493            }
494        }
495
496        if problems.is_empty() {
497            return Ok(());
498        }
499
500        // Stable sort by id: HashMap order is nondeterministic, and stability keeps a
501        // field's type problem before its default problem (matches Java's TreeMap order).
502        let message = problems
503            .into_iter()
504            .sorted_by_key(|(id, _)| *id)
505            .map(|(_, msg)| msg)
506            .join("\n- ");
507        Err(Error::new(
508            ErrorKind::DataInvalid,
509            format!("Invalid schema for {format_version}:\n- {message}"),
510        ))
511    }
512}
513
514impl Display for Schema {
515    fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result {
516        writeln!(f, "table {{")?;
517        for field in self.as_struct().fields() {
518            writeln!(f, "  {field}")?;
519        }
520        writeln!(f, "}}")
521    }
522}
523
524#[cfg(test)]
525mod tests {
526    use std::collections::HashMap;
527
528    use bimap::BiHashMap;
529
530    use crate::spec::datatypes::Type::{List, Map, Primitive, Struct, Variant};
531    use crate::spec::datatypes::{
532        ListType, MapType, NestedField, NestedFieldRef, PrimitiveType, StructType,
533    };
534    use crate::spec::schema::Schema;
535    use crate::spec::values::Map as MapValue;
536    use crate::spec::{Datum, Literal};
537
538    #[test]
539    fn test_check_format_compatibility() {
540        use crate::spec::{FormatVersion, PrimitiveLiteral, VariantType};
541
542        fn schema_with(fields: Vec<NestedFieldRef>) -> Schema {
543            Schema::builder().with_fields(fields).build().unwrap()
544        }
545
546        // Variant type requires v3.
547        let variant = schema_with(vec![
548            NestedField::optional(1, "v", Variant(VariantType)).into(),
549        ]);
550        assert!(
551            variant
552                .check_format_compatibility(FormatVersion::V2)
553                .is_err()
554        );
555        assert!(
556            variant
557                .check_format_compatibility(FormatVersion::V3)
558                .is_ok()
559        );
560
561        // A non-null initial default requires v3, even for a v1-compatible type.
562        let with_default = schema_with(vec![
563            NestedField::optional(1, "a", Primitive(PrimitiveType::Int))
564                .with_initial_default(Literal::Primitive(PrimitiveLiteral::Int(1)))
565                .into(),
566        ]);
567        let err = with_default
568            .check_format_compatibility(FormatVersion::V2)
569            .unwrap_err();
570        assert!(
571            err.message().contains("Invalid initial default for a"),
572            "{err}"
573        );
574        assert!(
575            with_default
576                .check_format_compatibility(FormatVersion::V3)
577                .is_ok()
578        );
579
580        // No default is fine at any version (an absent/null default never trips).
581        let no_default = schema_with(vec![
582            NestedField::optional(1, "a", Primitive(PrimitiveType::Int)).into(),
583        ]);
584        assert!(
585            no_default
586                .check_format_compatibility(FormatVersion::V1)
587                .is_ok()
588        );
589
590        // Recursion: a default on a field nested inside a struct is caught.
591        let nested = schema_with(vec![
592            NestedField::required(
593                1,
594                "s",
595                Struct(StructType::new(vec![
596                    NestedField::optional(2, "inner", Primitive(PrimitiveType::Long))
597                        .with_initial_default(Literal::Primitive(PrimitiveLiteral::Long(7)))
598                        .into(),
599                ])),
600            )
601            .into(),
602        ]);
603        let err = nested
604            .check_format_compatibility(FormatVersion::V2)
605            .unwrap_err();
606        assert!(err.message().contains("inner"), "{err}");
607
608        // A v3-only type nested in a container blames the leaf, never the container.
609        let nested_variant = schema_with(vec![
610            NestedField::required(
611                1,
612                "container",
613                Struct(StructType::new(vec![
614                    NestedField::optional(2, "v", Variant(VariantType)).into(),
615                ])),
616            )
617            .into(),
618        ]);
619        let err = nested_variant
620            .check_format_compatibility(FormatVersion::V2)
621            .unwrap_err();
622        assert!(err.message().contains("container.v"), "{err}");
623        assert!(
624            !err.message().contains("Invalid type for container:"),
625            "container must not be blamed: {err}"
626        );
627    }
628
629    #[test]
630    fn test_calc_min_compatible_format() {
631        use crate::spec::{FormatVersion, VariantType};
632
633        fn schema_with(fields: Vec<NestedFieldRef>) -> Schema {
634            Schema::builder().with_fields(fields).build().unwrap()
635        }
636
637        // All v1-compatible types → V1.
638        let v1 = schema_with(vec![
639            NestedField::required(1, "a", Primitive(PrimitiveType::Int)).into(),
640            NestedField::optional(2, "b", Primitive(PrimitiveType::String)).into(),
641        ]);
642        assert_eq!(v1.calc_min_compatible_format(), FormatVersion::V1);
643
644        // A top-level variant → V3.
645        let variant = schema_with(vec![
646            NestedField::optional(1, "v", Variant(VariantType)).into(),
647        ]);
648        assert_eq!(variant.calc_min_compatible_format(), FormatVersion::V3);
649
650        // A v3-only type nested inside a list inside a struct → V3 (flattened fields).
651        let nested = schema_with(vec![
652            NestedField::required(
653                1,
654                "s",
655                Struct(StructType::new(vec![
656                    NestedField::optional(
657                        2,
658                        "l",
659                        List(ListType::new(
660                            NestedField::required(
661                                3,
662                                "element",
663                                Primitive(PrimitiveType::TimestampNs),
664                            )
665                            .into(),
666                        )),
667                    )
668                    .into(),
669                ])),
670            )
671            .into(),
672        ]);
673        assert_eq!(nested.calc_min_compatible_format(), FormatVersion::V3);
674    }
675
676    #[test]
677    fn test_check_format_compatibility_message_order() {
678        use crate::spec::{FormatVersion, PrimitiveLiteral, VariantType};
679
680        // Fields declared out of id order; field 2 fails both checks (ts_ns type
681        // AND a non-null initial default). The error must list problems by field id,
682        // with the type problem before the initial-default problem for field 2.
683        let schema = Schema::builder()
684            .with_fields(vec![
685                NestedField::optional(3, "c", Variant(VariantType)).into(),
686                NestedField::optional(2, "b", Primitive(PrimitiveType::TimestampNs))
687                    .with_initial_default(Literal::Primitive(PrimitiveLiteral::Long(0)))
688                    .into(),
689                NestedField::required(1, "a", Primitive(PrimitiveType::Int)).into(),
690            ])
691            .build()
692            .unwrap();
693
694        let message = schema
695            .check_format_compatibility(FormatVersion::V2)
696            .unwrap_err()
697            .message()
698            .to_string();
699
700        let lines: Vec<&str> = message.lines().skip(1).collect();
701        assert_eq!(
702            lines,
703            vec![
704                "- Invalid type for b: timestamp_ns is not supported until v3 but format version is v2.",
705                "- Invalid initial default for b: non-null default (Primitive(Long(0))) is not supported until v3 but format version is v2.",
706                "- Invalid type for c: variant is not supported until v3 but format version is v2.",
707            ],
708            "{message}"
709        );
710    }
711
712    #[test]
713    fn test_construct_schema() {
714        let field1: NestedFieldRef =
715            NestedField::required(1, "f1", Primitive(PrimitiveType::Boolean)).into();
716        let field2: NestedFieldRef =
717            NestedField::optional(2, "f2", Primitive(PrimitiveType::Int)).into();
718
719        let schema = Schema::builder()
720            .with_fields(vec![field1.clone()])
721            .with_fields(vec![field2.clone()])
722            .with_schema_id(3)
723            .build()
724            .unwrap();
725
726        assert_eq!(3, schema.schema_id());
727        assert_eq!(2, schema.highest_field_id());
728        assert_eq!(Some(&field1), schema.field_by_id(1));
729        assert_eq!(Some(&field2), schema.field_by_id(2));
730        assert_eq!(None, schema.field_by_id(3));
731    }
732
733    pub fn table_schema_simple<'a>() -> (Schema, &'a str) {
734        let schema = Schema::builder()
735            .with_schema_id(1)
736            .with_identifier_field_ids(vec![2])
737            .with_fields(vec![
738                NestedField::optional(1, "foo", Primitive(PrimitiveType::String)).into(),
739                NestedField::required(2, "bar", Primitive(PrimitiveType::Int)).into(),
740                NestedField::optional(3, "baz", Primitive(PrimitiveType::Boolean)).into(),
741            ])
742            .build()
743            .unwrap();
744        let record = r#"{
745            "type":"struct",
746            "schema-id":1,
747            "fields":[
748                {
749                    "id":1,
750                    "name":"foo",
751                    "required":false,
752                    "type":"string"
753                },
754                {
755                    "id":2,
756                    "name":"bar",
757                    "required":true,
758                    "type":"int"
759                },
760                {
761                    "id":3,
762                    "name":"baz",
763                    "required":false,
764                    "type":"boolean"
765                }
766            ],
767            "identifier-field-ids":[2]
768        }"#;
769        (schema, record)
770    }
771
772    pub fn table_schema_nested() -> Schema {
773        Schema::builder()
774            .with_schema_id(1)
775            .with_identifier_field_ids(vec![2])
776            .with_fields(vec![
777                NestedField::optional(1, "foo", Primitive(PrimitiveType::String)).into(),
778                NestedField::required(2, "bar", Primitive(PrimitiveType::Int)).into(),
779                NestedField::optional(3, "baz", Primitive(PrimitiveType::Boolean)).into(),
780                NestedField::required(
781                    4,
782                    "qux",
783                    List(ListType {
784                        element_field: NestedField::list_element(
785                            5,
786                            Primitive(PrimitiveType::String),
787                            true,
788                        )
789                        .into(),
790                    }),
791                )
792                .into(),
793                NestedField::required(
794                    6,
795                    "quux",
796                    Map(MapType {
797                        key_field: NestedField::map_key_element(
798                            7,
799                            Primitive(PrimitiveType::String),
800                        )
801                        .into(),
802                        value_field: NestedField::map_value_element(
803                            8,
804                            Map(MapType {
805                                key_field: NestedField::map_key_element(
806                                    9,
807                                    Primitive(PrimitiveType::String),
808                                )
809                                .into(),
810                                value_field: NestedField::map_value_element(
811                                    10,
812                                    Primitive(PrimitiveType::Int),
813                                    true,
814                                )
815                                .into(),
816                            }),
817                            true,
818                        )
819                        .into(),
820                    }),
821                )
822                .into(),
823                NestedField::required(
824                    11,
825                    "location",
826                    List(ListType {
827                        element_field: NestedField::list_element(
828                            12,
829                            Struct(StructType::new(vec![
830                                NestedField::optional(
831                                    13,
832                                    "latitude",
833                                    Primitive(PrimitiveType::Float),
834                                )
835                                .into(),
836                                NestedField::optional(
837                                    14,
838                                    "longitude",
839                                    Primitive(PrimitiveType::Float),
840                                )
841                                .into(),
842                            ])),
843                            true,
844                        )
845                        .into(),
846                    }),
847                )
848                .into(),
849                NestedField::optional(
850                    15,
851                    "person",
852                    Struct(StructType::new(vec![
853                        NestedField::optional(16, "name", Primitive(PrimitiveType::String)).into(),
854                        NestedField::required(17, "age", Primitive(PrimitiveType::Int)).into(),
855                    ])),
856                )
857                .into(),
858            ])
859            .build()
860            .unwrap()
861    }
862
863    #[test]
864    fn test_schema_display() {
865        let expected_str = "
866table {
867  1: foo: optional string\x20
868  2: bar: required int\x20
869  3: baz: optional boolean\x20
870}
871";
872
873        assert_eq!(expected_str, format!("\n{}", table_schema_simple().0));
874    }
875
876    #[test]
877    fn test_schema_build_failed_on_duplicate_names() {
878        let ret = Schema::builder()
879            .with_schema_id(1)
880            .with_identifier_field_ids(vec![1])
881            .with_fields(vec![
882                NestedField::required(1, "foo", Primitive(PrimitiveType::String)).into(),
883                NestedField::required(2, "bar", Primitive(PrimitiveType::Int)).into(),
884                NestedField::optional(3, "baz", Primitive(PrimitiveType::Boolean)).into(),
885                NestedField::optional(4, "baz", Primitive(PrimitiveType::Boolean)).into(),
886            ])
887            .build();
888
889        assert!(
890            ret.unwrap_err()
891                .message()
892                .contains("Invalid schema: multiple fields for name baz")
893        );
894    }
895
896    #[test]
897    fn test_schema_into_builder() {
898        let original_schema = table_schema_nested();
899        let builder = original_schema.clone().into_builder();
900        let schema = builder.build().unwrap();
901
902        assert_eq!(original_schema, schema);
903    }
904
905    #[test]
906    fn test_schema_index_by_name() {
907        let expected_name_to_id = HashMap::from(
908            [
909                ("foo", 1),
910                ("bar", 2),
911                ("baz", 3),
912                ("qux", 4),
913                ("qux.element", 5),
914                ("quux", 6),
915                ("quux.key", 7),
916                ("quux.value", 8),
917                ("quux.value.key", 9),
918                ("quux.value.value", 10),
919                ("location", 11),
920                ("location.element", 12),
921                ("location.element.latitude", 13),
922                ("location.element.longitude", 14),
923                ("location.latitude", 13),
924                ("location.longitude", 14),
925                ("person", 15),
926                ("person.name", 16),
927                ("person.age", 17),
928            ]
929            .map(|e| (e.0.to_string(), e.1)),
930        );
931
932        let schema = table_schema_nested();
933        assert_eq!(&expected_name_to_id, &schema.name_to_id);
934    }
935
936    #[test]
937    fn test_schema_index_by_name_case_insensitive() {
938        let expected_name_to_id = HashMap::from(
939            [
940                ("fOo", 1),
941                ("Bar", 2),
942                ("BAz", 3),
943                ("quX", 4),
944                ("quX.ELEment", 5),
945                ("qUUx", 6),
946                ("QUUX.KEY", 7),
947                ("QUUX.Value", 8),
948                ("qUUX.VALUE.Key", 9),
949                ("qUux.VaLue.Value", 10),
950                ("lOCAtION", 11),
951                ("LOCAtioN.ELeMENt", 12),
952                ("LoCATion.element.LATitude", 13),
953                ("locatION.ElemeNT.LONgitude", 14),
954                ("LOCAtiON.LATITUDE", 13),
955                ("LOCATION.LONGITUDE", 14),
956                ("PERSon", 15),
957                ("PERSON.Name", 16),
958                ("peRSON.AGe", 17),
959            ]
960            .map(|e| (e.0.to_string(), e.1)),
961        );
962
963        let schema = table_schema_nested();
964        for (name, id) in expected_name_to_id {
965            assert_eq!(
966                Some(id),
967                schema.field_by_name_case_insensitive(&name).map(|f| f.id)
968            );
969        }
970    }
971
972    #[test]
973    fn test_schema_find_column_name() {
974        let expected_column_name = HashMap::from([
975            (1, "foo"),
976            (2, "bar"),
977            (3, "baz"),
978            (4, "qux"),
979            (5, "qux.element"),
980            (6, "quux"),
981            (7, "quux.key"),
982            (8, "quux.value"),
983            (9, "quux.value.key"),
984            (10, "quux.value.value"),
985            (11, "location"),
986            (12, "location.element"),
987            (13, "location.element.latitude"),
988            (14, "location.element.longitude"),
989        ]);
990
991        let schema = table_schema_nested();
992        for (id, name) in expected_column_name {
993            assert_eq!(
994                Some(name),
995                schema.name_by_field_id(id),
996                "Column name for field id {id} not match."
997            );
998        }
999    }
1000
1001    #[test]
1002    fn test_schema_find_column_name_not_found() {
1003        let schema = table_schema_nested();
1004
1005        assert!(schema.name_by_field_id(99).is_none());
1006    }
1007
1008    #[test]
1009    fn test_schema_find_column_name_by_id_simple() {
1010        let expected_id_to_name = HashMap::from([(1, "foo"), (2, "bar"), (3, "baz")]);
1011
1012        let schema = table_schema_simple().0;
1013
1014        for (id, name) in expected_id_to_name {
1015            assert_eq!(
1016                Some(name),
1017                schema.name_by_field_id(id),
1018                "Column name for field id {id} not match."
1019            );
1020        }
1021    }
1022
1023    #[test]
1024    fn test_schema_find_simple() {
1025        let schema = table_schema_simple().0;
1026
1027        assert_eq!(
1028            Some(schema.r#struct.fields()[0].clone()),
1029            schema.field_by_id(1).cloned()
1030        );
1031        assert_eq!(
1032            Some(schema.r#struct.fields()[1].clone()),
1033            schema.field_by_id(2).cloned()
1034        );
1035        assert_eq!(
1036            Some(schema.r#struct.fields()[2].clone()),
1037            schema.field_by_id(3).cloned()
1038        );
1039
1040        assert!(schema.field_by_id(4).is_none());
1041        assert!(schema.field_by_name("non exist").is_none());
1042    }
1043
1044    #[test]
1045    fn test_schema_find_nested() {
1046        let expected_id_to_field: HashMap<i32, NestedField> = HashMap::from([
1047            (
1048                1,
1049                NestedField::optional(1, "foo", Primitive(PrimitiveType::String)),
1050            ),
1051            (
1052                2,
1053                NestedField::required(2, "bar", Primitive(PrimitiveType::Int)),
1054            ),
1055            (
1056                3,
1057                NestedField::optional(3, "baz", Primitive(PrimitiveType::Boolean)),
1058            ),
1059            (
1060                4,
1061                NestedField::required(
1062                    4,
1063                    "qux",
1064                    List(ListType {
1065                        element_field: NestedField::list_element(
1066                            5,
1067                            Primitive(PrimitiveType::String),
1068                            true,
1069                        )
1070                        .into(),
1071                    }),
1072                ),
1073            ),
1074            (
1075                5,
1076                NestedField::required(5, "element", Primitive(PrimitiveType::String)),
1077            ),
1078            (
1079                6,
1080                NestedField::required(
1081                    6,
1082                    "quux",
1083                    Map(MapType {
1084                        key_field: NestedField::map_key_element(
1085                            7,
1086                            Primitive(PrimitiveType::String),
1087                        )
1088                        .into(),
1089                        value_field: NestedField::map_value_element(
1090                            8,
1091                            Map(MapType {
1092                                key_field: NestedField::map_key_element(
1093                                    9,
1094                                    Primitive(PrimitiveType::String),
1095                                )
1096                                .into(),
1097                                value_field: NestedField::map_value_element(
1098                                    10,
1099                                    Primitive(PrimitiveType::Int),
1100                                    true,
1101                                )
1102                                .into(),
1103                            }),
1104                            true,
1105                        )
1106                        .into(),
1107                    }),
1108                ),
1109            ),
1110            (
1111                7,
1112                NestedField::required(7, "key", Primitive(PrimitiveType::String)),
1113            ),
1114            (
1115                8,
1116                NestedField::required(
1117                    8,
1118                    "value",
1119                    Map(MapType {
1120                        key_field: NestedField::map_key_element(
1121                            9,
1122                            Primitive(PrimitiveType::String),
1123                        )
1124                        .into(),
1125                        value_field: NestedField::map_value_element(
1126                            10,
1127                            Primitive(PrimitiveType::Int),
1128                            true,
1129                        )
1130                        .into(),
1131                    }),
1132                ),
1133            ),
1134            (
1135                9,
1136                NestedField::required(9, "key", Primitive(PrimitiveType::String)),
1137            ),
1138            (
1139                10,
1140                NestedField::required(10, "value", Primitive(PrimitiveType::Int)),
1141            ),
1142            (
1143                11,
1144                NestedField::required(
1145                    11,
1146                    "location",
1147                    List(ListType {
1148                        element_field: NestedField::list_element(
1149                            12,
1150                            Struct(StructType::new(vec![
1151                                NestedField::optional(
1152                                    13,
1153                                    "latitude",
1154                                    Primitive(PrimitiveType::Float),
1155                                )
1156                                .into(),
1157                                NestedField::optional(
1158                                    14,
1159                                    "longitude",
1160                                    Primitive(PrimitiveType::Float),
1161                                )
1162                                .into(),
1163                            ])),
1164                            true,
1165                        )
1166                        .into(),
1167                    }),
1168                ),
1169            ),
1170            (
1171                12,
1172                NestedField::list_element(
1173                    12,
1174                    Struct(StructType::new(vec![
1175                        NestedField::optional(13, "latitude", Primitive(PrimitiveType::Float))
1176                            .into(),
1177                        NestedField::optional(14, "longitude", Primitive(PrimitiveType::Float))
1178                            .into(),
1179                    ])),
1180                    true,
1181                ),
1182            ),
1183            (
1184                13,
1185                NestedField::optional(13, "latitude", Primitive(PrimitiveType::Float)),
1186            ),
1187            (
1188                14,
1189                NestedField::optional(14, "longitude", Primitive(PrimitiveType::Float)),
1190            ),
1191            (
1192                15,
1193                NestedField::optional(
1194                    15,
1195                    "person",
1196                    Struct(StructType::new(vec![
1197                        NestedField::optional(16, "name", Primitive(PrimitiveType::String)).into(),
1198                        NestedField::required(17, "age", Primitive(PrimitiveType::Int)).into(),
1199                    ])),
1200                ),
1201            ),
1202            (
1203                16,
1204                NestedField::optional(16, "name", Primitive(PrimitiveType::String)),
1205            ),
1206            (
1207                17,
1208                NestedField::required(17, "age", Primitive(PrimitiveType::Int)),
1209            ),
1210        ]);
1211
1212        let schema = table_schema_nested();
1213        for (id, field) in expected_id_to_field {
1214            assert_eq!(
1215                Some(&field),
1216                schema.field_by_id(id).map(|f| f.as_ref()),
1217                "Field for {id} not match."
1218            );
1219        }
1220    }
1221
1222    #[test]
1223    fn test_build_accessors() {
1224        let schema = table_schema_nested();
1225
1226        let test_struct = crate::spec::Struct::from_iter(vec![
1227            Some(Literal::string("foo value")),
1228            Some(Literal::int(1002)),
1229            Some(Literal::bool(true)),
1230            Some(Literal::List(vec![
1231                Some(Literal::string("qux item 1")),
1232                Some(Literal::string("qux item 2")),
1233            ])),
1234            Some(Literal::Map(MapValue::from([(
1235                Literal::string("quux key 1"),
1236                Some(Literal::Map(MapValue::from([(
1237                    Literal::string("quux nested key 1"),
1238                    Some(Literal::int(1000)),
1239                )]))),
1240            )]))),
1241            Some(Literal::List(vec![Some(Literal::Struct(
1242                crate::spec::Struct::from_iter(vec![
1243                    Some(Literal::float(52.509_09)),
1244                    Some(Literal::float(-1.885_249)),
1245                ]),
1246            ))])),
1247            Some(Literal::Struct(crate::spec::Struct::from_iter(vec![
1248                Some(Literal::string("Testy McTest")),
1249                Some(Literal::int(33)),
1250            ]))),
1251        ]);
1252
1253        assert_eq!(
1254            schema
1255                .accessor_by_field_id(1)
1256                .unwrap()
1257                .get(&test_struct)
1258                .unwrap(),
1259            Some(Datum::string("foo value"))
1260        );
1261        assert_eq!(
1262            schema
1263                .accessor_by_field_id(2)
1264                .unwrap()
1265                .get(&test_struct)
1266                .unwrap(),
1267            Some(Datum::int(1002))
1268        );
1269        assert_eq!(
1270            schema
1271                .accessor_by_field_id(3)
1272                .unwrap()
1273                .get(&test_struct)
1274                .unwrap(),
1275            Some(Datum::bool(true))
1276        );
1277        assert_eq!(
1278            schema
1279                .accessor_by_field_id(16)
1280                .unwrap()
1281                .get(&test_struct)
1282                .unwrap(),
1283            Some(Datum::string("Testy McTest"))
1284        );
1285        assert_eq!(
1286            schema
1287                .accessor_by_field_id(17)
1288                .unwrap()
1289                .get(&test_struct)
1290                .unwrap(),
1291            Some(Datum::int(33))
1292        );
1293    }
1294
1295    #[test]
1296    fn test_highest_field_id() {
1297        let schema = table_schema_nested();
1298        assert_eq!(17, schema.highest_field_id());
1299
1300        let schema = table_schema_simple().0;
1301        assert_eq!(3, schema.highest_field_id());
1302    }
1303
1304    #[test]
1305    fn test_highest_field_id_no_fields() {
1306        let schema = Schema::builder().with_schema_id(1).build().unwrap();
1307        assert_eq!(0, schema.highest_field_id());
1308    }
1309
1310    #[test]
1311    fn test_field_ids_must_be_unique() {
1312        let reassigned_schema = Schema::builder()
1313            .with_schema_id(1)
1314            .with_identifier_field_ids(vec![5])
1315            .with_alias(BiHashMap::from_iter(vec![("bar_alias".to_string(), 3)]))
1316            .with_fields(vec![
1317                NestedField::required(5, "foo", Primitive(PrimitiveType::String)).into(),
1318                NestedField::optional(3, "bar", Primitive(PrimitiveType::Int)).into(),
1319                NestedField::optional(3, "baz", Primitive(PrimitiveType::Boolean)).into(),
1320            ])
1321            .build()
1322            .unwrap_err();
1323
1324        assert!(reassigned_schema.message().contains("'field.id' 3"));
1325    }
1326
1327    #[test]
1328    fn test_reassign_ids_empty_schema() {
1329        let schema = Schema::builder().with_schema_id(1).build().unwrap();
1330        let reassigned_schema = schema
1331            .clone()
1332            .into_builder()
1333            .with_reassigned_field_ids(0)
1334            .build()
1335            .unwrap();
1336
1337        assert_eq!(schema, reassigned_schema);
1338        assert_eq!(schema.highest_field_id(), 0);
1339    }
1340
1341    #[test]
1342    fn test_identifier_field_ids() {
1343        // field in map
1344        assert!(
1345            Schema::builder()
1346                .with_schema_id(1)
1347                .with_identifier_field_ids(vec![2])
1348                .with_fields(vec![
1349                    NestedField::required(
1350                        1,
1351                        "Map",
1352                        Map(MapType::new(
1353                            NestedField::map_key_element(2, Primitive(PrimitiveType::String))
1354                                .into(),
1355                            NestedField::map_value_element(
1356                                3,
1357                                Primitive(PrimitiveType::Boolean),
1358                                true,
1359                            )
1360                            .into(),
1361                        )),
1362                    )
1363                    .into()
1364                ])
1365                .build()
1366                .is_err()
1367        );
1368        assert!(
1369            Schema::builder()
1370                .with_schema_id(1)
1371                .with_identifier_field_ids(vec![3])
1372                .with_fields(vec![
1373                    NestedField::required(
1374                        1,
1375                        "Map",
1376                        Map(MapType::new(
1377                            NestedField::map_key_element(2, Primitive(PrimitiveType::String))
1378                                .into(),
1379                            NestedField::map_value_element(
1380                                3,
1381                                Primitive(PrimitiveType::Boolean),
1382                                true,
1383                            )
1384                            .into(),
1385                        )),
1386                    )
1387                    .into()
1388                ])
1389                .build()
1390                .is_err()
1391        );
1392
1393        // field in list
1394        assert!(
1395            Schema::builder()
1396                .with_schema_id(1)
1397                .with_identifier_field_ids(vec![2])
1398                .with_fields(vec![
1399                    NestedField::required(
1400                        1,
1401                        "List",
1402                        List(ListType::new(
1403                            NestedField::list_element(2, Primitive(PrimitiveType::String), true)
1404                                .into(),
1405                        )),
1406                    )
1407                    .into()
1408                ])
1409                .build()
1410                .is_err()
1411        );
1412
1413        // field in optional struct
1414        assert!(
1415            Schema::builder()
1416                .with_schema_id(1)
1417                .with_identifier_field_ids(vec![2])
1418                .with_fields(vec![
1419                    NestedField::optional(
1420                        1,
1421                        "Struct",
1422                        Struct(StructType::new(vec![
1423                            NestedField::required(2, "name", Primitive(PrimitiveType::String))
1424                                .into(),
1425                            NestedField::optional(3, "age", Primitive(PrimitiveType::Int)).into(),
1426                        ])),
1427                    )
1428                    .into()
1429                ])
1430                .build()
1431                .is_err()
1432        );
1433
1434        // float and double
1435        assert!(
1436            Schema::builder()
1437                .with_schema_id(1)
1438                .with_identifier_field_ids(vec![1])
1439                .with_fields(vec![
1440                    NestedField::required(1, "Float", Primitive(PrimitiveType::Float),).into()
1441                ])
1442                .build()
1443                .is_err()
1444        );
1445        assert!(
1446            Schema::builder()
1447                .with_schema_id(1)
1448                .with_identifier_field_ids(vec![1])
1449                .with_fields(vec![
1450                    NestedField::required(1, "Double", Primitive(PrimitiveType::Double),).into()
1451                ])
1452                .build()
1453                .is_err()
1454        );
1455
1456        // optional field
1457        assert!(
1458            Schema::builder()
1459                .with_schema_id(1)
1460                .with_identifier_field_ids(vec![1])
1461                .with_fields(vec![
1462                    NestedField::required(1, "Required", Primitive(PrimitiveType::String),).into()
1463                ])
1464                .build()
1465                .is_ok()
1466        );
1467        assert!(
1468            Schema::builder()
1469                .with_schema_id(1)
1470                .with_identifier_field_ids(vec![1])
1471                .with_fields(vec![
1472                    NestedField::optional(1, "Optional", Primitive(PrimitiveType::String),).into()
1473                ])
1474                .build()
1475                .is_err()
1476        );
1477    }
1478}
iceberg/spec/schema/mod.rs

iceberg/spec/schema/
mod.rs
