1use std::collections::{HashMap, HashSet};
21use std::fmt::{Display, Formatter};
22use std::sync::Arc;
23
24mod utils;
25mod visitor;
26pub use self::visitor::*;
27pub(super) mod _serde;
28mod assign_fresh_ids;
29mod id_reassigner;
30mod index;
31mod prune_columns;
32use bimap::BiHashMap;
33use itertools::{Itertools, zip_eq};
34use serde::{Deserialize, Serialize};
35
36use self::_serde::SchemaEnum;
37pub(crate) use self::assign_fresh_ids::assign_fresh_ids;
38use self::id_reassigner::ReassignFieldIds;
39use self::index::{IndexByName, index_by_id, index_parents};
40pub use self::prune_columns::prune_columns;
41use super::NestedField;
42use crate::error::{Result, invalid_data};
43use crate::expr::accessor::StructAccessor;
44use crate::spec::FormatVersion;
45use crate::spec::datatypes::{
46 LIST_FIELD_NAME, ListType, MAP_KEY_FIELD_NAME, MAP_VALUE_FIELD_NAME, MapType, NestedFieldRef,
47 PrimitiveType, StructType, Type,
48};
49use crate::{Error, ErrorKind, ensure_data_valid};
50
51pub type SchemaId = i32;
53pub type SchemaRef = Arc<Schema>;
55pub const DEFAULT_SCHEMA_ID: SchemaId = 0;
57pub const SCHEMA_NAME_DELIMITER: &str = ".";
59pub(crate) const DEFAULT_VALUES_MIN_FORMAT_VERSION: FormatVersion = FormatVersion::V3;
62
63#[derive(Debug, Serialize, Deserialize, Clone)]
65#[serde(try_from = "SchemaEnum", into = "SchemaEnum")]
66pub struct Schema {
67 r#struct: StructType,
68 schema_id: SchemaId,
69 highest_field_id: i32,
70 identifier_field_ids: HashSet<i32>,
71
72 alias_to_id: BiHashMap<String, i32>,
73 id_to_field: HashMap<i32, NestedFieldRef>,
74
75 name_to_id: HashMap<String, i32>,
76 lowercase_name_to_id: HashMap<String, Option<i32>>,
78 id_to_name: HashMap<i32, String>,
79
80 field_id_to_accessor: HashMap<i32, Arc<StructAccessor>>,
81}
82
83impl PartialEq for Schema {
84 fn eq(&self, other: &Self) -> bool {
85 self.r#struct == other.r#struct
86 && self.schema_id == other.schema_id
87 && self.identifier_field_ids == other.identifier_field_ids
88 }
89}
90
91impl Eq for Schema {}
92
93#[derive(Debug)]
95pub struct SchemaBuilder {
96 schema_id: i32,
97 fields: Vec<NestedFieldRef>,
98 alias_to_id: BiHashMap<String, i32>,
99 identifier_field_ids: HashSet<i32>,
100 reassign_field_ids_from: Option<i32>,
101}
102
103impl SchemaBuilder {
104 pub fn with_fields(mut self, fields: impl IntoIterator<Item = NestedFieldRef>) -> Self {
106 self.fields.extend(fields);
107 self
108 }
109
110 pub(crate) fn with_reassigned_field_ids(mut self, start_from: i32) -> Self {
115 self.reassign_field_ids_from = Some(start_from);
116 self
117 }
118
119 pub fn with_schema_id(mut self, schema_id: i32) -> Self {
121 self.schema_id = schema_id;
122 self
123 }
124
125 pub fn with_identifier_field_ids(mut self, ids: impl IntoIterator<Item = i32>) -> Self {
127 self.identifier_field_ids.extend(ids);
128 self
129 }
130
131 pub fn with_alias(mut self, alias_to_id: BiHashMap<String, i32>) -> Self {
133 self.alias_to_id = alias_to_id;
134 self
135 }
136
137 pub fn build(self) -> Result<Schema> {
139 let field_id_to_accessor = self.build_accessors();
140
141 let r#struct = StructType::new(self.fields);
142 let id_to_field = index_by_id(&r#struct)?;
143
144 Self::validate_identifier_ids(
145 &r#struct,
146 &id_to_field,
147 self.identifier_field_ids.iter().copied(),
148 )?;
149
150 let (name_to_id, id_to_name) = {
151 let mut index = IndexByName::default();
152 visit_struct(&r#struct, &mut index)?;
153 index.indexes()
154 };
155
156 let mut lowercase_name_to_id = HashMap::new();
157 for (name, field_id) in &name_to_id {
158 lowercase_name_to_id
159 .entry(name.to_lowercase())
160 .and_modify(|existing| *existing = None)
161 .or_insert(Some(*field_id));
162 }
163
164 let highest_field_id = id_to_field.keys().max().cloned().unwrap_or(0);
165
166 let mut schema = Schema {
167 r#struct,
168 schema_id: self.schema_id,
169 highest_field_id,
170 identifier_field_ids: self.identifier_field_ids,
171 alias_to_id: self.alias_to_id,
172 id_to_field,
173
174 name_to_id,
175 lowercase_name_to_id,
176 id_to_name,
177
178 field_id_to_accessor,
179 };
180
181 if let Some(start_from) = self.reassign_field_ids_from {
182 let mut id_reassigner = ReassignFieldIds::new(start_from);
183 let new_fields = id_reassigner.reassign_field_ids(schema.r#struct.fields().to_vec())?;
184 let new_identifier_field_ids =
185 id_reassigner.apply_to_identifier_fields(schema.identifier_field_ids)?;
186 let new_alias_to_id = id_reassigner.apply_to_aliases(schema.alias_to_id.clone())?;
187
188 schema = Schema::builder()
189 .with_schema_id(schema.schema_id)
190 .with_fields(new_fields)
191 .with_identifier_field_ids(new_identifier_field_ids)
192 .with_alias(new_alias_to_id)
193 .build()?;
194 }
195
196 Ok(schema)
197 }
198
199 fn build_accessors(&self) -> HashMap<i32, Arc<StructAccessor>> {
200 let mut map = HashMap::new();
201
202 for (pos, field) in self.fields.iter().enumerate() {
203 match field.field_type.as_ref() {
204 Type::Primitive(prim_type) => {
205 let accessor = Arc::new(StructAccessor::new(pos, prim_type.clone()));
207 map.insert(field.id, accessor.clone());
208 }
209
210 Type::Struct(nested) => {
211 for (field_id, accessor) in Self::build_accessors_nested(nested.fields()) {
213 let new_accessor = Arc::new(StructAccessor::wrap(pos, accessor));
214 map.insert(field_id, new_accessor.clone());
215 }
216 }
217 _ => {
218 }
220 }
221 }
222
223 map
224 }
225
226 fn build_accessors_nested(fields: &[NestedFieldRef]) -> Vec<(i32, Box<StructAccessor>)> {
227 let mut results = vec![];
228 for (pos, field) in fields.iter().enumerate() {
229 match field.field_type.as_ref() {
230 Type::Primitive(prim_type) => {
231 let accessor = Box::new(StructAccessor::new(pos, prim_type.clone()));
232 results.push((field.id, accessor));
233 }
234 Type::Struct(nested) => {
235 let nested_accessors = Self::build_accessors_nested(nested.fields());
236
237 let wrapped_nested_accessors =
238 nested_accessors.into_iter().map(|(id, accessor)| {
239 let new_accessor = Box::new(StructAccessor::wrap(pos, accessor));
240 (id, new_accessor.clone())
241 });
242
243 results.extend(wrapped_nested_accessors);
244 }
245 _ => {
246 }
248 }
249 }
250
251 results
252 }
253
254 fn validate_identifier_ids(
260 r#struct: &StructType,
261 id_to_field: &HashMap<i32, NestedFieldRef>,
262 identifier_field_ids: impl Iterator<Item = i32>,
263 ) -> Result<()> {
264 let id_to_parent = index_parents(r#struct)?;
265 for identifier_field_id in identifier_field_ids {
266 let field = id_to_field.get(&identifier_field_id).ok_or_else(|| {
267 invalid_data!(
268 "Cannot add identifier field {identifier_field_id}: field does not exist"
269 )
270 })?;
271 ensure_data_valid!(
272 field.required,
273 "Cannot add identifier field: {} is an optional field",
274 field.name
275 );
276 if let Type::Primitive(p) = field.field_type.as_ref() {
277 ensure_data_valid!(
278 !matches!(p, PrimitiveType::Double | PrimitiveType::Float),
279 "Cannot add identifier field {}: cannot be a float or double type",
280 field.name
281 );
282 } else {
283 return Err(invalid_data!(
284 "Cannot add field {} as an identifier field: not a primitive type field",
285 field.name
286 ));
287 }
288
289 let mut cur_field_id = identifier_field_id;
290 while let Some(parent) = id_to_parent.get(&cur_field_id) {
291 let parent_field = id_to_field
292 .get(parent)
293 .expect("Field id should not disappear.");
294 ensure_data_valid!(
295 parent_field.field_type.is_struct(),
296 "Cannot add field {} as an identifier field: must not be nested in {:?}",
297 field.name,
298 parent_field
299 );
300 ensure_data_valid!(
301 parent_field.required,
302 "Cannot add field {} as an identifier field: must not be nested in an optional field {}",
303 field.name,
304 parent_field
305 );
306 cur_field_id = *parent;
307 }
308 }
309
310 Ok(())
311 }
312}
313
314impl Schema {
315 pub fn builder() -> SchemaBuilder {
317 SchemaBuilder {
318 schema_id: DEFAULT_SCHEMA_ID,
319 fields: vec![],
320 identifier_field_ids: HashSet::default(),
321 alias_to_id: BiHashMap::default(),
322 reassign_field_ids_from: None,
323 }
324 }
325
326 pub fn into_builder(self) -> SchemaBuilder {
328 SchemaBuilder {
329 schema_id: self.schema_id,
330 fields: self.r#struct.fields().to_vec(),
331 alias_to_id: self.alias_to_id,
332 identifier_field_ids: self.identifier_field_ids,
333 reassign_field_ids_from: None,
334 }
335 }
336
337 pub fn field_by_id(&self, field_id: i32) -> Option<&NestedFieldRef> {
339 self.id_to_field.get(&field_id)
340 }
341
342 pub fn field_by_name(&self, field_name: &str) -> Option<&NestedFieldRef> {
346 self.name_to_id
347 .get(field_name)
348 .and_then(|id| self.field_by_id(*id))
349 }
350
351 pub fn field_by_name_case_insensitive(&self, field_name: &str) -> Option<&NestedFieldRef> {
357 self.lowercase_name_to_id
358 .get(&field_name.to_lowercase())
359 .copied()
360 .flatten()
361 .and_then(|id| self.field_by_id(id))
362 }
363
364 pub fn field_by_name_case_insensitive_checked(
372 &self,
373 field_name: &str,
374 ) -> Result<Option<&NestedFieldRef>> {
375 let lowercase_name = field_name.to_lowercase();
376 match self.lowercase_name_to_id.get(&lowercase_name) {
377 Some(Some(id)) => Ok(self.field_by_id(*id)),
378 Some(None) => {
379 let mut colliders: Vec<&str> = self
380 .name_to_id
381 .keys()
382 .filter(|name| name.to_lowercase() == lowercase_name)
383 .map(String::as_str)
384 .collect();
385 colliders.sort_unstable();
386 Err(invalid_data!(
387 "Multiple fields match {field_name} case-insensitively: {colliders:?}"
388 ))
389 }
390 None => Ok(None),
391 }
392 }
393
394 pub fn field_by_alias(&self, alias: &str) -> Option<&NestedFieldRef> {
396 self.alias_to_id
397 .get_by_left(alias)
398 .and_then(|id| self.field_by_id(*id))
399 }
400
401 #[inline]
403 pub fn highest_field_id(&self) -> i32 {
404 self.highest_field_id
405 }
406
407 #[inline]
409 pub fn schema_id(&self) -> SchemaId {
410 self.schema_id
411 }
412
413 #[inline]
415 pub fn as_struct(&self) -> &StructType {
416 &self.r#struct
417 }
418
419 #[inline]
421 pub fn identifier_field_ids(&self) -> impl ExactSizeIterator<Item = i32> + '_ {
422 self.identifier_field_ids.iter().copied()
423 }
424
425 pub fn field_id_by_name(&self, name: &str) -> Option<i32> {
427 self.name_to_id.get(name).copied()
428 }
429
430 pub fn name_by_field_id(&self, field_id: i32) -> Option<&str> {
432 self.id_to_name.get(&field_id).map(String::as_str)
433 }
434
435 pub fn accessor_by_field_id(&self, field_id: i32) -> Option<Arc<StructAccessor>> {
437 self.field_id_to_accessor.get(&field_id).cloned()
438 }
439
440 pub(crate) fn is_same_schema(&self, other: &SchemaRef) -> bool {
442 self.as_struct().eq(other.as_struct())
443 && self.identifier_field_ids().eq(other.identifier_field_ids())
444 }
445
446 pub(crate) fn with_schema_id(self, schema_id: SchemaId) -> Self {
450 Self { schema_id, ..self }
451 }
452
453 pub fn field_id_to_name_map(&self) -> &HashMap<i32, String> {
455 &self.id_to_name
456 }
457
458 pub fn field_id_to_fields(&self) -> &HashMap<i32, NestedFieldRef> {
460 &self.id_to_field
461 }
462
463 pub fn calc_min_compatible_format(&self) -> FormatVersion {
469 self.id_to_field
471 .values()
472 .map(|f| f.field_type.min_format_version())
473 .max()
474 .unwrap_or(FormatVersion::V1)
475 }
476
477 pub fn check_format_compatibility(&self, format_version: FormatVersion) -> Result<()> {
486 let mut problems: Vec<(i32, String)> = Vec::new();
488
489 for field in self.id_to_field.values() {
492 let min_version = field.field_type.min_format_version();
493 if format_version < min_version {
494 let name = self.name_by_field_id(field.id).ok_or_else(|| {
498 Error::new(
499 ErrorKind::Unexpected,
500 format!(
501 "Field id {} is missing from the schema's name index",
502 field.id
503 ),
504 )
505 })?;
506 problems.push((field.id, format!(
507 "Invalid type for {name}: {} is not supported until {min_version} but format version is {format_version}.",
508 field.field_type,
509 )));
510 }
511
512 if let Some(default) = &field.initial_default
513 && format_version < DEFAULT_VALUES_MIN_FORMAT_VERSION
514 {
515 let name = self.name_by_field_id(field.id).ok_or_else(|| {
516 Error::new(
517 ErrorKind::Unexpected,
518 format!(
519 "Field id {} is missing from the schema's name index",
520 field.id
521 ),
522 )
523 })?;
524 problems.push((field.id, format!(
525 "Invalid initial default for {name}: non-null default ({default:?}) is not supported until {DEFAULT_VALUES_MIN_FORMAT_VERSION} but format version is {format_version}."
526 )));
527 }
528 }
529
530 if problems.is_empty() {
531 return Ok(());
532 }
533
534 let message = problems
537 .into_iter()
538 .sorted_by_key(|(id, _)| *id)
539 .map(|(_, msg)| msg)
540 .join("\n- ");
541 Err(invalid_data!(
542 "Invalid schema for {format_version}:\n- {message}"
543 ))
544 }
545}
546
547impl Display for Schema {
548 fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result {
549 writeln!(f, "table {{")?;
550 for field in self.as_struct().fields() {
551 writeln!(f, " {field}")?;
552 }
553 writeln!(f, "}}")
554 }
555}
556
557#[cfg(test)]
558mod tests {
559 use std::collections::HashMap;
560
561 use bimap::BiHashMap;
562
563 use crate::spec::datatypes::Type::{List, Map, Primitive, Struct, Variant};
564 use crate::spec::datatypes::{
565 ListType, MapType, NestedField, NestedFieldRef, PrimitiveType, StructType,
566 };
567 use crate::spec::schema::Schema;
568 use crate::spec::values::Map as MapValue;
569 use crate::spec::{Datum, Literal};
570
571 #[test]
572 fn test_check_format_compatibility() {
573 use crate::spec::{FormatVersion, PrimitiveLiteral, VariantType};
574
575 fn schema_with(fields: Vec<NestedFieldRef>) -> Schema {
576 Schema::builder().with_fields(fields).build().unwrap()
577 }
578
579 let variant = schema_with(vec![
581 NestedField::optional(1, "v", Variant(VariantType)).into(),
582 ]);
583 assert!(
584 variant
585 .check_format_compatibility(FormatVersion::V2)
586 .is_err()
587 );
588 assert!(
589 variant
590 .check_format_compatibility(FormatVersion::V3)
591 .is_ok()
592 );
593
594 let with_default = schema_with(vec![
596 NestedField::optional(1, "a", Primitive(PrimitiveType::Int))
597 .with_initial_default(Literal::Primitive(PrimitiveLiteral::Int(1)))
598 .into(),
599 ]);
600 let err = with_default
601 .check_format_compatibility(FormatVersion::V2)
602 .unwrap_err();
603 assert!(
604 err.message().contains("Invalid initial default for a"),
605 "{err}"
606 );
607 assert!(
608 with_default
609 .check_format_compatibility(FormatVersion::V3)
610 .is_ok()
611 );
612
613 let no_default = schema_with(vec![
615 NestedField::optional(1, "a", Primitive(PrimitiveType::Int)).into(),
616 ]);
617 assert!(
618 no_default
619 .check_format_compatibility(FormatVersion::V1)
620 .is_ok()
621 );
622
623 let nested = schema_with(vec![
625 NestedField::required(
626 1,
627 "s",
628 Struct(StructType::new(vec![
629 NestedField::optional(2, "inner", Primitive(PrimitiveType::Long))
630 .with_initial_default(Literal::Primitive(PrimitiveLiteral::Long(7)))
631 .into(),
632 ])),
633 )
634 .into(),
635 ]);
636 let err = nested
637 .check_format_compatibility(FormatVersion::V2)
638 .unwrap_err();
639 assert!(err.message().contains("inner"), "{err}");
640
641 let nested_variant = schema_with(vec![
643 NestedField::required(
644 1,
645 "container",
646 Struct(StructType::new(vec![
647 NestedField::optional(2, "v", Variant(VariantType)).into(),
648 ])),
649 )
650 .into(),
651 ]);
652 let err = nested_variant
653 .check_format_compatibility(FormatVersion::V2)
654 .unwrap_err();
655 assert!(err.message().contains("container.v"), "{err}");
656 assert!(
657 !err.message().contains("Invalid type for container:"),
658 "container must not be blamed: {err}"
659 );
660 }
661
662 #[test]
663 fn test_calc_min_compatible_format() {
664 use crate::spec::{FormatVersion, GeometryType, VariantType};
665
666 fn schema_with(fields: Vec<NestedFieldRef>) -> Schema {
667 Schema::builder().with_fields(fields).build().unwrap()
668 }
669
670 let v1 = schema_with(vec![
672 NestedField::required(1, "a", Primitive(PrimitiveType::Int)).into(),
673 NestedField::optional(2, "b", Primitive(PrimitiveType::String)).into(),
674 ]);
675 assert_eq!(v1.calc_min_compatible_format(), FormatVersion::V1);
676
677 let variant = schema_with(vec![
679 NestedField::optional(1, "v", Variant(VariantType)).into(),
680 ]);
681 assert_eq!(variant.calc_min_compatible_format(), FormatVersion::V3);
682
683 let unknown = schema_with(vec![
685 NestedField::optional(1, "u", Primitive(PrimitiveType::Unknown)).into(),
686 ]);
687 assert_eq!(unknown.calc_min_compatible_format(), FormatVersion::V3);
688 assert!(
689 unknown
690 .check_format_compatibility(FormatVersion::V2)
691 .is_err()
692 );
693
694 let geometry = schema_with(vec![
696 NestedField::optional(
697 1,
698 "geom",
699 Primitive(PrimitiveType::Geometry(GeometryType::default())),
700 )
701 .into(),
702 ]);
703 assert_eq!(geometry.calc_min_compatible_format(), FormatVersion::V3);
704 assert!(
705 geometry
706 .check_format_compatibility(FormatVersion::V2)
707 .is_err()
708 );
709
710 let nested = schema_with(vec![
712 NestedField::required(
713 1,
714 "s",
715 Struct(StructType::new(vec![
716 NestedField::optional(
717 2,
718 "l",
719 List(ListType::new(
720 NestedField::required(
721 3,
722 "element",
723 Primitive(PrimitiveType::TimestampNs),
724 )
725 .into(),
726 )),
727 )
728 .into(),
729 ])),
730 )
731 .into(),
732 ]);
733 assert_eq!(nested.calc_min_compatible_format(), FormatVersion::V3);
734 }
735
736 #[test]
737 fn test_check_format_compatibility_message_order() {
738 use crate::spec::{FormatVersion, PrimitiveLiteral, VariantType};
739
740 let schema = Schema::builder()
744 .with_fields(vec![
745 NestedField::optional(3, "c", Variant(VariantType)).into(),
746 NestedField::optional(2, "b", Primitive(PrimitiveType::TimestampNs))
747 .with_initial_default(Literal::Primitive(PrimitiveLiteral::Long(0)))
748 .into(),
749 NestedField::required(1, "a", Primitive(PrimitiveType::Int)).into(),
750 ])
751 .build()
752 .unwrap();
753
754 let message = schema
755 .check_format_compatibility(FormatVersion::V2)
756 .unwrap_err()
757 .message()
758 .to_string();
759
760 let lines: Vec<&str> = message.lines().skip(1).collect();
761 assert_eq!(
762 lines,
763 vec![
764 "- Invalid type for b: timestamp_ns is not supported until v3 but format version is v2.",
765 "- Invalid initial default for b: non-null default (Primitive(Long(0))) is not supported until v3 but format version is v2.",
766 "- Invalid type for c: variant is not supported until v3 but format version is v2.",
767 ],
768 "{message}"
769 );
770 }
771
772 #[test]
773 fn test_construct_schema() {
774 let field1: NestedFieldRef =
775 NestedField::required(1, "f1", Primitive(PrimitiveType::Boolean)).into();
776 let field2: NestedFieldRef =
777 NestedField::optional(2, "f2", Primitive(PrimitiveType::Int)).into();
778
779 let schema = Schema::builder()
780 .with_fields(vec![field1.clone()])
781 .with_fields(vec![field2.clone()])
782 .with_schema_id(3)
783 .build()
784 .unwrap();
785
786 assert_eq!(3, schema.schema_id());
787 assert_eq!(2, schema.highest_field_id());
788 assert_eq!(Some(&field1), schema.field_by_id(1));
789 assert_eq!(Some(&field2), schema.field_by_id(2));
790 assert_eq!(None, schema.field_by_id(3));
791 }
792
793 pub fn table_schema_simple<'a>() -> (Schema, &'a str) {
794 let schema = Schema::builder()
795 .with_schema_id(1)
796 .with_identifier_field_ids(vec![2])
797 .with_fields(vec![
798 NestedField::optional(1, "foo", Primitive(PrimitiveType::String)).into(),
799 NestedField::required(2, "bar", Primitive(PrimitiveType::Int)).into(),
800 NestedField::optional(3, "baz", Primitive(PrimitiveType::Boolean)).into(),
801 ])
802 .build()
803 .unwrap();
804 let record = r#"{
805 "type":"struct",
806 "schema-id":1,
807 "fields":[
808 {
809 "id":1,
810 "name":"foo",
811 "required":false,
812 "type":"string"
813 },
814 {
815 "id":2,
816 "name":"bar",
817 "required":true,
818 "type":"int"
819 },
820 {
821 "id":3,
822 "name":"baz",
823 "required":false,
824 "type":"boolean"
825 }
826 ],
827 "identifier-field-ids":[2]
828 }"#;
829 (schema, record)
830 }
831
832 pub fn table_schema_nested() -> Schema {
833 Schema::builder()
834 .with_schema_id(1)
835 .with_identifier_field_ids(vec![2])
836 .with_fields(vec![
837 NestedField::optional(1, "foo", Primitive(PrimitiveType::String)).into(),
838 NestedField::required(2, "bar", Primitive(PrimitiveType::Int)).into(),
839 NestedField::optional(3, "baz", Primitive(PrimitiveType::Boolean)).into(),
840 NestedField::required(
841 4,
842 "qux",
843 List(ListType {
844 element_field: NestedField::list_element(
845 5,
846 Primitive(PrimitiveType::String),
847 true,
848 )
849 .into(),
850 }),
851 )
852 .into(),
853 NestedField::required(
854 6,
855 "quux",
856 Map(MapType {
857 key_field: NestedField::map_key_element(
858 7,
859 Primitive(PrimitiveType::String),
860 )
861 .into(),
862 value_field: NestedField::map_value_element(
863 8,
864 Map(MapType {
865 key_field: NestedField::map_key_element(
866 9,
867 Primitive(PrimitiveType::String),
868 )
869 .into(),
870 value_field: NestedField::map_value_element(
871 10,
872 Primitive(PrimitiveType::Int),
873 true,
874 )
875 .into(),
876 }),
877 true,
878 )
879 .into(),
880 }),
881 )
882 .into(),
883 NestedField::required(
884 11,
885 "location",
886 List(ListType {
887 element_field: NestedField::list_element(
888 12,
889 Struct(StructType::new(vec![
890 NestedField::optional(
891 13,
892 "latitude",
893 Primitive(PrimitiveType::Float),
894 )
895 .into(),
896 NestedField::optional(
897 14,
898 "longitude",
899 Primitive(PrimitiveType::Float),
900 )
901 .into(),
902 ])),
903 true,
904 )
905 .into(),
906 }),
907 )
908 .into(),
909 NestedField::optional(
910 15,
911 "person",
912 Struct(StructType::new(vec![
913 NestedField::optional(16, "name", Primitive(PrimitiveType::String)).into(),
914 NestedField::required(17, "age", Primitive(PrimitiveType::Int)).into(),
915 ])),
916 )
917 .into(),
918 ])
919 .build()
920 .unwrap()
921 }
922
923 #[test]
924 fn test_schema_display() {
925 let expected_str = "
926table {
927 1: foo: optional string\x20
928 2: bar: required int\x20
929 3: baz: optional boolean\x20
930}
931";
932
933 assert_eq!(expected_str, format!("\n{}", table_schema_simple().0));
934 }
935
936 #[test]
937 fn test_schema_build_failed_on_duplicate_names() {
938 let ret = Schema::builder()
939 .with_schema_id(1)
940 .with_identifier_field_ids(vec![1])
941 .with_fields(vec![
942 NestedField::required(1, "foo", Primitive(PrimitiveType::String)).into(),
943 NestedField::required(2, "bar", Primitive(PrimitiveType::Int)).into(),
944 NestedField::optional(3, "baz", Primitive(PrimitiveType::Boolean)).into(),
945 NestedField::optional(4, "baz", Primitive(PrimitiveType::Boolean)).into(),
946 ])
947 .build();
948
949 assert!(
950 ret.unwrap_err()
951 .message()
952 .contains("Invalid schema: multiple fields for name baz")
953 );
954 }
955
956 #[test]
957 fn test_schema_into_builder() {
958 let original_schema = table_schema_nested();
959 let builder = original_schema.clone().into_builder();
960 let schema = builder.build().unwrap();
961
962 assert_eq!(original_schema, schema);
963 }
964
965 #[test]
966 fn test_schema_index_by_name() {
967 let expected_name_to_id = HashMap::from(
968 [
969 ("foo", 1),
970 ("bar", 2),
971 ("baz", 3),
972 ("qux", 4),
973 ("qux.element", 5),
974 ("quux", 6),
975 ("quux.key", 7),
976 ("quux.value", 8),
977 ("quux.value.key", 9),
978 ("quux.value.value", 10),
979 ("location", 11),
980 ("location.element", 12),
981 ("location.element.latitude", 13),
982 ("location.element.longitude", 14),
983 ("location.latitude", 13),
984 ("location.longitude", 14),
985 ("person", 15),
986 ("person.name", 16),
987 ("person.age", 17),
988 ]
989 .map(|e| (e.0.to_string(), e.1)),
990 );
991
992 let schema = table_schema_nested();
993 assert_eq!(&expected_name_to_id, &schema.name_to_id);
994 }
995
996 #[test]
997 fn test_schema_index_by_name_case_insensitive() {
998 let expected_name_to_id = HashMap::from(
999 [
1000 ("fOo", 1),
1001 ("Bar", 2),
1002 ("BAz", 3),
1003 ("quX", 4),
1004 ("quX.ELEment", 5),
1005 ("qUUx", 6),
1006 ("QUUX.KEY", 7),
1007 ("QUUX.Value", 8),
1008 ("qUUX.VALUE.Key", 9),
1009 ("qUux.VaLue.Value", 10),
1010 ("lOCAtION", 11),
1011 ("LOCAtioN.ELeMENt", 12),
1012 ("LoCATion.element.LATitude", 13),
1013 ("locatION.ElemeNT.LONgitude", 14),
1014 ("LOCAtiON.LATITUDE", 13),
1015 ("LOCATION.LONGITUDE", 14),
1016 ("PERSon", 15),
1017 ("PERSON.Name", 16),
1018 ("peRSON.AGe", 17),
1019 ]
1020 .map(|e| (e.0.to_string(), e.1)),
1021 );
1022
1023 let schema = table_schema_nested();
1024 for (name, id) in expected_name_to_id {
1025 assert_eq!(
1026 Some(id),
1027 schema.field_by_name_case_insensitive(&name).map(|f| f.id)
1028 );
1029 }
1030 }
1031
1032 #[test]
1033 fn test_schema_field_by_name_case_insensitive_checked() {
1034 let schema = Schema::builder()
1035 .with_fields(vec![
1036 NestedField::optional(1, "id", Primitive(PrimitiveType::Int)).into(),
1037 NestedField::optional(2, "ID", Primitive(PrimitiveType::Int)).into(),
1038 NestedField::optional(3, "data", Primitive(PrimitiveType::String)).into(),
1039 ])
1040 .build()
1041 .expect("case-insensitive name ambiguity is valid in a schema");
1042
1043 assert_eq!(
1044 schema
1045 .field_by_name_case_insensitive_checked("DATA")
1046 .unwrap()
1047 .map(|field| field.id),
1048 Some(3)
1049 );
1050 assert!(
1051 schema
1052 .field_by_name_case_insensitive_checked("missing")
1053 .unwrap()
1054 .is_none()
1055 );
1056
1057 let error = schema
1058 .field_by_name_case_insensitive_checked("Id")
1059 .unwrap_err();
1060 assert_eq!(error.kind(), crate::ErrorKind::DataInvalid);
1061 assert!(error.message().contains("case-insensitively"), "{error}");
1062 assert!(error.message().contains("[\"ID\", \"id\"]"), "{error}");
1063 assert!(schema.field_by_name_case_insensitive("Id").is_none());
1064 }
1065
1066 #[test]
1067 fn test_schema_find_column_name() {
1068 let expected_column_name = HashMap::from([
1069 (1, "foo"),
1070 (2, "bar"),
1071 (3, "baz"),
1072 (4, "qux"),
1073 (5, "qux.element"),
1074 (6, "quux"),
1075 (7, "quux.key"),
1076 (8, "quux.value"),
1077 (9, "quux.value.key"),
1078 (10, "quux.value.value"),
1079 (11, "location"),
1080 (12, "location.element"),
1081 (13, "location.element.latitude"),
1082 (14, "location.element.longitude"),
1083 ]);
1084
1085 let schema = table_schema_nested();
1086 for (id, name) in expected_column_name {
1087 assert_eq!(
1088 Some(name),
1089 schema.name_by_field_id(id),
1090 "Column name for field id {id} not match."
1091 );
1092 }
1093 }
1094
1095 #[test]
1096 fn test_schema_find_column_name_not_found() {
1097 let schema = table_schema_nested();
1098
1099 assert!(schema.name_by_field_id(99).is_none());
1100 }
1101
1102 #[test]
1103 fn test_schema_find_column_name_by_id_simple() {
1104 let expected_id_to_name = HashMap::from([(1, "foo"), (2, "bar"), (3, "baz")]);
1105
1106 let schema = table_schema_simple().0;
1107
1108 for (id, name) in expected_id_to_name {
1109 assert_eq!(
1110 Some(name),
1111 schema.name_by_field_id(id),
1112 "Column name for field id {id} not match."
1113 );
1114 }
1115 }
1116
1117 #[test]
1118 fn test_schema_find_simple() {
1119 let schema = table_schema_simple().0;
1120
1121 assert_eq!(
1122 Some(schema.r#struct.fields()[0].clone()),
1123 schema.field_by_id(1).cloned()
1124 );
1125 assert_eq!(
1126 Some(schema.r#struct.fields()[1].clone()),
1127 schema.field_by_id(2).cloned()
1128 );
1129 assert_eq!(
1130 Some(schema.r#struct.fields()[2].clone()),
1131 schema.field_by_id(3).cloned()
1132 );
1133
1134 assert!(schema.field_by_id(4).is_none());
1135 assert!(schema.field_by_name("non exist").is_none());
1136 }
1137
1138 #[test]
1139 fn test_schema_find_nested() {
1140 let expected_id_to_field: HashMap<i32, NestedField> = HashMap::from([
1141 (
1142 1,
1143 NestedField::optional(1, "foo", Primitive(PrimitiveType::String)),
1144 ),
1145 (
1146 2,
1147 NestedField::required(2, "bar", Primitive(PrimitiveType::Int)),
1148 ),
1149 (
1150 3,
1151 NestedField::optional(3, "baz", Primitive(PrimitiveType::Boolean)),
1152 ),
1153 (
1154 4,
1155 NestedField::required(
1156 4,
1157 "qux",
1158 List(ListType {
1159 element_field: NestedField::list_element(
1160 5,
1161 Primitive(PrimitiveType::String),
1162 true,
1163 )
1164 .into(),
1165 }),
1166 ),
1167 ),
1168 (
1169 5,
1170 NestedField::required(5, "element", Primitive(PrimitiveType::String)),
1171 ),
1172 (
1173 6,
1174 NestedField::required(
1175 6,
1176 "quux",
1177 Map(MapType {
1178 key_field: NestedField::map_key_element(
1179 7,
1180 Primitive(PrimitiveType::String),
1181 )
1182 .into(),
1183 value_field: NestedField::map_value_element(
1184 8,
1185 Map(MapType {
1186 key_field: NestedField::map_key_element(
1187 9,
1188 Primitive(PrimitiveType::String),
1189 )
1190 .into(),
1191 value_field: NestedField::map_value_element(
1192 10,
1193 Primitive(PrimitiveType::Int),
1194 true,
1195 )
1196 .into(),
1197 }),
1198 true,
1199 )
1200 .into(),
1201 }),
1202 ),
1203 ),
1204 (
1205 7,
1206 NestedField::required(7, "key", Primitive(PrimitiveType::String)),
1207 ),
1208 (
1209 8,
1210 NestedField::required(
1211 8,
1212 "value",
1213 Map(MapType {
1214 key_field: NestedField::map_key_element(
1215 9,
1216 Primitive(PrimitiveType::String),
1217 )
1218 .into(),
1219 value_field: NestedField::map_value_element(
1220 10,
1221 Primitive(PrimitiveType::Int),
1222 true,
1223 )
1224 .into(),
1225 }),
1226 ),
1227 ),
1228 (
1229 9,
1230 NestedField::required(9, "key", Primitive(PrimitiveType::String)),
1231 ),
1232 (
1233 10,
1234 NestedField::required(10, "value", Primitive(PrimitiveType::Int)),
1235 ),
1236 (
1237 11,
1238 NestedField::required(
1239 11,
1240 "location",
1241 List(ListType {
1242 element_field: NestedField::list_element(
1243 12,
1244 Struct(StructType::new(vec![
1245 NestedField::optional(
1246 13,
1247 "latitude",
1248 Primitive(PrimitiveType::Float),
1249 )
1250 .into(),
1251 NestedField::optional(
1252 14,
1253 "longitude",
1254 Primitive(PrimitiveType::Float),
1255 )
1256 .into(),
1257 ])),
1258 true,
1259 )
1260 .into(),
1261 }),
1262 ),
1263 ),
1264 (
1265 12,
1266 NestedField::list_element(
1267 12,
1268 Struct(StructType::new(vec![
1269 NestedField::optional(13, "latitude", Primitive(PrimitiveType::Float))
1270 .into(),
1271 NestedField::optional(14, "longitude", Primitive(PrimitiveType::Float))
1272 .into(),
1273 ])),
1274 true,
1275 ),
1276 ),
1277 (
1278 13,
1279 NestedField::optional(13, "latitude", Primitive(PrimitiveType::Float)),
1280 ),
1281 (
1282 14,
1283 NestedField::optional(14, "longitude", Primitive(PrimitiveType::Float)),
1284 ),
1285 (
1286 15,
1287 NestedField::optional(
1288 15,
1289 "person",
1290 Struct(StructType::new(vec![
1291 NestedField::optional(16, "name", Primitive(PrimitiveType::String)).into(),
1292 NestedField::required(17, "age", Primitive(PrimitiveType::Int)).into(),
1293 ])),
1294 ),
1295 ),
1296 (
1297 16,
1298 NestedField::optional(16, "name", Primitive(PrimitiveType::String)),
1299 ),
1300 (
1301 17,
1302 NestedField::required(17, "age", Primitive(PrimitiveType::Int)),
1303 ),
1304 ]);
1305
1306 let schema = table_schema_nested();
1307 for (id, field) in expected_id_to_field {
1308 assert_eq!(
1309 Some(&field),
1310 schema.field_by_id(id).map(|f| f.as_ref()),
1311 "Field for {id} not match."
1312 );
1313 }
1314 }
1315
1316 #[test]
1317 fn test_build_accessors() {
1318 let schema = table_schema_nested();
1319
1320 let test_struct = crate::spec::Struct::from_iter(vec![
1321 Some(Literal::string("foo value")),
1322 Some(Literal::int(1002)),
1323 Some(Literal::bool(true)),
1324 Some(Literal::List(vec![
1325 Some(Literal::string("qux item 1")),
1326 Some(Literal::string("qux item 2")),
1327 ])),
1328 Some(Literal::Map(MapValue::from([(
1329 Literal::string("quux key 1"),
1330 Some(Literal::Map(MapValue::from([(
1331 Literal::string("quux nested key 1"),
1332 Some(Literal::int(1000)),
1333 )]))),
1334 )]))),
1335 Some(Literal::List(vec![Some(Literal::Struct(
1336 crate::spec::Struct::from_iter(vec![
1337 Some(Literal::float(52.509_09_f32)),
1338 Some(Literal::float(-1.885_249_f32)),
1339 ]),
1340 ))])),
1341 Some(Literal::Struct(crate::spec::Struct::from_iter(vec![
1342 Some(Literal::string("Testy McTest")),
1343 Some(Literal::int(33)),
1344 ]))),
1345 ]);
1346
1347 assert_eq!(
1348 schema
1349 .accessor_by_field_id(1)
1350 .unwrap()
1351 .get(&test_struct)
1352 .unwrap(),
1353 Some(Datum::string("foo value"))
1354 );
1355 assert_eq!(
1356 schema
1357 .accessor_by_field_id(2)
1358 .unwrap()
1359 .get(&test_struct)
1360 .unwrap(),
1361 Some(Datum::int(1002))
1362 );
1363 assert_eq!(
1364 schema
1365 .accessor_by_field_id(3)
1366 .unwrap()
1367 .get(&test_struct)
1368 .unwrap(),
1369 Some(Datum::bool(true))
1370 );
1371 assert_eq!(
1372 schema
1373 .accessor_by_field_id(16)
1374 .unwrap()
1375 .get(&test_struct)
1376 .unwrap(),
1377 Some(Datum::string("Testy McTest"))
1378 );
1379 assert_eq!(
1380 schema
1381 .accessor_by_field_id(17)
1382 .unwrap()
1383 .get(&test_struct)
1384 .unwrap(),
1385 Some(Datum::int(33))
1386 );
1387 }
1388
1389 #[test]
1390 fn test_highest_field_id() {
1391 let schema = table_schema_nested();
1392 assert_eq!(17, schema.highest_field_id());
1393
1394 let schema = table_schema_simple().0;
1395 assert_eq!(3, schema.highest_field_id());
1396 }
1397
1398 #[test]
1399 fn test_highest_field_id_no_fields() {
1400 let schema = Schema::builder().with_schema_id(1).build().unwrap();
1401 assert_eq!(0, schema.highest_field_id());
1402 }
1403
1404 #[test]
1405 fn test_field_ids_must_be_unique() {
1406 let reassigned_schema = Schema::builder()
1407 .with_schema_id(1)
1408 .with_identifier_field_ids(vec![5])
1409 .with_alias(BiHashMap::from_iter(vec![("bar_alias".to_string(), 3)]))
1410 .with_fields(vec![
1411 NestedField::required(5, "foo", Primitive(PrimitiveType::String)).into(),
1412 NestedField::optional(3, "bar", Primitive(PrimitiveType::Int)).into(),
1413 NestedField::optional(3, "baz", Primitive(PrimitiveType::Boolean)).into(),
1414 ])
1415 .build()
1416 .unwrap_err();
1417
1418 assert!(reassigned_schema.message().contains("'field.id' 3"));
1419 }
1420
1421 #[test]
1422 fn test_reassign_ids_empty_schema() {
1423 let schema = Schema::builder().with_schema_id(1).build().unwrap();
1424 let reassigned_schema = schema
1425 .clone()
1426 .into_builder()
1427 .with_reassigned_field_ids(0)
1428 .build()
1429 .unwrap();
1430
1431 assert_eq!(schema, reassigned_schema);
1432 assert_eq!(schema.highest_field_id(), 0);
1433 }
1434
1435 #[test]
1436 fn test_identifier_field_ids() {
1437 assert!(
1439 Schema::builder()
1440 .with_schema_id(1)
1441 .with_identifier_field_ids(vec![2])
1442 .with_fields(vec![
1443 NestedField::required(
1444 1,
1445 "Map",
1446 Map(MapType::new(
1447 NestedField::map_key_element(2, Primitive(PrimitiveType::String))
1448 .into(),
1449 NestedField::map_value_element(
1450 3,
1451 Primitive(PrimitiveType::Boolean),
1452 true,
1453 )
1454 .into(),
1455 )),
1456 )
1457 .into()
1458 ])
1459 .build()
1460 .is_err()
1461 );
1462 assert!(
1463 Schema::builder()
1464 .with_schema_id(1)
1465 .with_identifier_field_ids(vec![3])
1466 .with_fields(vec![
1467 NestedField::required(
1468 1,
1469 "Map",
1470 Map(MapType::new(
1471 NestedField::map_key_element(2, Primitive(PrimitiveType::String))
1472 .into(),
1473 NestedField::map_value_element(
1474 3,
1475 Primitive(PrimitiveType::Boolean),
1476 true,
1477 )
1478 .into(),
1479 )),
1480 )
1481 .into()
1482 ])
1483 .build()
1484 .is_err()
1485 );
1486
1487 assert!(
1489 Schema::builder()
1490 .with_schema_id(1)
1491 .with_identifier_field_ids(vec![2])
1492 .with_fields(vec![
1493 NestedField::required(
1494 1,
1495 "List",
1496 List(ListType::new(
1497 NestedField::list_element(2, Primitive(PrimitiveType::String), true)
1498 .into(),
1499 )),
1500 )
1501 .into()
1502 ])
1503 .build()
1504 .is_err()
1505 );
1506
1507 assert!(
1509 Schema::builder()
1510 .with_schema_id(1)
1511 .with_identifier_field_ids(vec![2])
1512 .with_fields(vec![
1513 NestedField::optional(
1514 1,
1515 "Struct",
1516 Struct(StructType::new(vec![
1517 NestedField::required(2, "name", Primitive(PrimitiveType::String))
1518 .into(),
1519 NestedField::optional(3, "age", Primitive(PrimitiveType::Int)).into(),
1520 ])),
1521 )
1522 .into()
1523 ])
1524 .build()
1525 .is_err()
1526 );
1527
1528 assert!(
1530 Schema::builder()
1531 .with_schema_id(1)
1532 .with_identifier_field_ids(vec![1])
1533 .with_fields(vec![
1534 NestedField::required(1, "Float", Primitive(PrimitiveType::Float),).into()
1535 ])
1536 .build()
1537 .is_err()
1538 );
1539 assert!(
1540 Schema::builder()
1541 .with_schema_id(1)
1542 .with_identifier_field_ids(vec![1])
1543 .with_fields(vec![
1544 NestedField::required(1, "Double", Primitive(PrimitiveType::Double),).into()
1545 ])
1546 .build()
1547 .is_err()
1548 );
1549
1550 assert!(
1552 Schema::builder()
1553 .with_schema_id(1)
1554 .with_identifier_field_ids(vec![1])
1555 .with_fields(vec![
1556 NestedField::required(1, "Required", Primitive(PrimitiveType::String),).into()
1557 ])
1558 .build()
1559 .is_ok()
1560 );
1561 assert!(
1562 Schema::builder()
1563 .with_schema_id(1)
1564 .with_identifier_field_ids(vec![1])
1565 .with_fields(vec![
1566 NestedField::optional(1, "Optional", Primitive(PrimitiveType::String),).into()
1567 ])
1568 .build()
1569 .is_err()
1570 );
1571 }
1572
1573 #[test]
1574 fn test_unknown_type_deserialization_rejects_non_null_default() {
1575 let field_json = serde_json::json!({
1576 "id": 1,
1577 "name": "empty",
1578 "required": false,
1579 "type": "unknown",
1580 "initial-default": 1
1581 });
1582
1583 let error = serde_json::from_value::<NestedField>(field_json.clone()).unwrap_err();
1584 assert!(
1585 error
1586 .to_string()
1587 .contains("Unknown type only supports null default values"),
1588 "unexpected error: {error}"
1589 );
1590
1591 let schema_json = serde_json::json!({
1592 "type": "struct",
1593 "schema-id": 1,
1594 "fields": [field_json]
1595 });
1596 assert!(serde_json::from_value::<Schema>(schema_json).is_err());
1597 }
1598
1599 #[test]
1600 fn test_unknown_type_deserialization_accepts_null_defaults() {
1601 let schema_json = serde_json::json!({
1602 "type": "struct",
1603 "schema-id": 1,
1604 "fields": [
1605 {
1606 "id": 1,
1607 "name": "empty",
1608 "required": false,
1609 "type": "unknown",
1610 "initial-default": null,
1611 "write-default": null
1612 }
1613 ]
1614 });
1615
1616 serde_json::from_value::<Schema>(schema_json).unwrap();
1617 }
1618
1619 #[test]
1620 fn test_unknown_type_accepts_null_container_defaults() {
1621 let cases = [
1622 (
1623 "struct",
1624 Struct(StructType::new(vec![
1625 NestedField::optional(2, "empty", Primitive(PrimitiveType::Unknown)).into(),
1626 ])),
1627 Literal::Struct(crate::spec::Struct::from_iter([None])),
1628 ),
1629 (
1630 "list",
1631 List(ListType::new(
1632 NestedField::list_element(2, Primitive(PrimitiveType::Unknown), false).into(),
1633 )),
1634 Literal::List(vec![None]),
1635 ),
1636 (
1637 "map",
1638 Map(MapType::optional(
1639 2,
1640 Primitive(PrimitiveType::String),
1641 3,
1642 Primitive(PrimitiveType::Unknown),
1643 )),
1644 Literal::Map(MapValue::from([(Literal::string("key"), None)])),
1645 ),
1646 ];
1647
1648 for (name, field_type, default) in cases {
1649 let schema = Schema::builder()
1650 .with_schema_id(1)
1651 .with_fields(vec![
1652 NestedField::optional(1, name, field_type)
1653 .with_write_default(default)
1654 .into(),
1655 ])
1656 .build()
1657 .unwrap();
1658 serde_json::to_value(schema).unwrap();
1659 }
1660 }
1661
1662 #[test]
1663 fn test_unknown_type_deserialization_rejects_non_null_container_defaults() {
1664 let cases = [
1665 (
1666 "struct",
1667 serde_json::json!({
1668 "type": "struct",
1669 "fields": [{
1670 "id": 2,
1671 "name": "empty",
1672 "required": false,
1673 "type": "unknown"
1674 }]
1675 }),
1676 serde_json::json!({"2": 1}),
1677 ),
1678 (
1679 "list",
1680 serde_json::json!({
1681 "type": "list",
1682 "element-id": 2,
1683 "element-required": false,
1684 "element": "unknown"
1685 }),
1686 serde_json::json!([1]),
1687 ),
1688 (
1689 "map",
1690 serde_json::json!({
1691 "type": "map",
1692 "key-id": 2,
1693 "key": "string",
1694 "value-id": 3,
1695 "value-required": false,
1696 "value": "unknown"
1697 }),
1698 serde_json::json!({"keys": ["key"], "values": [1]}),
1699 ),
1700 ];
1701
1702 for (name, field_type, default) in cases {
1703 let schema_json = serde_json::json!({
1704 "type": "struct",
1705 "schema-id": 1,
1706 "fields": [{
1707 "id": 1,
1708 "name": name,
1709 "required": false,
1710 "type": field_type,
1711 "initial-default": default
1712 }]
1713 });
1714
1715 assert!(
1716 serde_json::from_value::<Schema>(schema_json).is_err(),
1717 "non-null unknown default in {name} should be rejected"
1718 );
1719 }
1720 }
1721}