iceberg/
metadata_columns.rs

1// Licensed to the Apache Software Foundation (ASF) under one
2// or more contributor license agreements.  See the NOTICE file
3// distributed with this work for additional information
4// regarding copyright ownership.  The ASF licenses this file
5// to you under the Apache License, Version 2.0 (the
6// "License"); you may not use this file except in compliance
7// with the License.  You may obtain a copy of the License at
8//
9//   http://www.apache.org/licenses/LICENSE-2.0
10//
11// Unless required by applicable law or agreed to in writing,
12// software distributed under the License is distributed on an
13// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14// KIND, either express or implied.  See the License for the
15// specific language governing permissions and limitations
16// under the License.
17
18//! Metadata columns (virtual/reserved fields) for Iceberg tables.
19//!
20//! This module defines metadata columns that can be requested in projections
21//! but are not stored in data files. Instead, they are computed on-the-fly
22//! during reading. Examples include the _file column (file path) and future
23//! columns like partition values or row numbers.
24
25use std::sync::Arc;
26
27use once_cell::sync::Lazy;
28
29use crate::spec::{NestedField, NestedFieldRef, PrimitiveType, Type};
30use crate::{Error, ErrorKind, Result};
31
32/// Reserved field ID for the file path (_file) column per Iceberg spec
33pub const RESERVED_FIELD_ID_FILE: i32 = i32::MAX - 1;
34
35/// Reserved field ID for the position (_pos) column per Iceberg spec
36pub const RESERVED_FIELD_ID_POS: i32 = i32::MAX - 2;
37
38/// Reserved field ID for the deleted (_deleted) column per Iceberg spec
39pub const RESERVED_FIELD_ID_DELETED: i32 = i32::MAX - 3;
40
41/// Reserved field ID for the spec ID (_spec_id) column per Iceberg spec
42pub const RESERVED_FIELD_ID_SPEC_ID: i32 = i32::MAX - 4;
43
44/// Reserved field ID for the partition (_partition) column per Iceberg spec
45pub const RESERVED_FIELD_ID_PARTITION: i32 = i32::MAX - 5;
46
47/// Reserved field ID for the file path in position delete files
48pub const RESERVED_FIELD_ID_DELETE_FILE_PATH: i32 = i32::MAX - 101;
49
50/// Reserved field ID for the position in position delete files
51pub const RESERVED_FIELD_ID_DELETE_FILE_POS: i32 = i32::MAX - 102;
52
53/// Reserved field ID for the change type (_change_type) column per Iceberg spec
54pub const RESERVED_FIELD_ID_CHANGE_TYPE: i32 = i32::MAX - 104;
55
56/// Reserved field ID for the change ordinal (_change_ordinal) column per Iceberg spec
57pub const RESERVED_FIELD_ID_CHANGE_ORDINAL: i32 = i32::MAX - 105;
58
59/// Reserved field ID for the commit snapshot ID (_commit_snapshot_id) column per Iceberg spec
60pub const RESERVED_FIELD_ID_COMMIT_SNAPSHOT_ID: i32 = i32::MAX - 106;
61
62/// Reserved field ID for the row ID (_row_id) column per Iceberg spec
63pub const RESERVED_FIELD_ID_ROW_ID: i32 = i32::MAX - 107;
64
65/// Reserved field ID for the last updated sequence number (_last_updated_sequence_number) column per Iceberg spec
66pub const RESERVED_FIELD_ID_LAST_UPDATED_SEQUENCE_NUMBER: i32 = i32::MAX - 108;
67
68/// Reserved column name for the file path metadata column
69pub const RESERVED_COL_NAME_FILE: &str = "_file";
70
71/// Reserved column name for the position metadata column
72pub const RESERVED_COL_NAME_POS: &str = "_pos";
73
74/// Reserved column name for the deleted metadata column
75pub const RESERVED_COL_NAME_DELETED: &str = "_deleted";
76
77/// Reserved column name for the spec ID metadata column
78pub const RESERVED_COL_NAME_SPEC_ID: &str = "_spec_id";
79
80/// Reserved column name for the partition metadata column
81pub const RESERVED_COL_NAME_PARTITION: &str = "_partition";
82
83/// Reserved column name for the file path in position delete files
84pub const RESERVED_COL_NAME_DELETE_FILE_PATH: &str = "file_path";
85
86/// Reserved column name for the position in position delete files
87pub const RESERVED_COL_NAME_DELETE_FILE_POS: &str = "pos";
88
89/// Reserved column name for the change type metadata column
90pub const RESERVED_COL_NAME_CHANGE_TYPE: &str = "_change_type";
91
92/// Reserved column name for the change ordinal metadata column
93pub const RESERVED_COL_NAME_CHANGE_ORDINAL: &str = "_change_ordinal";
94
95/// Reserved column name for the commit snapshot ID metadata column
96pub const RESERVED_COL_NAME_COMMIT_SNAPSHOT_ID: &str = "_commit_snapshot_id";
97
98/// Reserved column name for the row ID metadata column
99pub const RESERVED_COL_NAME_ROW_ID: &str = "_row_id";
100
101/// Reserved column name for the last updated sequence number metadata column
102pub const RESERVED_COL_NAME_LAST_UPDATED_SEQUENCE_NUMBER: &str = "_last_updated_sequence_number";
103
104/// Lazy-initialized Iceberg field definition for the _file metadata column.
105/// This field represents the file path as a required string field.
106static FILE_FIELD: Lazy<NestedFieldRef> = Lazy::new(|| {
107    Arc::new(
108        NestedField::required(
109            RESERVED_FIELD_ID_FILE,
110            RESERVED_COL_NAME_FILE,
111            Type::Primitive(PrimitiveType::String),
112        )
113        .with_doc("Path of the file in which a row is stored"),
114    )
115});
116
117/// Lazy-initialized Iceberg field definition for the _pos metadata column.
118/// This field represents the ordinal position of a row in the source data file.
119static POS_FIELD: Lazy<NestedFieldRef> = Lazy::new(|| {
120    Arc::new(
121        NestedField::required(
122            RESERVED_FIELD_ID_POS,
123            RESERVED_COL_NAME_POS,
124            Type::Primitive(PrimitiveType::Long),
125        )
126        .with_doc("Ordinal position of a row in the source data file"),
127    )
128});
129
130/// Lazy-initialized Iceberg field definition for the _deleted metadata column.
131/// This field indicates whether a row has been deleted.
132static DELETED_FIELD: Lazy<NestedFieldRef> = Lazy::new(|| {
133    Arc::new(
134        NestedField::required(
135            RESERVED_FIELD_ID_DELETED,
136            RESERVED_COL_NAME_DELETED,
137            Type::Primitive(PrimitiveType::Boolean),
138        )
139        .with_doc("Whether the row has been deleted"),
140    )
141});
142
143/// Lazy-initialized Iceberg field definition for the _spec_id metadata column.
144/// This field represents the spec ID used to track the file containing a row.
145static SPEC_ID_FIELD: Lazy<NestedFieldRef> = Lazy::new(|| {
146    Arc::new(
147        NestedField::required(
148            RESERVED_FIELD_ID_SPEC_ID,
149            RESERVED_COL_NAME_SPEC_ID,
150            Type::Primitive(PrimitiveType::Int),
151        )
152        .with_doc("Spec ID used to track the file containing a row"),
153    )
154});
155
156/// Lazy-initialized Iceberg field definition for the file_path column in position delete files.
157/// This field represents the path of a file in position-based delete files.
158static DELETE_FILE_PATH_FIELD: Lazy<NestedFieldRef> = Lazy::new(|| {
159    Arc::new(
160        NestedField::required(
161            RESERVED_FIELD_ID_DELETE_FILE_PATH,
162            RESERVED_COL_NAME_DELETE_FILE_PATH,
163            Type::Primitive(PrimitiveType::String),
164        )
165        .with_doc("Path of a file, used in position-based delete files"),
166    )
167});
168
169/// Lazy-initialized Iceberg field definition for the pos column in position delete files.
170/// This field represents the ordinal position of a row in position-based delete files.
171static DELETE_FILE_POS_FIELD: Lazy<NestedFieldRef> = Lazy::new(|| {
172    Arc::new(
173        NestedField::required(
174            RESERVED_FIELD_ID_DELETE_FILE_POS,
175            RESERVED_COL_NAME_DELETE_FILE_POS,
176            Type::Primitive(PrimitiveType::Long),
177        )
178        .with_doc("Ordinal position of a row, used in position-based delete files"),
179    )
180});
181
182/// Lazy-initialized Iceberg field definition for the _change_type metadata column.
183/// This field represents the record type in the changelog.
184static CHANGE_TYPE_FIELD: Lazy<NestedFieldRef> = Lazy::new(|| {
185    Arc::new(
186        NestedField::required(
187            RESERVED_FIELD_ID_CHANGE_TYPE,
188            RESERVED_COL_NAME_CHANGE_TYPE,
189            Type::Primitive(PrimitiveType::String),
190        )
191        .with_doc(
192            "The record type in the changelog (INSERT, DELETE, UPDATE_BEFORE, or UPDATE_AFTER)",
193        ),
194    )
195});
196
197/// Lazy-initialized Iceberg field definition for the _change_ordinal metadata column.
198/// This field represents the order of the change.
199static CHANGE_ORDINAL_FIELD: Lazy<NestedFieldRef> = Lazy::new(|| {
200    Arc::new(
201        NestedField::required(
202            RESERVED_FIELD_ID_CHANGE_ORDINAL,
203            RESERVED_COL_NAME_CHANGE_ORDINAL,
204            Type::Primitive(PrimitiveType::Int),
205        )
206        .with_doc("The order of the change"),
207    )
208});
209
210/// Lazy-initialized Iceberg field definition for the _commit_snapshot_id metadata column.
211/// This field represents the snapshot ID in which the change occurred.
212static COMMIT_SNAPSHOT_ID_FIELD: Lazy<NestedFieldRef> = Lazy::new(|| {
213    Arc::new(
214        NestedField::required(
215            RESERVED_FIELD_ID_COMMIT_SNAPSHOT_ID,
216            RESERVED_COL_NAME_COMMIT_SNAPSHOT_ID,
217            Type::Primitive(PrimitiveType::Long),
218        )
219        .with_doc("The snapshot ID in which the change occurred"),
220    )
221});
222
223/// Lazy-initialized Iceberg field definition for the _row_id metadata column.
224/// This field represents a unique long assigned for row lineage.
225static ROW_ID_FIELD: Lazy<NestedFieldRef> = Lazy::new(|| {
226    Arc::new(
227        NestedField::optional(
228            RESERVED_FIELD_ID_ROW_ID,
229            RESERVED_COL_NAME_ROW_ID,
230            Type::Primitive(PrimitiveType::Long),
231        )
232        .with_doc("A unique long assigned for row lineage"),
233    )
234});
235
236/// Lazy-initialized Iceberg field definition for the _last_updated_sequence_number metadata column.
237/// This field represents the sequence number which last updated this row.
238static LAST_UPDATED_SEQUENCE_NUMBER_FIELD: Lazy<NestedFieldRef> = Lazy::new(|| {
239    Arc::new(
240        NestedField::optional(
241            RESERVED_FIELD_ID_LAST_UPDATED_SEQUENCE_NUMBER,
242            RESERVED_COL_NAME_LAST_UPDATED_SEQUENCE_NUMBER,
243            Type::Primitive(PrimitiveType::Long),
244        )
245        .with_doc("The sequence number which last updated this row"),
246    )
247});
248
249/// Returns the Iceberg field definition for the _file metadata column.
250///
251/// # Returns
252/// A reference to the _file field definition as an Iceberg NestedField
253pub fn file_field() -> &'static NestedFieldRef {
254    &FILE_FIELD
255}
256
257/// Returns the Iceberg field definition for the _pos metadata column.
258///
259/// # Returns
260/// A reference to the _pos field definition as an Iceberg NestedField
261pub fn pos_field() -> &'static NestedFieldRef {
262    &POS_FIELD
263}
264
265/// Returns the Iceberg field definition for the _deleted metadata column.
266///
267/// # Returns
268/// A reference to the _deleted field definition as an Iceberg NestedField
269pub fn deleted_field() -> &'static NestedFieldRef {
270    &DELETED_FIELD
271}
272
273/// Returns the Iceberg field definition for the _spec_id metadata column.
274///
275/// # Returns
276/// A reference to the _spec_id field definition as an Iceberg NestedField
277pub fn spec_id_field() -> &'static NestedFieldRef {
278    &SPEC_ID_FIELD
279}
280
281/// Returns the Iceberg field definition for the file_path column in position delete files.
282///
283/// # Returns
284/// A reference to the file_path field definition as an Iceberg NestedField
285pub fn delete_file_path_field() -> &'static NestedFieldRef {
286    &DELETE_FILE_PATH_FIELD
287}
288
289/// Returns the Iceberg field definition for the pos column in position delete files.
290///
291/// # Returns
292/// A reference to the pos field definition as an Iceberg NestedField
293pub fn delete_file_pos_field() -> &'static NestedFieldRef {
294    &DELETE_FILE_POS_FIELD
295}
296
297/// Returns the Iceberg field definition for the _change_type metadata column.
298///
299/// # Returns
300/// A reference to the _change_type field definition as an Iceberg NestedField
301pub fn change_type_field() -> &'static NestedFieldRef {
302    &CHANGE_TYPE_FIELD
303}
304
305/// Returns the Iceberg field definition for the _change_ordinal metadata column.
306///
307/// # Returns
308/// A reference to the _change_ordinal field definition as an Iceberg NestedField
309pub fn change_ordinal_field() -> &'static NestedFieldRef {
310    &CHANGE_ORDINAL_FIELD
311}
312
313/// Returns the Iceberg field definition for the _commit_snapshot_id metadata column.
314///
315/// # Returns
316/// A reference to the _commit_snapshot_id field definition as an Iceberg NestedField
317pub fn commit_snapshot_id_field() -> &'static NestedFieldRef {
318    &COMMIT_SNAPSHOT_ID_FIELD
319}
320
321/// Returns the Iceberg field definition for the _row_id metadata column.
322///
323/// # Returns
324/// A reference to the _row_id field definition as an Iceberg NestedField
325pub fn row_id_field() -> &'static NestedFieldRef {
326    &ROW_ID_FIELD
327}
328
329/// Returns the Iceberg field definition for the _last_updated_sequence_number metadata column.
330///
331/// # Returns
332/// A reference to the _last_updated_sequence_number field definition as an Iceberg NestedField
333pub fn last_updated_sequence_number_field() -> &'static NestedFieldRef {
334    &LAST_UPDATED_SEQUENCE_NUMBER_FIELD
335}
336
337/// Creates the Iceberg field definition for the _partition metadata column.
338///
339/// The _partition field is a struct whose fields depend on the partition spec.
340/// This function creates the field dynamically with the provided partition fields.
341///
342/// # Arguments
343/// * `partition_fields` - The fields that make up the partition struct
344///
345/// # Returns
346/// A new _partition field definition as an Iceberg NestedField
347///
348/// # Example
349/// ```
350/// use std::sync::Arc;
351///
352/// use iceberg::metadata_columns::partition_field;
353/// use iceberg::spec::{NestedField, PrimitiveType, Type};
354///
355/// let fields = vec![
356///     Arc::new(NestedField::required(
357///         1,
358///         "year",
359///         Type::Primitive(PrimitiveType::Int),
360///     )),
361///     Arc::new(NestedField::required(
362///         2,
363///         "month",
364///         Type::Primitive(PrimitiveType::Int),
365///     )),
366/// ];
367/// let partition_field = partition_field(fields);
368/// ```
369pub fn partition_field(partition_fields: Vec<NestedFieldRef>) -> NestedFieldRef {
370    use crate::spec::StructType;
371
372    Arc::new(
373        NestedField::required(
374            RESERVED_FIELD_ID_PARTITION,
375            RESERVED_COL_NAME_PARTITION,
376            Type::Struct(StructType::new(partition_fields)),
377        )
378        .with_doc("Partition to which a row belongs"),
379    )
380}
381
382/// Returns the Iceberg field definition for a metadata field ID.
383///
384/// Note: This function does not support `_partition` (field ID `i32::MAX - 5`) because
385/// it's a struct field that requires dynamic partition fields. Use `partition_field()`
386/// instead to create the `_partition` field with the appropriate partition fields.
387///
388/// # Arguments
389/// * `field_id` - The metadata field ID
390///
391/// # Returns
392/// The Iceberg field definition for the metadata column, or an error if not a metadata field
393pub fn get_metadata_field(field_id: i32) -> Result<&'static NestedFieldRef> {
394    match field_id {
395        RESERVED_FIELD_ID_FILE => Ok(file_field()),
396        RESERVED_FIELD_ID_POS => Ok(pos_field()),
397        RESERVED_FIELD_ID_DELETED => Ok(deleted_field()),
398        RESERVED_FIELD_ID_SPEC_ID => Ok(spec_id_field()),
399        RESERVED_FIELD_ID_PARTITION => Err(Error::new(
400            ErrorKind::Unexpected,
401            "The _partition field must be created using partition_field() with appropriate partition fields",
402        )),
403        RESERVED_FIELD_ID_DELETE_FILE_PATH => Ok(delete_file_path_field()),
404        RESERVED_FIELD_ID_DELETE_FILE_POS => Ok(delete_file_pos_field()),
405        RESERVED_FIELD_ID_CHANGE_TYPE => Ok(change_type_field()),
406        RESERVED_FIELD_ID_CHANGE_ORDINAL => Ok(change_ordinal_field()),
407        RESERVED_FIELD_ID_COMMIT_SNAPSHOT_ID => Ok(commit_snapshot_id_field()),
408        RESERVED_FIELD_ID_ROW_ID => Ok(row_id_field()),
409        RESERVED_FIELD_ID_LAST_UPDATED_SEQUENCE_NUMBER => Ok(last_updated_sequence_number_field()),
410        _ if is_metadata_field(field_id) => {
411            // Future metadata fields can be added here
412            Err(Error::new(
413                ErrorKind::Unexpected,
414                format!(
415                    "Metadata field ID {field_id} recognized but field definition not implemented"
416                ),
417            ))
418        }
419        _ => Err(Error::new(
420            ErrorKind::Unexpected,
421            format!("Field ID {field_id} is not a metadata field"),
422        )),
423    }
424}
425
426/// Returns the field ID for a metadata column name.
427///
428/// This maps every reserved name to its field id, including the position-delete-file
429/// internal columns `pos` and `file_path`, which are not `_`-prefixed. It is therefore
430/// not a membership test for data-table metadata columns: `get_metadata_field_id("pos")`
431/// returns `Ok` even though `pos` is a valid user data column name. To decide whether a
432/// name is a projectable data-table metadata column, use [`is_metadata_column_name`].
433///
434/// # Arguments
435/// * `column_name` - The metadata column name
436///
437/// # Returns
438/// The field ID of the metadata column, or an error if the column name is not recognized
439pub fn get_metadata_field_id(column_name: &str) -> Result<i32> {
440    match column_name {
441        RESERVED_COL_NAME_FILE => Ok(RESERVED_FIELD_ID_FILE),
442        RESERVED_COL_NAME_POS => Ok(RESERVED_FIELD_ID_POS),
443        RESERVED_COL_NAME_DELETED => Ok(RESERVED_FIELD_ID_DELETED),
444        RESERVED_COL_NAME_SPEC_ID => Ok(RESERVED_FIELD_ID_SPEC_ID),
445        RESERVED_COL_NAME_PARTITION => Ok(RESERVED_FIELD_ID_PARTITION),
446        RESERVED_COL_NAME_DELETE_FILE_PATH => Ok(RESERVED_FIELD_ID_DELETE_FILE_PATH),
447        RESERVED_COL_NAME_DELETE_FILE_POS => Ok(RESERVED_FIELD_ID_DELETE_FILE_POS),
448        RESERVED_COL_NAME_CHANGE_TYPE => Ok(RESERVED_FIELD_ID_CHANGE_TYPE),
449        RESERVED_COL_NAME_CHANGE_ORDINAL => Ok(RESERVED_FIELD_ID_CHANGE_ORDINAL),
450        RESERVED_COL_NAME_COMMIT_SNAPSHOT_ID => Ok(RESERVED_FIELD_ID_COMMIT_SNAPSHOT_ID),
451        RESERVED_COL_NAME_ROW_ID => Ok(RESERVED_FIELD_ID_ROW_ID),
452        RESERVED_COL_NAME_LAST_UPDATED_SEQUENCE_NUMBER => {
453            Ok(RESERVED_FIELD_ID_LAST_UPDATED_SEQUENCE_NUMBER)
454        }
455        _ => Err(Error::new(
456            ErrorKind::Unexpected,
457            format!("Unknown/unsupported metadata column name: {column_name}"),
458        )),
459    }
460}
461
462/// Checks if a field ID is a data-table metadata column.
463///
464/// Mirrors Java `MetadataColumns.isMetadataColumn(int)` (backed by `META_IDS`): only the
465/// field ids of columns projectable in a data-table scan return `true`. The
466/// position-delete-file internal columns (`file_path`/`pos`, ids `i32::MAX - 101/-102`)
467/// and the changelog columns (`_change_type`/`_change_ordinal`/`_commit_snapshot_id`) are
468/// deliberately excluded, so their ids return `false` here even though
469/// [`get_metadata_field`] can still resolve them.
470///
471/// # Arguments
472/// * `field_id` - The field ID to check
473///
474/// # Returns
475/// `true` if the field ID is a data-table metadata column, `false` otherwise
476pub fn is_metadata_field(field_id: i32) -> bool {
477    matches!(
478        field_id,
479        RESERVED_FIELD_ID_FILE
480            | RESERVED_FIELD_ID_POS
481            | RESERVED_FIELD_ID_DELETED
482            | RESERVED_FIELD_ID_SPEC_ID
483            | RESERVED_FIELD_ID_PARTITION
484            | RESERVED_FIELD_ID_ROW_ID
485            | RESERVED_FIELD_ID_LAST_UPDATED_SEQUENCE_NUMBER
486    )
487}
488
489/// Checks if a column name is a data-table metadata column.
490///
491/// Mirrors Java `MetadataColumns.isMetadataColumn(String)` (backed by its `META_COLUMNS`
492/// allowlist plus the `_partition` special case): a column name is a data-table metadata
493/// column only if it is one of the reserved names that a scan can project.
494///
495/// This deliberately excludes two groups of reserved names that Java also keeps out of
496/// `META_COLUMNS`:
497/// - the position-delete-file internal columns `pos`, `file_path`, and `row`, so a user
498///   data column of the same name is not shadowed during a scan (issue #2837);
499/// - the changelog columns `_change_type`, `_change_ordinal`, and `_commit_snapshot_id`.
500///
501/// [`get_metadata_field_id`] still maps the reserved names it knows (including `pos` and
502/// `file_path`) to their field ids; only this membership check excludes them. (`row` has
503/// no reserved id in this crate, so `get_metadata_field_id("row")` is an error.)
504///
505/// # Arguments
506/// * `column_name` - The column name to check
507///
508/// # Returns
509/// `true` if the column name is a data-table metadata column, `false` otherwise
510pub fn is_metadata_column_name(column_name: &str) -> bool {
511    matches!(
512        column_name,
513        RESERVED_COL_NAME_FILE
514            | RESERVED_COL_NAME_POS
515            | RESERVED_COL_NAME_DELETED
516            | RESERVED_COL_NAME_SPEC_ID
517            | RESERVED_COL_NAME_PARTITION
518            | RESERVED_COL_NAME_ROW_ID
519            | RESERVED_COL_NAME_LAST_UPDATED_SEQUENCE_NUMBER
520    )
521}
522
523#[cfg(test)]
524mod tests {
525    use super::*;
526    use crate::spec::PrimitiveType;
527
528    #[test]
529    fn test_partition_field_creation() {
530        // Create partition fields for a hypothetical year/month partition
531        let partition_fields = vec![
532            Arc::new(NestedField::required(
533                1000,
534                "year",
535                Type::Primitive(PrimitiveType::Int),
536            )),
537            Arc::new(NestedField::required(
538                1001,
539                "month",
540                Type::Primitive(PrimitiveType::Int),
541            )),
542        ];
543
544        // Create the _partition metadata field
545        let partition = partition_field(partition_fields);
546
547        // Verify field properties
548        assert_eq!(partition.id, RESERVED_FIELD_ID_PARTITION);
549        assert_eq!(partition.name, RESERVED_COL_NAME_PARTITION);
550        assert!(partition.required);
551
552        // Verify it's a struct type with correct fields
553        if let Type::Struct(struct_type) = partition.field_type.as_ref() {
554            assert_eq!(struct_type.fields().len(), 2);
555            assert_eq!(struct_type.fields()[0].name, "year");
556            assert_eq!(struct_type.fields()[1].name, "month");
557        } else {
558            panic!("Expected struct type for _partition field");
559        }
560    }
561
562    #[test]
563    fn test_partition_field_id_recognized() {
564        assert!(is_metadata_field(RESERVED_FIELD_ID_PARTITION));
565    }
566
567    #[test]
568    fn test_partition_field_name_recognized() {
569        assert_eq!(
570            get_metadata_field_id(RESERVED_COL_NAME_PARTITION).unwrap(),
571            RESERVED_FIELD_ID_PARTITION
572        );
573    }
574
575    #[test]
576    fn test_get_metadata_field_returns_error_for_partition() {
577        // partition field requires dynamic creation, so get_metadata_field should return an error
578        let result = get_metadata_field(RESERVED_FIELD_ID_PARTITION);
579        assert!(result.is_err());
580        assert!(
581            result
582                .unwrap_err()
583                .to_string()
584                .contains("partition_field()")
585        );
586    }
587
588    #[test]
589    fn test_all_metadata_field_ids() {
590        // Test that all non-partition metadata fields can be retrieved
591        assert!(get_metadata_field(RESERVED_FIELD_ID_FILE).is_ok());
592        assert!(get_metadata_field(RESERVED_FIELD_ID_POS).is_ok());
593        assert!(get_metadata_field(RESERVED_FIELD_ID_DELETED).is_ok());
594        assert!(get_metadata_field(RESERVED_FIELD_ID_SPEC_ID).is_ok());
595        assert!(get_metadata_field(RESERVED_FIELD_ID_DELETE_FILE_PATH).is_ok());
596        assert!(get_metadata_field(RESERVED_FIELD_ID_DELETE_FILE_POS).is_ok());
597        assert!(get_metadata_field(RESERVED_FIELD_ID_CHANGE_TYPE).is_ok());
598        assert!(get_metadata_field(RESERVED_FIELD_ID_CHANGE_ORDINAL).is_ok());
599        assert!(get_metadata_field(RESERVED_FIELD_ID_COMMIT_SNAPSHOT_ID).is_ok());
600        assert!(get_metadata_field(RESERVED_FIELD_ID_ROW_ID).is_ok());
601        assert!(get_metadata_field(RESERVED_FIELD_ID_LAST_UPDATED_SEQUENCE_NUMBER).is_ok());
602    }
603
604    #[test]
605    fn test_row_lineage_metadata_fields_are_optional() {
606        // The spec requires readers to produce null for these columns in
607        // legitimate cases (e.g. a data file with a null first_row_id), so the
608        // fields must be optional rather than required.
609        assert!(!row_id_field().required);
610        assert!(!last_updated_sequence_number_field().required);
611    }
612
613    #[test]
614    fn test_data_table_metadata_columns() {
615        // Both sides of the data-table metadata check (name and field id) must accept
616        // these seven columns. On the name side Java keeps six in `META_COLUMNS` and
617        // recognizes `_partition` via the `isMetadataColumn(String)` special case; on the
618        // id side all seven are in `META_IDS`.
619        assert!(is_metadata_column_name(RESERVED_COL_NAME_FILE));
620        assert!(is_metadata_column_name(RESERVED_COL_NAME_POS));
621        assert!(is_metadata_column_name(RESERVED_COL_NAME_DELETED));
622        assert!(is_metadata_column_name(RESERVED_COL_NAME_SPEC_ID));
623        assert!(is_metadata_column_name(RESERVED_COL_NAME_PARTITION));
624        assert!(is_metadata_column_name(RESERVED_COL_NAME_ROW_ID));
625        assert!(is_metadata_column_name(
626            RESERVED_COL_NAME_LAST_UPDATED_SEQUENCE_NUMBER
627        ));
628
629        assert!(is_metadata_field(RESERVED_FIELD_ID_FILE));
630        assert!(is_metadata_field(RESERVED_FIELD_ID_POS));
631        assert!(is_metadata_field(RESERVED_FIELD_ID_DELETED));
632        assert!(is_metadata_field(RESERVED_FIELD_ID_SPEC_ID));
633        assert!(is_metadata_field(RESERVED_FIELD_ID_PARTITION));
634        assert!(is_metadata_field(RESERVED_FIELD_ID_ROW_ID));
635        assert!(is_metadata_field(
636            RESERVED_FIELD_ID_LAST_UPDATED_SEQUENCE_NUMBER
637        ));
638    }
639
640    #[test]
641    fn test_changelog_columns_are_not_data_table_metadata_columns() {
642        // Changelog columns are not in Java's `META_COLUMNS`, so they are not projectable
643        // data-table metadata columns (`isMetadataColumn` is false for them).
644        assert!(!is_metadata_column_name(RESERVED_COL_NAME_CHANGE_TYPE));
645        assert!(!is_metadata_column_name(RESERVED_COL_NAME_CHANGE_ORDINAL));
646        assert!(!is_metadata_column_name(
647            RESERVED_COL_NAME_COMMIT_SNAPSHOT_ID
648        ));
649        assert!(!is_metadata_field(RESERVED_FIELD_ID_CHANGE_TYPE));
650        assert!(!is_metadata_field(RESERVED_FIELD_ID_CHANGE_ORDINAL));
651        assert!(!is_metadata_field(RESERVED_FIELD_ID_COMMIT_SNAPSHOT_ID));
652    }
653
654    #[test]
655    fn test_delete_file_columns_are_not_data_table_metadata_columns() {
656        // `pos`, `file_path`, and `row` are the internal columns of a position-delete file,
657        // not projectable metadata columns of a data table. They must not shadow real data
658        // columns of the same name during a scan (issue #2837). Java returns false for both
659        // their names and their field ids; mirror that on both sides here.
660        assert!(!is_metadata_column_name("pos"));
661        assert!(!is_metadata_column_name("file_path"));
662        assert!(!is_metadata_column_name("row"));
663        assert!(!is_metadata_field(RESERVED_FIELD_ID_DELETE_FILE_POS));
664        assert!(!is_metadata_field(RESERVED_FIELD_ID_DELETE_FILE_PATH));
665        // `row` (Java's DELETE_FILE_ROW_FIELD_ID, i32::MAX - 103) has no reserved id in
666        // this crate, so it is not a metadata field either.
667        assert!(!is_metadata_field(i32::MAX - 103));
668
669        // `get_metadata_field_id` intentionally still maps the two bare names it knows to
670        // their reserved field ids; only the data-table membership check excludes them.
671        assert_eq!(
672            get_metadata_field_id(RESERVED_COL_NAME_DELETE_FILE_POS).unwrap(),
673            RESERVED_FIELD_ID_DELETE_FILE_POS
674        );
675        assert_eq!(
676            get_metadata_field_id(RESERVED_COL_NAME_DELETE_FILE_PATH).unwrap(),
677            RESERVED_FIELD_ID_DELETE_FILE_PATH
678        );
679    }
680
681    #[test]
682    fn test_ordinary_data_column_names_are_not_metadata_columns() {
683        assert!(!is_metadata_column_name("id"));
684        assert!(!is_metadata_column_name("name"));
685        // A user column that merely starts with an underscore but is not reserved.
686        assert!(!is_metadata_column_name("_custom"));
687    }
688}
iceberg/metadata_columns.rs

iceberg/
metadata_columns.rs
