Skip to main content

iceberg/cow_rewrite/
mod.rs

1// Licensed to the Apache Software Foundation (ASF) under one
2// or more contributor license agreements.  See the NOTICE file
3// distributed with this work for additional information
4// regarding copyright ownership.  The ASF licenses this file
5// to you under the Apache License, Version 2.0 (the
6// "License"); you may not use this file except in compliance
7// with the License.  You may obtain a copy of the License at
8//
9//   http://www.apache.org/licenses/LICENSE-2.0
10//
11// Unless required by applicable law or agreed to in writing,
12// software distributed under the License is distributed on an
13// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14// KIND, either express or implied.  See the License for the
15// specific language governing permissions and limitations
16// under the License.
17
18//! Copy-on-write rewrite primitives.
19//!
20//! This module plans candidate data files, reads their visible rows, applies a
21//! caller-provided batch rewriter, and writes replacement data files. It returns
22//! old and new file sets that can be committed by an overwrite-style transaction
23//! action.
24//!
25//! The primitive does not parse SQL and does not commit metadata by itself.
26//! Rewriters must emit batches compatible with the schema rows were read in
27//! (the planned snapshot's schema) and must preserve each source file's
28//! partition values; this primitive does not repartition rewritten rows.
29//!
30//! Replacement files deliberately mirror their source file's layout instead of
31//! re-optimizing it — a deliberate divergence from Java's RewriteDataFiles and
32//! Spark DML: they are written in the planned snapshot's schema rather than
33//! promoted to the table's current schema, they keep the source file's
34//! partition spec rather than being repartitioned into the table's default
35//! spec, and they stay unsorted since no sort order is applied. An opt-in
36//! promote-to-current-schema rewrite is left for follow-up work.
37//!
38//! The result carries data files only. A commit adapter consuming these file
39//! lists may remove position delete files and deletion vectors only when they
40//! exclusively reference removed data files. Equality delete files must be
41//! retained while they can still apply to other live data files by sequence
42//! number.
43//!
44//! ```rust,no_run
45//! # use std::sync::Arc;
46//! # use arrow_array::RecordBatch;
47//! # use iceberg::cow_rewrite::{CowBatchRewrite, CowBatchRewriter, CowRewriteBuilder};
48//! # use iceberg::table::Table;
49//! # use iceberg::Result;
50//! struct KeepAll;
51//!
52//! impl CowBatchRewriter for KeepAll {
53//!     fn rewrite_batch(&self, batch: RecordBatch) -> Result<CowBatchRewrite> {
54//!         Ok(CowBatchRewrite {
55//!             output: Some(batch),
56//!             changed: false,
57//!         })
58//!     }
59//! }
60//!
61//! # async fn example(table: &Table) -> Result<()> {
62//! let result = CowRewriteBuilder::new(table)
63//!     .with_rewriter(Arc::new(KeepAll))
64//!     .rewrite()
65//!     .await?;
66//!
67//! assert!(!result.has_changes());
68//! # Ok(())
69//! # }
70//! ```
71
72mod plan;
73mod rewriter;
74pub(crate) mod writer;
75
76use std::sync::Arc;
77
78use arrow_array::RecordBatch;
79use futures::TryStreamExt;
80pub use plan::CowRewriteFile;
81pub use rewriter::{CowBatchRewrite, CowBatchRewriter};
82
83use crate::arrow::ArrowReaderOptions;
84use crate::expr::Predicate;
85use crate::scan::FileScanTaskStream;
86use crate::spec::{DataFile, PartitionKey};
87use crate::table::Table;
88use crate::{Error, ErrorKind, Result};
89
90/// Counters produced by a copy-on-write rewrite.
91#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)]
92pub struct CowRewriteStats {
93    /// Number of candidate files selected by planning.
94    pub candidate_files: usize,
95    /// Number of old files that have replacement output or are fully removed.
96    pub rewritten_files: usize,
97    /// Number of candidate files that did not change after row rewriting.
98    pub unchanged_files: usize,
99    /// Visible input row count read from candidate files.
100    pub input_rows: u64,
101    /// Output row count written to replacement files.
102    ///
103    /// Rows the rewriter emitted for files that turned out unchanged are not
104    /// counted, so this always matches the row counts of `added_data_files`.
105    pub output_rows: u64,
106    /// Number of input batches that changed, including batches the rewriter
107    /// dropped entirely (`output: None`) even if it did not flag them.
108    pub changed_batches: u64,
109}
110
111/// Result of a copy-on-write rewrite operation.
112#[derive(Debug, Default)]
113pub struct CowRewriteResult {
114    /// Old data files that should be removed by the commit action.
115    pub removed_data_files: Vec<DataFile>,
116    /// New data files that should be added by the commit action.
117    pub added_data_files: Vec<DataFile>,
118    /// Candidate files that were read and left unchanged.
119    ///
120    /// Files whose visible rows were all removed by delete files are NOT
121    /// included here: they read as zero rows and are reported in
122    /// `removed_data_files` with no replacement. A commit adapter may remove
123    /// position deletes and deletion vectors that exclusively reference these
124    /// removed files, but must retain equality deletes that can still apply to
125    /// other live files.
126    pub unchanged_data_files: Vec<DataFile>,
127    /// Rewrite counters.
128    pub stats: CowRewriteStats,
129}
130
131impl CowRewriteResult {
132    /// Returns true if the rewrite produced any table changes.
133    pub fn has_changes(&self) -> bool {
134        !self.removed_data_files.is_empty() || !self.added_data_files.is_empty()
135    }
136}
137
138/// Builder for orchestrating copy-on-write data file rewrites.
139pub struct CowRewriteBuilder<'a> {
140    table: &'a Table,
141    predicate: Predicate,
142    snapshot_id: Option<i64>,
143    batch_size: Option<usize>,
144    case_sensitive: bool,
145    rewriter: Option<Arc<dyn CowBatchRewriter>>,
146}
147
148impl<'a> CowRewriteBuilder<'a> {
149    /// Creates a copy-on-write rewrite builder for `table`.
150    pub fn new(table: &'a Table) -> Self {
151        Self {
152            table,
153            predicate: Predicate::AlwaysTrue,
154            snapshot_id: None,
155            batch_size: None,
156            case_sensitive: true,
157            rewriter: None,
158        }
159    }
160
161    /// Sets the row predicate used to plan candidate files.
162    pub fn with_predicate(mut self, predicate: Predicate) -> Self {
163        self.predicate = predicate;
164        self
165    }
166
167    /// Sets the snapshot id used to plan candidate files.
168    pub fn with_snapshot_id(mut self, snapshot_id: i64) -> Self {
169        self.snapshot_id = Some(snapshot_id);
170        self
171    }
172
173    /// Sets the Arrow reader batch size.
174    pub fn with_batch_size(mut self, batch_size: usize) -> Self {
175        self.batch_size = Some(batch_size);
176        self
177    }
178
179    /// Sets the case sensitivity used to bind the planning predicate.
180    pub fn with_case_sensitive(mut self, case_sensitive: bool) -> Self {
181        self.case_sensitive = case_sensitive;
182        self
183    }
184
185    /// Sets the record batch rewriter.
186    pub fn with_rewriter(mut self, rewriter: Arc<dyn CowBatchRewriter>) -> Self {
187        self.rewriter = Some(rewriter);
188        self
189    }
190
191    /// Plans, reads, rewrites, and writes replacement data files.
192    ///
193    /// # Memory usage
194    ///
195    /// Batches read before a file's first changed batch are buffered so that
196    /// no replacement file is emitted for a source file that turns out
197    /// unchanged. A file that never changes — or whose first change sits at
198    /// its very end — therefore buffers its entire decoded contents in
199    /// memory. Files are processed sequentially, so peak usage is one decoded
200    /// source file at a time, but that can still be several GB for a
201    /// compaction-sized file. A size-capped fallback that starts writing the
202    /// replacement once the buffer crosses a threshold is left for follow-up
203    /// work.
204    pub async fn rewrite(self) -> Result<CowRewriteResult> {
205        let rewriter = self.rewriter.ok_or_else(|| {
206            Error::new(
207                ErrorKind::PreconditionFailed,
208                "COW rewrite requires a batch rewriter",
209            )
210        })?;
211        let files = plan::plan_cow_rewrite_files(
212            self.table,
213            Some(self.predicate),
214            self.snapshot_id,
215            self.case_sensitive,
216        )
217        .await?;
218
219        let mut result = CowRewriteResult {
220            stats: CowRewriteStats {
221                candidate_files: files.len(),
222                ..CowRewriteStats::default()
223            },
224            ..CowRewriteResult::default()
225        };
226
227        for file in files {
228            let CowRewriteFile {
229                old_data_file,
230                scan_task,
231            } = file;
232            // Schema the rows are read in (the planned snapshot's schema). The
233            // replacement files must be written with this schema so that batches
234            // remain compatible when the table's current schema has evolved past
235            // the snapshot the source files belong to. This preserves the older
236            // schema in replacement files rather than promoting them to the
237            // table's current schema during this rewrite.
238            let write_schema = scan_task.schema_ref();
239            let has_delete_files = !scan_task.deletes().is_empty();
240
241            // Batches produced before the first changed batch. They are buffered
242            // rather than written immediately because the primitive must not
243            // emit a replacement file for a source file that turns out to be
244            // unchanged. Once a changed batch is observed the buffered prefix is
245            // flushed to the writer and all subsequent batches stream straight
246            // through.
247            //
248            // Worst-case footprint: for a file that never changes (or whose
249            // first change sits at its very end) the prefix holds the entire
250            // decoded source file in memory. Files are processed sequentially,
251            // so peak usage is one file at a time, but that can still be several
252            // GB for a compaction-sized file. A size-capped fallback that starts
253            // writing the replacement once the buffer crosses a threshold is
254            // left for follow-up work.
255            let mut prefix: Vec<RecordBatch> = Vec::new();
256            let mut file_changed = false;
257            let mut file_input_rows = 0_u64;
258            let mut file_output_rows = 0_u64;
259            let mut writer: Option<Box<dyn crate::writer::IcebergWriter>> = None;
260            // Debug-only tripwire for the "preserve partition values"
261            // contract; see `DebugPartitionGuard`.
262            #[cfg(debug_assertions)]
263            let partition_guard =
264                DebugPartitionGuard::new(self.table, &old_data_file, &write_schema)?;
265
266            // Planning already cleared the row predicate (see
267            // `ManifestEntryContext::into_cow_rewrite_file`), so this task
268            // reads every row of the source file.
269            let tasks = Box::pin(futures::stream::iter(vec![Ok(scan_task)])) as FileScanTaskStream;
270
271            // Each candidate file gets its own reader so the per-file prefix
272            // and lazy-writer semantics stay intact; the delete-file cache is
273            // therefore also per file, and equality deletes shared by several
274            // candidates are fetched once per file.
275            let mut reader_builder = self.table.reader_builder();
276            if let Some(batch_size) = self.batch_size {
277                reader_builder = reader_builder.with_options(
278                    ArrowReaderOptions::builder()
279                        .with_batch_size(batch_size)
280                        .build(),
281                );
282            }
283
284            let mut batches = reader_builder.build().read(tasks)?.stream();
285            while let Some(batch) = batches.try_next().await? {
286                result.stats.input_rows += batch.num_rows() as u64;
287                file_input_rows += batch.num_rows() as u64;
288
289                let rewrite = rewriter.rewrite_batch(batch)?;
290                // `output: None` means the batch is fully removed, which is
291                // itself a change. Derive the effective flag instead of
292                // trusting every rewriter to keep `changed` consistent with
293                // `output` — otherwise a `{changed: false, output: None}`
294                // batch would silently drop its rows while leaving the file
295                // marked unchanged.
296                let changed = rewrite.changed || rewrite.output.is_none();
297                if changed {
298                    file_changed = true;
299                    result.stats.changed_batches += 1;
300                }
301
302                // The guard must see every emitted batch, including ones that
303                // are only buffered into the prefix before the file flips to
304                // changed.
305                #[cfg(debug_assertions)]
306                if let (Some(guard), Some(output)) = (&partition_guard, &rewrite.output) {
307                    guard.check(output)?;
308                }
309
310                // A dropped batch can be the first change. Flush any kept
311                // prefix even when this batch has no output; defer opening the
312                // writer only if there are no rows to preserve yet.
313                if file_changed
314                    && writer.is_none()
315                    && (!prefix.is_empty() || rewrite.output.is_some())
316                {
317                    let partition_key =
318                        source_partition_key(self.table, &old_data_file, &write_schema)?;
319                    writer = Some(
320                        writer::build_replacement_writer(
321                            self.table,
322                            write_schema.clone(),
323                            Some(partition_key),
324                        )
325                        .await?,
326                    );
327                }
328
329                if let Some(writer) = writer.as_mut() {
330                    for prefix_batch in prefix.drain(..) {
331                        writer.write(prefix_batch).await?;
332                    }
333                    if let Some(output) = rewrite.output {
334                        file_output_rows += output.num_rows() as u64;
335                        writer.write(output).await?;
336                    }
337                } else if let Some(output) = rewrite.output {
338                    file_output_rows += output.num_rows() as u64;
339                    prefix.push(output);
340                }
341            }
342
343            // A candidate whose visible rows were all removed by its delete
344            // files reads as zero rows and the loop above never runs. Treat it
345            // the same as a rewriter that dropped every batch — changed with
346            // no replacement — so the file and its delete files can be
347            // compacted away instead of being pinned in the table forever.
348            let fully_removed_by_deletes =
349                !file_changed && file_input_rows == 0 && has_delete_files;
350
351            if file_changed || fully_removed_by_deletes {
352                result.stats.rewritten_files += 1;
353                result.stats.output_rows += file_output_rows;
354                result.removed_data_files.push(old_data_file);
355
356                if let Some(mut writer) = writer {
357                    let added_data_files = writer.close().await?;
358                    result.added_data_files.extend(added_data_files);
359                }
360                // If `writer` is `None`, no visible rows remain after delete
361                // files and batch rewriting, so no replacement is written.
362            } else {
363                result.stats.unchanged_files += 1;
364                result.unchanged_data_files.push(old_data_file);
365                // `prefix` is dropped here; no replacement file was written.
366            }
367        }
368
369        Ok(result)
370    }
371}
372
373fn source_partition_key(
374    table: &Table,
375    data_file: &DataFile,
376    schema: &crate::spec::SchemaRef,
377) -> Result<PartitionKey> {
378    let spec = table
379        .metadata()
380        .partition_spec_by_id(data_file.partition_spec_id)
381        .ok_or_else(|| {
382            Error::new(
383                ErrorKind::DataInvalid,
384                format!(
385                    "Missing partition spec {} for COW rewrite source file",
386                    data_file.partition_spec_id
387                ),
388            )
389        })?
390        .as_ref()
391        .clone();
392    // `PartitionKey::new` does not bind the spec to the schema, so validate the
393    // binding here: a spec that is incompatible with the planned snapshot
394    // schema should fail with a clear error instead of producing a bad
395    // partition path when the writer later calls `PartitionKey::to_path`.
396    spec.partition_type(schema).map_err(|err| {
397        Error::new(
398            ErrorKind::DataInvalid,
399            format!(
400                "Cannot bind partition spec {} to the planned snapshot schema for COW rewrite",
401                data_file.partition_spec_id
402            ),
403        )
404        .with_source(err)
405    })?;
406
407    Ok(PartitionKey::new(
408        spec,
409        schema.clone(),
410        data_file.partition().clone(),
411    ))
412}
413
414/// Debug-only tripwire for the rewriter contract that output batches preserve
415/// the source file's partition values: replacement files record the source
416/// partition verbatim, so a rewriter that mutates a partition-source column
417/// would silently write rows under a partition they do not belong to. Compiled
418/// out in release builds, where per-row partition calculation would be too
419/// expensive for the rewrite hot path.
420#[cfg(debug_assertions)]
421struct DebugPartitionGuard {
422    calculator: crate::arrow::PartitionValueCalculator,
423    partition_type: crate::spec::StructType,
424    expected: crate::spec::Struct,
425}
426
427#[cfg(debug_assertions)]
428impl DebugPartitionGuard {
429    /// Builds a guard for one candidate file, or `None` when the source
430    /// file's partition spec is unpartitioned. Mirrors the spec lookup and
431    /// schema binding of [`source_partition_key`].
432    fn new(
433        table: &Table,
434        data_file: &DataFile,
435        schema: &crate::spec::SchemaRef,
436    ) -> Result<Option<Self>> {
437        let spec = table
438            .metadata()
439            .partition_spec_by_id(data_file.partition_spec_id)
440            .ok_or_else(|| {
441                Error::new(
442                    ErrorKind::DataInvalid,
443                    format!(
444                        "Missing partition spec {} for COW rewrite source file",
445                        data_file.partition_spec_id
446                    ),
447                )
448            })?;
449        if spec.is_unpartitioned() {
450            return Ok(None);
451        }
452
453        Ok(Some(Self {
454            calculator: crate::arrow::PartitionValueCalculator::try_new(spec, schema)?,
455            partition_type: spec.partition_type(schema)?,
456            expected: data_file.partition().clone(),
457        }))
458    }
459
460    /// Asserts that every row in `output` computes to the source file's
461    /// partition under the source spec.
462    fn check(&self, output: &RecordBatch) -> Result<()> {
463        let partition_array = self.calculator.calculate(output)?;
464        let values = crate::arrow::arrow_struct_to_literal(&partition_array, &self.partition_type)?;
465        for value in values {
466            debug_assert_eq!(
467                value,
468                Some(crate::spec::Literal::Struct(self.expected.clone())),
469                "COW rewriter must preserve the source file's partition values: \
470                 replacement files record the source partition verbatim, so \
471                 mutating a partition-source column writes rows under a \
472                 partition they do not belong to"
473            );
474        }
475        Ok(())
476    }
477}
478
479#[cfg(test)]
480mod tests {
481    use std::collections::{HashMap, HashSet};
482    use std::sync::Arc;
483
484    use arrow_array::{Array, ArrayRef, BooleanArray, Int32Array, RecordBatch};
485    use arrow_schema::{DataType, Field, Schema as ArrowSchema};
486    use futures::TryStreamExt;
487    use parquet::arrow::PARQUET_FIELD_ID_META_KEY;
488    use tempfile::TempDir;
489
490    use crate::cow_rewrite::{CowBatchRewrite, CowBatchRewriter, CowRewriteBuilder};
491    use crate::io::LocalFsStorageFactory;
492    use crate::memory::{MEMORY_CATALOG_WAREHOUSE, MemoryCatalogBuilder};
493    use crate::scan::{FileScanTask, FileScanTaskStream};
494    use crate::spec::{
495        DataFile, Literal, NestedField, PrimitiveType, Schema, Struct, TableProperties, Transform,
496        Type,
497    };
498    use crate::table::Table;
499    use crate::test_utils::scan::TableTestFixture;
500    use crate::transaction::{AddColumn, ApplyTransactionAction, Transaction};
501    use crate::{Catalog, CatalogBuilder, Error, ErrorKind, NamespaceIdent, Result, TableCreation};
502
503    struct KeepAll;
504
505    impl CowBatchRewriter for KeepAll {
506        fn rewrite_batch(&self, batch: RecordBatch) -> Result<CowBatchRewrite> {
507            Ok(CowBatchRewrite {
508                output: Some(batch),
509                changed: false,
510            })
511        }
512    }
513
514    struct DeleteEvenIds;
515
516    impl CowBatchRewriter for DeleteEvenIds {
517        fn rewrite_batch(&self, batch: RecordBatch) -> Result<CowBatchRewrite> {
518            let ids = batch
519                .column_by_name("id")
520                .ok_or_else(|| Error::new(ErrorKind::DataInvalid, "missing id column"))?
521                .as_any()
522                .downcast_ref::<Int32Array>()
523                .ok_or_else(|| Error::new(ErrorKind::DataInvalid, "id must be Int32"))?;
524
525            let keep =
526                BooleanArray::from_iter((0..ids.len()).map(|row| Some(ids.value(row) % 2 != 0)));
527            let filtered = arrow_select::filter::filter_record_batch(&batch, &keep)
528                .map_err(|err| Error::new(ErrorKind::Unexpected, err.to_string()))?;
529
530            Ok(CowBatchRewrite {
531                changed: filtered.num_rows() != batch.num_rows(),
532                output: (filtered.num_rows() > 0).then_some(filtered),
533            })
534        }
535    }
536
537    struct IncrementValueForEvenIds;
538
539    impl CowBatchRewriter for IncrementValueForEvenIds {
540        fn rewrite_batch(&self, batch: RecordBatch) -> Result<CowBatchRewrite> {
541            let ids = batch
542                .column_by_name("id")
543                .ok_or_else(|| Error::new(ErrorKind::DataInvalid, "missing id column"))?
544                .as_any()
545                .downcast_ref::<Int32Array>()
546                .ok_or_else(|| Error::new(ErrorKind::DataInvalid, "id must be Int32"))?;
547            let values = batch
548                .column_by_name("value")
549                .ok_or_else(|| Error::new(ErrorKind::DataInvalid, "missing value column"))?
550                .as_any()
551                .downcast_ref::<Int32Array>()
552                .ok_or_else(|| Error::new(ErrorKind::DataInvalid, "value must be Int32"))?;
553
554            let mut changed = false;
555            let updated_values = Int32Array::from_iter((0..values.len()).map(|row| {
556                let value = values.value(row);
557                if ids.value(row) % 2 == 0 {
558                    changed = true;
559                    Some(value + 10)
560                } else {
561                    Some(value)
562                }
563            }));
564            let output = RecordBatch::try_new(batch.schema(), vec![
565                batch.column(0).clone(),
566                Arc::new(updated_values),
567            ])
568            .map_err(|err| Error::new(ErrorKind::Unexpected, err.to_string()))?;
569
570            Ok(CowBatchRewrite {
571                output: Some(output),
572                changed,
573            })
574        }
575    }
576
577    struct CowRewriteFixture {
578        _temp_dir: TempDir,
579        table: Table,
580    }
581
582    async fn test_table_with_ids(ids: Vec<i32>) -> Result<CowRewriteFixture> {
583        let temp_dir = TempDir::new().unwrap();
584        let warehouse = format!("file://{}", temp_dir.path().join("warehouse").display());
585        let catalog = MemoryCatalogBuilder::default()
586            .with_storage_factory(Arc::new(LocalFsStorageFactory))
587            .load(
588                "memory",
589                HashMap::from([(MEMORY_CATALOG_WAREHOUSE.to_string(), warehouse)]),
590            )
591            .await?;
592        let namespace = NamespaceIdent::new("ns".to_string());
593        catalog.create_namespace(&namespace, HashMap::new()).await?;
594
595        let schema = Schema::builder()
596            .with_fields(vec![
597                NestedField::required(1, "id", Type::Primitive(PrimitiveType::Int)).into(),
598            ])
599            .build()?;
600        let table = catalog
601            .create_table(
602                &namespace,
603                TableCreation::builder()
604                    .name("cow_rewrite_fixture".to_string())
605                    .schema(schema)
606                    .build(),
607            )
608            .await?;
609
610        let arrow_schema = Arc::new(ArrowSchema::new(vec![
611            Field::new("id", DataType::Int32, false).with_metadata(HashMap::from([(
612                PARQUET_FIELD_ID_META_KEY.to_string(),
613                "1".to_string(),
614            )])),
615        ]));
616        let batch = RecordBatch::try_new(arrow_schema, vec![
617            Arc::new(Int32Array::from(ids)) as ArrayRef
618        ])?;
619        let data_files = super::writer::write_replacement_batches(
620            &table,
621            table.metadata().current_schema().clone(),
622            None,
623            futures::stream::iter(vec![Ok(batch)]),
624        )
625        .await?;
626
627        let tx = Transaction::new(&table);
628        let tx = tx.fast_append().add_data_files(data_files).apply(tx)?;
629        let table = tx.commit(&catalog).await?;
630
631        Ok(CowRewriteFixture {
632            _temp_dir: temp_dir,
633            table,
634        })
635    }
636
637    async fn test_table_with_id_batches(batches: Vec<Vec<i32>>) -> Result<CowRewriteFixture> {
638        let temp_dir = TempDir::new().unwrap();
639        let warehouse = format!("file://{}", temp_dir.path().join("warehouse").display());
640        let catalog = MemoryCatalogBuilder::default()
641            .with_storage_factory(Arc::new(LocalFsStorageFactory))
642            .load(
643                "memory",
644                HashMap::from([(MEMORY_CATALOG_WAREHOUSE.to_string(), warehouse)]),
645            )
646            .await?;
647        let namespace = NamespaceIdent::new("ns".to_string());
648        catalog.create_namespace(&namespace, HashMap::new()).await?;
649
650        let schema = Schema::builder()
651            .with_fields(vec![
652                NestedField::required(1, "id", Type::Primitive(PrimitiveType::Int)).into(),
653            ])
654            .build()?;
655        let table = catalog
656            .create_table(
657                &namespace,
658                TableCreation::builder()
659                    .name("cow_rewrite_fixture".to_string())
660                    .schema(schema)
661                    .properties(HashMap::from([(
662                        TableProperties::PROPERTY_WRITE_TARGET_FILE_SIZE_BYTES.to_string(),
663                        "1".to_string(),
664                    )]))
665                    .build(),
666            )
667            .await?;
668
669        let arrow_schema = Arc::new(ArrowSchema::new(vec![
670            Field::new("id", DataType::Int32, false).with_metadata(HashMap::from([(
671                PARQUET_FIELD_ID_META_KEY.to_string(),
672                "1".to_string(),
673            )])),
674        ]));
675        let input = batches.into_iter().map(|ids| {
676            Ok(RecordBatch::try_new(arrow_schema.clone(), vec![
677                Arc::new(Int32Array::from(ids)) as ArrayRef,
678            ])?)
679        });
680        let data_files = super::writer::write_replacement_batches(
681            &table,
682            table.metadata().current_schema().clone(),
683            None,
684            futures::stream::iter(input),
685        )
686        .await?;
687
688        let tx = Transaction::new(&table);
689        let tx = tx.fast_append().add_data_files(data_files).apply(tx)?;
690        let table = tx.commit(&catalog).await?;
691
692        Ok(CowRewriteFixture {
693            _temp_dir: temp_dir,
694            table,
695        })
696    }
697
698    async fn test_table_with_id_value_rows(rows: Vec<(i32, i32)>) -> Result<CowRewriteFixture> {
699        let temp_dir = TempDir::new().unwrap();
700        let warehouse = format!("file://{}", temp_dir.path().join("warehouse").display());
701        let catalog = MemoryCatalogBuilder::default()
702            .with_storage_factory(Arc::new(LocalFsStorageFactory))
703            .load(
704                "memory",
705                HashMap::from([(MEMORY_CATALOG_WAREHOUSE.to_string(), warehouse)]),
706            )
707            .await?;
708        let namespace = NamespaceIdent::new("ns".to_string());
709        catalog.create_namespace(&namespace, HashMap::new()).await?;
710
711        let schema = Schema::builder()
712            .with_fields(vec![
713                NestedField::required(1, "id", Type::Primitive(PrimitiveType::Int)).into(),
714                NestedField::required(2, "value", Type::Primitive(PrimitiveType::Int)).into(),
715            ])
716            .build()?;
717        let table = catalog
718            .create_table(
719                &namespace,
720                TableCreation::builder()
721                    .name("cow_rewrite_fixture".to_string())
722                    .schema(schema)
723                    .build(),
724            )
725            .await?;
726
727        let arrow_schema = Arc::new(ArrowSchema::new(vec![
728            Field::new("id", DataType::Int32, false).with_metadata(HashMap::from([(
729                PARQUET_FIELD_ID_META_KEY.to_string(),
730                "1".to_string(),
731            )])),
732            Field::new("value", DataType::Int32, false).with_metadata(HashMap::from([(
733                PARQUET_FIELD_ID_META_KEY.to_string(),
734                "2".to_string(),
735            )])),
736        ]));
737        let ids = rows.iter().map(|(id, _)| *id).collect::<Vec<_>>();
738        let values = rows.iter().map(|(_, value)| *value).collect::<Vec<_>>();
739        let batch = RecordBatch::try_new(arrow_schema, vec![
740            Arc::new(Int32Array::from(ids)) as ArrayRef,
741            Arc::new(Int32Array::from(values)) as ArrayRef,
742        ])?;
743        let data_files = super::writer::write_replacement_batches(
744            &table,
745            table.metadata().current_schema().clone(),
746            None,
747            futures::stream::iter(vec![Ok(batch)]),
748        )
749        .await?;
750
751        let tx = Transaction::new(&table);
752        let tx = tx.fast_append().add_data_files(data_files).apply(tx)?;
753        let table = tx.commit(&catalog).await?;
754
755        Ok(CowRewriteFixture {
756            _temp_dir: temp_dir,
757            table,
758        })
759    }
760
761    async fn read_ids(table: &Table, files: &[DataFile]) -> Result<Vec<i32>> {
762        let schema = table.metadata().current_schema().clone();
763        let project_field_ids = schema
764            .as_struct()
765            .fields()
766            .iter()
767            .map(|field| field.id)
768            .collect::<Vec<_>>();
769        let tasks = files
770            .iter()
771            .map(|data_file| {
772                FileScanTask::builder()
773                    .with_file_size_in_bytes(data_file.file_size_in_bytes())
774                    .with_start(0)
775                    .with_length(data_file.file_size_in_bytes())
776                    .with_record_count(Some(data_file.record_count()))
777                    .with_data_file_path(data_file.file_path().to_string())
778                    .with_data_file_format(data_file.file_format())
779                    .with_schema(schema.clone())
780                    .with_project_field_ids(project_field_ids.clone())
781                    .with_case_sensitive(true)
782                    .build()
783            })
784            .collect::<Vec<_>>();
785        let task_stream = Box::pin(futures::stream::iter(tasks)) as FileScanTaskStream;
786        let batches = table
787            .reader_builder()
788            .build()
789            .read(task_stream)?
790            .stream()
791            .try_collect::<Vec<_>>()
792            .await?;
793
794        let mut ids = Vec::new();
795        for batch in batches {
796            let column = batch
797                .column_by_name("id")
798                .ok_or_else(|| Error::new(ErrorKind::DataInvalid, "missing id column"))?
799                .as_any()
800                .downcast_ref::<Int32Array>()
801                .ok_or_else(|| Error::new(ErrorKind::DataInvalid, "id must be Int32"))?;
802            ids.extend((0..column.len()).map(|row| column.value(row)));
803        }
804        ids.sort_unstable();
805        Ok(ids)
806    }
807
808    async fn read_ids_and_values(table: &Table, files: &[DataFile]) -> Result<Vec<(i32, i32)>> {
809        let schema = table.metadata().current_schema().clone();
810        let project_field_ids = schema
811            .as_struct()
812            .fields()
813            .iter()
814            .map(|field| field.id)
815            .collect::<Vec<_>>();
816        let tasks = files
817            .iter()
818            .map(|data_file| {
819                FileScanTask::builder()
820                    .with_file_size_in_bytes(data_file.file_size_in_bytes())
821                    .with_start(0)
822                    .with_length(data_file.file_size_in_bytes())
823                    .with_record_count(Some(data_file.record_count()))
824                    .with_data_file_path(data_file.file_path().to_string())
825                    .with_data_file_format(data_file.file_format())
826                    .with_schema(schema.clone())
827                    .with_project_field_ids(project_field_ids.clone())
828                    .with_case_sensitive(true)
829                    .build()
830            })
831            .collect::<Vec<_>>();
832        let task_stream = Box::pin(futures::stream::iter(tasks)) as FileScanTaskStream;
833        let batches = table
834            .reader_builder()
835            .build()
836            .read(task_stream)?
837            .stream()
838            .try_collect::<Vec<_>>()
839            .await?;
840
841        let mut rows = Vec::new();
842        for batch in batches {
843            let ids = batch
844                .column_by_name("id")
845                .ok_or_else(|| Error::new(ErrorKind::DataInvalid, "missing id column"))?
846                .as_any()
847                .downcast_ref::<Int32Array>()
848                .ok_or_else(|| Error::new(ErrorKind::DataInvalid, "id must be Int32"))?;
849            let values = batch
850                .column_by_name("value")
851                .ok_or_else(|| Error::new(ErrorKind::DataInvalid, "missing value column"))?
852                .as_any()
853                .downcast_ref::<Int32Array>()
854                .ok_or_else(|| Error::new(ErrorKind::DataInvalid, "value must be Int32"))?;
855            rows.extend((0..ids.len()).map(|row| (ids.value(row), values.value(row))));
856        }
857        rows.sort_unstable_by_key(|(id, _)| *id);
858        Ok(rows)
859    }
860
861    #[tokio::test]
862    async fn cow_rewrite_keep_all_produces_no_changes() -> Result<()> {
863        let fixture = test_table_with_ids(vec![1, 2, 3]).await?;
864
865        let result = CowRewriteBuilder::new(&fixture.table)
866            .with_predicate(crate::expr::Predicate::AlwaysTrue)
867            .with_rewriter(Arc::new(KeepAll))
868            .rewrite()
869            .await?;
870
871        assert!(!result.has_changes());
872        assert_eq!(result.removed_data_files.len(), 0);
873        assert_eq!(result.added_data_files.len(), 0);
874        assert_eq!(result.unchanged_data_files.len(), 1);
875        assert_eq!(result.stats.candidate_files, 1);
876        assert_eq!(result.stats.unchanged_files, 1);
877        assert_eq!(result.stats.input_rows, 3);
878        // Nothing was written, so rows emitted for the unchanged file are not
879        // counted as output.
880        assert_eq!(result.stats.output_rows, 0);
881
882        Ok(())
883    }
884
885    #[tokio::test]
886    async fn cow_rewrite_requires_rewriter() -> Result<()> {
887        let fixture = test_table_with_ids(vec![1]).await?;
888
889        let err = CowRewriteBuilder::new(&fixture.table)
890            .rewrite()
891            .await
892            .expect_err("missing rewriter should fail");
893
894        assert_eq!(err.kind(), ErrorKind::PreconditionFailed);
895
896        Ok(())
897    }
898
899    #[tokio::test]
900    async fn cow_rewrite_delete_rows_removes_old_file_and_adds_replacement() -> Result<()> {
901        let fixture = test_table_with_ids(vec![1, 2, 3, 4]).await?;
902
903        let result = CowRewriteBuilder::new(&fixture.table)
904            .with_predicate(crate::expr::Predicate::AlwaysTrue)
905            .with_rewriter(Arc::new(DeleteEvenIds))
906            .rewrite()
907            .await?;
908
909        assert!(result.has_changes());
910        assert_eq!(result.removed_data_files.len(), 1);
911        assert_eq!(result.added_data_files.len(), 1);
912        assert_eq!(result.stats.input_rows, 4);
913        assert_eq!(result.stats.output_rows, 2);
914
915        let ids = read_ids(&fixture.table, &result.added_data_files).await?;
916        assert_eq!(ids, vec![1, 3]);
917
918        Ok(())
919    }
920
921    #[tokio::test]
922    async fn cow_rewrite_update_rows_rewrites_file_with_updated_values() -> Result<()> {
923        let fixture =
924            test_table_with_id_value_rows(vec![(1, 10), (2, 20), (3, 30), (4, 40)]).await?;
925
926        let result = CowRewriteBuilder::new(&fixture.table)
927            .with_predicate(crate::expr::Predicate::AlwaysTrue)
928            .with_rewriter(Arc::new(IncrementValueForEvenIds))
929            .rewrite()
930            .await?;
931
932        assert_eq!(result.removed_data_files.len(), 1);
933        assert_eq!(result.added_data_files.len(), 1);
934        assert_eq!(result.stats.input_rows, 4);
935        assert_eq!(result.stats.output_rows, 4);
936
937        let rows = read_ids_and_values(&fixture.table, &result.added_data_files).await?;
938        assert_eq!(rows, vec![(1, 10), (2, 30), (3, 30), (4, 50)]);
939
940        Ok(())
941    }
942
943    #[tokio::test]
944    async fn cow_rewrite_full_file_delete_removes_old_file_without_replacement() -> Result<()> {
945        let fixture = test_table_with_ids(vec![2, 4]).await?;
946
947        let result = CowRewriteBuilder::new(&fixture.table)
948            .with_predicate(crate::expr::Predicate::AlwaysTrue)
949            .with_rewriter(Arc::new(DeleteEvenIds))
950            .rewrite()
951            .await?;
952
953        assert!(result.has_changes());
954        assert_eq!(result.removed_data_files.len(), 1);
955        assert_eq!(result.added_data_files.len(), 0);
956        assert_eq!(result.stats.input_rows, 2);
957        assert_eq!(result.stats.output_rows, 0);
958
959        Ok(())
960    }
961
962    #[tokio::test]
963    async fn cow_rewrite_removes_file_fully_covered_by_position_deletes() -> Result<()> {
964        let mut fixture = TableTestFixture::new();
965        let positions = (0..300).collect::<Vec<i64>>();
966        fixture.setup_multi_row_group_manifest(&positions).await;
967
968        let result = CowRewriteBuilder::new(&fixture.table)
969            .with_predicate(crate::expr::Predicate::AlwaysTrue)
970            .with_rewriter(Arc::new(KeepAll))
971            .rewrite()
972            .await?;
973
974        assert_eq!(result.stats.candidate_files, 1);
975        assert_eq!(result.stats.rewritten_files, 1);
976        assert_eq!(result.stats.input_rows, 0);
977        assert_eq!(result.stats.output_rows, 0);
978        assert_eq!(result.removed_data_files.len(), 1);
979        assert!(result.added_data_files.is_empty());
980        assert!(result.unchanged_data_files.is_empty());
981
982        Ok(())
983    }
984
985    #[tokio::test]
986    async fn cow_rewrite_delete_no_matching_rows_keeps_old_file() -> Result<()> {
987        let fixture = test_table_with_ids(vec![1, 3]).await?;
988
989        let result = CowRewriteBuilder::new(&fixture.table)
990            .with_predicate(crate::expr::Predicate::AlwaysTrue)
991            .with_rewriter(Arc::new(DeleteEvenIds))
992            .rewrite()
993            .await?;
994
995        assert!(!result.has_changes());
996        assert_eq!(result.removed_data_files.len(), 0);
997        assert_eq!(result.added_data_files.len(), 0);
998        assert_eq!(result.unchanged_data_files.len(), 1);
999        assert_eq!(result.stats.input_rows, 2);
1000        assert_eq!(result.stats.output_rows, 0);
1001
1002        Ok(())
1003    }
1004
1005    /// Drops every batch while reporting `changed: false`, violating the
1006    /// documented contract. The orchestrator must derive the change from
1007    /// `output: None` itself, otherwise the file would be kept as "unchanged"
1008    /// while its rows were meant to be dropped.
1009    struct SilentFullDrop;
1010
1011    impl CowBatchRewriter for SilentFullDrop {
1012        fn rewrite_batch(&self, _batch: RecordBatch) -> Result<CowBatchRewrite> {
1013            Ok(CowBatchRewrite {
1014                output: None,
1015                changed: false,
1016            })
1017        }
1018    }
1019
1020    #[tokio::test]
1021    async fn cow_rewrite_none_output_implies_change() -> Result<()> {
1022        let fixture = test_table_with_ids(vec![1, 2]).await?;
1023
1024        let result = CowRewriteBuilder::new(&fixture.table)
1025            .with_predicate(crate::expr::Predicate::AlwaysTrue)
1026            .with_rewriter(Arc::new(SilentFullDrop))
1027            .rewrite()
1028            .await?;
1029
1030        assert!(result.has_changes());
1031        assert_eq!(result.removed_data_files.len(), 1);
1032        assert_eq!(result.added_data_files.len(), 0);
1033        assert!(result.unchanged_data_files.is_empty());
1034        assert_eq!(result.stats.rewritten_files, 1);
1035        assert_eq!(result.stats.input_rows, 2);
1036        assert_eq!(result.stats.output_rows, 0);
1037        assert_eq!(result.stats.changed_batches, 1);
1038
1039        Ok(())
1040    }
1041
1042    /// Returns `output: None` with `changed: false` for the first batch, then
1043    /// flags the second batch as changed. The replacement must contain exactly
1044    /// the second batch's rows — the silently dropped first batch must not
1045    /// resurrect, and the original file must be removed.
1046    struct DropFirstBatchSilently {
1047        batches_seen: std::sync::atomic::AtomicUsize,
1048    }
1049
1050    impl CowBatchRewriter for DropFirstBatchSilently {
1051        fn rewrite_batch(&self, batch: RecordBatch) -> Result<CowBatchRewrite> {
1052            let seen = self
1053                .batches_seen
1054                .fetch_add(1, std::sync::atomic::Ordering::SeqCst);
1055            if seen == 0 {
1056                Ok(CowBatchRewrite {
1057                    output: None,
1058                    changed: false,
1059                })
1060            } else {
1061                Ok(CowBatchRewrite {
1062                    output: Some(batch),
1063                    changed: true,
1064                })
1065            }
1066        }
1067    }
1068
1069    #[tokio::test]
1070    async fn cow_rewrite_silent_drop_before_change_loses_no_rows() -> Result<()> {
1071        let fixture = test_table_with_ids(vec![1, 2, 3, 4]).await?;
1072
1073        let result = CowRewriteBuilder::new(&fixture.table)
1074            .with_predicate(crate::expr::Predicate::AlwaysTrue)
1075            .with_batch_size(2)
1076            .with_rewriter(Arc::new(DropFirstBatchSilently {
1077                batches_seen: std::sync::atomic::AtomicUsize::new(0),
1078            }))
1079            .rewrite()
1080            .await?;
1081
1082        assert!(result.has_changes());
1083        assert_eq!(result.removed_data_files.len(), 1);
1084        assert_eq!(result.added_data_files.len(), 1);
1085        assert_eq!(result.stats.input_rows, 4);
1086        assert_eq!(result.stats.output_rows, 2);
1087
1088        let ids = read_ids(&fixture.table, &result.added_data_files).await?;
1089        assert_eq!(ids, vec![3, 4]);
1090
1091        Ok(())
1092    }
1093
1094    #[tokio::test]
1095    async fn cow_rewrite_kept_prefix_survives_later_full_batch_delete() -> Result<()> {
1096        let fixture = test_table_with_ids(vec![1, 3, 2, 4]).await?;
1097
1098        let result = CowRewriteBuilder::new(&fixture.table)
1099            .with_predicate(crate::expr::Predicate::AlwaysTrue)
1100            .with_batch_size(2)
1101            .with_rewriter(Arc::new(DeleteEvenIds))
1102            .rewrite()
1103            .await?;
1104
1105        assert_eq!(result.removed_data_files.len(), 1);
1106        assert_eq!(result.stats.input_rows, 4);
1107        assert_eq!(result.stats.output_rows, 2);
1108        assert_eq!(result.added_data_files.len(), 1);
1109        assert!(result.unchanged_data_files.is_empty());
1110        assert_eq!(
1111            read_ids(&fixture.table, &result.added_data_files).await?,
1112            vec![1, 3]
1113        );
1114
1115        Ok(())
1116    }
1117
1118    #[tokio::test]
1119    async fn cow_rewrite_uses_unique_replacement_paths_for_multiple_source_files() -> Result<()> {
1120        let fixture = test_table_with_id_batches(vec![vec![1, 2], vec![3, 4]]).await?;
1121
1122        let result = CowRewriteBuilder::new(&fixture.table)
1123            .with_predicate(crate::expr::Predicate::AlwaysTrue)
1124            .with_rewriter(Arc::new(DeleteEvenIds))
1125            .rewrite()
1126            .await?;
1127
1128        assert_eq!(result.stats.candidate_files, 2);
1129        assert_eq!(result.removed_data_files.len(), 2);
1130        assert_eq!(result.added_data_files.len(), 2);
1131
1132        let added_paths = result
1133            .added_data_files
1134            .iter()
1135            .map(|file| file.file_path().to_string())
1136            .collect::<HashSet<_>>();
1137        let removed_paths = result
1138            .removed_data_files
1139            .iter()
1140            .map(|file| file.file_path().to_string())
1141            .collect::<HashSet<_>>();
1142
1143        assert_eq!(added_paths.len(), result.added_data_files.len());
1144        assert!(added_paths.is_disjoint(&removed_paths));
1145
1146        let ids = read_ids(&fixture.table, &result.added_data_files).await?;
1147        assert_eq!(ids, vec![1, 3]);
1148
1149        Ok(())
1150    }
1151
1152    #[test]
1153    fn cow_batch_rewriter_is_object_safe() {
1154        let _rewriter: Arc<dyn CowBatchRewriter> = Arc::new(KeepAll);
1155    }
1156
1157    #[test]
1158    fn cow_rewrite_result_reports_no_changes() {
1159        let result = crate::cow_rewrite::CowRewriteResult {
1160            removed_data_files: vec![],
1161            added_data_files: vec![],
1162            unchanged_data_files: vec![],
1163            stats: crate::cow_rewrite::CowRewriteStats {
1164                candidate_files: 0,
1165                rewritten_files: 0,
1166                unchanged_files: 0,
1167                input_rows: 0,
1168                output_rows: 0,
1169                changed_batches: 0,
1170            },
1171        };
1172
1173        assert!(!result.has_changes());
1174        assert_eq!(result.stats.candidate_files, 0);
1175    }
1176
1177    /// `DeleteIfModThree` keeps rows whose `id` is not divisible by 3.
1178    struct DeleteIfModThree;
1179
1180    impl CowBatchRewriter for DeleteIfModThree {
1181        fn rewrite_batch(&self, batch: RecordBatch) -> Result<CowBatchRewrite> {
1182            let ids = batch
1183                .column_by_name("id")
1184                .ok_or_else(|| Error::new(ErrorKind::DataInvalid, "missing id column"))?
1185                .as_any()
1186                .downcast_ref::<Int32Array>()
1187                .ok_or_else(|| Error::new(ErrorKind::DataInvalid, "id must be Int32"))?;
1188
1189            let keep =
1190                BooleanArray::from_iter((0..ids.len()).map(|row| Some(ids.value(row) % 3 != 0)));
1191            let filtered = arrow_select::filter::filter_record_batch(&batch, &keep)
1192                .map_err(|err| Error::new(ErrorKind::Unexpected, err.to_string()))?;
1193
1194            Ok(CowBatchRewrite {
1195                changed: filtered.num_rows() != batch.num_rows(),
1196                output: (filtered.num_rows() > 0).then_some(filtered),
1197            })
1198        }
1199    }
1200
1201    /// After adding an optional column (`value`) to the schema, a COW rewrite
1202    /// of the pre-existing data must not fail. The replacement writer must use
1203    /// the schema the batches were read in (the snapshot schema), not the
1204    /// table's evolved current schema, otherwise the parquet writer rejects
1205    /// the input batch whose column set does not include `value`.
1206    #[tokio::test]
1207    async fn cow_rewrite_after_optional_column_add() -> Result<()> {
1208        let temp_dir = TempDir::new().unwrap();
1209        let warehouse = format!("file://{}", temp_dir.path().join("warehouse").display());
1210        let catalog: Arc<dyn Catalog> = Arc::new(
1211            MemoryCatalogBuilder::default()
1212                .with_storage_factory(Arc::new(LocalFsStorageFactory))
1213                .load(
1214                    "memory",
1215                    HashMap::from([(MEMORY_CATALOG_WAREHOUSE.to_string(), warehouse)]),
1216                )
1217                .await?,
1218        );
1219        let namespace = NamespaceIdent::new("ns".to_string());
1220        catalog.create_namespace(&namespace, HashMap::new()).await?;
1221
1222        let schema = Schema::builder()
1223            .with_fields(vec![
1224                NestedField::required(1, "id", Type::Primitive(PrimitiveType::Int)).into(),
1225            ])
1226            .build()?;
1227        let table = catalog
1228            .create_table(
1229                &namespace,
1230                TableCreation::builder()
1231                    .name("evolved".to_string())
1232                    .schema(schema)
1233                    .build(),
1234            )
1235            .await?;
1236
1237        // Write the original data file with the {id}-only schema.
1238        let arrow_schema = Arc::new(ArrowSchema::new(vec![
1239            Field::new("id", DataType::Int32, false).with_metadata(HashMap::from([(
1240                PARQUET_FIELD_ID_META_KEY.to_string(),
1241                "1".to_string(),
1242            )])),
1243        ]));
1244        let batch = RecordBatch::try_new(arrow_schema, vec![Arc::new(Int32Array::from(vec![
1245            1, 2, 3, 4,
1246        ])) as ArrayRef])?;
1247        let data_files = super::writer::write_replacement_batches(
1248            &table,
1249            table.metadata().current_schema().clone(),
1250            None,
1251            futures::stream::iter(vec![Ok(batch)]),
1252        )
1253        .await?;
1254        let tx = Transaction::new(&table);
1255        let tx = tx.fast_append().add_data_files(data_files).apply(tx)?;
1256        let table = tx.commit(&*catalog).await?;
1257
1258        // Evolve the schema: add an optional `value` column.
1259        let tx = Transaction::new(&table);
1260        let tx = tx
1261            .update_schema()
1262            .add_column(AddColumn::optional(
1263                "value",
1264                Type::Primitive(PrimitiveType::Int),
1265            ))
1266            .apply(tx)?;
1267        let table = tx.commit(&*catalog).await?;
1268
1269        // The current schema now has {id, value} but the current snapshot's
1270        // schema is still the original {id} schema.
1271        let current_snapshot = table.metadata().current_snapshot().unwrap();
1272        assert_ne!(
1273            table.metadata().current_schema_id(),
1274            current_snapshot.schema_id().unwrap()
1275        );
1276
1277        // Before the fix this fails when the parquet writer rejects the {id}-only
1278        // batch while configured with the {id, value} current schema.
1279        let result = CowRewriteBuilder::new(&table)
1280            .with_predicate(crate::expr::Predicate::AlwaysTrue)
1281            .with_rewriter(Arc::new(DeleteIfModThree))
1282            .rewrite()
1283            .await?;
1284
1285        // id 3 is divisible by 3 and is dropped; remaining ids are 1, 2, 4.
1286        assert!(result.has_changes());
1287        assert_eq!(result.removed_data_files.len(), 1);
1288        assert_eq!(result.added_data_files.len(), 1);
1289
1290        let ids = read_ids(&table, &result.added_data_files).await?;
1291        assert_eq!(ids, vec![1, 2, 4]);
1292
1293        Ok(())
1294    }
1295
1296    /// Builds a two-file table whose files have disjoint id ranges so predicate
1297    /// metrics pruning can exclude one file entirely.
1298    async fn two_file_table_disjoint_ids() -> Result<CowRewriteFixture> {
1299        let temp_dir = TempDir::new().unwrap();
1300        let warehouse = format!("file://{}", temp_dir.path().join("warehouse").display());
1301        let catalog = MemoryCatalogBuilder::default()
1302            .with_storage_factory(Arc::new(LocalFsStorageFactory))
1303            .load(
1304                "memory",
1305                HashMap::from([(MEMORY_CATALOG_WAREHOUSE.to_string(), warehouse)]),
1306            )
1307            .await?;
1308        let namespace = NamespaceIdent::new("ns".to_string());
1309        catalog.create_namespace(&namespace, HashMap::new()).await?;
1310
1311        let schema = Schema::builder()
1312            .with_fields(vec![
1313                NestedField::required(1, "id", Type::Primitive(PrimitiveType::Int)).into(),
1314            ])
1315            .build()?;
1316        let table = catalog
1317            .create_table(
1318                &namespace,
1319                TableCreation::builder()
1320                    .name("disjoint".to_string())
1321                    .schema(schema)
1322                    .properties(HashMap::from([(
1323                        TableProperties::PROPERTY_WRITE_TARGET_FILE_SIZE_BYTES.to_string(),
1324                        "1".to_string(),
1325                    )]))
1326                    .build(),
1327            )
1328            .await?;
1329
1330        let arrow_schema = Arc::new(ArrowSchema::new(vec![
1331            Field::new("id", DataType::Int32, false).with_metadata(HashMap::from([(
1332                PARQUET_FIELD_ID_META_KEY.to_string(),
1333                "1".to_string(),
1334            )])),
1335        ]));
1336        // First batch ids 1..=4, second batch ids 100..=103. Force a split via
1337        // tiny target file size so each batch lands in its own data file.
1338        let input = vec![vec![1, 2, 3, 4], vec![100, 101, 102, 103]]
1339            .into_iter()
1340            .map(|ids| {
1341                Ok(RecordBatch::try_new(arrow_schema.clone(), vec![
1342                    Arc::new(Int32Array::from(ids)) as ArrayRef,
1343                ])?)
1344            });
1345        let data_files = super::writer::write_replacement_batches(
1346            &table,
1347            table.metadata().current_schema().clone(),
1348            None,
1349            futures::stream::iter(input),
1350        )
1351        .await?;
1352
1353        let tx = Transaction::new(&table);
1354        let tx = tx.fast_append().add_data_files(data_files).apply(tx)?;
1355        let table = tx.commit(&catalog).await?;
1356
1357        Ok(CowRewriteFixture {
1358            _temp_dir: temp_dir,
1359            table,
1360        })
1361    }
1362
1363    /// Predicate-based planning must exclude files whose metrics cannot match
1364    /// the predicate. With `id > 50`, the file holding ids 1..=4 must never be
1365    /// planned as a candidate.
1366    #[tokio::test]
1367    async fn cow_rewrite_predicate_prunes_non_matching_files() -> Result<()> {
1368        let fixture = two_file_table_disjoint_ids().await?;
1369
1370        let predicate = crate::expr::Reference::new("id").greater_than(crate::spec::Datum::int(50));
1371
1372        let result = CowRewriteBuilder::new(&fixture.table)
1373            .with_predicate(predicate)
1374            .with_rewriter(Arc::new(KeepAll))
1375            .rewrite()
1376            .await?;
1377
1378        // Only the high-id file should be a candidate.
1379        assert_eq!(result.stats.candidate_files, 1);
1380        // KeepAll does not change anything, so the single candidate is unchanged.
1381        assert_eq!(result.removed_data_files.len(), 0);
1382        assert_eq!(result.added_data_files.len(), 0);
1383        assert_eq!(result.unchanged_data_files.len(), 1);
1384
1385        // The surviving candidate must be the high-id file (ids 100..=103).
1386        let ids = read_ids(&fixture.table, &result.unchanged_data_files).await?;
1387        assert_eq!(ids, vec![100, 101, 102, 103]);
1388
1389        Ok(())
1390    }
1391
1392    /// Rewrites every batch without altering rows, so every candidate file is
1393    /// replaced. Useful for exercising the replacement-writing path itself.
1394    struct TouchEveryBatch;
1395
1396    impl CowBatchRewriter for TouchEveryBatch {
1397        fn rewrite_batch(&self, batch: RecordBatch) -> Result<CowBatchRewrite> {
1398            Ok(CowBatchRewrite {
1399                output: Some(batch),
1400                changed: true,
1401            })
1402        }
1403    }
1404
1405    /// An end-to-end rewrite on an identity-partitioned table must preserve
1406    /// each source file's partition: the replacement file carries the same
1407    /// partition values and partition spec id, and lands under the same
1408    /// partition directory — including the null partition.
1409    #[tokio::test]
1410    async fn cow_rewrite_preserves_partitions_end_to_end() -> Result<()> {
1411        let temp_dir = TempDir::new().unwrap();
1412        let warehouse = format!("file://{}", temp_dir.path().join("warehouse").display());
1413        let catalog = MemoryCatalogBuilder::default()
1414            .with_storage_factory(Arc::new(LocalFsStorageFactory))
1415            .load(
1416                "memory",
1417                HashMap::from([(MEMORY_CATALOG_WAREHOUSE.to_string(), warehouse)]),
1418            )
1419            .await?;
1420        let namespace = NamespaceIdent::new("ns".to_string());
1421        catalog.create_namespace(&namespace, HashMap::new()).await?;
1422
1423        let schema = Schema::builder()
1424            .with_fields(vec![
1425                NestedField::required(1, "id", Type::Primitive(PrimitiveType::Int)).into(),
1426                NestedField::optional(2, "value", Type::Primitive(PrimitiveType::Int)).into(),
1427            ])
1428            .build()?;
1429        let partition_spec = crate::spec::PartitionSpec::builder(Arc::new(schema.clone()))
1430            .add_partition_field("value", "value", Transform::Identity)?
1431            .build()?;
1432        let table = catalog
1433            .create_table(
1434                &namespace,
1435                TableCreation::builder()
1436                    .name("partitioned".to_string())
1437                    .schema(schema)
1438                    .partition_spec(partition_spec.into_unbound())
1439                    .build(),
1440            )
1441            .await?;
1442
1443        // Two source files: ids 1, 2 in the `value=1` partition and ids 3, 4
1444        // in the null partition.
1445        let arrow_schema = Arc::new(ArrowSchema::new(vec![
1446            Field::new("id", DataType::Int32, false).with_metadata(HashMap::from([(
1447                PARQUET_FIELD_ID_META_KEY.to_string(),
1448                "1".to_string(),
1449            )])),
1450            Field::new("value", DataType::Int32, true).with_metadata(HashMap::from([(
1451                PARQUET_FIELD_ID_META_KEY.to_string(),
1452                "2".to_string(),
1453            )])),
1454        ]));
1455        let current_schema = table.metadata().current_schema().clone();
1456        let default_spec = table.metadata().default_partition_spec().as_ref().clone();
1457
1458        let batch_one = RecordBatch::try_new(arrow_schema.clone(), vec![
1459            Arc::new(Int32Array::from(vec![1, 2])) as ArrayRef,
1460            Arc::new(Int32Array::from(vec![1, 1])),
1461        ])?;
1462        let one_partition = crate::spec::PartitionKey::new(
1463            default_spec.clone(),
1464            current_schema.clone(),
1465            Struct::from_iter([Some(Literal::int(1))]),
1466        );
1467        let files_one = super::writer::write_replacement_batches(
1468            &table,
1469            current_schema.clone(),
1470            Some(one_partition),
1471            futures::stream::iter(vec![Ok(batch_one)]),
1472        )
1473        .await?;
1474
1475        let batch_null = RecordBatch::try_new(arrow_schema.clone(), vec![
1476            Arc::new(Int32Array::from(vec![3, 4])) as ArrayRef,
1477            Arc::new(Int32Array::from(vec![None::<i32>, None])),
1478        ])?;
1479        let null_partition = crate::spec::PartitionKey::new(
1480            default_spec.clone(),
1481            current_schema.clone(),
1482            Struct::from_iter([None::<Literal>]),
1483        );
1484        let files_null = super::writer::write_replacement_batches(
1485            &table,
1486            current_schema.clone(),
1487            Some(null_partition),
1488            futures::stream::iter(vec![Ok(batch_null)]),
1489        )
1490        .await?;
1491
1492        let tx = Transaction::new(&table);
1493        let tx = tx
1494            .fast_append()
1495            .add_data_files([files_one, files_null].concat())
1496            .apply(tx)?;
1497        let table = tx.commit(&catalog).await?;
1498
1499        let result = CowRewriteBuilder::new(&table)
1500            .with_rewriter(Arc::new(TouchEveryBatch))
1501            .rewrite()
1502            .await?;
1503
1504        assert_eq!(result.stats.candidate_files, 2);
1505        assert_eq!(result.stats.rewritten_files, 2);
1506        assert_eq!(result.removed_data_files.len(), 2);
1507        assert_eq!(result.added_data_files.len(), 2);
1508        assert!(result.unchanged_data_files.is_empty());
1509
1510        // Partition values survive the rewrite unchanged.
1511        let one_struct = Struct::from_iter([Some(Literal::int(1))]);
1512        let null_struct = Struct::from_iter([None::<Literal>]);
1513        let mut removed_partitions = result
1514            .removed_data_files
1515            .iter()
1516            .map(|file| file.partition().clone())
1517            .collect::<Vec<_>>();
1518        let mut added_partitions = result
1519            .added_data_files
1520            .iter()
1521            .map(|file| file.partition().clone())
1522            .collect::<Vec<_>>();
1523        removed_partitions.sort_by_key(|partition| partition.is_null_at_index(0));
1524        added_partitions.sort_by_key(|partition| partition.is_null_at_index(0));
1525        assert_eq!(removed_partitions, added_partitions);
1526        assert_eq!(added_partitions[0], one_struct);
1527        assert_eq!(added_partitions[1], null_struct);
1528
1529        for file in &result.added_data_files {
1530            assert_eq!(file.partition_spec_id, 0);
1531            if file.partition() == &one_struct {
1532                assert!(file.file_path().contains("value=1"), "{}", file.file_path());
1533            } else {
1534                assert!(
1535                    file.file_path().contains("value=null"),
1536                    "{}",
1537                    file.file_path()
1538                );
1539            }
1540        }
1541
1542        // Replacement content matches the source rows.
1543        let ids = read_ids(&table, &result.added_data_files).await?;
1544        assert_eq!(ids, vec![1, 2, 3, 4]);
1545
1546        Ok(())
1547    }
1548
1549    /// Mutates the identity-partitioned `value` column while flagging the
1550    /// batch as changed — a violation of the partition-preservation contract.
1551    /// In debug builds the partition guard must trip instead of letting the
1552    /// replacement file record a partition that disagrees with its rows.
1553    #[cfg(debug_assertions)]
1554    struct MutatePartitionColumn;
1555
1556    #[cfg(debug_assertions)]
1557    impl CowBatchRewriter for MutatePartitionColumn {
1558        fn rewrite_batch(&self, batch: RecordBatch) -> Result<CowBatchRewrite> {
1559            let values = batch
1560                .column_by_name("value")
1561                .ok_or_else(|| Error::new(ErrorKind::DataInvalid, "missing value column"))?
1562                .as_any()
1563                .downcast_ref::<Int32Array>()
1564                .ok_or_else(|| Error::new(ErrorKind::DataInvalid, "value must be Int32"))?;
1565            let updated =
1566                Int32Array::from_iter((0..values.len()).map(|row| Some(values.value(row) + 100)));
1567            let output = RecordBatch::try_new(batch.schema(), vec![
1568                batch.column(0).clone(),
1569                Arc::new(updated),
1570            ])
1571            .map_err(|err| Error::new(ErrorKind::Unexpected, err.to_string()))?;
1572
1573            Ok(CowBatchRewrite {
1574                output: Some(output),
1575                changed: true,
1576            })
1577        }
1578    }
1579
1580    #[cfg(debug_assertions)]
1581    #[tokio::test]
1582    #[should_panic(expected = "must preserve the source file's partition values")]
1583    async fn cow_rewrite_partition_column_mutation_trips_debug_guard() {
1584        let temp_dir = TempDir::new().unwrap();
1585        let warehouse = format!("file://{}", temp_dir.path().join("warehouse").display());
1586        let catalog = MemoryCatalogBuilder::default()
1587            .with_storage_factory(Arc::new(LocalFsStorageFactory))
1588            .load(
1589                "memory",
1590                HashMap::from([(MEMORY_CATALOG_WAREHOUSE.to_string(), warehouse)]),
1591            )
1592            .await
1593            .unwrap();
1594        let namespace = NamespaceIdent::new("ns".to_string());
1595        catalog
1596            .create_namespace(&namespace, HashMap::new())
1597            .await
1598            .unwrap();
1599
1600        let schema = Schema::builder()
1601            .with_fields(vec![
1602                NestedField::required(1, "id", Type::Primitive(PrimitiveType::Int)).into(),
1603                NestedField::required(2, "value", Type::Primitive(PrimitiveType::Int)).into(),
1604            ])
1605            .build()
1606            .unwrap();
1607        let partition_spec = crate::spec::PartitionSpec::builder(Arc::new(schema.clone()))
1608            .add_partition_field("value", "value", Transform::Identity)
1609            .unwrap()
1610            .build()
1611            .unwrap();
1612        let table = catalog
1613            .create_table(
1614                &namespace,
1615                TableCreation::builder()
1616                    .name("partition_guard".to_string())
1617                    .schema(schema)
1618                    .partition_spec(partition_spec.into_unbound())
1619                    .build(),
1620            )
1621            .await
1622            .unwrap();
1623
1624        // One source file whose rows all sit in the `value=1` partition.
1625        let arrow_schema = Arc::new(ArrowSchema::new(vec![
1626            Field::new("id", DataType::Int32, false).with_metadata(HashMap::from([(
1627                PARQUET_FIELD_ID_META_KEY.to_string(),
1628                "1".to_string(),
1629            )])),
1630            Field::new("value", DataType::Int32, false).with_metadata(HashMap::from([(
1631                PARQUET_FIELD_ID_META_KEY.to_string(),
1632                "2".to_string(),
1633            )])),
1634        ]));
1635        let batch = RecordBatch::try_new(arrow_schema, vec![
1636            Arc::new(Int32Array::from(vec![1, 2])) as ArrayRef,
1637            Arc::new(Int32Array::from(vec![1, 1])),
1638        ])
1639        .unwrap();
1640        let partition_key = crate::spec::PartitionKey::new(
1641            table.metadata().default_partition_spec().as_ref().clone(),
1642            table.metadata().current_schema().clone(),
1643            Struct::from_iter([Some(Literal::int(1))]),
1644        );
1645        let data_files = super::writer::write_replacement_batches(
1646            &table,
1647            table.metadata().current_schema().clone(),
1648            Some(partition_key),
1649            futures::stream::iter(vec![Ok(batch)]),
1650        )
1651        .await
1652        .unwrap();
1653
1654        let tx = Transaction::new(&table);
1655        let tx = tx
1656            .fast_append()
1657            .add_data_files(data_files)
1658            .apply(tx)
1659            .unwrap();
1660        let table = tx.commit(&catalog).await.unwrap();
1661
1662        // The rewriter bumps every `value` by 100, so the output batch
1663        // computes to a different partition than the source file records.
1664        // The debug guard must panic here; without it the rewrite would
1665        // silently commit a replacement file under `value=1` holding rows
1666        // with `value=101`.
1667        let _ = CowRewriteBuilder::new(&table)
1668            .with_rewriter(Arc::new(MutatePartitionColumn))
1669            .rewrite()
1670            .await;
1671    }
1672}