1mod plan;
73mod rewriter;
74pub(crate) mod writer;
75
76use std::sync::Arc;
77
78use arrow_array::RecordBatch;
79use futures::TryStreamExt;
80pub use plan::CowRewriteFile;
81pub use rewriter::{CowBatchRewrite, CowBatchRewriter};
82
83use crate::arrow::ArrowReaderOptions;
84use crate::expr::Predicate;
85use crate::scan::FileScanTaskStream;
86use crate::spec::{DataFile, PartitionKey};
87use crate::table::Table;
88use crate::{Error, ErrorKind, Result};
89
90#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)]
92pub struct CowRewriteStats {
93 pub candidate_files: usize,
95 pub rewritten_files: usize,
97 pub unchanged_files: usize,
99 pub input_rows: u64,
101 pub output_rows: u64,
106 pub changed_batches: u64,
109}
110
111#[derive(Debug, Default)]
113pub struct CowRewriteResult {
114 pub removed_data_files: Vec<DataFile>,
116 pub added_data_files: Vec<DataFile>,
118 pub unchanged_data_files: Vec<DataFile>,
127 pub stats: CowRewriteStats,
129}
130
131impl CowRewriteResult {
132 pub fn has_changes(&self) -> bool {
134 !self.removed_data_files.is_empty() || !self.added_data_files.is_empty()
135 }
136}
137
138pub struct CowRewriteBuilder<'a> {
140 table: &'a Table,
141 predicate: Predicate,
142 snapshot_id: Option<i64>,
143 batch_size: Option<usize>,
144 case_sensitive: bool,
145 rewriter: Option<Arc<dyn CowBatchRewriter>>,
146}
147
148impl<'a> CowRewriteBuilder<'a> {
149 pub fn new(table: &'a Table) -> Self {
151 Self {
152 table,
153 predicate: Predicate::AlwaysTrue,
154 snapshot_id: None,
155 batch_size: None,
156 case_sensitive: true,
157 rewriter: None,
158 }
159 }
160
161 pub fn with_predicate(mut self, predicate: Predicate) -> Self {
163 self.predicate = predicate;
164 self
165 }
166
167 pub fn with_snapshot_id(mut self, snapshot_id: i64) -> Self {
169 self.snapshot_id = Some(snapshot_id);
170 self
171 }
172
173 pub fn with_batch_size(mut self, batch_size: usize) -> Self {
175 self.batch_size = Some(batch_size);
176 self
177 }
178
179 pub fn with_case_sensitive(mut self, case_sensitive: bool) -> Self {
181 self.case_sensitive = case_sensitive;
182 self
183 }
184
185 pub fn with_rewriter(mut self, rewriter: Arc<dyn CowBatchRewriter>) -> Self {
187 self.rewriter = Some(rewriter);
188 self
189 }
190
191 pub async fn rewrite(self) -> Result<CowRewriteResult> {
205 let rewriter = self.rewriter.ok_or_else(|| {
206 Error::new(
207 ErrorKind::PreconditionFailed,
208 "COW rewrite requires a batch rewriter",
209 )
210 })?;
211 let files = plan::plan_cow_rewrite_files(
212 self.table,
213 Some(self.predicate),
214 self.snapshot_id,
215 self.case_sensitive,
216 )
217 .await?;
218
219 let mut result = CowRewriteResult {
220 stats: CowRewriteStats {
221 candidate_files: files.len(),
222 ..CowRewriteStats::default()
223 },
224 ..CowRewriteResult::default()
225 };
226
227 for file in files {
228 let CowRewriteFile {
229 old_data_file,
230 scan_task,
231 } = file;
232 let write_schema = scan_task.schema_ref();
239 let has_delete_files = !scan_task.deletes().is_empty();
240
241 let mut prefix: Vec<RecordBatch> = Vec::new();
256 let mut file_changed = false;
257 let mut file_input_rows = 0_u64;
258 let mut file_output_rows = 0_u64;
259 let mut writer: Option<Box<dyn crate::writer::IcebergWriter>> = None;
260 #[cfg(debug_assertions)]
263 let partition_guard =
264 DebugPartitionGuard::new(self.table, &old_data_file, &write_schema)?;
265
266 let tasks = Box::pin(futures::stream::iter(vec![Ok(scan_task)])) as FileScanTaskStream;
270
271 let mut reader_builder = self.table.reader_builder();
276 if let Some(batch_size) = self.batch_size {
277 reader_builder = reader_builder.with_options(
278 ArrowReaderOptions::builder()
279 .with_batch_size(batch_size)
280 .build(),
281 );
282 }
283
284 let mut batches = reader_builder.build().read(tasks)?.stream();
285 while let Some(batch) = batches.try_next().await? {
286 result.stats.input_rows += batch.num_rows() as u64;
287 file_input_rows += batch.num_rows() as u64;
288
289 let rewrite = rewriter.rewrite_batch(batch)?;
290 let changed = rewrite.changed || rewrite.output.is_none();
297 if changed {
298 file_changed = true;
299 result.stats.changed_batches += 1;
300 }
301
302 #[cfg(debug_assertions)]
306 if let (Some(guard), Some(output)) = (&partition_guard, &rewrite.output) {
307 guard.check(output)?;
308 }
309
310 if file_changed
314 && writer.is_none()
315 && (!prefix.is_empty() || rewrite.output.is_some())
316 {
317 let partition_key =
318 source_partition_key(self.table, &old_data_file, &write_schema)?;
319 writer = Some(
320 writer::build_replacement_writer(
321 self.table,
322 write_schema.clone(),
323 Some(partition_key),
324 )
325 .await?,
326 );
327 }
328
329 if let Some(writer) = writer.as_mut() {
330 for prefix_batch in prefix.drain(..) {
331 writer.write(prefix_batch).await?;
332 }
333 if let Some(output) = rewrite.output {
334 file_output_rows += output.num_rows() as u64;
335 writer.write(output).await?;
336 }
337 } else if let Some(output) = rewrite.output {
338 file_output_rows += output.num_rows() as u64;
339 prefix.push(output);
340 }
341 }
342
343 let fully_removed_by_deletes =
349 !file_changed && file_input_rows == 0 && has_delete_files;
350
351 if file_changed || fully_removed_by_deletes {
352 result.stats.rewritten_files += 1;
353 result.stats.output_rows += file_output_rows;
354 result.removed_data_files.push(old_data_file);
355
356 if let Some(mut writer) = writer {
357 let added_data_files = writer.close().await?;
358 result.added_data_files.extend(added_data_files);
359 }
360 } else {
363 result.stats.unchanged_files += 1;
364 result.unchanged_data_files.push(old_data_file);
365 }
367 }
368
369 Ok(result)
370 }
371}
372
373fn source_partition_key(
374 table: &Table,
375 data_file: &DataFile,
376 schema: &crate::spec::SchemaRef,
377) -> Result<PartitionKey> {
378 let spec = table
379 .metadata()
380 .partition_spec_by_id(data_file.partition_spec_id)
381 .ok_or_else(|| {
382 Error::new(
383 ErrorKind::DataInvalid,
384 format!(
385 "Missing partition spec {} for COW rewrite source file",
386 data_file.partition_spec_id
387 ),
388 )
389 })?
390 .as_ref()
391 .clone();
392 spec.partition_type(schema).map_err(|err| {
397 Error::new(
398 ErrorKind::DataInvalid,
399 format!(
400 "Cannot bind partition spec {} to the planned snapshot schema for COW rewrite",
401 data_file.partition_spec_id
402 ),
403 )
404 .with_source(err)
405 })?;
406
407 Ok(PartitionKey::new(
408 spec,
409 schema.clone(),
410 data_file.partition().clone(),
411 ))
412}
413
414#[cfg(debug_assertions)]
421struct DebugPartitionGuard {
422 calculator: crate::arrow::PartitionValueCalculator,
423 partition_type: crate::spec::StructType,
424 expected: crate::spec::Struct,
425}
426
427#[cfg(debug_assertions)]
428impl DebugPartitionGuard {
429 fn new(
433 table: &Table,
434 data_file: &DataFile,
435 schema: &crate::spec::SchemaRef,
436 ) -> Result<Option<Self>> {
437 let spec = table
438 .metadata()
439 .partition_spec_by_id(data_file.partition_spec_id)
440 .ok_or_else(|| {
441 Error::new(
442 ErrorKind::DataInvalid,
443 format!(
444 "Missing partition spec {} for COW rewrite source file",
445 data_file.partition_spec_id
446 ),
447 )
448 })?;
449 if spec.is_unpartitioned() {
450 return Ok(None);
451 }
452
453 Ok(Some(Self {
454 calculator: crate::arrow::PartitionValueCalculator::try_new(spec, schema)?,
455 partition_type: spec.partition_type(schema)?,
456 expected: data_file.partition().clone(),
457 }))
458 }
459
460 fn check(&self, output: &RecordBatch) -> Result<()> {
463 let partition_array = self.calculator.calculate(output)?;
464 let values = crate::arrow::arrow_struct_to_literal(&partition_array, &self.partition_type)?;
465 for value in values {
466 debug_assert_eq!(
467 value,
468 Some(crate::spec::Literal::Struct(self.expected.clone())),
469 "COW rewriter must preserve the source file's partition values: \
470 replacement files record the source partition verbatim, so \
471 mutating a partition-source column writes rows under a \
472 partition they do not belong to"
473 );
474 }
475 Ok(())
476 }
477}
478
479#[cfg(test)]
480mod tests {
481 use std::collections::{HashMap, HashSet};
482 use std::sync::Arc;
483
484 use arrow_array::{Array, ArrayRef, BooleanArray, Int32Array, RecordBatch};
485 use arrow_schema::{DataType, Field, Schema as ArrowSchema};
486 use futures::TryStreamExt;
487 use parquet::arrow::PARQUET_FIELD_ID_META_KEY;
488 use tempfile::TempDir;
489
490 use crate::cow_rewrite::{CowBatchRewrite, CowBatchRewriter, CowRewriteBuilder};
491 use crate::io::LocalFsStorageFactory;
492 use crate::memory::{MEMORY_CATALOG_WAREHOUSE, MemoryCatalogBuilder};
493 use crate::scan::{FileScanTask, FileScanTaskStream};
494 use crate::spec::{
495 DataFile, Literal, NestedField, PrimitiveType, Schema, Struct, TableProperties, Transform,
496 Type,
497 };
498 use crate::table::Table;
499 use crate::test_utils::scan::TableTestFixture;
500 use crate::transaction::{AddColumn, ApplyTransactionAction, Transaction};
501 use crate::{Catalog, CatalogBuilder, Error, ErrorKind, NamespaceIdent, Result, TableCreation};
502
503 struct KeepAll;
504
505 impl CowBatchRewriter for KeepAll {
506 fn rewrite_batch(&self, batch: RecordBatch) -> Result<CowBatchRewrite> {
507 Ok(CowBatchRewrite {
508 output: Some(batch),
509 changed: false,
510 })
511 }
512 }
513
514 struct DeleteEvenIds;
515
516 impl CowBatchRewriter for DeleteEvenIds {
517 fn rewrite_batch(&self, batch: RecordBatch) -> Result<CowBatchRewrite> {
518 let ids = batch
519 .column_by_name("id")
520 .ok_or_else(|| Error::new(ErrorKind::DataInvalid, "missing id column"))?
521 .as_any()
522 .downcast_ref::<Int32Array>()
523 .ok_or_else(|| Error::new(ErrorKind::DataInvalid, "id must be Int32"))?;
524
525 let keep =
526 BooleanArray::from_iter((0..ids.len()).map(|row| Some(ids.value(row) % 2 != 0)));
527 let filtered = arrow_select::filter::filter_record_batch(&batch, &keep)
528 .map_err(|err| Error::new(ErrorKind::Unexpected, err.to_string()))?;
529
530 Ok(CowBatchRewrite {
531 changed: filtered.num_rows() != batch.num_rows(),
532 output: (filtered.num_rows() > 0).then_some(filtered),
533 })
534 }
535 }
536
537 struct IncrementValueForEvenIds;
538
539 impl CowBatchRewriter for IncrementValueForEvenIds {
540 fn rewrite_batch(&self, batch: RecordBatch) -> Result<CowBatchRewrite> {
541 let ids = batch
542 .column_by_name("id")
543 .ok_or_else(|| Error::new(ErrorKind::DataInvalid, "missing id column"))?
544 .as_any()
545 .downcast_ref::<Int32Array>()
546 .ok_or_else(|| Error::new(ErrorKind::DataInvalid, "id must be Int32"))?;
547 let values = batch
548 .column_by_name("value")
549 .ok_or_else(|| Error::new(ErrorKind::DataInvalid, "missing value column"))?
550 .as_any()
551 .downcast_ref::<Int32Array>()
552 .ok_or_else(|| Error::new(ErrorKind::DataInvalid, "value must be Int32"))?;
553
554 let mut changed = false;
555 let updated_values = Int32Array::from_iter((0..values.len()).map(|row| {
556 let value = values.value(row);
557 if ids.value(row) % 2 == 0 {
558 changed = true;
559 Some(value + 10)
560 } else {
561 Some(value)
562 }
563 }));
564 let output = RecordBatch::try_new(batch.schema(), vec![
565 batch.column(0).clone(),
566 Arc::new(updated_values),
567 ])
568 .map_err(|err| Error::new(ErrorKind::Unexpected, err.to_string()))?;
569
570 Ok(CowBatchRewrite {
571 output: Some(output),
572 changed,
573 })
574 }
575 }
576
577 struct CowRewriteFixture {
578 _temp_dir: TempDir,
579 table: Table,
580 }
581
582 async fn test_table_with_ids(ids: Vec<i32>) -> Result<CowRewriteFixture> {
583 let temp_dir = TempDir::new().unwrap();
584 let warehouse = format!("file://{}", temp_dir.path().join("warehouse").display());
585 let catalog = MemoryCatalogBuilder::default()
586 .with_storage_factory(Arc::new(LocalFsStorageFactory))
587 .load(
588 "memory",
589 HashMap::from([(MEMORY_CATALOG_WAREHOUSE.to_string(), warehouse)]),
590 )
591 .await?;
592 let namespace = NamespaceIdent::new("ns".to_string());
593 catalog.create_namespace(&namespace, HashMap::new()).await?;
594
595 let schema = Schema::builder()
596 .with_fields(vec![
597 NestedField::required(1, "id", Type::Primitive(PrimitiveType::Int)).into(),
598 ])
599 .build()?;
600 let table = catalog
601 .create_table(
602 &namespace,
603 TableCreation::builder()
604 .name("cow_rewrite_fixture".to_string())
605 .schema(schema)
606 .build(),
607 )
608 .await?;
609
610 let arrow_schema = Arc::new(ArrowSchema::new(vec![
611 Field::new("id", DataType::Int32, false).with_metadata(HashMap::from([(
612 PARQUET_FIELD_ID_META_KEY.to_string(),
613 "1".to_string(),
614 )])),
615 ]));
616 let batch = RecordBatch::try_new(arrow_schema, vec![
617 Arc::new(Int32Array::from(ids)) as ArrayRef
618 ])?;
619 let data_files = super::writer::write_replacement_batches(
620 &table,
621 table.metadata().current_schema().clone(),
622 None,
623 futures::stream::iter(vec![Ok(batch)]),
624 )
625 .await?;
626
627 let tx = Transaction::new(&table);
628 let tx = tx.fast_append().add_data_files(data_files).apply(tx)?;
629 let table = tx.commit(&catalog).await?;
630
631 Ok(CowRewriteFixture {
632 _temp_dir: temp_dir,
633 table,
634 })
635 }
636
637 async fn test_table_with_id_batches(batches: Vec<Vec<i32>>) -> Result<CowRewriteFixture> {
638 let temp_dir = TempDir::new().unwrap();
639 let warehouse = format!("file://{}", temp_dir.path().join("warehouse").display());
640 let catalog = MemoryCatalogBuilder::default()
641 .with_storage_factory(Arc::new(LocalFsStorageFactory))
642 .load(
643 "memory",
644 HashMap::from([(MEMORY_CATALOG_WAREHOUSE.to_string(), warehouse)]),
645 )
646 .await?;
647 let namespace = NamespaceIdent::new("ns".to_string());
648 catalog.create_namespace(&namespace, HashMap::new()).await?;
649
650 let schema = Schema::builder()
651 .with_fields(vec![
652 NestedField::required(1, "id", Type::Primitive(PrimitiveType::Int)).into(),
653 ])
654 .build()?;
655 let table = catalog
656 .create_table(
657 &namespace,
658 TableCreation::builder()
659 .name("cow_rewrite_fixture".to_string())
660 .schema(schema)
661 .properties(HashMap::from([(
662 TableProperties::PROPERTY_WRITE_TARGET_FILE_SIZE_BYTES.to_string(),
663 "1".to_string(),
664 )]))
665 .build(),
666 )
667 .await?;
668
669 let arrow_schema = Arc::new(ArrowSchema::new(vec![
670 Field::new("id", DataType::Int32, false).with_metadata(HashMap::from([(
671 PARQUET_FIELD_ID_META_KEY.to_string(),
672 "1".to_string(),
673 )])),
674 ]));
675 let input = batches.into_iter().map(|ids| {
676 Ok(RecordBatch::try_new(arrow_schema.clone(), vec![
677 Arc::new(Int32Array::from(ids)) as ArrayRef,
678 ])?)
679 });
680 let data_files = super::writer::write_replacement_batches(
681 &table,
682 table.metadata().current_schema().clone(),
683 None,
684 futures::stream::iter(input),
685 )
686 .await?;
687
688 let tx = Transaction::new(&table);
689 let tx = tx.fast_append().add_data_files(data_files).apply(tx)?;
690 let table = tx.commit(&catalog).await?;
691
692 Ok(CowRewriteFixture {
693 _temp_dir: temp_dir,
694 table,
695 })
696 }
697
698 async fn test_table_with_id_value_rows(rows: Vec<(i32, i32)>) -> Result<CowRewriteFixture> {
699 let temp_dir = TempDir::new().unwrap();
700 let warehouse = format!("file://{}", temp_dir.path().join("warehouse").display());
701 let catalog = MemoryCatalogBuilder::default()
702 .with_storage_factory(Arc::new(LocalFsStorageFactory))
703 .load(
704 "memory",
705 HashMap::from([(MEMORY_CATALOG_WAREHOUSE.to_string(), warehouse)]),
706 )
707 .await?;
708 let namespace = NamespaceIdent::new("ns".to_string());
709 catalog.create_namespace(&namespace, HashMap::new()).await?;
710
711 let schema = Schema::builder()
712 .with_fields(vec![
713 NestedField::required(1, "id", Type::Primitive(PrimitiveType::Int)).into(),
714 NestedField::required(2, "value", Type::Primitive(PrimitiveType::Int)).into(),
715 ])
716 .build()?;
717 let table = catalog
718 .create_table(
719 &namespace,
720 TableCreation::builder()
721 .name("cow_rewrite_fixture".to_string())
722 .schema(schema)
723 .build(),
724 )
725 .await?;
726
727 let arrow_schema = Arc::new(ArrowSchema::new(vec![
728 Field::new("id", DataType::Int32, false).with_metadata(HashMap::from([(
729 PARQUET_FIELD_ID_META_KEY.to_string(),
730 "1".to_string(),
731 )])),
732 Field::new("value", DataType::Int32, false).with_metadata(HashMap::from([(
733 PARQUET_FIELD_ID_META_KEY.to_string(),
734 "2".to_string(),
735 )])),
736 ]));
737 let ids = rows.iter().map(|(id, _)| *id).collect::<Vec<_>>();
738 let values = rows.iter().map(|(_, value)| *value).collect::<Vec<_>>();
739 let batch = RecordBatch::try_new(arrow_schema, vec![
740 Arc::new(Int32Array::from(ids)) as ArrayRef,
741 Arc::new(Int32Array::from(values)) as ArrayRef,
742 ])?;
743 let data_files = super::writer::write_replacement_batches(
744 &table,
745 table.metadata().current_schema().clone(),
746 None,
747 futures::stream::iter(vec![Ok(batch)]),
748 )
749 .await?;
750
751 let tx = Transaction::new(&table);
752 let tx = tx.fast_append().add_data_files(data_files).apply(tx)?;
753 let table = tx.commit(&catalog).await?;
754
755 Ok(CowRewriteFixture {
756 _temp_dir: temp_dir,
757 table,
758 })
759 }
760
761 async fn read_ids(table: &Table, files: &[DataFile]) -> Result<Vec<i32>> {
762 let schema = table.metadata().current_schema().clone();
763 let project_field_ids = schema
764 .as_struct()
765 .fields()
766 .iter()
767 .map(|field| field.id)
768 .collect::<Vec<_>>();
769 let tasks = files
770 .iter()
771 .map(|data_file| {
772 FileScanTask::builder()
773 .with_file_size_in_bytes(data_file.file_size_in_bytes())
774 .with_start(0)
775 .with_length(data_file.file_size_in_bytes())
776 .with_record_count(Some(data_file.record_count()))
777 .with_data_file_path(data_file.file_path().to_string())
778 .with_data_file_format(data_file.file_format())
779 .with_schema(schema.clone())
780 .with_project_field_ids(project_field_ids.clone())
781 .with_case_sensitive(true)
782 .build()
783 })
784 .collect::<Vec<_>>();
785 let task_stream = Box::pin(futures::stream::iter(tasks)) as FileScanTaskStream;
786 let batches = table
787 .reader_builder()
788 .build()
789 .read(task_stream)?
790 .stream()
791 .try_collect::<Vec<_>>()
792 .await?;
793
794 let mut ids = Vec::new();
795 for batch in batches {
796 let column = batch
797 .column_by_name("id")
798 .ok_or_else(|| Error::new(ErrorKind::DataInvalid, "missing id column"))?
799 .as_any()
800 .downcast_ref::<Int32Array>()
801 .ok_or_else(|| Error::new(ErrorKind::DataInvalid, "id must be Int32"))?;
802 ids.extend((0..column.len()).map(|row| column.value(row)));
803 }
804 ids.sort_unstable();
805 Ok(ids)
806 }
807
808 async fn read_ids_and_values(table: &Table, files: &[DataFile]) -> Result<Vec<(i32, i32)>> {
809 let schema = table.metadata().current_schema().clone();
810 let project_field_ids = schema
811 .as_struct()
812 .fields()
813 .iter()
814 .map(|field| field.id)
815 .collect::<Vec<_>>();
816 let tasks = files
817 .iter()
818 .map(|data_file| {
819 FileScanTask::builder()
820 .with_file_size_in_bytes(data_file.file_size_in_bytes())
821 .with_start(0)
822 .with_length(data_file.file_size_in_bytes())
823 .with_record_count(Some(data_file.record_count()))
824 .with_data_file_path(data_file.file_path().to_string())
825 .with_data_file_format(data_file.file_format())
826 .with_schema(schema.clone())
827 .with_project_field_ids(project_field_ids.clone())
828 .with_case_sensitive(true)
829 .build()
830 })
831 .collect::<Vec<_>>();
832 let task_stream = Box::pin(futures::stream::iter(tasks)) as FileScanTaskStream;
833 let batches = table
834 .reader_builder()
835 .build()
836 .read(task_stream)?
837 .stream()
838 .try_collect::<Vec<_>>()
839 .await?;
840
841 let mut rows = Vec::new();
842 for batch in batches {
843 let ids = batch
844 .column_by_name("id")
845 .ok_or_else(|| Error::new(ErrorKind::DataInvalid, "missing id column"))?
846 .as_any()
847 .downcast_ref::<Int32Array>()
848 .ok_or_else(|| Error::new(ErrorKind::DataInvalid, "id must be Int32"))?;
849 let values = batch
850 .column_by_name("value")
851 .ok_or_else(|| Error::new(ErrorKind::DataInvalid, "missing value column"))?
852 .as_any()
853 .downcast_ref::<Int32Array>()
854 .ok_or_else(|| Error::new(ErrorKind::DataInvalid, "value must be Int32"))?;
855 rows.extend((0..ids.len()).map(|row| (ids.value(row), values.value(row))));
856 }
857 rows.sort_unstable_by_key(|(id, _)| *id);
858 Ok(rows)
859 }
860
861 #[tokio::test]
862 async fn cow_rewrite_keep_all_produces_no_changes() -> Result<()> {
863 let fixture = test_table_with_ids(vec![1, 2, 3]).await?;
864
865 let result = CowRewriteBuilder::new(&fixture.table)
866 .with_predicate(crate::expr::Predicate::AlwaysTrue)
867 .with_rewriter(Arc::new(KeepAll))
868 .rewrite()
869 .await?;
870
871 assert!(!result.has_changes());
872 assert_eq!(result.removed_data_files.len(), 0);
873 assert_eq!(result.added_data_files.len(), 0);
874 assert_eq!(result.unchanged_data_files.len(), 1);
875 assert_eq!(result.stats.candidate_files, 1);
876 assert_eq!(result.stats.unchanged_files, 1);
877 assert_eq!(result.stats.input_rows, 3);
878 assert_eq!(result.stats.output_rows, 0);
881
882 Ok(())
883 }
884
885 #[tokio::test]
886 async fn cow_rewrite_requires_rewriter() -> Result<()> {
887 let fixture = test_table_with_ids(vec![1]).await?;
888
889 let err = CowRewriteBuilder::new(&fixture.table)
890 .rewrite()
891 .await
892 .expect_err("missing rewriter should fail");
893
894 assert_eq!(err.kind(), ErrorKind::PreconditionFailed);
895
896 Ok(())
897 }
898
899 #[tokio::test]
900 async fn cow_rewrite_delete_rows_removes_old_file_and_adds_replacement() -> Result<()> {
901 let fixture = test_table_with_ids(vec![1, 2, 3, 4]).await?;
902
903 let result = CowRewriteBuilder::new(&fixture.table)
904 .with_predicate(crate::expr::Predicate::AlwaysTrue)
905 .with_rewriter(Arc::new(DeleteEvenIds))
906 .rewrite()
907 .await?;
908
909 assert!(result.has_changes());
910 assert_eq!(result.removed_data_files.len(), 1);
911 assert_eq!(result.added_data_files.len(), 1);
912 assert_eq!(result.stats.input_rows, 4);
913 assert_eq!(result.stats.output_rows, 2);
914
915 let ids = read_ids(&fixture.table, &result.added_data_files).await?;
916 assert_eq!(ids, vec![1, 3]);
917
918 Ok(())
919 }
920
921 #[tokio::test]
922 async fn cow_rewrite_update_rows_rewrites_file_with_updated_values() -> Result<()> {
923 let fixture =
924 test_table_with_id_value_rows(vec![(1, 10), (2, 20), (3, 30), (4, 40)]).await?;
925
926 let result = CowRewriteBuilder::new(&fixture.table)
927 .with_predicate(crate::expr::Predicate::AlwaysTrue)
928 .with_rewriter(Arc::new(IncrementValueForEvenIds))
929 .rewrite()
930 .await?;
931
932 assert_eq!(result.removed_data_files.len(), 1);
933 assert_eq!(result.added_data_files.len(), 1);
934 assert_eq!(result.stats.input_rows, 4);
935 assert_eq!(result.stats.output_rows, 4);
936
937 let rows = read_ids_and_values(&fixture.table, &result.added_data_files).await?;
938 assert_eq!(rows, vec![(1, 10), (2, 30), (3, 30), (4, 50)]);
939
940 Ok(())
941 }
942
943 #[tokio::test]
944 async fn cow_rewrite_full_file_delete_removes_old_file_without_replacement() -> Result<()> {
945 let fixture = test_table_with_ids(vec![2, 4]).await?;
946
947 let result = CowRewriteBuilder::new(&fixture.table)
948 .with_predicate(crate::expr::Predicate::AlwaysTrue)
949 .with_rewriter(Arc::new(DeleteEvenIds))
950 .rewrite()
951 .await?;
952
953 assert!(result.has_changes());
954 assert_eq!(result.removed_data_files.len(), 1);
955 assert_eq!(result.added_data_files.len(), 0);
956 assert_eq!(result.stats.input_rows, 2);
957 assert_eq!(result.stats.output_rows, 0);
958
959 Ok(())
960 }
961
962 #[tokio::test]
963 async fn cow_rewrite_removes_file_fully_covered_by_position_deletes() -> Result<()> {
964 let mut fixture = TableTestFixture::new();
965 let positions = (0..300).collect::<Vec<i64>>();
966 fixture.setup_multi_row_group_manifest(&positions).await;
967
968 let result = CowRewriteBuilder::new(&fixture.table)
969 .with_predicate(crate::expr::Predicate::AlwaysTrue)
970 .with_rewriter(Arc::new(KeepAll))
971 .rewrite()
972 .await?;
973
974 assert_eq!(result.stats.candidate_files, 1);
975 assert_eq!(result.stats.rewritten_files, 1);
976 assert_eq!(result.stats.input_rows, 0);
977 assert_eq!(result.stats.output_rows, 0);
978 assert_eq!(result.removed_data_files.len(), 1);
979 assert!(result.added_data_files.is_empty());
980 assert!(result.unchanged_data_files.is_empty());
981
982 Ok(())
983 }
984
985 #[tokio::test]
986 async fn cow_rewrite_delete_no_matching_rows_keeps_old_file() -> Result<()> {
987 let fixture = test_table_with_ids(vec![1, 3]).await?;
988
989 let result = CowRewriteBuilder::new(&fixture.table)
990 .with_predicate(crate::expr::Predicate::AlwaysTrue)
991 .with_rewriter(Arc::new(DeleteEvenIds))
992 .rewrite()
993 .await?;
994
995 assert!(!result.has_changes());
996 assert_eq!(result.removed_data_files.len(), 0);
997 assert_eq!(result.added_data_files.len(), 0);
998 assert_eq!(result.unchanged_data_files.len(), 1);
999 assert_eq!(result.stats.input_rows, 2);
1000 assert_eq!(result.stats.output_rows, 0);
1001
1002 Ok(())
1003 }
1004
1005 struct SilentFullDrop;
1010
1011 impl CowBatchRewriter for SilentFullDrop {
1012 fn rewrite_batch(&self, _batch: RecordBatch) -> Result<CowBatchRewrite> {
1013 Ok(CowBatchRewrite {
1014 output: None,
1015 changed: false,
1016 })
1017 }
1018 }
1019
1020 #[tokio::test]
1021 async fn cow_rewrite_none_output_implies_change() -> Result<()> {
1022 let fixture = test_table_with_ids(vec![1, 2]).await?;
1023
1024 let result = CowRewriteBuilder::new(&fixture.table)
1025 .with_predicate(crate::expr::Predicate::AlwaysTrue)
1026 .with_rewriter(Arc::new(SilentFullDrop))
1027 .rewrite()
1028 .await?;
1029
1030 assert!(result.has_changes());
1031 assert_eq!(result.removed_data_files.len(), 1);
1032 assert_eq!(result.added_data_files.len(), 0);
1033 assert!(result.unchanged_data_files.is_empty());
1034 assert_eq!(result.stats.rewritten_files, 1);
1035 assert_eq!(result.stats.input_rows, 2);
1036 assert_eq!(result.stats.output_rows, 0);
1037 assert_eq!(result.stats.changed_batches, 1);
1038
1039 Ok(())
1040 }
1041
1042 struct DropFirstBatchSilently {
1047 batches_seen: std::sync::atomic::AtomicUsize,
1048 }
1049
1050 impl CowBatchRewriter for DropFirstBatchSilently {
1051 fn rewrite_batch(&self, batch: RecordBatch) -> Result<CowBatchRewrite> {
1052 let seen = self
1053 .batches_seen
1054 .fetch_add(1, std::sync::atomic::Ordering::SeqCst);
1055 if seen == 0 {
1056 Ok(CowBatchRewrite {
1057 output: None,
1058 changed: false,
1059 })
1060 } else {
1061 Ok(CowBatchRewrite {
1062 output: Some(batch),
1063 changed: true,
1064 })
1065 }
1066 }
1067 }
1068
1069 #[tokio::test]
1070 async fn cow_rewrite_silent_drop_before_change_loses_no_rows() -> Result<()> {
1071 let fixture = test_table_with_ids(vec![1, 2, 3, 4]).await?;
1072
1073 let result = CowRewriteBuilder::new(&fixture.table)
1074 .with_predicate(crate::expr::Predicate::AlwaysTrue)
1075 .with_batch_size(2)
1076 .with_rewriter(Arc::new(DropFirstBatchSilently {
1077 batches_seen: std::sync::atomic::AtomicUsize::new(0),
1078 }))
1079 .rewrite()
1080 .await?;
1081
1082 assert!(result.has_changes());
1083 assert_eq!(result.removed_data_files.len(), 1);
1084 assert_eq!(result.added_data_files.len(), 1);
1085 assert_eq!(result.stats.input_rows, 4);
1086 assert_eq!(result.stats.output_rows, 2);
1087
1088 let ids = read_ids(&fixture.table, &result.added_data_files).await?;
1089 assert_eq!(ids, vec![3, 4]);
1090
1091 Ok(())
1092 }
1093
1094 #[tokio::test]
1095 async fn cow_rewrite_kept_prefix_survives_later_full_batch_delete() -> Result<()> {
1096 let fixture = test_table_with_ids(vec![1, 3, 2, 4]).await?;
1097
1098 let result = CowRewriteBuilder::new(&fixture.table)
1099 .with_predicate(crate::expr::Predicate::AlwaysTrue)
1100 .with_batch_size(2)
1101 .with_rewriter(Arc::new(DeleteEvenIds))
1102 .rewrite()
1103 .await?;
1104
1105 assert_eq!(result.removed_data_files.len(), 1);
1106 assert_eq!(result.stats.input_rows, 4);
1107 assert_eq!(result.stats.output_rows, 2);
1108 assert_eq!(result.added_data_files.len(), 1);
1109 assert!(result.unchanged_data_files.is_empty());
1110 assert_eq!(
1111 read_ids(&fixture.table, &result.added_data_files).await?,
1112 vec![1, 3]
1113 );
1114
1115 Ok(())
1116 }
1117
1118 #[tokio::test]
1119 async fn cow_rewrite_uses_unique_replacement_paths_for_multiple_source_files() -> Result<()> {
1120 let fixture = test_table_with_id_batches(vec![vec![1, 2], vec![3, 4]]).await?;
1121
1122 let result = CowRewriteBuilder::new(&fixture.table)
1123 .with_predicate(crate::expr::Predicate::AlwaysTrue)
1124 .with_rewriter(Arc::new(DeleteEvenIds))
1125 .rewrite()
1126 .await?;
1127
1128 assert_eq!(result.stats.candidate_files, 2);
1129 assert_eq!(result.removed_data_files.len(), 2);
1130 assert_eq!(result.added_data_files.len(), 2);
1131
1132 let added_paths = result
1133 .added_data_files
1134 .iter()
1135 .map(|file| file.file_path().to_string())
1136 .collect::<HashSet<_>>();
1137 let removed_paths = result
1138 .removed_data_files
1139 .iter()
1140 .map(|file| file.file_path().to_string())
1141 .collect::<HashSet<_>>();
1142
1143 assert_eq!(added_paths.len(), result.added_data_files.len());
1144 assert!(added_paths.is_disjoint(&removed_paths));
1145
1146 let ids = read_ids(&fixture.table, &result.added_data_files).await?;
1147 assert_eq!(ids, vec![1, 3]);
1148
1149 Ok(())
1150 }
1151
1152 #[test]
1153 fn cow_batch_rewriter_is_object_safe() {
1154 let _rewriter: Arc<dyn CowBatchRewriter> = Arc::new(KeepAll);
1155 }
1156
1157 #[test]
1158 fn cow_rewrite_result_reports_no_changes() {
1159 let result = crate::cow_rewrite::CowRewriteResult {
1160 removed_data_files: vec![],
1161 added_data_files: vec![],
1162 unchanged_data_files: vec![],
1163 stats: crate::cow_rewrite::CowRewriteStats {
1164 candidate_files: 0,
1165 rewritten_files: 0,
1166 unchanged_files: 0,
1167 input_rows: 0,
1168 output_rows: 0,
1169 changed_batches: 0,
1170 },
1171 };
1172
1173 assert!(!result.has_changes());
1174 assert_eq!(result.stats.candidate_files, 0);
1175 }
1176
1177 struct DeleteIfModThree;
1179
1180 impl CowBatchRewriter for DeleteIfModThree {
1181 fn rewrite_batch(&self, batch: RecordBatch) -> Result<CowBatchRewrite> {
1182 let ids = batch
1183 .column_by_name("id")
1184 .ok_or_else(|| Error::new(ErrorKind::DataInvalid, "missing id column"))?
1185 .as_any()
1186 .downcast_ref::<Int32Array>()
1187 .ok_or_else(|| Error::new(ErrorKind::DataInvalid, "id must be Int32"))?;
1188
1189 let keep =
1190 BooleanArray::from_iter((0..ids.len()).map(|row| Some(ids.value(row) % 3 != 0)));
1191 let filtered = arrow_select::filter::filter_record_batch(&batch, &keep)
1192 .map_err(|err| Error::new(ErrorKind::Unexpected, err.to_string()))?;
1193
1194 Ok(CowBatchRewrite {
1195 changed: filtered.num_rows() != batch.num_rows(),
1196 output: (filtered.num_rows() > 0).then_some(filtered),
1197 })
1198 }
1199 }
1200
1201 #[tokio::test]
1207 async fn cow_rewrite_after_optional_column_add() -> Result<()> {
1208 let temp_dir = TempDir::new().unwrap();
1209 let warehouse = format!("file://{}", temp_dir.path().join("warehouse").display());
1210 let catalog: Arc<dyn Catalog> = Arc::new(
1211 MemoryCatalogBuilder::default()
1212 .with_storage_factory(Arc::new(LocalFsStorageFactory))
1213 .load(
1214 "memory",
1215 HashMap::from([(MEMORY_CATALOG_WAREHOUSE.to_string(), warehouse)]),
1216 )
1217 .await?,
1218 );
1219 let namespace = NamespaceIdent::new("ns".to_string());
1220 catalog.create_namespace(&namespace, HashMap::new()).await?;
1221
1222 let schema = Schema::builder()
1223 .with_fields(vec![
1224 NestedField::required(1, "id", Type::Primitive(PrimitiveType::Int)).into(),
1225 ])
1226 .build()?;
1227 let table = catalog
1228 .create_table(
1229 &namespace,
1230 TableCreation::builder()
1231 .name("evolved".to_string())
1232 .schema(schema)
1233 .build(),
1234 )
1235 .await?;
1236
1237 let arrow_schema = Arc::new(ArrowSchema::new(vec![
1239 Field::new("id", DataType::Int32, false).with_metadata(HashMap::from([(
1240 PARQUET_FIELD_ID_META_KEY.to_string(),
1241 "1".to_string(),
1242 )])),
1243 ]));
1244 let batch = RecordBatch::try_new(arrow_schema, vec![Arc::new(Int32Array::from(vec![
1245 1, 2, 3, 4,
1246 ])) as ArrayRef])?;
1247 let data_files = super::writer::write_replacement_batches(
1248 &table,
1249 table.metadata().current_schema().clone(),
1250 None,
1251 futures::stream::iter(vec![Ok(batch)]),
1252 )
1253 .await?;
1254 let tx = Transaction::new(&table);
1255 let tx = tx.fast_append().add_data_files(data_files).apply(tx)?;
1256 let table = tx.commit(&*catalog).await?;
1257
1258 let tx = Transaction::new(&table);
1260 let tx = tx
1261 .update_schema()
1262 .add_column(AddColumn::optional(
1263 "value",
1264 Type::Primitive(PrimitiveType::Int),
1265 ))
1266 .apply(tx)?;
1267 let table = tx.commit(&*catalog).await?;
1268
1269 let current_snapshot = table.metadata().current_snapshot().unwrap();
1272 assert_ne!(
1273 table.metadata().current_schema_id(),
1274 current_snapshot.schema_id().unwrap()
1275 );
1276
1277 let result = CowRewriteBuilder::new(&table)
1280 .with_predicate(crate::expr::Predicate::AlwaysTrue)
1281 .with_rewriter(Arc::new(DeleteIfModThree))
1282 .rewrite()
1283 .await?;
1284
1285 assert!(result.has_changes());
1287 assert_eq!(result.removed_data_files.len(), 1);
1288 assert_eq!(result.added_data_files.len(), 1);
1289
1290 let ids = read_ids(&table, &result.added_data_files).await?;
1291 assert_eq!(ids, vec![1, 2, 4]);
1292
1293 Ok(())
1294 }
1295
1296 async fn two_file_table_disjoint_ids() -> Result<CowRewriteFixture> {
1299 let temp_dir = TempDir::new().unwrap();
1300 let warehouse = format!("file://{}", temp_dir.path().join("warehouse").display());
1301 let catalog = MemoryCatalogBuilder::default()
1302 .with_storage_factory(Arc::new(LocalFsStorageFactory))
1303 .load(
1304 "memory",
1305 HashMap::from([(MEMORY_CATALOG_WAREHOUSE.to_string(), warehouse)]),
1306 )
1307 .await?;
1308 let namespace = NamespaceIdent::new("ns".to_string());
1309 catalog.create_namespace(&namespace, HashMap::new()).await?;
1310
1311 let schema = Schema::builder()
1312 .with_fields(vec![
1313 NestedField::required(1, "id", Type::Primitive(PrimitiveType::Int)).into(),
1314 ])
1315 .build()?;
1316 let table = catalog
1317 .create_table(
1318 &namespace,
1319 TableCreation::builder()
1320 .name("disjoint".to_string())
1321 .schema(schema)
1322 .properties(HashMap::from([(
1323 TableProperties::PROPERTY_WRITE_TARGET_FILE_SIZE_BYTES.to_string(),
1324 "1".to_string(),
1325 )]))
1326 .build(),
1327 )
1328 .await?;
1329
1330 let arrow_schema = Arc::new(ArrowSchema::new(vec![
1331 Field::new("id", DataType::Int32, false).with_metadata(HashMap::from([(
1332 PARQUET_FIELD_ID_META_KEY.to_string(),
1333 "1".to_string(),
1334 )])),
1335 ]));
1336 let input = vec![vec![1, 2, 3, 4], vec![100, 101, 102, 103]]
1339 .into_iter()
1340 .map(|ids| {
1341 Ok(RecordBatch::try_new(arrow_schema.clone(), vec![
1342 Arc::new(Int32Array::from(ids)) as ArrayRef,
1343 ])?)
1344 });
1345 let data_files = super::writer::write_replacement_batches(
1346 &table,
1347 table.metadata().current_schema().clone(),
1348 None,
1349 futures::stream::iter(input),
1350 )
1351 .await?;
1352
1353 let tx = Transaction::new(&table);
1354 let tx = tx.fast_append().add_data_files(data_files).apply(tx)?;
1355 let table = tx.commit(&catalog).await?;
1356
1357 Ok(CowRewriteFixture {
1358 _temp_dir: temp_dir,
1359 table,
1360 })
1361 }
1362
1363 #[tokio::test]
1367 async fn cow_rewrite_predicate_prunes_non_matching_files() -> Result<()> {
1368 let fixture = two_file_table_disjoint_ids().await?;
1369
1370 let predicate = crate::expr::Reference::new("id").greater_than(crate::spec::Datum::int(50));
1371
1372 let result = CowRewriteBuilder::new(&fixture.table)
1373 .with_predicate(predicate)
1374 .with_rewriter(Arc::new(KeepAll))
1375 .rewrite()
1376 .await?;
1377
1378 assert_eq!(result.stats.candidate_files, 1);
1380 assert_eq!(result.removed_data_files.len(), 0);
1382 assert_eq!(result.added_data_files.len(), 0);
1383 assert_eq!(result.unchanged_data_files.len(), 1);
1384
1385 let ids = read_ids(&fixture.table, &result.unchanged_data_files).await?;
1387 assert_eq!(ids, vec![100, 101, 102, 103]);
1388
1389 Ok(())
1390 }
1391
1392 struct TouchEveryBatch;
1395
1396 impl CowBatchRewriter for TouchEveryBatch {
1397 fn rewrite_batch(&self, batch: RecordBatch) -> Result<CowBatchRewrite> {
1398 Ok(CowBatchRewrite {
1399 output: Some(batch),
1400 changed: true,
1401 })
1402 }
1403 }
1404
1405 #[tokio::test]
1410 async fn cow_rewrite_preserves_partitions_end_to_end() -> Result<()> {
1411 let temp_dir = TempDir::new().unwrap();
1412 let warehouse = format!("file://{}", temp_dir.path().join("warehouse").display());
1413 let catalog = MemoryCatalogBuilder::default()
1414 .with_storage_factory(Arc::new(LocalFsStorageFactory))
1415 .load(
1416 "memory",
1417 HashMap::from([(MEMORY_CATALOG_WAREHOUSE.to_string(), warehouse)]),
1418 )
1419 .await?;
1420 let namespace = NamespaceIdent::new("ns".to_string());
1421 catalog.create_namespace(&namespace, HashMap::new()).await?;
1422
1423 let schema = Schema::builder()
1424 .with_fields(vec![
1425 NestedField::required(1, "id", Type::Primitive(PrimitiveType::Int)).into(),
1426 NestedField::optional(2, "value", Type::Primitive(PrimitiveType::Int)).into(),
1427 ])
1428 .build()?;
1429 let partition_spec = crate::spec::PartitionSpec::builder(Arc::new(schema.clone()))
1430 .add_partition_field("value", "value", Transform::Identity)?
1431 .build()?;
1432 let table = catalog
1433 .create_table(
1434 &namespace,
1435 TableCreation::builder()
1436 .name("partitioned".to_string())
1437 .schema(schema)
1438 .partition_spec(partition_spec.into_unbound())
1439 .build(),
1440 )
1441 .await?;
1442
1443 let arrow_schema = Arc::new(ArrowSchema::new(vec![
1446 Field::new("id", DataType::Int32, false).with_metadata(HashMap::from([(
1447 PARQUET_FIELD_ID_META_KEY.to_string(),
1448 "1".to_string(),
1449 )])),
1450 Field::new("value", DataType::Int32, true).with_metadata(HashMap::from([(
1451 PARQUET_FIELD_ID_META_KEY.to_string(),
1452 "2".to_string(),
1453 )])),
1454 ]));
1455 let current_schema = table.metadata().current_schema().clone();
1456 let default_spec = table.metadata().default_partition_spec().as_ref().clone();
1457
1458 let batch_one = RecordBatch::try_new(arrow_schema.clone(), vec![
1459 Arc::new(Int32Array::from(vec![1, 2])) as ArrayRef,
1460 Arc::new(Int32Array::from(vec![1, 1])),
1461 ])?;
1462 let one_partition = crate::spec::PartitionKey::new(
1463 default_spec.clone(),
1464 current_schema.clone(),
1465 Struct::from_iter([Some(Literal::int(1))]),
1466 );
1467 let files_one = super::writer::write_replacement_batches(
1468 &table,
1469 current_schema.clone(),
1470 Some(one_partition),
1471 futures::stream::iter(vec![Ok(batch_one)]),
1472 )
1473 .await?;
1474
1475 let batch_null = RecordBatch::try_new(arrow_schema.clone(), vec![
1476 Arc::new(Int32Array::from(vec![3, 4])) as ArrayRef,
1477 Arc::new(Int32Array::from(vec![None::<i32>, None])),
1478 ])?;
1479 let null_partition = crate::spec::PartitionKey::new(
1480 default_spec.clone(),
1481 current_schema.clone(),
1482 Struct::from_iter([None::<Literal>]),
1483 );
1484 let files_null = super::writer::write_replacement_batches(
1485 &table,
1486 current_schema.clone(),
1487 Some(null_partition),
1488 futures::stream::iter(vec![Ok(batch_null)]),
1489 )
1490 .await?;
1491
1492 let tx = Transaction::new(&table);
1493 let tx = tx
1494 .fast_append()
1495 .add_data_files([files_one, files_null].concat())
1496 .apply(tx)?;
1497 let table = tx.commit(&catalog).await?;
1498
1499 let result = CowRewriteBuilder::new(&table)
1500 .with_rewriter(Arc::new(TouchEveryBatch))
1501 .rewrite()
1502 .await?;
1503
1504 assert_eq!(result.stats.candidate_files, 2);
1505 assert_eq!(result.stats.rewritten_files, 2);
1506 assert_eq!(result.removed_data_files.len(), 2);
1507 assert_eq!(result.added_data_files.len(), 2);
1508 assert!(result.unchanged_data_files.is_empty());
1509
1510 let one_struct = Struct::from_iter([Some(Literal::int(1))]);
1512 let null_struct = Struct::from_iter([None::<Literal>]);
1513 let mut removed_partitions = result
1514 .removed_data_files
1515 .iter()
1516 .map(|file| file.partition().clone())
1517 .collect::<Vec<_>>();
1518 let mut added_partitions = result
1519 .added_data_files
1520 .iter()
1521 .map(|file| file.partition().clone())
1522 .collect::<Vec<_>>();
1523 removed_partitions.sort_by_key(|partition| partition.is_null_at_index(0));
1524 added_partitions.sort_by_key(|partition| partition.is_null_at_index(0));
1525 assert_eq!(removed_partitions, added_partitions);
1526 assert_eq!(added_partitions[0], one_struct);
1527 assert_eq!(added_partitions[1], null_struct);
1528
1529 for file in &result.added_data_files {
1530 assert_eq!(file.partition_spec_id, 0);
1531 if file.partition() == &one_struct {
1532 assert!(file.file_path().contains("value=1"), "{}", file.file_path());
1533 } else {
1534 assert!(
1535 file.file_path().contains("value=null"),
1536 "{}",
1537 file.file_path()
1538 );
1539 }
1540 }
1541
1542 let ids = read_ids(&table, &result.added_data_files).await?;
1544 assert_eq!(ids, vec![1, 2, 3, 4]);
1545
1546 Ok(())
1547 }
1548
1549 #[cfg(debug_assertions)]
1554 struct MutatePartitionColumn;
1555
1556 #[cfg(debug_assertions)]
1557 impl CowBatchRewriter for MutatePartitionColumn {
1558 fn rewrite_batch(&self, batch: RecordBatch) -> Result<CowBatchRewrite> {
1559 let values = batch
1560 .column_by_name("value")
1561 .ok_or_else(|| Error::new(ErrorKind::DataInvalid, "missing value column"))?
1562 .as_any()
1563 .downcast_ref::<Int32Array>()
1564 .ok_or_else(|| Error::new(ErrorKind::DataInvalid, "value must be Int32"))?;
1565 let updated =
1566 Int32Array::from_iter((0..values.len()).map(|row| Some(values.value(row) + 100)));
1567 let output = RecordBatch::try_new(batch.schema(), vec![
1568 batch.column(0).clone(),
1569 Arc::new(updated),
1570 ])
1571 .map_err(|err| Error::new(ErrorKind::Unexpected, err.to_string()))?;
1572
1573 Ok(CowBatchRewrite {
1574 output: Some(output),
1575 changed: true,
1576 })
1577 }
1578 }
1579
1580 #[cfg(debug_assertions)]
1581 #[tokio::test]
1582 #[should_panic(expected = "must preserve the source file's partition values")]
1583 async fn cow_rewrite_partition_column_mutation_trips_debug_guard() {
1584 let temp_dir = TempDir::new().unwrap();
1585 let warehouse = format!("file://{}", temp_dir.path().join("warehouse").display());
1586 let catalog = MemoryCatalogBuilder::default()
1587 .with_storage_factory(Arc::new(LocalFsStorageFactory))
1588 .load(
1589 "memory",
1590 HashMap::from([(MEMORY_CATALOG_WAREHOUSE.to_string(), warehouse)]),
1591 )
1592 .await
1593 .unwrap();
1594 let namespace = NamespaceIdent::new("ns".to_string());
1595 catalog
1596 .create_namespace(&namespace, HashMap::new())
1597 .await
1598 .unwrap();
1599
1600 let schema = Schema::builder()
1601 .with_fields(vec![
1602 NestedField::required(1, "id", Type::Primitive(PrimitiveType::Int)).into(),
1603 NestedField::required(2, "value", Type::Primitive(PrimitiveType::Int)).into(),
1604 ])
1605 .build()
1606 .unwrap();
1607 let partition_spec = crate::spec::PartitionSpec::builder(Arc::new(schema.clone()))
1608 .add_partition_field("value", "value", Transform::Identity)
1609 .unwrap()
1610 .build()
1611 .unwrap();
1612 let table = catalog
1613 .create_table(
1614 &namespace,
1615 TableCreation::builder()
1616 .name("partition_guard".to_string())
1617 .schema(schema)
1618 .partition_spec(partition_spec.into_unbound())
1619 .build(),
1620 )
1621 .await
1622 .unwrap();
1623
1624 let arrow_schema = Arc::new(ArrowSchema::new(vec![
1626 Field::new("id", DataType::Int32, false).with_metadata(HashMap::from([(
1627 PARQUET_FIELD_ID_META_KEY.to_string(),
1628 "1".to_string(),
1629 )])),
1630 Field::new("value", DataType::Int32, false).with_metadata(HashMap::from([(
1631 PARQUET_FIELD_ID_META_KEY.to_string(),
1632 "2".to_string(),
1633 )])),
1634 ]));
1635 let batch = RecordBatch::try_new(arrow_schema, vec![
1636 Arc::new(Int32Array::from(vec![1, 2])) as ArrayRef,
1637 Arc::new(Int32Array::from(vec![1, 1])),
1638 ])
1639 .unwrap();
1640 let partition_key = crate::spec::PartitionKey::new(
1641 table.metadata().default_partition_spec().as_ref().clone(),
1642 table.metadata().current_schema().clone(),
1643 Struct::from_iter([Some(Literal::int(1))]),
1644 );
1645 let data_files = super::writer::write_replacement_batches(
1646 &table,
1647 table.metadata().current_schema().clone(),
1648 Some(partition_key),
1649 futures::stream::iter(vec![Ok(batch)]),
1650 )
1651 .await
1652 .unwrap();
1653
1654 let tx = Transaction::new(&table);
1655 let tx = tx
1656 .fast_append()
1657 .add_data_files(data_files)
1658 .apply(tx)
1659 .unwrap();
1660 let table = tx.commit(&catalog).await.unwrap();
1661
1662 let _ = CowRewriteBuilder::new(&table)
1668 .with_rewriter(Arc::new(MutatePartitionColumn))
1669 .rewrite()
1670 .await;
1671 }
1672}