1use std::collections::{HashMap, HashSet};
21
22use futures::{StreamExt, TryStreamExt, stream};
23
24use crate::Result;
25use crate::io::FileIO;
26use crate::spec::{ManifestFile, TableMetadata};
27use crate::table::Table;
28
29const DELETE_CONCURRENCY: usize = 10;
30
31pub async fn drop_table_data(table_info: &Table) -> Result<()> {
41 let mut manifest_lists_to_delete: HashSet<String> = HashSet::new();
42 let mut manifests_to_delete: HashMap<String, ManifestFile> = HashMap::new();
43
44 let metadata = table_info.metadata_ref();
45 let io = table_info.file_io();
46 let results: Vec<_> =
48 futures::future::try_join_all(metadata.snapshots().map(|snapshot| async {
49 let manifest_list = table_info.manifest_list_reader(snapshot).load().await?;
50 Ok::<_, crate::Error>((snapshot.manifest_list().to_string(), manifest_list))
51 }))
52 .await?;
53
54 for (manifest_list_location, manifest_list) in results {
55 if !manifest_list_location.is_empty() {
56 manifest_lists_to_delete.insert(manifest_list_location);
57 }
58 for manifest_file in manifest_list.entries() {
59 manifests_to_delete.insert(manifest_file.manifest_path.clone(), manifest_file.clone());
60 }
61 }
62
63 if metadata.table_properties().gc_enabled()? {
65 delete_data_files(table_info, &manifests_to_delete).await?;
66 }
67
68 let manifest_paths: Vec<String> = manifests_to_delete.into_keys().collect();
70 io.delete_stream(stream::iter(manifest_paths)).await?;
71
72 io.delete_stream(stream::iter(manifest_lists_to_delete))
74 .await?;
75
76 let prev_metadata_paths: Vec<String> = metadata
78 .metadata_log()
79 .iter()
80 .map(|m| m.metadata_file.clone())
81 .collect();
82 io.delete_stream(stream::iter(prev_metadata_paths)).await?;
83
84 let stats_paths: Vec<String> = metadata
86 .statistics_iter()
87 .map(|s| s.statistics_path.clone())
88 .collect();
89 io.delete_stream(stream::iter(stats_paths)).await?;
90
91 let partition_stats_paths: Vec<String> = metadata
93 .partition_statistics_iter()
94 .map(|s| s.statistics_path.clone())
95 .collect();
96 io.delete_stream(stream::iter(partition_stats_paths))
97 .await?;
98
99 if let Some(location) = table_info.metadata_location() {
101 io.delete(location).await?;
102 }
103
104 Ok(())
105}
106
107pub async fn delete_removed_metadata_files(io: &FileIO, base: &TableMetadata, new: &TableMetadata) {
114 match new
115 .table_properties()
116 .metadata_delete_after_commit_enabled()
117 {
118 Ok(true) => {}
119 Ok(false) => return,
120 Err(e) => {
121 tracing::warn!("Skipping deletion of removed metadata files: {e}");
122 return;
123 }
124 }
125
126 let retained: HashSet<&str> = new
127 .metadata_log()
128 .iter()
129 .map(|log| log.metadata_file.as_str())
130 .collect();
131 let removed = base
132 .metadata_log()
133 .iter()
134 .map(|log| log.metadata_file.as_str())
135 .filter(|path| !retained.contains(path));
136
137 stream::iter(removed)
138 .for_each_concurrent(DELETE_CONCURRENCY, |path| async move {
139 if let Err(e) = io.delete(path).await {
140 tracing::warn!("Failed to delete removed metadata file {path}: {e}");
141 }
142 })
143 .await;
144}
145
146async fn delete_data_files(
148 table: &Table,
149 manifest_files: &HashMap<String, ManifestFile>,
150) -> Result<()> {
151 let io = table.file_io();
152 stream::iter(manifest_files.values().map(Ok))
153 .try_for_each_concurrent(DELETE_CONCURRENCY, |manifest_file| async move {
154 let manifest = table.manifest_reader().read(manifest_file).await?;
155 let data_file_paths = manifest
156 .entries()
157 .iter()
158 .map(|entry| entry.data_file.file_path().to_string())
159 .collect::<Vec<_>>();
160
161 io.delete_stream(stream::iter(data_file_paths)).await
162 })
163 .await
164}