1use std::collections::HashMap;
21use std::sync::Arc;
22
23use arrow_schema::SchemaRef as ArrowSchemaRef;
24use bytes::Bytes;
25use futures::future::BoxFuture;
26use itertools::Itertools;
27use parquet::arrow::AsyncArrowWriter;
28use parquet::arrow::async_reader::AsyncFileReader;
29use parquet::arrow::async_writer::AsyncFileWriter as ArrowAsyncFileWriter;
30use parquet::basic::{BrotliLevel, Compression, GzipLevel, ZstdLevel};
31use parquet::encryption::encrypt::FileEncryptionProperties;
32use parquet::file::metadata::{ColumnChunkMetaData, ParquetMetaData};
33use parquet::file::properties::{CdcOptions, WriterProperties};
34use parquet::file::statistics::Statistics;
35use parquet::geospatial::bounding_box::BoundingBox;
36use parquet_geospatial::interval::{Interval, IntervalTrait, WraparoundInterval};
37
38use super::{FileWriter, FileWriterBuilder};
39use crate::arrow::{
40 ArrowFileReader, DEFAULT_MAP_FIELD_NAME, FieldMatchMode, NanValueCountVisitor,
41 get_parquet_stat_max_as_datum, get_parquet_stat_min_as_datum,
42};
43use crate::compression::CompressionCodec;
44use crate::encryption::{EncryptionManager, StandardKeyMetadata};
45use crate::error::invalid_data;
46use crate::io::{FileIO, FileWrite, OutputFile};
47use crate::spec::{
48 DataContentType, DataFileBuilder, DataFileFormat, Datum, ListType, Literal, MapType,
49 NestedFieldRef, PartitionSpec, PrimitiveLiteral, PrimitiveType, Schema, SchemaRef,
50 SchemaVisitor, Struct, StructType, TableMetadata, TableProperties, Type, VariantType,
51 visit_schema,
52};
53use crate::transform::create_transform_function;
54use crate::writer::{CurrentFileStatus, DataFile};
55use crate::{Error, ErrorKind, Result};
56
57#[derive(Clone, Debug)]
59pub struct ParquetWriterBuilder {
60 props: WriterProperties,
61 schema: SchemaRef,
62 match_mode: FieldMatchMode,
63 encryption_manager: Option<Arc<EncryptionManager>>,
64}
65
66impl ParquetWriterBuilder {
67 pub fn new(props: WriterProperties, schema: SchemaRef) -> Self {
74 Self::new_with_match_mode(props, schema, FieldMatchMode::Id)
75 }
76
77 pub fn new_with_match_mode(
79 props: WriterProperties,
80 schema: SchemaRef,
81 match_mode: FieldMatchMode,
82 ) -> Self {
83 Self {
84 props,
85 schema,
86 match_mode,
87 encryption_manager: None,
88 }
89 }
90
91 pub fn from_table_properties(
95 table_props: &TableProperties<'_>,
96 schema: SchemaRef,
97 ) -> Result<Self> {
98 let cdc = if table_props.cdc_enabled()? {
99 Some(CdcOptions {
100 min_chunk_size: table_props.cdc_min_chunk_size()?,
101 max_chunk_size: table_props.cdc_max_chunk_size()?,
102 norm_level: table_props.cdc_norm_level()?,
103 })
104 } else {
105 None
106 };
107 let compression = parquet_compression(table_props.parquet_compression_codec()?)?;
108 let props = WriterProperties::builder()
109 .set_content_defined_chunking(cdc)
110 .set_compression(compression)
111 .set_max_row_group_bytes(Some(table_props.parquet_row_group_size_bytes()?))
112 .set_data_page_size_limit(table_props.parquet_page_size_bytes()?)
113 .set_data_page_row_count_limit(table_props.parquet_page_row_limit()?)
114 .set_dictionary_page_size_limit(table_props.parquet_dict_size_bytes()?)
115 .build();
116 Ok(Self::new_with_match_mode(props, schema, FieldMatchMode::Id))
117 }
118
119 pub fn with_match_mode(mut self, match_mode: FieldMatchMode) -> Self {
124 self.match_mode = match_mode;
125 self
126 }
127
128 pub fn with_encryption_manager(mut self, encryption_manager: Arc<EncryptionManager>) -> Self {
130 self.encryption_manager = Some(encryption_manager);
131 self
132 }
133}
134
135fn parquet_compression(codec: CompressionCodec) -> Result<Compression> {
136 let compression = match codec {
137 CompressionCodec::None => Compression::UNCOMPRESSED,
138 CompressionCodec::Snappy => Compression::SNAPPY,
139 CompressionCodec::Lzo => Compression::LZO,
140 CompressionCodec::Lz4 => Compression::LZ4,
141 CompressionCodec::Lz4Raw => Compression::LZ4_RAW,
142 CompressionCodec::Zstd(level) => {
143 let level =
144 ZstdLevel::try_new(level as i32).map_err(|e| invalid_level_error("zstd", e))?;
145 Compression::ZSTD(level)
146 }
147 CompressionCodec::Gzip(level) => {
148 let level =
149 GzipLevel::try_new(level as u32).map_err(|e| invalid_level_error("gzip", e))?;
150 Compression::GZIP(level)
151 }
152 CompressionCodec::Brotli(level) => {
153 let level =
154 BrotliLevel::try_new(level as u32).map_err(|e| invalid_level_error("brotli", e))?;
155 Compression::BROTLI(level)
156 }
157 };
158 Ok(compression)
159}
160
161fn invalid_level_error(codec: &str, source: impl Into<anyhow::Error>) -> Error {
162 invalid_data!("Invalid {codec} compression level").with_source(source)
163}
164
165impl FileWriterBuilder for ParquetWriterBuilder {
166 type R = ParquetWriter;
167
168 async fn build(&self, output_file: OutputFile) -> Result<Self::R> {
169 let key_metadata = self
170 .encryption_manager
171 .as_ref()
172 .map(|em| em.generate_key_metadata());
173 let writer_properties =
174 resolve_writer_properties(self.props.clone(), key_metadata.as_ref())?;
175 Ok(ParquetWriter {
176 schema: self.schema.clone(),
177 inner_writer: None,
178 writer_properties,
179 current_row_num: 0,
180 output_file,
181 nan_value_count_visitor: NanValueCountVisitor::new_with_match_mode(self.match_mode),
182 key_metadata,
183 })
184 }
185}
186
187struct IndexByParquetPathName {
189 name_to_id: HashMap<String, i32>,
190
191 field_names: Vec<String>,
192
193 field_id: i32,
194}
195
196impl IndexByParquetPathName {
197 pub fn new() -> Self {
199 Self {
200 name_to_id: HashMap::new(),
201 field_names: Vec::new(),
202 field_id: 0,
203 }
204 }
205
206 pub fn get(&self, name: &str) -> Option<&i32> {
208 self.name_to_id.get(name)
209 }
210}
211
212impl Default for IndexByParquetPathName {
213 fn default() -> Self {
214 Self::new()
215 }
216}
217
218impl SchemaVisitor for IndexByParquetPathName {
219 type T = ();
220
221 fn before_struct_field(&mut self, field: &NestedFieldRef) -> Result<()> {
222 self.field_names.push(field.name.to_string());
223 self.field_id = field.id;
224 Ok(())
225 }
226
227 fn after_struct_field(&mut self, _field: &NestedFieldRef) -> Result<()> {
228 self.field_names.pop();
229 Ok(())
230 }
231
232 fn before_list_element(&mut self, field: &NestedFieldRef) -> Result<()> {
233 self.field_names.push(format!("list.{}", field.name));
234 self.field_id = field.id;
235 Ok(())
236 }
237
238 fn after_list_element(&mut self, _field: &NestedFieldRef) -> Result<()> {
239 self.field_names.pop();
240 Ok(())
241 }
242
243 fn before_map_key(&mut self, field: &NestedFieldRef) -> Result<()> {
244 self.field_names
245 .push(format!("{DEFAULT_MAP_FIELD_NAME}.key"));
246 self.field_id = field.id;
247 Ok(())
248 }
249
250 fn after_map_key(&mut self, _field: &NestedFieldRef) -> Result<()> {
251 self.field_names.pop();
252 Ok(())
253 }
254
255 fn before_map_value(&mut self, field: &NestedFieldRef) -> Result<()> {
256 self.field_names
257 .push(format!("{DEFAULT_MAP_FIELD_NAME}.value"));
258 self.field_id = field.id;
259 Ok(())
260 }
261
262 fn after_map_value(&mut self, _field: &NestedFieldRef) -> Result<()> {
263 self.field_names.pop();
264 Ok(())
265 }
266
267 fn schema(&mut self, _schema: &Schema, _value: Self::T) -> Result<Self::T> {
268 Ok(())
269 }
270
271 fn field(&mut self, _field: &NestedFieldRef, _value: Self::T) -> Result<Self::T> {
272 Ok(())
273 }
274
275 fn r#struct(&mut self, _struct: &StructType, _results: Vec<Self::T>) -> Result<Self::T> {
276 Ok(())
277 }
278
279 fn list(&mut self, _list: &ListType, _value: Self::T) -> Result<Self::T> {
280 Ok(())
281 }
282
283 fn map(&mut self, _map: &MapType, _key_value: Self::T, _value: Self::T) -> Result<Self::T> {
284 Ok(())
285 }
286
287 fn primitive(&mut self, _p: &PrimitiveType) -> Result<Self::T> {
288 let full_name = self.field_names.iter().map(String::as_str).join(".");
289 let field_id = self.field_id;
290 if let Some(existing_field_id) = self.name_to_id.get(full_name.as_str()) {
291 return Err(invalid_data!(
292 "Invalid schema: multiple fields for name {full_name}: {field_id} and {existing_field_id}"
293 ));
294 } else {
295 self.name_to_id.insert(full_name, field_id);
296 }
297
298 Ok(())
299 }
300
301 fn variant(&mut self, _v: &VariantType) -> Result<Self::T> {
302 Err(Error::new(
303 ErrorKind::FeatureUnsupported,
304 "Writing variant columns to Parquet is not supported yet",
305 ))
306 }
307}
308
309pub struct ParquetWriter {
311 schema: SchemaRef,
312 output_file: OutputFile,
313 inner_writer: Option<AsyncArrowWriter<AsyncFileWriter>>,
314 writer_properties: WriterProperties,
315 current_row_num: usize,
316 nan_value_count_visitor: NanValueCountVisitor,
317 key_metadata: Option<StandardKeyMetadata>,
318}
319
320struct MinMaxColAggregator {
322 lower_bounds: HashMap<i32, Datum>,
323 upper_bounds: HashMap<i32, Datum>,
324 geo_bounds: HashMap<i32, (PrimitiveType, Option<GeospatialBounds>)>,
326 schema: SchemaRef,
327}
328
329struct GeospatialBounds {
330 x: WraparoundInterval,
331 y: Interval,
332 z: Interval,
333 m: Interval,
334}
335
336impl From<&BoundingBox> for GeospatialBounds {
337 fn from(bbox: &BoundingBox) -> Self {
338 Self {
339 x: WraparoundInterval::new(bbox.get_xmin(), bbox.get_xmax()),
340 y: Interval::new(bbox.get_ymin(), bbox.get_ymax()),
341 z: bbox
342 .get_zmin()
343 .zip(bbox.get_zmax())
344 .map(|(lo, hi)| Interval::new(lo, hi))
345 .unwrap_or_else(Interval::empty),
346 m: bbox
347 .get_mmin()
348 .zip(bbox.get_mmax())
349 .map(|(lo, hi)| Interval::new(lo, hi))
350 .unwrap_or_else(Interval::empty),
351 }
352 }
353}
354
355impl GeospatialBounds {
356 fn merge(&mut self, bbox: &BoundingBox) {
357 let other = Self::from(bbox);
358 self.x = self.x.merge_interval(&other.x);
359 self.y.update_interval(&other.y);
360 self.z.update_interval(&other.z);
361 self.m.update_interval(&other.m);
362 }
363
364 fn into_datums(self, ty: PrimitiveType) -> (Datum, Datum) {
365 let (xmin, xmax) = if matches!(ty, PrimitiveType::Geography(_)) {
366 (self.x.lo().max(-180.0), self.x.hi().min(180.0))
368 } else if self.x.is_wraparound() {
369 (f64::NEG_INFINITY, f64::INFINITY)
371 } else {
372 (self.x.lo(), self.x.hi())
373 };
374 let dimensions = if !self.m.is_empty() {
375 4
376 } else if !self.z.is_empty() {
377 3
378 } else {
379 2
380 };
381 let (zmin, zmax) = if self.z.is_empty() {
382 (f64::NAN, f64::NAN)
383 } else {
384 (self.z.lo(), self.z.hi())
385 };
386 let encode = |point: [f64; 4]| {
388 PrimitiveLiteral::Binary(
389 point[..dimensions]
390 .iter()
391 .flat_map(|v| v.to_le_bytes())
392 .collect(),
393 )
394 };
395 (
396 Datum::new(ty.clone(), encode([xmin, self.y.lo(), zmin, self.m.lo()])),
397 Datum::new(ty, encode([xmax, self.y.hi(), zmax, self.m.hi()])),
398 )
399 }
400}
401
402impl MinMaxColAggregator {
403 fn new(schema: SchemaRef) -> Self {
405 Self {
406 lower_bounds: HashMap::new(),
407 upper_bounds: HashMap::new(),
408 geo_bounds: HashMap::new(),
409 schema,
410 }
411 }
412
413 fn update_state_min(&mut self, field_id: i32, datum: Datum) {
414 self.lower_bounds
415 .entry(field_id)
416 .and_modify(|e| {
417 if *e > datum {
418 *e = datum.clone()
419 }
420 })
421 .or_insert(datum);
422 }
423
424 fn update_state_max(&mut self, field_id: i32, datum: Datum) {
425 self.upper_bounds
426 .entry(field_id)
427 .and_modify(|e| {
428 if *e < datum {
429 *e = datum.clone()
430 }
431 })
432 .or_insert(datum);
433 }
434
435 fn update(&mut self, field_id: i32, column: &ColumnChunkMetaData) -> Result<()> {
437 let Some(ty) = self
438 .schema
439 .field_by_id(field_id)
440 .map(|f| f.field_type.as_ref())
441 else {
442 return Ok(());
445 };
446 let Type::Primitive(ty) = ty else {
447 return Err(Error::new(
448 ErrorKind::Unexpected,
449 format!("Composed type {ty} is not supported for min max aggregation."),
450 ));
451 };
452
453 match ty {
454 PrimitiveType::Geometry(_) | PrimitiveType::Geography(_) => {
455 if column.num_values() == 0
457 || column
458 .statistics()
459 .and_then(Statistics::null_count_opt)
460 .is_some_and(|nulls| nulls == column.num_values() as u64)
461 {
462 return Ok(());
463 }
464 let bbox = column
466 .geo_statistics()
467 .and_then(|stats| stats.bounding_box());
468 self.geo_bounds
469 .entry(field_id)
470 .and_modify(|(_, state)| {
471 if let (Some(bounds), Some(bbox)) = (state.as_mut(), bbox) {
472 bounds.merge(bbox);
473 } else {
474 *state = None;
475 }
476 })
477 .or_insert_with(|| (ty.clone(), bbox.map(GeospatialBounds::from)));
478 return Ok(());
479 }
480 _ => {}
481 }
482 let Some(value) = column.statistics() else {
483 return Ok(());
484 };
485 let ty = ty.clone();
486
487 if value.min_is_exact() {
488 let Some(min_datum) = get_parquet_stat_min_as_datum(&ty, value)? else {
489 return Err(Error::new(
490 ErrorKind::Unexpected,
491 format!("Statistics {value} is not match with field type {ty}."),
492 ));
493 };
494
495 if !min_datum.is_nan() {
497 self.update_state_min(field_id, min_datum);
498 }
499 }
500
501 if value.max_is_exact() {
502 let Some(max_datum) = get_parquet_stat_max_as_datum(&ty, value)? else {
503 return Err(Error::new(
504 ErrorKind::Unexpected,
505 format!("Statistics {value} is not match with field type {ty}."),
506 ));
507 };
508
509 if !max_datum.is_nan() {
510 self.update_state_max(field_id, max_datum);
511 }
512 }
513
514 Ok(())
515 }
516
517 fn produce(mut self) -> (HashMap<i32, Datum>, HashMap<i32, Datum>) {
519 for (field_id, (ty, bounds)) in self.geo_bounds {
520 if let Some(bounds) = bounds {
521 let (lower, upper) = bounds.into_datums(ty);
522 self.lower_bounds.insert(field_id, lower);
523 self.upper_bounds.insert(field_id, upper);
524 }
525 }
526 (self.lower_bounds, self.upper_bounds)
527 }
528}
529
530impl ParquetWriter {
531 #[allow(dead_code)]
533 pub(crate) async fn parquet_files_to_data_files(
534 file_io: &FileIO,
535 file_paths: Vec<String>,
536 table_metadata: &TableMetadata,
537 ) -> Result<Vec<DataFile>> {
538 let mut data_files: Vec<DataFile> = Vec::new();
540
541 for file_path in file_paths {
542 let input_file = file_io.new_input(&file_path)?;
543 let file_metadata = input_file.metadata().await?;
544 let file_size_in_bytes = file_metadata.size as usize;
545 let reader = input_file.reader().await?;
546
547 let mut parquet_reader = ArrowFileReader::new(file_metadata, reader);
548 let parquet_metadata = parquet_reader
549 .get_metadata(None)
550 .await
551 .map_err(|err| invalid_data!("Error reading Parquet metadata: {err}"))?;
552 let mut builder = ParquetWriter::parquet_to_data_file_builder(
553 table_metadata.current_schema().clone(),
554 parquet_metadata,
555 file_size_in_bytes,
556 file_path,
557 HashMap::new(),
559 None,
560 )?;
561 builder.partition_spec_id(table_metadata.default_partition_spec_id());
562 let data_file = builder.build().unwrap();
563 data_files.push(data_file);
564 }
565
566 Ok(data_files)
567 }
568
569 pub(crate) fn parquet_to_data_file_builder(
571 schema: SchemaRef,
572 metadata: Arc<ParquetMetaData>,
573 written_size: usize,
574 file_path: String,
575 nan_value_counts: HashMap<i32, u64>,
576 key_metadata: Option<StandardKeyMetadata>,
577 ) -> Result<DataFileBuilder> {
578 let index_by_parquet_path = {
579 let mut visitor = IndexByParquetPathName::new();
580 visit_schema(&schema, &mut visitor)?;
581 visitor
582 };
583
584 let (column_sizes, value_counts, null_value_counts, (lower_bounds, upper_bounds)) = {
585 let mut per_col_size: HashMap<i32, u64> = HashMap::new();
586 let mut per_col_val_num: HashMap<i32, u64> = HashMap::new();
587 let mut per_col_null_val_num: HashMap<i32, u64> = HashMap::new();
588 let mut min_max_agg = MinMaxColAggregator::new(schema);
589
590 for row_group in metadata.row_groups() {
591 for column_chunk_metadata in row_group.columns() {
592 let parquet_path = column_chunk_metadata.column_descr().path().string();
593
594 let Some(&field_id) = index_by_parquet_path.get(&parquet_path) else {
595 continue;
596 };
597
598 *per_col_size.entry(field_id).or_insert(0) +=
599 column_chunk_metadata.compressed_size() as u64;
600 *per_col_val_num.entry(field_id).or_insert(0) +=
601 column_chunk_metadata.num_values() as u64;
602
603 min_max_agg.update(field_id, column_chunk_metadata)?;
604
605 if let Some(null_count) = column_chunk_metadata
606 .statistics()
607 .and_then(Statistics::null_count_opt)
608 {
609 *per_col_null_val_num.entry(field_id).or_insert(0) += null_count;
610 }
611 }
612 }
613 (
614 per_col_size,
615 per_col_val_num,
616 per_col_null_val_num,
617 min_max_agg.produce(),
618 )
619 };
620
621 let key_metadata = match key_metadata {
622 Some(m) => Some(m.encode()?.into_vec()),
623 None => None,
624 };
625
626 let mut builder = DataFileBuilder::default();
627 builder
628 .content(DataContentType::Data)
629 .file_path(file_path)
630 .file_format(DataFileFormat::Parquet)
631 .partition(Struct::empty())
632 .record_count(metadata.file_metadata().num_rows() as u64)
633 .file_size_in_bytes(written_size as u64)
634 .column_sizes(column_sizes)
635 .value_counts(value_counts)
636 .null_value_counts(null_value_counts)
637 .nan_value_counts(nan_value_counts)
638 .lower_bounds(lower_bounds)
641 .upper_bounds(upper_bounds)
642 .key_metadata(key_metadata)
643 .split_offsets(Some(
644 metadata
645 .row_groups()
646 .iter()
647 .filter_map(|group| group.file_offset())
648 .collect(),
649 ));
650
651 Ok(builder)
652 }
653
654 #[allow(dead_code)]
655 fn partition_value_from_bounds(
656 table_spec: Arc<PartitionSpec>,
657 lower_bounds: &HashMap<i32, Datum>,
658 upper_bounds: &HashMap<i32, Datum>,
659 ) -> Result<Struct> {
660 let mut partition_literals: Vec<Option<Literal>> = Vec::new();
661
662 for field in table_spec.fields() {
663 if let (Some(lower), Some(upper)) = (
664 lower_bounds.get(&field.source_id),
665 upper_bounds.get(&field.source_id),
666 ) {
667 if !field.transform.preserves_order() {
668 return Err(invalid_data!(
669 "cannot infer partition value for non linear partition field (needs to preserve order): {} with transform {}",
670 field.name,
671 field.transform
672 ));
673 }
674
675 if lower != upper {
676 return Err(invalid_data!(
677 "multiple partition values for field {}: lower: {:?}, upper: {:?}",
678 field.name,
679 lower,
680 upper
681 ));
682 }
683
684 let transform_fn = create_transform_function(&field.transform)?;
685 let transform_literal =
686 Literal::from(transform_fn.transform_literal_result(lower)?);
687
688 partition_literals.push(Some(transform_literal));
689 } else {
690 partition_literals.push(None);
691 }
692 }
693
694 let partition_struct = Struct::from_iter(partition_literals);
695
696 Ok(partition_struct)
697 }
698}
699
700fn resolve_writer_properties(
701 writer_properties: WriterProperties,
702 key_metadata: Option<&StandardKeyMetadata>,
703) -> Result<WriterProperties> {
704 let Some(key_metadata) = key_metadata else {
705 return Ok(writer_properties);
706 };
707
708 if writer_properties.file_encryption_properties().is_some() {
709 return Err(Error::new(
710 ErrorKind::Unexpected,
711 "Parquet writer properties already have file encryption properties set",
712 ));
713 }
714
715 let mut builder =
716 FileEncryptionProperties::builder(key_metadata.encryption_key().as_bytes().to_vec());
717 if let Some(aad) = key_metadata.aad_prefix() {
718 builder = builder.with_aad_prefix(aad.to_vec());
719 }
720 let file_encryption_properties = builder.build().map_err(|e| {
721 Error::new(
722 ErrorKind::Unexpected,
723 "Failed to build parquet file encryption properties",
724 )
725 .with_source(e)
726 })?;
727
728 Ok(writer_properties
729 .into_builder()
730 .with_file_encryption_properties(file_encryption_properties)
731 .build())
732}
733
734impl FileWriter for ParquetWriter {
735 async fn write(&mut self, batch: &arrow_array::RecordBatch) -> Result<()> {
736 if batch.num_rows() == 0 {
738 return Ok(());
739 }
740
741 self.current_row_num += batch.num_rows();
742
743 let batch_c = batch.clone();
744 self.nan_value_count_visitor
745 .compute(self.schema.clone(), batch_c)?;
746
747 let writer = if let Some(writer) = &mut self.inner_writer {
749 writer
750 } else {
751 let arrow_schema: ArrowSchemaRef = Arc::new(self.schema.as_ref().try_into()?);
752 let inner_writer = self.output_file.writer().await?;
753 let async_writer = AsyncFileWriter::new(inner_writer);
754 let writer = AsyncArrowWriter::try_new(
755 async_writer,
756 arrow_schema.clone(),
757 Some(self.writer_properties.clone()),
758 )
759 .map_err(|err| {
760 Error::new(ErrorKind::Unexpected, "Failed to build parquet writer.")
761 .with_source(err)
762 })?;
763 self.inner_writer = Some(writer);
764 self.inner_writer.as_mut().unwrap()
765 };
766
767 writer.write(batch).await.map_err(|err| {
768 Error::new(
769 ErrorKind::Unexpected,
770 "Failed to write using parquet writer.",
771 )
772 .with_source(err)
773 })?;
774
775 Ok(())
776 }
777
778 async fn close(mut self) -> Result<Vec<DataFileBuilder>> {
779 let mut writer = match self.inner_writer.take() {
780 Some(writer) => writer,
781 None => return Ok(vec![]),
782 };
783
784 let metadata = writer.finish().await.map_err(|err| {
785 Error::new(ErrorKind::Unexpected, "Failed to finish parquet writer.").with_source(err)
786 })?;
787
788 let written_size = writer.bytes_written();
789
790 if self.current_row_num == 0 {
791 self.output_file.delete().await.map_err(|err| {
792 Error::new(
793 ErrorKind::Unexpected,
794 "Failed to delete empty parquet file.",
795 )
796 .with_source(err)
797 })?;
798 Ok(vec![])
799 } else {
800 let parquet_metadata = Arc::new(metadata);
801
802 Ok(vec![Self::parquet_to_data_file_builder(
803 self.schema,
804 parquet_metadata,
805 written_size,
806 self.output_file.location().to_string(),
807 self.nan_value_count_visitor.nan_value_counts,
808 self.key_metadata,
809 )?])
810 }
811 }
812}
813
814impl CurrentFileStatus for ParquetWriter {
815 fn current_file_path(&self) -> String {
816 self.output_file.location().to_string()
817 }
818
819 fn current_row_num(&self) -> usize {
820 self.current_row_num
821 }
822
823 fn current_written_size(&self) -> usize {
824 if let Some(inner) = self.inner_writer.as_ref() {
825 inner.bytes_written() + inner.in_progress_size()
828 } else {
829 0
831 }
832 }
833}
834
835struct AsyncFileWriter(Box<dyn FileWrite>);
841
842impl AsyncFileWriter {
843 pub fn new(writer: Box<dyn FileWrite>) -> Self {
845 Self(writer)
846 }
847}
848
849impl ArrowAsyncFileWriter for AsyncFileWriter {
850 fn write(&mut self, bs: Bytes) -> BoxFuture<'_, parquet::errors::Result<()>> {
851 Box::pin(async {
852 self.0
853 .write(bs)
854 .await
855 .map_err(|err| parquet::errors::ParquetError::External(Box::new(err)))
856 })
857 }
858
859 fn complete(&mut self) -> BoxFuture<'_, parquet::errors::Result<()>> {
860 Box::pin(async {
861 self.0
863 .close()
864 .await
865 .map(|_| ())
866 .map_err(|err| parquet::errors::ParquetError::External(Box::new(err)))
867 })
868 }
869}
870
871#[cfg(test)]
872mod tests {
873 use std::collections::HashMap;
874 use std::sync::Arc;
875
876 use anyhow::Result;
877 use arrow_array::builder::{Float32Builder, Int32Builder, MapBuilder};
878 use arrow_array::types::{Float32Type, Int64Type};
879 use arrow_array::{
880 Array, ArrayRef, BooleanArray, Decimal128Array, Float32Array, Float64Array, Int32Array,
881 Int64Array, ListArray, MapArray, RecordBatch, StructArray,
882 };
883 use arrow_schema::{DataType, Field, Fields, SchemaRef as ArrowSchemaRef};
884 use arrow_select::concat::concat_batches;
885 use futures::TryStreamExt;
886 use parquet::arrow::PARQUET_FIELD_ID_META_KEY;
887 use parquet::arrow::arrow_reader::ParquetRecordBatchReaderBuilder;
888 use parquet::basic::{
889 BrotliLevel, Compression, EdgeInterpolationAlgorithm, GzipLevel, LogicalType, ZstdLevel,
890 };
891 use parquet::file::statistics::ValueStatistics;
892 use parquet::schema::types::ColumnPath;
893 use parquet_geospatial::testing::wkb_point_xy;
894 use tempfile::TempDir;
895 use uuid::Uuid;
896
897 use super::*;
898 use crate::Runtime;
899 use crate::arrow::{ArrowReaderBuilder, schema_to_arrow_schema};
900 use crate::io::FileIO;
901 use crate::scan::{FileScanTask, FileScanTaskStream};
902 use crate::spec::decimal_utils::{decimal_mantissa, decimal_new, decimal_scale};
903 use crate::spec::geospatial::EdgeInterpolationAlgorithm as IcebergEdgeInterpolationAlgorithm;
904 use crate::spec::{PrimitiveLiteral, Struct, *};
905 use crate::test_utils::make_encryption_manager;
906 use crate::writer::file_writer::location_generator::{
907 DefaultFileNameGenerator, DefaultLocationGenerator, FileNameGenerator, LocationGenerator,
908 };
909 use crate::writer::tests::check_parquet_data_file;
910
911 fn schema_for_all_type() -> Schema {
912 Schema::builder()
913 .with_schema_id(1)
914 .with_fields(vec![
915 NestedField::optional(0, "boolean", Type::Primitive(PrimitiveType::Boolean)).into(),
916 NestedField::optional(1, "int", Type::Primitive(PrimitiveType::Int)).into(),
917 NestedField::optional(2, "long", Type::Primitive(PrimitiveType::Long)).into(),
918 NestedField::optional(3, "float", Type::Primitive(PrimitiveType::Float)).into(),
919 NestedField::optional(4, "double", Type::Primitive(PrimitiveType::Double)).into(),
920 NestedField::optional(5, "string", Type::Primitive(PrimitiveType::String)).into(),
921 NestedField::optional(6, "binary", Type::Primitive(PrimitiveType::Binary)).into(),
922 NestedField::optional(7, "date", Type::Primitive(PrimitiveType::Date)).into(),
923 NestedField::optional(8, "time", Type::Primitive(PrimitiveType::Time)).into(),
924 NestedField::optional(9, "timestamp", Type::Primitive(PrimitiveType::Timestamp))
925 .into(),
926 NestedField::optional(
927 10,
928 "timestamptz",
929 Type::Primitive(PrimitiveType::Timestamptz),
930 )
931 .into(),
932 NestedField::optional(
933 11,
934 "timestamp_ns",
935 Type::Primitive(PrimitiveType::TimestampNs),
936 )
937 .into(),
938 NestedField::optional(
939 12,
940 "timestamptz_ns",
941 Type::Primitive(PrimitiveType::TimestamptzNs),
942 )
943 .into(),
944 NestedField::optional(
945 13,
946 "decimal",
947 Type::Primitive(PrimitiveType::Decimal {
948 precision: 10,
949 scale: 5,
950 }),
951 )
952 .into(),
953 NestedField::optional(14, "uuid", Type::Primitive(PrimitiveType::Uuid)).into(),
954 NestedField::optional(15, "fixed", Type::Primitive(PrimitiveType::Fixed(10)))
955 .into(),
956 NestedField::optional(
959 16,
960 "decimal_38",
961 Type::Primitive(PrimitiveType::Decimal {
962 precision: 38,
963 scale: 5,
964 }),
965 )
966 .into(),
967 ])
968 .build()
969 .unwrap()
970 }
971
972 fn nested_schema_for_test() -> Schema {
973 Schema::builder()
975 .with_schema_id(1)
976 .with_fields(vec![
977 NestedField::required(0, "col0", Type::Primitive(PrimitiveType::Long)).into(),
978 NestedField::required(
979 1,
980 "col1",
981 Type::Struct(StructType::new(vec![
982 NestedField::required(5, "col_1_5", Type::Primitive(PrimitiveType::Long))
983 .into(),
984 NestedField::required(6, "col_1_6", Type::Primitive(PrimitiveType::Long))
985 .into(),
986 ])),
987 )
988 .into(),
989 NestedField::required(2, "col2", Type::Primitive(PrimitiveType::String)).into(),
990 NestedField::required(
991 3,
992 "col3",
993 Type::List(ListType::new(
994 NestedField::required(7, "element", Type::Primitive(PrimitiveType::Long))
995 .into(),
996 )),
997 )
998 .into(),
999 NestedField::required(
1000 4,
1001 "col4",
1002 Type::Struct(StructType::new(vec![
1003 NestedField::required(
1004 8,
1005 "col_4_8",
1006 Type::Struct(StructType::new(vec![
1007 NestedField::required(
1008 9,
1009 "col_4_8_9",
1010 Type::Primitive(PrimitiveType::Long),
1011 )
1012 .into(),
1013 ])),
1014 )
1015 .into(),
1016 ])),
1017 )
1018 .into(),
1019 NestedField::required(
1020 10,
1021 "col5",
1022 Type::Map(MapType::new(
1023 NestedField::required(11, "key", Type::Primitive(PrimitiveType::String))
1024 .into(),
1025 NestedField::required(
1026 12,
1027 "value",
1028 Type::List(ListType::new(
1029 NestedField::required(
1030 13,
1031 "item",
1032 Type::Primitive(PrimitiveType::Long),
1033 )
1034 .into(),
1035 )),
1036 )
1037 .into(),
1038 )),
1039 )
1040 .into(),
1041 ])
1042 .build()
1043 .unwrap()
1044 }
1045
1046 #[tokio::test]
1047 async fn test_index_by_parquet_path() {
1048 let expect = HashMap::from([
1049 ("col0".to_string(), 0),
1050 ("col1.col_1_5".to_string(), 5),
1051 ("col1.col_1_6".to_string(), 6),
1052 ("col2".to_string(), 2),
1053 ("col3.list.element".to_string(), 7),
1054 ("col4.col_4_8.col_4_8_9".to_string(), 9),
1055 ("col5.key_value.key".to_string(), 11),
1056 ("col5.key_value.value.list.item".to_string(), 13),
1057 ]);
1058 let mut visitor = IndexByParquetPathName::new();
1059 visit_schema(&nested_schema_for_test(), &mut visitor).unwrap();
1060 assert_eq!(visitor.name_to_id, expect);
1061 }
1062
1063 #[test]
1064 fn test_index_by_parquet_path_variant_is_unsupported() {
1065 let schema = Schema::builder()
1068 .with_fields(vec![
1069 NestedField::optional(1, "v", Type::Variant(VariantType)).into(),
1070 ])
1071 .build()
1072 .unwrap();
1073 let mut visitor = IndexByParquetPathName::new();
1074 let err = visit_schema(&schema, &mut visitor).unwrap_err();
1075 assert_eq!(err.kind(), ErrorKind::FeatureUnsupported, "{err}");
1076 }
1077
1078 #[tokio::test]
1079 async fn test_parquet_writer() -> Result<()> {
1080 let temp_dir = TempDir::new().unwrap();
1081 let file_io = FileIO::new_with_fs();
1082 let location_gen = DefaultLocationGenerator::with_data_location(
1083 temp_dir.path().to_str().unwrap().to_string(),
1084 );
1085 let file_name_gen =
1086 DefaultFileNameGenerator::new("test".to_string(), None, DataFileFormat::Parquet);
1087
1088 let schema = {
1090 let fields =
1091 vec![
1092 Field::new("col", DataType::Int64, true).with_metadata(HashMap::from([(
1093 PARQUET_FIELD_ID_META_KEY.to_string(),
1094 "0".to_string(),
1095 )])),
1096 ];
1097 Arc::new(arrow_schema::Schema::new(fields))
1098 };
1099 let col = Arc::new(Int64Array::from_iter_values(0..1024)) as ArrayRef;
1100 let null_col = Arc::new(Int64Array::new_null(1024)) as ArrayRef;
1101 let to_write = RecordBatch::try_new(schema.clone(), vec![col]).unwrap();
1102 let to_write_null = RecordBatch::try_new(schema.clone(), vec![null_col]).unwrap();
1103
1104 let output_file = file_io.new_output(
1105 location_gen.generate_location(None, &file_name_gen.generate_file_name()),
1106 )?;
1107
1108 let mut pw = ParquetWriterBuilder::new(
1110 WriterProperties::builder()
1111 .set_max_row_group_row_count(Some(128))
1112 .build(),
1113 Arc::new(to_write.schema().as_ref().try_into().unwrap()),
1114 )
1115 .build(output_file)
1116 .await?;
1117 pw.write(&to_write).await?;
1118 pw.write(&to_write_null).await?;
1119 let res = pw.close().await?;
1120 assert_eq!(res.len(), 1);
1121 let data_file = res
1122 .into_iter()
1123 .next()
1124 .unwrap()
1125 .content(DataContentType::Data)
1127 .partition(Struct::empty())
1128 .partition_spec_id(0)
1129 .build()
1130 .unwrap();
1131
1132 assert_eq!(data_file.record_count(), 2048);
1134 assert_eq!(*data_file.value_counts(), HashMap::from([(0, 2048)]));
1135 assert_eq!(
1136 *data_file.lower_bounds(),
1137 HashMap::from([(0, Datum::long(0))])
1138 );
1139 assert_eq!(
1140 *data_file.upper_bounds(),
1141 HashMap::from([(0, Datum::long(1023))])
1142 );
1143 assert_eq!(*data_file.null_value_counts(), HashMap::from([(0, 1024)]));
1144
1145 let expect_batch = concat_batches(&schema, vec![&to_write, &to_write_null]).unwrap();
1147 check_parquet_data_file(&file_io, &data_file, &expect_batch).await;
1148
1149 Ok(())
1150 }
1151
1152 #[tokio::test]
1153 async fn test_parquet_writer_encrypted_roundtrip() -> Result<()> {
1154 let temp_dir = TempDir::new().unwrap();
1155 let file_io = FileIO::new_with_fs();
1156 let location_gen = DefaultLocationGenerator::with_data_location(
1157 temp_dir.path().to_str().unwrap().to_string(),
1158 );
1159 let file_name_gen =
1160 DefaultFileNameGenerator::new("test".to_string(), None, DataFileFormat::Parquet);
1161
1162 let arrow_schema = Arc::new(arrow_schema::Schema::new(vec![
1163 Field::new("id", DataType::Int32, false).with_metadata(HashMap::from([(
1164 PARQUET_FIELD_ID_META_KEY.to_string(),
1165 "1".to_string(),
1166 )])),
1167 ]));
1168 let iceberg_schema: SchemaRef = Arc::new(arrow_schema.as_ref().try_into().unwrap());
1169 let batch = RecordBatch::try_new(arrow_schema, vec![Arc::new(Int32Array::from(vec![
1170 10, 20, 30,
1171 ])) as ArrayRef])
1172 .unwrap();
1173
1174 let output_file = file_io.new_output(
1175 location_gen.generate_location(None, &file_name_gen.generate_file_name()),
1176 )?;
1177 let raw_properties = HashMap::from([(
1178 TableProperties::PROPERTY_ENCRYPTION_KEY_ID.to_string(),
1179 "test-key".to_string(),
1180 )]);
1181 let table_properties = TableProperties::new(&raw_properties);
1182 let mut parquet_writer =
1183 ParquetWriterBuilder::from_table_properties(&table_properties, iceberg_schema.clone())?
1184 .with_encryption_manager(make_encryption_manager("test-key"))
1185 .build(output_file)
1186 .await?;
1187 parquet_writer.write(&batch).await?;
1188 let data_file = parquet_writer
1189 .close()
1190 .await?
1191 .into_iter()
1192 .next()
1193 .unwrap()
1194 .content(DataContentType::Data)
1195 .partition(Struct::empty())
1196 .partition_spec_id(0)
1197 .build()
1198 .unwrap();
1199
1200 let raw = file_io
1201 .new_input(data_file.file_path.clone())?
1202 .read()
1203 .await?;
1204 assert!(
1205 ParquetRecordBatchReaderBuilder::try_new(raw).is_err(),
1206 "an encrypted parquet file must not be readable without decryption"
1207 );
1208
1209 let reader = ArrowReaderBuilder::new(file_io, Runtime::current()).build();
1210 let task = FileScanTask::builder()
1211 .with_file_size_in_bytes(data_file.file_size_in_bytes())
1212 .with_start(0)
1213 .with_length(0)
1214 .with_data_file_path(data_file.file_path.clone())
1215 .with_data_file_format(DataFileFormat::Parquet)
1216 .with_schema(iceberg_schema)
1217 .with_project_field_ids(vec![1])
1218 .with_case_sensitive(false)
1219 .with_key_metadata(data_file.key_metadata().map(Box::from))
1220 .build()
1221 .unwrap();
1222 let tasks = Box::pin(futures::stream::iter(vec![Ok(task)])) as FileScanTaskStream;
1223 let batches: Vec<RecordBatch> = reader
1224 .read(tasks)
1225 .unwrap()
1226 .stream()
1227 .try_collect()
1228 .await
1229 .unwrap();
1230
1231 assert_eq!(batches.len(), 1);
1232 let ids = batches[0]
1233 .column(0)
1234 .as_any()
1235 .downcast_ref::<Int32Array>()
1236 .unwrap();
1237 assert_eq!(ids.values(), &[10, 20, 30]);
1238
1239 Ok(())
1240 }
1241
1242 #[tokio::test]
1243 async fn test_parquet_writer_with_complex_schema() -> Result<()> {
1244 let temp_dir = TempDir::new().unwrap();
1245 let file_io = FileIO::new_with_fs();
1246 let location_gen = DefaultLocationGenerator::with_data_location(
1247 temp_dir.path().to_str().unwrap().to_string(),
1248 );
1249 let file_name_gen =
1250 DefaultFileNameGenerator::new("test".to_string(), None, DataFileFormat::Parquet);
1251
1252 let schema = nested_schema_for_test();
1254 let arrow_schema: ArrowSchemaRef = Arc::new((&schema).try_into().unwrap());
1255 let col0 = Arc::new(Int64Array::from_iter_values(0..1024)) as ArrayRef;
1256 let col1 = Arc::new(StructArray::new(
1257 {
1258 if let DataType::Struct(fields) = arrow_schema.field(1).data_type() {
1259 fields.clone()
1260 } else {
1261 unreachable!()
1262 }
1263 },
1264 vec![
1265 Arc::new(Int64Array::from_iter_values(0..1024)),
1266 Arc::new(Int64Array::from_iter_values(0..1024)),
1267 ],
1268 None,
1269 ));
1270 let col2 = Arc::new(arrow_array::StringArray::from_iter_values(
1271 (0..1024).map(|n| n.to_string()),
1272 )) as ArrayRef;
1273 let col3 = Arc::new({
1274 let list_parts = ListArray::from_iter_primitive::<Int64Type, _, _>(
1275 (0..1024).map(|n| Some(vec![Some(n)])),
1276 )
1277 .into_parts();
1278 ListArray::new(
1279 {
1280 if let DataType::List(field) = arrow_schema.field(3).data_type() {
1281 field.clone()
1282 } else {
1283 unreachable!()
1284 }
1285 },
1286 list_parts.1,
1287 list_parts.2,
1288 list_parts.3,
1289 )
1290 }) as ArrayRef;
1291 let col4 = Arc::new(StructArray::new(
1292 {
1293 if let DataType::Struct(fields) = arrow_schema.field(4).data_type() {
1294 fields.clone()
1295 } else {
1296 unreachable!()
1297 }
1298 },
1299 vec![Arc::new(StructArray::new(
1300 {
1301 if let DataType::Struct(fields) = arrow_schema.field(4).data_type() {
1302 if let DataType::Struct(fields) = fields[0].data_type() {
1303 fields.clone()
1304 } else {
1305 unreachable!()
1306 }
1307 } else {
1308 unreachable!()
1309 }
1310 },
1311 vec![Arc::new(Int64Array::from_iter_values(0..1024))],
1312 None,
1313 ))],
1314 None,
1315 ));
1316 let col5 = Arc::new({
1317 let mut map_array_builder = MapBuilder::new(
1318 None,
1319 arrow_array::builder::StringBuilder::new(),
1320 arrow_array::builder::ListBuilder::new(arrow_array::builder::PrimitiveBuilder::<
1321 Int64Type,
1322 >::new()),
1323 );
1324 for i in 0..1024 {
1325 map_array_builder.keys().append_value(i.to_string());
1326 map_array_builder
1327 .values()
1328 .append_value(vec![Some(i as i64); i + 1]);
1329 map_array_builder.append(true)?;
1330 }
1331 let (_, offset_buffer, struct_array, null_buffer, ordered) =
1332 map_array_builder.finish().into_parts();
1333 let struct_array = {
1334 let (_, mut arrays, nulls) = struct_array.into_parts();
1335 let list_array = {
1336 let list_array = arrays[1]
1337 .as_any()
1338 .downcast_ref::<ListArray>()
1339 .unwrap()
1340 .clone();
1341 let (_, offsets, array, nulls) = list_array.into_parts();
1342 let list_field = {
1343 if let DataType::Map(map_field, _) = arrow_schema.field(5).data_type() {
1344 if let DataType::Struct(fields) = map_field.data_type() {
1345 if let DataType::List(list_field) = fields[1].data_type() {
1346 list_field.clone()
1347 } else {
1348 unreachable!()
1349 }
1350 } else {
1351 unreachable!()
1352 }
1353 } else {
1354 unreachable!()
1355 }
1356 };
1357 ListArray::new(list_field, offsets, array, nulls)
1358 };
1359 arrays[1] = Arc::new(list_array) as ArrayRef;
1360 StructArray::new(
1361 {
1362 if let DataType::Map(map_field, _) = arrow_schema.field(5).data_type() {
1363 if let DataType::Struct(fields) = map_field.data_type() {
1364 fields.clone()
1365 } else {
1366 unreachable!()
1367 }
1368 } else {
1369 unreachable!()
1370 }
1371 },
1372 arrays,
1373 nulls,
1374 )
1375 };
1376 MapArray::new(
1377 {
1378 if let DataType::Map(map_field, _) = arrow_schema.field(5).data_type() {
1379 map_field.clone()
1380 } else {
1381 unreachable!()
1382 }
1383 },
1384 offset_buffer,
1385 struct_array,
1386 null_buffer,
1387 ordered,
1388 )
1389 }) as ArrayRef;
1390 let to_write = RecordBatch::try_new(arrow_schema.clone(), vec![
1391 col0, col1, col2, col3, col4, col5,
1392 ])
1393 .unwrap();
1394 let output_file = file_io.new_output(
1395 location_gen.generate_location(None, &file_name_gen.generate_file_name()),
1396 )?;
1397
1398 let mut pw =
1400 ParquetWriterBuilder::new(WriterProperties::builder().build(), Arc::new(schema))
1401 .build(output_file)
1402 .await?;
1403 pw.write(&to_write).await?;
1404 let res = pw.close().await?;
1405 assert_eq!(res.len(), 1);
1406 let data_file = res
1407 .into_iter()
1408 .next()
1409 .unwrap()
1410 .content(DataContentType::Data)
1412 .partition(Struct::empty())
1413 .partition_spec_id(0)
1414 .build()
1415 .unwrap();
1416
1417 assert_eq!(data_file.record_count(), 1024);
1419 assert_eq!(
1420 *data_file.value_counts(),
1421 HashMap::from([
1422 (0, 1024),
1423 (5, 1024),
1424 (6, 1024),
1425 (2, 1024),
1426 (7, 1024),
1427 (9, 1024),
1428 (11, 1024),
1429 (13, (1..1025).sum()),
1430 ])
1431 );
1432 assert_eq!(
1433 *data_file.lower_bounds(),
1434 HashMap::from([
1435 (0, Datum::long(0)),
1436 (5, Datum::long(0)),
1437 (6, Datum::long(0)),
1438 (2, Datum::string("0")),
1439 (7, Datum::long(0)),
1440 (9, Datum::long(0)),
1441 (11, Datum::string("0")),
1442 (13, Datum::long(0))
1443 ])
1444 );
1445 assert_eq!(
1446 *data_file.upper_bounds(),
1447 HashMap::from([
1448 (0, Datum::long(1023)),
1449 (5, Datum::long(1023)),
1450 (6, Datum::long(1023)),
1451 (2, Datum::string("999")),
1452 (7, Datum::long(1023)),
1453 (9, Datum::long(1023)),
1454 (11, Datum::string("999")),
1455 (13, Datum::long(1023))
1456 ])
1457 );
1458
1459 check_parquet_data_file(&file_io, &data_file, &to_write).await;
1461
1462 Ok(())
1463 }
1464
1465 #[tokio::test]
1466 async fn test_all_type_for_write() -> Result<()> {
1467 let temp_dir = TempDir::new().unwrap();
1468 let file_io = FileIO::new_with_fs();
1469 let location_gen = DefaultLocationGenerator::with_data_location(
1470 temp_dir.path().to_str().unwrap().to_string(),
1471 );
1472 let file_name_gen =
1473 DefaultFileNameGenerator::new("test".to_string(), None, DataFileFormat::Parquet);
1474
1475 let schema = schema_for_all_type();
1478 let arrow_schema: ArrowSchemaRef = Arc::new((&schema).try_into().unwrap());
1479 let col0 = Arc::new(BooleanArray::from(vec![
1480 Some(true),
1481 Some(false),
1482 None,
1483 Some(true),
1484 ])) as ArrayRef;
1485 let col1 = Arc::new(Int32Array::from(vec![Some(1), Some(2), None, Some(4)])) as ArrayRef;
1486 let col2 = Arc::new(Int64Array::from(vec![Some(1), Some(2), None, Some(4)])) as ArrayRef;
1487 let col3 = Arc::new(Float32Array::from(vec![
1488 Some(0.5),
1489 Some(2.0),
1490 None,
1491 Some(3.5),
1492 ])) as ArrayRef;
1493 let col4 = Arc::new(Float64Array::from(vec![
1494 Some(0.5),
1495 Some(2.0),
1496 None,
1497 Some(3.5),
1498 ])) as ArrayRef;
1499 let col5 = Arc::new(arrow_array::StringArray::from(vec![
1500 Some("a"),
1501 Some("b"),
1502 None,
1503 Some("d"),
1504 ])) as ArrayRef;
1505 let col6 = Arc::new(arrow_array::BinaryArray::from_opt_vec(vec![
1506 Some(b"one"),
1507 None,
1508 Some(b""),
1509 Some(b"zzzz"),
1510 ])) as ArrayRef;
1511 let col7 = Arc::new(arrow_array::Date32Array::from(vec![
1512 Some(0),
1513 Some(1),
1514 None,
1515 Some(3),
1516 ])) as ArrayRef;
1517 let col8 = Arc::new(arrow_array::Time64MicrosecondArray::from(vec![
1518 Some(0),
1519 Some(1),
1520 None,
1521 Some(3),
1522 ])) as ArrayRef;
1523 let col9 = Arc::new(arrow_array::TimestampMicrosecondArray::from(vec![
1524 Some(0),
1525 Some(1),
1526 None,
1527 Some(3),
1528 ])) as ArrayRef;
1529 let col10 = Arc::new(
1530 arrow_array::TimestampMicrosecondArray::from(vec![Some(0), Some(1), None, Some(3)])
1531 .with_timezone_utc(),
1532 ) as ArrayRef;
1533 let col11 = Arc::new(arrow_array::TimestampNanosecondArray::from(vec![
1534 Some(0),
1535 Some(1),
1536 None,
1537 Some(3),
1538 ])) as ArrayRef;
1539 let col12 = Arc::new(
1540 arrow_array::TimestampNanosecondArray::from(vec![Some(0), Some(1), None, Some(3)])
1541 .with_timezone_utc(),
1542 ) as ArrayRef;
1543 let col13 = Arc::new(
1544 Decimal128Array::from(vec![Some(1), Some(2), None, Some(100)])
1545 .with_precision_and_scale(10, 5)
1546 .unwrap(),
1547 ) as ArrayRef;
1548 let col14 = Arc::new(
1549 arrow_array::FixedSizeBinaryArray::try_from_sparse_iter_with_size(
1550 vec![
1551 Some(Uuid::from_u128(0).as_bytes().to_vec()),
1552 Some(Uuid::from_u128(1).as_bytes().to_vec()),
1553 None,
1554 Some(Uuid::from_u128(3).as_bytes().to_vec()),
1555 ]
1556 .into_iter(),
1557 16,
1558 )
1559 .unwrap(),
1560 ) as ArrayRef;
1561 let col15 = Arc::new(
1562 arrow_array::FixedSizeBinaryArray::try_from_sparse_iter_with_size(
1563 vec![
1564 Some(vec![1, 2, 3, 4, 5, 6, 7, 8, 9, 10]),
1565 Some(vec![11, 12, 13, 14, 15, 16, 17, 18, 19, 20]),
1566 None,
1567 Some(vec![21, 22, 23, 24, 25, 26, 27, 28, 29, 30]),
1568 ]
1569 .into_iter(),
1570 10,
1571 )
1572 .unwrap(),
1573 ) as ArrayRef;
1574 let col16 = Arc::new(
1575 Decimal128Array::from(vec![Some(1), Some(2), None, Some(100)])
1576 .with_precision_and_scale(38, 5)
1577 .unwrap(),
1578 ) as ArrayRef;
1579 let to_write = RecordBatch::try_new(arrow_schema.clone(), vec![
1580 col0, col1, col2, col3, col4, col5, col6, col7, col8, col9, col10, col11, col12, col13,
1581 col14, col15, col16,
1582 ])
1583 .unwrap();
1584 let output_file = file_io.new_output(
1585 location_gen.generate_location(None, &file_name_gen.generate_file_name()),
1586 )?;
1587
1588 let mut pw =
1590 ParquetWriterBuilder::new(WriterProperties::builder().build(), Arc::new(schema))
1591 .build(output_file)
1592 .await?;
1593 pw.write(&to_write).await?;
1594 let res = pw.close().await?;
1595 assert_eq!(res.len(), 1);
1596 let data_file = res
1597 .into_iter()
1598 .next()
1599 .unwrap()
1600 .content(DataContentType::Data)
1602 .partition(Struct::empty())
1603 .partition_spec_id(0)
1604 .build()
1605 .unwrap();
1606
1607 assert_eq!(data_file.record_count(), 4);
1609 assert!(data_file.value_counts().iter().all(|(_, &v)| { v == 4 }));
1610 assert!(
1611 data_file
1612 .null_value_counts()
1613 .iter()
1614 .all(|(_, &v)| { v == 1 })
1615 );
1616 assert_eq!(
1617 *data_file.lower_bounds(),
1618 HashMap::from([
1619 (0, Datum::bool(false)),
1620 (1, Datum::int(1)),
1621 (2, Datum::long(1)),
1622 (3, Datum::float(0.5_f32)),
1623 (4, Datum::double(0.5)),
1624 (5, Datum::string("a")),
1625 (6, Datum::binary(vec![])),
1626 (7, Datum::date(0)),
1627 (8, Datum::time_micros(0).unwrap()),
1628 (9, Datum::timestamp_micros(0)),
1629 (10, Datum::timestamptz_micros(0)),
1630 (11, Datum::timestamp_nanos(0)),
1631 (12, Datum::timestamptz_nanos(0)),
1632 (
1633 13,
1634 Datum::new(
1635 PrimitiveType::Decimal {
1636 precision: 10,
1637 scale: 5
1638 },
1639 PrimitiveLiteral::Int128(1)
1640 )
1641 ),
1642 (14, Datum::uuid(Uuid::from_u128(0))),
1643 (15, Datum::fixed(vec![1, 2, 3, 4, 5, 6, 7, 8, 9, 10])),
1644 (
1645 16,
1646 Datum::new(
1647 PrimitiveType::Decimal {
1648 precision: 38,
1649 scale: 5
1650 },
1651 PrimitiveLiteral::Int128(1)
1652 )
1653 ),
1654 ])
1655 );
1656 assert_eq!(
1657 *data_file.upper_bounds(),
1658 HashMap::from([
1659 (0, Datum::bool(true)),
1660 (1, Datum::int(4)),
1661 (2, Datum::long(4)),
1662 (3, Datum::float(3.5_f32)),
1663 (4, Datum::double(3.5)),
1664 (5, Datum::string("d")),
1665 (6, Datum::binary(vec![122, 122, 122, 122])),
1666 (7, Datum::date(3)),
1667 (8, Datum::time_micros(3).unwrap()),
1668 (9, Datum::timestamp_micros(3)),
1669 (10, Datum::timestamptz_micros(3)),
1670 (11, Datum::timestamp_nanos(3)),
1671 (12, Datum::timestamptz_nanos(3)),
1672 (
1673 13,
1674 Datum::new(
1675 PrimitiveType::Decimal {
1676 precision: 10,
1677 scale: 5
1678 },
1679 PrimitiveLiteral::Int128(100)
1680 )
1681 ),
1682 (14, Datum::uuid(Uuid::from_u128(3))),
1683 (
1684 15,
1685 Datum::fixed(vec![21, 22, 23, 24, 25, 26, 27, 28, 29, 30])
1686 ),
1687 (
1688 16,
1689 Datum::new(
1690 PrimitiveType::Decimal {
1691 precision: 38,
1692 scale: 5
1693 },
1694 PrimitiveLiteral::Int128(100)
1695 )
1696 ),
1697 ])
1698 );
1699
1700 check_parquet_data_file(&file_io, &data_file, &to_write).await;
1702
1703 Ok(())
1704 }
1705
1706 #[tokio::test]
1707 async fn test_decimal_bound() -> Result<()> {
1708 let temp_dir = TempDir::new().unwrap();
1709 let file_io = FileIO::new_with_fs();
1710 let location_gen = DefaultLocationGenerator::with_data_location(
1711 temp_dir.path().to_str().unwrap().to_string(),
1712 );
1713 let file_name_gen =
1714 DefaultFileNameGenerator::new("test".to_string(), None, DataFileFormat::Parquet);
1715
1716 let schema = Arc::new(
1718 Schema::builder()
1719 .with_fields(vec![
1720 NestedField::optional(
1721 0,
1722 "decimal",
1723 Type::Primitive(PrimitiveType::Decimal {
1724 precision: 28,
1725 scale: 10,
1726 }),
1727 )
1728 .into(),
1729 ])
1730 .build()
1731 .unwrap(),
1732 );
1733 let arrow_schema: ArrowSchemaRef = Arc::new(schema_to_arrow_schema(&schema).unwrap());
1734 let output_file = file_io.new_output(
1735 location_gen.generate_location(None, &file_name_gen.generate_file_name()),
1736 )?;
1737 let mut pw = ParquetWriterBuilder::new(WriterProperties::builder().build(), schema.clone())
1738 .build(output_file)
1739 .await?;
1740 let col0 = Arc::new(
1741 Decimal128Array::from(vec![Some(22000000000), Some(11000000000)])
1742 .with_data_type(DataType::Decimal128(28, 10)),
1743 ) as ArrayRef;
1744 let to_write = RecordBatch::try_new(arrow_schema.clone(), vec![col0]).unwrap();
1745 pw.write(&to_write).await?;
1746 let res = pw.close().await?;
1747 assert_eq!(res.len(), 1);
1748 let data_file = res
1749 .into_iter()
1750 .next()
1751 .unwrap()
1752 .content(DataContentType::Data)
1753 .partition(Struct::empty())
1754 .partition_spec_id(0)
1755 .build()
1756 .unwrap();
1757 assert_eq!(
1758 data_file.upper_bounds().get(&0),
1759 Some(Datum::decimal_with_precision(decimal_new(22000000000_i64, 10), 28).unwrap())
1760 .as_ref()
1761 );
1762 assert_eq!(
1763 data_file.lower_bounds().get(&0),
1764 Some(Datum::decimal_with_precision(decimal_new(11000000000_i64, 10), 28).unwrap())
1765 .as_ref()
1766 );
1767
1768 let schema = Arc::new(
1770 Schema::builder()
1771 .with_fields(vec![
1772 NestedField::optional(
1773 0,
1774 "decimal",
1775 Type::Primitive(PrimitiveType::Decimal {
1776 precision: 28,
1777 scale: 10,
1778 }),
1779 )
1780 .into(),
1781 ])
1782 .build()
1783 .unwrap(),
1784 );
1785 let arrow_schema: ArrowSchemaRef = Arc::new(schema_to_arrow_schema(&schema).unwrap());
1786 let output_file = file_io.new_output(
1787 location_gen.generate_location(None, &file_name_gen.generate_file_name()),
1788 )?;
1789 let mut pw = ParquetWriterBuilder::new(WriterProperties::builder().build(), schema.clone())
1790 .build(output_file)
1791 .await?;
1792 let col0 = Arc::new(
1793 Decimal128Array::from(vec![Some(-22000000000), Some(-11000000000)])
1794 .with_data_type(DataType::Decimal128(28, 10)),
1795 ) as ArrayRef;
1796 let to_write = RecordBatch::try_new(arrow_schema.clone(), vec![col0]).unwrap();
1797 pw.write(&to_write).await?;
1798 let res = pw.close().await?;
1799 assert_eq!(res.len(), 1);
1800 let data_file = res
1801 .into_iter()
1802 .next()
1803 .unwrap()
1804 .content(DataContentType::Data)
1805 .partition(Struct::empty())
1806 .partition_spec_id(0)
1807 .build()
1808 .unwrap();
1809 assert_eq!(
1810 data_file.upper_bounds().get(&0),
1811 Some(Datum::decimal_with_precision(decimal_new(-11000000000_i64, 10), 28).unwrap())
1812 .as_ref()
1813 );
1814 assert_eq!(
1815 data_file.lower_bounds().get(&0),
1816 Some(Datum::decimal_with_precision(decimal_new(-22000000000_i64, 10), 28).unwrap())
1817 .as_ref()
1818 );
1819
1820 use crate::spec::decimal_utils::decimal_from_str_exact;
1823 let decimal_max = decimal_from_str_exact("99999999999999999999999999999999999999").unwrap();
1824 let decimal_min =
1825 decimal_from_str_exact("-99999999999999999999999999999999999999").unwrap();
1826 assert_eq!(decimal_scale(&decimal_max), decimal_scale(&decimal_min));
1827 let schema = Arc::new(
1828 Schema::builder()
1829 .with_fields(vec![
1830 NestedField::optional(
1831 0,
1832 "decimal",
1833 Type::Primitive(PrimitiveType::Decimal {
1834 precision: 38,
1835 scale: decimal_scale(&decimal_max),
1836 }),
1837 )
1838 .into(),
1839 ])
1840 .build()
1841 .unwrap(),
1842 );
1843 let arrow_schema: ArrowSchemaRef = Arc::new(schema_to_arrow_schema(&schema).unwrap());
1844 let output_file = file_io.new_output(
1845 location_gen.generate_location(None, &file_name_gen.generate_file_name()),
1846 )?;
1847 let mut pw = ParquetWriterBuilder::new(WriterProperties::builder().build(), schema)
1848 .build(output_file)
1849 .await?;
1850 let col0 = Arc::new(
1851 Decimal128Array::from(vec![
1852 Some(decimal_mantissa(&decimal_max)),
1853 Some(decimal_mantissa(&decimal_min)),
1854 ])
1855 .with_data_type(DataType::Decimal128(38, 0)),
1856 ) as ArrayRef;
1857 let to_write = RecordBatch::try_new(arrow_schema.clone(), vec![col0]).unwrap();
1858 pw.write(&to_write).await?;
1859 let res = pw.close().await?;
1860 assert_eq!(res.len(), 1);
1861 let data_file = res
1862 .into_iter()
1863 .next()
1864 .unwrap()
1865 .content(DataContentType::Data)
1866 .partition(Struct::empty())
1867 .partition_spec_id(0)
1868 .build()
1869 .unwrap();
1870 assert_eq!(
1871 data_file.upper_bounds().get(&0),
1872 Some(Datum::decimal(decimal_max).unwrap()).as_ref()
1873 );
1874 assert_eq!(
1875 data_file.lower_bounds().get(&0),
1876 Some(Datum::decimal(decimal_min).unwrap()).as_ref()
1877 );
1878
1879 Ok(())
1949 }
1950
1951 #[tokio::test]
1952 async fn test_empty_write() -> Result<()> {
1953 let temp_dir = TempDir::new().unwrap();
1954 let file_io = FileIO::new_with_fs();
1955 let location_gen = DefaultLocationGenerator::with_data_location(
1956 temp_dir.path().to_str().unwrap().to_string(),
1957 );
1958 let file_name_gen =
1959 DefaultFileNameGenerator::new("test".to_string(), None, DataFileFormat::Parquet);
1960
1961 let schema = {
1963 let fields =
1964 vec![
1965 Field::new("col", DataType::Int64, true).with_metadata(HashMap::from([(
1966 PARQUET_FIELD_ID_META_KEY.to_string(),
1967 "0".to_string(),
1968 )])),
1969 ];
1970 Arc::new(arrow_schema::Schema::new(fields))
1971 };
1972 let col = Arc::new(Int64Array::from_iter_values(0..1024)) as ArrayRef;
1973 let to_write = RecordBatch::try_new(schema.clone(), vec![col]).unwrap();
1974 let file_path = location_gen.generate_location(None, &file_name_gen.generate_file_name());
1975 let output_file = file_io.new_output(&file_path)?;
1976 let mut pw = ParquetWriterBuilder::new(
1977 WriterProperties::builder().build(),
1978 Arc::new(to_write.schema().as_ref().try_into().unwrap()),
1979 )
1980 .build(output_file)
1981 .await?;
1982 pw.write(&to_write).await?;
1983 pw.close().await.unwrap();
1984 assert!(file_io.exists(&file_path).await.unwrap());
1985
1986 let file_name_gen =
1988 DefaultFileNameGenerator::new("test_empty".to_string(), None, DataFileFormat::Parquet);
1989 let file_path = location_gen.generate_location(None, &file_name_gen.generate_file_name());
1990 let output_file = file_io.new_output(&file_path)?;
1991 let pw = ParquetWriterBuilder::new(
1992 WriterProperties::builder().build(),
1993 Arc::new(to_write.schema().as_ref().try_into().unwrap()),
1994 )
1995 .build(output_file)
1996 .await?;
1997 pw.close().await.unwrap();
1998 assert!(!file_io.exists(&file_path).await.unwrap());
1999
2000 Ok(())
2001 }
2002
2003 #[tokio::test]
2004 async fn test_nan_val_cnts_primitive_type() -> Result<()> {
2005 let temp_dir = TempDir::new().unwrap();
2006 let file_io = FileIO::new_with_fs();
2007 let location_gen = DefaultLocationGenerator::with_data_location(
2008 temp_dir.path().to_str().unwrap().to_string(),
2009 );
2010 let file_name_gen =
2011 DefaultFileNameGenerator::new("test".to_string(), None, DataFileFormat::Parquet);
2012
2013 let arrow_schema = {
2015 let fields = vec![
2016 Field::new("col", DataType::Float32, false).with_metadata(HashMap::from([(
2017 PARQUET_FIELD_ID_META_KEY.to_string(),
2018 "0".to_string(),
2019 )])),
2020 Field::new("col2", DataType::Float64, false).with_metadata(HashMap::from([(
2021 PARQUET_FIELD_ID_META_KEY.to_string(),
2022 "1".to_string(),
2023 )])),
2024 ];
2025 Arc::new(arrow_schema::Schema::new(fields))
2026 };
2027
2028 let float_32_col = Arc::new(Float32Array::from_iter_values_with_nulls(
2029 [1.0_f32, f32::NAN, 2.0, 2.0].into_iter(),
2030 None,
2031 )) as ArrayRef;
2032
2033 let float_64_col = Arc::new(Float64Array::from_iter_values_with_nulls(
2034 [1.0_f64, f64::NAN, 2.0, 2.0].into_iter(),
2035 None,
2036 )) as ArrayRef;
2037
2038 let to_write =
2039 RecordBatch::try_new(arrow_schema.clone(), vec![float_32_col, float_64_col]).unwrap();
2040 let output_file = file_io.new_output(
2041 location_gen.generate_location(None, &file_name_gen.generate_file_name()),
2042 )?;
2043
2044 let mut pw = ParquetWriterBuilder::new(
2046 WriterProperties::builder().build(),
2047 Arc::new(to_write.schema().as_ref().try_into().unwrap()),
2048 )
2049 .build(output_file)
2050 .await?;
2051
2052 pw.write(&to_write).await?;
2053 let res = pw.close().await?;
2054 assert_eq!(res.len(), 1);
2055 let data_file = res
2056 .into_iter()
2057 .next()
2058 .unwrap()
2059 .content(DataContentType::Data)
2061 .partition(Struct::empty())
2062 .partition_spec_id(0)
2063 .build()
2064 .unwrap();
2065
2066 assert_eq!(data_file.record_count(), 4);
2068 assert_eq!(*data_file.value_counts(), HashMap::from([(0, 4), (1, 4)]));
2069 assert_eq!(
2070 *data_file.lower_bounds(),
2071 HashMap::from([(0, Datum::float(1.0_f32)), (1, Datum::double(1.0)),])
2072 );
2073 assert_eq!(
2074 *data_file.upper_bounds(),
2075 HashMap::from([(0, Datum::float(2.0_f32)), (1, Datum::double(2.0)),])
2076 );
2077 assert_eq!(
2078 *data_file.null_value_counts(),
2079 HashMap::from([(0, 0), (1, 0)])
2080 );
2081 assert_eq!(
2082 *data_file.nan_value_counts(),
2083 HashMap::from([(0, 1), (1, 1)])
2084 );
2085
2086 let expect_batch = concat_batches(&arrow_schema, vec![&to_write]).unwrap();
2088 check_parquet_data_file(&file_io, &data_file, &expect_batch).await;
2089
2090 Ok(())
2091 }
2092
2093 #[tokio::test]
2094 async fn test_nan_val_cnts_struct_type() -> Result<()> {
2095 let temp_dir = TempDir::new().unwrap();
2096 let file_io = FileIO::new_with_fs();
2097 let location_gen = DefaultLocationGenerator::with_data_location(
2098 temp_dir.path().to_str().unwrap().to_string(),
2099 );
2100 let file_name_gen =
2101 DefaultFileNameGenerator::new("test".to_string(), None, DataFileFormat::Parquet);
2102
2103 let schema_struct_float_fields = Fields::from(vec![
2104 Field::new("col4", DataType::Float32, false).with_metadata(HashMap::from([(
2105 PARQUET_FIELD_ID_META_KEY.to_string(),
2106 "4".to_string(),
2107 )])),
2108 ]);
2109
2110 let schema_struct_nested_float_fields = Fields::from(vec![
2111 Field::new("col7", DataType::Float32, false).with_metadata(HashMap::from([(
2112 PARQUET_FIELD_ID_META_KEY.to_string(),
2113 "7".to_string(),
2114 )])),
2115 ]);
2116
2117 let schema_struct_nested_fields = Fields::from(vec![
2118 Field::new(
2119 "col6",
2120 DataType::Struct(schema_struct_nested_float_fields.clone()),
2121 false,
2122 )
2123 .with_metadata(HashMap::from([(
2124 PARQUET_FIELD_ID_META_KEY.to_string(),
2125 "6".to_string(),
2126 )])),
2127 ]);
2128
2129 let arrow_schema = {
2131 let fields = vec![
2132 Field::new(
2133 "col3",
2134 DataType::Struct(schema_struct_float_fields.clone()),
2135 false,
2136 )
2137 .with_metadata(HashMap::from([(
2138 PARQUET_FIELD_ID_META_KEY.to_string(),
2139 "3".to_string(),
2140 )])),
2141 Field::new(
2142 "col5",
2143 DataType::Struct(schema_struct_nested_fields.clone()),
2144 false,
2145 )
2146 .with_metadata(HashMap::from([(
2147 PARQUET_FIELD_ID_META_KEY.to_string(),
2148 "5".to_string(),
2149 )])),
2150 ];
2151 Arc::new(arrow_schema::Schema::new(fields))
2152 };
2153
2154 let float_32_col = Arc::new(Float32Array::from_iter_values_with_nulls(
2155 [1.0_f32, f32::NAN, 2.0, 2.0].into_iter(),
2156 None,
2157 )) as ArrayRef;
2158
2159 let struct_float_field_col = Arc::new(StructArray::new(
2160 schema_struct_float_fields,
2161 vec![float_32_col.clone()],
2162 None,
2163 )) as ArrayRef;
2164
2165 let struct_nested_float_field_col = Arc::new(StructArray::new(
2166 schema_struct_nested_fields,
2167 vec![Arc::new(StructArray::new(
2168 schema_struct_nested_float_fields,
2169 vec![float_32_col.clone()],
2170 None,
2171 )) as ArrayRef],
2172 None,
2173 )) as ArrayRef;
2174
2175 let to_write = RecordBatch::try_new(arrow_schema.clone(), vec![
2176 struct_float_field_col,
2177 struct_nested_float_field_col,
2178 ])
2179 .unwrap();
2180 let output_file = file_io.new_output(
2181 location_gen.generate_location(None, &file_name_gen.generate_file_name()),
2182 )?;
2183
2184 let mut pw = ParquetWriterBuilder::new(
2186 WriterProperties::builder().build(),
2187 Arc::new(to_write.schema().as_ref().try_into().unwrap()),
2188 )
2189 .build(output_file)
2190 .await?;
2191
2192 pw.write(&to_write).await?;
2193 let res = pw.close().await?;
2194 assert_eq!(res.len(), 1);
2195 let data_file = res
2196 .into_iter()
2197 .next()
2198 .unwrap()
2199 .content(DataContentType::Data)
2201 .partition(Struct::empty())
2202 .partition_spec_id(0)
2203 .build()
2204 .unwrap();
2205
2206 assert_eq!(data_file.record_count(), 4);
2208 assert_eq!(*data_file.value_counts(), HashMap::from([(4, 4), (7, 4)]));
2209 assert_eq!(
2210 *data_file.lower_bounds(),
2211 HashMap::from([(4, Datum::float(1.0_f32)), (7, Datum::float(1.0_f32)),])
2212 );
2213 assert_eq!(
2214 *data_file.upper_bounds(),
2215 HashMap::from([(4, Datum::float(2.0_f32)), (7, Datum::float(2.0_f32)),])
2216 );
2217 assert_eq!(
2218 *data_file.null_value_counts(),
2219 HashMap::from([(4, 0), (7, 0)])
2220 );
2221 assert_eq!(
2222 *data_file.nan_value_counts(),
2223 HashMap::from([(4, 1), (7, 1)])
2224 );
2225
2226 let expect_batch = concat_batches(&arrow_schema, vec![&to_write]).unwrap();
2228 check_parquet_data_file(&file_io, &data_file, &expect_batch).await;
2229
2230 Ok(())
2231 }
2232
2233 #[tokio::test]
2234 async fn test_nan_val_cnts_list_type() -> Result<()> {
2235 let temp_dir = TempDir::new().unwrap();
2236 let file_io = FileIO::new_with_fs();
2237 let location_gen = DefaultLocationGenerator::with_data_location(
2238 temp_dir.path().to_str().unwrap().to_string(),
2239 );
2240 let file_name_gen =
2241 DefaultFileNameGenerator::new("test".to_string(), None, DataFileFormat::Parquet);
2242
2243 let schema_list_float_field = Field::new("element", DataType::Float32, true).with_metadata(
2244 HashMap::from([(PARQUET_FIELD_ID_META_KEY.to_string(), "1".to_string())]),
2245 );
2246
2247 let schema_struct_list_float_field = Field::new("element", DataType::Float32, true)
2248 .with_metadata(HashMap::from([(
2249 PARQUET_FIELD_ID_META_KEY.to_string(),
2250 "4".to_string(),
2251 )]));
2252
2253 let schema_struct_list_field = Fields::from(vec![
2254 Field::new_list("col2", schema_struct_list_float_field.clone(), true).with_metadata(
2255 HashMap::from([(PARQUET_FIELD_ID_META_KEY.to_string(), "3".to_string())]),
2256 ),
2257 ]);
2258
2259 let arrow_schema = {
2260 let fields = vec![
2261 Field::new_list("col0", schema_list_float_field.clone(), true).with_metadata(
2262 HashMap::from([(PARQUET_FIELD_ID_META_KEY.to_string(), "0".to_string())]),
2263 ),
2264 Field::new_struct("col1", schema_struct_list_field.clone(), true)
2265 .with_metadata(HashMap::from([(
2266 PARQUET_FIELD_ID_META_KEY.to_string(),
2267 "2".to_string(),
2268 )]))
2269 .clone(),
2270 ];
2274 Arc::new(arrow_schema::Schema::new(fields))
2275 };
2276
2277 let list_parts = ListArray::from_iter_primitive::<Float32Type, _, _>(vec![Some(vec![
2278 Some(1.0_f32),
2279 Some(f32::NAN),
2280 Some(2.0),
2281 Some(2.0),
2282 ])])
2283 .into_parts();
2284
2285 let list_float_field_col = Arc::new({
2286 let list_parts = list_parts.clone();
2287 ListArray::new(
2288 {
2289 if let DataType::List(field) = arrow_schema.field(0).data_type() {
2290 field.clone()
2291 } else {
2292 unreachable!()
2293 }
2294 },
2295 list_parts.1,
2296 list_parts.2,
2297 list_parts.3,
2298 )
2299 }) as ArrayRef;
2300
2301 let struct_list_fields_schema =
2302 if let DataType::Struct(fields) = arrow_schema.field(1).data_type() {
2303 fields.clone()
2304 } else {
2305 unreachable!()
2306 };
2307
2308 let struct_list_float_field_col = Arc::new({
2309 ListArray::new(
2310 {
2311 if let DataType::List(field) = struct_list_fields_schema
2312 .first()
2313 .expect("could not find first list field")
2314 .data_type()
2315 {
2316 field.clone()
2317 } else {
2318 unreachable!()
2319 }
2320 },
2321 list_parts.1,
2322 list_parts.2,
2323 list_parts.3,
2324 )
2325 }) as ArrayRef;
2326
2327 let struct_list_float_field_col = Arc::new(StructArray::new(
2328 struct_list_fields_schema,
2329 vec![struct_list_float_field_col.clone()],
2330 None,
2331 )) as ArrayRef;
2332
2333 let to_write = RecordBatch::try_new(arrow_schema.clone(), vec![
2334 list_float_field_col,
2335 struct_list_float_field_col,
2336 ])
2338 .expect("Could not form record batch");
2339 let output_file = file_io.new_output(
2340 location_gen.generate_location(None, &file_name_gen.generate_file_name()),
2341 )?;
2342
2343 let mut pw = ParquetWriterBuilder::new(
2345 WriterProperties::builder().build(),
2346 Arc::new(
2347 to_write
2348 .schema()
2349 .as_ref()
2350 .try_into()
2351 .expect("Could not convert iceberg schema"),
2352 ),
2353 )
2354 .build(output_file)
2355 .await?;
2356
2357 pw.write(&to_write).await?;
2358 let res = pw.close().await?;
2359 assert_eq!(res.len(), 1);
2360 let data_file = res
2361 .into_iter()
2362 .next()
2363 .unwrap()
2364 .content(DataContentType::Data)
2365 .partition(Struct::empty())
2366 .partition_spec_id(0)
2367 .build()
2368 .unwrap();
2369
2370 assert_eq!(data_file.record_count(), 1);
2372 assert_eq!(*data_file.value_counts(), HashMap::from([(1, 4), (4, 4)]));
2373 assert_eq!(
2374 *data_file.lower_bounds(),
2375 HashMap::from([(1, Datum::float(1.0_f32)), (4, Datum::float(1.0_f32))])
2376 );
2377 assert_eq!(
2378 *data_file.upper_bounds(),
2379 HashMap::from([(1, Datum::float(2.0_f32)), (4, Datum::float(2.0_f32))])
2380 );
2381 assert_eq!(
2382 *data_file.null_value_counts(),
2383 HashMap::from([(1, 0), (4, 0)])
2384 );
2385 assert_eq!(
2386 *data_file.nan_value_counts(),
2387 HashMap::from([(1, 1), (4, 1)])
2388 );
2389
2390 let expect_batch = concat_batches(&arrow_schema, vec![&to_write]).unwrap();
2392 check_parquet_data_file(&file_io, &data_file, &expect_batch).await;
2393
2394 Ok(())
2395 }
2396
2397 macro_rules! construct_map_arr {
2398 ($map_key_field_schema:ident, $map_value_field_schema:ident) => {{
2399 let int_builder = Int32Builder::new();
2400 let float_builder = Float32Builder::with_capacity(4);
2401 let mut builder = MapBuilder::new(None, int_builder, float_builder);
2402 builder.keys().append_value(1);
2403 builder.values().append_value(1.0_f32);
2404 builder.append(true).unwrap();
2405 builder.keys().append_value(2);
2406 builder.values().append_value(f32::NAN);
2407 builder.append(true).unwrap();
2408 builder.keys().append_value(3);
2409 builder.values().append_value(2.0);
2410 builder.append(true).unwrap();
2411 builder.keys().append_value(4);
2412 builder.values().append_value(2.0);
2413 builder.append(true).unwrap();
2414 let array = builder.finish();
2415
2416 let (_field, offsets, entries, nulls, ordered) = array.into_parts();
2417 let new_struct_fields_schema =
2418 Fields::from(vec![$map_key_field_schema, $map_value_field_schema]);
2419
2420 let entries = {
2421 let (_, arrays, nulls) = entries.into_parts();
2422 StructArray::new(new_struct_fields_schema.clone(), arrays, nulls)
2423 };
2424
2425 let field = Arc::new(Field::new(
2426 DEFAULT_MAP_FIELD_NAME,
2427 DataType::Struct(new_struct_fields_schema),
2428 false,
2429 ));
2430
2431 Arc::new(MapArray::new(field, offsets, entries, nulls, ordered))
2432 }};
2433 }
2434
2435 #[tokio::test]
2436 async fn test_nan_val_cnts_map_type() -> Result<()> {
2437 let temp_dir = TempDir::new().unwrap();
2438 let file_io = FileIO::new_with_fs();
2439 let location_gen = DefaultLocationGenerator::with_data_location(
2440 temp_dir.path().to_str().unwrap().to_string(),
2441 );
2442 let file_name_gen =
2443 DefaultFileNameGenerator::new("test".to_string(), None, DataFileFormat::Parquet);
2444
2445 let map_key_field_schema =
2446 Field::new(MAP_KEY_FIELD_NAME, DataType::Int32, false).with_metadata(HashMap::from([
2447 (PARQUET_FIELD_ID_META_KEY.to_string(), "1".to_string()),
2448 ]));
2449
2450 let map_value_field_schema =
2451 Field::new(MAP_VALUE_FIELD_NAME, DataType::Float32, true).with_metadata(HashMap::from(
2452 [(PARQUET_FIELD_ID_META_KEY.to_string(), "2".to_string())],
2453 ));
2454
2455 let struct_map_key_field_schema =
2456 Field::new(MAP_KEY_FIELD_NAME, DataType::Int32, false).with_metadata(HashMap::from([
2457 (PARQUET_FIELD_ID_META_KEY.to_string(), "6".to_string()),
2458 ]));
2459
2460 let struct_map_value_field_schema =
2461 Field::new(MAP_VALUE_FIELD_NAME, DataType::Float32, true).with_metadata(HashMap::from(
2462 [(PARQUET_FIELD_ID_META_KEY.to_string(), "7".to_string())],
2463 ));
2464
2465 let schema_struct_map_field = Fields::from(vec![
2466 Field::new_map(
2467 "col3",
2468 DEFAULT_MAP_FIELD_NAME,
2469 struct_map_key_field_schema.clone(),
2470 struct_map_value_field_schema.clone(),
2471 false,
2472 false,
2473 )
2474 .with_metadata(HashMap::from([(
2475 PARQUET_FIELD_ID_META_KEY.to_string(),
2476 "5".to_string(),
2477 )])),
2478 ]);
2479
2480 let arrow_schema = {
2481 let fields = vec![
2482 Field::new_map(
2483 "col0",
2484 DEFAULT_MAP_FIELD_NAME,
2485 map_key_field_schema.clone(),
2486 map_value_field_schema.clone(),
2487 false,
2488 false,
2489 )
2490 .with_metadata(HashMap::from([(
2491 PARQUET_FIELD_ID_META_KEY.to_string(),
2492 "0".to_string(),
2493 )])),
2494 Field::new_struct("col1", schema_struct_map_field.clone(), true)
2495 .with_metadata(HashMap::from([(
2496 PARQUET_FIELD_ID_META_KEY.to_string(),
2497 "3".to_string(),
2498 )]))
2499 .clone(),
2500 ];
2501 Arc::new(arrow_schema::Schema::new(fields))
2502 };
2503
2504 let map_array = construct_map_arr!(map_key_field_schema, map_value_field_schema);
2505
2506 let struct_map_arr =
2507 construct_map_arr!(struct_map_key_field_schema, struct_map_value_field_schema);
2508
2509 let struct_list_float_field_col = Arc::new(StructArray::new(
2510 schema_struct_map_field,
2511 vec![struct_map_arr],
2512 None,
2513 )) as ArrayRef;
2514
2515 let to_write = RecordBatch::try_new(arrow_schema.clone(), vec![
2516 map_array,
2517 struct_list_float_field_col,
2518 ])
2519 .expect("Could not form record batch");
2520 let output_file = file_io.new_output(
2521 location_gen.generate_location(None, &file_name_gen.generate_file_name()),
2522 )?;
2523
2524 let mut pw = ParquetWriterBuilder::new(
2526 WriterProperties::builder().build(),
2527 Arc::new(
2528 to_write
2529 .schema()
2530 .as_ref()
2531 .try_into()
2532 .expect("Could not convert iceberg schema"),
2533 ),
2534 )
2535 .build(output_file)
2536 .await?;
2537
2538 pw.write(&to_write).await?;
2539 let res = pw.close().await?;
2540 assert_eq!(res.len(), 1);
2541 let data_file = res
2542 .into_iter()
2543 .next()
2544 .unwrap()
2545 .content(DataContentType::Data)
2546 .partition(Struct::empty())
2547 .partition_spec_id(0)
2548 .build()
2549 .unwrap();
2550
2551 assert_eq!(data_file.record_count(), 4);
2553 assert_eq!(
2554 *data_file.value_counts(),
2555 HashMap::from([(1, 4), (2, 4), (6, 4), (7, 4)])
2556 );
2557 assert_eq!(
2558 *data_file.lower_bounds(),
2559 HashMap::from([
2560 (1, Datum::int(1)),
2561 (2, Datum::float(1.0_f32)),
2562 (6, Datum::int(1)),
2563 (7, Datum::float(1.0_f32))
2564 ])
2565 );
2566 assert_eq!(
2567 *data_file.upper_bounds(),
2568 HashMap::from([
2569 (1, Datum::int(4)),
2570 (2, Datum::float(2.0_f32)),
2571 (6, Datum::int(4)),
2572 (7, Datum::float(2.0_f32))
2573 ])
2574 );
2575 assert_eq!(
2576 *data_file.null_value_counts(),
2577 HashMap::from([(1, 0), (2, 0), (6, 0), (7, 0)])
2578 );
2579 assert_eq!(
2580 *data_file.nan_value_counts(),
2581 HashMap::from([(2, 1), (7, 1)])
2582 );
2583
2584 let expect_batch = concat_batches(&arrow_schema, vec![&to_write]).unwrap();
2586 check_parquet_data_file(&file_io, &data_file, &expect_batch).await;
2587
2588 Ok(())
2589 }
2590
2591 #[tokio::test]
2592 async fn test_write_empty_parquet_file() {
2593 let temp_dir = TempDir::new().unwrap();
2594 let file_io = FileIO::new_with_fs();
2595 let location_gen = DefaultLocationGenerator::with_data_location(
2596 temp_dir.path().to_str().unwrap().to_string(),
2597 );
2598 let file_name_gen =
2599 DefaultFileNameGenerator::new("test".to_string(), None, DataFileFormat::Parquet);
2600 let output_file = file_io
2601 .new_output(location_gen.generate_location(None, &file_name_gen.generate_file_name()))
2602 .unwrap();
2603
2604 let pw = ParquetWriterBuilder::new(
2606 WriterProperties::builder().build(),
2607 Arc::new(
2608 Schema::builder()
2609 .with_schema_id(1)
2610 .with_fields(vec![
2611 NestedField::required(0, "col", Type::Primitive(PrimitiveType::Long))
2612 .with_id(0)
2613 .into(),
2614 ])
2615 .build()
2616 .expect("Failed to create schema"),
2617 ),
2618 )
2619 .build(output_file)
2620 .await
2621 .unwrap();
2622
2623 let res = pw.close().await.unwrap();
2624 assert_eq!(res.len(), 0);
2625
2626 assert_eq!(std::fs::read_dir(temp_dir.path()).unwrap().count(), 0);
2628 }
2629
2630 #[tokio::test]
2631 async fn test_parquet_writer_uuid_logical_type() {
2632 let temp_dir = TempDir::new().unwrap();
2633 let file_io = FileIO::new_with_fs();
2634 let location_gen = DefaultLocationGenerator::with_data_location(
2635 temp_dir.path().to_str().unwrap().to_string(),
2636 );
2637 let file_name_gen =
2638 DefaultFileNameGenerator::new("test".to_string(), None, DataFileFormat::Parquet);
2639 let output_file = file_io
2640 .new_output(location_gen.generate_location(None, &file_name_gen.generate_file_name()))
2641 .unwrap();
2642
2643 let schema = Arc::new(
2644 Schema::builder()
2645 .with_schema_id(1)
2646 .with_fields(vec![
2647 NestedField::required(0, "uuid", Type::Primitive(PrimitiveType::Uuid)).into(),
2648 ])
2649 .build()
2650 .unwrap(),
2651 );
2652 let arrow_schema: ArrowSchemaRef = Arc::new(schema_to_arrow_schema(&schema).unwrap());
2653 let col = Arc::new(
2654 arrow_array::FixedSizeBinaryArray::try_from_iter(
2655 [Uuid::from_u128(0), Uuid::from_u128(1)]
2656 .into_iter()
2657 .map(|uuid| uuid.into_bytes()),
2658 )
2659 .unwrap(),
2660 ) as ArrayRef;
2661 let to_write = RecordBatch::try_new(arrow_schema, vec![col]).unwrap();
2662
2663 let mut pw = ParquetWriterBuilder::new(WriterProperties::builder().build(), schema)
2664 .build(output_file)
2665 .await
2666 .unwrap();
2667 pw.write(&to_write).await.unwrap();
2668 let data_file = pw
2669 .close()
2670 .await
2671 .unwrap()
2672 .into_iter()
2673 .next()
2674 .unwrap()
2675 .content(DataContentType::Data)
2676 .partition(Struct::empty())
2677 .partition_spec_id(0)
2678 .build()
2679 .unwrap();
2680
2681 let raw = file_io
2682 .new_input(data_file.file_path.clone())
2683 .unwrap()
2684 .read()
2685 .await
2686 .unwrap();
2687 let reader = ParquetRecordBatchReaderBuilder::try_new(raw).unwrap();
2688 let column = reader.metadata().file_metadata().schema_descr().column(0);
2689 assert_eq!(
2690 (
2691 column.physical_type(),
2692 column.type_length(),
2693 column.logical_type_ref()
2694 ),
2695 (
2696 parquet::basic::Type::FIXED_LEN_BYTE_ARRAY,
2697 16,
2698 Some(&LogicalType::Uuid)
2699 )
2700 );
2701 }
2702
2703 struct GeospatialTestFile {
2704 temp_dir: TempDir,
2705 file_io: FileIO,
2706 schema: SchemaRef,
2707 batch: RecordBatch,
2708 data_file: DataFile,
2709 }
2710
2711 async fn write_geospatial_test_file() -> Result<GeospatialTestFile> {
2712 let temp_dir = TempDir::new().unwrap();
2713 let file_io = FileIO::new_with_fs();
2714 let geom_type = PrimitiveType::Geometry(GeometryType::new(Some("srid:0".to_string()))?);
2715 let schema = Arc::new(
2716 Schema::builder()
2717 .with_fields(vec![
2718 NestedField::required(1, "geom", Type::Primitive(geom_type)).into(),
2719 NestedField::optional(
2720 2,
2721 "geog",
2722 Type::Primitive(PrimitiveType::Geography(GeographyType::new(
2723 None,
2724 IcebergEdgeInterpolationAlgorithm::Karney,
2725 )?)),
2726 )
2727 .into(),
2728 ])
2729 .build()?,
2730 );
2731 let geometry = [
2732 parquet_geospatial::testing::wkb_point_xyzm(1.0, 20.0, f64::NAN, 40.0),
2733 parquet_geospatial::testing::wkb_point_xyzm(-3.0, 4.0, f64::NAN, 70.0),
2734 ];
2735 let geography = wkb_point_xy(3.0, 4.0);
2736 let batch = RecordBatch::try_new(Arc::new(schema_to_arrow_schema(&schema)?), vec![
2737 Arc::new(arrow_array::LargeBinaryArray::from_iter_values(&geometry)),
2738 Arc::new(arrow_array::LargeBinaryArray::from(vec![
2739 Some(geography.as_slice()),
2740 None,
2741 ])),
2742 ])?;
2743 let output = file_io.new_output(temp_dir.path().join("geo.parquet").to_str().unwrap())?;
2744 let props = WriterProperties::builder()
2745 .set_max_row_group_row_count(Some(1))
2746 .build();
2747 let mut writer = ParquetWriterBuilder::new(props, schema.clone())
2748 .build(output)
2749 .await?;
2750 writer.write(&batch).await?;
2751 let data_file = writer
2752 .close()
2753 .await?
2754 .into_iter()
2755 .next()
2756 .unwrap()
2757 .content(DataContentType::Data)
2758 .partition(Struct::empty())
2759 .partition_spec_id(0)
2760 .build()
2761 .unwrap();
2762
2763 Ok(GeospatialTestFile {
2764 temp_dir,
2765 file_io,
2766 schema,
2767 batch,
2768 data_file,
2769 })
2770 }
2771
2772 #[tokio::test]
2773 async fn test_geospatial_parquet_roundtrip() -> Result<()> {
2774 let file = write_geospatial_test_file().await?;
2775 let input = file.file_io.new_input(file.data_file.file_path())?;
2776 let mut parquet_reader =
2777 ArrowFileReader::new(input.metadata().await?, input.reader().await?);
2778 let metadata = parquet_reader.get_metadata(None).await?;
2779 let schema_descr = metadata.file_metadata().schema_descr();
2780 assert_eq!(metadata.num_row_groups(), 2);
2781 let logical_types: Vec<_> = schema_descr
2782 .columns()
2783 .iter()
2784 .map(|column| column.logical_type_ref())
2785 .collect();
2786 let expected = [
2787 LogicalType::geometry(Some("srid:0".to_string())),
2788 LogicalType::geography(None, Some(EdgeInterpolationAlgorithm::KARNEY)),
2789 ];
2790 assert_eq!(logical_types, expected.iter().map(Some).collect::<Vec<_>>());
2791
2792 let arrow_schema = parquet::arrow::parquet_to_arrow_schema(
2793 schema_descr,
2794 metadata.file_metadata().key_value_metadata(),
2795 )?;
2796 let unset_metadata: serde_json::Value =
2797 serde_json::from_str(arrow_schema.field(0).extension_type_metadata().unwrap())?;
2798 assert!(unset_metadata.get("crs").is_none());
2799
2800 let task = FileScanTask::builder()
2801 .with_file_size_in_bytes(file.data_file.file_size_in_bytes())
2802 .with_start(0)
2803 .with_length(0)
2804 .with_data_file_path(file.data_file.file_path.clone())
2805 .with_data_file_format(DataFileFormat::Parquet)
2806 .with_schema(file.schema)
2807 .with_project_field_ids(vec![1, 2])
2808 .with_case_sensitive(false)
2809 .build()?;
2810 let tasks = Box::pin(futures::stream::iter(vec![Ok(task)])) as FileScanTaskStream;
2811 let reader = ArrowReaderBuilder::new(file.file_io, Runtime::current()).build();
2812 let batches: Vec<RecordBatch> = reader.read(tasks)?.stream().try_collect().await?;
2813 let read_back = concat_batches(batches[0].schema_ref(), &batches)?;
2814 assert_eq!(read_back.columns(), file.batch.columns());
2815 Ok(())
2816 }
2817
2818 fn coordinate_bytes(coordinates: &[f64]) -> Vec<u8> {
2819 coordinates
2820 .iter()
2821 .flat_map(|value| value.to_le_bytes())
2822 .collect()
2823 }
2824
2825 #[tokio::test]
2826 async fn test_geometry_bounds_manifest_roundtrip() -> Result<()> {
2827 let file = write_geospatial_test_file().await?;
2828 let path = file.temp_dir.path().join("manifest.avro");
2829 let output = file.file_io.new_output(path.to_str().unwrap())?;
2830 let mut writer = ManifestWriterBuilder::new(
2831 output,
2832 Some(1),
2833 file.schema.clone(),
2834 PartitionSpec::builder(file.schema).build()?,
2835 )
2836 .build_v3_data();
2837 writer.add_file(file.data_file, 1)?;
2838 writer.write_manifest_file().await?;
2839 let manifest = Manifest::parse_avro(&std::fs::read(path).unwrap())?;
2840 let data_file = manifest.entries()[0].data_file();
2841 let ty = PrimitiveType::Geometry(GeometryType::new(Some("srid:0".to_string()))?);
2842 let expected_lower = HashMap::from([(
2843 1,
2844 Datum::new(
2845 ty.clone(),
2846 PrimitiveLiteral::Binary(coordinate_bytes(&[-3.0, 4.0, f64::NAN, 40.0])),
2847 ),
2848 )]);
2849 let expected_upper = HashMap::from([(
2850 1,
2851 Datum::new(
2852 ty,
2853 PrimitiveLiteral::Binary(coordinate_bytes(&[1.0, 20.0, f64::NAN, 70.0])),
2854 ),
2855 )]);
2856 assert_eq!(
2857 (data_file.lower_bounds(), data_file.upper_bounds()),
2858 (&expected_lower, &expected_upper),
2859 );
2860 Ok(())
2861 }
2862
2863 #[tokio::test]
2864 async fn test_geometry_bounds_do_not_drive_scalar_metrics() -> Result<()> {
2865 use crate::expr::visitors::inclusive_metrics_evaluator::InclusiveMetricsEvaluator;
2866 use crate::expr::visitors::strict_metrics_evaluator::StrictMetricsEvaluator;
2867 use crate::expr::{Bind, Reference};
2868
2869 let file = write_geospatial_test_file().await?;
2870 let values = file
2871 .batch
2872 .column(0)
2873 .as_any()
2874 .downcast_ref::<arrow_array::LargeBinaryArray>()
2875 .unwrap();
2876 let literal = Datum::try_from_bytes(
2877 values.value(0),
2878 file.schema
2879 .field_by_id(1)
2880 .unwrap()
2881 .field_type
2882 .as_primitive_type()
2883 .unwrap()
2884 .clone(),
2885 )?;
2886 let inequality = Reference::new("geom")
2887 .less_than(literal.clone())
2888 .bind(file.schema.clone(), true)?;
2889 let exclusion = Reference::new("geom")
2890 .not_equal_to(literal)
2891 .bind(file.schema, true)?;
2892 assert_eq!(
2893 (
2894 InclusiveMetricsEvaluator::eval(&inequality, &file.data_file, true)?,
2895 StrictMetricsEvaluator::eval(&exclusion, &file.data_file)?,
2896 ),
2897 (true, false),
2898 );
2899 Ok(())
2900 }
2901
2902 fn geometry_bound_bytes(bbox: BoundingBox) -> (Vec<u8>, Vec<u8>) {
2903 let ty = PrimitiveType::Geometry(GeometryType::default());
2904 let (lower, upper) = GeospatialBounds::from(&bbox).into_datums(ty);
2905 (
2906 lower.to_bytes().unwrap().into_vec(),
2907 upper.to_bytes().unwrap().into_vec(),
2908 )
2909 }
2910
2911 #[test]
2912 fn test_geometry_xy_bounds() {
2913 assert_eq!(
2914 geometry_bound_bytes(BoundingBox::new(-3.0, 1.0, 4.0, 20.0)),
2915 (
2916 coordinate_bytes(&[-3.0, 4.0]),
2917 coordinate_bytes(&[1.0, 20.0])
2918 ),
2919 );
2920 }
2921
2922 #[test]
2923 fn test_geometry_xyz_bounds() {
2924 assert_eq!(
2925 geometry_bound_bytes(BoundingBox::new(-3.0, 1.0, 4.0, 20.0).with_zrange(2.0, 9.0)),
2926 (
2927 coordinate_bytes(&[-3.0, 4.0, 2.0]),
2928 coordinate_bytes(&[1.0, 20.0, 9.0])
2929 ),
2930 );
2931 }
2932
2933 #[test]
2934 fn test_wrapping_geometry_bounds_cover_all_x() {
2935 assert_eq!(
2936 geometry_bound_bytes(BoundingBox::new(170.0, -170.0, -10.0, 10.0)),
2937 (
2938 coordinate_bytes(&[f64::NEG_INFINITY, -10.0]),
2939 coordinate_bytes(&[f64::INFINITY, 10.0]),
2940 ),
2941 );
2942 }
2943
2944 fn geospatial_stats_fixture(ty: PrimitiveType) -> (SchemaRef, ColumnChunkMetaData) {
2945 let schema = Arc::new(
2946 Schema::builder()
2947 .with_fields(vec![
2948 NestedField::optional(1, "geo", Type::Primitive(ty)).into(),
2949 ])
2950 .build()
2951 .unwrap(),
2952 );
2953 let parquet_schema = parquet::arrow::ArrowSchemaConverter::new()
2954 .convert(&schema_to_arrow_schema(&schema).unwrap())
2955 .unwrap();
2956 let column = ColumnChunkMetaData::builder(parquet_schema.column(0))
2957 .set_num_values(1)
2958 .build()
2959 .unwrap();
2960 (schema, column)
2961 }
2962
2963 fn with_geo_box(column: &ColumnChunkMetaData, bbox: BoundingBox) -> ColumnChunkMetaData {
2964 use parquet::geospatial::statistics::GeospatialStatistics;
2965 column
2966 .clone()
2967 .into_builder()
2968 .set_geo_statistics(Box::new(GeospatialStatistics::new(Some(bbox), None)))
2969 .build()
2970 .unwrap()
2971 }
2972
2973 #[test]
2974 fn test_geometry_bounds_merge_across_null_row_group() -> Result<()> {
2975 let (schema, column) =
2976 geospatial_stats_fixture(PrimitiveType::Geometry(GeometryType::default()));
2977 let first = with_geo_box(
2978 &column,
2979 BoundingBox::new(-3.0, 1.0, 4.0, 20.0).with_zrange(4.0, 9.0),
2980 );
2981 let second = with_geo_box(
2982 &column,
2983 BoundingBox::new(0.0, 5.0, -20.0, 5.0).with_mrange(1.0, 3.0),
2984 );
2985 let nulls = column
2986 .into_builder()
2987 .set_statistics(Statistics::ByteArray(ValueStatistics::new(
2988 None,
2989 None,
2990 None,
2991 Some(1),
2992 false,
2993 )))
2994 .build()?;
2995 let mut bounds = MinMaxColAggregator::new(schema);
2996 for column in [&first, &nulls, &second] {
2997 bounds.update(1, column)?;
2998 }
2999 let (lower, upper) = bounds.produce();
3000 assert_eq!(
3001 (
3002 lower[&1].to_bytes()?.into_vec(),
3003 upper[&1].to_bytes()?.into_vec()
3004 ),
3005 (
3006 coordinate_bytes(&[-3.0, -20.0, 4.0, 1.0]),
3007 coordinate_bytes(&[5.0, 20.0, 9.0, 3.0])
3008 ),
3009 );
3010 Ok(())
3011 }
3012
3013 #[test]
3014 fn test_geometry_bounds_require_all_non_null_row_groups() -> Result<()> {
3015 let (schema, unknown) =
3016 geospatial_stats_fixture(PrimitiveType::Geometry(GeometryType::default()));
3017 let known = with_geo_box(&unknown, BoundingBox::new(1.0, 2.0, 3.0, 4.0));
3018 let mut bounds = MinMaxColAggregator::new(schema);
3019 for column in [&known, &unknown, &known] {
3020 bounds.update(1, column)?;
3021 }
3022 assert_eq!(bounds.produce(), (HashMap::new(), HashMap::new()));
3023 Ok(())
3024 }
3025
3026 #[test]
3027 fn test_geography_bounds_merge_from_parquet_statistics() -> Result<()> {
3028 let (schema, column) =
3029 geospatial_stats_fixture(PrimitiveType::Geography(GeographyType::default()));
3030 let first = with_geo_box(&column, BoundingBox::new(170.0, -170.0, -20.0, 20.0));
3031 let second = with_geo_box(&column, BoundingBox::new(175.0, 178.0, -10.0, 30.0));
3032 let mut bounds = MinMaxColAggregator::new(schema);
3033 bounds.update(1, &first)?;
3034 bounds.update(1, &second)?;
3035 let (lower, upper) = bounds.produce();
3036 assert_eq!(
3037 (
3038 lower[&1].to_bytes()?.into_vec(),
3039 upper[&1].to_bytes()?.into_vec()
3040 ),
3041 (
3042 coordinate_bytes(&[170.0, -20.0]),
3043 coordinate_bytes(&[-170.0, 30.0])
3044 ),
3045 );
3046 Ok(())
3047 }
3048
3049 #[test]
3050 fn test_geography_bounds_full_longitude_range() -> Result<()> {
3051 let (schema, column) =
3052 geospatial_stats_fixture(PrimitiveType::Geography(GeographyType::default()));
3053 let first = with_geo_box(&column, BoundingBox::new(170.0, -170.0, -20.0, 20.0));
3054 let second = with_geo_box(&column, BoundingBox::new(-175.0, 175.0, -10.0, 30.0));
3055 let mut bounds = MinMaxColAggregator::new(schema);
3056 bounds.update(1, &first)?;
3057 bounds.update(1, &second)?;
3058 let (lower, upper) = bounds.produce();
3059 assert_eq!(
3060 (
3061 lower[&1].to_bytes()?.into_vec(),
3062 upper[&1].to_bytes()?.into_vec()
3063 ),
3064 (
3065 coordinate_bytes(&[-180.0, -20.0]),
3066 coordinate_bytes(&[180.0, 30.0])
3067 ),
3068 );
3069 Ok(())
3070 }
3071
3072 #[test]
3073 fn test_min_max_aggregator() {
3074 let schema = Arc::new(
3075 Schema::builder()
3076 .with_schema_id(1)
3077 .with_fields(vec![
3078 NestedField::required(0, "col", Type::Primitive(PrimitiveType::Int))
3079 .with_id(0)
3080 .into(),
3081 ])
3082 .build()
3083 .expect("Failed to create schema"),
3084 );
3085 let parquet_schema = parquet::arrow::ArrowSchemaConverter::new()
3086 .convert(&schema_to_arrow_schema(&schema).unwrap())
3087 .unwrap();
3088 let mut min_max_agg = MinMaxColAggregator::new(schema);
3089 let create_statistics = |min, max| {
3090 ColumnChunkMetaData::builder(parquet_schema.column(0))
3091 .set_statistics(Statistics::Int32(ValueStatistics::new(
3092 min, max, None, None, false,
3093 )))
3094 .build()
3095 .unwrap()
3096 };
3097 min_max_agg
3098 .update(0, &create_statistics(None, Some(42)))
3099 .unwrap();
3100 min_max_agg
3101 .update(0, &create_statistics(Some(0), Some(i32::MAX)))
3102 .unwrap();
3103 min_max_agg
3104 .update(0, &create_statistics(Some(i32::MIN), None))
3105 .unwrap();
3106 min_max_agg
3107 .update(0, &create_statistics(None, None))
3108 .unwrap();
3109
3110 let (lower_bounds, upper_bounds) = min_max_agg.produce();
3111
3112 assert_eq!(lower_bounds, HashMap::from([(0, Datum::int(i32::MIN))]));
3113 assert_eq!(upper_bounds, HashMap::from([(0, Datum::int(i32::MAX))]));
3114 }
3115
3116 #[test]
3117 fn test_min_max_aggregator_skips_nan_bounds() {
3118 let schema = Arc::new(
3119 Schema::builder()
3120 .with_schema_id(1)
3121 .with_fields(vec![
3122 NestedField::required(0, "f", Type::Primitive(PrimitiveType::Float)).into(),
3123 NestedField::required(1, "d", Type::Primitive(PrimitiveType::Double)).into(),
3124 ])
3125 .build()
3126 .expect("Failed to create schema"),
3127 );
3128 let parquet_schema = parquet::arrow::ArrowSchemaConverter::new()
3129 .convert(&schema_to_arrow_schema(&schema).unwrap())
3130 .unwrap();
3131 let mut min_max_agg = MinMaxColAggregator::new(schema);
3132 let float_stats = |min, max| {
3133 ColumnChunkMetaData::builder(parquet_schema.column(0))
3134 .set_statistics(Statistics::Float(ValueStatistics::new(
3135 min, max, None, None, false,
3136 )))
3137 .build()
3138 .unwrap()
3139 };
3140 let double_stats = |min, max| {
3141 ColumnChunkMetaData::builder(parquet_schema.column(1))
3142 .set_statistics(Statistics::Double(ValueStatistics::new(
3143 min, max, None, None, false,
3144 )))
3145 .build()
3146 .unwrap()
3147 };
3148
3149 min_max_agg
3150 .update(0, &float_stats(Some(-f32::NAN), Some(f32::NAN)))
3151 .unwrap();
3152 min_max_agg
3153 .update(0, &float_stats(Some(1.5), Some(2.5)))
3154 .unwrap();
3155 min_max_agg
3156 .update(1, &double_stats(Some(f64::NAN), Some(f64::NAN)))
3157 .unwrap();
3158
3159 let (lower_bounds, upper_bounds) = min_max_agg.produce();
3160
3161 assert_eq!(lower_bounds, HashMap::from([(0, Datum::float(1.5_f32))]));
3162 assert_eq!(upper_bounds, HashMap::from([(0, Datum::float(2.5_f32))]));
3163 }
3164
3165 fn cdc_test_schema() -> SchemaRef {
3166 Arc::new(
3167 Schema::builder()
3168 .with_schema_id(1)
3169 .with_fields(vec![
3170 NestedField::required(1, "id", Type::Primitive(PrimitiveType::Long)).into(),
3171 NestedField::required(2, "payload", Type::Primitive(PrimitiveType::String))
3172 .into(),
3173 ])
3174 .build()
3175 .unwrap(),
3176 )
3177 }
3178
3179 #[test]
3180 fn test_from_table_properties_no_cdc_by_default() {
3181 let raw_properties = HashMap::new();
3182 let tp = TableProperties::new(&raw_properties);
3183 let builder = ParquetWriterBuilder::from_table_properties(&tp, cdc_test_schema()).unwrap();
3184 assert!(builder.props.content_defined_chunking().is_none());
3185 }
3186
3187 #[tokio::test]
3188 async fn test_from_table_properties_without_encryption_writes_plaintext() {
3189 let raw_properties = HashMap::new();
3190 let tp = TableProperties::new(&raw_properties);
3191 let tmp = TempDir::new().unwrap();
3192 let output = FileIO::new_with_fs()
3193 .new_output(format!("{}/plain.parquet", tmp.path().to_str().unwrap()))
3194 .unwrap();
3195 let writer = ParquetWriterBuilder::from_table_properties(&tp, cdc_test_schema())
3196 .unwrap()
3197 .build(output)
3198 .await
3199 .unwrap();
3200
3201 assert!(writer.key_metadata.is_none());
3202 assert!(
3203 writer
3204 .writer_properties
3205 .file_encryption_properties()
3206 .is_none()
3207 );
3208 }
3209
3210 #[tokio::test]
3211 async fn test_from_table_properties_propagate_to_writer() {
3212 let raw_properties = HashMap::from([
3221 (
3222 TableProperties::PROPERTY_PARQUET_CDC_ENABLED.to_string(),
3223 "true".to_string(),
3224 ),
3225 (
3226 TableProperties::PROPERTY_PARQUET_CDC_MIN_CHUNK_SIZE.to_string(),
3227 "4096".to_string(),
3228 ),
3229 (
3230 TableProperties::PROPERTY_PARQUET_CDC_MAX_CHUNK_SIZE.to_string(),
3231 "8192".to_string(),
3232 ),
3233 (
3234 TableProperties::PROPERTY_PARQUET_CDC_NORM_LEVEL.to_string(),
3235 "2".to_string(),
3236 ),
3237 ]);
3238 let tp = TableProperties::new(&raw_properties);
3239
3240 let tmp = TempDir::new().unwrap();
3241 let output = FileIO::new_with_fs()
3242 .new_output(format!("{}/cdc.parquet", tmp.path().to_str().unwrap()))
3243 .unwrap();
3244 let writer = ParquetWriterBuilder::from_table_properties(&tp, cdc_test_schema())
3245 .unwrap()
3246 .build(output)
3247 .await
3248 .unwrap();
3249
3250 let cdc = writer
3251 .writer_properties
3252 .content_defined_chunking()
3253 .copied()
3254 .expect("CDC should be enabled on the built writer");
3255 assert_eq!(cdc.min_chunk_size, 4096);
3256 assert_eq!(cdc.max_chunk_size, 8192);
3257 assert_eq!(cdc.norm_level, 2);
3258 }
3259
3260 #[test]
3261 fn test_from_table_properties_sizing_defaults() {
3262 let raw_properties = HashMap::new();
3265 let tp = TableProperties::new(&raw_properties);
3266 let props = ParquetWriterBuilder::from_table_properties(&tp, cdc_test_schema())
3267 .unwrap()
3268 .props;
3269
3270 assert_eq!(
3271 props.max_row_group_bytes(),
3272 Some(TableProperties::PROPERTY_PARQUET_ROW_GROUP_SIZE_BYTES_DEFAULT)
3273 );
3274 assert_eq!(
3275 props.data_page_size_limit(),
3276 TableProperties::PROPERTY_PARQUET_PAGE_SIZE_BYTES_DEFAULT
3277 );
3278 assert_eq!(
3279 props.data_page_row_count_limit(),
3280 TableProperties::PROPERTY_PARQUET_PAGE_ROW_LIMIT_DEFAULT
3281 );
3282 assert_eq!(
3283 props.dictionary_page_size_limit(),
3284 TableProperties::PROPERTY_PARQUET_DICT_SIZE_BYTES_DEFAULT
3285 );
3286 assert_eq!(
3288 props.compression(&ColumnPath::from("id")),
3289 Compression::ZSTD(ZstdLevel::try_new(3).unwrap())
3290 );
3291 }
3292
3293 #[test]
3294 fn test_from_table_properties_sizing_and_compression_overrides() {
3295 let raw_properties = HashMap::from([
3296 (
3297 TableProperties::PROPERTY_PARQUET_ROW_GROUP_SIZE_BYTES.to_string(),
3298 "1048576".to_string(),
3299 ),
3300 (
3301 TableProperties::PROPERTY_PARQUET_PAGE_SIZE_BYTES.to_string(),
3302 "65536".to_string(),
3303 ),
3304 (
3305 TableProperties::PROPERTY_PARQUET_PAGE_ROW_LIMIT.to_string(),
3306 "5000".to_string(),
3307 ),
3308 (
3309 TableProperties::PROPERTY_PARQUET_DICT_SIZE_BYTES.to_string(),
3310 "131072".to_string(),
3311 ),
3312 (
3313 TableProperties::PROPERTY_PARQUET_COMPRESSION_CODEC.to_string(),
3314 "gzip".to_string(),
3315 ),
3316 (
3317 TableProperties::PROPERTY_PARQUET_COMPRESSION_LEVEL.to_string(),
3318 "9".to_string(),
3319 ),
3320 ]);
3321 let tp = TableProperties::new(&raw_properties);
3322 let props = ParquetWriterBuilder::from_table_properties(&tp, cdc_test_schema())
3323 .unwrap()
3324 .props;
3325
3326 assert_eq!(props.max_row_group_bytes(), Some(1048576));
3327 assert_eq!(props.data_page_size_limit(), 65536);
3328 assert_eq!(props.data_page_row_count_limit(), 5000);
3329 assert_eq!(props.dictionary_page_size_limit(), 131072);
3330 assert_eq!(
3331 props.compression(&ColumnPath::from("id")),
3332 Compression::GZIP(GzipLevel::try_new(9).unwrap())
3333 );
3334 }
3335
3336 #[test]
3337 fn test_from_table_properties_invalid_codec_errors() {
3338 let entries = HashMap::from([(
3339 TableProperties::PROPERTY_PARQUET_COMPRESSION_CODEC.to_string(),
3340 "bogus".to_string(),
3341 )]);
3342 let err = ParquetWriterBuilder::from_table_properties(
3343 &TableProperties::new(&entries),
3344 cdc_test_schema(),
3345 )
3346 .unwrap_err();
3347 assert_eq!(err.kind(), ErrorKind::DataInvalid);
3348 assert!(err.to_string().contains("bogus"));
3349 }
3350
3351 #[test]
3352 fn test_parquet_compression_mapping() {
3353 assert_eq!(
3355 parquet_compression(CompressionCodec::None).unwrap(),
3356 Compression::UNCOMPRESSED
3357 );
3358 assert_eq!(
3359 parquet_compression(CompressionCodec::Snappy).unwrap(),
3360 Compression::SNAPPY
3361 );
3362 assert_eq!(
3363 parquet_compression(CompressionCodec::Lz4).unwrap(),
3364 Compression::LZ4
3365 );
3366 assert_eq!(
3367 parquet_compression(CompressionCodec::Lz4Raw).unwrap(),
3368 Compression::LZ4_RAW
3369 );
3370 assert_eq!(
3371 parquet_compression(CompressionCodec::Lzo).unwrap(),
3372 Compression::LZO
3373 );
3374
3375 assert_eq!(
3377 parquet_compression(CompressionCodec::zstd_default()).unwrap(),
3378 Compression::ZSTD(ZstdLevel::try_new(3).unwrap())
3379 );
3380 assert_eq!(
3381 parquet_compression(CompressionCodec::gzip_default()).unwrap(),
3382 Compression::GZIP(GzipLevel::try_new(6).unwrap())
3383 );
3384 assert_eq!(
3385 parquet_compression(CompressionCodec::brotli_default()).unwrap(),
3386 Compression::BROTLI(BrotliLevel::try_new(1).unwrap())
3387 );
3388
3389 assert_eq!(
3391 parquet_compression(CompressionCodec::Zstd(10)).unwrap(),
3392 Compression::ZSTD(ZstdLevel::try_new(10).unwrap())
3393 );
3394 }
3395
3396 #[test]
3397 fn test_parquet_compression_invalid_level() {
3398 let err = parquet_compression(CompressionCodec::Zstd(99)).unwrap_err();
3400 assert_eq!(err.kind(), ErrorKind::DataInvalid);
3401 assert!(err.to_string().contains("zstd"));
3402 }
3403}