iceberg/io/
mod.rs

1// Licensed to the Apache Software Foundation (ASF) under one
2// or more contributor license agreements.  See the NOTICE file
3// distributed with this work for additional information
4// regarding copyright ownership.  The ASF licenses this file
5// to you under the Apache License, Version 2.0 (the
6// "License"); you may not use this file except in compliance
7// with the License.  You may obtain a copy of the License at
8//
9//   http://www.apache.org/licenses/LICENSE-2.0
10//
11// Unless required by applicable law or agreed to in writing,
12// software distributed under the License is distributed on an
13// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14// KIND, either express or implied.  See the License for the
15// specific language governing permissions and limitations
16// under the License.
17
18//! File io implementation.
19//!
20//! # How to build `FileIO`
21//!
22//! We provided a `FileIOBuilder` to build `FileIO` from scratch. For example:
23//!
24//! ```rust
25//! use iceberg::Result;
26//! use iceberg::io::{FileIOBuilder, S3_REGION};
27//!
28//! # fn test() -> Result<()> {
29//! // Build a memory file io.
30//! let file_io = FileIOBuilder::new("memory").build()?;
31//! // Build an fs file io.
32//! let file_io = FileIOBuilder::new("fs").build()?;
33//! // Build an s3 file io.
34//! let file_io = FileIOBuilder::new("s3")
35//!     .with_prop(S3_REGION, "us-east-1")
36//!     .build()?;
37//! # Ok(())
38//! # }
39//! ```
40//!
41//! Or you can pass a path to ask `FileIO` to infer schema for you:
42//!
43//! ```rust
44//! use iceberg::Result;
45//! use iceberg::io::{FileIO, S3_REGION};
46//!
47//! # fn test() -> Result<()> {
48//! // Build a memory file io.
49//! let file_io = FileIO::from_path("memory:///")?.build()?;
50//! // Build an fs file io.
51//! let file_io = FileIO::from_path("fs:///tmp")?.build()?;
52//! // Build an s3 file io.
53//! let file_io = FileIO::from_path("s3://bucket/a")?
54//!     .with_prop(S3_REGION, "us-east-1")
55//!     .build()?;
56//! # Ok(())
57//! # }
58//! ```
59//!
60//! # How to use `FileIO`
61//!
62//! Currently `FileIO` provides simple methods for file operations:
63//!
64//! - `delete`: Delete file.
65//! - `exists`: Check if file exists.
66//! - `new_input`: Create input file for reading.
67//! - `new_output`: Create output file for writing.
68
69mod config;
70mod file_io;
71mod local_fs;
72mod memory;
73mod opendal;
74mod storage;
75
76pub use config::*;
77pub use file_io::*;
78#[cfg(feature = "storage-s3")]
79pub use opendal::CustomAwsCredentialLoader;
80pub use opendal::{OpenDalStorage, OpenDalStorageFactory};
81pub use storage::{Storage, StorageConfig, StorageFactory};
82
83pub(crate) mod object_cache;
84
85pub(crate) fn is_truthy(value: &str) -> bool {
86    ["true", "t", "1", "on"].contains(&value.to_lowercase().as_str())
87}
iceberg/io/mod.rs

iceberg/io/
mod.rs
