Skip to main content

iceberg/spec/values/
decimal_utils.rs

1// Licensed to the Apache Software Foundation (ASF) under one
2// or more contributor license agreements.  See the NOTICE file
3// distributed with this work for additional information
4// regarding copyright ownership.  The ASF licenses this file
5// to you under the Apache License, Version 2.0 (the
6// "License"); you may not use this file except in compliance
7// with the License.  You may obtain a copy of the License at
8//
9//   http://www.apache.org/licenses/LICENSE-2.0
10//
11// Unless required by applicable law or agreed to in writing,
12// software distributed under the License is distributed on an
13// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14// KIND, either express or implied.  See the License for the
15// specific language governing permissions and limitations
16// under the License.
17
18//! Compatibility layer for decimal operations.
19//!
20//! Provides rust_decimal-compatible API using fastnum's D128 internally.
21//! D128 supports 38-digit precision, meeting the Iceberg spec requirement.
22
23use fastnum::D128;
24use fastnum::decimal::Context;
25
26use crate::error::invalid_data;
27use crate::{Result, ensure_data_valid};
28
29/// Re-export D128 as the Decimal type for use throughout the crate.
30pub type Decimal = D128;
31
32/// Create a D128 from mantissa (i128) and scale (u32).
33///
34/// This is equivalent to rust_decimal's `Decimal::from_i128_with_scale`.
35/// The value is computed as: mantissa * 10^(-scale)
36///
37/// For example:
38/// - mantissa=12345, scale=2 => 123.45
39/// - mantissa=-456, scale=3 => -0.456
40pub fn decimal_from_i128_with_scale(mantissa: i128, scale: u32) -> Decimal {
41    if scale == 0 {
42        return D128::from_i128(mantissa).expect("i128 always fits in D128");
43    }
44
45    // Convert mantissa to string and insert decimal point at the right position
46    let is_negative = mantissa < 0;
47    let abs_str = mantissa.unsigned_abs().to_string();
48    let scale_usize = scale as usize;
49
50    let decimal_str = if abs_str.len() <= scale_usize {
51        // Need leading zeros: e.g., mantissa=456, scale=3 => "0.456"
52        // Or mantissa=5, scale=3 => "0.005"
53        let zeros_needed = scale_usize - abs_str.len();
54        format!(
55            "{}0.{}{}",
56            if is_negative { "-" } else { "" },
57            "0".repeat(zeros_needed),
58            abs_str
59        )
60    } else {
61        // Insert decimal point: e.g., mantissa=12345, scale=2 => "123.45"
62        let decimal_pos = abs_str.len() - scale_usize;
63        format!(
64            "{}{}.{}",
65            if is_negative { "-" } else { "" },
66            &abs_str[..decimal_pos],
67            &abs_str[decimal_pos..]
68        )
69    };
70
71    D128::from_str(&decimal_str, Context::default())
72        .expect("constructed decimal string is always valid")
73}
74
75/// Try to create a D128 from mantissa and scale, with validation.
76///
77/// This is equivalent to rust_decimal's `Decimal::try_from_i128_with_scale`.
78/// Currently always succeeds for 38-digit decimals.
79pub fn try_decimal_from_i128_with_scale(mantissa: i128, scale: u32) -> Result<Decimal> {
80    // For now, always succeeds since D128 supports full 38-digit precision
81    Ok(decimal_from_i128_with_scale(mantissa, scale))
82}
83
84/// Create a D128 from i64 mantissa and scale.
85///
86/// This is equivalent to rust_decimal's `Decimal::new`.
87#[allow(dead_code)]
88pub fn decimal_new(mantissa: i64, scale: u32) -> Decimal {
89    decimal_from_i128_with_scale(mantissa as i128, scale)
90}
91
92/// Parse a decimal from string with exact representation.
93///
94/// This is equivalent to rust_decimal's `Decimal::from_str_exact`.
95pub fn decimal_from_str_exact(s: &str) -> Result<Decimal> {
96    let decimal = D128::from_str(s, Context::default())
97        .map_err(|e| invalid_data!("Can't parse decimal: {e}"))?;
98    ensure_data_valid!(
99        decimal.is_finite(),
100        "Decimal value must be finite: {}",
101        decimal
102    );
103    Ok(decimal)
104}
105
106/// Get the mantissa (unscaled coefficient) as i128.
107///
108/// This is equivalent to rust_decimal's `decimal.mantissa()`.
109///
110/// The mantissa is signed: negative decimals return negative mantissa.
111pub fn decimal_mantissa(d: &Decimal) -> i128 {
112    // digits() returns unsigned coefficient as UInt<N>
113    // For Iceberg decimals (max 38 digits), this always fits in u128/i128
114    let digits = d.digits();
115
116    // Convert UInt<2> to u128 - this always succeeds for Iceberg-compliant decimals
117    // since 38 digits requires ~127 bits and u128 has 128 bits
118    let unsigned: u128 = digits
119        .to_u128()
120        .expect("Iceberg decimals (max 38 digits) always fit in u128");
121
122    let signed = unsigned as i128;
123    if d.is_sign_negative() {
124        -signed
125    } else {
126        signed
127    }
128}
129
130/// Get the decimal digit precision of a mantissa.
131///
132/// A mantissa of 0 has a precision of 1.
133pub fn decimal_precision(mantissa: i128) -> u32 {
134    mantissa
135        .unsigned_abs()
136        .checked_ilog10()
137        .map_or(1, |x| x + 1)
138}
139
140/// Get the scale (number of digits after decimal point).
141///
142/// This is equivalent to rust_decimal's `decimal.scale()`.
143pub fn decimal_scale(d: &Decimal) -> u32 {
144    let frac = d.fractional_digits_count();
145    if frac < 0 { 0 } else { frac as u32 }
146}
147
148/// Rescale a decimal to the given scale, returning the rescaled value.
149///
150/// This is equivalent to rust_decimal's `decimal.rescale(scale)`.
151pub fn decimal_rescale(d: Decimal, scale: u32) -> Decimal {
152    d.rescale(scale as i16)
153}
154
155/// Convert big-endian signed bytes to i128.
156///
157/// This handles variable-length byte arrays (up to 16 bytes) with sign extension.
158/// Returns None if the byte array is longer than 16 bytes.
159pub fn i128_from_be_bytes(bytes: &[u8]) -> Option<i128> {
160    if bytes.is_empty() {
161        return Some(0);
162    }
163    if bytes.len() > 16 {
164        return None; // Too large for i128
165    }
166
167    // Check sign bit (most significant bit of first byte)
168    let is_negative = bytes[0] & 0x80 != 0;
169
170    // Pad to 16 bytes with sign extension
171    let mut padded = if is_negative { [0xFF; 16] } else { [0; 16] };
172    let start = 16 - bytes.len();
173    padded[start..].copy_from_slice(bytes);
174
175    Some(i128::from_be_bytes(padded))
176}
177
178/// Convert i128 to big-endian signed bytes with minimum length.
179///
180/// This produces the shortest two's complement representation of the value.
181/// The result is suitable for Iceberg decimal binary serialization.
182pub fn i128_to_be_bytes_min(value: i128) -> Vec<u8> {
183    let bytes = value.to_be_bytes();
184
185    // Find the first significant byte
186    // For positive numbers, skip leading 0x00 bytes (but keep sign bit)
187    // For negative numbers, skip leading 0xFF bytes (but keep sign bit)
188    let is_negative = value < 0;
189    let skip_byte = if is_negative { 0xFF } else { 0x00 };
190
191    let mut start = 0;
192    while start < 15 && bytes[start] == skip_byte {
193        // Check if the next byte has the correct sign bit
194        let next_byte = bytes[start + 1];
195        let next_is_negative = (next_byte & 0x80) != 0;
196        if next_is_negative == is_negative {
197            start += 1;
198        } else {
199            break;
200        }
201    }
202
203    bytes[start..].to_vec()
204}
205
206/// Encode an i128 as exactly `len` big-endian two's complement bytes, matching a
207/// Parquet `FIXED_LEN_BYTE_ARRAY` column of that declared `type_length`.
208///
209/// Returns `None` if `value` does not fit in `len` bytes, since a truncated
210/// encoding would represent a different number.
211pub(crate) fn decimal_to_fixed_length_bytes_exact(value: i128, len: usize) -> Option<Vec<u8>> {
212    if len == 0 || len > 16 {
213        return None;
214    }
215
216    let be_bytes = value.to_be_bytes();
217    let offset = 16 - len;
218    let sign_byte = if value < 0 { 0xFF } else { 0x00 };
219
220    // The bytes being dropped must be pure sign extension.
221    if be_bytes[..offset].iter().any(|&b| b != sign_byte) {
222        return None;
223    }
224
225    // The retained leading byte must carry the value's sign.
226    if (be_bytes[offset] & 0x80 != 0) != (value < 0) {
227        return None;
228    }
229
230    Some(be_bytes[offset..].to_vec())
231}
232
233#[cfg(test)]
234mod tests {
235    use super::*;
236
237    #[test]
238    fn test_decimal_from_i128_with_scale() {
239        let d = decimal_from_i128_with_scale(12345, 2);
240        assert_eq!(d.to_string(), "123.45");
241
242        let d = decimal_from_i128_with_scale(-12345, 2);
243        assert_eq!(d.to_string(), "-123.45");
244
245        let d = decimal_from_i128_with_scale(0, 5);
246        assert_eq!(d.to_string(), "0.00000");
247    }
248
249    #[test]
250    fn test_decimal_new() {
251        let d = decimal_new(123, 2);
252        assert_eq!(d.to_string(), "1.23");
253
254        let d = decimal_new(-456, 3);
255        assert_eq!(d.to_string(), "-0.456");
256    }
257
258    #[test]
259    fn test_decimal_from_str_exact() {
260        let d = decimal_from_str_exact("123.45").unwrap();
261        assert_eq!(d.to_string(), "123.45");
262
263        let d = decimal_from_str_exact("-0.001").unwrap();
264        assert_eq!(d.to_string(), "-0.001");
265
266        let d = decimal_from_str_exact("99999999999999999999999999999999999999").unwrap();
267        assert_eq!(d.to_string(), "99999999999999999999999999999999999999");
268    }
269
270    #[test]
271    fn test_decimal_mantissa() {
272        let d = decimal_from_i128_with_scale(12345, 2);
273        assert_eq!(decimal_mantissa(&d), 12345);
274
275        let d = decimal_from_i128_with_scale(-12345, 2);
276        assert_eq!(decimal_mantissa(&d), -12345);
277    }
278
279    #[test]
280    fn test_decimal_precision() {
281        assert_eq!(decimal_precision(0), 1);
282        assert_eq!(decimal_precision(5), 1);
283        assert_eq!(decimal_precision(42), 2);
284        assert_eq!(decimal_precision(-42), 2);
285        // power-of-10 boundaries, where off-by-one bugs in log10 logic live
286        assert_eq!(decimal_precision(9), 1);
287        assert_eq!(decimal_precision(10), 2);
288        assert_eq!(decimal_precision(99), 2);
289        assert_eq!(decimal_precision(100), 3);
290
291        // max Iceberg decimal precision
292        assert_eq!(
293            decimal_precision(99999999999999999999999999999999999999),
294            38
295        );
296        assert_eq!(
297            decimal_precision(-99999999999999999999999999999999999999),
298            38
299        );
300
301        // i128 extremes
302        assert_eq!(decimal_precision(i128::MAX), 39);
303        assert_eq!(decimal_precision(i128::MIN), 39);
304    }
305
306    #[test]
307    fn test_decimal_scale() {
308        let d = decimal_from_i128_with_scale(12345, 2);
309        assert_eq!(decimal_scale(&d), 2);
310
311        let d = decimal_from_i128_with_scale(12345, 0);
312        assert_eq!(decimal_scale(&d), 0);
313    }
314
315    #[test]
316    fn test_decimal_rescale() {
317        let d = decimal_from_str_exact("123.45").unwrap();
318        let rescaled = decimal_rescale(d, 4);
319        assert_eq!(decimal_scale(&rescaled), 4);
320        assert_eq!(decimal_mantissa(&rescaled), 1234500);
321    }
322
323    #[test]
324    fn test_38_digit_precision() {
325        // Test that we can handle 38-digit decimals (Iceberg spec requirement)
326        let max_38_digits = "99999999999999999999999999999999999999";
327        let d = decimal_from_str_exact(max_38_digits).unwrap();
328        assert_eq!(d.to_string(), max_38_digits);
329
330        let min_38_digits = "-99999999999999999999999999999999999999";
331        let d = decimal_from_str_exact(min_38_digits).unwrap();
332        assert_eq!(d.to_string(), min_38_digits);
333    }
334
335    #[test]
336    fn test_i128_from_be_bytes() {
337        // Empty bytes
338        assert_eq!(i128_from_be_bytes(&[]), Some(0));
339
340        // Positive values
341        assert_eq!(i128_from_be_bytes(&[0x01]), Some(1));
342        assert_eq!(i128_from_be_bytes(&[0x7F]), Some(127));
343        assert_eq!(i128_from_be_bytes(&[0x00, 0xFF]), Some(255));
344        assert_eq!(i128_from_be_bytes(&[0x04, 0xD2]), Some(1234));
345
346        // Negative values (sign extension)
347        assert_eq!(i128_from_be_bytes(&[0xFF]), Some(-1));
348        assert_eq!(i128_from_be_bytes(&[0x80]), Some(-128));
349        assert_eq!(i128_from_be_bytes(&[0xFB, 0x2E]), Some(-1234));
350
351        // Too large (> 16 bytes)
352        assert_eq!(i128_from_be_bytes(&[0; 17]), None);
353    }
354
355    #[test]
356    fn test_i128_to_be_bytes_min() {
357        // Positive values
358        assert_eq!(i128_to_be_bytes_min(0), vec![0x00]);
359        assert_eq!(i128_to_be_bytes_min(1), vec![0x01]);
360        assert_eq!(i128_to_be_bytes_min(127), vec![0x7F]);
361        assert_eq!(i128_to_be_bytes_min(128), vec![0x00, 0x80]);
362        assert_eq!(i128_to_be_bytes_min(255), vec![0x00, 0xFF]);
363        assert_eq!(i128_to_be_bytes_min(1234), vec![0x04, 0xD2]);
364
365        // Negative values
366        assert_eq!(i128_to_be_bytes_min(-1), vec![0xFF]);
367        assert_eq!(i128_to_be_bytes_min(-128), vec![0x80]);
368        assert_eq!(i128_to_be_bytes_min(-129), vec![0xFF, 0x7F]);
369        assert_eq!(i128_to_be_bytes_min(-1234), vec![0xFB, 0x2E]);
370
371        // Round trip test
372        for val in [
373            0i128,
374            1,
375            -1,
376            127,
377            -128,
378            255,
379            -256,
380            12345,
381            -12345,
382            i128::MAX,
383            i128::MIN,
384        ] {
385            let bytes = i128_to_be_bytes_min(val);
386            assert_eq!(
387                i128_from_be_bytes(&bytes),
388                Some(val),
389                "Round trip failed for {val}"
390            );
391        }
392    }
393
394    #[test]
395    fn test_decimal_to_fixed_length_bytes_exact_positive() {
396        let bytes = decimal_to_fixed_length_bytes_exact(12345, 9).unwrap();
397        assert_eq!(bytes.len(), 9);
398        // Big-endian, zero-padded on the left: 12345 = 0x3039
399        assert_eq!(bytes[7], 0x30);
400        assert_eq!(bytes[8], 0x39);
401        assert!(bytes[..7].iter().all(|&b| b == 0x00));
402    }
403
404    #[test]
405    fn test_decimal_to_fixed_length_bytes_exact_negative() {
406        let bytes = decimal_to_fixed_length_bytes_exact(-1, 9).unwrap();
407        assert_eq!(bytes.len(), 9);
408        assert!(bytes.iter().all(|&b| b == 0xFF));
409    }
410
411    #[test]
412    fn test_decimal_to_fixed_length_bytes_exact_round_trip() {
413        for value in [
414            0i128,
415            1,
416            -1,
417            12345,
418            -12345,
419            i64::MAX as i128,
420            i64::MIN as i128,
421        ] {
422            let bytes = decimal_to_fixed_length_bytes_exact(value, 16).unwrap();
423            assert_eq!(bytes.len(), 16);
424            assert_eq!(
425                i128_from_be_bytes(&bytes),
426                Some(value),
427                "Round trip failed for value={value}"
428            );
429        }
430    }
431
432    /// The length must be honoured exactly: a value needing more bytes cannot be
433    /// truncated into the column's width, because the truncation would encode a
434    /// different number and probe the wrong bloom filter slot.
435    #[test]
436    fn test_decimal_to_fixed_length_bytes_exact_rejects_overflow() {
437        // 200 needs a leading zero byte to stay positive in two's complement.
438        assert_eq!(decimal_to_fixed_length_bytes_exact(200, 1), None);
439        assert_eq!(
440            decimal_to_fixed_length_bytes_exact(200, 2),
441            Some(vec![0x00, 0xC8])
442        );
443
444        assert_eq!(decimal_to_fixed_length_bytes_exact(-200, 1), None);
445        assert_eq!(
446            decimal_to_fixed_length_bytes_exact(-200, 2),
447            Some(vec![0xFF, 0x38])
448        );
449
450        // Exactly representable boundaries.
451        assert_eq!(
452            decimal_to_fixed_length_bytes_exact(127, 1),
453            Some(vec![0x7F])
454        );
455        assert_eq!(decimal_to_fixed_length_bytes_exact(128, 1), None);
456        assert_eq!(
457            decimal_to_fixed_length_bytes_exact(-128, 1),
458            Some(vec![0x80])
459        );
460        assert_eq!(decimal_to_fixed_length_bytes_exact(-129, 1), None);
461
462        assert_eq!(decimal_to_fixed_length_bytes_exact(i128::MAX, 15), None);
463        assert_eq!(decimal_to_fixed_length_bytes_exact(0, 0), None);
464        assert_eq!(decimal_to_fixed_length_bytes_exact(0, 17), None);
465    }
466}