Faster int<->float conversions.

library/compiler-builtins/build.rs

@@ -281,10 +281,7 @@ mod c {
 
         if target_env == "msvc" {
             if target_arch == "x86_64" {
-                sources.extend(&[
+                sources.extend(&[("__floatdixf", "x86_64/floatdixf.c")]);
-                    ("__floatdisf", "x86_64/floatdisf.c"),
-                    ("__floatdixf", "x86_64/floatdixf.c"),
-                ]);
             }
         } else {
             // None of these seem to be used on x86_64 windows, and they've all
@@ -292,10 +289,7 @@ mod c {
             if target_os != "windows" {
                 if target_arch == "x86_64" {
                     sources.extend(&[
-                        ("__floatdisf", "x86_64/floatdisf.c"),
                         ("__floatdixf", "x86_64/floatdixf.c"),
-                        ("__floatundidf", "x86_64/floatundidf.S"),
-                        ("__floatundisf", "x86_64/floatundisf.S"),
                         ("__floatundixf", "x86_64/floatundixf.S"),
                     ]);
                 }
@@ -306,11 +300,7 @@ mod c {
                     ("__ashldi3", "i386/ashldi3.S"),
                     ("__ashrdi3", "i386/ashrdi3.S"),
                     ("__divdi3", "i386/divdi3.S"),
-                    ("__floatdidf", "i386/floatdidf.S"),
-                    ("__floatdisf", "i386/floatdisf.S"),
                     ("__floatdixf", "i386/floatdixf.S"),
-                    ("__floatundidf", "i386/floatundidf.S"),
-                    ("__floatundisf", "i386/floatundisf.S"),
                     ("__floatundixf", "i386/floatundixf.S"),
                     ("__lshrdi3", "i386/lshrdi3.S"),
                     ("__moddi3", "i386/moddi3.S"),

library/compiler-builtins/src/float/conv.rs

@@ -1,286 +1,351 @@
-use float::Float;
+/// Conversions from integers to floats.
-use int::{CastInto, Int};
+///
-
+/// These are hand-optimized bit twiddling code,
-fn int_to_float<I: Int, F: Float>(i: I) -> F
+/// which unfortunately isn't the easiest kind of code to read.
-where
+///
-    F::Int: CastInto<u32>,
+/// The algorithm is explained here: https://blog.m-ou.se/floats/
-    F::Int: CastInto<I>,
+mod int_to_float {
-    I::UnsignedInt: CastInto<F::Int>,
+    pub fn u32_to_f32_bits(i: u32) -> u32 {
-    u32: CastInto<F::Int>,
+        if i == 0 {
-{
+            return 0;
-    if i == I::ZERO {
+        }
-        return F::ZERO;
+        let n = i.leading_zeros();
+        let a = i << n >> 8; // Significant bits, with bit 24 still in tact.
+        let b = i << n << 24; // Insignificant bits, only relevant for rounding.
+        let m = a + ((b - (b >> 31 & !a)) >> 31); // Add one when we need to round up. Break ties to even.
+        let e = 157 - n as u32; // Exponent plus 127, minus one.
+        (e << 23) + m // + not |, so the mantissa can overflow into the exponent.
     }
 
-    let two = I::UnsignedInt::ONE + I::UnsignedInt::ONE;
+    pub fn u32_to_f64_bits(i: u32) -> u64 {
-    let four = two + two;
+        if i == 0 {
-    let sign = i < I::ZERO;
+            return 0;
-    let mut x = Int::abs_diff(i, I::ZERO);
+        }
-
+        let n = i.leading_zeros();
-    // number of significant digits in the integer
+        let m = (i as u64) << (21 + n); // Significant bits, with bit 53 still in tact.
-    let i_sd = I::BITS - x.leading_zeros();
+        let e = 1053 - n as u64; // Exponent plus 1023, minus one.
-    // significant digits for the float, including implicit bit
+        (e << 52) + m // Bit 53 of m will overflow into e.
-    let f_sd = F::SIGNIFICAND_BITS + 1;
-
-    // exponent
-    let mut exp = i_sd - 1;
-
-    if I::BITS < f_sd {
-        return F::from_parts(
-            sign,
-            (exp + F::EXPONENT_BIAS).cast(),
-            x.cast() << (f_sd - exp - 1),
-        );
     }
 
-    x = if i_sd > f_sd {
+    pub fn u64_to_f32_bits(i: u64) -> u32 {
-        // start:  0000000000000000000001xxxxxxxxxxxxxxxxxxxxxxPQxxxxxxxxxxxxxxxxxx
+        let n = i.leading_zeros();
-        // finish: 000000000000000000000000000000000000001xxxxxxxxxxxxxxxxxxxxxxPQR
+        let y = i.wrapping_shl(n);
-        //                                               12345678901234567890123456
+        let a = (y >> 40) as u32; // Significant bits, with bit 24 still in tact.
-        // 1 = the implicit bit
+        let b = (y >> 8 | y & 0xFFFF) as u32; // Insignificant bits, only relevant for rounding.
-        // P = bit f_sd-1 bits to the right of 1
+        let m = a + ((b - (b >> 31 & !a)) >> 31); // Add one when we need to round up. Break ties to even.
-        // Q = bit f_sd bits to the right of 1
+        let e = if i == 0 { 0 } else { 189 - n }; // Exponent plus 127, minus one, except for zero.
-        // R = "or" of all bits to the right of Q
+        (e << 23) + m // + not |, so the mantissa can overflow into the exponent.
-        let f_sd_add2 = f_sd + 2;
-        x = if i_sd == (f_sd + 1) {
-            x << 1
-        } else if i_sd == f_sd_add2 {
-            x
-        } else {
-            (x >> (i_sd - f_sd_add2))
-                | Int::from_bool(
-                    (x & I::UnsignedInt::MAX).wrapping_shl((I::BITS + f_sd_add2) - i_sd)
-                        != Int::ZERO,
-                )
-        };
-
-        // R |= P
-        x |= Int::from_bool((x & four) != I::UnsignedInt::ZERO);
-        // round - this step may add a significant bit
-        x += Int::ONE;
-        // dump Q and R
-        x >>= 2;
-
-        // a is now rounded to f_sd or f_sd+1 bits
-        if (x & (I::UnsignedInt::ONE << f_sd)) != Int::ZERO {
-            x >>= 1;
-            exp += 1;
     }
-        x
-    } else {
-        x.wrapping_shl(f_sd - i_sd)
-    };
 
-    F::from_parts(sign, (exp + F::EXPONENT_BIAS).cast(), x.cast())
+    pub fn u64_to_f64_bits(i: u64) -> u64 {
+        if i == 0 {
+            return 0;
+        }
+        let n = i.leading_zeros();
+        let a = (i << n >> 11) as u64; // Significant bits, with bit 53 still in tact.
+        let b = (i << n << 53) as u64; // Insignificant bits, only relevant for rounding.
+        let m = a + ((b - (b >> 63 & !a)) >> 63); // Add one when we need to round up. Break ties to even.
+        let e = 1085 - n as u64; // Exponent plus 1023, minus one.
+        (e << 52) + m // + not |, so the mantissa can overflow into the exponent.
+    }
+
+    pub fn u128_to_f32_bits(i: u128) -> u32 {
+        let n = i.leading_zeros();
+        let y = i.wrapping_shl(n);
+        let a = (y >> 104) as u32; // Significant bits, with bit 24 still in tact.
+        let b = (y >> 72) as u32 | (y << 32 >> 32 != 0) as u32; // Insignificant bits, only relevant for rounding.
+        let m = a + ((b - (b >> 31 & !a)) >> 31); // Add one when we need to round up. Break ties to even.
+        let e = if i == 0 { 0 } else { 253 - n }; // Exponent plus 127, minus one, except for zero.
+        (e << 23) + m // + not |, so the mantissa can overflow into the exponent.
+    }
+
+    pub fn u128_to_f64_bits(i: u128) -> u64 {
+        let n = i.leading_zeros();
+        let y = i.wrapping_shl(n);
+        let a = (y >> 75) as u64; // Significant bits, with bit 53 still in tact.
+        let b = (y >> 11 | y & 0xFFFF_FFFF) as u64; // Insignificant bits, only relevant for rounding.
+        let m = a + ((b - (b >> 63 & !a)) >> 63); // Add one when we need to round up. Break ties to even.
+        let e = if i == 0 { 0 } else { 1149 - n as u64 }; // Exponent plus 1023, minus one, except for zero.
+        (e << 52) + m // + not |, so the mantissa can overflow into the exponent.
+    }
 }
 
+// Conversions from unsigned integers to floats.
 intrinsics! {
-    #[arm_aeabi_alias = __aeabi_i2f]
-    pub extern "C" fn __floatsisf(i: i32) -> f32 {
-        int_to_float(i)
-    }
-
-    #[arm_aeabi_alias = __aeabi_i2d]
-    pub extern "C" fn __floatsidf(i: i32) -> f64 {
-        int_to_float(i)
-    }
-
-    #[maybe_use_optimized_c_shim]
-    #[arm_aeabi_alias = __aeabi_l2f]
-    pub extern "C" fn __floatdisf(i: i64) -> f32 {
-        // On x86_64 LLVM will use native instructions for this conversion, we
-        // can just do it directly
-        if cfg!(target_arch = "x86_64") {
-            i as f32
-        } else {
-            int_to_float(i)
-        }
-    }
-
-    #[maybe_use_optimized_c_shim]
-    #[arm_aeabi_alias = __aeabi_l2d]
-    pub extern "C" fn __floatdidf(i: i64) -> f64 {
-        // On x86_64 LLVM will use native instructions for this conversion, we
-        // can just do it directly
-        if cfg!(target_arch = "x86_64") {
-            i as f64
-        } else {
-            int_to_float(i)
-        }
-    }
-
     #[arm_aeabi_alias = __aeabi_ui2f]
     pub extern "C" fn __floatunsisf(i: u32) -> f32 {
-        int_to_float(i)
+        f32::from_bits(int_to_float::u32_to_f32_bits(i))
     }
 
     #[arm_aeabi_alias = __aeabi_ui2d]
     pub extern "C" fn __floatunsidf(i: u32) -> f64 {
-        int_to_float(i)
+        f64::from_bits(int_to_float::u32_to_f64_bits(i))
     }
 
-    #[maybe_use_optimized_c_shim]
     #[arm_aeabi_alias = __aeabi_ul2f]
     pub extern "C" fn __floatundisf(i: u64) -> f32 {
-        int_to_float(i)
+        f32::from_bits(int_to_float::u64_to_f32_bits(i))
     }
 
-    #[maybe_use_optimized_c_shim]
     #[arm_aeabi_alias = __aeabi_ul2d]
     pub extern "C" fn __floatundidf(i: u64) -> f64 {
-        int_to_float(i)
+        f64::from_bits(int_to_float::u64_to_f64_bits(i))
-    }
-}
-
-fn float_to_int<F: Float, I: Int>(f: F) -> I
-where
-    F::ExpInt: CastInto<u32>,
-    u32: CastInto<F::ExpInt>,
-    F::Int: CastInto<I>,
-{
-    // converting NaNs is UB, so we don't consider them
-
-    let sign = f.sign();
-    let mut exp = f.exp();
-
-    // if less than one or unsigned & negative
-    if (exp < F::EXPONENT_BIAS.cast()) || (!I::SIGNED && sign) {
-        return I::ZERO;
-    }
-    exp -= F::EXPONENT_BIAS.cast();
-
-    // If the value is too large for `I`, saturate.
-    let bits: F::ExpInt = I::BITS.cast();
-    let max = if I::SIGNED {
-        bits - F::ExpInt::ONE
-    } else {
-        bits
-    };
-    if max <= exp {
-        return if sign {
-            // It happens that I::MIN is handled correctly
-            I::MIN
-        } else {
-            I::MAX
-        };
-    };
-
-    // `0 <= exp < max`
-
-    // If 0 <= exponent < F::SIGNIFICAND_BITS, right shift to get the result. Otherwise, shift left.
-    let sig_bits: F::ExpInt = F::SIGNIFICAND_BITS.cast();
-    // The larger integer has to be casted into, or else the shift overflows
-    let r: I = if F::Int::BITS < I::BITS {
-        let tmp: I = if exp < sig_bits {
-            f.imp_frac().cast() >> (sig_bits - exp).cast()
-        } else {
-            f.imp_frac().cast() << (exp - sig_bits).cast()
-        };
-        tmp
-    } else {
-        let tmp: F::Int = if exp < sig_bits {
-            f.imp_frac() >> (sig_bits - exp).cast()
-        } else {
-            f.imp_frac() << (exp - sig_bits).cast()
-        };
-        tmp.cast()
-    };
-
-    if sign {
-        r.wrapping_neg()
-    } else {
-        r
-    }
-}
-
-intrinsics! {
-    #[arm_aeabi_alias = __aeabi_f2iz]
-    pub extern "C" fn __fixsfsi(f: f32) -> i32 {
-        float_to_int(f)
-    }
-
-    #[arm_aeabi_alias = __aeabi_f2lz]
-    pub extern "C" fn __fixsfdi(f: f32) -> i64 {
-        float_to_int(f)
-    }
-
-    #[arm_aeabi_alias = __aeabi_d2iz]
-    pub extern "C" fn __fixdfsi(f: f64) -> i32 {
-        float_to_int(f)
-    }
-
-    #[arm_aeabi_alias = __aeabi_d2lz]
-    pub extern "C" fn __fixdfdi(f: f64) -> i64 {
-        float_to_int(f)
-    }
-
-    #[arm_aeabi_alias = __aeabi_f2uiz]
-    pub extern "C" fn __fixunssfsi(f: f32) -> u32 {
-        float_to_int(f)
-    }
-
-    #[arm_aeabi_alias = __aeabi_f2ulz]
-    pub extern "C" fn __fixunssfdi(f: f32) -> u64 {
-        float_to_int(f)
-    }
-
-    #[arm_aeabi_alias = __aeabi_d2uiz]
-    pub extern "C" fn __fixunsdfsi(f: f64) -> u32 {
-        float_to_int(f)
-    }
-
-    #[arm_aeabi_alias = __aeabi_d2ulz]
-    pub extern "C" fn __fixunsdfdi(f: f64) -> u64 {
-        float_to_int(f)
-    }
-}
-
-// The ABI for the following intrinsics changed in LLVM 14. On Win64, they now
-// use Win64 ABI rather than unadjusted ABI. Pick the correct ABI based on the
-// llvm14-builtins-abi target feature.
-
-intrinsics! {
-    #[cfg_attr(not(target_feature = "llvm14-builtins-abi"), unadjusted_on_win64)]
-    pub extern "C" fn __floattisf(i: i128) -> f32 {
-        int_to_float(i)
-    }
-
-    #[cfg_attr(not(target_feature = "llvm14-builtins-abi"), unadjusted_on_win64)]
-    pub extern "C" fn __floattidf(i: i128) -> f64 {
-        int_to_float(i)
     }
 
     #[cfg_attr(not(target_feature = "llvm14-builtins-abi"), unadjusted_on_win64)]
     pub extern "C" fn __floatuntisf(i: u128) -> f32 {
-        int_to_float(i)
+        f32::from_bits(int_to_float::u128_to_f32_bits(i))
     }
 
     #[cfg_attr(not(target_feature = "llvm14-builtins-abi"), unadjusted_on_win64)]
     pub extern "C" fn __floatuntidf(i: u128) -> f64 {
-        int_to_float(i)
+        f64::from_bits(int_to_float::u128_to_f64_bits(i))
+    }
+}
+
+// Conversions from signed integers to floats.
+intrinsics! {
+    #[arm_aeabi_alias = __aeabi_i2f]
+    pub extern "C" fn __floatsisf(i: i32) -> f32 {
+        let sign_bit = ((i >> 31) as u32) << 31;
+        f32::from_bits(int_to_float::u32_to_f32_bits(i.unsigned_abs()) | sign_bit)
     }
 
-    #[cfg_attr(target_feature = "llvm14-builtins-abi", win64_128bit_abi_hack)]
+    #[arm_aeabi_alias = __aeabi_i2d]
-    #[cfg_attr(not(target_feature = "llvm14-builtins-abi"), unadjusted_on_win64)]
+    pub extern "C" fn __floatsidf(i: i32) -> f64 {
-    pub extern "C" fn __fixsfti(f: f32) -> i128 {
+        let sign_bit = ((i >> 31) as u64) << 63;
-        float_to_int(f)
+        f64::from_bits(int_to_float::u32_to_f64_bits(i.unsigned_abs()) | sign_bit)
+    }
+
+    #[arm_aeabi_alias = __aeabi_l2f]
+    pub extern "C" fn __floatdisf(i: i64) -> f32 {
+        let sign_bit = ((i >> 63) as u32) << 31;
+        f32::from_bits(int_to_float::u64_to_f32_bits(i.unsigned_abs()) | sign_bit)
+    }
+
+    #[arm_aeabi_alias = __aeabi_l2d]
+    pub extern "C" fn __floatdidf(i: i64) -> f64 {
+        let sign_bit = ((i >> 63) as u64) << 63;
+        f64::from_bits(int_to_float::u64_to_f64_bits(i.unsigned_abs()) | sign_bit)
     }
 
-    #[cfg_attr(target_feature = "llvm14-builtins-abi", win64_128bit_abi_hack)]
     #[cfg_attr(not(target_feature = "llvm14-builtins-abi"), unadjusted_on_win64)]
-    pub extern "C" fn __fixdfti(f: f64) -> i128 {
+    pub extern "C" fn __floattisf(i: i128) -> f32 {
-        float_to_int(f)
+        let sign_bit = ((i >> 127) as u32) << 31;
+        f32::from_bits(int_to_float::u128_to_f32_bits(i.unsigned_abs()) | sign_bit)
+    }
+
+    #[cfg_attr(not(target_feature = "llvm14-builtins-abi"), unadjusted_on_win64)]
+    pub extern "C" fn __floattidf(i: i128) -> f64 {
+        let sign_bit = ((i >> 127) as u64) << 63;
+        f64::from_bits(int_to_float::u128_to_f64_bits(i.unsigned_abs()) | sign_bit)
+    }
+}
+
+// Conversions from floats to unsigned integers.
+intrinsics! {
+    #[arm_aeabi_alias = __aeabi_f2uiz]
+    pub extern "C" fn __fixunssfsi(f: f32) -> u32 {
+        let fbits = f.to_bits();
+        if fbits < 127 << 23 { // >= 0, < 1
+            0
+        } else if fbits < 159 << 23 { // >= 1, < max
+            let m = 1 << 31 | fbits << 8; // Mantissa and the implicit 1-bit.
+            let s = 158 - (fbits >> 23); // Shift based on the exponent and bias.
+            m >> s
+        } else if fbits <= 255 << 23 { // >= max (incl. inf)
+            u32::MAX
+        } else { // Negative or NaN
+            0
+        }
+    }
+
+    #[arm_aeabi_alias = __aeabi_f2ulz]
+    pub extern "C" fn __fixunssfdi(f: f32) -> u64 {
+        let fbits = f.to_bits();
+        if fbits < 127 << 23 { // >= 0, < 1
+            0
+        } else if fbits < 191 << 23 { // >= 1, < max
+            let m = 1 << 63 | (fbits as u64) << 40; // Mantissa and the implicit 1-bit.
+            let s = 190 - (fbits >> 23); // Shift based on the exponent and bias.
+            m >> s
+        } else if fbits <= 255 << 23 { // >= max (incl. inf)
+            u64::MAX
+        } else { // Negative or NaN
+            0
+        }
     }
 
     #[cfg_attr(target_feature = "llvm14-builtins-abi", win64_128bit_abi_hack)]
     #[cfg_attr(not(target_feature = "llvm14-builtins-abi"), unadjusted_on_win64)]
     pub extern "C" fn __fixunssfti(f: f32) -> u128 {
-        float_to_int(f)
+        let fbits = f.to_bits();
+        if fbits < 127 << 23 { // >= 0, < 1
+            0
+        } else if fbits < 255 << 23 { // >= 1, < inf
+            let m = 1 << 127 | (fbits as u128) << 104; // Mantissa and the implicit 1-bit.
+            let s = 254 - (fbits >> 23); // Shift based on the exponent and bias.
+            m >> s
+        } else if fbits == 255 << 23 { // == inf
+            u128::MAX
+        } else { // Negative or NaN
+            0
+        }
+    }
+
+    #[arm_aeabi_alias = __aeabi_d2uiz]
+    pub extern "C" fn __fixunsdfsi(f: f64) -> u32 {
+        let fbits = f.to_bits();
+        if fbits < 1023 << 52 { // >= 0, < 1
+            0
+        } else if fbits < 1055 << 52 { // >= 1, < max
+            let m = 1 << 31 | (fbits >> 21) as u32; // Mantissa and the implicit 1-bit.
+            let s = 1054 - (fbits >> 52); // Shift based on the exponent and bias.
+            m >> s
+        } else if fbits <= 2047 << 52 { // >= max (incl. inf)
+            u32::MAX
+        } else { // Negative or NaN
+            0
+        }
+    }
+
+    #[arm_aeabi_alias = __aeabi_d2ulz]
+    pub extern "C" fn __fixunsdfdi(f: f64) -> u64 {
+        let fbits = f.to_bits();
+        if fbits < 1023 << 52 { // >= 0, < 1
+            0
+        } else if fbits < 1087 << 52 { // >= 1, < max
+            let m = 1 << 63 | fbits << 11; // Mantissa and the implicit 1-bit.
+            let s = 1086 - (fbits >> 52); // Shift based on the exponent and bias.
+            m >> s
+        } else if fbits <= 2047 << 52 { // >= max (incl. inf)
+            u64::MAX
+        } else { // Negative or NaN
+            0
+        }
     }
 
     #[cfg_attr(target_feature = "llvm14-builtins-abi", win64_128bit_abi_hack)]
     #[cfg_attr(not(target_feature = "llvm14-builtins-abi"), unadjusted_on_win64)]
     pub extern "C" fn __fixunsdfti(f: f64) -> u128 {
-        float_to_int(f)
+        let fbits = f.to_bits();
+        if fbits < 1023 << 52 { // >= 0, < 1
+            0
+        } else if fbits < 1151 << 52 { // >= 1, < max
+            let m = 1 << 127 | (fbits as u128) << 75; // Mantissa and the implicit 1-bit.
+            let s = 1150 - (fbits >> 52); // Shift based on the exponent and bias.
+            m >> s
+        } else if fbits <= 2047 << 52 { // >= max (incl. inf)
+            u128::MAX
+        } else { // Negative or NaN
+            0
+        }
+    }
+}
+
+// Conversions from floats to signed integers.
+intrinsics! {
+    #[arm_aeabi_alias = __aeabi_f2iz]
+    pub extern "C" fn __fixsfsi(f: f32) -> i32 {
+        let fbits = f.to_bits() & !0 >> 1; // Remove sign bit.
+        if fbits < 127 << 23 { // >= 0, < 1
+            0
+        } else if fbits < 158 << 23 { // >= 1, < max
+            let m = 1 << 31 | fbits << 8; // Mantissa and the implicit 1-bit.
+            let s = 158 - (fbits >> 23); // Shift based on the exponent and bias.
+            let u = (m >> s) as i32; // Unsigned result.
+            if f.is_sign_negative() { -u } else { u }
+        } else if fbits <= 255 << 23 { // >= max (incl. inf)
+            if f.is_sign_negative() { i32::MIN } else { i32::MAX }
+        } else { // NaN
+            0
+        }
+    }
+
+    #[arm_aeabi_alias = __aeabi_f2lz]
+    pub extern "C" fn __fixsfdi(f: f32) -> i64 {
+        let fbits = f.to_bits() & !0 >> 1; // Remove sign bit.
+        if fbits < 127 << 23 { // >= 0, < 1
+            0
+        } else if fbits < 190 << 23 { // >= 1, < max
+            let m = 1 << 63 | (fbits as u64) << 40; // Mantissa and the implicit 1-bit.
+            let s = 190 - (fbits >> 23); // Shift based on the exponent and bias.
+            let u = (m >> s) as i64; // Unsigned result.
+            if f.is_sign_negative() { -u } else { u }
+        } else if fbits <= 255 << 23 { // >= max (incl. inf)
+            if f.is_sign_negative() { i64::MIN } else { i64::MAX }
+        } else { // NaN
+            0
+        }
+    }
+
+    #[cfg_attr(target_feature = "llvm14-builtins-abi", win64_128bit_abi_hack)]
+    #[cfg_attr(not(target_feature = "llvm14-builtins-abi"), unadjusted_on_win64)]
+    pub extern "C" fn __fixsfti(f: f32) -> i128 {
+        let fbits = f.to_bits() & !0 >> 1; // Remove sign bit.
+        if fbits < 127 << 23 { // >= 0, < 1
+            0
+        } else if fbits < 254 << 23 { // >= 1, < max
+            let m = 1 << 127 | (fbits as u128) << 104; // Mantissa and the implicit 1-bit.
+            let s = 254 - (fbits >> 23); // Shift based on the exponent and bias.
+            let u = (m >> s) as i128; // Unsigned result.
+            if f.is_sign_negative() { -u } else { u }
+        } else if fbits <= 255 << 23 { // >= max (incl. inf)
+            if f.is_sign_negative() { i128::MIN } else { i128::MAX }
+        } else { // NaN
+            0
+        }
+    }
+
+    #[arm_aeabi_alias = __aeabi_d2iz]
+    pub extern "C" fn __fixdfsi(f: f64) -> i32 {
+        let fbits = f.to_bits() & !0 >> 1; // Remove sign bit.
+        if fbits < 1023 << 52 { // >= 0, < 1
+            0
+        } else if fbits < 1054 << 52 { // >= 1, < max
+            let m = 1 << 31 | (fbits >> 21) as u32; // Mantissa and the implicit 1-bit.
+            let s = 1054 - (fbits >> 52); // Shift based on the exponent and bias.
+            let u = (m >> s) as i32; // Unsigned result.
+            if f.is_sign_negative() { -u } else { u }
+        } else if fbits <= 2047 << 52 { // >= max (incl. inf)
+            if f.is_sign_negative() { i32::MIN } else { i32::MAX }
+        } else { // NaN
+            0
+        }
+    }
+
+    #[arm_aeabi_alias = __aeabi_d2lz]
+    pub extern "C" fn __fixdfdi(f: f64) -> i64 {
+        let fbits = f.to_bits() & !0 >> 1; // Remove sign bit.
+        if fbits < 1023 << 52 { // >= 0, < 1
+            0
+        } else if fbits < 1086 << 52 { // >= 1, < max
+            let m = 1 << 63 | fbits << 11; // Mantissa and the implicit 1-bit.
+            let s = 1086 - (fbits >> 52); // Shift based on the exponent and bias.
+            let u = (m >> s) as i64; // Unsigned result.
+            if f.is_sign_negative() { -u } else { u }
+        } else if fbits <= 2047 << 52 { // >= max (incl. inf)
+            if f.is_sign_negative() { i64::MIN } else { i64::MAX }
+        } else { // NaN
+            0
+        }
+    }
+
+    #[cfg_attr(target_feature = "llvm14-builtins-abi", win64_128bit_abi_hack)]
+    #[cfg_attr(not(target_feature = "llvm14-builtins-abi"), unadjusted_on_win64)]
+    pub extern "C" fn __fixdfti(f: f64) -> i128 {
+        let fbits = f.to_bits() & !0 >> 1; // Remove sign bit.
+        if fbits < 1023 << 52 { // >= 0, < 1
+            0
+        } else if fbits < 1150 << 52 { // >= 1, < max
+            let m = 1 << 127 | (fbits as u128) << 75; // Mantissa and the implicit 1-bit.
+            let s = 1150 - (fbits >> 52); // Shift based on the exponent and bias.
+            let u = (m >> s) as i128; // Unsigned result.
+            if f.is_sign_negative() { -u } else { u }
+        } else if fbits <= 2047 << 52 { // >= max (incl. inf)
+            if f.is_sign_negative() { i128::MIN } else { i128::MAX }
+        } else { // NaN
+            0
+        }
     }
 }