Lazily allocate+initialize locks.

This commit is contained in:
Mara Bos
2022-06-03 17:04:14 +02:00
parent ac5aa1ded5
commit 6a417d4828
17 changed files with 145 additions and 36 deletions

View File

@@ -1,4 +1,5 @@
use crate::sys::locks::Mutex; use crate::sys::locks::Mutex;
use crate::sys_common::lazy_box::{LazyBox, LazyInit};
use crate::time::Duration; use crate::time::Duration;
use super::waitqueue::{SpinMutex, WaitQueue, WaitVariable}; use super::waitqueue::{SpinMutex, WaitQueue, WaitVariable};
@@ -7,16 +8,19 @@ pub struct Condvar {
inner: SpinMutex<WaitVariable<()>>, inner: SpinMutex<WaitVariable<()>>,
} }
pub type MovableCondvar = Box<Condvar>; pub(crate) type MovableCondvar = LazyBox<Condvar>;
impl LazyInit for Condvar {
fn init() -> Box<Self> {
Box::new(Self::new())
}
}
impl Condvar { impl Condvar {
pub const fn new() -> Condvar { pub const fn new() -> Condvar {
Condvar { inner: SpinMutex::new(WaitVariable::new(())) } Condvar { inner: SpinMutex::new(WaitVariable::new(())) }
} }
#[inline]
pub unsafe fn init(&mut self) {}
#[inline] #[inline]
pub unsafe fn notify_one(&self) { pub unsafe fn notify_one(&self) {
let _ = WaitQueue::notify_one(self.inner.lock()); let _ = WaitQueue::notify_one(self.inner.lock());

View File

@@ -1,11 +1,18 @@
use super::waitqueue::{try_lock_or_false, SpinMutex, WaitQueue, WaitVariable}; use super::waitqueue::{try_lock_or_false, SpinMutex, WaitQueue, WaitVariable};
use crate::sys_common::lazy_box::{LazyBox, LazyInit};
pub struct Mutex { pub struct Mutex {
inner: SpinMutex<WaitVariable<bool>>, inner: SpinMutex<WaitVariable<bool>>,
} }
// not movable: see UnsafeList implementation // not movable: see UnsafeList implementation
pub type MovableMutex = Box<Mutex>; pub(crate) type MovableMutex = LazyBox<Mutex>;
impl LazyInit for Mutex {
fn init() -> Box<Self> {
Box::new(Self::new())
}
}
// Implementation according to “Operating Systems: Three Easy Pieces”, chapter 28 // Implementation according to “Operating Systems: Three Easy Pieces”, chapter 28
impl Mutex { impl Mutex {

View File

@@ -2,6 +2,7 @@
mod tests; mod tests;
use crate::num::NonZeroUsize; use crate::num::NonZeroUsize;
use crate::sys_common::lazy_box::{LazyBox, LazyInit};
use super::waitqueue::{ use super::waitqueue::{
try_lock_or_false, NotifiedTcs, SpinMutex, SpinMutexGuard, WaitQueue, WaitVariable, try_lock_or_false, NotifiedTcs, SpinMutex, SpinMutexGuard, WaitQueue, WaitVariable,
@@ -13,7 +14,13 @@ pub struct RwLock {
writer: SpinMutex<WaitVariable<bool>>, writer: SpinMutex<WaitVariable<bool>>,
} }
pub type MovableRwLock = Box<RwLock>; pub(crate) type MovableRwLock = LazyBox<RwLock>;
impl LazyInit for RwLock {
fn init() -> Box<Self> {
Box::new(Self::new())
}
}
// Check at compile time that RwLock size matches C definition (see test_c_rwlock_initializer below) // Check at compile time that RwLock size matches C definition (see test_c_rwlock_initializer below)
// //

View File

@@ -115,9 +115,6 @@ impl Condvar {
Self { futex: AtomicU32::new(0) } Self { futex: AtomicU32::new(0) }
} }
#[inline]
pub unsafe fn init(&mut self) {}
// All the memory orderings here are `Relaxed`, // All the memory orderings here are `Relaxed`,
// because synchronization is done by unlocking and locking the mutex. // because synchronization is done by unlocking and locking the mutex.

View File

@@ -9,14 +9,14 @@ cfg_if::cfg_if! {
))] { ))] {
mod futex; mod futex;
mod futex_rwlock; mod futex_rwlock;
pub use futex::{Mutex, MovableMutex, Condvar, MovableCondvar}; pub(crate) use futex::{Mutex, MovableMutex, MovableCondvar};
pub use futex_rwlock::{RwLock, MovableRwLock}; pub(crate) use futex_rwlock::{RwLock, MovableRwLock};
} else { } else {
mod pthread_mutex; mod pthread_mutex;
mod pthread_rwlock; mod pthread_rwlock;
mod pthread_condvar; mod pthread_condvar;
pub use pthread_mutex::{Mutex, MovableMutex}; pub(crate) use pthread_mutex::{Mutex, MovableMutex};
pub use pthread_rwlock::{RwLock, MovableRwLock}; pub(crate) use pthread_rwlock::{RwLock, MovableRwLock};
pub use pthread_condvar::{Condvar, MovableCondvar}; pub(crate) use pthread_condvar::MovableCondvar;
} }
} }

View File

@@ -1,12 +1,13 @@
use crate::cell::UnsafeCell; use crate::cell::UnsafeCell;
use crate::sys::locks::{pthread_mutex, Mutex}; use crate::sys::locks::{pthread_mutex, Mutex};
use crate::sys_common::lazy_box::{LazyBox, LazyInit};
use crate::time::Duration; use crate::time::Duration;
pub struct Condvar { pub struct Condvar {
inner: UnsafeCell<libc::pthread_cond_t>, inner: UnsafeCell<libc::pthread_cond_t>,
} }
pub type MovableCondvar = Box<Condvar>; pub(crate) type MovableCondvar = LazyBox<Condvar>;
unsafe impl Send for Condvar {} unsafe impl Send for Condvar {}
unsafe impl Sync for Condvar {} unsafe impl Sync for Condvar {}
@@ -18,6 +19,14 @@ fn saturating_cast_to_time_t(value: u64) -> libc::time_t {
if value > <libc::time_t>::MAX as u64 { <libc::time_t>::MAX } else { value as libc::time_t } if value > <libc::time_t>::MAX as u64 { <libc::time_t>::MAX } else { value as libc::time_t }
} }
impl LazyInit for Condvar {
fn init() -> Box<Self> {
let mut condvar = Box::new(Self::new());
unsafe { condvar.init() };
condvar
}
}
impl Condvar { impl Condvar {
pub const fn new() -> Condvar { pub const fn new() -> Condvar {
// Might be moved and address is changing it is better to avoid // Might be moved and address is changing it is better to avoid
@@ -32,14 +41,14 @@ impl Condvar {
target_os = "android", target_os = "android",
target_os = "redox" target_os = "redox"
))] ))]
pub unsafe fn init(&mut self) {} unsafe fn init(&mut self) {}
// NOTE: ESP-IDF's PTHREAD_COND_INITIALIZER support is not released yet // NOTE: ESP-IDF's PTHREAD_COND_INITIALIZER support is not released yet
// So on that platform, init() should always be called // So on that platform, init() should always be called
// Moreover, that platform does not have pthread_condattr_setclock support, // Moreover, that platform does not have pthread_condattr_setclock support,
// hence that initialization should be skipped as well // hence that initialization should be skipped as well
#[cfg(target_os = "espidf")] #[cfg(target_os = "espidf")]
pub unsafe fn init(&mut self) { unsafe fn init(&mut self) {
let r = libc::pthread_cond_init(self.inner.get(), crate::ptr::null()); let r = libc::pthread_cond_init(self.inner.get(), crate::ptr::null());
assert_eq!(r, 0); assert_eq!(r, 0);
} }
@@ -52,7 +61,7 @@ impl Condvar {
target_os = "redox", target_os = "redox",
target_os = "espidf" target_os = "espidf"
)))] )))]
pub unsafe fn init(&mut self) { unsafe fn init(&mut self) {
use crate::mem::MaybeUninit; use crate::mem::MaybeUninit;
let mut attr = MaybeUninit::<libc::pthread_condattr_t>::uninit(); let mut attr = MaybeUninit::<libc::pthread_condattr_t>::uninit();
let r = libc::pthread_condattr_init(attr.as_mut_ptr()); let r = libc::pthread_condattr_init(attr.as_mut_ptr());

View File

@@ -1,12 +1,13 @@
use crate::cell::UnsafeCell; use crate::cell::UnsafeCell;
use crate::mem::MaybeUninit; use crate::mem::MaybeUninit;
use crate::sys::cvt_nz; use crate::sys::cvt_nz;
use crate::sys_common::lazy_box::{LazyBox, LazyInit};
pub struct Mutex { pub struct Mutex {
inner: UnsafeCell<libc::pthread_mutex_t>, inner: UnsafeCell<libc::pthread_mutex_t>,
} }
pub type MovableMutex = Box<Mutex>; pub(crate) type MovableMutex = LazyBox<Mutex>;
#[inline] #[inline]
pub unsafe fn raw(m: &Mutex) -> *mut libc::pthread_mutex_t { pub unsafe fn raw(m: &Mutex) -> *mut libc::pthread_mutex_t {
@@ -16,6 +17,14 @@ pub unsafe fn raw(m: &Mutex) -> *mut libc::pthread_mutex_t {
unsafe impl Send for Mutex {} unsafe impl Send for Mutex {}
unsafe impl Sync for Mutex {} unsafe impl Sync for Mutex {}
impl LazyInit for Mutex {
fn init() -> Box<Self> {
let mut mutex = Box::new(Self::new());
unsafe { mutex.init() };
mutex
}
}
impl Mutex { impl Mutex {
pub const fn new() -> Mutex { pub const fn new() -> Mutex {
// Might be moved to a different address, so it is better to avoid // Might be moved to a different address, so it is better to avoid

View File

@@ -1,5 +1,6 @@
use crate::cell::UnsafeCell; use crate::cell::UnsafeCell;
use crate::sync::atomic::{AtomicUsize, Ordering}; use crate::sync::atomic::{AtomicUsize, Ordering};
use crate::sys_common::lazy_box::{LazyBox, LazyInit};
pub struct RwLock { pub struct RwLock {
inner: UnsafeCell<libc::pthread_rwlock_t>, inner: UnsafeCell<libc::pthread_rwlock_t>,
@@ -7,11 +8,17 @@ pub struct RwLock {
num_readers: AtomicUsize, num_readers: AtomicUsize,
} }
pub type MovableRwLock = Box<RwLock>; pub(crate) type MovableRwLock = LazyBox<RwLock>;
unsafe impl Send for RwLock {} unsafe impl Send for RwLock {}
unsafe impl Sync for RwLock {} unsafe impl Sync for RwLock {}
impl LazyInit for RwLock {
fn init() -> Box<Self> {
Box::new(Self::new())
}
}
impl RwLock { impl RwLock {
pub const fn new() -> RwLock { pub const fn new() -> RwLock {
RwLock { RwLock {

View File

@@ -10,9 +10,6 @@ impl Condvar {
Condvar {} Condvar {}
} }
#[inline]
pub unsafe fn init(&mut self) {}
#[inline] #[inline]
pub unsafe fn notify_one(&self) {} pub unsafe fn notify_one(&self) {}

View File

@@ -54,8 +54,8 @@ cfg_if::cfg_if! {
#![allow(unsafe_op_in_unsafe_fn)] #![allow(unsafe_op_in_unsafe_fn)]
mod futex; mod futex;
mod futex_rwlock; mod futex_rwlock;
pub use futex::{Mutex, MovableMutex, Condvar, MovableCondvar}; pub(crate) use futex::{Mutex, MovableMutex, Condvar, MovableCondvar};
pub use futex_rwlock::{RwLock, MovableRwLock}; pub(crate) use futex_rwlock::{RwLock, MovableRwLock};
} }
#[path = "atomics/futex.rs"] #[path = "atomics/futex.rs"]
pub mod futex; pub mod futex;

View File

@@ -18,9 +18,6 @@ impl Condvar {
Condvar { inner: UnsafeCell::new(c::CONDITION_VARIABLE_INIT) } Condvar { inner: UnsafeCell::new(c::CONDITION_VARIABLE_INIT) }
} }
#[inline]
pub unsafe fn init(&mut self) {}
#[inline] #[inline]
pub unsafe fn wait(&self, mutex: &Mutex) { pub unsafe fn wait(&self, mutex: &Mutex) {
let r = c::SleepConditionVariableSRW(self.inner.get(), mutex::raw(mutex), c::INFINITE, 0); let r = c::SleepConditionVariableSRW(self.inner.get(), mutex::raw(mutex), c::INFINITE, 0);

View File

@@ -15,9 +15,7 @@ pub struct Condvar {
impl Condvar { impl Condvar {
/// Creates a new condition variable for use. /// Creates a new condition variable for use.
pub fn new() -> Self { pub fn new() -> Self {
let mut c = imp::MovableCondvar::from(imp::Condvar::new()); Self { inner: imp::MovableCondvar::new(), check: CondvarCheck::new() }
unsafe { c.init() };
Self { inner: c, check: CondvarCheck::new() }
} }
/// Signals one waiter on this condition variable to wake up. /// Signals one waiter on this condition variable to wake up.

View File

@@ -1,6 +1,7 @@
use crate::ptr; use crate::ptr;
use crate::sync::atomic::{AtomicPtr, Ordering}; use crate::sync::atomic::{AtomicPtr, Ordering};
use crate::sys::locks as imp; use crate::sys::locks as imp;
use crate::sys_common::lazy_box::{LazyBox, LazyInit};
use crate::sys_common::mutex::MovableMutex; use crate::sys_common::mutex::MovableMutex;
pub trait CondvarCheck { pub trait CondvarCheck {
@@ -9,7 +10,7 @@ pub trait CondvarCheck {
/// For boxed mutexes, a `Condvar` will check it's only ever used with the same /// For boxed mutexes, a `Condvar` will check it's only ever used with the same
/// mutex, based on its (stable) address. /// mutex, based on its (stable) address.
impl CondvarCheck for Box<imp::Mutex> { impl<T: LazyInit> CondvarCheck for LazyBox<T> {
type Check = SameMutexCheck; type Check = SameMutexCheck;
} }

View File

@@ -0,0 +1,77 @@
#![allow(dead_code)] // Only used on some platforms.
// This is used to wrap pthread {Mutex, Condvar, RwLock} in.
use crate::marker::PhantomData;
use crate::ops::{Deref, DerefMut};
use crate::ptr::null_mut;
use crate::sync::atomic::{
AtomicPtr,
Ordering::{AcqRel, Acquire},
};
pub(crate) struct LazyBox<T: LazyInit> {
ptr: AtomicPtr<T>,
_phantom: PhantomData<T>,
}
pub(crate) trait LazyInit {
/// This is called before the box is allocated, to provide the value to
/// move into the new box.
///
/// It might be called more than once per LazyBox, as multiple threads
/// might race to initialize it concurrently, each constructing and initializing
/// their own box. (All but one of them will be destroyed right after.)
fn init() -> Box<Self>;
}
impl<T: LazyInit> LazyBox<T> {
#[inline]
pub const fn new() -> Self {
Self { ptr: AtomicPtr::new(null_mut()), _phantom: PhantomData }
}
#[inline]
fn get_pointer(&self) -> *mut T {
let ptr = self.ptr.load(Acquire);
if ptr.is_null() { self.initialize() } else { ptr }
}
#[cold]
fn initialize(&self) -> *mut T {
let new_ptr = Box::into_raw(T::init());
match self.ptr.compare_exchange(null_mut(), new_ptr, AcqRel, Acquire) {
Ok(_) => new_ptr,
Err(ptr) => {
// Lost the race to another thread.
// Drop the box we created, and use the one from the other thread instead.
drop(unsafe { Box::from_raw(new_ptr) });
ptr
}
}
}
}
impl<T: LazyInit> Deref for LazyBox<T> {
type Target = T;
#[inline]
fn deref(&self) -> &T {
unsafe { &*self.get_pointer() }
}
}
impl<T: LazyInit> DerefMut for LazyBox<T> {
#[inline]
fn deref_mut(&mut self) -> &mut T {
unsafe { &mut *self.get_pointer() }
}
}
impl<T: LazyInit> Drop for LazyBox<T> {
fn drop(&mut self) {
let ptr = *self.ptr.get_mut();
if !ptr.is_null() {
drop(unsafe { Box::from_raw(ptr) });
}
}
}

View File

@@ -24,6 +24,7 @@ pub mod backtrace;
pub mod condvar; pub mod condvar;
pub mod fs; pub mod fs;
pub mod io; pub mod io;
pub mod lazy_box;
pub mod memchr; pub mod memchr;
pub mod mutex; pub mod mutex;
pub mod process; pub mod process;

View File

@@ -61,9 +61,7 @@ unsafe impl Sync for MovableMutex {}
impl MovableMutex { impl MovableMutex {
/// Creates a new mutex. /// Creates a new mutex.
pub fn new() -> Self { pub fn new() -> Self {
let mut mutex = imp::MovableMutex::from(imp::Mutex::new()); Self(imp::MovableMutex::new())
unsafe { mutex.init() };
Self(mutex)
} }
pub(super) fn raw(&self) -> &imp::Mutex { pub(super) fn raw(&self) -> &imp::Mutex {

View File

@@ -74,7 +74,7 @@ pub struct MovableRwLock(imp::MovableRwLock);
impl MovableRwLock { impl MovableRwLock {
/// Creates a new reader-writer lock for use. /// Creates a new reader-writer lock for use.
pub fn new() -> Self { pub fn new() -> Self {
Self(imp::MovableRwLock::from(imp::RwLock::new())) Self(imp::MovableRwLock::new())
} }
/// Acquires shared access to the underlying lock, blocking the current /// Acquires shared access to the underlying lock, blocking the current