rust-lang/rust
1
0
Fork 0
Code Issues Pull Requests Actions 1 Packages Projects Releases Wiki Activity

Auto merge of #87123 - RalfJung:miri-provenance-overhaul, r=oli-obk

Browse Source 
CTFE/Miri engine Pointer type overhaul

This fixes the long-standing problem that we are using `Scalar` as a type to represent pointers that might be integer values (since they point to a ZST). The main problem is that with int-to-ptr casts, there are multiple ways to represent the same pointer as a `Scalar` and it is unclear if "normalization" (i.e., the cast) already happened or not. This leads to ugly methods like `force_mplace_ptr` and `force_op_ptr`.
Another problem this solves is that in Miri, it would make a lot more sense to have the `Pointer::offset` field represent the full absolute address (instead of being relative to the `AllocId`). This means we can do ptr-to-int casts without access to any machine state, and it means that the overflow checks on pointer arithmetic are (finally!) accurate.

To solve this, the `Pointer` type is made entirely parametric over the provenance, so that we can use `Pointer<AllocId>` inside `Scalar` but use `Pointer<Option<AllocId>>` when accessing memory (where `None` represents the case that we could not figure out an `AllocId`; in that case the `offset` is an absolute address). Moreover, the `Provenance` trait determines if a pointer with a given provenance can be cast to an integer by simply dropping the provenance.

I hope this can be read commit-by-commit, but the first commit does the bulk of the work. It introduces some FIXMEs that are resolved later.
Fixes https://github.com/rust-lang/miri/issues/841
Miri PR: https://github.com/rust-lang/miri/pull/1851
r? `@oli-obk`
This commit is contained in:
bors
2021-07-17 15:26:27 +00:00
parent f502bd3abd efbee50600
commit c78ebb7bdc
106 changed files with 1317 additions and 1407 deletions
Download Patch File Download Diff File Expand all files Collapse all files

412
compiler/rustc_mir/src/interpret/memory.rs
View File

@@ -9,7 +9,7 @@
use std::assert_matches::assert_matches;
use std::borrow::Cow;
use std::collections::VecDeque;
use std::convert::{TryFrom, TryInto};
use std::convert::TryFrom;
use std::fmt;
use std::ptr;
@@ -20,7 +20,8 @@ use rustc_target::abi::{Align, HasDataLayout, Size, TargetDataLayout};
use super::{
alloc_range, AllocId, AllocMap, AllocRange, Allocation, CheckInAllocMsg, GlobalAlloc,
InterpResult, Machine, MayLeak, Pointer, PointerArithmetic, Scalar, ScalarMaybeUninit,
InterpResult, Machine, MayLeak, Pointer, PointerArithmetic, Provenance, Scalar,
ScalarMaybeUninit,
};
use crate::util::pretty;
@@ -163,25 +164,22 @@ impl<'mir, 'tcx, M: Machine<'mir, 'tcx>> Memory<'mir, 'tcx, M> {
#[inline]
pub fn global_base_pointer(
&self,
mut ptr: Pointer,
ptr: Pointer<AllocId>,
) -> InterpResult<'tcx, Pointer<M::PointerTag>> {
// We know `offset` is relative to the allocation, so we can use `into_parts`.
let (alloc_id, offset) = ptr.into_parts();
// We need to handle `extern static`.
let ptr = match self.tcx.get_global_alloc(ptr.alloc_id) {
match self.tcx.get_global_alloc(alloc_id) {
Some(GlobalAlloc::Static(def_id)) if self.tcx.is_thread_local_static(def_id) => {
bug!("global memory cannot point to thread-local static")
}
Some(GlobalAlloc::Static(def_id)) if self.tcx.is_foreign_item(def_id) => {
ptr.alloc_id = M::extern_static_alloc_id(self, def_id)?;
ptr
return M::extern_static_base_pointer(self, def_id);
}
_ => {
// No need to change the `AllocId`.
ptr
}
};
_ => {}
}
// And we need to get the tag.
let tag = M::tag_global_base_pointer(&self.extra, ptr.alloc_id);
Ok(ptr.with_tag(tag))
Ok(M::tag_alloc_base_pointer(self, Pointer::new(alloc_id, offset)))
}
pub fn create_fn_alloc(
@@ -235,21 +233,21 @@ impl<'mir, 'tcx, M: Machine<'mir, 'tcx>> Memory<'mir, 'tcx, M> {
M::GLOBAL_KIND.map(MemoryKind::Machine),
"dynamically allocating global memory"
);
// This is a new allocation, not a new global one, so no `global_base_ptr`.
let (alloc, tag) = M::init_allocation_extra(&self.extra, id, Cow::Owned(alloc), Some(kind));
let alloc = M::init_allocation_extra(self, id, Cow::Owned(alloc), Some(kind));
self.alloc_map.insert(id, (kind, alloc.into_owned()));
Pointer::from(id).with_tag(tag)
M::tag_alloc_base_pointer(self, Pointer::from(id))
}
pub fn reallocate(
&mut self,
ptr: Pointer<M::PointerTag>,
ptr: Pointer<Option<M::PointerTag>>,
old_size_and_align: Option<(Size, Align)>,
new_size: Size,
new_align: Align,
kind: MemoryKind<M::MemoryKind>,
) -> InterpResult<'tcx, Pointer<M::PointerTag>> {
if ptr.offset.bytes() != 0 {
let (alloc_id, offset, ptr) = self.ptr_get_alloc(ptr)?;
if offset.bytes() != 0 {
throw_ub_format!(
"reallocating {:?} which does not point to the beginning of an object",
ptr
@@ -261,7 +259,7 @@ impl<'mir, 'tcx, M: Machine<'mir, 'tcx>> Memory<'mir, 'tcx, M> {
let new_ptr = self.allocate(new_size, new_align, kind)?;
let old_size = match old_size_and_align {
Some((size, _align)) => size,
None => self.get_raw(ptr.alloc_id)?.size(),
None => self.get_raw(alloc_id)?.size(),
};
// This will also call the access hooks.
self.copy(
@@ -272,50 +270,51 @@ impl<'mir, 'tcx, M: Machine<'mir, 'tcx>> Memory<'mir, 'tcx, M> {
old_size.min(new_size),
/*nonoverlapping*/ true,
)?;
self.deallocate(ptr, old_size_and_align, kind)?;
self.deallocate(ptr.into(), old_size_and_align, kind)?;
Ok(new_ptr)
}
pub fn deallocate(
&mut self,
ptr: Pointer<M::PointerTag>,
ptr: Pointer<Option<M::PointerTag>>,
old_size_and_align: Option<(Size, Align)>,
kind: MemoryKind<M::MemoryKind>,
) -> InterpResult<'tcx> {
trace!("deallocating: {}", ptr.alloc_id);
let (alloc_id, offset, ptr) = self.ptr_get_alloc(ptr)?;
trace!("deallocating: {}", alloc_id);
if ptr.offset.bytes() != 0 {
if offset.bytes() != 0 {
throw_ub_format!(
"deallocating {:?} which does not point to the beginning of an object",
ptr
);
}
let (alloc_kind, mut alloc) = match self.alloc_map.remove(&ptr.alloc_id) {
let (alloc_kind, mut alloc) = match self.alloc_map.remove(&alloc_id) {
Some(alloc) => alloc,
None => {
// Deallocating global memory -- always an error
return Err(match self.tcx.get_global_alloc(ptr.alloc_id) {
return Err(match self.tcx.get_global_alloc(alloc_id) {
Some(GlobalAlloc::Function(..)) => {
err_ub_format!("deallocating {}, which is a function", ptr.alloc_id)
err_ub_format!("deallocating {}, which is a function", alloc_id)
}
Some(GlobalAlloc::Static(..) | GlobalAlloc::Memory(..)) => {
err_ub_format!("deallocating {}, which is static memory", ptr.alloc_id)
err_ub_format!("deallocating {}, which is static memory", alloc_id)
}
None => err_ub!(PointerUseAfterFree(ptr.alloc_id)),
None => err_ub!(PointerUseAfterFree(alloc_id)),
}
.into());
}
};
if alloc.mutability == Mutability::Not {
throw_ub_format!("deallocating immutable allocation {}", ptr.alloc_id);
throw_ub_format!("deallocating immutable allocation {}", alloc_id);
}
if alloc_kind != kind {
throw_ub_format!(
"deallocating {}, which is {} memory, using {} deallocation operation",
ptr.alloc_id,
alloc_id,
alloc_kind,
kind
);
@@ -324,7 +323,7 @@ impl<'mir, 'tcx, M: Machine<'mir, 'tcx>> Memory<'mir, 'tcx, M> {
if size != alloc.size() || align != alloc.align {
throw_ub_format!(
"incorrect layout on deallocation: {} has size {} and alignment {}, but gave size {} and alignment {}",
ptr.alloc_id,
alloc_id,
alloc.size().bytes(),
alloc.align.bytes(),
size.bytes(),
@@ -335,10 +334,15 @@ impl<'mir, 'tcx, M: Machine<'mir, 'tcx>> Memory<'mir, 'tcx, M> {
// Let the machine take some extra action
let size = alloc.size();
M::memory_deallocated(&mut self.extra, &mut alloc.extra, ptr, size)?;
M::memory_deallocated(
&mut self.extra,
&mut alloc.extra,
ptr.provenance,
alloc_range(Size::ZERO, size),
)?;
// Don't forget to remember size and align of this now-dead allocation
let old = self.dead_alloc_map.insert(ptr.alloc_id, (size, alloc.align));
let old = self.dead_alloc_map.insert(alloc_id, (size, alloc.align));
if old.is_some() {
bug!("Nothing can be deallocated twice");
}
@@ -346,52 +350,69 @@ impl<'mir, 'tcx, M: Machine<'mir, 'tcx>> Memory<'mir, 'tcx, M> {
Ok(())
}
/// Internal helper function for APIs that offer memory access based on `Scalar` pointers.
/// Internal helper function to determine the allocation and offset of a pointer (if any).
#[inline(always)]
pub(super) fn check_ptr_access(
fn get_ptr_access(
&self,
sptr: Scalar<M::PointerTag>,
ptr: Pointer<Option<M::PointerTag>>,
size: Size,
align: Align,
) -> InterpResult<'tcx, Option<Pointer<M::PointerTag>>> {
) -> InterpResult<'tcx, Option<(AllocId, Size, Pointer<M::PointerTag>)>> {
let align = M::enforce_alignment(&self.extra).then_some(align);
self.check_and_deref_ptr(sptr, size, align, CheckInAllocMsg::MemoryAccessTest, |ptr| {
let (size, align) =
self.get_size_and_align(ptr.alloc_id, AllocCheck::Dereferenceable)?;
Ok((size, align, ptr))
})
self.check_and_deref_ptr(
ptr,
size,
align,
CheckInAllocMsg::MemoryAccessTest,
|alloc_id, offset, ptr| {
let (size, align) =
self.get_size_and_align(alloc_id, AllocCheck::Dereferenceable)?;
Ok((size, align, (alloc_id, offset, ptr)))
},
)
}
/// Check if the given scalar is allowed to do a memory access of given `size` and `align`
/// Check if the given pointerpoints to live memory of given `size` and `align`
/// (ignoring `M::enforce_alignment`). The caller can control the error message for the
/// out-of-bounds case.
#[inline(always)]
pub fn check_ptr_access_align(
&self,
sptr: Scalar<M::PointerTag>,
ptr: Pointer<Option<M::PointerTag>>,
size: Size,
align: Align,
msg: CheckInAllocMsg,
) -> InterpResult<'tcx> {
self.check_and_deref_ptr(sptr, size, Some(align), msg, |ptr| {
let (size, align) =
self.get_size_and_align(ptr.alloc_id, AllocCheck::Dereferenceable)?;
self.check_and_deref_ptr(ptr, size, Some(align), msg, |alloc_id, _, _| {
let check = match msg {
CheckInAllocMsg::DerefTest | CheckInAllocMsg::MemoryAccessTest => {
AllocCheck::Dereferenceable
}
CheckInAllocMsg::PointerArithmeticTest | CheckInAllocMsg::InboundsTest => {
AllocCheck::Live
}
};
let (size, align) = self.get_size_and_align(alloc_id, check)?;
Ok((size, align, ()))
})?;
Ok(())
}
/// Low-level helper function to check if a ptr is in-bounds and potentially return a reference
/// to the allocation it points to. Supports both shared and mutable references, to the actual
/// to the allocation it points to. Supports both shared and mutable references, as the actual
/// checking is offloaded to a helper closure. `align` defines whether and which alignment check
/// is done. Returns `None` for size 0, and otherwise `Some` of what `alloc_size` returned.
fn check_and_deref_ptr<T>(
&self,
sptr: Scalar<M::PointerTag>,
ptr: Pointer<Option<M::PointerTag>>,
size: Size,
align: Option<Align>,
msg: CheckInAllocMsg,
alloc_size: impl FnOnce(Pointer<M::PointerTag>) -> InterpResult<'tcx, (Size, Align, T)>,
alloc_size: impl FnOnce(
AllocId,
Size,
Pointer<M::PointerTag>,
) -> InterpResult<'tcx, (Size, Align, T)>,
) -> InterpResult<'tcx, Option<T>> {
fn check_offset_align(offset: u64, align: Align) -> InterpResult<'static> {
if offset % align.bytes() == 0 {
@@ -406,53 +427,50 @@ impl<'mir, 'tcx, M: Machine<'mir, 'tcx>> Memory<'mir, 'tcx, M> {
}
}
// Normalize to a `Pointer` if we definitely need one.
let normalized = if size.bytes() == 0 {
// Can be an integer, just take what we got. We do NOT `force_bits` here;
// if this is already a `Pointer` we want to do the bounds checks!
sptr
// Extract from the pointer an `Option<AllocId>` and an offset, which is relative to the
// allocation or (if that is `None`) an absolute address.
let ptr_or_addr = if size.bytes() == 0 {
// Let's see what we can do, but don't throw errors if there's nothing there.
self.ptr_try_get_alloc(ptr)
} else {
// A "real" access, we must get a pointer to be able to check the bounds.
Scalar::from(self.force_ptr(sptr)?)
// A "real" access, we insist on getting an `AllocId`.
Ok(self.ptr_get_alloc(ptr)?)
};
Ok(match normalized.to_bits_or_ptr(self.pointer_size(), self) {
Ok(bits) => {
let bits = u64::try_from(bits).unwrap(); // it's ptr-sized
assert!(size.bytes() == 0);
Ok(match ptr_or_addr {
Err(addr) => {
// No memory is actually being accessed.
debug_assert!(size.bytes() == 0);
// Must be non-null.
if bits == 0 {
if addr == 0 {
throw_ub!(DanglingIntPointer(0, msg))
}
// Must be aligned.
if let Some(align) = align {
check_offset_align(bits, align)?;
check_offset_align(addr, align)?;
}
None
}
Err(ptr) => {
let (allocation_size, alloc_align, ret_val) = alloc_size(ptr)?;
Ok((alloc_id, offset, ptr)) => {
let (allocation_size, alloc_align, ret_val) = alloc_size(alloc_id, offset, ptr)?;
// Test bounds. This also ensures non-null.
// It is sufficient to check this for the end pointer. The addition
// checks for overflow.
let end_ptr = ptr.offset(size, self)?;
if end_ptr.offset > allocation_size {
// equal is okay!
throw_ub!(PointerOutOfBounds { ptr: end_ptr.erase_tag(), msg, allocation_size })
// It is sufficient to check this for the end pointer. Also check for overflow!
if offset.checked_add(size, &self.tcx).map_or(true, |end| end > allocation_size) {
throw_ub!(PointerOutOfBounds { alloc_id, offset, size, allocation_size, msg })
}
// Test align. Check this last; if both bounds and alignment are violated
// we want the error to be about the bounds.
if let Some(align) = align {
if M::force_int_for_alignment_check(&self.extra) {
let bits = self
.force_bits(ptr.into(), self.pointer_size())
let addr = Scalar::from_pointer(ptr, &self.tcx)
.to_machine_usize(&self.tcx)
.expect("ptr-to-int cast for align check should never fail");
check_offset_align(bits.try_into().unwrap(), align)?;
check_offset_align(addr, align)?;
} else {
// Check allocation alignment and offset alignment.
if alloc_align.bytes() < align.bytes() {
throw_ub!(AlignmentCheckFailed { has: alloc_align, required: align });
}
check_offset_align(ptr.offset.bytes(), align)?;
check_offset_align(offset.bytes(), align)?;
}
}
@@ -464,13 +482,18 @@ impl<'mir, 'tcx, M: Machine<'mir, 'tcx>> Memory<'mir, 'tcx, M> {
}
/// Test if the pointer might be null.
pub fn ptr_may_be_null(&self, ptr: Pointer<M::PointerTag>) -> bool {
let (size, _align) = self
.get_size_and_align(ptr.alloc_id, AllocCheck::MaybeDead)
.expect("alloc info with MaybeDead cannot fail");
// If the pointer is out-of-bounds, it may be null.
// Note that one-past-the-end (offset == size) is still inbounds, and never null.
ptr.offset > size
pub fn ptr_may_be_null(&self, ptr: Pointer<Option<M::PointerTag>>) -> bool {
match self.ptr_try_get_alloc(ptr) {
Ok((alloc_id, offset, _)) => {
let (size, _align) = self
.get_size_and_align(alloc_id, AllocCheck::MaybeDead)
.expect("alloc info with MaybeDead cannot fail");
// If the pointer is out-of-bounds, it may be null.
// Note that one-past-the-end (offset == size) is still inbounds, and never null.
offset > size
}
Err(offset) => offset == 0,
}
}
}
@@ -482,12 +505,11 @@ impl<'mir, 'tcx, M: Machine<'mir, 'tcx>> Memory<'mir, 'tcx, M> {
/// this machine use the same pointer tag, so it is indirected through
/// `M::tag_allocation`.
fn get_global_alloc(
memory_extra: &M::MemoryExtra,
tcx: TyCtxt<'tcx>,
&self,
id: AllocId,
is_write: bool,
) -> InterpResult<'tcx, Cow<'tcx, Allocation<M::PointerTag, M::AllocExtra>>> {
let (alloc, def_id) = match tcx.get_global_alloc(id) {
let (alloc, def_id) = match self.tcx.get_global_alloc(id) {
Some(GlobalAlloc::Memory(mem)) => {
// Memory of a constant or promoted or anonymous memory referenced by a static.
(mem, None)
@@ -495,8 +517,8 @@ impl<'mir, 'tcx, M: Machine<'mir, 'tcx>> Memory<'mir, 'tcx, M> {
Some(GlobalAlloc::Function(..)) => throw_ub!(DerefFunctionPointer(id)),
None => throw_ub!(PointerUseAfterFree(id)),
Some(GlobalAlloc::Static(def_id)) => {
assert!(tcx.is_static(def_id));
assert!(!tcx.is_thread_local_static(def_id));
assert!(self.tcx.is_static(def_id));
assert!(!self.tcx.is_thread_local_static(def_id));
// Notice that every static has two `AllocId` that will resolve to the same
// thing here: one maps to `GlobalAlloc::Static`, this is the "lazy" ID,
// and the other one is maps to `GlobalAlloc::Memory`, this is returned by
@@ -507,24 +529,22 @@ impl<'mir, 'tcx, M: Machine<'mir, 'tcx>> Memory<'mir, 'tcx, M> {
// The `GlobalAlloc::Memory` branch here is still reachable though; when a static
// contains a reference to memory that was created during its evaluation (i.e., not
// to another static), those inner references only exist in "resolved" form.
if tcx.is_foreign_item(def_id) {
if self.tcx.is_foreign_item(def_id) {
throw_unsup!(ReadExternStatic(def_id));
}
(tcx.eval_static_initializer(def_id)?, Some(def_id))
(self.tcx.eval_static_initializer(def_id)?, Some(def_id))
}
};
M::before_access_global(memory_extra, id, alloc, def_id, is_write)?;
M::before_access_global(&self.extra, id, alloc, def_id, is_write)?;
let alloc = Cow::Borrowed(alloc);
// We got tcx memory. Let the machine initialize its "extra" stuff.
let (alloc, tag) = M::init_allocation_extra(
memory_extra,
let alloc = M::init_allocation_extra(
self,
id, // always use the ID we got as input, not the "hidden" one.
alloc,
M::GLOBAL_KIND.map(MemoryKind::Machine),
);
// Sanity check that this is the same pointer we would have gotten via `global_base_pointer`.
debug_assert_eq!(tag, M::tag_global_base_pointer(memory_extra, id));
Ok(alloc)
}
@@ -539,8 +559,7 @@ impl<'mir, 'tcx, M: Machine<'mir, 'tcx>> Memory<'mir, 'tcx, M> {
// `get_global_alloc` that we can actually use directly without inserting anything anywhere.
// So the error type is `InterpResult<'tcx, &Allocation<M::PointerTag>>`.
let a = self.alloc_map.get_or(id, || {
let alloc = Self::get_global_alloc(&self.extra, self.tcx, id, /*is_write*/ false)
.map_err(Err)?;
let alloc = self.get_global_alloc(id, /*is_write*/ false).map_err(Err)?;
match alloc {
Cow::Borrowed(alloc) => {
// We got a ref, cheaply return that as an "error" so that the
@@ -567,30 +586,30 @@ impl<'mir, 'tcx, M: Machine<'mir, 'tcx>> Memory<'mir, 'tcx, M> {
/// "Safe" (bounds and align-checked) allocation access.
pub fn get<'a>(
&'a self,
sptr: Scalar<M::PointerTag>,
ptr: Pointer<Option<M::PointerTag>>,
size: Size,
align: Align,
) -> InterpResult<'tcx, Option<AllocRef<'a, 'tcx, M::PointerTag, M::AllocExtra>>> {
let align = M::enforce_alignment(&self.extra).then_some(align);
let ptr_and_alloc = self.check_and_deref_ptr(
sptr,
ptr,
size,
align,
CheckInAllocMsg::MemoryAccessTest,
|ptr| {
let alloc = self.get_raw(ptr.alloc_id)?;
Ok((alloc.size(), alloc.align, (ptr, alloc)))
|alloc_id, offset, ptr| {
let alloc = self.get_raw(alloc_id)?;
Ok((alloc.size(), alloc.align, (alloc_id, offset, ptr, alloc)))
},
)?;
if let Some((ptr, alloc)) = ptr_and_alloc {
M::memory_read(&self.extra, &alloc.extra, ptr, size)?;
let range = alloc_range(ptr.offset, size);
Ok(Some(AllocRef { alloc, range, tcx: self.tcx, alloc_id: ptr.alloc_id }))
if let Some((alloc_id, offset, ptr, alloc)) = ptr_and_alloc {
let range = alloc_range(offset, size);
M::memory_read(&self.extra, &alloc.extra, ptr.provenance, range)?;
Ok(Some(AllocRef { alloc, range, tcx: self.tcx, alloc_id }))
} else {
// Even in this branch we have to be sure that we actually access the allocation, in
// order to ensure that `static FOO: Type = FOO;` causes a cycle error instead of
// magically pulling *any* ZST value from the ether. However, the `get_raw` above is
// always called when `sptr` is truly a `Pointer`, so we are good.
// always called when `ptr` has an `AllocId`.
Ok(None)
}
}
@@ -610,48 +629,44 @@ impl<'mir, 'tcx, M: Machine<'mir, 'tcx>> Memory<'mir, 'tcx, M> {
id: AllocId,
) -> InterpResult<'tcx, (&mut Allocation<M::PointerTag, M::AllocExtra>, &mut M::MemoryExtra)>
{
let tcx = self.tcx;
let memory_extra = &mut self.extra;
let a = self.alloc_map.get_mut_or(id, || {
// Need to make a copy, even if `get_global_alloc` is able
// to give us a cheap reference.
let alloc = Self::get_global_alloc(memory_extra, tcx, id, /*is_write*/ true)?;
// We have "NLL problem case #3" here, which cannot be worked around without loss of
// efficiency even for the common case where the key is in the map.
// <https://rust-lang.github.io/rfcs/2094-nll.html#problem-case-3-conditional-control-flow-across-functions>
// (Cannot use `get_mut_or` since `get_global_alloc` needs `&self`.)
if self.alloc_map.get_mut(id).is_none() {
// Slow path.
// Allocation not found locally, go look global.
let alloc = self.get_global_alloc(id, /*is_write*/ true)?;
let kind = M::GLOBAL_KIND.expect(
"I got a global allocation that I have to copy but the machine does \
not expect that to happen",
);
Ok((MemoryKind::Machine(kind), alloc.into_owned()))
});
// Unpack the error type manually because type inference doesn't
// work otherwise (and we cannot help it because `impl Trait`)
match a {
Err(e) => Err(e),
Ok(a) => {
let a = &mut a.1;
if a.mutability == Mutability::Not {
throw_ub!(WriteToReadOnly(id))
}
Ok((a, memory_extra))
}
self.alloc_map.insert(id, (MemoryKind::Machine(kind), alloc.into_owned()));
}
let (_kind, alloc) = self.alloc_map.get_mut(id).unwrap();
if alloc.mutability == Mutability::Not {
throw_ub!(WriteToReadOnly(id))
}
Ok((alloc, &mut self.extra))
}
/// "Safe" (bounds and align-checked) allocation access.
pub fn get_mut<'a>(
&'a mut self,
sptr: Scalar<M::PointerTag>,
ptr: Pointer<Option<M::PointerTag>>,
size: Size,
align: Align,
) -> InterpResult<'tcx, Option<AllocRefMut<'a, 'tcx, M::PointerTag, M::AllocExtra>>> {
let ptr = self.check_ptr_access(sptr, size, align)?;
if let Some(ptr) = ptr {
let parts = self.get_ptr_access(ptr, size, align)?;
if let Some((alloc_id, offset, ptr)) = parts {
let tcx = self.tcx;
// FIXME: can we somehow avoid looking up the allocation twice here?
// We cannot call `get_raw_mut` inside `check_and_deref_ptr` as that would duplicate `&mut self`.
let (alloc, extra) = self.get_raw_mut(ptr.alloc_id)?;
M::memory_written(extra, &mut alloc.extra, ptr, size)?;
let range = alloc_range(ptr.offset, size);
Ok(Some(AllocRefMut { alloc, range, tcx, alloc_id: ptr.alloc_id }))
let (alloc, extra) = self.get_raw_mut(alloc_id)?;
let range = alloc_range(offset, size);
M::memory_written(extra, &mut alloc.extra, ptr.provenance, range)?;
Ok(Some(AllocRefMut { alloc, range, tcx, alloc_id }))
} else {
Ok(None)
}
@@ -728,7 +743,6 @@ impl<'mir, 'tcx, M: Machine<'mir, 'tcx>> Memory<'mir, 'tcx, M> {
}
fn get_fn_alloc(&self, id: AllocId) -> Option<FnVal<'tcx, M::ExtraFnVal>> {
trace!("reading fn ptr: {}", id);
if let Some(extra) = self.extra_fn_ptr_map.get(&id) {
Some(FnVal::Other(*extra))
} else {
@@ -741,14 +755,15 @@ impl<'mir, 'tcx, M: Machine<'mir, 'tcx>> Memory<'mir, 'tcx, M> {
pub fn get_fn(
&self,
ptr: Scalar<M::PointerTag>,
ptr: Pointer<Option<M::PointerTag>>,
) -> InterpResult<'tcx, FnVal<'tcx, M::ExtraFnVal>> {
let ptr = self.force_ptr(ptr)?; // We definitely need a pointer value.
if ptr.offset.bytes() != 0 {
throw_ub!(InvalidFunctionPointer(ptr.erase_tag()))
trace!("get_fn({:?})", ptr);
let (alloc_id, offset, _ptr) = self.ptr_get_alloc(ptr)?;
if offset.bytes() != 0 {
throw_ub!(InvalidFunctionPointer(Pointer::new(alloc_id, offset)))
}
self.get_fn_alloc(ptr.alloc_id)
.ok_or_else(|| err_ub!(InvalidFunctionPointer(ptr.erase_tag())).into())
self.get_fn_alloc(alloc_id)
.ok_or_else(|| err_ub!(InvalidFunctionPointer(Pointer::new(alloc_id, offset))).into())
}
pub fn mark_immutable(&mut self, id: AllocId) -> InterpResult<'tcx> {
@@ -787,7 +802,7 @@ impl<'mir, 'tcx, M: Machine<'mir, 'tcx>> Memory<'mir, 'tcx, M> {
if reachable.insert(id) {
// This is a new allocation, add its relocations to `todo`.
if let Some((_, alloc)) = self.alloc_map.get(id) {
todo.extend(alloc.relocations().values().map(|&(_, target_id)| target_id));
todo.extend(alloc.relocations().values().map(|tag| tag.get_alloc_id()));
}
}
}
@@ -821,14 +836,14 @@ pub struct DumpAllocs<'a, 'mir, 'tcx, M: Machine<'mir, 'tcx>> {
impl<'a, 'mir, 'tcx, M: Machine<'mir, 'tcx>> std::fmt::Debug for DumpAllocs<'a, 'mir, 'tcx, M> {
fn fmt(&self, fmt: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
// Cannot be a closure because it is generic in `Tag`, `Extra`.
fn write_allocation_track_relocs<'tcx, Tag: Copy + fmt::Debug, Extra>(
fn write_allocation_track_relocs<'tcx, Tag: Provenance, Extra>(
fmt: &mut std::fmt::Formatter<'_>,
tcx: TyCtxt<'tcx>,
allocs_to_print: &mut VecDeque<AllocId>,
alloc: &Allocation<Tag, Extra>,
) -> std::fmt::Result {
for &(_, target_id) in alloc.relocations().values() {
allocs_to_print.push_back(target_id);
for alloc_id in alloc.relocations().values().map(|tag| tag.get_alloc_id()) {
allocs_to_print.push_back(alloc_id);
}
write!(fmt, "{}", pretty::display_allocation(tcx, alloc))
}
@@ -931,8 +946,12 @@ impl<'mir, 'tcx, M: Machine<'mir, 'tcx>> Memory<'mir, 'tcx, M> {
/// Reads the given number of bytes from memory. Returns them as a slice.
///
/// Performs appropriate bounds checks.
pub fn read_bytes(&self, sptr: Scalar<M::PointerTag>, size: Size) -> InterpResult<'tcx, &[u8]> {
let alloc_ref = match self.get(sptr, size, Align::ONE)? {
pub fn read_bytes(
&self,
ptr: Pointer<Option<M::PointerTag>>,
size: Size,
) -> InterpResult<'tcx, &[u8]> {
let alloc_ref = match self.get(ptr, size, Align::ONE)? {
Some(a) => a,
None => return Ok(&[]), // zero-sized access
};
@@ -949,7 +968,7 @@ impl<'mir, 'tcx, M: Machine<'mir, 'tcx>> Memory<'mir, 'tcx, M> {
/// Performs appropriate bounds checks.
pub fn write_bytes(
&mut self,
sptr: Scalar<M::PointerTag>,
ptr: Pointer<Option<M::PointerTag>>,
src: impl IntoIterator<Item = u8>,
) -> InterpResult<'tcx> {
let mut src = src.into_iter();
@@ -958,7 +977,7 @@ impl<'mir, 'tcx, M: Machine<'mir, 'tcx>> Memory<'mir, 'tcx, M> {
assert_eq!(lower, len, "can only write iterators with a precise length");
let size = Size::from_bytes(len);
let alloc_ref = match self.get_mut(sptr, size, Align::ONE)? {
let alloc_ref = match self.get_mut(ptr, size, Align::ONE)? {
Some(alloc_ref) => alloc_ref,
None => {
// zero-sized access
@@ -985,9 +1004,9 @@ impl<'mir, 'tcx, M: Machine<'mir, 'tcx>> Memory<'mir, 'tcx, M> {
pub fn copy(
&mut self,
src: Scalar<M::PointerTag>,
src: Pointer<Option<M::PointerTag>>,
src_align: Align,
dest: Scalar<M::PointerTag>,
dest: Pointer<Option<M::PointerTag>>,
dest_align: Align,
size: Size,
nonoverlapping: bool,
@@ -997,9 +1016,9 @@ impl<'mir, 'tcx, M: Machine<'mir, 'tcx>> Memory<'mir, 'tcx, M> {
pub fn copy_repeatedly(
&mut self,
src: Scalar<M::PointerTag>,
src: Pointer<Option<M::PointerTag>>,
src_align: Align,
dest: Scalar<M::PointerTag>,
dest: Pointer<Option<M::PointerTag>>,
dest_align: Align,
size: Size,
num_copies: u64,
@@ -1007,22 +1026,23 @@ impl<'mir, 'tcx, M: Machine<'mir, 'tcx>> Memory<'mir, 'tcx, M> {
) -> InterpResult<'tcx> {
let tcx = self.tcx;
// We need to do our own bounds-checks.
let src = self.check_ptr_access(src, size, src_align)?;
let dest = self.check_ptr_access(dest, size * num_copies, dest_align)?; // `Size` multiplication
let src_parts = self.get_ptr_access(src, size, src_align)?;
let dest_parts = self.get_ptr_access(dest, size * num_copies, dest_align)?; // `Size` multiplication
// FIXME: we look up both allocations twice here, once ebfore for the `check_ptr_access`
// and once below to get the underlying `&[mut] Allocation`.
// Source alloc preparations and access hooks.
let src = match src {
let (src_alloc_id, src_offset, src) = match src_parts {
None => return Ok(()), // Zero-sized *source*, that means dst is also zero-sized and we have nothing to do.
Some(src_ptr) => src_ptr,
};
let src_alloc = self.get_raw(src.alloc_id)?;
M::memory_read(&self.extra, &src_alloc.extra, src, size)?;
let src_alloc = self.get_raw(src_alloc_id)?;
let src_range = alloc_range(src_offset, size);
M::memory_read(&self.extra, &src_alloc.extra, src.provenance, src_range)?;
// We need the `dest` ptr for the next operation, so we get it now.
// We already did the source checks and called the hooks so we are good to return early.
let dest = match dest {
let (dest_alloc_id, dest_offset, dest) = match dest_parts {
None => return Ok(()), // Zero-sized *destiantion*.
Some(dest_ptr) => dest_ptr,
};
@@ -1032,26 +1052,21 @@ impl<'mir, 'tcx, M: Machine<'mir, 'tcx>> Memory<'mir, 'tcx, M> {
// since we don't want to keep any relocations at the target.
// (`get_bytes_with_uninit_and_ptr` below checks that there are no
// relocations overlapping the edges; those would not be handled correctly).
let relocations = src_alloc.prepare_relocation_copy(
self,
alloc_range(src.offset, size),
dest.offset,
num_copies,
);
let relocations =
src_alloc.prepare_relocation_copy(self, src_range, dest_offset, num_copies);
// Prepare a copy of the initialization mask.
let compressed = src_alloc.compress_uninit_range(alloc_range(src.offset, size));
let compressed = src_alloc.compress_uninit_range(src_range);
// This checks relocation edges on the src.
let src_bytes = src_alloc
.get_bytes_with_uninit_and_ptr(&tcx, alloc_range(src.offset, size))
.map_err(|e| e.to_interp_error(src.alloc_id))?
.get_bytes_with_uninit_and_ptr(&tcx, src_range)
.map_err(|e| e.to_interp_error(src_alloc_id))?
.as_ptr(); // raw ptr, so we can also get a ptr to the destination allocation
// Destination alloc preparations and access hooks.
let (dest_alloc, extra) = self.get_raw_mut(dest.alloc_id)?;
M::memory_written(extra, &mut dest_alloc.extra, dest, size * num_copies)?;
let dest_bytes = dest_alloc
.get_bytes_mut_ptr(&tcx, alloc_range(dest.offset, size * num_copies))
.as_mut_ptr();
let (dest_alloc, extra) = self.get_raw_mut(dest_alloc_id)?;
let dest_range = alloc_range(dest_offset, size * num_copies);
M::memory_written(extra, &mut dest_alloc.extra, dest.provenance, dest_range)?;
let dest_bytes = dest_alloc.get_bytes_mut_ptr(&tcx, dest_range).as_mut_ptr();
if compressed.no_bytes_init() {
// Fast path: If all bytes are `uninit` then there is nothing to copy. The target range
@@ -1060,7 +1075,7 @@ impl<'mir, 'tcx, M: Machine<'mir, 'tcx>> Memory<'mir, 'tcx, M> {
// This also avoids writing to the target bytes so that the backing allocation is never
// touched if the bytes stay uninitialized for the whole interpreter execution. On contemporary
// operating system this can avoid physically allocating the page.
dest_alloc.mark_init(alloc_range(dest.offset, size * num_copies), false); // `Size` multiplication
dest_alloc.mark_init(dest_range, false); // `Size` multiplication
dest_alloc.mark_relocation_range(relocations);
return Ok(());
}
@@ -1071,11 +1086,11 @@ impl<'mir, 'tcx, M: Machine<'mir, 'tcx>> Memory<'mir, 'tcx, M> {
// The pointers above remain valid even if the `HashMap` table is moved around because they
// point into the `Vec` storing the bytes.
unsafe {
if src.alloc_id == dest.alloc_id {
if src_alloc_id == dest_alloc_id {
if nonoverlapping {
// `Size` additions
if (src.offset <= dest.offset && src.offset + size > dest.offset)
|| (dest.offset <= src.offset && dest.offset + size > src.offset)
if (src_offset <= dest_offset && src_offset + size > dest_offset)
|| (dest_offset <= src_offset && dest_offset + size > src_offset)
{
throw_ub_format!("copy_nonoverlapping called on overlapping ranges")
}
@@ -1102,7 +1117,7 @@ impl<'mir, 'tcx, M: Machine<'mir, 'tcx>> Memory<'mir, 'tcx, M> {
// now fill in all the "init" data
dest_alloc.mark_compressed_init_range(
&compressed,
alloc_range(dest.offset, size),
alloc_range(dest_offset, size), // just a single copy (i.e., not full `dest_range`)
num_copies,
);
// copy the relocations to the destination
@@ -1114,24 +1129,41 @@ impl<'mir, 'tcx, M: Machine<'mir, 'tcx>> Memory<'mir, 'tcx, M> {
/// Machine pointer introspection.
impl<'mir, 'tcx, M: Machine<'mir, 'tcx>> Memory<'mir, 'tcx, M> {
pub fn force_ptr(
&self,
scalar: Scalar<M::PointerTag>,
) -> InterpResult<'tcx, Pointer<M::PointerTag>> {
match scalar {
Scalar::Ptr(ptr) => Ok(ptr),
_ => M::int_to_ptr(&self, scalar.to_machine_usize(self)?),
pub fn scalar_to_ptr(&self, scalar: Scalar<M::PointerTag>) -> Pointer<Option<M::PointerTag>> {
// We use `to_bits_or_ptr_internal` since we are just implementing the method people need to
// call to force getting out a pointer.
match scalar.to_bits_or_ptr_internal(self.pointer_size()) {
Err(ptr) => ptr.into(),
Ok(bits) => {
let addr = u64::try_from(bits).unwrap();
M::ptr_from_addr(&self, addr)
}
}
}
pub fn force_bits(
/// Turning a "maybe pointer" into a proper pointer (and some information
/// about where it points), or an absolute address.
pub fn ptr_try_get_alloc(
&self,
scalar: Scalar<M::PointerTag>,
size: Size,
) -> InterpResult<'tcx, u128> {
match scalar.to_bits_or_ptr(size, self) {
Ok(bits) => Ok(bits),
Err(ptr) => Ok(M::ptr_to_int(&self, ptr)?.into()),
ptr: Pointer<Option<M::PointerTag>>,
) -> Result<(AllocId, Size, Pointer<M::PointerTag>), u64> {
match ptr.into_pointer_or_addr() {
Ok(ptr) => {
let (alloc_id, offset) = M::ptr_get_alloc(self, ptr);
Ok((alloc_id, offset, ptr))
}
Err(addr) => Err(addr.bytes()),
}
}
/// Turning a "maybe pointer" into a proper pointer (and some information about where it points).
#[inline(always)]
pub fn ptr_get_alloc(
&self,
ptr: Pointer<Option<M::PointerTag>>,
) -> InterpResult<'tcx, (AllocId, Size, Pointer<M::PointerTag>)> {
self.ptr_try_get_alloc(ptr).map_err(|offset| {
err_ub!(DanglingIntPointer(offset, CheckInAllocMsg::InboundsTest)).into()
})
}
}