Rename rustc_mir_build::build to builder

compiler/rustc_mir_build/src/builder/block.rs

@@ -0,0 +1,365 @@
+use rustc_middle::middle::region::Scope;
+use rustc_middle::mir::*;
+use rustc_middle::thir::*;
+use rustc_middle::{span_bug, ty};
+use rustc_span::Span;
+use tracing::debug;
+
+use crate::builder::ForGuard::OutsideGuard;
+use crate::builder::matches::{DeclareLetBindings, EmitStorageLive, ScheduleDrops};
+use crate::builder::{BlockAnd, BlockAndExtension, BlockFrame, Builder};
+
+impl<'a, 'tcx> Builder<'a, 'tcx> {
+    pub(crate) fn ast_block(
+        &mut self,
+        destination: Place<'tcx>,
+        block: BasicBlock,
+        ast_block: BlockId,
+        source_info: SourceInfo,
+    ) -> BlockAnd<()> {
+        let Block { region_scope, span, ref stmts, expr, targeted_by_break, safety_mode: _ } =
+            self.thir[ast_block];
+        self.in_scope((region_scope, source_info), LintLevel::Inherited, move |this| {
+            if targeted_by_break {
+                this.in_breakable_scope(None, destination, span, |this| {
+                    Some(this.ast_block_stmts(destination, block, span, stmts, expr, region_scope))
+                })
+            } else {
+                this.ast_block_stmts(destination, block, span, stmts, expr, region_scope)
+            }
+        })
+    }
+
+    fn ast_block_stmts(
+        &mut self,
+        destination: Place<'tcx>,
+        mut block: BasicBlock,
+        span: Span,
+        stmts: &[StmtId],
+        expr: Option<ExprId>,
+        region_scope: Scope,
+    ) -> BlockAnd<()> {
+        let this = self;
+
+        // This convoluted structure is to avoid using recursion as we walk down a list
+        // of statements. Basically, the structure we get back is something like:
+        //
+        //    let x = <init> in {
+        //       expr1;
+        //       let y = <init> in {
+        //           expr2;
+        //           expr3;
+        //           ...
+        //       }
+        //    }
+        //
+        // The let bindings are valid till the end of block so all we have to do is to pop all
+        // the let-scopes at the end.
+        //
+        // First we build all the statements in the block.
+        let mut let_scope_stack = Vec::with_capacity(8);
+        let outer_source_scope = this.source_scope;
+        // This scope information is kept for breaking out of the parent remainder scope in case
+        // one let-else pattern matching fails.
+        // By doing so, we can be sure that even temporaries that receive extended lifetime
+        // assignments are dropped, too.
+        let mut last_remainder_scope = region_scope;
+
+        let source_info = this.source_info(span);
+        for stmt in stmts {
+            let Stmt { ref kind } = this.thir[*stmt];
+            match kind {
+                StmtKind::Expr { scope, expr } => {
+                    this.block_context.push(BlockFrame::Statement { ignores_expr_result: true });
+                    let si = (*scope, source_info);
+                    block = this
+                        .in_scope(si, LintLevel::Inherited, |this| {
+                            this.stmt_expr(block, *expr, Some(*scope))
+                        })
+                        .into_block();
+                }
+                StmtKind::Let {
+                    remainder_scope,
+                    init_scope,
+                    pattern,
+                    initializer: Some(initializer),
+                    lint_level,
+                    else_block: Some(else_block),
+                    span: _,
+                } => {
+                    // When lowering the statement `let <pat> = <expr> else { <else> };`,
+                    // the `<else>` block is nested in the parent scope enclosing this statement.
+                    // That scope is usually either the enclosing block scope,
+                    // or the remainder scope of the last statement.
+                    // This is to make sure that temporaries instantiated in `<expr>` are dropped
+                    // as well.
+                    // In addition, even though bindings in `<pat>` only come into scope if
+                    // the pattern matching passes, in the MIR building the storages for them
+                    // are declared as live any way.
+                    // This is similar to `let x;` statements without an initializer expression,
+                    // where the value of `x` in this example may or may be assigned,
+                    // because the storage for their values may not be live after all due to
+                    // failure in pattern matching.
+                    // For this reason, we declare those storages as live but we do not schedule
+                    // any drop yet- they are scheduled later after the pattern matching.
+                    // The generated MIR will have `StorageDead` whenever the control flow breaks out
+                    // of the parent scope, regardless of the result of the pattern matching.
+                    // However, the drops are inserted in MIR only when the control flow breaks out of
+                    // the scope of the remainder scope associated with this `let .. else` statement.
+                    // Pictorial explanation of the scope structure:
+                    // ┌─────────────────────────────────┐
+                    // │  Scope of the enclosing block,  │
+                    // │  or the last remainder scope    │
+                    // │  ┌───────────────────────────┐  │
+                    // │  │  Scope for <else> block   │  │
+                    // │  └───────────────────────────┘  │
+                    // │  ┌───────────────────────────┐  │
+                    // │  │  Remainder scope of       │  │
+                    // │  │  this let-else statement  │  │
+                    // │  │  ┌─────────────────────┐  │  │
+                    // │  │  │ <expr> scope        │  │  │
+                    // │  │  └─────────────────────┘  │  │
+                    // │  │  extended temporaries in  │  │
+                    // │  │  <expr> lives in this     │  │
+                    // │  │  scope                    │  │
+                    // │  │  ┌─────────────────────┐  │  │
+                    // │  │  │ Scopes for the rest │  │  │
+                    // │  │  └─────────────────────┘  │  │
+                    // │  └───────────────────────────┘  │
+                    // └─────────────────────────────────┘
+                    // Generated control flow:
+                    //          │ let Some(x) = y() else { return; }
+                    //          │
+                    // ┌────────▼───────┐
+                    // │ evaluate y()   │
+                    // └────────┬───────┘
+                    //          │              ┌────────────────┐
+                    // ┌────────▼───────┐      │Drop temporaries│
+                    // │Test the pattern├──────►in y()          │
+                    // └────────┬───────┘      │because breaking│
+                    //          │              │out of <expr>   │
+                    // ┌────────▼───────┐      │scope           │
+                    // │Move value into │      └───────┬────────┘
+                    // │binding x       │              │
+                    // └────────┬───────┘      ┌───────▼────────┐
+                    //          │              │Drop extended   │
+                    // ┌────────▼───────┐      │temporaries in  │
+                    // │Drop temporaries│      │<expr> because  │
+                    // │in y()          │      │breaking out of │
+                    // │because breaking│      │remainder scope │
+                    // │out of <expr>   │      └───────┬────────┘
+                    // │scope           │              │
+                    // └────────┬───────┘      ┌───────▼────────┐
+                    //          │              │Enter <else>    ├────────►
+                    // ┌────────▼───────┐      │block           │ return;
+                    // │Continue...     │      └────────────────┘
+                    // └────────────────┘
+
+                    let ignores_expr_result = matches!(pattern.kind, PatKind::Wild);
+                    this.block_context.push(BlockFrame::Statement { ignores_expr_result });
+
+                    // Lower the `else` block first because its parent scope is actually
+                    // enclosing the rest of the `let .. else ..` parts.
+                    let else_block_span = this.thir[*else_block].span;
+                    // This place is not really used because this destination place
+                    // should never be used to take values at the end of the failure
+                    // block.
+                    let dummy_place = this.temp(this.tcx.types.never, else_block_span);
+                    let failure_entry = this.cfg.start_new_block();
+                    let failure_block;
+                    failure_block = this
+                        .ast_block(
+                            dummy_place,
+                            failure_entry,
+                            *else_block,
+                            this.source_info(else_block_span),
+                        )
+                        .into_block();
+                    this.cfg.terminate(
+                        failure_block,
+                        this.source_info(else_block_span),
+                        TerminatorKind::Unreachable,
+                    );
+
+                    // Declare the bindings, which may create a source scope.
+                    let remainder_span = remainder_scope.span(this.tcx, this.region_scope_tree);
+                    this.push_scope((*remainder_scope, source_info));
+                    let_scope_stack.push(remainder_scope);
+
+                    let visibility_scope =
+                        Some(this.new_source_scope(remainder_span, LintLevel::Inherited));
+
+                    let initializer_span = this.thir[*initializer].span;
+                    let scope = (*init_scope, source_info);
+                    let failure_and_block = this.in_scope(scope, *lint_level, |this| {
+                        this.declare_bindings(
+                            visibility_scope,
+                            remainder_span,
+                            pattern,
+                            None,
+                            Some((Some(&destination), initializer_span)),
+                        );
+                        this.visit_primary_bindings(
+                            pattern,
+                            UserTypeProjections::none(),
+                            &mut |this, _, _, node, span, _, _| {
+                                this.storage_live_binding(
+                                    block,
+                                    node,
+                                    span,
+                                    OutsideGuard,
+                                    ScheduleDrops::Yes,
+                                );
+                            },
+                        );
+                        let else_block_span = this.thir[*else_block].span;
+                        let (matching, failure) =
+                            this.in_if_then_scope(last_remainder_scope, else_block_span, |this| {
+                                this.lower_let_expr(
+                                    block,
+                                    *initializer,
+                                    pattern,
+                                    None,
+                                    initializer_span,
+                                    DeclareLetBindings::No,
+                                    EmitStorageLive::No,
+                                )
+                            });
+                        matching.and(failure)
+                    });
+                    let failure = unpack!(block = failure_and_block);
+                    this.cfg.goto(failure, source_info, failure_entry);
+
+                    if let Some(source_scope) = visibility_scope {
+                        this.source_scope = source_scope;
+                    }
+                    last_remainder_scope = *remainder_scope;
+                }
+                StmtKind::Let { init_scope, initializer: None, else_block: Some(_), .. } => {
+                    span_bug!(
+                        init_scope.span(this.tcx, this.region_scope_tree),
+                        "initializer is missing, but else block is present in this let binding",
+                    )
+                }
+                StmtKind::Let {
+                    remainder_scope,
+                    init_scope,
+                    ref pattern,
+                    initializer,
+                    lint_level,
+                    else_block: None,
+                    span: _,
+                } => {
+                    let ignores_expr_result = matches!(pattern.kind, PatKind::Wild);
+                    this.block_context.push(BlockFrame::Statement { ignores_expr_result });
+
+                    // Enter the remainder scope, i.e., the bindings' destruction scope.
+                    this.push_scope((*remainder_scope, source_info));
+                    let_scope_stack.push(remainder_scope);
+
+                    // Declare the bindings, which may create a source scope.
+                    let remainder_span = remainder_scope.span(this.tcx, this.region_scope_tree);
+
+                    let visibility_scope =
+                        Some(this.new_source_scope(remainder_span, LintLevel::Inherited));
+
+                    // Evaluate the initializer, if present.
+                    if let Some(init) = *initializer {
+                        let initializer_span = this.thir[init].span;
+                        let scope = (*init_scope, source_info);
+
+                        block = this
+                            .in_scope(scope, *lint_level, |this| {
+                                this.declare_bindings(
+                                    visibility_scope,
+                                    remainder_span,
+                                    pattern,
+                                    None,
+                                    Some((None, initializer_span)),
+                                );
+                                this.expr_into_pattern(block, &pattern, init)
+                                // irrefutable pattern
+                            })
+                            .into_block();
+                    } else {
+                        let scope = (*init_scope, source_info);
+                        let _: BlockAnd<()> = this.in_scope(scope, *lint_level, |this| {
+                            this.declare_bindings(
+                                visibility_scope,
+                                remainder_span,
+                                pattern,
+                                None,
+                                None,
+                            );
+                            block.unit()
+                        });
+
+                        debug!("ast_block_stmts: pattern={:?}", pattern);
+                        this.visit_primary_bindings(
+                            pattern,
+                            UserTypeProjections::none(),
+                            &mut |this, _, _, node, span, _, _| {
+                                this.storage_live_binding(
+                                    block,
+                                    node,
+                                    span,
+                                    OutsideGuard,
+                                    ScheduleDrops::Yes,
+                                );
+                                this.schedule_drop_for_binding(node, span, OutsideGuard);
+                            },
+                        )
+                    }
+
+                    // Enter the visibility scope, after evaluating the initializer.
+                    if let Some(source_scope) = visibility_scope {
+                        this.source_scope = source_scope;
+                    }
+                    last_remainder_scope = *remainder_scope;
+                }
+            }
+
+            let popped = this.block_context.pop();
+            assert!(popped.is_some_and(|bf| bf.is_statement()));
+        }
+
+        // Then, the block may have an optional trailing expression which is a “return” value
+        // of the block, which is stored into `destination`.
+        let tcx = this.tcx;
+        let destination_ty = destination.ty(&this.local_decls, tcx).ty;
+        if let Some(expr_id) = expr {
+            let expr = &this.thir[expr_id];
+            let tail_result_is_ignored =
+                destination_ty.is_unit() || this.block_context.currently_ignores_tail_results();
+            this.block_context
+                .push(BlockFrame::TailExpr { tail_result_is_ignored, span: expr.span });
+
+            block = this.expr_into_dest(destination, block, expr_id).into_block();
+            let popped = this.block_context.pop();
+
+            assert!(popped.is_some_and(|bf| bf.is_tail_expr()));
+        } else {
+            // If a block has no trailing expression, then it is given an implicit return type.
+            // This return type is usually `()`, unless the block is diverging, in which case the
+            // return type is `!`. For the unit type, we need to actually return the unit, but in
+            // the case of `!`, no return value is required, as the block will never return.
+            // Opaque types of empty bodies also need this unit assignment, in order to infer that their
+            // type is actually unit. Otherwise there will be no defining use found in the MIR.
+            if destination_ty.is_unit()
+                || matches!(destination_ty.kind(), ty::Alias(ty::Opaque, ..))
+            {
+                // We only want to assign an implicit `()` as the return value of the block if the
+                // block does not diverge. (Otherwise, we may try to assign a unit to a `!`-type.)
+                this.cfg.push_assign_unit(block, source_info, destination, this.tcx);
+            }
+        }
+        // Finally, we pop all the let scopes before exiting out from the scope of block
+        // itself.
+        for scope in let_scope_stack.into_iter().rev() {
+            block = this.pop_scope((*scope, source_info), block).into_block();
+        }
+        // Restore the original source scope.
+        this.source_scope = outer_source_scope;
+        block.unit()
+    }
+}

compiler/rustc_mir_build/src/builder/cfg.rs

@@ -0,0 +1,137 @@
+//! Routines for manipulating the control-flow graph.
+
+use rustc_middle::mir::*;
+use rustc_middle::ty::TyCtxt;
+use tracing::debug;
+
+use crate::builder::CFG;
+
+impl<'tcx> CFG<'tcx> {
+    pub(crate) fn block_data(&self, blk: BasicBlock) -> &BasicBlockData<'tcx> {
+        &self.basic_blocks[blk]
+    }
+
+    pub(crate) fn block_data_mut(&mut self, blk: BasicBlock) -> &mut BasicBlockData<'tcx> {
+        &mut self.basic_blocks[blk]
+    }
+
+    // llvm.org/PR32488 makes this function use an excess of stack space. Mark
+    // it as #[inline(never)] to keep rustc's stack use in check.
+    #[inline(never)]
+    pub(crate) fn start_new_block(&mut self) -> BasicBlock {
+        self.basic_blocks.push(BasicBlockData::new(None))
+    }
+
+    pub(crate) fn start_new_cleanup_block(&mut self) -> BasicBlock {
+        let bb = self.start_new_block();
+        self.block_data_mut(bb).is_cleanup = true;
+        bb
+    }
+
+    pub(crate) fn push(&mut self, block: BasicBlock, statement: Statement<'tcx>) {
+        debug!("push({:?}, {:?})", block, statement);
+        self.block_data_mut(block).statements.push(statement);
+    }
+
+    pub(crate) fn push_assign(
+        &mut self,
+        block: BasicBlock,
+        source_info: SourceInfo,
+        place: Place<'tcx>,
+        rvalue: Rvalue<'tcx>,
+    ) {
+        self.push(block, Statement {
+            source_info,
+            kind: StatementKind::Assign(Box::new((place, rvalue))),
+        });
+    }
+
+    pub(crate) fn push_assign_constant(
+        &mut self,
+        block: BasicBlock,
+        source_info: SourceInfo,
+        temp: Place<'tcx>,
+        constant: ConstOperand<'tcx>,
+    ) {
+        self.push_assign(
+            block,
+            source_info,
+            temp,
+            Rvalue::Use(Operand::Constant(Box::new(constant))),
+        );
+    }
+
+    pub(crate) fn push_assign_unit(
+        &mut self,
+        block: BasicBlock,
+        source_info: SourceInfo,
+        place: Place<'tcx>,
+        tcx: TyCtxt<'tcx>,
+    ) {
+        self.push_assign(
+            block,
+            source_info,
+            place,
+            Rvalue::Use(Operand::Constant(Box::new(ConstOperand {
+                span: source_info.span,
+                user_ty: None,
+                const_: Const::zero_sized(tcx.types.unit),
+            }))),
+        );
+    }
+
+    pub(crate) fn push_fake_read(
+        &mut self,
+        block: BasicBlock,
+        source_info: SourceInfo,
+        cause: FakeReadCause,
+        place: Place<'tcx>,
+    ) {
+        let kind = StatementKind::FakeRead(Box::new((cause, place)));
+        let stmt = Statement { source_info, kind };
+        self.push(block, stmt);
+    }
+
+    pub(crate) fn push_place_mention(
+        &mut self,
+        block: BasicBlock,
+        source_info: SourceInfo,
+        place: Place<'tcx>,
+    ) {
+        let kind = StatementKind::PlaceMention(Box::new(place));
+        let stmt = Statement { source_info, kind };
+        self.push(block, stmt);
+    }
+
+    /// Adds a dummy statement whose only role is to associate a span with its
+    /// enclosing block for the purposes of coverage instrumentation.
+    ///
+    /// This results in more accurate coverage reports for certain kinds of
+    /// syntax (e.g. `continue` or `if !`) that would otherwise not appear in MIR.
+    pub(crate) fn push_coverage_span_marker(&mut self, block: BasicBlock, source_info: SourceInfo) {
+        let kind = StatementKind::Coverage(coverage::CoverageKind::SpanMarker);
+        let stmt = Statement { source_info, kind };
+        self.push(block, stmt);
+    }
+
+    pub(crate) fn terminate(
+        &mut self,
+        block: BasicBlock,
+        source_info: SourceInfo,
+        kind: TerminatorKind<'tcx>,
+    ) {
+        debug!("terminating block {:?} <- {:?}", block, kind);
+        debug_assert!(
+            self.block_data(block).terminator.is_none(),
+            "terminate: block {:?}={:?} already has a terminator set",
+            block,
+            self.block_data(block)
+        );
+        self.block_data_mut(block).terminator = Some(Terminator { source_info, kind });
+    }
+
+    /// In the `origin` block, push a `goto -> target` terminator.
+    pub(crate) fn goto(&mut self, origin: BasicBlock, source_info: SourceInfo, target: BasicBlock) {
+        self.terminate(origin, source_info, TerminatorKind::Goto { target })
+    }
+}

compiler/rustc_mir_build/src/builder/coverageinfo.rs

@@ -0,0 +1,310 @@
+use std::assert_matches::assert_matches;
+use std::collections::hash_map::Entry;
+
+use rustc_data_structures::fx::FxHashMap;
+use rustc_middle::mir::coverage::{BlockMarkerId, BranchSpan, CoverageInfoHi, CoverageKind};
+use rustc_middle::mir::{self, BasicBlock, SourceInfo, UnOp};
+use rustc_middle::thir::{ExprId, ExprKind, Pat, Thir};
+use rustc_middle::ty::TyCtxt;
+use rustc_span::def_id::LocalDefId;
+
+use crate::builder::coverageinfo::mcdc::MCDCInfoBuilder;
+use crate::builder::{Builder, CFG};
+
+mod mcdc;
+
+/// Collects coverage-related information during MIR building, to eventually be
+/// turned into a function's [`CoverageInfoHi`] when MIR building is complete.
+pub(crate) struct CoverageInfoBuilder {
+    /// Maps condition expressions to their enclosing `!`, for better instrumentation.
+    nots: FxHashMap<ExprId, NotInfo>,
+
+    markers: BlockMarkerGen,
+
+    /// Present if branch coverage is enabled.
+    branch_info: Option<BranchInfo>,
+    /// Present if MC/DC coverage is enabled.
+    mcdc_info: Option<MCDCInfoBuilder>,
+}
+
+#[derive(Default)]
+struct BranchInfo {
+    branch_spans: Vec<BranchSpan>,
+}
+
+#[derive(Clone, Copy)]
+struct NotInfo {
+    /// When visiting the associated expression as a branch condition, treat this
+    /// enclosing `!` as the branch condition instead.
+    enclosing_not: ExprId,
+    /// True if the associated expression is nested within an odd number of `!`
+    /// expressions relative to `enclosing_not` (inclusive of `enclosing_not`).
+    is_flipped: bool,
+}
+
+#[derive(Default)]
+struct BlockMarkerGen {
+    num_block_markers: usize,
+}
+
+impl BlockMarkerGen {
+    fn next_block_marker_id(&mut self) -> BlockMarkerId {
+        let id = BlockMarkerId::from_usize(self.num_block_markers);
+        self.num_block_markers += 1;
+        id
+    }
+
+    fn inject_block_marker(
+        &mut self,
+        cfg: &mut CFG<'_>,
+        source_info: SourceInfo,
+        block: BasicBlock,
+    ) -> BlockMarkerId {
+        let id = self.next_block_marker_id();
+        let marker_statement = mir::Statement {
+            source_info,
+            kind: mir::StatementKind::Coverage(CoverageKind::BlockMarker { id }),
+        };
+        cfg.push(block, marker_statement);
+
+        id
+    }
+}
+
+impl CoverageInfoBuilder {
+    /// Creates a new coverage info builder, but only if coverage instrumentation
+    /// is enabled and `def_id` represents a function that is eligible for coverage.
+    pub(crate) fn new_if_enabled(tcx: TyCtxt<'_>, def_id: LocalDefId) -> Option<Self> {
+        if !tcx.sess.instrument_coverage() || !tcx.is_eligible_for_coverage(def_id) {
+            return None;
+        }
+
+        Some(Self {
+            nots: FxHashMap::default(),
+            markers: BlockMarkerGen::default(),
+            branch_info: tcx.sess.instrument_coverage_branch().then(BranchInfo::default),
+            mcdc_info: tcx.sess.instrument_coverage_mcdc().then(MCDCInfoBuilder::new),
+        })
+    }
+
+    /// Unary `!` expressions inside an `if` condition are lowered by lowering
+    /// their argument instead, and then reversing the then/else arms of that `if`.
+    ///
+    /// That's awkward for branch coverage instrumentation, so to work around that
+    /// we pre-emptively visit any affected `!` expressions, and record extra
+    /// information that [`Builder::visit_coverage_branch_condition`] can use to
+    /// synthesize branch instrumentation for the enclosing `!`.
+    pub(crate) fn visit_unary_not(&mut self, thir: &Thir<'_>, unary_not: ExprId) {
+        assert_matches!(thir[unary_not].kind, ExprKind::Unary { op: UnOp::Not, .. });
+
+        // The information collected by this visitor is only needed when branch
+        // coverage or higher is enabled.
+        if self.branch_info.is_none() {
+            return;
+        }
+
+        self.visit_with_not_info(
+            thir,
+            unary_not,
+            // Set `is_flipped: false` for the `!` itself, so that its enclosed
+            // expression will have `is_flipped: true`.
+            NotInfo { enclosing_not: unary_not, is_flipped: false },
+        );
+    }
+
+    fn visit_with_not_info(&mut self, thir: &Thir<'_>, expr_id: ExprId, not_info: NotInfo) {
+        match self.nots.entry(expr_id) {
+            // This expression has already been marked by an enclosing `!`.
+            Entry::Occupied(_) => return,
+            Entry::Vacant(entry) => entry.insert(not_info),
+        };
+
+        match thir[expr_id].kind {
+            ExprKind::Unary { op: UnOp::Not, arg } => {
+                // Invert the `is_flipped` flag for the contents of this `!`.
+                let not_info = NotInfo { is_flipped: !not_info.is_flipped, ..not_info };
+                self.visit_with_not_info(thir, arg, not_info);
+            }
+            ExprKind::Scope { value, .. } => self.visit_with_not_info(thir, value, not_info),
+            ExprKind::Use { source } => self.visit_with_not_info(thir, source, not_info),
+            // All other expressions (including `&&` and `||`) don't need any
+            // special handling of their contents, so stop visiting.
+            _ => {}
+        }
+    }
+
+    fn register_two_way_branch<'tcx>(
+        &mut self,
+        tcx: TyCtxt<'tcx>,
+        cfg: &mut CFG<'tcx>,
+        source_info: SourceInfo,
+        true_block: BasicBlock,
+        false_block: BasicBlock,
+    ) {
+        // Separate path for handling branches when MC/DC is enabled.
+        if let Some(mcdc_info) = self.mcdc_info.as_mut() {
+            let inject_block_marker =
+                |source_info, block| self.markers.inject_block_marker(cfg, source_info, block);
+            mcdc_info.visit_evaluated_condition(
+                tcx,
+                source_info,
+                true_block,
+                false_block,
+                inject_block_marker,
+            );
+            return;
+        }
+
+        // Bail out if branch coverage is not enabled.
+        let Some(branch_info) = self.branch_info.as_mut() else { return };
+
+        let true_marker = self.markers.inject_block_marker(cfg, source_info, true_block);
+        let false_marker = self.markers.inject_block_marker(cfg, source_info, false_block);
+
+        branch_info.branch_spans.push(BranchSpan {
+            span: source_info.span,
+            true_marker,
+            false_marker,
+        });
+    }
+
+    pub(crate) fn into_done(self) -> Box<CoverageInfoHi> {
+        let Self { nots: _, markers: BlockMarkerGen { num_block_markers }, branch_info, mcdc_info } =
+            self;
+
+        let branch_spans =
+            branch_info.map(|branch_info| branch_info.branch_spans).unwrap_or_default();
+
+        let (mcdc_spans, mcdc_degraded_branch_spans) =
+            mcdc_info.map(MCDCInfoBuilder::into_done).unwrap_or_default();
+
+        // For simplicity, always return an info struct (without Option), even
+        // if there's nothing interesting in it.
+        Box::new(CoverageInfoHi {
+            num_block_markers,
+            branch_spans,
+            mcdc_degraded_branch_spans,
+            mcdc_spans,
+        })
+    }
+}
+
+impl<'tcx> Builder<'_, 'tcx> {
+    /// If condition coverage is enabled, inject extra blocks and marker statements
+    /// that will let us track the value of the condition in `place`.
+    pub(crate) fn visit_coverage_standalone_condition(
+        &mut self,
+        mut expr_id: ExprId,     // Expression giving the span of the condition
+        place: mir::Place<'tcx>, // Already holds the boolean condition value
+        block: &mut BasicBlock,
+    ) {
+        // Bail out if condition coverage is not enabled for this function.
+        let Some(coverage_info) = self.coverage_info.as_mut() else { return };
+        if !self.tcx.sess.instrument_coverage_condition() {
+            return;
+        };
+
+        // Remove any wrappers, so that we can inspect the real underlying expression.
+        while let ExprKind::Use { source: inner } | ExprKind::Scope { value: inner, .. } =
+            self.thir[expr_id].kind
+        {
+            expr_id = inner;
+        }
+        // If the expression is a lazy logical op, it will naturally get branch
+        // coverage as part of its normal lowering, so we can disregard it here.
+        if let ExprKind::LogicalOp { .. } = self.thir[expr_id].kind {
+            return;
+        }
+
+        let source_info = SourceInfo { span: self.thir[expr_id].span, scope: self.source_scope };
+
+        // Using the boolean value that has already been stored in `place`, set up
+        // control flow in the shape of a diamond, so that we can place separate
+        // marker statements in the true and false blocks. The coverage MIR pass
+        // will use those markers to inject coverage counters as appropriate.
+        //
+        //          block
+        //         /     \
+        // true_block   false_block
+        //  (marker)     (marker)
+        //         \     /
+        //        join_block
+
+        let true_block = self.cfg.start_new_block();
+        let false_block = self.cfg.start_new_block();
+        self.cfg.terminate(
+            *block,
+            source_info,
+            mir::TerminatorKind::if_(mir::Operand::Copy(place), true_block, false_block),
+        );
+
+        // Separate path for handling branches when MC/DC is enabled.
+        coverage_info.register_two_way_branch(
+            self.tcx,
+            &mut self.cfg,
+            source_info,
+            true_block,
+            false_block,
+        );
+
+        let join_block = self.cfg.start_new_block();
+        self.cfg.goto(true_block, source_info, join_block);
+        self.cfg.goto(false_block, source_info, join_block);
+        // Any subsequent codegen in the caller should use the new join block.
+        *block = join_block;
+    }
+
+    /// If branch coverage is enabled, inject marker statements into `then_block`
+    /// and `else_block`, and record their IDs in the table of branch spans.
+    pub(crate) fn visit_coverage_branch_condition(
+        &mut self,
+        mut expr_id: ExprId,
+        mut then_block: BasicBlock,
+        mut else_block: BasicBlock,
+    ) {
+        // Bail out if coverage is not enabled for this function.
+        let Some(coverage_info) = self.coverage_info.as_mut() else { return };
+
+        // If this condition expression is nested within one or more `!` expressions,
+        // replace it with the enclosing `!` collected by `visit_unary_not`.
+        if let Some(&NotInfo { enclosing_not, is_flipped }) = coverage_info.nots.get(&expr_id) {
+            expr_id = enclosing_not;
+            if is_flipped {
+                std::mem::swap(&mut then_block, &mut else_block);
+            }
+        }
+
+        let source_info = SourceInfo { span: self.thir[expr_id].span, scope: self.source_scope };
+
+        coverage_info.register_two_way_branch(
+            self.tcx,
+            &mut self.cfg,
+            source_info,
+            then_block,
+            else_block,
+        );
+    }
+
+    /// If branch coverage is enabled, inject marker statements into `true_block`
+    /// and `false_block`, and record their IDs in the table of branches.
+    ///
+    /// Used to instrument let-else and if-let (including let-chains) for branch coverage.
+    pub(crate) fn visit_coverage_conditional_let(
+        &mut self,
+        pattern: &Pat<'tcx>, // Pattern that has been matched when the true path is taken
+        true_block: BasicBlock,
+        false_block: BasicBlock,
+    ) {
+        // Bail out if coverage is not enabled for this function.
+        let Some(coverage_info) = self.coverage_info.as_mut() else { return };
+
+        let source_info = SourceInfo { span: pattern.span, scope: self.source_scope };
+        coverage_info.register_two_way_branch(
+            self.tcx,
+            &mut self.cfg,
+            source_info,
+            true_block,
+            false_block,
+        );
+    }
+}

compiler/rustc_mir_build/src/builder/coverageinfo/mcdc.rs

@@ -0,0 +1,295 @@
+use std::collections::VecDeque;
+
+use rustc_middle::bug;
+use rustc_middle::mir::coverage::{
+    BlockMarkerId, ConditionId, ConditionInfo, MCDCBranchSpan, MCDCDecisionSpan,
+};
+use rustc_middle::mir::{BasicBlock, SourceInfo};
+use rustc_middle::thir::LogicalOp;
+use rustc_middle::ty::TyCtxt;
+use rustc_span::Span;
+
+use crate::builder::Builder;
+use crate::errors::MCDCExceedsConditionLimit;
+
+/// LLVM uses `i16` to represent condition id. Hence `i16::MAX` is the hard limit for number of
+/// conditions in a decision.
+const MAX_CONDITIONS_IN_DECISION: usize = i16::MAX as usize;
+
+#[derive(Default)]
+struct MCDCDecisionCtx {
+    /// To construct condition evaluation tree.
+    decision_stack: VecDeque<ConditionInfo>,
+    processing_decision: Option<MCDCDecisionSpan>,
+    conditions: Vec<MCDCBranchSpan>,
+}
+
+struct MCDCState {
+    decision_ctx_stack: Vec<MCDCDecisionCtx>,
+}
+
+impl MCDCState {
+    fn new() -> Self {
+        Self { decision_ctx_stack: vec![MCDCDecisionCtx::default()] }
+    }
+
+    /// Decision depth is given as a u16 to reduce the size of the `CoverageKind`,
+    /// as it is very unlikely that the depth ever reaches 2^16.
+    #[inline]
+    fn decision_depth(&self) -> u16 {
+        match u16::try_from(self.decision_ctx_stack.len())
+            .expect(
+                "decision depth did not fit in u16, this is likely to be an instrumentation error",
+            )
+            .checked_sub(1)
+        {
+            Some(d) => d,
+            None => bug!("Unexpected empty decision stack"),
+        }
+    }
+
+    // At first we assign ConditionIds for each sub expression.
+    // If the sub expression is composite, re-assign its ConditionId to its LHS and generate a new ConditionId for its RHS.
+    //
+    // Example: "x = (A && B) || (C && D) || (D && F)"
+    //
+    //      Visit Depth1:
+    //              (A && B) || (C && D) || (D && F)
+    //              ^-------LHS--------^    ^-RHS--^
+    //                      ID=1              ID=2
+    //
+    //      Visit LHS-Depth2:
+    //              (A && B) || (C && D)
+    //              ^-LHS--^    ^-RHS--^
+    //                ID=1        ID=3
+    //
+    //      Visit LHS-Depth3:
+    //               (A && B)
+    //               LHS   RHS
+    //               ID=1  ID=4
+    //
+    //      Visit RHS-Depth3:
+    //                         (C && D)
+    //                         LHS   RHS
+    //                         ID=3  ID=5
+    //
+    //      Visit RHS-Depth2:              (D && F)
+    //                                     LHS   RHS
+    //                                     ID=2  ID=6
+    //
+    //      Visit Depth1:
+    //              (A && B)  || (C && D)  || (D && F)
+    //              ID=1  ID=4   ID=3  ID=5   ID=2  ID=6
+    //
+    // A node ID of '0' always means MC/DC isn't being tracked.
+    //
+    // If a "next" node ID is '0', it means it's the end of the test vector.
+    //
+    // As the compiler tracks expression in pre-order, we can ensure that condition info of parents are always properly assigned when their children are visited.
+    // - If the op is AND, the "false_next" of LHS and RHS should be the parent's "false_next". While "true_next" of the LHS is the RHS, the "true next" of RHS is the parent's "true_next".
+    // - If the op is OR, the "true_next" of LHS and RHS should be the parent's "true_next". While "false_next" of the LHS is the RHS, the "false next" of RHS is the parent's "false_next".
+    fn record_conditions(&mut self, op: LogicalOp, span: Span) {
+        let decision_depth = self.decision_depth();
+        let Some(decision_ctx) = self.decision_ctx_stack.last_mut() else {
+            bug!("Unexpected empty decision_ctx_stack")
+        };
+        let decision = match decision_ctx.processing_decision.as_mut() {
+            Some(decision) => {
+                decision.span = decision.span.to(span);
+                decision
+            }
+            None => decision_ctx.processing_decision.insert(MCDCDecisionSpan {
+                span,
+                num_conditions: 0,
+                end_markers: vec![],
+                decision_depth,
+            }),
+        };
+
+        let parent_condition = decision_ctx.decision_stack.pop_back().unwrap_or_else(|| {
+            assert_eq!(
+                decision.num_conditions, 0,
+                "decision stack must be empty only for empty decision"
+            );
+            decision.num_conditions += 1;
+            ConditionInfo {
+                condition_id: ConditionId::START,
+                true_next_id: None,
+                false_next_id: None,
+            }
+        });
+        let lhs_id = parent_condition.condition_id;
+
+        let rhs_condition_id = ConditionId::from(decision.num_conditions);
+        decision.num_conditions += 1;
+        let (lhs, rhs) = match op {
+            LogicalOp::And => {
+                let lhs = ConditionInfo {
+                    condition_id: lhs_id,
+                    true_next_id: Some(rhs_condition_id),
+                    false_next_id: parent_condition.false_next_id,
+                };
+                let rhs = ConditionInfo {
+                    condition_id: rhs_condition_id,
+                    true_next_id: parent_condition.true_next_id,
+                    false_next_id: parent_condition.false_next_id,
+                };
+                (lhs, rhs)
+            }
+            LogicalOp::Or => {
+                let lhs = ConditionInfo {
+                    condition_id: lhs_id,
+                    true_next_id: parent_condition.true_next_id,
+                    false_next_id: Some(rhs_condition_id),
+                };
+                let rhs = ConditionInfo {
+                    condition_id: rhs_condition_id,
+                    true_next_id: parent_condition.true_next_id,
+                    false_next_id: parent_condition.false_next_id,
+                };
+                (lhs, rhs)
+            }
+        };
+        // We visit expressions tree in pre-order, so place the left-hand side on the top.
+        decision_ctx.decision_stack.push_back(rhs);
+        decision_ctx.decision_stack.push_back(lhs);
+    }
+
+    fn try_finish_decision(
+        &mut self,
+        span: Span,
+        true_marker: BlockMarkerId,
+        false_marker: BlockMarkerId,
+        degraded_branches: &mut Vec<MCDCBranchSpan>,
+    ) -> Option<(MCDCDecisionSpan, Vec<MCDCBranchSpan>)> {
+        let Some(decision_ctx) = self.decision_ctx_stack.last_mut() else {
+            bug!("Unexpected empty decision_ctx_stack")
+        };
+        let Some(condition_info) = decision_ctx.decision_stack.pop_back() else {
+            let branch = MCDCBranchSpan {
+                span,
+                condition_info: ConditionInfo {
+                    condition_id: ConditionId::START,
+                    true_next_id: None,
+                    false_next_id: None,
+                },
+                true_marker,
+                false_marker,
+            };
+            degraded_branches.push(branch);
+            return None;
+        };
+        let Some(decision) = decision_ctx.processing_decision.as_mut() else {
+            bug!("Processing decision should have been created before any conditions are taken");
+        };
+        if condition_info.true_next_id.is_none() {
+            decision.end_markers.push(true_marker);
+        }
+        if condition_info.false_next_id.is_none() {
+            decision.end_markers.push(false_marker);
+        }
+        decision_ctx.conditions.push(MCDCBranchSpan {
+            span,
+            condition_info,
+            true_marker,
+            false_marker,
+        });
+
+        if decision_ctx.decision_stack.is_empty() {
+            let conditions = std::mem::take(&mut decision_ctx.conditions);
+            decision_ctx.processing_decision.take().map(|decision| (decision, conditions))
+        } else {
+            None
+        }
+    }
+}
+
+pub(crate) struct MCDCInfoBuilder {
+    degraded_spans: Vec<MCDCBranchSpan>,
+    mcdc_spans: Vec<(MCDCDecisionSpan, Vec<MCDCBranchSpan>)>,
+    state: MCDCState,
+}
+
+impl MCDCInfoBuilder {
+    pub(crate) fn new() -> Self {
+        Self { degraded_spans: vec![], mcdc_spans: vec![], state: MCDCState::new() }
+    }
+
+    pub(crate) fn visit_evaluated_condition(
+        &mut self,
+        tcx: TyCtxt<'_>,
+        source_info: SourceInfo,
+        true_block: BasicBlock,
+        false_block: BasicBlock,
+        mut inject_block_marker: impl FnMut(SourceInfo, BasicBlock) -> BlockMarkerId,
+    ) {
+        let true_marker = inject_block_marker(source_info, true_block);
+        let false_marker = inject_block_marker(source_info, false_block);
+
+        // take_condition() returns Some for decision_result when the decision stack
+        // is empty, i.e. when all the conditions of the decision were instrumented,
+        // and the decision is "complete".
+        if let Some((decision, conditions)) = self.state.try_finish_decision(
+            source_info.span,
+            true_marker,
+            false_marker,
+            &mut self.degraded_spans,
+        ) {
+            let num_conditions = conditions.len();
+            assert_eq!(
+                num_conditions, decision.num_conditions,
+                "final number of conditions is not correct"
+            );
+            match num_conditions {
+                0 => {
+                    unreachable!("Decision with no condition is not expected");
+                }
+                1..=MAX_CONDITIONS_IN_DECISION => {
+                    self.mcdc_spans.push((decision, conditions));
+                }
+                _ => {
+                    self.degraded_spans.extend(conditions);
+
+                    tcx.dcx().emit_warn(MCDCExceedsConditionLimit {
+                        span: decision.span,
+                        num_conditions,
+                        max_conditions: MAX_CONDITIONS_IN_DECISION,
+                    });
+                }
+            }
+        }
+    }
+
+    pub(crate) fn into_done(
+        self,
+    ) -> (Vec<(MCDCDecisionSpan, Vec<MCDCBranchSpan>)>, Vec<MCDCBranchSpan>) {
+        (self.mcdc_spans, self.degraded_spans)
+    }
+}
+
+impl Builder<'_, '_> {
+    pub(crate) fn visit_coverage_branch_operation(&mut self, logical_op: LogicalOp, span: Span) {
+        if let Some(coverage_info) = self.coverage_info.as_mut()
+            && let Some(mcdc_info) = coverage_info.mcdc_info.as_mut()
+        {
+            mcdc_info.state.record_conditions(logical_op, span);
+        }
+    }
+
+    pub(crate) fn mcdc_increment_depth_if_enabled(&mut self) {
+        if let Some(coverage_info) = self.coverage_info.as_mut()
+            && let Some(mcdc_info) = coverage_info.mcdc_info.as_mut()
+        {
+            mcdc_info.state.decision_ctx_stack.push(MCDCDecisionCtx::default());
+        };
+    }
+
+    pub(crate) fn mcdc_decrement_depth_if_enabled(&mut self) {
+        if let Some(coverage_info) = self.coverage_info.as_mut()
+            && let Some(mcdc_info) = coverage_info.mcdc_info.as_mut()
+            && mcdc_info.state.decision_ctx_stack.pop().is_none()
+        {
+            bug!("Unexpected empty decision stack");
+        };
+    }
+}

compiler/rustc_mir_build/src/builder/custom/mod.rs

@@ -0,0 +1,176 @@
+//! Provides the implementation of the `custom_mir` attribute.
+//!
+//! Up until MIR building, this attribute has absolutely no effect. The `mir!` macro is a normal
+//! decl macro that expands like any other, and the code goes through parsing, name resolution and
+//! type checking like all other code. In MIR building we finally detect whether this attribute is
+//! present, and if so we branch off into this module, which implements the attribute by
+//! implementing a custom lowering from THIR to MIR.
+//!
+//! The result of this lowering is returned "normally" from the `build_mir` hook, with the only
+//! notable difference being that the `injected` field in the body is set. Various components of the
+//! MIR pipeline, like borrowck and the pass manager will then consult this field (via
+//! `body.should_skip()`) to skip the parts of the MIR pipeline that precede the MIR phase the user
+//! specified.
+//!
+//! This file defines the general framework for the custom parsing. The parsing for all the
+//! "top-level" constructs can be found in the `parse` submodule, while the parsing for statements,
+//! terminators, and everything below can be found in the `parse::instruction` submodule.
+//!
+
+use rustc_data_structures::fx::FxHashMap;
+use rustc_hir::def_id::DefId;
+use rustc_hir::{Attribute, HirId};
+use rustc_index::{IndexSlice, IndexVec};
+use rustc_middle::mir::*;
+use rustc_middle::span_bug;
+use rustc_middle::thir::*;
+use rustc_middle::ty::{self, Ty, TyCtxt};
+use rustc_span::Span;
+
+mod parse;
+
+pub(super) fn build_custom_mir<'tcx>(
+    tcx: TyCtxt<'tcx>,
+    did: DefId,
+    hir_id: HirId,
+    thir: &Thir<'tcx>,
+    expr: ExprId,
+    params: &IndexSlice<ParamId, Param<'tcx>>,
+    return_ty: Ty<'tcx>,
+    return_ty_span: Span,
+    span: Span,
+    attr: &Attribute,
+) -> Body<'tcx> {
+    let mut body = Body {
+        basic_blocks: BasicBlocks::new(IndexVec::new()),
+        source: MirSource::item(did),
+        phase: MirPhase::Built,
+        source_scopes: IndexVec::new(),
+        coroutine: None,
+        local_decls: IndexVec::new(),
+        user_type_annotations: IndexVec::new(),
+        arg_count: params.len(),
+        spread_arg: None,
+        var_debug_info: Vec::new(),
+        span,
+        required_consts: None,
+        mentioned_items: None,
+        is_polymorphic: false,
+        tainted_by_errors: None,
+        injection_phase: None,
+        pass_count: 0,
+        coverage_info_hi: None,
+        function_coverage_info: None,
+    };
+
+    body.local_decls.push(LocalDecl::new(return_ty, return_ty_span));
+    body.basic_blocks_mut().push(BasicBlockData::new(None));
+    body.source_scopes.push(SourceScopeData {
+        span,
+        parent_scope: None,
+        inlined: None,
+        inlined_parent_scope: None,
+        local_data: ClearCrossCrate::Set(SourceScopeLocalData { lint_root: hir_id }),
+    });
+    body.injection_phase = Some(parse_attribute(attr));
+
+    let mut pctxt = ParseCtxt {
+        tcx,
+        typing_env: body.typing_env(tcx),
+        thir,
+        source_scope: OUTERMOST_SOURCE_SCOPE,
+        body: &mut body,
+        local_map: FxHashMap::default(),
+        block_map: FxHashMap::default(),
+    };
+
+    let res: PResult<_> = try {
+        pctxt.parse_args(params)?;
+        pctxt.parse_body(expr)?;
+    };
+    if let Err(err) = res {
+        tcx.dcx().span_fatal(
+            err.span,
+            format!("Could not parse {}, found: {:?}", err.expected, err.item_description),
+        )
+    }
+
+    body
+}
+
+fn parse_attribute(attr: &Attribute) -> MirPhase {
+    let meta_items = attr.meta_item_list().unwrap();
+    let mut dialect: Option<String> = None;
+    let mut phase: Option<String> = None;
+
+    for nested in meta_items {
+        let name = nested.name_or_empty();
+        let value = nested.value_str().unwrap().as_str().to_string();
+        match name.as_str() {
+            "dialect" => {
+                assert!(dialect.is_none());
+                dialect = Some(value);
+            }
+            "phase" => {
+                assert!(phase.is_none());
+                phase = Some(value);
+            }
+            other => {
+                span_bug!(
+                    nested.span(),
+                    "Unexpected key while parsing custom_mir attribute: '{}'",
+                    other
+                );
+            }
+        }
+    }
+
+    let Some(dialect) = dialect else {
+        assert!(phase.is_none());
+        return MirPhase::Built;
+    };
+
+    MirPhase::parse(dialect, phase)
+}
+
+struct ParseCtxt<'a, 'tcx> {
+    tcx: TyCtxt<'tcx>,
+    typing_env: ty::TypingEnv<'tcx>,
+    thir: &'a Thir<'tcx>,
+    source_scope: SourceScope,
+    body: &'a mut Body<'tcx>,
+    local_map: FxHashMap<LocalVarId, Local>,
+    block_map: FxHashMap<LocalVarId, BasicBlock>,
+}
+
+struct ParseError {
+    span: Span,
+    item_description: String,
+    expected: String,
+}
+
+impl<'a, 'tcx> ParseCtxt<'a, 'tcx> {
+    fn expr_error(&self, expr: ExprId, expected: &'static str) -> ParseError {
+        let expr = &self.thir[expr];
+        ParseError {
+            span: expr.span,
+            item_description: format!("{:?}", expr.kind),
+            expected: expected.to_string(),
+        }
+    }
+
+    fn stmt_error(&self, stmt: StmtId, expected: &'static str) -> ParseError {
+        let stmt = &self.thir[stmt];
+        let span = match stmt.kind {
+            StmtKind::Expr { expr, .. } => self.thir[expr].span,
+            StmtKind::Let { span, .. } => span,
+        };
+        ParseError {
+            span,
+            item_description: format!("{:?}", stmt.kind),
+            expected: expected.to_string(),
+        }
+    }
+}
+
+type PResult<T> = Result<T, ParseError>;

compiler/rustc_mir_build/src/builder/custom/parse.rs

@@ -0,0 +1,333 @@
+use rustc_index::IndexSlice;
+use rustc_middle::mir::*;
+use rustc_middle::thir::*;
+use rustc_middle::ty::{self, Ty};
+use rustc_span::Span;
+
+use super::{PResult, ParseCtxt, ParseError};
+
+mod instruction;
+
+/// Helper macro for parsing custom MIR.
+///
+/// Example usage looks something like:
+/// ```rust,ignore (incomplete example)
+/// parse_by_kind!(
+///     self, // : &ParseCtxt
+///     expr_id, // what you're matching against
+///     "assignment", // the thing you're trying to parse
+///     @call("mir_assign", args) => { args[0] }, // match invocations of the `mir_assign` special function
+///     ExprKind::Assign { lhs, .. } => { lhs }, // match thir assignment expressions
+///     // no need for fallthrough case - reasonable error is automatically generated
+/// )
+/// ```
+macro_rules! parse_by_kind {
+    (
+        $self:ident,
+        $expr_id:expr,
+        $expr_name:pat,
+        $expected:literal,
+        $(
+            @call($name:ident, $args:ident) => $call_expr:expr,
+        )*
+        $(
+            @variant($adt:ident, $variant:ident) => $variant_expr:expr,
+        )*
+        $(
+            $pat:pat $(if $guard:expr)? => $expr:expr,
+        )*
+    ) => {{
+        let expr_id = $self.preparse($expr_id);
+        let expr = &$self.thir[expr_id];
+        tracing::debug!("Trying to parse {:?} as {}", expr.kind, $expected);
+        let $expr_name = expr;
+        match &expr.kind {
+            $(
+                ExprKind::Call { ty, fun: _, args: $args, .. } if {
+                    match ty.kind() {
+                        ty::FnDef(did, _) => {
+                            $self.tcx.is_diagnostic_item(rustc_span::sym::$name, *did)
+                        }
+                        _ => false,
+                    }
+                } => $call_expr,
+            )*
+            $(
+                ExprKind::Adt(box AdtExpr { adt_def, variant_index, .. }) if {
+                    $self.tcx.is_diagnostic_item(rustc_span::sym::$adt, adt_def.did()) &&
+                    adt_def.variants()[*variant_index].name == rustc_span::sym::$variant
+                } => $variant_expr,
+            )*
+            $(
+                $pat $(if $guard)? => $expr,
+            )*
+            #[allow(unreachable_patterns)]
+            _ => return Err($self.expr_error(expr_id, $expected))
+        }
+    }};
+}
+pub(crate) use parse_by_kind;
+
+impl<'a, 'tcx> ParseCtxt<'a, 'tcx> {
+    /// Expressions should only ever be matched on after preparsing them. This removes extra scopes
+    /// we don't care about.
+    fn preparse(&self, expr_id: ExprId) -> ExprId {
+        let expr = &self.thir[expr_id];
+        match expr.kind {
+            ExprKind::Scope { value, .. } => self.preparse(value),
+            _ => expr_id,
+        }
+    }
+
+    fn statement_as_expr(&self, stmt_id: StmtId) -> PResult<ExprId> {
+        match &self.thir[stmt_id].kind {
+            StmtKind::Expr { expr, .. } => Ok(*expr),
+            kind @ StmtKind::Let { pattern, .. } => Err(ParseError {
+                span: pattern.span,
+                item_description: format!("{kind:?}"),
+                expected: "expression".to_string(),
+            }),
+        }
+    }
+
+    pub(crate) fn parse_args(&mut self, params: &IndexSlice<ParamId, Param<'tcx>>) -> PResult<()> {
+        for param in params.iter() {
+            let (var, span) = {
+                let pat = param.pat.as_ref().unwrap();
+                match &pat.kind {
+                    PatKind::Binding { var, .. } => (*var, pat.span),
+                    _ => {
+                        return Err(ParseError {
+                            span: pat.span,
+                            item_description: format!("{:?}", pat.kind),
+                            expected: "local".to_string(),
+                        });
+                    }
+                }
+            };
+            let decl = LocalDecl::new(param.ty, span);
+            let local = self.body.local_decls.push(decl);
+            self.local_map.insert(var, local);
+        }
+
+        Ok(())
+    }
+
+    /// Bodies are of the form:
+    ///
+    /// ```text
+    /// {
+    ///     let bb1: BasicBlock;
+    ///     let bb2: BasicBlock;
+    ///     {
+    ///         let RET: _;
+    ///         let local1;
+    ///         let local2;
+    ///
+    ///         {
+    ///             { // entry block
+    ///                 statement1;
+    ///                 terminator1
+    ///             };
+    ///
+    ///             bb1 = {
+    ///                 statement2;
+    ///                 terminator2
+    ///             };
+    ///
+    ///             bb2 = {
+    ///                 statement3;
+    ///                 terminator3
+    ///             }
+    ///
+    ///             RET
+    ///         }
+    ///     }
+    /// }
+    /// ```
+    ///
+    /// This allows us to easily parse the basic blocks declarations, local declarations, and
+    /// basic block definitions in order.
+    pub(crate) fn parse_body(&mut self, expr_id: ExprId) -> PResult<()> {
+        let body = parse_by_kind!(self, expr_id, _, "whole body",
+            ExprKind::Block { block } => self.thir[*block].expr.unwrap(),
+        );
+        let (block_decls, rest) = parse_by_kind!(self, body, _, "body with block decls",
+            ExprKind::Block { block } => {
+                let block = &self.thir[*block];
+                (&block.stmts, block.expr.unwrap())
+            },
+        );
+        self.parse_block_decls(block_decls.iter().copied())?;
+
+        let (local_decls, rest) = parse_by_kind!(self, rest, _, "body with local decls",
+            ExprKind::Block { block } => {
+                let block = &self.thir[*block];
+                (&block.stmts, block.expr.unwrap())
+            },
+        );
+        self.parse_local_decls(local_decls.iter().copied())?;
+
+        let (debuginfo, rest) = parse_by_kind!(self, rest, _, "body with debuginfo",
+            ExprKind::Block { block } => {
+                let block = &self.thir[*block];
+                (&block.stmts, block.expr.unwrap())
+            },
+        );
+        self.parse_debuginfo(debuginfo.iter().copied())?;
+
+        let block_defs = parse_by_kind!(self, rest, _, "body with block defs",
+            ExprKind::Block { block } => &self.thir[*block].stmts,
+        );
+        for (i, block_def) in block_defs.iter().enumerate() {
+            let is_cleanup = self.body.basic_blocks_mut()[BasicBlock::from_usize(i)].is_cleanup;
+            let block = self.parse_block_def(self.statement_as_expr(*block_def)?, is_cleanup)?;
+            self.body.basic_blocks_mut()[BasicBlock::from_usize(i)] = block;
+        }
+
+        Ok(())
+    }
+
+    fn parse_block_decls(&mut self, stmts: impl Iterator<Item = StmtId>) -> PResult<()> {
+        for stmt in stmts {
+            self.parse_basic_block_decl(stmt)?;
+        }
+        Ok(())
+    }
+
+    fn parse_basic_block_decl(&mut self, stmt: StmtId) -> PResult<()> {
+        match &self.thir[stmt].kind {
+            StmtKind::Let { pattern, initializer: Some(initializer), .. } => {
+                let (var, ..) = self.parse_var(pattern)?;
+                let mut data = BasicBlockData::new(None);
+                data.is_cleanup = parse_by_kind!(self, *initializer, _, "basic block declaration",
+                    @variant(mir_basic_block, Normal) => false,
+                    @variant(mir_basic_block, Cleanup) => true,
+                );
+                let block = self.body.basic_blocks_mut().push(data);
+                self.block_map.insert(var, block);
+                Ok(())
+            }
+            _ => Err(self.stmt_error(stmt, "let statement with an initializer")),
+        }
+    }
+
+    fn parse_local_decls(&mut self, mut stmts: impl Iterator<Item = StmtId>) -> PResult<()> {
+        let (ret_var, ..) = self.parse_let_statement(stmts.next().unwrap())?;
+        self.local_map.insert(ret_var, Local::ZERO);
+
+        for stmt in stmts {
+            let (var, ty, span) = self.parse_let_statement(stmt)?;
+            let decl = LocalDecl::new(ty, span);
+            let local = self.body.local_decls.push(decl);
+            self.local_map.insert(var, local);
+        }
+
+        Ok(())
+    }
+
+    fn parse_debuginfo(&mut self, stmts: impl Iterator<Item = StmtId>) -> PResult<()> {
+        for stmt in stmts {
+            let stmt = &self.thir[stmt];
+            let expr = match stmt.kind {
+                StmtKind::Let { span, .. } => {
+                    return Err(ParseError {
+                        span,
+                        item_description: format!("{:?}", stmt),
+                        expected: "debuginfo".to_string(),
+                    });
+                }
+                StmtKind::Expr { expr, .. } => expr,
+            };
+            let span = self.thir[expr].span;
+            let (name, operand) = parse_by_kind!(self, expr, _, "debuginfo",
+                @call(mir_debuginfo, args) => {
+                    (args[0], args[1])
+                },
+            );
+            let name = parse_by_kind!(self, name, _, "debuginfo",
+                ExprKind::Literal { lit, neg: false } => lit,
+            );
+            let Some(name) = name.node.str() else {
+                return Err(ParseError {
+                    span,
+                    item_description: format!("{:?}", name),
+                    expected: "string".to_string(),
+                });
+            };
+            let operand = self.parse_operand(operand)?;
+            let value = match operand {
+                Operand::Constant(c) => VarDebugInfoContents::Const(*c),
+                Operand::Copy(p) | Operand::Move(p) => VarDebugInfoContents::Place(p),
+            };
+            let dbginfo = VarDebugInfo {
+                name,
+                source_info: SourceInfo { span, scope: self.source_scope },
+                composite: None,
+                argument_index: None,
+                value,
+            };
+            self.body.var_debug_info.push(dbginfo);
+        }
+
+        Ok(())
+    }
+
+    fn parse_let_statement(&mut self, stmt_id: StmtId) -> PResult<(LocalVarId, Ty<'tcx>, Span)> {
+        let pattern = match &self.thir[stmt_id].kind {
+            StmtKind::Let { pattern, .. } => pattern,
+            StmtKind::Expr { expr, .. } => {
+                return Err(self.expr_error(*expr, "let statement"));
+            }
+        };
+
+        self.parse_var(pattern)
+    }
+
+    fn parse_var(&mut self, mut pat: &Pat<'tcx>) -> PResult<(LocalVarId, Ty<'tcx>, Span)> {
+        // Make sure we throw out any `AscribeUserType` we find
+        loop {
+            match &pat.kind {
+                PatKind::Binding { var, ty, .. } => break Ok((*var, *ty, pat.span)),
+                PatKind::AscribeUserType { subpattern, .. } => {
+                    pat = subpattern;
+                }
+                _ => {
+                    break Err(ParseError {
+                        span: pat.span,
+                        item_description: format!("{:?}", pat.kind),
+                        expected: "local".to_string(),
+                    });
+                }
+            }
+        }
+    }
+
+    fn parse_block_def(&self, expr_id: ExprId, is_cleanup: bool) -> PResult<BasicBlockData<'tcx>> {
+        let block = parse_by_kind!(self, expr_id, _, "basic block",
+            ExprKind::Block { block } => &self.thir[*block],
+        );
+
+        let mut data = BasicBlockData::new(None);
+        data.is_cleanup = is_cleanup;
+        for stmt_id in &*block.stmts {
+            let stmt = self.statement_as_expr(*stmt_id)?;
+            let span = self.thir[stmt].span;
+            let statement = self.parse_statement(stmt)?;
+            data.statements.push(Statement {
+                source_info: SourceInfo { span, scope: self.source_scope },
+                kind: statement,
+            });
+        }
+
+        let Some(trailing) = block.expr else { return Err(self.expr_error(expr_id, "terminator")) };
+        let span = self.thir[trailing].span;
+        let terminator = self.parse_terminator(trailing)?;
+        data.terminator = Some(Terminator {
+            source_info: SourceInfo { span, scope: self.source_scope },
+            kind: terminator,
+        });
+
+        Ok(data)
+    }
+}

compiler/rustc_mir_build/src/builder/custom/parse/instruction.rs

@@ -0,0 +1,400 @@
+use rustc_abi::{FieldIdx, VariantIdx};
+use rustc_middle::mir::interpret::Scalar;
+use rustc_middle::mir::tcx::PlaceTy;
+use rustc_middle::mir::*;
+use rustc_middle::thir::*;
+use rustc_middle::ty;
+use rustc_middle::ty::cast::mir_cast_kind;
+use rustc_span::Span;
+use rustc_span::source_map::Spanned;
+
+use super::{PResult, ParseCtxt, parse_by_kind};
+use crate::builder::custom::ParseError;
+use crate::builder::expr::as_constant::as_constant_inner;
+
+impl<'a, 'tcx> ParseCtxt<'a, 'tcx> {
+    pub(crate) fn parse_statement(&self, expr_id: ExprId) -> PResult<StatementKind<'tcx>> {
+        parse_by_kind!(self, expr_id, _, "statement",
+            @call(mir_storage_live, args) => {
+                Ok(StatementKind::StorageLive(self.parse_local(args[0])?))
+            },
+            @call(mir_storage_dead, args) => {
+                Ok(StatementKind::StorageDead(self.parse_local(args[0])?))
+            },
+            @call(mir_assume, args) => {
+                let op = self.parse_operand(args[0])?;
+                Ok(StatementKind::Intrinsic(Box::new(NonDivergingIntrinsic::Assume(op))))
+            },
+            @call(mir_deinit, args) => {
+                Ok(StatementKind::Deinit(Box::new(self.parse_place(args[0])?)))
+            },
+            @call(mir_retag, args) => {
+                Ok(StatementKind::Retag(RetagKind::Default, Box::new(self.parse_place(args[0])?)))
+            },
+            @call(mir_set_discriminant, args) => {
+                let place = self.parse_place(args[0])?;
+                let var = self.parse_integer_literal(args[1])? as u32;
+                Ok(StatementKind::SetDiscriminant {
+                    place: Box::new(place),
+                    variant_index: VariantIdx::from_u32(var),
+                })
+            },
+            ExprKind::Assign { lhs, rhs } => {
+                let lhs = self.parse_place(*lhs)?;
+                let rhs = self.parse_rvalue(*rhs)?;
+                Ok(StatementKind::Assign(Box::new((lhs, rhs))))
+            },
+        )
+    }
+
+    pub(crate) fn parse_terminator(&self, expr_id: ExprId) -> PResult<TerminatorKind<'tcx>> {
+        parse_by_kind!(self, expr_id, expr, "terminator",
+            @call(mir_return, _args) => {
+                Ok(TerminatorKind::Return)
+            },
+            @call(mir_goto, args) => {
+                Ok(TerminatorKind::Goto { target: self.parse_block(args[0])? } )
+            },
+            @call(mir_unreachable, _args) => {
+                Ok(TerminatorKind::Unreachable)
+            },
+            @call(mir_unwind_resume, _args) => {
+                Ok(TerminatorKind::UnwindResume)
+            },
+            @call(mir_unwind_terminate, args) => {
+                Ok(TerminatorKind::UnwindTerminate(self.parse_unwind_terminate_reason(args[0])?))
+            },
+            @call(mir_drop, args) => {
+                Ok(TerminatorKind::Drop {
+                    place: self.parse_place(args[0])?,
+                    target: self.parse_return_to(args[1])?,
+                    unwind: self.parse_unwind_action(args[2])?,
+                    replace: false,
+                })
+            },
+            @call(mir_call, args) => {
+                self.parse_call(args)
+            },
+            @call(mir_tail_call, args) => {
+                self.parse_tail_call(args)
+            },
+            ExprKind::Match { scrutinee, arms, .. } => {
+                let discr = self.parse_operand(*scrutinee)?;
+                self.parse_match(arms, expr.span).map(|t| TerminatorKind::SwitchInt { discr, targets: t })
+            },
+        )
+    }
+
+    fn parse_unwind_terminate_reason(&self, expr_id: ExprId) -> PResult<UnwindTerminateReason> {
+        parse_by_kind!(self, expr_id, _, "unwind terminate reason",
+            @variant(mir_unwind_terminate_reason, Abi) => {
+                Ok(UnwindTerminateReason::Abi)
+            },
+            @variant(mir_unwind_terminate_reason, InCleanup) => {
+                Ok(UnwindTerminateReason::InCleanup)
+            },
+        )
+    }
+
+    fn parse_unwind_action(&self, expr_id: ExprId) -> PResult<UnwindAction> {
+        parse_by_kind!(self, expr_id, _, "unwind action",
+            @call(mir_unwind_continue, _args) => {
+                Ok(UnwindAction::Continue)
+            },
+            @call(mir_unwind_unreachable, _args) => {
+                Ok(UnwindAction::Unreachable)
+            },
+            @call(mir_unwind_terminate, args) => {
+                Ok(UnwindAction::Terminate(self.parse_unwind_terminate_reason(args[0])?))
+            },
+            @call(mir_unwind_cleanup, args) => {
+                Ok(UnwindAction::Cleanup(self.parse_block(args[0])?))
+            },
+        )
+    }
+
+    fn parse_return_to(&self, expr_id: ExprId) -> PResult<BasicBlock> {
+        parse_by_kind!(self, expr_id, _, "return block",
+            @call(mir_return_to, args) => {
+                self.parse_block(args[0])
+            },
+        )
+    }
+
+    fn parse_match(&self, arms: &[ArmId], span: Span) -> PResult<SwitchTargets> {
+        let Some((otherwise, rest)) = arms.split_last() else {
+            return Err(ParseError {
+                span,
+                item_description: "no arms".to_string(),
+                expected: "at least one arm".to_string(),
+            });
+        };
+
+        let otherwise = &self.thir[*otherwise];
+        let PatKind::Wild = otherwise.pattern.kind else {
+            return Err(ParseError {
+                span: otherwise.span,
+                item_description: format!("{:?}", otherwise.pattern.kind),
+                expected: "wildcard pattern".to_string(),
+            });
+        };
+        let otherwise = self.parse_block(otherwise.body)?;
+
+        let mut values = Vec::new();
+        let mut targets = Vec::new();
+        for arm in rest {
+            let arm = &self.thir[*arm];
+            let value = match arm.pattern.kind {
+                PatKind::Constant { value } => value,
+                PatKind::ExpandedConstant { ref subpattern, def_id: _, is_inline: false }
+                    if let PatKind::Constant { value } = subpattern.kind =>
+                {
+                    value
+                }
+                _ => {
+                    return Err(ParseError {
+                        span: arm.pattern.span,
+                        item_description: format!("{:?}", arm.pattern.kind),
+                        expected: "constant pattern".to_string(),
+                    });
+                }
+            };
+            values.push(value.eval_bits(self.tcx, self.typing_env));
+            targets.push(self.parse_block(arm.body)?);
+        }
+
+        Ok(SwitchTargets::new(values.into_iter().zip(targets), otherwise))
+    }
+
+    fn parse_call(&self, args: &[ExprId]) -> PResult<TerminatorKind<'tcx>> {
+        let (destination, call) = parse_by_kind!(self, args[0], _, "function call",
+            ExprKind::Assign { lhs, rhs } => (*lhs, *rhs),
+        );
+        let destination = self.parse_place(destination)?;
+        let target = self.parse_return_to(args[1])?;
+        let unwind = self.parse_unwind_action(args[2])?;
+
+        parse_by_kind!(self, call, _, "function call",
+            ExprKind::Call { fun, args, from_hir_call, fn_span, .. } => {
+                let fun = self.parse_operand(*fun)?;
+                let args = args
+                    .iter()
+                    .map(|arg|
+                        Ok(Spanned { node: self.parse_operand(*arg)?, span: self.thir.exprs[*arg].span  } )
+                    )
+                    .collect::<PResult<Box<[_]>>>()?;
+                Ok(TerminatorKind::Call {
+                    func: fun,
+                    args,
+                    destination,
+                    target: Some(target),
+                    unwind,
+                    call_source: if *from_hir_call { CallSource::Normal } else {
+                        CallSource::OverloadedOperator
+                    },
+                    fn_span: *fn_span,
+                })
+            },
+        )
+    }
+
+    fn parse_tail_call(&self, args: &[ExprId]) -> PResult<TerminatorKind<'tcx>> {
+        parse_by_kind!(self, args[0], _, "tail call",
+            ExprKind::Call { fun, args, fn_span, .. } => {
+                let fun = self.parse_operand(*fun)?;
+                let args = args
+                    .iter()
+                    .map(|arg|
+                        Ok(Spanned { node: self.parse_operand(*arg)?, span: self.thir.exprs[*arg].span  } )
+                    )
+                    .collect::<PResult<Box<[_]>>>()?;
+                Ok(TerminatorKind::TailCall {
+                    func: fun,
+                    args,
+                    fn_span: *fn_span,
+                })
+            },
+        )
+    }
+
+    fn parse_rvalue(&self, expr_id: ExprId) -> PResult<Rvalue<'tcx>> {
+        parse_by_kind!(self, expr_id, expr, "rvalue",
+            @call(mir_discriminant, args) => self.parse_place(args[0]).map(Rvalue::Discriminant),
+            @call(mir_cast_transmute, args) => {
+                let source = self.parse_operand(args[0])?;
+                Ok(Rvalue::Cast(CastKind::Transmute, source, expr.ty))
+            },
+            @call(mir_cast_ptr_to_ptr, args) => {
+                let source = self.parse_operand(args[0])?;
+                Ok(Rvalue::Cast(CastKind::PtrToPtr, source, expr.ty))
+            },
+            @call(mir_checked, args) => {
+                parse_by_kind!(self, args[0], _, "binary op",
+                    ExprKind::Binary { op, lhs, rhs } => {
+                        if let Some(op_with_overflow) = op.wrapping_to_overflowing() {
+                            Ok(Rvalue::BinaryOp(
+                                op_with_overflow, Box::new((self.parse_operand(*lhs)?, self.parse_operand(*rhs)?))
+                            ))
+                        } else {
+                            Err(self.expr_error(expr_id, "No WithOverflow form of this operator"))
+                        }
+                    },
+                )
+            },
+            @call(mir_offset, args) => {
+                let ptr = self.parse_operand(args[0])?;
+                let offset = self.parse_operand(args[1])?;
+                Ok(Rvalue::BinaryOp(BinOp::Offset, Box::new((ptr, offset))))
+            },
+            @call(mir_len, args) => Ok(Rvalue::Len(self.parse_place(args[0])?)),
+            @call(mir_ptr_metadata, args) => Ok(Rvalue::UnaryOp(UnOp::PtrMetadata, self.parse_operand(args[0])?)),
+            @call(mir_copy_for_deref, args) => Ok(Rvalue::CopyForDeref(self.parse_place(args[0])?)),
+            ExprKind::Borrow { borrow_kind, arg } => Ok(
+                Rvalue::Ref(self.tcx.lifetimes.re_erased, *borrow_kind, self.parse_place(*arg)?)
+            ),
+            ExprKind::RawBorrow { mutability, arg } => Ok(
+                Rvalue::RawPtr(*mutability, self.parse_place(*arg)?)
+            ),
+            ExprKind::Binary { op, lhs, rhs } =>  Ok(
+                Rvalue::BinaryOp(*op, Box::new((self.parse_operand(*lhs)?, self.parse_operand(*rhs)?)))
+            ),
+            ExprKind::Unary { op, arg } => Ok(
+                Rvalue::UnaryOp(*op, self.parse_operand(*arg)?)
+            ),
+            ExprKind::Repeat { value, count } => Ok(
+                Rvalue::Repeat(self.parse_operand(*value)?, *count)
+            ),
+            ExprKind::Cast { source } => {
+                let source = self.parse_operand(*source)?;
+                let source_ty = source.ty(self.body.local_decls(), self.tcx);
+                let cast_kind = mir_cast_kind(source_ty, expr.ty);
+                Ok(Rvalue::Cast(cast_kind, source, expr.ty))
+            },
+            ExprKind::Tuple { fields } => Ok(
+                Rvalue::Aggregate(
+                    Box::new(AggregateKind::Tuple),
+                    fields.iter().map(|e| self.parse_operand(*e)).collect::<Result<_, _>>()?
+                )
+            ),
+            ExprKind::Array { fields } => {
+                let elem_ty = expr.ty.builtin_index().expect("ty must be an array");
+                Ok(Rvalue::Aggregate(
+                    Box::new(AggregateKind::Array(elem_ty)),
+                    fields.iter().map(|e| self.parse_operand(*e)).collect::<Result<_, _>>()?
+                ))
+            },
+            ExprKind::Adt(box AdtExpr { adt_def, variant_index, args, fields, .. }) => {
+                let is_union = adt_def.is_union();
+                let active_field_index = is_union.then(|| fields[0].name);
+
+                Ok(Rvalue::Aggregate(
+                    Box::new(AggregateKind::Adt(adt_def.did(), *variant_index, args, None, active_field_index)),
+                    fields.iter().map(|f| self.parse_operand(f.expr)).collect::<Result<_, _>>()?
+                ))
+            },
+            _ => self.parse_operand(expr_id).map(Rvalue::Use),
+        )
+    }
+
+    pub(crate) fn parse_operand(&self, expr_id: ExprId) -> PResult<Operand<'tcx>> {
+        parse_by_kind!(self, expr_id, expr, "operand",
+            @call(mir_move, args) => self.parse_place(args[0]).map(Operand::Move),
+            @call(mir_static, args) => self.parse_static(args[0]),
+            @call(mir_static_mut, args) => self.parse_static(args[0]),
+            ExprKind::Literal { .. }
+            | ExprKind::NamedConst { .. }
+            | ExprKind::NonHirLiteral { .. }
+            | ExprKind::ZstLiteral { .. }
+            | ExprKind::ConstParam { .. }
+            | ExprKind::ConstBlock { .. } => {
+                Ok(Operand::Constant(Box::new(
+                    as_constant_inner(expr, |_| None, self.tcx)
+                )))
+            },
+            _ => self.parse_place(expr_id).map(Operand::Copy),
+        )
+    }
+
+    fn parse_place(&self, expr_id: ExprId) -> PResult<Place<'tcx>> {
+        self.parse_place_inner(expr_id).map(|(x, _)| x)
+    }
+
+    fn parse_place_inner(&self, expr_id: ExprId) -> PResult<(Place<'tcx>, PlaceTy<'tcx>)> {
+        let (parent, proj) = parse_by_kind!(self, expr_id, expr, "place",
+            @call(mir_field, args) => {
+                let (parent, ty) = self.parse_place_inner(args[0])?;
+                let field = FieldIdx::from_u32(self.parse_integer_literal(args[1])? as u32);
+                let field_ty = ty.field_ty(self.tcx, field);
+                let proj = PlaceElem::Field(field, field_ty);
+                let place = parent.project_deeper(&[proj], self.tcx);
+                return Ok((place, PlaceTy::from_ty(field_ty)));
+            },
+            @call(mir_variant, args) => {
+                (args[0], PlaceElem::Downcast(
+                    None,
+                    VariantIdx::from_u32(self.parse_integer_literal(args[1])? as u32)
+                ))
+            },
+            ExprKind::Deref { arg } => {
+                parse_by_kind!(self, *arg, _, "does not matter",
+                    @call(mir_make_place, args) => return self.parse_place_inner(args[0]),
+                    _ => (*arg, PlaceElem::Deref),
+                )
+            },
+            ExprKind::Index { lhs, index } => (*lhs, PlaceElem::Index(self.parse_local(*index)?)),
+            ExprKind::Field { lhs, name: field, .. } => (*lhs, PlaceElem::Field(*field, expr.ty)),
+            _ => {
+                let place = self.parse_local(expr_id).map(Place::from)?;
+                return Ok((place, PlaceTy::from_ty(expr.ty)))
+            },
+        );
+        let (parent, ty) = self.parse_place_inner(parent)?;
+        let place = parent.project_deeper(&[proj], self.tcx);
+        let ty = ty.projection_ty(self.tcx, proj);
+        Ok((place, ty))
+    }
+
+    fn parse_local(&self, expr_id: ExprId) -> PResult<Local> {
+        parse_by_kind!(self, expr_id, _, "local",
+            ExprKind::VarRef { id } => Ok(self.local_map[id]),
+        )
+    }
+
+    fn parse_block(&self, expr_id: ExprId) -> PResult<BasicBlock> {
+        parse_by_kind!(self, expr_id, _, "basic block",
+            ExprKind::VarRef { id } => Ok(self.block_map[id]),
+        )
+    }
+
+    fn parse_static(&self, expr_id: ExprId) -> PResult<Operand<'tcx>> {
+        let expr_id = parse_by_kind!(self, expr_id, _, "static",
+            ExprKind::Deref { arg } => *arg,
+        );
+
+        parse_by_kind!(self, expr_id, expr, "static",
+            ExprKind::StaticRef { alloc_id, ty, .. } => {
+                let const_val =
+                    ConstValue::Scalar(Scalar::from_pointer((*alloc_id).into(), &self.tcx));
+                let const_ = Const::Val(const_val, *ty);
+
+                Ok(Operand::Constant(Box::new(ConstOperand {
+                    span: expr.span,
+                    user_ty: None,
+                    const_
+                })))
+            },
+        )
+    }
+
+    fn parse_integer_literal(&self, expr_id: ExprId) -> PResult<u128> {
+        parse_by_kind!(self, expr_id, expr, "constant",
+            ExprKind::Literal { .. }
+            | ExprKind::NamedConst { .. }
+            | ExprKind::NonHirLiteral { .. }
+            | ExprKind::ConstBlock { .. } => Ok({
+                let value = as_constant_inner(expr, |_| None, self.tcx);
+                value.const_.eval_bits(self.tcx, self.typing_env)
+            }),
+        )
+    }
+}

compiler/rustc_mir_build/src/builder/expr/as_constant.rs

@@ -0,0 +1,173 @@
+//! See docs in build/expr/mod.rs
+
+use rustc_abi::Size;
+use rustc_ast as ast;
+use rustc_hir::LangItem;
+use rustc_middle::mir::interpret::{
+    Allocation, CTFE_ALLOC_SALT, LitToConstError, LitToConstInput, Scalar,
+};
+use rustc_middle::mir::*;
+use rustc_middle::thir::*;
+use rustc_middle::ty::{
+    self, CanonicalUserType, CanonicalUserTypeAnnotation, Ty, TyCtxt, UserTypeAnnotationIndex,
+};
+use rustc_middle::{bug, mir, span_bug};
+use tracing::{instrument, trace};
+
+use crate::builder::{Builder, parse_float_into_constval};
+
+impl<'a, 'tcx> Builder<'a, 'tcx> {
+    /// Compile `expr`, yielding a compile-time constant. Assumes that
+    /// `expr` is a valid compile-time constant!
+    pub(crate) fn as_constant(&mut self, expr: &Expr<'tcx>) -> ConstOperand<'tcx> {
+        let this = self;
+        let tcx = this.tcx;
+        let Expr { ty, temp_lifetime: _, span, ref kind } = *expr;
+        match kind {
+            ExprKind::Scope { region_scope: _, lint_level: _, value } => {
+                this.as_constant(&this.thir[*value])
+            }
+            _ => as_constant_inner(
+                expr,
+                |user_ty| {
+                    Some(this.canonical_user_type_annotations.push(CanonicalUserTypeAnnotation {
+                        span,
+                        user_ty: user_ty.clone(),
+                        inferred_ty: ty,
+                    }))
+                },
+                tcx,
+            ),
+        }
+    }
+}
+
+pub(crate) fn as_constant_inner<'tcx>(
+    expr: &Expr<'tcx>,
+    push_cuta: impl FnMut(&Box<CanonicalUserType<'tcx>>) -> Option<UserTypeAnnotationIndex>,
+    tcx: TyCtxt<'tcx>,
+) -> ConstOperand<'tcx> {
+    let Expr { ty, temp_lifetime: _, span, ref kind } = *expr;
+    match *kind {
+        ExprKind::Literal { lit, neg } => {
+            let const_ = match lit_to_mir_constant(tcx, LitToConstInput { lit: &lit.node, ty, neg })
+            {
+                Ok(c) => c,
+                Err(LitToConstError::Reported(guar)) => {
+                    Const::Ty(Ty::new_error(tcx, guar), ty::Const::new_error(tcx, guar))
+                }
+                Err(LitToConstError::TypeError) => {
+                    bug!("encountered type error in `lit_to_mir_constant`")
+                }
+            };
+
+            ConstOperand { span, user_ty: None, const_ }
+        }
+        ExprKind::NonHirLiteral { lit, ref user_ty } => {
+            let user_ty = user_ty.as_ref().and_then(push_cuta);
+
+            let const_ = Const::Val(ConstValue::Scalar(Scalar::Int(lit)), ty);
+
+            ConstOperand { span, user_ty, const_ }
+        }
+        ExprKind::ZstLiteral { ref user_ty } => {
+            let user_ty = user_ty.as_ref().and_then(push_cuta);
+
+            let const_ = Const::Val(ConstValue::ZeroSized, ty);
+
+            ConstOperand { span, user_ty, const_ }
+        }
+        ExprKind::NamedConst { def_id, args, ref user_ty } => {
+            let user_ty = user_ty.as_ref().and_then(push_cuta);
+
+            let uneval = mir::UnevaluatedConst::new(def_id, args);
+            let const_ = Const::Unevaluated(uneval, ty);
+
+            ConstOperand { user_ty, span, const_ }
+        }
+        ExprKind::ConstParam { param, def_id: _ } => {
+            let const_param = ty::Const::new_param(tcx, param);
+            let const_ = Const::Ty(expr.ty, const_param);
+
+            ConstOperand { user_ty: None, span, const_ }
+        }
+        ExprKind::ConstBlock { did: def_id, args } => {
+            let uneval = mir::UnevaluatedConst::new(def_id, args);
+            let const_ = Const::Unevaluated(uneval, ty);
+
+            ConstOperand { user_ty: None, span, const_ }
+        }
+        ExprKind::StaticRef { alloc_id, ty, .. } => {
+            let const_val = ConstValue::Scalar(Scalar::from_pointer(alloc_id.into(), &tcx));
+            let const_ = Const::Val(const_val, ty);
+
+            ConstOperand { span, user_ty: None, const_ }
+        }
+        _ => span_bug!(span, "expression is not a valid constant {:?}", kind),
+    }
+}
+
+#[instrument(skip(tcx, lit_input))]
+fn lit_to_mir_constant<'tcx>(
+    tcx: TyCtxt<'tcx>,
+    lit_input: LitToConstInput<'tcx>,
+) -> Result<Const<'tcx>, LitToConstError> {
+    let LitToConstInput { lit, ty, neg } = lit_input;
+    let trunc = |n| {
+        let width = match tcx.layout_of(ty::TypingEnv::fully_monomorphized().as_query_input(ty)) {
+            Ok(layout) => layout.size,
+            Err(_) => {
+                tcx.dcx().bug(format!("couldn't compute width of literal: {:?}", lit_input.lit))
+            }
+        };
+        trace!("trunc {} with size {} and shift {}", n, width.bits(), 128 - width.bits());
+        let result = width.truncate(n);
+        trace!("trunc result: {}", result);
+        Ok(ConstValue::Scalar(Scalar::from_uint(result, width)))
+    };
+
+    let value = match (lit, ty.kind()) {
+        (ast::LitKind::Str(s, _), ty::Ref(_, inner_ty, _)) if inner_ty.is_str() => {
+            let s = s.as_str();
+            let allocation = Allocation::from_bytes_byte_aligned_immutable(s.as_bytes());
+            let allocation = tcx.mk_const_alloc(allocation);
+            ConstValue::Slice { data: allocation, meta: allocation.inner().size().bytes() }
+        }
+        (ast::LitKind::ByteStr(data, _), ty::Ref(_, inner_ty, _))
+            if matches!(inner_ty.kind(), ty::Slice(_)) =>
+        {
+            let allocation = Allocation::from_bytes_byte_aligned_immutable(data as &[u8]);
+            let allocation = tcx.mk_const_alloc(allocation);
+            ConstValue::Slice { data: allocation, meta: allocation.inner().size().bytes() }
+        }
+        (ast::LitKind::ByteStr(data, _), ty::Ref(_, inner_ty, _)) if inner_ty.is_array() => {
+            let id = tcx.allocate_bytes_dedup(data, CTFE_ALLOC_SALT);
+            ConstValue::Scalar(Scalar::from_pointer(id.into(), &tcx))
+        }
+        (ast::LitKind::CStr(data, _), ty::Ref(_, inner_ty, _)) if matches!(inner_ty.kind(), ty::Adt(def, _) if tcx.is_lang_item(def.did(), LangItem::CStr)) =>
+        {
+            let allocation = Allocation::from_bytes_byte_aligned_immutable(data as &[u8]);
+            let allocation = tcx.mk_const_alloc(allocation);
+            ConstValue::Slice { data: allocation, meta: allocation.inner().size().bytes() }
+        }
+        (ast::LitKind::Byte(n), ty::Uint(ty::UintTy::U8)) => {
+            ConstValue::Scalar(Scalar::from_uint(*n, Size::from_bytes(1)))
+        }
+        (ast::LitKind::Int(n, _), ty::Uint(_)) | (ast::LitKind::Int(n, _), ty::Int(_)) => {
+            trunc(if neg { (n.get() as i128).overflowing_neg().0 as u128 } else { n.get() })?
+        }
+        (ast::LitKind::Float(n, _), ty::Float(fty)) => parse_float_into_constval(*n, *fty, neg)
+            .ok_or_else(|| {
+                LitToConstError::Reported(
+                    tcx.dcx()
+                        .delayed_bug(format!("couldn't parse float literal: {:?}", lit_input.lit)),
+                )
+            })?,
+        (ast::LitKind::Bool(b), ty::Bool) => ConstValue::Scalar(Scalar::from_bool(*b)),
+        (ast::LitKind::Char(c), ty::Char) => ConstValue::Scalar(Scalar::from_char(*c)),
+        (ast::LitKind::Err(guar), _) => return Err(LitToConstError::Reported(*guar)),
+        _ => return Err(LitToConstError::TypeError),
+    };
+
+    Ok(Const::Val(value, ty))
+}

compiler/rustc_mir_build/src/builder/expr/as_operand.rs

@@ -0,0 +1,200 @@
+//! See docs in build/expr/mod.rs
+
+use rustc_middle::mir::*;
+use rustc_middle::thir::*;
+use tracing::{debug, instrument};
+
+use crate::builder::expr::category::Category;
+use crate::builder::{BlockAnd, BlockAndExtension, Builder, NeedsTemporary};
+
+impl<'a, 'tcx> Builder<'a, 'tcx> {
+    /// Construct a temporary lifetime restricted to just the local scope
+    pub(crate) fn local_temp_lifetime(&self) -> TempLifetime {
+        let local_scope = self.local_scope();
+        TempLifetime { temp_lifetime: Some(local_scope), backwards_incompatible: None }
+    }
+
+    /// Returns an operand suitable for use until the end of the current
+    /// scope expression.
+    ///
+    /// The operand returned from this function will *not be valid*
+    /// after the current enclosing `ExprKind::Scope` has ended, so
+    /// please do *not* return it from functions to avoid bad
+    /// miscompiles.
+    pub(crate) fn as_local_operand(
+        &mut self,
+        block: BasicBlock,
+        expr_id: ExprId,
+    ) -> BlockAnd<Operand<'tcx>> {
+        self.as_operand(
+            block,
+            self.local_temp_lifetime(),
+            expr_id,
+            LocalInfo::Boring,
+            NeedsTemporary::Maybe,
+        )
+    }
+
+    /// Returns an operand suitable for use until the end of the current scope expression and
+    /// suitable also to be passed as function arguments.
+    ///
+    /// The operand returned from this function will *not be valid* after an ExprKind::Scope is
+    /// passed, so please do *not* return it from functions to avoid bad miscompiles. Returns an
+    /// operand suitable for use as a call argument. This is almost always equivalent to
+    /// `as_operand`, except for the particular case of passing values of (potentially) unsized
+    /// types "by value" (see details below).
+    ///
+    /// The operand returned from this function will *not be valid*
+    /// after the current enclosing `ExprKind::Scope` has ended, so
+    /// please do *not* return it from functions to avoid bad
+    /// miscompiles.
+    ///
+    /// # Parameters of unsized types
+    ///
+    /// We tweak the handling of parameters of unsized type slightly to avoid the need to create a
+    /// local variable of unsized type. For example, consider this program:
+    ///
+    /// ```
+    /// #![feature(unsized_locals, unsized_fn_params)]
+    /// # use core::fmt::Debug;
+    /// fn foo(p: dyn Debug) { dbg!(p); }
+    ///
+    /// fn bar(box_p: Box<dyn Debug>) { foo(*box_p); }
+    /// ```
+    ///
+    /// Ordinarily, for sized types, we would compile the call `foo(*p)` like so:
+    ///
+    /// ```ignore (illustrative)
+    /// let tmp0 = *box_p; // tmp0 would be the operand returned by this function call
+    /// foo(tmp0)
+    /// ```
+    ///
+    /// But because the parameter to `foo` is of the unsized type `dyn Debug`, and because it is
+    /// being moved the deref of a box, we compile it slightly differently. The temporary `tmp0`
+    /// that we create *stores the entire box*, and the parameter to the call itself will be
+    /// `*tmp0`:
+    ///
+    /// ```ignore (illustrative)
+    /// let tmp0 = box_p; call foo(*tmp0)
+    /// ```
+    ///
+    /// This way, the temporary `tmp0` that we create has type `Box<dyn Debug>`, which is sized.
+    /// The value passed to the call (`*tmp0`) still has the `dyn Debug` type -- but the way that
+    /// calls are compiled means that this parameter will be passed "by reference", meaning that we
+    /// will actually provide a pointer to the interior of the box, and not move the `dyn Debug`
+    /// value to the stack.
+    ///
+    /// See #68304 for more details.
+    pub(crate) fn as_local_call_operand(
+        &mut self,
+        block: BasicBlock,
+        expr: ExprId,
+    ) -> BlockAnd<Operand<'tcx>> {
+        self.as_call_operand(block, self.local_temp_lifetime(), expr)
+    }
+
+    /// Compile `expr` into a value that can be used as an operand.
+    /// If `expr` is a place like `x`, this will introduce a
+    /// temporary `tmp = x`, so that we capture the value of `x` at
+    /// this time.
+    ///
+    /// If we end up needing to create a temporary, then we will use
+    /// `local_info` as its `LocalInfo`, unless `as_temporary`
+    /// has already assigned it a non-`None` `LocalInfo`.
+    /// Normally, you should use `None` for `local_info`
+    ///
+    /// The operand is known to be live until the end of `scope`.
+    ///
+    /// Like `as_local_call_operand`, except that the argument will
+    /// not be valid once `scope` ends.
+    #[instrument(level = "debug", skip(self, scope))]
+    pub(crate) fn as_operand(
+        &mut self,
+        mut block: BasicBlock,
+        scope: TempLifetime,
+        expr_id: ExprId,
+        local_info: LocalInfo<'tcx>,
+        needs_temporary: NeedsTemporary,
+    ) -> BlockAnd<Operand<'tcx>> {
+        let this = self;
+
+        let expr = &this.thir[expr_id];
+        if let ExprKind::Scope { region_scope, lint_level, value } = expr.kind {
+            let source_info = this.source_info(expr.span);
+            let region_scope = (region_scope, source_info);
+            return this.in_scope(region_scope, lint_level, |this| {
+                this.as_operand(block, scope, value, local_info, needs_temporary)
+            });
+        }
+
+        let category = Category::of(&expr.kind).unwrap();
+        debug!(?category, ?expr.kind);
+        match category {
+            Category::Constant
+                if matches!(needs_temporary, NeedsTemporary::No)
+                    || !expr.ty.needs_drop(this.tcx, this.typing_env()) =>
+            {
+                let constant = this.as_constant(expr);
+                block.and(Operand::Constant(Box::new(constant)))
+            }
+            Category::Constant | Category::Place | Category::Rvalue(..) => {
+                let operand = unpack!(block = this.as_temp(block, scope, expr_id, Mutability::Mut));
+                // Overwrite temp local info if we have something more interesting to record.
+                if !matches!(local_info, LocalInfo::Boring) {
+                    let decl_info =
+                        this.local_decls[operand].local_info.as_mut().assert_crate_local();
+                    if let LocalInfo::Boring | LocalInfo::BlockTailTemp(_) = **decl_info {
+                        **decl_info = local_info;
+                    }
+                }
+                block.and(Operand::Move(Place::from(operand)))
+            }
+        }
+    }
+
+    pub(crate) fn as_call_operand(
+        &mut self,
+        mut block: BasicBlock,
+        scope: TempLifetime,
+        expr_id: ExprId,
+    ) -> BlockAnd<Operand<'tcx>> {
+        let this = self;
+        let expr = &this.thir[expr_id];
+        debug!("as_call_operand(block={:?}, expr={:?})", block, expr);
+
+        if let ExprKind::Scope { region_scope, lint_level, value } = expr.kind {
+            let source_info = this.source_info(expr.span);
+            let region_scope = (region_scope, source_info);
+            return this.in_scope(region_scope, lint_level, |this| {
+                this.as_call_operand(block, scope, value)
+            });
+        }
+
+        let tcx = this.tcx;
+
+        if tcx.features().unsized_fn_params() {
+            let ty = expr.ty;
+            if !ty.is_sized(tcx, this.typing_env()) {
+                // !sized means !copy, so this is an unsized move
+                assert!(!tcx.type_is_copy_modulo_regions(this.typing_env(), ty));
+
+                // As described above, detect the case where we are passing a value of unsized
+                // type, and that value is coming from the deref of a box.
+                if let ExprKind::Deref { arg } = expr.kind {
+                    // Generate let tmp0 = arg0
+                    let operand = unpack!(block = this.as_temp(block, scope, arg, Mutability::Mut));
+
+                    // Return the operand *tmp0 to be used as the call argument
+                    let place = Place {
+                        local: operand,
+                        projection: tcx.mk_place_elems(&[PlaceElem::Deref]),
+                    };
+
+                    return block.and(Operand::Move(place));
+                }
+            }
+        }
+
+        this.as_operand(block, scope, expr_id, LocalInfo::Boring, NeedsTemporary::Maybe)
+    }
+}

compiler/rustc_mir_build/src/builder/expr/as_place.rs

@@ -0,0 +1,832 @@
+//! See docs in build/expr/mod.rs
+
+use std::assert_matches::assert_matches;
+use std::iter;
+
+use rustc_abi::{FIRST_VARIANT, FieldIdx, VariantIdx};
+use rustc_hir::def_id::LocalDefId;
+use rustc_middle::hir::place::{Projection as HirProjection, ProjectionKind as HirProjectionKind};
+use rustc_middle::mir::AssertKind::BoundsCheck;
+use rustc_middle::mir::*;
+use rustc_middle::thir::*;
+use rustc_middle::ty::{self, AdtDef, CanonicalUserTypeAnnotation, Ty, Variance};
+use rustc_middle::{bug, span_bug};
+use rustc_span::{DesugaringKind, Span};
+use tracing::{debug, instrument, trace};
+
+use crate::builder::ForGuard::{OutsideGuard, RefWithinGuard};
+use crate::builder::expr::category::Category;
+use crate::builder::{BlockAnd, BlockAndExtension, Builder, Capture, CaptureMap};
+
+/// The "outermost" place that holds this value.
+#[derive(Copy, Clone, Debug, PartialEq)]
+pub(crate) enum PlaceBase {
+    /// Denotes the start of a `Place`.
+    Local(Local),
+
+    /// When building place for an expression within a closure, the place might start off a
+    /// captured path. When `capture_disjoint_fields` is enabled, we might not know the capture
+    /// index (within the desugared closure) of the captured path until most of the projections
+    /// are applied. We use `PlaceBase::Upvar` to keep track of the root variable off of which the
+    /// captured path starts, the closure the capture belongs to and the trait the closure
+    /// implements.
+    ///
+    /// Once we have figured out the capture index, we can convert the place builder to start from
+    /// `PlaceBase::Local`.
+    ///
+    /// Consider the following example
+    /// ```rust
+    /// let t = (((10, 10), 10), 10);
+    ///
+    /// let c = || {
+    ///     println!("{}", t.0.0.0);
+    /// };
+    /// ```
+    /// Here the THIR expression for `t.0.0.0` will be something like
+    ///
+    /// ```ignore (illustrative)
+    /// * Field(0)
+    ///     * Field(0)
+    ///         * Field(0)
+    ///             * UpvarRef(t)
+    /// ```
+    ///
+    /// When `capture_disjoint_fields` is enabled, `t.0.0.0` is captured and we won't be able to
+    /// figure out that it is captured until all the `Field` projections are applied.
+    Upvar {
+        /// HirId of the upvar
+        var_hir_id: LocalVarId,
+        /// DefId of the closure
+        closure_def_id: LocalDefId,
+    },
+}
+
+/// `PlaceBuilder` is used to create places during MIR construction. It allows you to "build up" a
+/// place by pushing more and more projections onto the end, and then convert the final set into a
+/// place using the `to_place` method.
+///
+/// This is used internally when building a place for an expression like `a.b.c`. The fields `b`
+/// and `c` can be progressively pushed onto the place builder that is created when converting `a`.
+#[derive(Clone, Debug, PartialEq)]
+pub(in crate::builder) struct PlaceBuilder<'tcx> {
+    base: PlaceBase,
+    projection: Vec<PlaceElem<'tcx>>,
+}
+
+/// Given a list of MIR projections, convert them to list of HIR ProjectionKind.
+/// The projections are truncated to represent a path that might be captured by a
+/// closure/coroutine. This implies the vector returned from this function doesn't contain
+/// ProjectionElems `Downcast`, `ConstantIndex`, `Index`, or `Subslice` because those will never be
+/// part of a path that is captured by a closure. We stop applying projections once we see the first
+/// projection that isn't captured by a closure.
+fn convert_to_hir_projections_and_truncate_for_capture(
+    mir_projections: &[PlaceElem<'_>],
+) -> Vec<HirProjectionKind> {
+    let mut hir_projections = Vec::new();
+    let mut variant = None;
+
+    for mir_projection in mir_projections {
+        let hir_projection = match mir_projection {
+            ProjectionElem::Deref => HirProjectionKind::Deref,
+            ProjectionElem::Field(field, _) => {
+                let variant = variant.unwrap_or(FIRST_VARIANT);
+                HirProjectionKind::Field(*field, variant)
+            }
+            ProjectionElem::Downcast(.., idx) => {
+                // We don't expect to see multi-variant enums here, as earlier
+                // phases will have truncated them already. However, there can
+                // still be downcasts, thanks to single-variant enums.
+                // We keep track of VariantIdx so we can use this information
+                // if the next ProjectionElem is a Field.
+                variant = Some(*idx);
+                continue;
+            }
+            // These do not affect anything, they just make sure we know the right type.
+            ProjectionElem::OpaqueCast(_) | ProjectionElem::Subtype(..) => continue,
+            ProjectionElem::Index(..)
+            | ProjectionElem::ConstantIndex { .. }
+            | ProjectionElem::Subslice { .. } => {
+                // We don't capture array-access projections.
+                // We can stop here as arrays are captured completely.
+                break;
+            }
+        };
+        variant = None;
+        hir_projections.push(hir_projection);
+    }
+
+    hir_projections
+}
+
+/// Return true if the `proj_possible_ancestor` represents an ancestor path
+/// to `proj_capture` or `proj_possible_ancestor` is same as `proj_capture`,
+/// assuming they both start off of the same root variable.
+///
+/// **Note:** It's the caller's responsibility to ensure that both lists of projections
+///           start off of the same root variable.
+///
+/// Eg: 1. `foo.x` which is represented using `projections=[Field(x)]` is an ancestor of
+///        `foo.x.y` which is represented using `projections=[Field(x), Field(y)]`.
+///        Note both `foo.x` and `foo.x.y` start off of the same root variable `foo`.
+///     2. Since we only look at the projections here function will return `bar.x` as a valid
+///        ancestor of `foo.x.y`. It's the caller's responsibility to ensure that both projections
+///        list are being applied to the same root variable.
+fn is_ancestor_or_same_capture(
+    proj_possible_ancestor: &[HirProjectionKind],
+    proj_capture: &[HirProjectionKind],
+) -> bool {
+    // We want to make sure `is_ancestor_or_same_capture("x.0.0", "x.0")` to return false.
+    // Therefore we can't just check if all projections are same in the zipped iterator below.
+    if proj_possible_ancestor.len() > proj_capture.len() {
+        return false;
+    }
+
+    iter::zip(proj_possible_ancestor, proj_capture).all(|(a, b)| a == b)
+}
+
+/// Given a closure, returns the index of a capture within the desugared closure struct and the
+/// `ty::CapturedPlace` which is the ancestor of the Place represented using the `var_hir_id`
+/// and `projection`.
+///
+/// Note there will be at most one ancestor for any given Place.
+///
+/// Returns None, when the ancestor is not found.
+fn find_capture_matching_projections<'a, 'tcx>(
+    upvars: &'a CaptureMap<'tcx>,
+    var_hir_id: LocalVarId,
+    projections: &[PlaceElem<'tcx>],
+) -> Option<(usize, &'a Capture<'tcx>)> {
+    let hir_projections = convert_to_hir_projections_and_truncate_for_capture(projections);
+
+    upvars.get_by_key_enumerated(var_hir_id.0).find(|(_, capture)| {
+        let possible_ancestor_proj_kinds: Vec<_> =
+            capture.captured_place.place.projections.iter().map(|proj| proj.kind).collect();
+        is_ancestor_or_same_capture(&possible_ancestor_proj_kinds, &hir_projections)
+    })
+}
+
+/// Takes an upvar place and tries to resolve it into a `PlaceBuilder`
+/// with `PlaceBase::Local`
+#[instrument(level = "trace", skip(cx), ret)]
+fn to_upvars_resolved_place_builder<'tcx>(
+    cx: &Builder<'_, 'tcx>,
+    var_hir_id: LocalVarId,
+    closure_def_id: LocalDefId,
+    projection: &[PlaceElem<'tcx>],
+) -> Option<PlaceBuilder<'tcx>> {
+    let Some((capture_index, capture)) =
+        find_capture_matching_projections(&cx.upvars, var_hir_id, projection)
+    else {
+        let closure_span = cx.tcx.def_span(closure_def_id);
+        if !enable_precise_capture(closure_span) {
+            bug!(
+                "No associated capture found for {:?}[{:#?}] even though \
+                    capture_disjoint_fields isn't enabled",
+                var_hir_id,
+                projection
+            )
+        } else {
+            debug!("No associated capture found for {:?}[{:#?}]", var_hir_id, projection,);
+        }
+        return None;
+    };
+
+    // Access the capture by accessing the field within the Closure struct.
+    let capture_info = &cx.upvars[capture_index];
+
+    let mut upvar_resolved_place_builder = PlaceBuilder::from(capture_info.use_place);
+
+    // We used some of the projections to build the capture itself,
+    // now we apply the remaining to the upvar resolved place.
+    trace!(?capture.captured_place, ?projection);
+    let remaining_projections = strip_prefix(
+        capture.captured_place.place.base_ty,
+        projection,
+        &capture.captured_place.place.projections,
+    );
+    upvar_resolved_place_builder.projection.extend(remaining_projections);
+
+    Some(upvar_resolved_place_builder)
+}
+
+/// Returns projections remaining after stripping an initial prefix of HIR
+/// projections.
+///
+/// Supports only HIR projection kinds that represent a path that might be
+/// captured by a closure or a coroutine, i.e., an `Index` or a `Subslice`
+/// projection kinds are unsupported.
+fn strip_prefix<'a, 'tcx>(
+    mut base_ty: Ty<'tcx>,
+    projections: &'a [PlaceElem<'tcx>],
+    prefix_projections: &[HirProjection<'tcx>],
+) -> impl Iterator<Item = PlaceElem<'tcx>> + 'a {
+    let mut iter = projections
+        .iter()
+        .copied()
+        // Filter out opaque casts, they are unnecessary in the prefix.
+        .filter(|elem| !matches!(elem, ProjectionElem::OpaqueCast(..)));
+    for projection in prefix_projections {
+        match projection.kind {
+            HirProjectionKind::Deref => {
+                assert_matches!(iter.next(), Some(ProjectionElem::Deref));
+            }
+            HirProjectionKind::Field(..) => {
+                if base_ty.is_enum() {
+                    assert_matches!(iter.next(), Some(ProjectionElem::Downcast(..)));
+                }
+                assert_matches!(iter.next(), Some(ProjectionElem::Field(..)));
+            }
+            HirProjectionKind::OpaqueCast => {
+                assert_matches!(iter.next(), Some(ProjectionElem::OpaqueCast(..)));
+            }
+            HirProjectionKind::Index | HirProjectionKind::Subslice => {
+                bug!("unexpected projection kind: {:?}", projection);
+            }
+        }
+        base_ty = projection.ty;
+    }
+    iter
+}
+
+impl<'tcx> PlaceBuilder<'tcx> {
+    pub(in crate::builder) fn to_place(&self, cx: &Builder<'_, 'tcx>) -> Place<'tcx> {
+        self.try_to_place(cx).unwrap_or_else(|| match self.base {
+            PlaceBase::Local(local) => span_bug!(
+                cx.local_decls[local].source_info.span,
+                "could not resolve local: {local:#?} + {:?}",
+                self.projection
+            ),
+            PlaceBase::Upvar { var_hir_id, closure_def_id: _ } => span_bug!(
+                cx.tcx.hir().span(var_hir_id.0),
+                "could not resolve upvar: {var_hir_id:?} + {:?}",
+                self.projection
+            ),
+        })
+    }
+
+    /// Creates a `Place` or returns `None` if an upvar cannot be resolved
+    pub(in crate::builder) fn try_to_place(&self, cx: &Builder<'_, 'tcx>) -> Option<Place<'tcx>> {
+        let resolved = self.resolve_upvar(cx);
+        let builder = resolved.as_ref().unwrap_or(self);
+        let PlaceBase::Local(local) = builder.base else { return None };
+        let projection = cx.tcx.mk_place_elems(&builder.projection);
+        Some(Place { local, projection })
+    }
+
+    /// Attempts to resolve the `PlaceBuilder`.
+    /// Returns `None` if this is not an upvar.
+    ///
+    /// Upvars resolve may fail for a `PlaceBuilder` when attempting to
+    /// resolve a disjoint field whose root variable is not captured
+    /// (destructured assignments) or when attempting to resolve a root
+    /// variable (discriminant matching with only wildcard arm) that is
+    /// not captured. This can happen because the final mir that will be
+    /// generated doesn't require a read for this place. Failures will only
+    /// happen inside closures.
+    pub(in crate::builder) fn resolve_upvar(
+        &self,
+        cx: &Builder<'_, 'tcx>,
+    ) -> Option<PlaceBuilder<'tcx>> {
+        let PlaceBase::Upvar { var_hir_id, closure_def_id } = self.base else {
+            return None;
+        };
+        to_upvars_resolved_place_builder(cx, var_hir_id, closure_def_id, &self.projection)
+    }
+
+    pub(crate) fn base(&self) -> PlaceBase {
+        self.base
+    }
+
+    pub(crate) fn projection(&self) -> &[PlaceElem<'tcx>] {
+        &self.projection
+    }
+
+    pub(crate) fn field(self, f: FieldIdx, ty: Ty<'tcx>) -> Self {
+        self.project(PlaceElem::Field(f, ty))
+    }
+
+    pub(crate) fn deref(self) -> Self {
+        self.project(PlaceElem::Deref)
+    }
+
+    pub(crate) fn downcast(self, adt_def: AdtDef<'tcx>, variant_index: VariantIdx) -> Self {
+        self.project(PlaceElem::Downcast(Some(adt_def.variant(variant_index).name), variant_index))
+    }
+
+    fn index(self, index: Local) -> Self {
+        self.project(PlaceElem::Index(index))
+    }
+
+    pub(crate) fn project(mut self, elem: PlaceElem<'tcx>) -> Self {
+        self.projection.push(elem);
+        self
+    }
+
+    /// Same as `.clone().project(..)` but more efficient
+    pub(crate) fn clone_project(&self, elem: PlaceElem<'tcx>) -> Self {
+        Self {
+            base: self.base,
+            projection: Vec::from_iter(self.projection.iter().copied().chain([elem])),
+        }
+    }
+}
+
+impl<'tcx> From<Local> for PlaceBuilder<'tcx> {
+    fn from(local: Local) -> Self {
+        Self { base: PlaceBase::Local(local), projection: Vec::new() }
+    }
+}
+
+impl<'tcx> From<PlaceBase> for PlaceBuilder<'tcx> {
+    fn from(base: PlaceBase) -> Self {
+        Self { base, projection: Vec::new() }
+    }
+}
+
+impl<'tcx> From<Place<'tcx>> for PlaceBuilder<'tcx> {
+    fn from(p: Place<'tcx>) -> Self {
+        Self { base: PlaceBase::Local(p.local), projection: p.projection.to_vec() }
+    }
+}
+
+impl<'a, 'tcx> Builder<'a, 'tcx> {
+    /// Compile `expr`, yielding a place that we can move from etc.
+    ///
+    /// WARNING: Any user code might:
+    /// * Invalidate any slice bounds checks performed.
+    /// * Change the address that this `Place` refers to.
+    /// * Modify the memory that this place refers to.
+    /// * Invalidate the memory that this place refers to, this will be caught
+    ///   by borrow checking.
+    ///
+    /// Extra care is needed if any user code is allowed to run between calling
+    /// this method and using it, as is the case for `match` and index
+    /// expressions.
+    pub(crate) fn as_place(
+        &mut self,
+        mut block: BasicBlock,
+        expr_id: ExprId,
+    ) -> BlockAnd<Place<'tcx>> {
+        let place_builder = unpack!(block = self.as_place_builder(block, expr_id));
+        block.and(place_builder.to_place(self))
+    }
+
+    /// This is used when constructing a compound `Place`, so that we can avoid creating
+    /// intermediate `Place` values until we know the full set of projections.
+    pub(crate) fn as_place_builder(
+        &mut self,
+        block: BasicBlock,
+        expr_id: ExprId,
+    ) -> BlockAnd<PlaceBuilder<'tcx>> {
+        self.expr_as_place(block, expr_id, Mutability::Mut, None)
+    }
+
+    /// Compile `expr`, yielding a place that we can move from etc.
+    /// Mutability note: The caller of this method promises only to read from the resulting
+    /// place. The place itself may or may not be mutable:
+    /// * If this expr is a place expr like a.b, then we will return that place.
+    /// * Otherwise, a temporary is created: in that event, it will be an immutable temporary.
+    pub(crate) fn as_read_only_place(
+        &mut self,
+        mut block: BasicBlock,
+        expr_id: ExprId,
+    ) -> BlockAnd<Place<'tcx>> {
+        let place_builder = unpack!(block = self.as_read_only_place_builder(block, expr_id));
+        block.and(place_builder.to_place(self))
+    }
+
+    /// This is used when constructing a compound `Place`, so that we can avoid creating
+    /// intermediate `Place` values until we know the full set of projections.
+    /// Mutability note: The caller of this method promises only to read from the resulting
+    /// place. The place itself may or may not be mutable:
+    /// * If this expr is a place expr like a.b, then we will return that place.
+    /// * Otherwise, a temporary is created: in that event, it will be an immutable temporary.
+    fn as_read_only_place_builder(
+        &mut self,
+        block: BasicBlock,
+        expr_id: ExprId,
+    ) -> BlockAnd<PlaceBuilder<'tcx>> {
+        self.expr_as_place(block, expr_id, Mutability::Not, None)
+    }
+
+    fn expr_as_place(
+        &mut self,
+        mut block: BasicBlock,
+        expr_id: ExprId,
+        mutability: Mutability,
+        fake_borrow_temps: Option<&mut Vec<Local>>,
+    ) -> BlockAnd<PlaceBuilder<'tcx>> {
+        let expr = &self.thir[expr_id];
+        debug!("expr_as_place(block={:?}, expr={:?}, mutability={:?})", block, expr, mutability);
+
+        let this = self;
+        let expr_span = expr.span;
+        let source_info = this.source_info(expr_span);
+        match expr.kind {
+            ExprKind::Scope { region_scope, lint_level, value } => {
+                this.in_scope((region_scope, source_info), lint_level, |this| {
+                    this.expr_as_place(block, value, mutability, fake_borrow_temps)
+                })
+            }
+            ExprKind::Field { lhs, variant_index, name } => {
+                let lhs_expr = &this.thir[lhs];
+                let mut place_builder =
+                    unpack!(block = this.expr_as_place(block, lhs, mutability, fake_borrow_temps,));
+                if let ty::Adt(adt_def, _) = lhs_expr.ty.kind() {
+                    if adt_def.is_enum() {
+                        place_builder = place_builder.downcast(*adt_def, variant_index);
+                    }
+                }
+                block.and(place_builder.field(name, expr.ty))
+            }
+            ExprKind::Deref { arg } => {
+                let place_builder =
+                    unpack!(block = this.expr_as_place(block, arg, mutability, fake_borrow_temps,));
+                block.and(place_builder.deref())
+            }
+            ExprKind::Index { lhs, index } => this.lower_index_expression(
+                block,
+                lhs,
+                index,
+                mutability,
+                fake_borrow_temps,
+                expr.temp_lifetime,
+                expr_span,
+                source_info,
+            ),
+            ExprKind::UpvarRef { closure_def_id, var_hir_id } => {
+                this.lower_captured_upvar(block, closure_def_id.expect_local(), var_hir_id)
+            }
+
+            ExprKind::VarRef { id } => {
+                let place_builder = if this.is_bound_var_in_guard(id) {
+                    let index = this.var_local_id(id, RefWithinGuard);
+                    PlaceBuilder::from(index).deref()
+                } else {
+                    let index = this.var_local_id(id, OutsideGuard);
+                    PlaceBuilder::from(index)
+                };
+                block.and(place_builder)
+            }
+
+            ExprKind::PlaceTypeAscription { source, ref user_ty, user_ty_span } => {
+                let place_builder = unpack!(
+                    block = this.expr_as_place(block, source, mutability, fake_borrow_temps,)
+                );
+                if let Some(user_ty) = user_ty {
+                    let ty_source_info = this.source_info(user_ty_span);
+                    let annotation_index =
+                        this.canonical_user_type_annotations.push(CanonicalUserTypeAnnotation {
+                            span: user_ty_span,
+                            user_ty: user_ty.clone(),
+                            inferred_ty: expr.ty,
+                        });
+
+                    let place = place_builder.to_place(this);
+                    this.cfg.push(block, Statement {
+                        source_info: ty_source_info,
+                        kind: StatementKind::AscribeUserType(
+                            Box::new((place, UserTypeProjection {
+                                base: annotation_index,
+                                projs: vec![],
+                            })),
+                            Variance::Invariant,
+                        ),
+                    });
+                }
+                block.and(place_builder)
+            }
+            ExprKind::ValueTypeAscription { source, ref user_ty, user_ty_span } => {
+                let source_expr = &this.thir[source];
+                let temp = unpack!(
+                    block = this.as_temp(block, source_expr.temp_lifetime, source, mutability)
+                );
+                if let Some(user_ty) = user_ty {
+                    let ty_source_info = this.source_info(user_ty_span);
+                    let annotation_index =
+                        this.canonical_user_type_annotations.push(CanonicalUserTypeAnnotation {
+                            span: user_ty_span,
+                            user_ty: user_ty.clone(),
+                            inferred_ty: expr.ty,
+                        });
+                    this.cfg.push(block, Statement {
+                        source_info: ty_source_info,
+                        kind: StatementKind::AscribeUserType(
+                            Box::new((Place::from(temp), UserTypeProjection {
+                                base: annotation_index,
+                                projs: vec![],
+                            })),
+                            Variance::Invariant,
+                        ),
+                    });
+                }
+                block.and(PlaceBuilder::from(temp))
+            }
+
+            ExprKind::Array { .. }
+            | ExprKind::Tuple { .. }
+            | ExprKind::Adt { .. }
+            | ExprKind::Closure { .. }
+            | ExprKind::Unary { .. }
+            | ExprKind::Binary { .. }
+            | ExprKind::LogicalOp { .. }
+            | ExprKind::Box { .. }
+            | ExprKind::Cast { .. }
+            | ExprKind::Use { .. }
+            | ExprKind::NeverToAny { .. }
+            | ExprKind::PointerCoercion { .. }
+            | ExprKind::Repeat { .. }
+            | ExprKind::Borrow { .. }
+            | ExprKind::RawBorrow { .. }
+            | ExprKind::Match { .. }
+            | ExprKind::If { .. }
+            | ExprKind::Loop { .. }
+            | ExprKind::Block { .. }
+            | ExprKind::Let { .. }
+            | ExprKind::Assign { .. }
+            | ExprKind::AssignOp { .. }
+            | ExprKind::Break { .. }
+            | ExprKind::Continue { .. }
+            | ExprKind::Return { .. }
+            | ExprKind::Become { .. }
+            | ExprKind::Literal { .. }
+            | ExprKind::NamedConst { .. }
+            | ExprKind::NonHirLiteral { .. }
+            | ExprKind::ZstLiteral { .. }
+            | ExprKind::ConstParam { .. }
+            | ExprKind::ConstBlock { .. }
+            | ExprKind::StaticRef { .. }
+            | ExprKind::InlineAsm { .. }
+            | ExprKind::OffsetOf { .. }
+            | ExprKind::Yield { .. }
+            | ExprKind::ThreadLocalRef(_)
+            | ExprKind::Call { .. } => {
+                // these are not places, so we need to make a temporary.
+                debug_assert!(!matches!(Category::of(&expr.kind), Some(Category::Place)));
+                let temp =
+                    unpack!(block = this.as_temp(block, expr.temp_lifetime, expr_id, mutability));
+                block.and(PlaceBuilder::from(temp))
+            }
+        }
+    }
+
+    /// Lower a captured upvar. Note we might not know the actual capture index,
+    /// so we create a place starting from `PlaceBase::Upvar`, which will be resolved
+    /// once all projections that allow us to identify a capture have been applied.
+    fn lower_captured_upvar(
+        &mut self,
+        block: BasicBlock,
+        closure_def_id: LocalDefId,
+        var_hir_id: LocalVarId,
+    ) -> BlockAnd<PlaceBuilder<'tcx>> {
+        block.and(PlaceBuilder::from(PlaceBase::Upvar { var_hir_id, closure_def_id }))
+    }
+
+    /// Lower an index expression
+    ///
+    /// This has two complications;
+    ///
+    /// * We need to do a bounds check.
+    /// * We need to ensure that the bounds check can't be invalidated using an
+    ///   expression like `x[1][{x = y; 2}]`. We use fake borrows here to ensure
+    ///   that this is the case.
+    fn lower_index_expression(
+        &mut self,
+        mut block: BasicBlock,
+        base: ExprId,
+        index: ExprId,
+        mutability: Mutability,
+        fake_borrow_temps: Option<&mut Vec<Local>>,
+        temp_lifetime: TempLifetime,
+        expr_span: Span,
+        source_info: SourceInfo,
+    ) -> BlockAnd<PlaceBuilder<'tcx>> {
+        let base_fake_borrow_temps = &mut Vec::new();
+        let is_outermost_index = fake_borrow_temps.is_none();
+        let fake_borrow_temps = fake_borrow_temps.unwrap_or(base_fake_borrow_temps);
+
+        let base_place =
+            unpack!(block = self.expr_as_place(block, base, mutability, Some(fake_borrow_temps),));
+
+        // Making this a *fresh* temporary means we do not have to worry about
+        // the index changing later: Nothing will ever change this temporary.
+        // The "retagging" transformation (for Stacked Borrows) relies on this.
+        let idx = unpack!(block = self.as_temp(block, temp_lifetime, index, Mutability::Not));
+
+        block = self.bounds_check(block, &base_place, idx, expr_span, source_info);
+
+        if is_outermost_index {
+            self.read_fake_borrows(block, fake_borrow_temps, source_info)
+        } else {
+            self.add_fake_borrows_of_base(
+                base_place.to_place(self),
+                block,
+                fake_borrow_temps,
+                expr_span,
+                source_info,
+            );
+        }
+
+        block.and(base_place.index(idx))
+    }
+
+    /// Given a place that's either an array or a slice, returns an operand
+    /// with the length of the array/slice.
+    ///
+    /// For arrays it'll be `Operand::Constant` with the actual length;
+    /// For slices it'll be `Operand::Move` of a local using `PtrMetadata`.
+    fn len_of_slice_or_array(
+        &mut self,
+        block: BasicBlock,
+        place: Place<'tcx>,
+        span: Span,
+        source_info: SourceInfo,
+    ) -> Operand<'tcx> {
+        let place_ty = place.ty(&self.local_decls, self.tcx).ty;
+        let usize_ty = self.tcx.types.usize;
+
+        match place_ty.kind() {
+            ty::Array(_elem_ty, len_const) => {
+                let ty_const = if let Some((_, len_ty)) = len_const.try_to_valtree()
+                    && len_ty != self.tcx.types.usize
+                {
+                    // Bad const generics can give us a constant from the type that's
+                    // not actually a `usize`, so in that case give an error instead.
+                    // FIXME: It'd be nice if the type checker made sure this wasn't
+                    // possible, instead.
+                    let err = self.tcx.dcx().span_delayed_bug(
+                        span,
+                        format!(
+                            "Array length should have already been a type error, as it's {len_ty:?}"
+                        ),
+                    );
+                    ty::Const::new_error(self.tcx, err)
+                } else {
+                    // We know how long an array is, so just use that as a constant
+                    // directly -- no locals needed. We do need one statement so
+                    // that borrow- and initialization-checking consider it used,
+                    // though. FIXME: Do we really *need* to count this as a use?
+                    // Could partial array tracking work off something else instead?
+                    self.cfg.push_fake_read(block, source_info, FakeReadCause::ForIndex, place);
+                    *len_const
+                };
+
+                let const_ = Const::from_ty_const(ty_const, usize_ty, self.tcx);
+                Operand::Constant(Box::new(ConstOperand { span, user_ty: None, const_ }))
+            }
+            ty::Slice(_elem_ty) => {
+                let ptr_or_ref = if let [PlaceElem::Deref] = place.projection[..]
+                    && let local_ty = self.local_decls[place.local].ty
+                    && local_ty.is_trivially_pure_clone_copy()
+                {
+                    // It's extremely common that we have something that can be
+                    // directly passed to `PtrMetadata`, so avoid an unnecessary
+                    // temporary and statement in those cases. Note that we can
+                    // only do that for `Copy` types -- not `&mut [_]` -- because
+                    // the MIR we're building here needs to pass NLL later.
+                    Operand::Copy(Place::from(place.local))
+                } else {
+                    let len_span = self.tcx.with_stable_hashing_context(|hcx| {
+                        let span = source_info.span;
+                        span.mark_with_reason(
+                            None,
+                            DesugaringKind::IndexBoundsCheckReborrow,
+                            span.edition(),
+                            hcx,
+                        )
+                    });
+                    let ptr_ty = Ty::new_imm_ptr(self.tcx, place_ty);
+                    let slice_ptr = self.temp(ptr_ty, span);
+                    self.cfg.push_assign(
+                        block,
+                        SourceInfo { span: len_span, ..source_info },
+                        slice_ptr,
+                        Rvalue::RawPtr(Mutability::Not, place),
+                    );
+                    Operand::Move(slice_ptr)
+                };
+
+                let len = self.temp(usize_ty, span);
+                self.cfg.push_assign(
+                    block,
+                    source_info,
+                    len,
+                    Rvalue::UnaryOp(UnOp::PtrMetadata, ptr_or_ref),
+                );
+
+                Operand::Move(len)
+            }
+            _ => {
+                span_bug!(span, "len called on place of type {place_ty:?}")
+            }
+        }
+    }
+
+    fn bounds_check(
+        &mut self,
+        block: BasicBlock,
+        slice: &PlaceBuilder<'tcx>,
+        index: Local,
+        expr_span: Span,
+        source_info: SourceInfo,
+    ) -> BasicBlock {
+        let slice = slice.to_place(self);
+
+        // len = len(slice)
+        let len = self.len_of_slice_or_array(block, slice, expr_span, source_info);
+
+        // lt = idx < len
+        let bool_ty = self.tcx.types.bool;
+        let lt = self.temp(bool_ty, expr_span);
+        self.cfg.push_assign(
+            block,
+            source_info,
+            lt,
+            Rvalue::BinaryOp(
+                BinOp::Lt,
+                Box::new((Operand::Copy(Place::from(index)), len.to_copy())),
+            ),
+        );
+        let msg = BoundsCheck { len, index: Operand::Copy(Place::from(index)) };
+
+        // assert!(lt, "...")
+        self.assert(block, Operand::Move(lt), true, msg, expr_span)
+    }
+
+    fn add_fake_borrows_of_base(
+        &mut self,
+        base_place: Place<'tcx>,
+        block: BasicBlock,
+        fake_borrow_temps: &mut Vec<Local>,
+        expr_span: Span,
+        source_info: SourceInfo,
+    ) {
+        let tcx = self.tcx;
+
+        let place_ty = base_place.ty(&self.local_decls, tcx);
+        if let ty::Slice(_) = place_ty.ty.kind() {
+            // We need to create fake borrows to ensure that the bounds
+            // check that we just did stays valid. Since we can't assign to
+            // unsized values, we only need to ensure that none of the
+            // pointers in the base place are modified.
+            for (base_place, elem) in base_place.iter_projections().rev() {
+                match elem {
+                    ProjectionElem::Deref => {
+                        let fake_borrow_deref_ty = base_place.ty(&self.local_decls, tcx).ty;
+                        let fake_borrow_ty =
+                            Ty::new_imm_ref(tcx, tcx.lifetimes.re_erased, fake_borrow_deref_ty);
+                        let fake_borrow_temp =
+                            self.local_decls.push(LocalDecl::new(fake_borrow_ty, expr_span));
+                        let projection = tcx.mk_place_elems(base_place.projection);
+                        self.cfg.push_assign(
+                            block,
+                            source_info,
+                            fake_borrow_temp.into(),
+                            Rvalue::Ref(
+                                tcx.lifetimes.re_erased,
+                                BorrowKind::Fake(FakeBorrowKind::Shallow),
+                                Place { local: base_place.local, projection },
+                            ),
+                        );
+                        fake_borrow_temps.push(fake_borrow_temp);
+                    }
+                    ProjectionElem::Index(_) => {
+                        let index_ty = base_place.ty(&self.local_decls, tcx);
+                        match index_ty.ty.kind() {
+                            // The previous index expression has already
+                            // done any index expressions needed here.
+                            ty::Slice(_) => break,
+                            ty::Array(..) => (),
+                            _ => bug!("unexpected index base"),
+                        }
+                    }
+                    ProjectionElem::Field(..)
+                    | ProjectionElem::Downcast(..)
+                    | ProjectionElem::OpaqueCast(..)
+                    | ProjectionElem::Subtype(..)
+                    | ProjectionElem::ConstantIndex { .. }
+                    | ProjectionElem::Subslice { .. } => (),
+                }
+            }
+        }
+    }
+
+    fn read_fake_borrows(
+        &mut self,
+        bb: BasicBlock,
+        fake_borrow_temps: &mut Vec<Local>,
+        source_info: SourceInfo,
+    ) {
+        // All indexes have been evaluated now, read all of the
+        // fake borrows so that they are live across those index
+        // expressions.
+        for temp in fake_borrow_temps {
+            self.cfg.push_fake_read(bb, source_info, FakeReadCause::ForIndex, Place::from(*temp));
+        }
+    }
+}
+
+/// Precise capture is enabled if user is using Rust Edition 2021 or higher.
+fn enable_precise_capture(closure_span: Span) -> bool {
+    closure_span.at_least_rust_2021()
+}

compiler/rustc_mir_build/src/builder/expr/as_rvalue.rs

@@ -0,0 +1,840 @@
+//! See docs in `build/expr/mod.rs`.
+
+use rustc_abi::{BackendRepr, FieldIdx, Primitive};
+use rustc_hir::lang_items::LangItem;
+use rustc_index::{Idx, IndexVec};
+use rustc_middle::bug;
+use rustc_middle::middle::region;
+use rustc_middle::mir::interpret::Scalar;
+use rustc_middle::mir::*;
+use rustc_middle::thir::*;
+use rustc_middle::ty::cast::{CastTy, mir_cast_kind};
+use rustc_middle::ty::layout::IntegerExt;
+use rustc_middle::ty::util::IntTypeExt;
+use rustc_middle::ty::{self, Ty, UpvarArgs};
+use rustc_span::source_map::Spanned;
+use rustc_span::{DUMMY_SP, Span};
+use tracing::debug;
+
+use crate::builder::expr::as_place::PlaceBase;
+use crate::builder::expr::category::{Category, RvalueFunc};
+use crate::builder::{BlockAnd, BlockAndExtension, Builder, NeedsTemporary};
+
+impl<'a, 'tcx> Builder<'a, 'tcx> {
+    /// Returns an rvalue suitable for use until the end of the current
+    /// scope expression.
+    ///
+    /// The operand returned from this function will *not be valid* after
+    /// an ExprKind::Scope is passed, so please do *not* return it from
+    /// functions to avoid bad miscompiles.
+    pub(crate) fn as_local_rvalue(
+        &mut self,
+        block: BasicBlock,
+        expr_id: ExprId,
+    ) -> BlockAnd<Rvalue<'tcx>> {
+        let local_scope = self.local_scope();
+        self.as_rvalue(
+            block,
+            TempLifetime { temp_lifetime: Some(local_scope), backwards_incompatible: None },
+            expr_id,
+        )
+    }
+
+    /// Compile `expr`, yielding an rvalue.
+    pub(crate) fn as_rvalue(
+        &mut self,
+        mut block: BasicBlock,
+        scope: TempLifetime,
+        expr_id: ExprId,
+    ) -> BlockAnd<Rvalue<'tcx>> {
+        let this = self;
+        let expr = &this.thir[expr_id];
+        debug!("expr_as_rvalue(block={:?}, scope={:?}, expr={:?})", block, scope, expr);
+
+        let expr_span = expr.span;
+        let source_info = this.source_info(expr_span);
+
+        match expr.kind {
+            ExprKind::ThreadLocalRef(did) => block.and(Rvalue::ThreadLocalRef(did)),
+            ExprKind::Scope { region_scope, lint_level, value } => {
+                let region_scope = (region_scope, source_info);
+                this.in_scope(region_scope, lint_level, |this| this.as_rvalue(block, scope, value))
+            }
+            ExprKind::Repeat { value, count } => {
+                if Some(0) == count.try_to_target_usize(this.tcx) {
+                    this.build_zero_repeat(block, value, scope, source_info)
+                } else {
+                    let value_operand = unpack!(
+                        block = this.as_operand(
+                            block,
+                            scope,
+                            value,
+                            LocalInfo::Boring,
+                            NeedsTemporary::No
+                        )
+                    );
+                    block.and(Rvalue::Repeat(value_operand, count))
+                }
+            }
+            ExprKind::Binary { op, lhs, rhs } => {
+                let lhs = unpack!(
+                    block = this.as_operand(
+                        block,
+                        scope,
+                        lhs,
+                        LocalInfo::Boring,
+                        NeedsTemporary::Maybe
+                    )
+                );
+                let rhs = unpack!(
+                    block =
+                        this.as_operand(block, scope, rhs, LocalInfo::Boring, NeedsTemporary::No)
+                );
+                this.build_binary_op(block, op, expr_span, expr.ty, lhs, rhs)
+            }
+            ExprKind::Unary { op, arg } => {
+                let arg = unpack!(
+                    block =
+                        this.as_operand(block, scope, arg, LocalInfo::Boring, NeedsTemporary::No)
+                );
+                // Check for -MIN on signed integers
+                if this.check_overflow && op == UnOp::Neg && expr.ty.is_signed() {
+                    let bool_ty = this.tcx.types.bool;
+
+                    let minval = this.minval_literal(expr_span, expr.ty);
+                    let is_min = this.temp(bool_ty, expr_span);
+
+                    this.cfg.push_assign(
+                        block,
+                        source_info,
+                        is_min,
+                        Rvalue::BinaryOp(BinOp::Eq, Box::new((arg.to_copy(), minval))),
+                    );
+
+                    block = this.assert(
+                        block,
+                        Operand::Move(is_min),
+                        false,
+                        AssertKind::OverflowNeg(arg.to_copy()),
+                        expr_span,
+                    );
+                }
+                block.and(Rvalue::UnaryOp(op, arg))
+            }
+            ExprKind::Box { value } => {
+                let value_ty = this.thir[value].ty;
+                let tcx = this.tcx;
+                let source_info = this.source_info(expr_span);
+
+                let size = this.temp(tcx.types.usize, expr_span);
+                this.cfg.push_assign(
+                    block,
+                    source_info,
+                    size,
+                    Rvalue::NullaryOp(NullOp::SizeOf, value_ty),
+                );
+
+                let align = this.temp(tcx.types.usize, expr_span);
+                this.cfg.push_assign(
+                    block,
+                    source_info,
+                    align,
+                    Rvalue::NullaryOp(NullOp::AlignOf, value_ty),
+                );
+
+                // malloc some memory of suitable size and align:
+                let exchange_malloc = Operand::function_handle(
+                    tcx,
+                    tcx.require_lang_item(LangItem::ExchangeMalloc, Some(expr_span)),
+                    [],
+                    expr_span,
+                );
+                let storage = this.temp(Ty::new_mut_ptr(tcx, tcx.types.u8), expr_span);
+                let success = this.cfg.start_new_block();
+                this.cfg.terminate(block, source_info, TerminatorKind::Call {
+                    func: exchange_malloc,
+                    args: [Spanned { node: Operand::Move(size), span: DUMMY_SP }, Spanned {
+                        node: Operand::Move(align),
+                        span: DUMMY_SP,
+                    }]
+                    .into(),
+                    destination: storage,
+                    target: Some(success),
+                    unwind: UnwindAction::Continue,
+                    call_source: CallSource::Misc,
+                    fn_span: expr_span,
+                });
+                this.diverge_from(block);
+                block = success;
+
+                // The `Box<T>` temporary created here is not a part of the HIR,
+                // and therefore is not considered during coroutine auto-trait
+                // determination. See the comment about `box` at `yield_in_scope`.
+                let result = this.local_decls.push(LocalDecl::new(expr.ty, expr_span));
+                this.cfg.push(block, Statement {
+                    source_info,
+                    kind: StatementKind::StorageLive(result),
+                });
+                if let Some(scope) = scope.temp_lifetime {
+                    // schedule a shallow free of that memory, lest we unwind:
+                    this.schedule_drop_storage_and_value(expr_span, scope, result);
+                }
+
+                // Transmute `*mut u8` to the box (thus far, uninitialized):
+                let box_ = Rvalue::ShallowInitBox(Operand::Move(storage), value_ty);
+                this.cfg.push_assign(block, source_info, Place::from(result), box_);
+
+                // initialize the box contents:
+                block = this
+                    .expr_into_dest(this.tcx.mk_place_deref(Place::from(result)), block, value)
+                    .into_block();
+                block.and(Rvalue::Use(Operand::Move(Place::from(result))))
+            }
+            ExprKind::Cast { source } => {
+                let source_expr = &this.thir[source];
+
+                // Casting an enum to an integer is equivalent to computing the discriminant and casting the
+                // discriminant. Previously every backend had to repeat the logic for this operation. Now we
+                // create all the steps directly in MIR with operations all backends need to support anyway.
+                let (source, ty) = if let ty::Adt(adt_def, ..) = source_expr.ty.kind()
+                    && adt_def.is_enum()
+                {
+                    let discr_ty = adt_def.repr().discr_type().to_ty(this.tcx);
+                    let temp = unpack!(block = this.as_temp(block, scope, source, Mutability::Not));
+                    let layout =
+                        this.tcx.layout_of(this.typing_env().as_query_input(source_expr.ty));
+                    let discr = this.temp(discr_ty, source_expr.span);
+                    this.cfg.push_assign(
+                        block,
+                        source_info,
+                        discr,
+                        Rvalue::Discriminant(temp.into()),
+                    );
+                    let (op, ty) = (Operand::Move(discr), discr_ty);
+
+                    if let BackendRepr::Scalar(scalar) = layout.unwrap().backend_repr
+                        && !scalar.is_always_valid(&this.tcx)
+                        && let Primitive::Int(int_width, _signed) = scalar.primitive()
+                    {
+                        let unsigned_ty = int_width.to_ty(this.tcx, false);
+                        let unsigned_place = this.temp(unsigned_ty, expr_span);
+                        this.cfg.push_assign(
+                            block,
+                            source_info,
+                            unsigned_place,
+                            Rvalue::Cast(CastKind::IntToInt, Operand::Copy(discr), unsigned_ty),
+                        );
+
+                        let bool_ty = this.tcx.types.bool;
+                        let range = scalar.valid_range(&this.tcx);
+                        let merge_op =
+                            if range.start <= range.end { BinOp::BitAnd } else { BinOp::BitOr };
+
+                        let mut comparer = |range: u128, bin_op: BinOp| -> Place<'tcx> {
+                            // We can use `ty::TypingEnv::fully_monomorphized()` here
+                            // as we only need it to compute the layout of a primitive.
+                            let range_val = Const::from_bits(
+                                this.tcx,
+                                range,
+                                ty::TypingEnv::fully_monomorphized(),
+                                unsigned_ty,
+                            );
+                            let lit_op = this.literal_operand(expr.span, range_val);
+                            let is_bin_op = this.temp(bool_ty, expr_span);
+                            this.cfg.push_assign(
+                                block,
+                                source_info,
+                                is_bin_op,
+                                Rvalue::BinaryOp(
+                                    bin_op,
+                                    Box::new((Operand::Copy(unsigned_place), lit_op)),
+                                ),
+                            );
+                            is_bin_op
+                        };
+                        let assert_place = if range.start == 0 {
+                            comparer(range.end, BinOp::Le)
+                        } else {
+                            let start_place = comparer(range.start, BinOp::Ge);
+                            let end_place = comparer(range.end, BinOp::Le);
+                            let merge_place = this.temp(bool_ty, expr_span);
+                            this.cfg.push_assign(
+                                block,
+                                source_info,
+                                merge_place,
+                                Rvalue::BinaryOp(
+                                    merge_op,
+                                    Box::new((
+                                        Operand::Move(start_place),
+                                        Operand::Move(end_place),
+                                    )),
+                                ),
+                            );
+                            merge_place
+                        };
+                        this.cfg.push(block, Statement {
+                            source_info,
+                            kind: StatementKind::Intrinsic(Box::new(
+                                NonDivergingIntrinsic::Assume(Operand::Move(assert_place)),
+                            )),
+                        });
+                    }
+
+                    (op, ty)
+                } else {
+                    let ty = source_expr.ty;
+                    let source = unpack!(
+                        block = this.as_operand(
+                            block,
+                            scope,
+                            source,
+                            LocalInfo::Boring,
+                            NeedsTemporary::No
+                        )
+                    );
+                    (source, ty)
+                };
+                let from_ty = CastTy::from_ty(ty);
+                let cast_ty = CastTy::from_ty(expr.ty);
+                debug!("ExprKind::Cast from_ty={from_ty:?}, cast_ty={:?}/{cast_ty:?}", expr.ty);
+                let cast_kind = mir_cast_kind(ty, expr.ty);
+                block.and(Rvalue::Cast(cast_kind, source, expr.ty))
+            }
+            ExprKind::PointerCoercion { cast, source, is_from_as_cast } => {
+                let source = unpack!(
+                    block = this.as_operand(
+                        block,
+                        scope,
+                        source,
+                        LocalInfo::Boring,
+                        NeedsTemporary::No
+                    )
+                );
+                let origin =
+                    if is_from_as_cast { CoercionSource::AsCast } else { CoercionSource::Implicit };
+                block.and(Rvalue::Cast(CastKind::PointerCoercion(cast, origin), source, expr.ty))
+            }
+            ExprKind::Array { ref fields } => {
+                // (*) We would (maybe) be closer to codegen if we
+                // handled this and other aggregate cases via
+                // `into()`, not `as_rvalue` -- in that case, instead
+                // of generating
+                //
+                //     let tmp1 = ...1;
+                //     let tmp2 = ...2;
+                //     dest = Rvalue::Aggregate(Foo, [tmp1, tmp2])
+                //
+                // we could just generate
+                //
+                //     dest.f = ...1;
+                //     dest.g = ...2;
+                //
+                // The problem is that then we would need to:
+                //
+                // (a) have a more complex mechanism for handling
+                //     partial cleanup;
+                // (b) distinguish the case where the type `Foo` has a
+                //     destructor, in which case creating an instance
+                //     as a whole "arms" the destructor, and you can't
+                //     write individual fields; and,
+                // (c) handle the case where the type Foo has no
+                //     fields. We don't want `let x: ();` to compile
+                //     to the same MIR as `let x = ();`.
+
+                // first process the set of fields
+                let el_ty = expr.ty.sequence_element_type(this.tcx);
+                let fields: IndexVec<FieldIdx, _> = fields
+                    .into_iter()
+                    .copied()
+                    .map(|f| {
+                        unpack!(
+                            block = this.as_operand(
+                                block,
+                                scope,
+                                f,
+                                LocalInfo::Boring,
+                                NeedsTemporary::Maybe
+                            )
+                        )
+                    })
+                    .collect();
+
+                block.and(Rvalue::Aggregate(Box::new(AggregateKind::Array(el_ty)), fields))
+            }
+            ExprKind::Tuple { ref fields } => {
+                // see (*) above
+                // first process the set of fields
+                let fields: IndexVec<FieldIdx, _> = fields
+                    .into_iter()
+                    .copied()
+                    .map(|f| {
+                        unpack!(
+                            block = this.as_operand(
+                                block,
+                                scope,
+                                f,
+                                LocalInfo::Boring,
+                                NeedsTemporary::Maybe
+                            )
+                        )
+                    })
+                    .collect();
+
+                block.and(Rvalue::Aggregate(Box::new(AggregateKind::Tuple), fields))
+            }
+            ExprKind::Closure(box ClosureExpr {
+                closure_id,
+                args,
+                ref upvars,
+                ref fake_reads,
+                movability: _,
+            }) => {
+                // Convert the closure fake reads, if any, from `ExprRef` to mir `Place`
+                // and push the fake reads.
+                // This must come before creating the operands. This is required in case
+                // there is a fake read and a borrow of the same path, since otherwise the
+                // fake read might interfere with the borrow. Consider an example like this
+                // one:
+                // ```
+                // let mut x = 0;
+                // let c = || {
+                //     &mut x; // mutable borrow of `x`
+                //     match x { _ => () } // fake read of `x`
+                // };
+                // ```
+                //
+                for (thir_place, cause, hir_id) in fake_reads.into_iter() {
+                    let place_builder = unpack!(block = this.as_place_builder(block, *thir_place));
+
+                    if let Some(mir_place) = place_builder.try_to_place(this) {
+                        this.cfg.push_fake_read(
+                            block,
+                            this.source_info(this.tcx.hir().span(*hir_id)),
+                            *cause,
+                            mir_place,
+                        );
+                    }
+                }
+
+                // see (*) above
+                let operands: IndexVec<FieldIdx, _> = upvars
+                    .into_iter()
+                    .copied()
+                    .map(|upvar| {
+                        let upvar_expr = &this.thir[upvar];
+                        match Category::of(&upvar_expr.kind) {
+                            // Use as_place to avoid creating a temporary when
+                            // moving a variable into a closure, so that
+                            // borrowck knows which variables to mark as being
+                            // used as mut. This is OK here because the upvar
+                            // expressions have no side effects and act on
+                            // disjoint places.
+                            // This occurs when capturing by copy/move, while
+                            // by reference captures use as_operand
+                            Some(Category::Place) => {
+                                let place = unpack!(block = this.as_place(block, upvar));
+                                this.consume_by_copy_or_move(place)
+                            }
+                            _ => {
+                                // Turn mutable borrow captures into unique
+                                // borrow captures when capturing an immutable
+                                // variable. This is sound because the mutation
+                                // that caused the capture will cause an error.
+                                match upvar_expr.kind {
+                                    ExprKind::Borrow {
+                                        borrow_kind:
+                                            BorrowKind::Mut { kind: MutBorrowKind::Default },
+                                        arg,
+                                    } => unpack!(
+                                        block = this.limit_capture_mutability(
+                                            upvar_expr.span,
+                                            upvar_expr.ty,
+                                            scope.temp_lifetime,
+                                            block,
+                                            arg,
+                                        )
+                                    ),
+                                    _ => {
+                                        unpack!(
+                                            block = this.as_operand(
+                                                block,
+                                                scope,
+                                                upvar,
+                                                LocalInfo::Boring,
+                                                NeedsTemporary::Maybe
+                                            )
+                                        )
+                                    }
+                                }
+                            }
+                        }
+                    })
+                    .collect();
+
+                let result = match args {
+                    UpvarArgs::Coroutine(args) => {
+                        Box::new(AggregateKind::Coroutine(closure_id.to_def_id(), args))
+                    }
+                    UpvarArgs::Closure(args) => {
+                        Box::new(AggregateKind::Closure(closure_id.to_def_id(), args))
+                    }
+                    UpvarArgs::CoroutineClosure(args) => {
+                        Box::new(AggregateKind::CoroutineClosure(closure_id.to_def_id(), args))
+                    }
+                };
+                block.and(Rvalue::Aggregate(result, operands))
+            }
+            ExprKind::Assign { .. } | ExprKind::AssignOp { .. } => {
+                block = this.stmt_expr(block, expr_id, None).into_block();
+                block.and(Rvalue::Use(Operand::Constant(Box::new(ConstOperand {
+                    span: expr_span,
+                    user_ty: None,
+                    const_: Const::zero_sized(this.tcx.types.unit),
+                }))))
+            }
+
+            ExprKind::OffsetOf { container, fields } => {
+                block.and(Rvalue::NullaryOp(NullOp::OffsetOf(fields), container))
+            }
+
+            ExprKind::Literal { .. }
+            | ExprKind::NamedConst { .. }
+            | ExprKind::NonHirLiteral { .. }
+            | ExprKind::ZstLiteral { .. }
+            | ExprKind::ConstParam { .. }
+            | ExprKind::ConstBlock { .. }
+            | ExprKind::StaticRef { .. } => {
+                let constant = this.as_constant(expr);
+                block.and(Rvalue::Use(Operand::Constant(Box::new(constant))))
+            }
+
+            ExprKind::Yield { .. }
+            | ExprKind::Block { .. }
+            | ExprKind::Match { .. }
+            | ExprKind::If { .. }
+            | ExprKind::NeverToAny { .. }
+            | ExprKind::Use { .. }
+            | ExprKind::Borrow { .. }
+            | ExprKind::RawBorrow { .. }
+            | ExprKind::Adt { .. }
+            | ExprKind::Loop { .. }
+            | ExprKind::LogicalOp { .. }
+            | ExprKind::Call { .. }
+            | ExprKind::Field { .. }
+            | ExprKind::Let { .. }
+            | ExprKind::Deref { .. }
+            | ExprKind::Index { .. }
+            | ExprKind::VarRef { .. }
+            | ExprKind::UpvarRef { .. }
+            | ExprKind::Break { .. }
+            | ExprKind::Continue { .. }
+            | ExprKind::Return { .. }
+            | ExprKind::Become { .. }
+            | ExprKind::InlineAsm { .. }
+            | ExprKind::PlaceTypeAscription { .. }
+            | ExprKind::ValueTypeAscription { .. } => {
+                // these do not have corresponding `Rvalue` variants,
+                // so make an operand and then return that
+                debug_assert!(!matches!(
+                    Category::of(&expr.kind),
+                    Some(Category::Rvalue(RvalueFunc::AsRvalue) | Category::Constant)
+                ));
+                let operand = unpack!(
+                    block = this.as_operand(
+                        block,
+                        scope,
+                        expr_id,
+                        LocalInfo::Boring,
+                        NeedsTemporary::No,
+                    )
+                );
+                block.and(Rvalue::Use(operand))
+            }
+        }
+    }
+
+    pub(crate) fn build_binary_op(
+        &mut self,
+        mut block: BasicBlock,
+        op: BinOp,
+        span: Span,
+        ty: Ty<'tcx>,
+        lhs: Operand<'tcx>,
+        rhs: Operand<'tcx>,
+    ) -> BlockAnd<Rvalue<'tcx>> {
+        let source_info = self.source_info(span);
+        let bool_ty = self.tcx.types.bool;
+        let rvalue = match op {
+            BinOp::Add | BinOp::Sub | BinOp::Mul if self.check_overflow && ty.is_integral() => {
+                let result_tup = Ty::new_tup(self.tcx, &[ty, bool_ty]);
+                let result_value = self.temp(result_tup, span);
+
+                let op_with_overflow = op.wrapping_to_overflowing().unwrap();
+
+                self.cfg.push_assign(
+                    block,
+                    source_info,
+                    result_value,
+                    Rvalue::BinaryOp(op_with_overflow, Box::new((lhs.to_copy(), rhs.to_copy()))),
+                );
+                let val_fld = FieldIdx::ZERO;
+                let of_fld = FieldIdx::new(1);
+
+                let tcx = self.tcx;
+                let val = tcx.mk_place_field(result_value, val_fld, ty);
+                let of = tcx.mk_place_field(result_value, of_fld, bool_ty);
+
+                let err = AssertKind::Overflow(op, lhs, rhs);
+                block = self.assert(block, Operand::Move(of), false, err, span);
+
+                Rvalue::Use(Operand::Move(val))
+            }
+            BinOp::Shl | BinOp::Shr if self.check_overflow && ty.is_integral() => {
+                // For an unsigned RHS, the shift is in-range for `rhs < bits`.
+                // For a signed RHS, `IntToInt` cast to the equivalent unsigned
+                // type and do that same comparison.
+                // A negative value will be *at least* 128 after the cast (that's i8::MIN),
+                // and 128 is an overflowing shift amount for all our currently existing types,
+                // so this cast can never make us miss an overflow.
+                let (lhs_size, _) = ty.int_size_and_signed(self.tcx);
+                assert!(lhs_size.bits() <= 128);
+                let rhs_ty = rhs.ty(&self.local_decls, self.tcx);
+                let (rhs_size, _) = rhs_ty.int_size_and_signed(self.tcx);
+
+                let (unsigned_rhs, unsigned_ty) = match rhs_ty.kind() {
+                    ty::Uint(_) => (rhs.to_copy(), rhs_ty),
+                    ty::Int(int_width) => {
+                        let uint_ty = Ty::new_uint(self.tcx, int_width.to_unsigned());
+                        let rhs_temp = self.temp(uint_ty, span);
+                        self.cfg.push_assign(
+                            block,
+                            source_info,
+                            rhs_temp,
+                            Rvalue::Cast(CastKind::IntToInt, rhs.to_copy(), uint_ty),
+                        );
+                        (Operand::Move(rhs_temp), uint_ty)
+                    }
+                    _ => unreachable!("only integers are shiftable"),
+                };
+
+                // This can't overflow because the largest shiftable types are 128-bit,
+                // which fits in `u8`, the smallest possible `unsigned_ty`.
+                let lhs_bits = Operand::const_from_scalar(
+                    self.tcx,
+                    unsigned_ty,
+                    Scalar::from_uint(lhs_size.bits(), rhs_size),
+                    span,
+                );
+
+                let inbounds = self.temp(bool_ty, span);
+                self.cfg.push_assign(
+                    block,
+                    source_info,
+                    inbounds,
+                    Rvalue::BinaryOp(BinOp::Lt, Box::new((unsigned_rhs, lhs_bits))),
+                );
+
+                let overflow_err = AssertKind::Overflow(op, lhs.to_copy(), rhs.to_copy());
+                block = self.assert(block, Operand::Move(inbounds), true, overflow_err, span);
+                Rvalue::BinaryOp(op, Box::new((lhs, rhs)))
+            }
+            BinOp::Div | BinOp::Rem if ty.is_integral() => {
+                // Checking division and remainder is more complex, since we 1. always check
+                // and 2. there are two possible failure cases, divide-by-zero and overflow.
+
+                let zero_err = if op == BinOp::Div {
+                    AssertKind::DivisionByZero(lhs.to_copy())
+                } else {
+                    AssertKind::RemainderByZero(lhs.to_copy())
+                };
+                let overflow_err = AssertKind::Overflow(op, lhs.to_copy(), rhs.to_copy());
+
+                // Check for / 0
+                let is_zero = self.temp(bool_ty, span);
+                let zero = self.zero_literal(span, ty);
+                self.cfg.push_assign(
+                    block,
+                    source_info,
+                    is_zero,
+                    Rvalue::BinaryOp(BinOp::Eq, Box::new((rhs.to_copy(), zero))),
+                );
+
+                block = self.assert(block, Operand::Move(is_zero), false, zero_err, span);
+
+                // We only need to check for the overflow in one case:
+                // MIN / -1, and only for signed values.
+                if ty.is_signed() {
+                    let neg_1 = self.neg_1_literal(span, ty);
+                    let min = self.minval_literal(span, ty);
+
+                    let is_neg_1 = self.temp(bool_ty, span);
+                    let is_min = self.temp(bool_ty, span);
+                    let of = self.temp(bool_ty, span);
+
+                    // this does (rhs == -1) & (lhs == MIN). It could short-circuit instead
+
+                    self.cfg.push_assign(
+                        block,
+                        source_info,
+                        is_neg_1,
+                        Rvalue::BinaryOp(BinOp::Eq, Box::new((rhs.to_copy(), neg_1))),
+                    );
+                    self.cfg.push_assign(
+                        block,
+                        source_info,
+                        is_min,
+                        Rvalue::BinaryOp(BinOp::Eq, Box::new((lhs.to_copy(), min))),
+                    );
+
+                    let is_neg_1 = Operand::Move(is_neg_1);
+                    let is_min = Operand::Move(is_min);
+                    self.cfg.push_assign(
+                        block,
+                        source_info,
+                        of,
+                        Rvalue::BinaryOp(BinOp::BitAnd, Box::new((is_neg_1, is_min))),
+                    );
+
+                    block = self.assert(block, Operand::Move(of), false, overflow_err, span);
+                }
+
+                Rvalue::BinaryOp(op, Box::new((lhs, rhs)))
+            }
+            _ => Rvalue::BinaryOp(op, Box::new((lhs, rhs))),
+        };
+        block.and(rvalue)
+    }
+
+    fn build_zero_repeat(
+        &mut self,
+        mut block: BasicBlock,
+        value: ExprId,
+        scope: TempLifetime,
+        outer_source_info: SourceInfo,
+    ) -> BlockAnd<Rvalue<'tcx>> {
+        let this = self;
+        let value_expr = &this.thir[value];
+        let elem_ty = value_expr.ty;
+        if let Some(Category::Constant) = Category::of(&value_expr.kind) {
+            // Repeating a const does nothing
+        } else {
+            // For a non-const, we may need to generate an appropriate `Drop`
+            let value_operand = unpack!(
+                block = this.as_operand(block, scope, value, LocalInfo::Boring, NeedsTemporary::No)
+            );
+            if let Operand::Move(to_drop) = value_operand {
+                let success = this.cfg.start_new_block();
+                this.cfg.terminate(block, outer_source_info, TerminatorKind::Drop {
+                    place: to_drop,
+                    target: success,
+                    unwind: UnwindAction::Continue,
+                    replace: false,
+                });
+                this.diverge_from(block);
+                block = success;
+            }
+            this.record_operands_moved(&[Spanned { node: value_operand, span: DUMMY_SP }]);
+        }
+        block.and(Rvalue::Aggregate(Box::new(AggregateKind::Array(elem_ty)), IndexVec::new()))
+    }
+
+    fn limit_capture_mutability(
+        &mut self,
+        upvar_span: Span,
+        upvar_ty: Ty<'tcx>,
+        temp_lifetime: Option<region::Scope>,
+        mut block: BasicBlock,
+        arg: ExprId,
+    ) -> BlockAnd<Operand<'tcx>> {
+        let this = self;
+
+        let source_info = this.source_info(upvar_span);
+        let temp = this.local_decls.push(LocalDecl::new(upvar_ty, upvar_span));
+
+        this.cfg.push(block, Statement { source_info, kind: StatementKind::StorageLive(temp) });
+
+        let arg_place_builder = unpack!(block = this.as_place_builder(block, arg));
+
+        let mutability = match arg_place_builder.base() {
+            // We are capturing a path that starts off a local variable in the parent.
+            // The mutability of the current capture is same as the mutability
+            // of the local declaration in the parent.
+            PlaceBase::Local(local) => this.local_decls[local].mutability,
+            // Parent is a closure and we are capturing a path that is captured
+            // by the parent itself. The mutability of the current capture
+            // is same as that of the capture in the parent closure.
+            PlaceBase::Upvar { .. } => {
+                let enclosing_upvars_resolved = arg_place_builder.to_place(this);
+
+                match enclosing_upvars_resolved.as_ref() {
+                    PlaceRef {
+                        local,
+                        projection: &[ProjectionElem::Field(upvar_index, _), ..],
+                    }
+                    | PlaceRef {
+                        local,
+                        projection:
+                            &[ProjectionElem::Deref, ProjectionElem::Field(upvar_index, _), ..],
+                    } => {
+                        // Not in a closure
+                        debug_assert!(
+                            local == ty::CAPTURE_STRUCT_LOCAL,
+                            "Expected local to be Local(1), found {local:?}"
+                        );
+                        // Not in a closure
+                        debug_assert!(
+                            this.upvars.len() > upvar_index.index(),
+                            "Unexpected capture place, upvars={:#?}, upvar_index={:?}",
+                            this.upvars,
+                            upvar_index
+                        );
+                        this.upvars[upvar_index.index()].mutability
+                    }
+                    _ => bug!("Unexpected capture place"),
+                }
+            }
+        };
+
+        let borrow_kind = match mutability {
+            Mutability::Not => BorrowKind::Mut { kind: MutBorrowKind::ClosureCapture },
+            Mutability::Mut => BorrowKind::Mut { kind: MutBorrowKind::Default },
+        };
+
+        let arg_place = arg_place_builder.to_place(this);
+
+        this.cfg.push_assign(
+            block,
+            source_info,
+            Place::from(temp),
+            Rvalue::Ref(this.tcx.lifetimes.re_erased, borrow_kind, arg_place),
+        );
+
+        // See the comment in `expr_as_temp` and on the `rvalue_scopes` field for why
+        // this can be `None`.
+        if let Some(temp_lifetime) = temp_lifetime {
+            this.schedule_drop_storage_and_value(upvar_span, temp_lifetime, temp);
+        }
+
+        block.and(Operand::Move(Place::from(temp)))
+    }
+
+    // Helper to get a `-1` value of the appropriate type
+    fn neg_1_literal(&mut self, span: Span, ty: Ty<'tcx>) -> Operand<'tcx> {
+        let typing_env = ty::TypingEnv::fully_monomorphized();
+        let size = self.tcx.layout_of(typing_env.as_query_input(ty)).unwrap().size;
+        let literal = Const::from_bits(self.tcx, size.unsigned_int_max(), typing_env, ty);
+
+        self.literal_operand(span, literal)
+    }
+
+    // Helper to get the minimum value of the appropriate type
+    fn minval_literal(&mut self, span: Span, ty: Ty<'tcx>) -> Operand<'tcx> {
+        assert!(ty.is_signed());
+        let typing_env = ty::TypingEnv::fully_monomorphized();
+        let bits = self.tcx.layout_of(typing_env.as_query_input(ty)).unwrap().size.bits();
+        let n = 1 << (bits - 1);
+        let literal = Const::from_bits(self.tcx, n, typing_env, ty);
+
+        self.literal_operand(span, literal)
+    }
+}

compiler/rustc_mir_build/src/builder/expr/as_temp.rs

@@ -0,0 +1,139 @@
+//! See docs in build/expr/mod.rs
+
+use rustc_data_structures::stack::ensure_sufficient_stack;
+use rustc_hir::HirId;
+use rustc_middle::middle::region::{Scope, ScopeData};
+use rustc_middle::mir::*;
+use rustc_middle::thir::*;
+use tracing::{debug, instrument};
+
+use crate::builder::scope::DropKind;
+use crate::builder::{BlockAnd, BlockAndExtension, Builder};
+
+impl<'a, 'tcx> Builder<'a, 'tcx> {
+    /// Compile `expr` into a fresh temporary. This is used when building
+    /// up rvalues so as to freeze the value that will be consumed.
+    pub(crate) fn as_temp(
+        &mut self,
+        block: BasicBlock,
+        temp_lifetime: TempLifetime,
+        expr_id: ExprId,
+        mutability: Mutability,
+    ) -> BlockAnd<Local> {
+        // this is the only place in mir building that we need to truly need to worry about
+        // infinite recursion. Everything else does recurse, too, but it always gets broken up
+        // at some point by inserting an intermediate temporary
+        ensure_sufficient_stack(|| self.as_temp_inner(block, temp_lifetime, expr_id, mutability))
+    }
+
+    #[instrument(skip(self), level = "debug")]
+    fn as_temp_inner(
+        &mut self,
+        mut block: BasicBlock,
+        temp_lifetime: TempLifetime,
+        expr_id: ExprId,
+        mutability: Mutability,
+    ) -> BlockAnd<Local> {
+        let this = self;
+
+        let expr = &this.thir[expr_id];
+        let expr_span = expr.span;
+        let source_info = this.source_info(expr_span);
+        if let ExprKind::Scope { region_scope, lint_level, value } = expr.kind {
+            return this.in_scope((region_scope, source_info), lint_level, |this| {
+                this.as_temp(block, temp_lifetime, value, mutability)
+            });
+        }
+
+        let expr_ty = expr.ty;
+        let deduplicate_temps = this.fixed_temps_scope.is_some()
+            && this.fixed_temps_scope == temp_lifetime.temp_lifetime;
+        let temp = if deduplicate_temps && let Some(temp_index) = this.fixed_temps.get(&expr_id) {
+            *temp_index
+        } else {
+            let mut local_decl = LocalDecl::new(expr_ty, expr_span);
+            if mutability.is_not() {
+                local_decl = local_decl.immutable();
+            }
+
+            debug!("creating temp {:?} with block_context: {:?}", local_decl, this.block_context);
+            let local_info = match expr.kind {
+                ExprKind::StaticRef { def_id, .. } => {
+                    assert!(!this.tcx.is_thread_local_static(def_id));
+                    LocalInfo::StaticRef { def_id, is_thread_local: false }
+                }
+                ExprKind::ThreadLocalRef(def_id) => {
+                    assert!(this.tcx.is_thread_local_static(def_id));
+                    LocalInfo::StaticRef { def_id, is_thread_local: true }
+                }
+                ExprKind::NamedConst { def_id, .. } | ExprKind::ConstParam { def_id, .. } => {
+                    LocalInfo::ConstRef { def_id }
+                }
+                // Find out whether this temp is being created within the
+                // tail expression of a block whose result is ignored.
+                _ if let Some(tail_info) = this.block_context.currently_in_block_tail() => {
+                    LocalInfo::BlockTailTemp(tail_info)
+                }
+
+                _ if let Some(Scope { data: ScopeData::IfThenRescope, id }) =
+                    temp_lifetime.temp_lifetime =>
+                {
+                    LocalInfo::IfThenRescopeTemp {
+                        if_then: HirId { owner: this.hir_id.owner, local_id: id },
+                    }
+                }
+
+                _ => LocalInfo::Boring,
+            };
+            **local_decl.local_info.as_mut().assert_crate_local() = local_info;
+            this.local_decls.push(local_decl)
+        };
+        debug!(?temp);
+        if deduplicate_temps {
+            this.fixed_temps.insert(expr_id, temp);
+        }
+        let temp_place = Place::from(temp);
+
+        match expr.kind {
+            // Don't bother with StorageLive and Dead for these temporaries,
+            // they are never assigned.
+            ExprKind::Break { .. } | ExprKind::Continue { .. } | ExprKind::Return { .. } => (),
+            ExprKind::Block { block }
+                if let Block { expr: None, targeted_by_break: false, .. } = this.thir[block]
+                    && expr_ty.is_never() => {}
+            _ => {
+                this.cfg
+                    .push(block, Statement { source_info, kind: StatementKind::StorageLive(temp) });
+
+                // In constants, `temp_lifetime` is `None` for temporaries that
+                // live for the `'static` lifetime. Thus we do not drop these
+                // temporaries and simply leak them.
+                // This is equivalent to what `let x = &foo();` does in
+                // functions. The temporary is lifted to their surrounding
+                // scope. In a function that means the temporary lives until
+                // just before the function returns. In constants that means it
+                // outlives the constant's initialization value computation.
+                // Anything outliving a constant must have the `'static`
+                // lifetime and live forever.
+                // Anything with a shorter lifetime (e.g the `&foo()` in
+                // `bar(&foo())` or anything within a block will keep the
+                // regular drops just like runtime code.
+                if let Some(temp_lifetime) = temp_lifetime.temp_lifetime {
+                    this.schedule_drop(expr_span, temp_lifetime, temp, DropKind::Storage);
+                }
+            }
+        }
+
+        block = this.expr_into_dest(temp_place, block, expr_id).into_block();
+
+        if let Some(temp_lifetime) = temp_lifetime.temp_lifetime {
+            this.schedule_drop(expr_span, temp_lifetime, temp, DropKind::Value);
+        }
+
+        if let Some(backwards_incompatible) = temp_lifetime.backwards_incompatible {
+            this.schedule_backwards_incompatible_drop(expr_span, backwards_incompatible, temp);
+        }
+
+        block.and(temp)
+    }
+}

compiler/rustc_mir_build/src/builder/expr/category.rs

@@ -0,0 +1,94 @@
+use rustc_middle::thir::*;
+
+#[derive(Debug, PartialEq)]
+pub(crate) enum Category {
+    /// An assignable memory location like `x`, `x.f`, `foo()[3]`, that
+    /// sort of thing. Something that could appear on the LHS of an `=`
+    /// sign.
+    Place,
+
+    /// A literal like `23` or `"foo"`. Does not include constant
+    /// expressions like `3 + 5`.
+    Constant,
+
+    /// Something that generates a new value at runtime, like `x + y`
+    /// or `foo()`.
+    Rvalue(RvalueFunc),
+}
+
+/// Rvalues fall into different "styles" that will determine which fn
+/// is best suited to generate them.
+#[derive(Debug, PartialEq)]
+pub(crate) enum RvalueFunc {
+    /// Best generated by `into`. This is generally exprs that
+    /// cause branching, like `match`, but also includes calls.
+    Into,
+
+    /// Best generated by `as_rvalue`. This is usually the case.
+    AsRvalue,
+}
+
+impl Category {
+    /// Determines the category for a given expression. Note that scope
+    /// and paren expressions have no category.
+    pub(crate) fn of(ek: &ExprKind<'_>) -> Option<Category> {
+        match *ek {
+            ExprKind::Scope { .. } => None,
+
+            ExprKind::Field { .. }
+            | ExprKind::Deref { .. }
+            | ExprKind::Index { .. }
+            | ExprKind::UpvarRef { .. }
+            | ExprKind::VarRef { .. }
+            | ExprKind::PlaceTypeAscription { .. }
+            | ExprKind::ValueTypeAscription { .. } => Some(Category::Place),
+
+            ExprKind::LogicalOp { .. }
+            | ExprKind::Match { .. }
+            | ExprKind::If { .. }
+            | ExprKind::Let { .. }
+            | ExprKind::NeverToAny { .. }
+            | ExprKind::Use { .. }
+            | ExprKind::Adt { .. }
+            | ExprKind::Borrow { .. }
+            | ExprKind::RawBorrow { .. }
+            | ExprKind::Yield { .. }
+            | ExprKind::Call { .. }
+            | ExprKind::InlineAsm { .. } => Some(Category::Rvalue(RvalueFunc::Into)),
+
+            ExprKind::Array { .. }
+            | ExprKind::Tuple { .. }
+            | ExprKind::Closure { .. }
+            | ExprKind::Unary { .. }
+            | ExprKind::Binary { .. }
+            | ExprKind::Box { .. }
+            | ExprKind::Cast { .. }
+            | ExprKind::PointerCoercion { .. }
+            | ExprKind::Repeat { .. }
+            | ExprKind::Assign { .. }
+            | ExprKind::AssignOp { .. }
+            | ExprKind::ThreadLocalRef(_)
+            | ExprKind::OffsetOf { .. } => Some(Category::Rvalue(RvalueFunc::AsRvalue)),
+
+            ExprKind::ConstBlock { .. }
+            | ExprKind::Literal { .. }
+            | ExprKind::NonHirLiteral { .. }
+            | ExprKind::ZstLiteral { .. }
+            | ExprKind::ConstParam { .. }
+            | ExprKind::StaticRef { .. }
+            | ExprKind::NamedConst { .. } => Some(Category::Constant),
+
+            ExprKind::Loop { .. }
+            | ExprKind::Block { .. }
+            | ExprKind::Break { .. }
+            | ExprKind::Continue { .. }
+            | ExprKind::Return { .. }
+            | ExprKind::Become { .. } =>
+            // FIXME(#27840) these probably want their own
+            // category, like "nonterminating"
+            {
+                Some(Category::Rvalue(RvalueFunc::Into))
+            }
+        }
+    }
+}

compiler/rustc_mir_build/src/builder/expr/into.rs

@@ -0,0 +1,650 @@
+//! See docs in build/expr/mod.rs
+
+use rustc_ast::{AsmMacro, InlineAsmOptions};
+use rustc_data_structures::fx::FxHashMap;
+use rustc_data_structures::stack::ensure_sufficient_stack;
+use rustc_hir as hir;
+use rustc_middle::mir::*;
+use rustc_middle::span_bug;
+use rustc_middle::thir::*;
+use rustc_middle::ty::CanonicalUserTypeAnnotation;
+use rustc_span::source_map::Spanned;
+use tracing::{debug, instrument};
+
+use crate::builder::expr::category::{Category, RvalueFunc};
+use crate::builder::matches::DeclareLetBindings;
+use crate::builder::{BlockAnd, BlockAndExtension, BlockFrame, Builder, NeedsTemporary};
+
+impl<'a, 'tcx> Builder<'a, 'tcx> {
+    /// Compile `expr`, storing the result into `destination`, which
+    /// is assumed to be uninitialized.
+    #[instrument(level = "debug", skip(self))]
+    pub(crate) fn expr_into_dest(
+        &mut self,
+        destination: Place<'tcx>,
+        mut block: BasicBlock,
+        expr_id: ExprId,
+    ) -> BlockAnd<()> {
+        // since we frequently have to reference `self` from within a
+        // closure, where `self` would be shadowed, it's easier to
+        // just use the name `this` uniformly
+        let this = self;
+        let expr = &this.thir[expr_id];
+        let expr_span = expr.span;
+        let source_info = this.source_info(expr_span);
+
+        let expr_is_block_or_scope =
+            matches!(expr.kind, ExprKind::Block { .. } | ExprKind::Scope { .. });
+
+        if !expr_is_block_or_scope {
+            this.block_context.push(BlockFrame::SubExpr);
+        }
+
+        let block_and = match expr.kind {
+            ExprKind::Scope { region_scope, lint_level, value } => {
+                let region_scope = (region_scope, source_info);
+                ensure_sufficient_stack(|| {
+                    this.in_scope(region_scope, lint_level, |this| {
+                        this.expr_into_dest(destination, block, value)
+                    })
+                })
+            }
+            ExprKind::Block { block: ast_block } => {
+                this.ast_block(destination, block, ast_block, source_info)
+            }
+            ExprKind::Match { scrutinee, ref arms, .. } => this.match_expr(
+                destination,
+                block,
+                scrutinee,
+                arms,
+                expr_span,
+                this.thir[scrutinee].span,
+            ),
+            ExprKind::If { cond, then, else_opt, if_then_scope } => {
+                let then_span = this.thir[then].span;
+                let then_source_info = this.source_info(then_span);
+                let condition_scope = this.local_scope();
+
+                let then_and_else_blocks = this.in_scope(
+                    (if_then_scope, then_source_info),
+                    LintLevel::Inherited,
+                    |this| {
+                        // FIXME: Does this need extra logic to handle let-chains?
+                        let source_info = if this.is_let(cond) {
+                            let variable_scope =
+                                this.new_source_scope(then_span, LintLevel::Inherited);
+                            this.source_scope = variable_scope;
+                            SourceInfo { span: then_span, scope: variable_scope }
+                        } else {
+                            this.source_info(then_span)
+                        };
+
+                        // Lower the condition, and have it branch into `then` and `else` blocks.
+                        let (then_block, else_block) =
+                            this.in_if_then_scope(condition_scope, then_span, |this| {
+                                let then_blk = this
+                                    .then_else_break(
+                                        block,
+                                        cond,
+                                        Some(condition_scope), // Temp scope
+                                        source_info,
+                                        DeclareLetBindings::Yes, // Declare `let` bindings normally
+                                    )
+                                    .into_block();
+
+                                // Lower the `then` arm into its block.
+                                this.expr_into_dest(destination, then_blk, then)
+                            });
+
+                        // Pack `(then_block, else_block)` into `BlockAnd<BasicBlock>`.
+                        then_block.and(else_block)
+                    },
+                );
+
+                // Unpack `BlockAnd<BasicBlock>` into `(then_blk, else_blk)`.
+                let (then_blk, mut else_blk);
+                else_blk = unpack!(then_blk = then_and_else_blocks);
+
+                // If there is an `else` arm, lower it into `else_blk`.
+                if let Some(else_expr) = else_opt {
+                    else_blk = this.expr_into_dest(destination, else_blk, else_expr).into_block();
+                } else {
+                    // There is no `else` arm, so we know both arms have type `()`.
+                    // Generate the implicit `else {}` by assigning unit.
+                    let correct_si = this.source_info(expr_span.shrink_to_hi());
+                    this.cfg.push_assign_unit(else_blk, correct_si, destination, this.tcx);
+                }
+
+                // The `then` and `else` arms have been lowered into their respective
+                // blocks, so make both of them meet up in a new block.
+                let join_block = this.cfg.start_new_block();
+                this.cfg.goto(then_blk, source_info, join_block);
+                this.cfg.goto(else_blk, source_info, join_block);
+                join_block.unit()
+            }
+            ExprKind::Let { .. } => {
+                // After desugaring, `let` expressions should only appear inside `if`
+                // expressions or `match` guards, possibly nested within a let-chain.
+                // In both cases they are specifically handled by the lowerings of
+                // those expressions, so this case is currently unreachable.
+                span_bug!(expr_span, "unexpected let expression outside of if or match-guard");
+            }
+            ExprKind::NeverToAny { source } => {
+                let source_expr = &this.thir[source];
+                let is_call =
+                    matches!(source_expr.kind, ExprKind::Call { .. } | ExprKind::InlineAsm { .. });
+
+                // (#66975) Source could be a const of type `!`, so has to
+                // exist in the generated MIR.
+                unpack!(
+                    block =
+                        this.as_temp(block, this.local_temp_lifetime(), source, Mutability::Mut)
+                );
+
+                // This is an optimization. If the expression was a call then we already have an
+                // unreachable block. Don't bother to terminate it and create a new one.
+                if is_call {
+                    block.unit()
+                } else {
+                    this.cfg.terminate(block, source_info, TerminatorKind::Unreachable);
+                    let end_block = this.cfg.start_new_block();
+                    end_block.unit()
+                }
+            }
+            ExprKind::LogicalOp { op, lhs, rhs } => {
+                let condition_scope = this.local_scope();
+                let source_info = this.source_info(expr.span);
+
+                this.visit_coverage_branch_operation(op, expr.span);
+
+                // We first evaluate the left-hand side of the predicate ...
+                let (then_block, else_block) =
+                    this.in_if_then_scope(condition_scope, expr.span, |this| {
+                        this.then_else_break(
+                            block,
+                            lhs,
+                            Some(condition_scope), // Temp scope
+                            source_info,
+                            // This flag controls how inner `let` expressions are lowered,
+                            // but either way there shouldn't be any of those in here.
+                            DeclareLetBindings::LetNotPermitted,
+                        )
+                    });
+                let (short_circuit, continuation, constant) = match op {
+                    LogicalOp::And => (else_block, then_block, false),
+                    LogicalOp::Or => (then_block, else_block, true),
+                };
+                // At this point, the control flow splits into a short-circuiting path
+                // and a continuation path.
+                // - If the operator is `&&`, passing `lhs` leads to continuation of evaluation on `rhs`;
+                //   failing it leads to the short-circuting path which assigns `false` to the place.
+                // - If the operator is `||`, failing `lhs` leads to continuation of evaluation on `rhs`;
+                //   passing it leads to the short-circuting path which assigns `true` to the place.
+                this.cfg.push_assign_constant(
+                    short_circuit,
+                    source_info,
+                    destination,
+                    ConstOperand {
+                        span: expr.span,
+                        user_ty: None,
+                        const_: Const::from_bool(this.tcx, constant),
+                    },
+                );
+                let mut rhs_block =
+                    this.expr_into_dest(destination, continuation, rhs).into_block();
+                // Instrument the lowered RHS's value for condition coverage.
+                // (Does nothing if condition coverage is not enabled.)
+                this.visit_coverage_standalone_condition(rhs, destination, &mut rhs_block);
+
+                let target = this.cfg.start_new_block();
+                this.cfg.goto(rhs_block, source_info, target);
+                this.cfg.goto(short_circuit, source_info, target);
+                target.unit()
+            }
+            ExprKind::Loop { body } => {
+                // [block]
+                //    |
+                //   [loop_block] -> [body_block] -/eval. body/-> [body_block_end]
+                //    |        ^                                         |
+                // false link  |                                         |
+                //    |        +-----------------------------------------+
+                //    +-> [diverge_cleanup]
+                // The false link is required to make sure borrowck considers unwinds through the
+                // body, even when the exact code in the body cannot unwind
+
+                let loop_block = this.cfg.start_new_block();
+
+                // Start the loop.
+                this.cfg.goto(block, source_info, loop_block);
+
+                this.in_breakable_scope(Some(loop_block), destination, expr_span, move |this| {
+                    // conduct the test, if necessary
+                    let body_block = this.cfg.start_new_block();
+                    this.cfg.terminate(loop_block, source_info, TerminatorKind::FalseUnwind {
+                        real_target: body_block,
+                        unwind: UnwindAction::Continue,
+                    });
+                    this.diverge_from(loop_block);
+
+                    // The “return” value of the loop body must always be a unit. We therefore
+                    // introduce a unit temporary as the destination for the loop body.
+                    let tmp = this.get_unit_temp();
+                    // Execute the body, branching back to the test.
+                    let body_block_end = this.expr_into_dest(tmp, body_block, body).into_block();
+                    this.cfg.goto(body_block_end, source_info, loop_block);
+
+                    // Loops are only exited by `break` expressions.
+                    None
+                })
+            }
+            ExprKind::Call { ty: _, fun, ref args, from_hir_call, fn_span } => {
+                let fun = unpack!(block = this.as_local_operand(block, fun));
+                let args: Box<[_]> = args
+                    .into_iter()
+                    .copied()
+                    .map(|arg| Spanned {
+                        node: unpack!(block = this.as_local_call_operand(block, arg)),
+                        span: this.thir.exprs[arg].span,
+                    })
+                    .collect();
+
+                let success = this.cfg.start_new_block();
+
+                this.record_operands_moved(&args);
+
+                debug!("expr_into_dest: fn_span={:?}", fn_span);
+
+                this.cfg.terminate(block, source_info, TerminatorKind::Call {
+                    func: fun,
+                    args,
+                    unwind: UnwindAction::Continue,
+                    destination,
+                    // The presence or absence of a return edge affects control-flow sensitive
+                    // MIR checks and ultimately whether code is accepted or not. We can only
+                    // omit the return edge if a return type is visibly uninhabited to a module
+                    // that makes the call.
+                    target: expr
+                        .ty
+                        .is_inhabited_from(
+                            this.tcx,
+                            this.parent_module,
+                            this.infcx.typing_env(this.param_env),
+                        )
+                        .then_some(success),
+                    call_source: if from_hir_call {
+                        CallSource::Normal
+                    } else {
+                        CallSource::OverloadedOperator
+                    },
+                    fn_span,
+                });
+                this.diverge_from(block);
+                success.unit()
+            }
+            ExprKind::Use { source } => this.expr_into_dest(destination, block, source),
+            ExprKind::Borrow { arg, borrow_kind } => {
+                // We don't do this in `as_rvalue` because we use `as_place`
+                // for borrow expressions, so we cannot create an `RValue` that
+                // remains valid across user code. `as_rvalue` is usually called
+                // by this method anyway, so this shouldn't cause too many
+                // unnecessary temporaries.
+                let arg_place = match borrow_kind {
+                    BorrowKind::Shared => {
+                        unpack!(block = this.as_read_only_place(block, arg))
+                    }
+                    _ => unpack!(block = this.as_place(block, arg)),
+                };
+                let borrow = Rvalue::Ref(this.tcx.lifetimes.re_erased, borrow_kind, arg_place);
+                this.cfg.push_assign(block, source_info, destination, borrow);
+                block.unit()
+            }
+            ExprKind::RawBorrow { mutability, arg } => {
+                let place = match mutability {
+                    hir::Mutability::Not => this.as_read_only_place(block, arg),
+                    hir::Mutability::Mut => this.as_place(block, arg),
+                };
+                let address_of = Rvalue::RawPtr(mutability, unpack!(block = place));
+                this.cfg.push_assign(block, source_info, destination, address_of);
+                block.unit()
+            }
+            ExprKind::Adt(box AdtExpr {
+                adt_def,
+                variant_index,
+                args,
+                ref user_ty,
+                ref fields,
+                ref base,
+            }) => {
+                // See the notes for `ExprKind::Array` in `as_rvalue` and for
+                // `ExprKind::Borrow` above.
+                let is_union = adt_def.is_union();
+                let active_field_index = is_union.then(|| fields[0].name);
+
+                let scope = this.local_temp_lifetime();
+
+                // first process the set of fields that were provided
+                // (evaluating them in order given by user)
+                let fields_map: FxHashMap<_, _> = fields
+                    .into_iter()
+                    .map(|f| {
+                        (
+                            f.name,
+                            unpack!(
+                                block = this.as_operand(
+                                    block,
+                                    scope,
+                                    f.expr,
+                                    LocalInfo::AggregateTemp,
+                                    NeedsTemporary::Maybe,
+                                )
+                            ),
+                        )
+                    })
+                    .collect();
+
+                let variant = adt_def.variant(variant_index);
+                let field_names = variant.fields.indices();
+
+                let fields = match base {
+                    AdtExprBase::None => {
+                        field_names.filter_map(|n| fields_map.get(&n).cloned()).collect()
+                    }
+                    AdtExprBase::Base(FruInfo { base, field_types }) => {
+                        let place_builder = unpack!(block = this.as_place_builder(block, *base));
+
+                        // We desugar FRU as we lower to MIR, so for each
+                        // base-supplied field, generate an operand that
+                        // reads it from the base.
+                        itertools::zip_eq(field_names, &**field_types)
+                            .map(|(n, ty)| match fields_map.get(&n) {
+                                Some(v) => v.clone(),
+                                None => {
+                                    let place =
+                                        place_builder.clone_project(PlaceElem::Field(n, *ty));
+                                    this.consume_by_copy_or_move(place.to_place(this))
+                                }
+                            })
+                            .collect()
+                    }
+                    AdtExprBase::DefaultFields(field_types) => {
+                        itertools::zip_eq(field_names, field_types)
+                            .map(|(n, &ty)| match fields_map.get(&n) {
+                                Some(v) => v.clone(),
+                                None => match variant.fields[n].value {
+                                    Some(def) => {
+                                        let value = Const::Unevaluated(
+                                            UnevaluatedConst::new(def, args),
+                                            ty,
+                                        );
+                                        Operand::Constant(Box::new(ConstOperand {
+                                            span: expr_span,
+                                            user_ty: None,
+                                            const_: value,
+                                        }))
+                                    }
+                                    None => {
+                                        let name = variant.fields[n].name;
+                                        span_bug!(
+                                            expr_span,
+                                            "missing mandatory field `{name}` of type `{ty}`",
+                                        );
+                                    }
+                                },
+                            })
+                            .collect()
+                    }
+                };
+
+                let inferred_ty = expr.ty;
+                let user_ty = user_ty.as_ref().map(|user_ty| {
+                    this.canonical_user_type_annotations.push(CanonicalUserTypeAnnotation {
+                        span: source_info.span,
+                        user_ty: user_ty.clone(),
+                        inferred_ty,
+                    })
+                });
+                let adt = Box::new(AggregateKind::Adt(
+                    adt_def.did(),
+                    variant_index,
+                    args,
+                    user_ty,
+                    active_field_index,
+                ));
+                this.cfg.push_assign(
+                    block,
+                    source_info,
+                    destination,
+                    Rvalue::Aggregate(adt, fields),
+                );
+                block.unit()
+            }
+            ExprKind::InlineAsm(box InlineAsmExpr {
+                asm_macro,
+                template,
+                ref operands,
+                options,
+                line_spans,
+            }) => {
+                use rustc_middle::{mir, thir};
+
+                let destination_block = this.cfg.start_new_block();
+                let mut targets =
+                    if asm_macro.diverges(options) { vec![] } else { vec![destination_block] };
+
+                let operands = operands
+                    .into_iter()
+                    .map(|op| match *op {
+                        thir::InlineAsmOperand::In { reg, expr } => mir::InlineAsmOperand::In {
+                            reg,
+                            value: unpack!(block = this.as_local_operand(block, expr)),
+                        },
+                        thir::InlineAsmOperand::Out { reg, late, expr } => {
+                            mir::InlineAsmOperand::Out {
+                                reg,
+                                late,
+                                place: expr.map(|expr| unpack!(block = this.as_place(block, expr))),
+                            }
+                        }
+                        thir::InlineAsmOperand::InOut { reg, late, expr } => {
+                            let place = unpack!(block = this.as_place(block, expr));
+                            mir::InlineAsmOperand::InOut {
+                                reg,
+                                late,
+                                // This works because asm operands must be Copy
+                                in_value: Operand::Copy(place),
+                                out_place: Some(place),
+                            }
+                        }
+                        thir::InlineAsmOperand::SplitInOut { reg, late, in_expr, out_expr } => {
+                            mir::InlineAsmOperand::InOut {
+                                reg,
+                                late,
+                                in_value: unpack!(block = this.as_local_operand(block, in_expr)),
+                                out_place: out_expr.map(|out_expr| {
+                                    unpack!(block = this.as_place(block, out_expr))
+                                }),
+                            }
+                        }
+                        thir::InlineAsmOperand::Const { value, span } => {
+                            mir::InlineAsmOperand::Const {
+                                value: Box::new(ConstOperand {
+                                    span,
+                                    user_ty: None,
+                                    const_: value,
+                                }),
+                            }
+                        }
+                        thir::InlineAsmOperand::SymFn { value, span } => {
+                            mir::InlineAsmOperand::SymFn {
+                                value: Box::new(ConstOperand {
+                                    span,
+                                    user_ty: None,
+                                    const_: value,
+                                }),
+                            }
+                        }
+                        thir::InlineAsmOperand::SymStatic { def_id } => {
+                            mir::InlineAsmOperand::SymStatic { def_id }
+                        }
+                        thir::InlineAsmOperand::Label { block } => {
+                            let target = this.cfg.start_new_block();
+                            let target_index = targets.len();
+                            targets.push(target);
+
+                            let tmp = this.get_unit_temp();
+                            let target =
+                                this.ast_block(tmp, target, block, source_info).into_block();
+                            this.cfg.terminate(target, source_info, TerminatorKind::Goto {
+                                target: destination_block,
+                            });
+
+                            mir::InlineAsmOperand::Label { target_index }
+                        }
+                    })
+                    .collect();
+
+                if !expr.ty.is_never() {
+                    this.cfg.push_assign_unit(block, source_info, destination, this.tcx);
+                }
+
+                let asm_macro = match asm_macro {
+                    AsmMacro::Asm => InlineAsmMacro::Asm,
+                    AsmMacro::GlobalAsm => {
+                        span_bug!(expr_span, "unexpected global_asm! in inline asm")
+                    }
+                    AsmMacro::NakedAsm => InlineAsmMacro::NakedAsm,
+                };
+
+                this.cfg.terminate(block, source_info, TerminatorKind::InlineAsm {
+                    asm_macro,
+                    template,
+                    operands,
+                    options,
+                    line_spans,
+                    targets: targets.into_boxed_slice(),
+                    unwind: if options.contains(InlineAsmOptions::MAY_UNWIND) {
+                        UnwindAction::Continue
+                    } else {
+                        UnwindAction::Unreachable
+                    },
+                });
+                if options.contains(InlineAsmOptions::MAY_UNWIND) {
+                    this.diverge_from(block);
+                }
+                destination_block.unit()
+            }
+
+            // These cases don't actually need a destination
+            ExprKind::Assign { .. } | ExprKind::AssignOp { .. } => {
+                block = this.stmt_expr(block, expr_id, None).into_block();
+                this.cfg.push_assign_unit(block, source_info, destination, this.tcx);
+                block.unit()
+            }
+
+            ExprKind::Continue { .. }
+            | ExprKind::Break { .. }
+            | ExprKind::Return { .. }
+            | ExprKind::Become { .. } => {
+                block = this.stmt_expr(block, expr_id, None).into_block();
+                // No assign, as these have type `!`.
+                block.unit()
+            }
+
+            // Avoid creating a temporary
+            ExprKind::VarRef { .. }
+            | ExprKind::UpvarRef { .. }
+            | ExprKind::PlaceTypeAscription { .. }
+            | ExprKind::ValueTypeAscription { .. } => {
+                debug_assert!(Category::of(&expr.kind) == Some(Category::Place));
+
+                let place = unpack!(block = this.as_place(block, expr_id));
+                let rvalue = Rvalue::Use(this.consume_by_copy_or_move(place));
+                this.cfg.push_assign(block, source_info, destination, rvalue);
+                block.unit()
+            }
+            ExprKind::Index { .. } | ExprKind::Deref { .. } | ExprKind::Field { .. } => {
+                debug_assert_eq!(Category::of(&expr.kind), Some(Category::Place));
+
+                // Create a "fake" temporary variable so that we check that the
+                // value is Sized. Usually, this is caught in type checking, but
+                // in the case of box expr there is no such check.
+                if !destination.projection.is_empty() {
+                    this.local_decls.push(LocalDecl::new(expr.ty, expr.span));
+                }
+
+                let place = unpack!(block = this.as_place(block, expr_id));
+                let rvalue = Rvalue::Use(this.consume_by_copy_or_move(place));
+                this.cfg.push_assign(block, source_info, destination, rvalue);
+                block.unit()
+            }
+
+            ExprKind::Yield { value } => {
+                let scope = this.local_temp_lifetime();
+                let value = unpack!(
+                    block =
+                        this.as_operand(block, scope, value, LocalInfo::Boring, NeedsTemporary::No)
+                );
+                let resume = this.cfg.start_new_block();
+                this.cfg.terminate(block, source_info, TerminatorKind::Yield {
+                    value,
+                    resume,
+                    resume_arg: destination,
+                    drop: None,
+                });
+                this.coroutine_drop_cleanup(block);
+                resume.unit()
+            }
+
+            // these are the cases that are more naturally handled by some other mode
+            ExprKind::Unary { .. }
+            | ExprKind::Binary { .. }
+            | ExprKind::Box { .. }
+            | ExprKind::Cast { .. }
+            | ExprKind::PointerCoercion { .. }
+            | ExprKind::Repeat { .. }
+            | ExprKind::Array { .. }
+            | ExprKind::Tuple { .. }
+            | ExprKind::Closure { .. }
+            | ExprKind::ConstBlock { .. }
+            | ExprKind::Literal { .. }
+            | ExprKind::NamedConst { .. }
+            | ExprKind::NonHirLiteral { .. }
+            | ExprKind::ZstLiteral { .. }
+            | ExprKind::ConstParam { .. }
+            | ExprKind::ThreadLocalRef(_)
+            | ExprKind::StaticRef { .. }
+            | ExprKind::OffsetOf { .. } => {
+                debug_assert!(match Category::of(&expr.kind).unwrap() {
+                    // should be handled above
+                    Category::Rvalue(RvalueFunc::Into) => false,
+
+                    // must be handled above or else we get an
+                    // infinite loop in the builder; see
+                    // e.g., `ExprKind::VarRef` above
+                    Category::Place => false,
+
+                    _ => true,
+                });
+
+                let rvalue = unpack!(block = this.as_local_rvalue(block, expr_id));
+                this.cfg.push_assign(block, source_info, destination, rvalue);
+                block.unit()
+            }
+        };
+
+        if !expr_is_block_or_scope {
+            let popped = this.block_context.pop();
+            assert!(popped.is_some());
+        }
+
+        block_and
+    }
+
+    fn is_let(&self, expr: ExprId) -> bool {
+        match self.thir[expr].kind {
+            ExprKind::Let { .. } => true,
+            ExprKind::Scope { value, .. } => self.is_let(value),
+            _ => false,
+        }
+    }
+}

compiler/rustc_mir_build/src/builder/expr/mod.rs

@@ -0,0 +1,70 @@
+//! Builds MIR from expressions. As a caller into this module, you
+//! have many options, but the first thing you have to decide is
+//! whether you are evaluating this expression for its *value*, its
+//! *location*, or as a *constant*.
+//!
+//! Typically, you want the value: e.g., if you are doing `expr_a +
+//! expr_b`, you want the values of those expressions. In that case,
+//! you want one of the following functions. Note that if the expr has
+//! a type that is not `Copy`, then using any of these functions will
+//! "move" the value out of its current home (if any).
+//!
+//! - `expr_into_dest` -- writes the value into a specific location, which
+//!   should be uninitialized
+//! - `as_operand` -- evaluates the value and yields an `Operand`,
+//!   suitable for use as an argument to an `Rvalue`
+//! - `as_temp` -- evaluates into a temporary; this is similar to `as_operand`
+//!   except it always returns a fresh place, even for constants
+//! - `as_rvalue` -- yields an `Rvalue`, suitable for use in an assignment;
+//!   as of this writing, never needed outside of the `expr` module itself
+//!
+//! Sometimes though want the expression's *location*. An example
+//! would be during a match statement, or the operand of the `&`
+//! operator. In that case, you want `as_place`. This will create a
+//! temporary if necessary.
+//!
+//! Finally, if it's a constant you seek, then call
+//! `as_constant`. This creates a `Constant<H>`, but naturally it can
+//! only be used on constant expressions and hence is needed only in
+//! very limited contexts.
+//!
+//! ### Implementation notes
+//!
+//! For any given kind of expression, there is generally one way that
+//! can be lowered most naturally. This is specified by the
+//! `Category::of` function in the `category` module. For example, a
+//! struct expression (or other expression that creates a new value)
+//! is typically easiest to write in terms of `as_rvalue` or `into`,
+//! whereas a reference to a field is easiest to write in terms of
+//! `as_place`. (The exception to this is scope and paren
+//! expressions, which have no category.)
+//!
+//! Therefore, the various functions above make use of one another in
+//! a descending fashion. For any given expression, you should pick
+//! the most suitable spot to implement it, and then just let the
+//! other fns cycle around. The handoff works like this:
+//!
+//! - `into(place)` -> fallback is to create an rvalue with `as_rvalue` and assign it to `place`
+//! - `as_rvalue` -> fallback is to create an Operand with `as_operand` and use `Rvalue::use`
+//! - `as_operand` -> either invokes `as_constant` or `as_temp`
+//! - `as_constant` -> (no fallback)
+//! - `as_temp` -> creates a temporary and either calls `as_place` or `into`
+//! - `as_place` -> for rvalues, falls back to `as_temp` and returns that
+//!
+//! As you can see, there is a cycle where `into` can (in theory) fallback to `as_temp`
+//! which can fallback to `into`. So if one of the `ExprKind` variants is not, in fact,
+//! implemented in the category where it is supposed to be, there will be a problem.
+//!
+//! Of those fallbacks, the most interesting one is `into`, because
+//! it discriminates based on the category of the expression. This is
+//! basically the point where the "by value" operations are bridged
+//! over to the "by reference" mode (`as_place`).
+
+pub(crate) mod as_constant;
+mod as_operand;
+pub(crate) mod as_place;
+mod as_rvalue;
+mod as_temp;
+pub(crate) mod category;
+mod into;
+mod stmt;

compiler/rustc_mir_build/src/builder/expr/stmt.rs

@@ -0,0 +1,196 @@
+use rustc_middle::middle::region;
+use rustc_middle::mir::*;
+use rustc_middle::span_bug;
+use rustc_middle::thir::*;
+use rustc_span::source_map::Spanned;
+use tracing::debug;
+
+use crate::builder::scope::BreakableTarget;
+use crate::builder::{BlockAnd, BlockAndExtension, BlockFrame, Builder};
+
+impl<'a, 'tcx> Builder<'a, 'tcx> {
+    /// Builds a block of MIR statements to evaluate the THIR `expr`.
+    ///
+    /// The `statement_scope` is used if a statement temporary must be dropped.
+    pub(crate) fn stmt_expr(
+        &mut self,
+        mut block: BasicBlock,
+        expr_id: ExprId,
+        statement_scope: Option<region::Scope>,
+    ) -> BlockAnd<()> {
+        let this = self;
+        let expr = &this.thir[expr_id];
+        let expr_span = expr.span;
+        let source_info = this.source_info(expr.span);
+        // Handle a number of expressions that don't need a destination at all. This
+        // avoids needing a mountain of temporary `()` variables.
+        match expr.kind {
+            ExprKind::Scope { region_scope, lint_level, value } => {
+                this.in_scope((region_scope, source_info), lint_level, |this| {
+                    this.stmt_expr(block, value, statement_scope)
+                })
+            }
+            ExprKind::Assign { lhs, rhs } => {
+                let lhs_expr = &this.thir[lhs];
+
+                // Note: we evaluate assignments right-to-left. This
+                // is better for borrowck interaction with overloaded
+                // operators like x[j] = x[i].
+
+                debug!("stmt_expr Assign block_context.push(SubExpr) : {:?}", expr);
+                this.block_context.push(BlockFrame::SubExpr);
+
+                // Generate better code for things that don't need to be
+                // dropped.
+                if lhs_expr.ty.needs_drop(this.tcx, this.typing_env()) {
+                    let rhs = unpack!(block = this.as_local_rvalue(block, rhs));
+                    let lhs = unpack!(block = this.as_place(block, lhs));
+                    block =
+                        this.build_drop_and_replace(block, lhs_expr.span, lhs, rhs).into_block();
+                } else {
+                    let rhs = unpack!(block = this.as_local_rvalue(block, rhs));
+                    let lhs = unpack!(block = this.as_place(block, lhs));
+                    this.cfg.push_assign(block, source_info, lhs, rhs);
+                }
+
+                this.block_context.pop();
+                block.unit()
+            }
+            ExprKind::AssignOp { op, lhs, rhs } => {
+                // FIXME(#28160) there is an interesting semantics
+                // question raised here -- should we "freeze" the
+                // value of the lhs here?  I'm inclined to think not,
+                // since it seems closer to the semantics of the
+                // overloaded version, which takes `&mut self`. This
+                // only affects weird things like `x += {x += 1; x}`
+                // -- is that equal to `x + (x + 1)` or `2*(x+1)`?
+
+                let lhs_ty = this.thir[lhs].ty;
+
+                debug!("stmt_expr AssignOp block_context.push(SubExpr) : {:?}", expr);
+                this.block_context.push(BlockFrame::SubExpr);
+
+                // As above, RTL.
+                let rhs = unpack!(block = this.as_local_operand(block, rhs));
+                let lhs = unpack!(block = this.as_place(block, lhs));
+
+                // we don't have to drop prior contents or anything
+                // because AssignOp is only legal for Copy types
+                // (overloaded ops should be desugared into a call).
+                let result = unpack!(
+                    block =
+                        this.build_binary_op(block, op, expr_span, lhs_ty, Operand::Copy(lhs), rhs)
+                );
+                this.cfg.push_assign(block, source_info, lhs, result);
+
+                this.block_context.pop();
+                block.unit()
+            }
+            ExprKind::Continue { label } => {
+                this.break_scope(block, None, BreakableTarget::Continue(label), source_info)
+            }
+            ExprKind::Break { label, value } => {
+                this.break_scope(block, value, BreakableTarget::Break(label), source_info)
+            }
+            ExprKind::Return { value } => {
+                this.break_scope(block, value, BreakableTarget::Return, source_info)
+            }
+            ExprKind::Become { value } => {
+                let v = &this.thir[value];
+                let ExprKind::Scope { value, lint_level, region_scope } = v.kind else {
+                    span_bug!(v.span, "`thir_check_tail_calls` should have disallowed this {v:?}")
+                };
+
+                let v = &this.thir[value];
+                let ExprKind::Call { ref args, fun, fn_span, .. } = v.kind else {
+                    span_bug!(v.span, "`thir_check_tail_calls` should have disallowed this {v:?}")
+                };
+
+                this.in_scope((region_scope, source_info), lint_level, |this| {
+                    let fun = unpack!(block = this.as_local_operand(block, fun));
+                    let args: Box<[_]> = args
+                        .into_iter()
+                        .copied()
+                        .map(|arg| Spanned {
+                            node: unpack!(block = this.as_local_call_operand(block, arg)),
+                            span: this.thir.exprs[arg].span,
+                        })
+                        .collect();
+
+                    this.record_operands_moved(&args);
+
+                    debug!("expr_into_dest: fn_span={:?}", fn_span);
+
+                    unpack!(block = this.break_for_tail_call(block, &args, source_info));
+
+                    this.cfg.terminate(block, source_info, TerminatorKind::TailCall {
+                        func: fun,
+                        args,
+                        fn_span,
+                    });
+
+                    this.cfg.start_new_block().unit()
+                })
+            }
+            _ => {
+                assert!(
+                    statement_scope.is_some(),
+                    "Should not be calling `stmt_expr` on a general expression \
+                     without a statement scope",
+                );
+
+                // Issue #54382: When creating temp for the value of
+                // expression like:
+                //
+                // `{ side_effects(); { let l = stuff(); the_value } }`
+                //
+                // it is usually better to focus on `the_value` rather
+                // than the entirety of block(s) surrounding it.
+                let adjusted_span = if let ExprKind::Block { block } = expr.kind
+                    && let Some(tail_ex) = this.thir[block].expr
+                {
+                    let mut expr = &this.thir[tail_ex];
+                    loop {
+                        match expr.kind {
+                            ExprKind::Block { block }
+                                if let Some(nested_expr) = this.thir[block].expr =>
+                            {
+                                expr = &this.thir[nested_expr];
+                            }
+                            ExprKind::Scope { value: nested_expr, .. } => {
+                                expr = &this.thir[nested_expr];
+                            }
+                            _ => break,
+                        }
+                    }
+                    this.block_context.push(BlockFrame::TailExpr {
+                        tail_result_is_ignored: true,
+                        span: expr.span,
+                    });
+                    Some(expr.span)
+                } else {
+                    None
+                };
+
+                let temp = unpack!(
+                    block = this.as_temp(
+                        block,
+                        TempLifetime {
+                            temp_lifetime: statement_scope,
+                            backwards_incompatible: None
+                        },
+                        expr_id,
+                        Mutability::Not
+                    )
+                );
+
+                if let Some(span) = adjusted_span {
+                    this.local_decls[temp].source_info.span = span;
+                    this.block_context.pop();
+                }
+
+                block.unit()
+            }
+        }
+    }
+}

compiler/rustc_mir_build/src/builder/matches/match_pair.rs

@@ -0,0 +1,260 @@
+use rustc_middle::mir::*;
+use rustc_middle::thir::{self, *};
+use rustc_middle::ty::{self, Ty, TypeVisitableExt};
+
+use crate::builder::Builder;
+use crate::builder::expr::as_place::{PlaceBase, PlaceBuilder};
+use crate::builder::matches::{FlatPat, MatchPairTree, TestCase};
+
+impl<'a, 'tcx> Builder<'a, 'tcx> {
+    /// Builds and returns [`MatchPairTree`] subtrees, one for each pattern in
+    /// `subpatterns`, representing the fields of a [`PatKind::Variant`] or
+    /// [`PatKind::Leaf`].
+    ///
+    /// Used internally by [`MatchPairTree::for_pattern`].
+    fn field_match_pairs<'pat>(
+        &mut self,
+        place: PlaceBuilder<'tcx>,
+        subpatterns: &'pat [FieldPat<'tcx>],
+    ) -> Vec<MatchPairTree<'pat, 'tcx>> {
+        subpatterns
+            .iter()
+            .map(|fieldpat| {
+                let place =
+                    place.clone_project(PlaceElem::Field(fieldpat.field, fieldpat.pattern.ty));
+                MatchPairTree::for_pattern(place, &fieldpat.pattern, self)
+            })
+            .collect()
+    }
+
+    /// Builds [`MatchPairTree`] subtrees for the prefix/middle/suffix parts of an
+    /// array pattern or slice pattern, and adds those trees to `match_pairs`.
+    ///
+    /// Used internally by [`MatchPairTree::for_pattern`].
+    fn prefix_slice_suffix<'pat>(
+        &mut self,
+        match_pairs: &mut Vec<MatchPairTree<'pat, 'tcx>>,
+        place: &PlaceBuilder<'tcx>,
+        prefix: &'pat [Box<Pat<'tcx>>],
+        opt_slice: &'pat Option<Box<Pat<'tcx>>>,
+        suffix: &'pat [Box<Pat<'tcx>>],
+    ) {
+        let tcx = self.tcx;
+        let (min_length, exact_size) = if let Some(place_resolved) = place.try_to_place(self) {
+            match place_resolved.ty(&self.local_decls, tcx).ty.kind() {
+                ty::Array(_, length) => (
+                    length
+                        .try_to_target_usize(tcx)
+                        .expect("expected len of array pat to be definite"),
+                    true,
+                ),
+                _ => ((prefix.len() + suffix.len()).try_into().unwrap(), false),
+            }
+        } else {
+            ((prefix.len() + suffix.len()).try_into().unwrap(), false)
+        };
+
+        match_pairs.extend(prefix.iter().enumerate().map(|(idx, subpattern)| {
+            let elem =
+                ProjectionElem::ConstantIndex { offset: idx as u64, min_length, from_end: false };
+            MatchPairTree::for_pattern(place.clone_project(elem), subpattern, self)
+        }));
+
+        if let Some(subslice_pat) = opt_slice {
+            let suffix_len = suffix.len() as u64;
+            let subslice = place.clone_project(PlaceElem::Subslice {
+                from: prefix.len() as u64,
+                to: if exact_size { min_length - suffix_len } else { suffix_len },
+                from_end: !exact_size,
+            });
+            match_pairs.push(MatchPairTree::for_pattern(subslice, subslice_pat, self));
+        }
+
+        match_pairs.extend(suffix.iter().rev().enumerate().map(|(idx, subpattern)| {
+            let end_offset = (idx + 1) as u64;
+            let elem = ProjectionElem::ConstantIndex {
+                offset: if exact_size { min_length - end_offset } else { end_offset },
+                min_length,
+                from_end: !exact_size,
+            };
+            let place = place.clone_project(elem);
+            MatchPairTree::for_pattern(place, subpattern, self)
+        }));
+    }
+}
+
+impl<'pat, 'tcx> MatchPairTree<'pat, 'tcx> {
+    /// Recursively builds a match pair tree for the given pattern and its
+    /// subpatterns.
+    pub(in crate::builder) fn for_pattern(
+        mut place_builder: PlaceBuilder<'tcx>,
+        pattern: &'pat Pat<'tcx>,
+        cx: &mut Builder<'_, 'tcx>,
+    ) -> MatchPairTree<'pat, 'tcx> {
+        // Force the place type to the pattern's type.
+        // FIXME(oli-obk): can we use this to simplify slice/array pattern hacks?
+        if let Some(resolved) = place_builder.resolve_upvar(cx) {
+            place_builder = resolved;
+        }
+
+        // Only add the OpaqueCast projection if the given place is an opaque type and the
+        // expected type from the pattern is not.
+        let may_need_cast = match place_builder.base() {
+            PlaceBase::Local(local) => {
+                let ty =
+                    Place::ty_from(local, place_builder.projection(), &cx.local_decls, cx.tcx).ty;
+                ty != pattern.ty && ty.has_opaque_types()
+            }
+            _ => true,
+        };
+        if may_need_cast {
+            place_builder = place_builder.project(ProjectionElem::OpaqueCast(pattern.ty));
+        }
+
+        let place = place_builder.try_to_place(cx);
+        let default_irrefutable = || TestCase::Irrefutable { binding: None, ascription: None };
+        let mut subpairs = Vec::new();
+        let test_case = match pattern.kind {
+            PatKind::Wild | PatKind::Error(_) => default_irrefutable(),
+
+            PatKind::Or { ref pats } => TestCase::Or {
+                pats: pats.iter().map(|pat| FlatPat::new(place_builder.clone(), pat, cx)).collect(),
+            },
+
+            PatKind::Range(ref range) => {
+                if range.is_full_range(cx.tcx) == Some(true) {
+                    default_irrefutable()
+                } else {
+                    TestCase::Range(range)
+                }
+            }
+
+            PatKind::Constant { value } => TestCase::Constant { value },
+
+            PatKind::AscribeUserType {
+                ascription: thir::Ascription { ref annotation, variance },
+                ref subpattern,
+                ..
+            } => {
+                // Apply the type ascription to the value at `match_pair.place`
+                let ascription = place.map(|source| super::Ascription {
+                    annotation: annotation.clone(),
+                    source,
+                    variance,
+                });
+
+                subpairs.push(MatchPairTree::for_pattern(place_builder, subpattern, cx));
+                TestCase::Irrefutable { ascription, binding: None }
+            }
+
+            PatKind::Binding { mode, var, ref subpattern, .. } => {
+                let binding = place.map(|source| super::Binding {
+                    span: pattern.span,
+                    source,
+                    var_id: var,
+                    binding_mode: mode,
+                });
+
+                if let Some(subpattern) = subpattern.as_ref() {
+                    // this is the `x @ P` case; have to keep matching against `P` now
+                    subpairs.push(MatchPairTree::for_pattern(place_builder, subpattern, cx));
+                }
+                TestCase::Irrefutable { ascription: None, binding }
+            }
+
+            PatKind::ExpandedConstant { subpattern: ref pattern, def_id: _, is_inline: false } => {
+                subpairs.push(MatchPairTree::for_pattern(place_builder, pattern, cx));
+                default_irrefutable()
+            }
+            PatKind::ExpandedConstant { subpattern: ref pattern, def_id, is_inline: true } => {
+                // Apply a type ascription for the inline constant to the value at `match_pair.place`
+                let ascription = place.map(|source| {
+                    let span = pattern.span;
+                    let parent_id = cx.tcx.typeck_root_def_id(cx.def_id.to_def_id());
+                    let args = ty::InlineConstArgs::new(cx.tcx, ty::InlineConstArgsParts {
+                        parent_args: ty::GenericArgs::identity_for_item(cx.tcx, parent_id),
+                        ty: cx.infcx.next_ty_var(span),
+                    })
+                    .args;
+                    let user_ty = cx.infcx.canonicalize_user_type_annotation(ty::UserType::new(
+                        ty::UserTypeKind::TypeOf(def_id, ty::UserArgs { args, user_self_ty: None }),
+                    ));
+                    let annotation = ty::CanonicalUserTypeAnnotation {
+                        inferred_ty: pattern.ty,
+                        span,
+                        user_ty: Box::new(user_ty),
+                    };
+                    super::Ascription { annotation, source, variance: ty::Contravariant }
+                });
+
+                subpairs.push(MatchPairTree::for_pattern(place_builder, pattern, cx));
+                TestCase::Irrefutable { ascription, binding: None }
+            }
+
+            PatKind::Array { ref prefix, ref slice, ref suffix } => {
+                cx.prefix_slice_suffix(&mut subpairs, &place_builder, prefix, slice, suffix);
+                default_irrefutable()
+            }
+            PatKind::Slice { ref prefix, ref slice, ref suffix } => {
+                cx.prefix_slice_suffix(&mut subpairs, &place_builder, prefix, slice, suffix);
+
+                if prefix.is_empty() && slice.is_some() && suffix.is_empty() {
+                    default_irrefutable()
+                } else {
+                    TestCase::Slice {
+                        len: prefix.len() + suffix.len(),
+                        variable_length: slice.is_some(),
+                    }
+                }
+            }
+
+            PatKind::Variant { adt_def, variant_index, args, ref subpatterns } => {
+                let downcast_place = place_builder.downcast(adt_def, variant_index); // `(x as Variant)`
+                subpairs = cx.field_match_pairs(downcast_place, subpatterns);
+
+                let irrefutable = adt_def.variants().iter_enumerated().all(|(i, v)| {
+                    i == variant_index
+                        || !v
+                            .inhabited_predicate(cx.tcx, adt_def)
+                            .instantiate(cx.tcx, args)
+                            .apply_ignore_module(cx.tcx, cx.infcx.typing_env(cx.param_env))
+                }) && (adt_def.did().is_local()
+                    || !adt_def.is_variant_list_non_exhaustive());
+                if irrefutable {
+                    default_irrefutable()
+                } else {
+                    TestCase::Variant { adt_def, variant_index }
+                }
+            }
+
+            PatKind::Leaf { ref subpatterns } => {
+                subpairs = cx.field_match_pairs(place_builder, subpatterns);
+                default_irrefutable()
+            }
+
+            PatKind::Deref { ref subpattern } => {
+                subpairs.push(MatchPairTree::for_pattern(place_builder.deref(), subpattern, cx));
+                default_irrefutable()
+            }
+
+            PatKind::DerefPattern { ref subpattern, mutability } => {
+                // Create a new temporary for each deref pattern.
+                // FIXME(deref_patterns): dedup temporaries to avoid multiple `deref()` calls?
+                let temp = cx.temp(
+                    Ty::new_ref(cx.tcx, cx.tcx.lifetimes.re_erased, subpattern.ty, mutability),
+                    pattern.span,
+                );
+                subpairs.push(MatchPairTree::for_pattern(
+                    PlaceBuilder::from(temp).deref(),
+                    subpattern,
+                    cx,
+                ));
+                TestCase::Deref { temp, mutability }
+            }
+
+            PatKind::Never => TestCase::Never,
+        };
+
+        MatchPairTree { place, test_case, subpairs, pattern }
+    }
+}

compiler/rustc_mir_build/src/builder/matches/simplify.rs

@@ -0,0 +1,71 @@
+//! Simplifying Candidates
+//!
+//! *Simplifying* a match pair `place @ pattern` means breaking it down
+//! into bindings or other, simpler match pairs. For example:
+//!
+//! - `place @ (P1, P2)` can be simplified to `[place.0 @ P1, place.1 @ P2]`
+//! - `place @ x` can be simplified to `[]` by binding `x` to `place`
+//!
+//! The `simplify_match_pairs` routine just repeatedly applies these
+//! sort of simplifications until there is nothing left to
+//! simplify. Match pairs cannot be simplified if they require some
+//! sort of test: for example, testing which variant an enum is, or
+//! testing a value against a constant.
+
+use std::mem;
+
+use tracing::{debug, instrument};
+
+use crate::builder::Builder;
+use crate::builder::matches::{MatchPairTree, PatternExtraData, TestCase};
+
+impl<'a, 'tcx> Builder<'a, 'tcx> {
+    /// Simplify a list of match pairs so they all require a test. Stores relevant bindings and
+    /// ascriptions in `extra_data`.
+    #[instrument(skip(self), level = "debug")]
+    pub(super) fn simplify_match_pairs<'pat>(
+        &mut self,
+        match_pairs: &mut Vec<MatchPairTree<'pat, 'tcx>>,
+        extra_data: &mut PatternExtraData<'tcx>,
+    ) {
+        // In order to please the borrow checker, in a pattern like `x @ pat` we must lower the
+        // bindings in `pat` before `x`. E.g. (#69971):
+        //
+        // struct NonCopyStruct {
+        //     copy_field: u32,
+        // }
+        //
+        // fn foo1(x: NonCopyStruct) {
+        //     let y @ NonCopyStruct { copy_field: z } = x;
+        //     // the above should turn into
+        //     let z = x.copy_field;
+        //     let y = x;
+        // }
+        //
+        // We therefore lower bindings from left-to-right, except we lower the `x` in `x @ pat`
+        // after any bindings in `pat`. This doesn't work for or-patterns: the current structure of
+        // match lowering forces us to lower bindings inside or-patterns last.
+        for mut match_pair in mem::take(match_pairs) {
+            self.simplify_match_pairs(&mut match_pair.subpairs, extra_data);
+            if let TestCase::Irrefutable { binding, ascription } = match_pair.test_case {
+                if let Some(binding) = binding {
+                    extra_data.bindings.push(binding);
+                }
+                if let Some(ascription) = ascription {
+                    extra_data.ascriptions.push(ascription);
+                }
+                // Simplifiable pattern; we replace it with its already simplified subpairs.
+                match_pairs.append(&mut match_pair.subpairs);
+            } else {
+                // Unsimplifiable pattern; we keep it.
+                match_pairs.push(match_pair);
+            }
+        }
+
+        // Move or-patterns to the end, because they can result in us
+        // creating additional candidates, so we want to test them as
+        // late as possible.
+        match_pairs.sort_by_key(|pair| matches!(pair.test_case, TestCase::Or { .. }));
+        debug!(simplified = ?match_pairs, "simplify_match_pairs");
+    }
+}

compiler/rustc_mir_build/src/builder/matches/test.rs

@@ -0,0 +1,784 @@
+// Testing candidates
+//
+// After candidates have been simplified, the only match pairs that
+// remain are those that require some sort of test. The functions here
+// identify what tests are needed, perform the tests, and then filter
+// the candidates based on the result.
+
+use std::cmp::Ordering;
+
+use rustc_data_structures::fx::FxIndexMap;
+use rustc_hir::{LangItem, RangeEnd};
+use rustc_middle::mir::*;
+use rustc_middle::ty::adjustment::PointerCoercion;
+use rustc_middle::ty::util::IntTypeExt;
+use rustc_middle::ty::{self, GenericArg, Ty, TyCtxt};
+use rustc_middle::{bug, span_bug};
+use rustc_span::def_id::DefId;
+use rustc_span::source_map::Spanned;
+use rustc_span::symbol::{Symbol, sym};
+use rustc_span::{DUMMY_SP, Span};
+use tracing::{debug, instrument};
+
+use crate::builder::Builder;
+use crate::builder::matches::{Candidate, MatchPairTree, Test, TestBranch, TestCase, TestKind};
+
+impl<'a, 'tcx> Builder<'a, 'tcx> {
+    /// Identifies what test is needed to decide if `match_pair` is applicable.
+    ///
+    /// It is a bug to call this with a not-fully-simplified pattern.
+    pub(super) fn pick_test_for_match_pair<'pat>(
+        &mut self,
+        match_pair: &MatchPairTree<'pat, 'tcx>,
+    ) -> Test<'tcx> {
+        let kind = match match_pair.test_case {
+            TestCase::Variant { adt_def, variant_index: _ } => TestKind::Switch { adt_def },
+
+            TestCase::Constant { .. } if match_pair.pattern.ty.is_bool() => TestKind::If,
+            TestCase::Constant { .. } if is_switch_ty(match_pair.pattern.ty) => TestKind::SwitchInt,
+            TestCase::Constant { value } => TestKind::Eq { value, ty: match_pair.pattern.ty },
+
+            TestCase::Range(range) => {
+                assert_eq!(range.ty, match_pair.pattern.ty);
+                TestKind::Range(Box::new(range.clone()))
+            }
+
+            TestCase::Slice { len, variable_length } => {
+                let op = if variable_length { BinOp::Ge } else { BinOp::Eq };
+                TestKind::Len { len: len as u64, op }
+            }
+
+            TestCase::Deref { temp, mutability } => TestKind::Deref { temp, mutability },
+
+            TestCase::Never => TestKind::Never,
+
+            // Or-patterns are not tested directly; instead they are expanded into subcandidates,
+            // which are then distinguished by testing whatever non-or patterns they contain.
+            TestCase::Or { .. } => bug!("or-patterns should have already been handled"),
+
+            TestCase::Irrefutable { .. } => span_bug!(
+                match_pair.pattern.span,
+                "simplifiable pattern found: {:?}",
+                match_pair.pattern
+            ),
+        };
+
+        Test { span: match_pair.pattern.span, kind }
+    }
+
+    #[instrument(skip(self, target_blocks, place), level = "debug")]
+    pub(super) fn perform_test(
+        &mut self,
+        match_start_span: Span,
+        scrutinee_span: Span,
+        block: BasicBlock,
+        otherwise_block: BasicBlock,
+        place: Place<'tcx>,
+        test: &Test<'tcx>,
+        target_blocks: FxIndexMap<TestBranch<'tcx>, BasicBlock>,
+    ) {
+        let place_ty = place.ty(&self.local_decls, self.tcx);
+        debug!(?place, ?place_ty);
+        let target_block = |branch| target_blocks.get(&branch).copied().unwrap_or(otherwise_block);
+
+        let source_info = self.source_info(test.span);
+        match test.kind {
+            TestKind::Switch { adt_def } => {
+                let otherwise_block = target_block(TestBranch::Failure);
+                let switch_targets = SwitchTargets::new(
+                    adt_def.discriminants(self.tcx).filter_map(|(idx, discr)| {
+                        if let Some(&block) = target_blocks.get(&TestBranch::Variant(idx)) {
+                            Some((discr.val, block))
+                        } else {
+                            None
+                        }
+                    }),
+                    otherwise_block,
+                );
+                debug!("num_enum_variants: {}", adt_def.variants().len());
+                let discr_ty = adt_def.repr().discr_type().to_ty(self.tcx);
+                let discr = self.temp(discr_ty, test.span);
+                self.cfg.push_assign(
+                    block,
+                    self.source_info(scrutinee_span),
+                    discr,
+                    Rvalue::Discriminant(place),
+                );
+                self.cfg.terminate(
+                    block,
+                    self.source_info(match_start_span),
+                    TerminatorKind::SwitchInt {
+                        discr: Operand::Move(discr),
+                        targets: switch_targets,
+                    },
+                );
+            }
+
+            TestKind::SwitchInt => {
+                // The switch may be inexhaustive so we have a catch-all block
+                let otherwise_block = target_block(TestBranch::Failure);
+                let switch_targets = SwitchTargets::new(
+                    target_blocks.iter().filter_map(|(&branch, &block)| {
+                        if let TestBranch::Constant(_, bits) = branch {
+                            Some((bits, block))
+                        } else {
+                            None
+                        }
+                    }),
+                    otherwise_block,
+                );
+                let terminator = TerminatorKind::SwitchInt {
+                    discr: Operand::Copy(place),
+                    targets: switch_targets,
+                };
+                self.cfg.terminate(block, self.source_info(match_start_span), terminator);
+            }
+
+            TestKind::If => {
+                let success_block = target_block(TestBranch::Success);
+                let fail_block = target_block(TestBranch::Failure);
+                let terminator =
+                    TerminatorKind::if_(Operand::Copy(place), success_block, fail_block);
+                self.cfg.terminate(block, self.source_info(match_start_span), terminator);
+            }
+
+            TestKind::Eq { value, ty } => {
+                let tcx = self.tcx;
+                let success_block = target_block(TestBranch::Success);
+                let fail_block = target_block(TestBranch::Failure);
+                if let ty::Adt(def, _) = ty.kind()
+                    && tcx.is_lang_item(def.did(), LangItem::String)
+                {
+                    if !tcx.features().string_deref_patterns() {
+                        span_bug!(
+                            test.span,
+                            "matching on `String` went through without enabling string_deref_patterns"
+                        );
+                    }
+                    let re_erased = tcx.lifetimes.re_erased;
+                    let ref_str_ty = Ty::new_imm_ref(tcx, re_erased, tcx.types.str_);
+                    let ref_str = self.temp(ref_str_ty, test.span);
+                    let eq_block = self.cfg.start_new_block();
+                    // `let ref_str: &str = <String as Deref>::deref(&place);`
+                    self.call_deref(
+                        block,
+                        eq_block,
+                        place,
+                        Mutability::Not,
+                        ty,
+                        ref_str,
+                        test.span,
+                    );
+                    self.non_scalar_compare(
+                        eq_block,
+                        success_block,
+                        fail_block,
+                        source_info,
+                        value,
+                        ref_str,
+                        ref_str_ty,
+                    );
+                } else if !ty.is_scalar() {
+                    // Use `PartialEq::eq` instead of `BinOp::Eq`
+                    // (the binop can only handle primitives)
+                    self.non_scalar_compare(
+                        block,
+                        success_block,
+                        fail_block,
+                        source_info,
+                        value,
+                        place,
+                        ty,
+                    );
+                } else {
+                    assert_eq!(value.ty(), ty);
+                    let expect = self.literal_operand(test.span, value);
+                    let val = Operand::Copy(place);
+                    self.compare(
+                        block,
+                        success_block,
+                        fail_block,
+                        source_info,
+                        BinOp::Eq,
+                        expect,
+                        val,
+                    );
+                }
+            }
+
+            TestKind::Range(ref range) => {
+                let success = target_block(TestBranch::Success);
+                let fail = target_block(TestBranch::Failure);
+                // Test `val` by computing `lo <= val && val <= hi`, using primitive comparisons.
+                let val = Operand::Copy(place);
+
+                let intermediate_block = if !range.lo.is_finite() {
+                    block
+                } else if !range.hi.is_finite() {
+                    success
+                } else {
+                    self.cfg.start_new_block()
+                };
+
+                if let Some(lo) = range.lo.as_finite() {
+                    let lo = self.literal_operand(test.span, lo);
+                    self.compare(
+                        block,
+                        intermediate_block,
+                        fail,
+                        source_info,
+                        BinOp::Le,
+                        lo,
+                        val.clone(),
+                    );
+                };
+
+                if let Some(hi) = range.hi.as_finite() {
+                    let hi = self.literal_operand(test.span, hi);
+                    let op = match range.end {
+                        RangeEnd::Included => BinOp::Le,
+                        RangeEnd::Excluded => BinOp::Lt,
+                    };
+                    self.compare(intermediate_block, success, fail, source_info, op, val, hi);
+                }
+            }
+
+            TestKind::Len { len, op } => {
+                let usize_ty = self.tcx.types.usize;
+                let actual = self.temp(usize_ty, test.span);
+
+                // actual = len(place)
+                self.cfg.push_assign(block, source_info, actual, Rvalue::Len(place));
+
+                // expected = <N>
+                let expected = self.push_usize(block, source_info, len);
+
+                let success_block = target_block(TestBranch::Success);
+                let fail_block = target_block(TestBranch::Failure);
+                // result = actual == expected OR result = actual < expected
+                // branch based on result
+                self.compare(
+                    block,
+                    success_block,
+                    fail_block,
+                    source_info,
+                    op,
+                    Operand::Move(actual),
+                    Operand::Move(expected),
+                );
+            }
+
+            TestKind::Deref { temp, mutability } => {
+                let ty = place_ty.ty;
+                let target = target_block(TestBranch::Success);
+                self.call_deref(block, target, place, mutability, ty, temp, test.span);
+            }
+
+            TestKind::Never => {
+                // Check that the place is initialized.
+                // FIXME(never_patterns): Also assert validity of the data at `place`.
+                self.cfg.push_fake_read(
+                    block,
+                    source_info,
+                    FakeReadCause::ForMatchedPlace(None),
+                    place,
+                );
+                // A never pattern is only allowed on an uninhabited type, so validity of the data
+                // implies unreachability.
+                self.cfg.terminate(block, source_info, TerminatorKind::Unreachable);
+            }
+        }
+    }
+
+    /// Perform `let temp = <ty as Deref>::deref(&place)`.
+    /// or `let temp = <ty as DerefMut>::deref_mut(&mut place)`.
+    pub(super) fn call_deref(
+        &mut self,
+        block: BasicBlock,
+        target_block: BasicBlock,
+        place: Place<'tcx>,
+        mutability: Mutability,
+        ty: Ty<'tcx>,
+        temp: Place<'tcx>,
+        span: Span,
+    ) {
+        let (trait_item, method) = match mutability {
+            Mutability::Not => (LangItem::Deref, sym::deref),
+            Mutability::Mut => (LangItem::DerefMut, sym::deref_mut),
+        };
+        let borrow_kind = super::util::ref_pat_borrow_kind(mutability);
+        let source_info = self.source_info(span);
+        let re_erased = self.tcx.lifetimes.re_erased;
+        let trait_item = self.tcx.require_lang_item(trait_item, None);
+        let method = trait_method(self.tcx, trait_item, method, [ty]);
+        let ref_src = self.temp(Ty::new_ref(self.tcx, re_erased, ty, mutability), span);
+        // `let ref_src = &src_place;`
+        // or `let ref_src = &mut src_place;`
+        self.cfg.push_assign(
+            block,
+            source_info,
+            ref_src,
+            Rvalue::Ref(re_erased, borrow_kind, place),
+        );
+        // `let temp = <Ty as Deref>::deref(ref_src);`
+        // or `let temp = <Ty as DerefMut>::deref_mut(ref_src);`
+        self.cfg.terminate(block, source_info, TerminatorKind::Call {
+            func: Operand::Constant(Box::new(ConstOperand { span, user_ty: None, const_: method })),
+            args: [Spanned { node: Operand::Move(ref_src), span }].into(),
+            destination: temp,
+            target: Some(target_block),
+            unwind: UnwindAction::Continue,
+            call_source: CallSource::Misc,
+            fn_span: source_info.span,
+        });
+    }
+
+    /// Compare using the provided built-in comparison operator
+    fn compare(
+        &mut self,
+        block: BasicBlock,
+        success_block: BasicBlock,
+        fail_block: BasicBlock,
+        source_info: SourceInfo,
+        op: BinOp,
+        left: Operand<'tcx>,
+        right: Operand<'tcx>,
+    ) {
+        let bool_ty = self.tcx.types.bool;
+        let result = self.temp(bool_ty, source_info.span);
+
+        // result = op(left, right)
+        self.cfg.push_assign(
+            block,
+            source_info,
+            result,
+            Rvalue::BinaryOp(op, Box::new((left, right))),
+        );
+
+        // branch based on result
+        self.cfg.terminate(
+            block,
+            source_info,
+            TerminatorKind::if_(Operand::Move(result), success_block, fail_block),
+        );
+    }
+
+    /// Compare two values using `<T as std::compare::PartialEq>::eq`.
+    /// If the values are already references, just call it directly, otherwise
+    /// take a reference to the values first and then call it.
+    fn non_scalar_compare(
+        &mut self,
+        block: BasicBlock,
+        success_block: BasicBlock,
+        fail_block: BasicBlock,
+        source_info: SourceInfo,
+        value: Const<'tcx>,
+        mut val: Place<'tcx>,
+        mut ty: Ty<'tcx>,
+    ) {
+        let mut expect = self.literal_operand(source_info.span, value);
+
+        // If we're using `b"..."` as a pattern, we need to insert an
+        // unsizing coercion, as the byte string has the type `&[u8; N]`.
+        //
+        // We want to do this even when the scrutinee is a reference to an
+        // array, so we can call `<[u8]>::eq` rather than having to find an
+        // `<[u8; N]>::eq`.
+        let unsize = |ty: Ty<'tcx>| match ty.kind() {
+            ty::Ref(region, rty, _) => match rty.kind() {
+                ty::Array(inner_ty, n) => Some((region, inner_ty, n)),
+                _ => None,
+            },
+            _ => None,
+        };
+        let opt_ref_ty = unsize(ty);
+        let opt_ref_test_ty = unsize(value.ty());
+        match (opt_ref_ty, opt_ref_test_ty) {
+            // nothing to do, neither is an array
+            (None, None) => {}
+            (Some((region, elem_ty, _)), _) | (None, Some((region, elem_ty, _))) => {
+                let tcx = self.tcx;
+                // make both a slice
+                ty = Ty::new_imm_ref(tcx, *region, Ty::new_slice(tcx, *elem_ty));
+                if opt_ref_ty.is_some() {
+                    let temp = self.temp(ty, source_info.span);
+                    self.cfg.push_assign(
+                        block,
+                        source_info,
+                        temp,
+                        Rvalue::Cast(
+                            CastKind::PointerCoercion(
+                                PointerCoercion::Unsize,
+                                CoercionSource::Implicit,
+                            ),
+                            Operand::Copy(val),
+                            ty,
+                        ),
+                    );
+                    val = temp;
+                }
+                if opt_ref_test_ty.is_some() {
+                    let slice = self.temp(ty, source_info.span);
+                    self.cfg.push_assign(
+                        block,
+                        source_info,
+                        slice,
+                        Rvalue::Cast(
+                            CastKind::PointerCoercion(
+                                PointerCoercion::Unsize,
+                                CoercionSource::Implicit,
+                            ),
+                            expect,
+                            ty,
+                        ),
+                    );
+                    expect = Operand::Move(slice);
+                }
+            }
+        }
+
+        // Figure out the type on which we are calling `PartialEq`. This involves an extra wrapping
+        // reference: we can only compare two `&T`, and then compare_ty will be `T`.
+        // Make sure that we do *not* call any user-defined code here.
+        // The only types that can end up here are string and byte literals,
+        // which have their comparison defined in `core`.
+        // (Interestingly this means that exhaustiveness analysis relies, for soundness,
+        // on the `PartialEq` impls for `str` and `[u8]` to b correct!)
+        let compare_ty = match *ty.kind() {
+            ty::Ref(_, deref_ty, _)
+                if deref_ty == self.tcx.types.str_ || deref_ty != self.tcx.types.u8 =>
+            {
+                deref_ty
+            }
+            _ => span_bug!(source_info.span, "invalid type for non-scalar compare: {}", ty),
+        };
+
+        let eq_def_id = self.tcx.require_lang_item(LangItem::PartialEq, Some(source_info.span));
+        let method = trait_method(self.tcx, eq_def_id, sym::eq, [compare_ty, compare_ty]);
+
+        let bool_ty = self.tcx.types.bool;
+        let eq_result = self.temp(bool_ty, source_info.span);
+        let eq_block = self.cfg.start_new_block();
+        self.cfg.terminate(block, source_info, TerminatorKind::Call {
+            func: Operand::Constant(Box::new(ConstOperand {
+                span: source_info.span,
+
+                // FIXME(#54571): This constant comes from user input (a
+                // constant in a pattern). Are there forms where users can add
+                // type annotations here?  For example, an associated constant?
+                // Need to experiment.
+                user_ty: None,
+
+                const_: method,
+            })),
+            args: [Spanned { node: Operand::Copy(val), span: DUMMY_SP }, Spanned {
+                node: expect,
+                span: DUMMY_SP,
+            }]
+            .into(),
+            destination: eq_result,
+            target: Some(eq_block),
+            unwind: UnwindAction::Continue,
+            call_source: CallSource::MatchCmp,
+            fn_span: source_info.span,
+        });
+        self.diverge_from(block);
+
+        // check the result
+        self.cfg.terminate(
+            eq_block,
+            source_info,
+            TerminatorKind::if_(Operand::Move(eq_result), success_block, fail_block),
+        );
+    }
+
+    /// Given that we are performing `test` against `test_place`, this job
+    /// sorts out what the status of `candidate` will be after the test. See
+    /// `test_candidates` for the usage of this function. The candidate may
+    /// be modified to update its `match_pairs`.
+    ///
+    /// So, for example, if this candidate is `x @ Some(P0)` and the `Test` is
+    /// a variant test, then we would modify the candidate to be `(x as
+    /// Option).0 @ P0` and return the index corresponding to the variant
+    /// `Some`.
+    ///
+    /// However, in some cases, the test may just not be relevant to candidate.
+    /// For example, suppose we are testing whether `foo.x == 22`, but in one
+    /// match arm we have `Foo { x: _, ... }`... in that case, the test for
+    /// the value of `x` has no particular relevance to this candidate. In
+    /// such cases, this function just returns None without doing anything.
+    /// This is used by the overall `match_candidates` algorithm to structure
+    /// the match as a whole. See `match_candidates` for more details.
+    ///
+    /// FIXME(#29623). In some cases, we have some tricky choices to make. for
+    /// example, if we are testing that `x == 22`, but the candidate is `x @
+    /// 13..55`, what should we do? In the event that the test is true, we know
+    /// that the candidate applies, but in the event of false, we don't know
+    /// that it *doesn't* apply. For now, we return false, indicate that the
+    /// test does not apply to this candidate, but it might be we can get
+    /// tighter match code if we do something a bit different.
+    pub(super) fn sort_candidate(
+        &mut self,
+        test_place: Place<'tcx>,
+        test: &Test<'tcx>,
+        candidate: &mut Candidate<'_, 'tcx>,
+        sorted_candidates: &FxIndexMap<TestBranch<'tcx>, Vec<&mut Candidate<'_, 'tcx>>>,
+    ) -> Option<TestBranch<'tcx>> {
+        // Find the match_pair for this place (if any). At present,
+        // afaik, there can be at most one. (In the future, if we
+        // adopted a more general `@` operator, there might be more
+        // than one, but it'd be very unusual to have two sides that
+        // both require tests; you'd expect one side to be simplified
+        // away.)
+        let (match_pair_index, match_pair) = candidate
+            .match_pairs
+            .iter()
+            .enumerate()
+            .find(|&(_, mp)| mp.place == Some(test_place))?;
+
+        // If true, the match pair is completely entailed by its corresponding test
+        // branch, so it can be removed. If false, the match pair is _compatible_
+        // with its test branch, but still needs a more specific test.
+        let fully_matched;
+        let ret = match (&test.kind, &match_pair.test_case) {
+            // If we are performing a variant switch, then this
+            // informs variant patterns, but nothing else.
+            (
+                &TestKind::Switch { adt_def: tested_adt_def },
+                &TestCase::Variant { adt_def, variant_index },
+            ) => {
+                assert_eq!(adt_def, tested_adt_def);
+                fully_matched = true;
+                Some(TestBranch::Variant(variant_index))
+            }
+
+            // If we are performing a switch over integers, then this informs integer
+            // equality, but nothing else.
+            //
+            // FIXME(#29623) we could use PatKind::Range to rule
+            // things out here, in some cases.
+            (TestKind::SwitchInt, &TestCase::Constant { value })
+                if is_switch_ty(match_pair.pattern.ty) =>
+            {
+                // An important invariant of candidate sorting is that a candidate
+                // must not match in multiple branches. For `SwitchInt` tests, adding
+                // a new value might invalidate that property for range patterns that
+                // have already been sorted into the failure arm, so we must take care
+                // not to add such values here.
+                let is_covering_range = |test_case: &TestCase<'_, 'tcx>| {
+                    test_case.as_range().is_some_and(|range| {
+                        matches!(
+                            range.contains(value, self.tcx, self.typing_env()),
+                            None | Some(true)
+                        )
+                    })
+                };
+                let is_conflicting_candidate = |candidate: &&mut Candidate<'_, 'tcx>| {
+                    candidate
+                        .match_pairs
+                        .iter()
+                        .any(|mp| mp.place == Some(test_place) && is_covering_range(&mp.test_case))
+                };
+                if sorted_candidates
+                    .get(&TestBranch::Failure)
+                    .is_some_and(|candidates| candidates.iter().any(is_conflicting_candidate))
+                {
+                    fully_matched = false;
+                    None
+                } else {
+                    fully_matched = true;
+                    let bits = value.eval_bits(self.tcx, self.typing_env());
+                    Some(TestBranch::Constant(value, bits))
+                }
+            }
+            (TestKind::SwitchInt, TestCase::Range(range)) => {
+                // When performing a `SwitchInt` test, a range pattern can be
+                // sorted into the failure arm if it doesn't contain _any_ of
+                // the values being tested. (This restricts what values can be
+                // added to the test by subsequent candidates.)
+                fully_matched = false;
+                let not_contained =
+                    sorted_candidates.keys().filter_map(|br| br.as_constant()).copied().all(
+                        |val| {
+                            matches!(range.contains(val, self.tcx, self.typing_env()), Some(false))
+                        },
+                    );
+
+                not_contained.then(|| {
+                    // No switch values are contained in the pattern range,
+                    // so the pattern can be matched only if this test fails.
+                    TestBranch::Failure
+                })
+            }
+
+            (TestKind::If, TestCase::Constant { value }) => {
+                fully_matched = true;
+                let value = value.try_eval_bool(self.tcx, self.typing_env()).unwrap_or_else(|| {
+                    span_bug!(test.span, "expected boolean value but got {value:?}")
+                });
+                Some(if value { TestBranch::Success } else { TestBranch::Failure })
+            }
+
+            (
+                &TestKind::Len { len: test_len, op: BinOp::Eq },
+                &TestCase::Slice { len, variable_length },
+            ) => {
+                match (test_len.cmp(&(len as u64)), variable_length) {
+                    (Ordering::Equal, false) => {
+                        // on true, min_len = len = $actual_length,
+                        // on false, len != $actual_length
+                        fully_matched = true;
+                        Some(TestBranch::Success)
+                    }
+                    (Ordering::Less, _) => {
+                        // test_len < pat_len. If $actual_len = test_len,
+                        // then $actual_len < pat_len and we don't have
+                        // enough elements.
+                        fully_matched = false;
+                        Some(TestBranch::Failure)
+                    }
+                    (Ordering::Equal | Ordering::Greater, true) => {
+                        // This can match both if $actual_len = test_len >= pat_len,
+                        // and if $actual_len > test_len. We can't advance.
+                        fully_matched = false;
+                        None
+                    }
+                    (Ordering::Greater, false) => {
+                        // test_len != pat_len, so if $actual_len = test_len, then
+                        // $actual_len != pat_len.
+                        fully_matched = false;
+                        Some(TestBranch::Failure)
+                    }
+                }
+            }
+            (
+                &TestKind::Len { len: test_len, op: BinOp::Ge },
+                &TestCase::Slice { len, variable_length },
+            ) => {
+                // the test is `$actual_len >= test_len`
+                match (test_len.cmp(&(len as u64)), variable_length) {
+                    (Ordering::Equal, true) => {
+                        // $actual_len >= test_len = pat_len,
+                        // so we can match.
+                        fully_matched = true;
+                        Some(TestBranch::Success)
+                    }
+                    (Ordering::Less, _) | (Ordering::Equal, false) => {
+                        // test_len <= pat_len. If $actual_len < test_len,
+                        // then it is also < pat_len, so the test passing is
+                        // necessary (but insufficient).
+                        fully_matched = false;
+                        Some(TestBranch::Success)
+                    }
+                    (Ordering::Greater, false) => {
+                        // test_len > pat_len. If $actual_len >= test_len > pat_len,
+                        // then we know we won't have a match.
+                        fully_matched = false;
+                        Some(TestBranch::Failure)
+                    }
+                    (Ordering::Greater, true) => {
+                        // test_len < pat_len, and is therefore less
+                        // strict. This can still go both ways.
+                        fully_matched = false;
+                        None
+                    }
+                }
+            }
+
+            (TestKind::Range(test), &TestCase::Range(pat)) => {
+                if test.as_ref() == pat {
+                    fully_matched = true;
+                    Some(TestBranch::Success)
+                } else {
+                    fully_matched = false;
+                    // If the testing range does not overlap with pattern range,
+                    // the pattern can be matched only if this test fails.
+                    if !test.overlaps(pat, self.tcx, self.typing_env())? {
+                        Some(TestBranch::Failure)
+                    } else {
+                        None
+                    }
+                }
+            }
+            (TestKind::Range(range), &TestCase::Constant { value }) => {
+                fully_matched = false;
+                if !range.contains(value, self.tcx, self.typing_env())? {
+                    // `value` is not contained in the testing range,
+                    // so `value` can be matched only if this test fails.
+                    Some(TestBranch::Failure)
+                } else {
+                    None
+                }
+            }
+
+            (TestKind::Eq { value: test_val, .. }, TestCase::Constant { value: case_val }) => {
+                if test_val == case_val {
+                    fully_matched = true;
+                    Some(TestBranch::Success)
+                } else {
+                    fully_matched = false;
+                    Some(TestBranch::Failure)
+                }
+            }
+
+            (TestKind::Deref { temp: test_temp, .. }, TestCase::Deref { temp, .. })
+                if test_temp == temp =>
+            {
+                fully_matched = true;
+                Some(TestBranch::Success)
+            }
+
+            (TestKind::Never, _) => {
+                fully_matched = true;
+                Some(TestBranch::Success)
+            }
+
+            (
+                TestKind::Switch { .. }
+                | TestKind::SwitchInt { .. }
+                | TestKind::If
+                | TestKind::Len { .. }
+                | TestKind::Range { .. }
+                | TestKind::Eq { .. }
+                | TestKind::Deref { .. },
+                _,
+            ) => {
+                fully_matched = false;
+                None
+            }
+        };
+
+        if fully_matched {
+            // Replace the match pair by its sub-pairs.
+            let match_pair = candidate.match_pairs.remove(match_pair_index);
+            candidate.match_pairs.extend(match_pair.subpairs);
+            // Move or-patterns to the end.
+            candidate.match_pairs.sort_by_key(|pair| matches!(pair.test_case, TestCase::Or { .. }));
+        }
+
+        ret
+    }
+}
+
+fn is_switch_ty(ty: Ty<'_>) -> bool {
+    ty.is_integral() || ty.is_char()
+}
+
+fn trait_method<'tcx>(
+    tcx: TyCtxt<'tcx>,
+    trait_def_id: DefId,
+    method_name: Symbol,
+    args: impl IntoIterator<Item: Into<GenericArg<'tcx>>>,
+) -> Const<'tcx> {
+    // The unhygienic comparison here is acceptable because this is only
+    // used on known traits.
+    let item = tcx
+        .associated_items(trait_def_id)
+        .filter_by_name_unhygienic(method_name)
+        .find(|item| item.kind == ty::AssocKind::Fn)
+        .expect("trait method not found");
+
+    let method_ty = Ty::new_fn_def(tcx, item.def_id, args);
+
+    Const::zero_sized(method_ty)
+}

compiler/rustc_mir_build/src/builder/matches/util.rs

@@ -0,0 +1,231 @@
+use rustc_data_structures::fx::FxIndexMap;
+use rustc_middle::mir::*;
+use rustc_middle::ty::Ty;
+use rustc_span::Span;
+use tracing::debug;
+
+use crate::builder::Builder;
+use crate::builder::expr::as_place::PlaceBase;
+use crate::builder::matches::{Binding, Candidate, FlatPat, MatchPairTree, TestCase};
+
+impl<'a, 'tcx> Builder<'a, 'tcx> {
+    /// Creates a false edge to `imaginary_target` and a real edge to
+    /// real_target. If `imaginary_target` is none, or is the same as the real
+    /// target, a Goto is generated instead to simplify the generated MIR.
+    pub(crate) fn false_edges(
+        &mut self,
+        from_block: BasicBlock,
+        real_target: BasicBlock,
+        imaginary_target: BasicBlock,
+        source_info: SourceInfo,
+    ) {
+        if imaginary_target != real_target {
+            self.cfg.terminate(from_block, source_info, TerminatorKind::FalseEdge {
+                real_target,
+                imaginary_target,
+            });
+        } else {
+            self.cfg.goto(from_block, source_info, real_target)
+        }
+    }
+}
+
+/// Determine the set of places that have to be stable across match guards.
+///
+/// Returns a list of places that need a fake borrow along with a local to store it.
+///
+/// Match exhaustiveness checking is not able to handle the case where the place being matched on is
+/// mutated in the guards. We add "fake borrows" to the guards that prevent any mutation of the
+/// place being matched. There are a some subtleties:
+///
+/// 1. Borrowing `*x` doesn't prevent assigning to `x`. If `x` is a shared reference, the borrow
+///    isn't even tracked. As such we have to add fake borrows of any prefixes of a place.
+/// 2. We don't want `match x { (Some(_), _) => (), .. }` to conflict with mutable borrows of `x.1`, so we
+///    only add fake borrows for places which are bound or tested by the match.
+/// 3. We don't want `match x { Some(_) => (), .. }` to conflict with mutable borrows of `(x as
+///    Some).0`, so the borrows are a special shallow borrow that only affects the place and not its
+///    projections.
+///    ```rust
+///    let mut x = (Some(0), true);
+///    match x {
+///        (Some(_), false) => {}
+///        _ if { if let Some(ref mut y) = x.0 { *y += 1 }; true } => {}
+///        _ => {}
+///    }
+///    ```
+/// 4. The fake borrows may be of places in inactive variants, e.g. here we need to fake borrow `x`
+///    and `(x as Some).0`, but when we reach the guard `x` may not be `Some`.
+///    ```rust
+///    let mut x = (Some(Some(0)), true);
+///    match x {
+///        (Some(Some(_)), false) => {}
+///        _ if { if let Some(Some(ref mut y)) = x.0 { *y += 1 }; true } => {}
+///        _ => {}
+///    }
+///    ```
+///    So it would be UB to generate code for the fake borrows. They therefore have to be removed by
+///    a MIR pass run after borrow checking.
+pub(super) fn collect_fake_borrows<'tcx>(
+    cx: &mut Builder<'_, 'tcx>,
+    candidates: &[Candidate<'_, 'tcx>],
+    temp_span: Span,
+    scrutinee_base: PlaceBase,
+) -> Vec<(Place<'tcx>, Local, FakeBorrowKind)> {
+    if candidates.iter().all(|candidate| !candidate.has_guard) {
+        // Fake borrows are only used when there is a guard.
+        return Vec::new();
+    }
+    let mut collector =
+        FakeBorrowCollector { cx, scrutinee_base, fake_borrows: FxIndexMap::default() };
+    for candidate in candidates.iter() {
+        collector.visit_candidate(candidate);
+    }
+    let fake_borrows = collector.fake_borrows;
+    debug!("add_fake_borrows fake_borrows = {:?}", fake_borrows);
+    let tcx = cx.tcx;
+    fake_borrows
+        .iter()
+        .map(|(matched_place, borrow_kind)| {
+            let fake_borrow_deref_ty = matched_place.ty(&cx.local_decls, tcx).ty;
+            let fake_borrow_ty =
+                Ty::new_imm_ref(tcx, tcx.lifetimes.re_erased, fake_borrow_deref_ty);
+            let mut fake_borrow_temp = LocalDecl::new(fake_borrow_ty, temp_span);
+            fake_borrow_temp.local_info = ClearCrossCrate::Set(Box::new(LocalInfo::FakeBorrow));
+            let fake_borrow_temp = cx.local_decls.push(fake_borrow_temp);
+            (*matched_place, fake_borrow_temp, *borrow_kind)
+        })
+        .collect()
+}
+
+pub(super) struct FakeBorrowCollector<'a, 'b, 'tcx> {
+    cx: &'a mut Builder<'b, 'tcx>,
+    /// Base of the scrutinee place. Used to distinguish bindings inside the scrutinee place from
+    /// bindings inside deref patterns.
+    scrutinee_base: PlaceBase,
+    /// Store for each place the kind of borrow to take. In case of conflicts, we take the strongest
+    /// borrow (i.e. Deep > Shallow).
+    /// Invariant: for any place in `fake_borrows`, all the prefixes of this place that are
+    /// dereferences are also borrowed with the same of stronger borrow kind.
+    fake_borrows: FxIndexMap<Place<'tcx>, FakeBorrowKind>,
+}
+
+impl<'a, 'b, 'tcx> FakeBorrowCollector<'a, 'b, 'tcx> {
+    // Fake borrow this place and its dereference prefixes.
+    fn fake_borrow(&mut self, place: Place<'tcx>, kind: FakeBorrowKind) {
+        if self.fake_borrows.get(&place).is_some_and(|k| *k >= kind) {
+            return;
+        }
+        self.fake_borrows.insert(place, kind);
+        // Also fake borrow the prefixes of any fake borrow.
+        self.fake_borrow_deref_prefixes(place, kind);
+    }
+
+    // Fake borrow the prefixes of this place that are dereferences.
+    fn fake_borrow_deref_prefixes(&mut self, place: Place<'tcx>, kind: FakeBorrowKind) {
+        for (place_ref, elem) in place.as_ref().iter_projections().rev() {
+            if let ProjectionElem::Deref = elem {
+                // Insert a shallow borrow after a deref. For other projections the borrow of
+                // `place_ref` will conflict with any mutation of `place.base`.
+                let place = place_ref.to_place(self.cx.tcx);
+                if self.fake_borrows.get(&place).is_some_and(|k| *k >= kind) {
+                    return;
+                }
+                self.fake_borrows.insert(place, kind);
+            }
+        }
+    }
+
+    fn visit_candidate(&mut self, candidate: &Candidate<'_, 'tcx>) {
+        for binding in &candidate.extra_data.bindings {
+            self.visit_binding(binding);
+        }
+        for match_pair in &candidate.match_pairs {
+            self.visit_match_pair(match_pair);
+        }
+    }
+
+    fn visit_flat_pat(&mut self, flat_pat: &FlatPat<'_, 'tcx>) {
+        for binding in &flat_pat.extra_data.bindings {
+            self.visit_binding(binding);
+        }
+        for match_pair in &flat_pat.match_pairs {
+            self.visit_match_pair(match_pair);
+        }
+    }
+
+    fn visit_match_pair(&mut self, match_pair: &MatchPairTree<'_, 'tcx>) {
+        if let TestCase::Or { pats, .. } = &match_pair.test_case {
+            for flat_pat in pats.iter() {
+                self.visit_flat_pat(flat_pat)
+            }
+        } else if matches!(match_pair.test_case, TestCase::Deref { .. }) {
+            // The subpairs of a deref pattern are all places relative to the deref temporary, so we
+            // don't fake borrow them. Problem is, if we only shallowly fake-borrowed
+            // `match_pair.place`, this would allow:
+            // ```
+            // let mut b = Box::new(false);
+            // match b {
+            //     deref!(true) => {} // not reached because `*b == false`
+            //     _ if { *b = true; false } => {} // not reached because the guard is `false`
+            //     deref!(false) => {} // not reached because the guard changed it
+            //     // UB because we reached the unreachable.
+            // }
+            // ```
+            // Hence we fake borrow using a deep borrow.
+            if let Some(place) = match_pair.place {
+                self.fake_borrow(place, FakeBorrowKind::Deep);
+            }
+        } else {
+            // Insert a Shallow borrow of any place that is switched on.
+            if let Some(place) = match_pair.place {
+                self.fake_borrow(place, FakeBorrowKind::Shallow);
+            }
+
+            for subpair in &match_pair.subpairs {
+                self.visit_match_pair(subpair);
+            }
+        }
+    }
+
+    fn visit_binding(&mut self, Binding { source, .. }: &Binding<'tcx>) {
+        if let PlaceBase::Local(l) = self.scrutinee_base
+            && l != source.local
+        {
+            // The base of this place is a temporary created for deref patterns. We don't emit fake
+            // borrows for these as they are not initialized in all branches.
+            return;
+        }
+
+        // Insert a borrows of prefixes of places that are bound and are
+        // behind a dereference projection.
+        //
+        // These borrows are taken to avoid situations like the following:
+        //
+        // match x[10] {
+        //     _ if { x = &[0]; false } => (),
+        //     y => (), // Out of bounds array access!
+        // }
+        //
+        // match *x {
+        //     // y is bound by reference in the guard and then by copy in the
+        //     // arm, so y is 2 in the arm!
+        //     y if { y == 1 && (x = &2) == () } => y,
+        //     _ => 3,
+        // }
+        //
+        // We don't just fake borrow the whole place because this is allowed:
+        // match u {
+        //     _ if { u = true; false } => (),
+        //     x => (),
+        // }
+        self.fake_borrow_deref_prefixes(*source, FakeBorrowKind::Shallow);
+    }
+}
+
+#[must_use]
+pub(crate) fn ref_pat_borrow_kind(ref_mutability: Mutability) -> BorrowKind {
+    match ref_mutability {
+        Mutability::Mut => BorrowKind::Mut { kind: MutBorrowKind::Default },
+        Mutability::Not => BorrowKind::Shared,
+    }
+}

compiler/rustc_mir_build/src/builder/misc.rs

@@ -0,0 +1,65 @@
+//! Miscellaneous builder routines that are not specific to building any particular
+//! kind of thing.
+
+use rustc_middle::mir::*;
+use rustc_middle::ty::{self, Ty};
+use rustc_span::Span;
+use rustc_trait_selection::infer::InferCtxtExt;
+use tracing::debug;
+
+use crate::builder::Builder;
+
+impl<'a, 'tcx> Builder<'a, 'tcx> {
+    /// Adds a new temporary value of type `ty` storing the result of
+    /// evaluating `expr`.
+    ///
+    /// N.B., **No cleanup is scheduled for this temporary.** You should
+    /// call `schedule_drop` once the temporary is initialized.
+    pub(crate) fn temp(&mut self, ty: Ty<'tcx>, span: Span) -> Place<'tcx> {
+        let temp = self.local_decls.push(LocalDecl::new(ty, span));
+        let place = Place::from(temp);
+        debug!("temp: created temp {:?} with type {:?}", place, self.local_decls[temp].ty);
+        place
+    }
+
+    /// Convenience function for creating a literal operand, one
+    /// without any user type annotation.
+    pub(crate) fn literal_operand(&mut self, span: Span, const_: Const<'tcx>) -> Operand<'tcx> {
+        let constant = Box::new(ConstOperand { span, user_ty: None, const_ });
+        Operand::Constant(constant)
+    }
+
+    /// Returns a zero literal operand for the appropriate type, works for
+    /// bool, char and integers.
+    pub(crate) fn zero_literal(&mut self, span: Span, ty: Ty<'tcx>) -> Operand<'tcx> {
+        let literal = Const::from_bits(self.tcx, 0, ty::TypingEnv::fully_monomorphized(), ty);
+
+        self.literal_operand(span, literal)
+    }
+
+    pub(crate) fn push_usize(
+        &mut self,
+        block: BasicBlock,
+        source_info: SourceInfo,
+        value: u64,
+    ) -> Place<'tcx> {
+        let usize_ty = self.tcx.types.usize;
+        let temp = self.temp(usize_ty, source_info.span);
+        self.cfg.push_assign_constant(block, source_info, temp, ConstOperand {
+            span: source_info.span,
+            user_ty: None,
+            const_: Const::from_usize(self.tcx, value),
+        });
+        temp
+    }
+
+    pub(crate) fn consume_by_copy_or_move(&self, place: Place<'tcx>) -> Operand<'tcx> {
+        let tcx = self.tcx;
+        let ty = place.ty(&self.local_decls, tcx).ty;
+        if !self.infcx.type_is_copy_modulo_regions(self.param_env, ty) {
+            Operand::Move(place)
+        } else {
+            Operand::Copy(place)
+        }
+    }
+}