From: David Woodhouse <[email protected]> All derived from the reference host's behaviour, and verified against its VSM unit-test suite (still green) and Windows Server 2025 VBS boot:
- Forced entry into a higher plane is edge-triggered, as on the reference host, which attempts one forced entry per posted interrupt. Delivery of a fixed vector or NMI to a suspended higher plane sets hv_apic_event_pending; the evaluation triggers on that marker and the consumable requests (stimer, NMI, APIC timer), not on a scan of the plane's IRR — a vector still latched after a refused entry waits for the target's next cooperative entry or next posted event instead of retrying on every run of the lower plane. The marker is cleared only by a run that actually reaches event evaluation; a run aborted by the VTL-exit or KVM_REQ_PLANE_RESCHED bail-outs leaves it set, since delivery never re-arms it. A vector masked by the higher VTL's TPR still forces exactly one entry. - One exception to single-shot: a plane that suspends (VtlReturn or the run-state ioctl) with a vector still latched in its IRR re-arms its marker, via kvm_arch_vcpu_plane_stopped() from kvm_vcpu_set_plane_stopped(). Evaluating does not imply delivering: Windows hibernation posts vector 0xE0 to a twin mid-run with IF clear, the run consumes the marker without injecting, and the suspended twin's vector is stranded — VTL0 freezes all processors, spins on the secure kernel and wedges the VM at 100% host CPU. A re-armed marker cannot storm: the deliverability gate still applies, and a plane that reached suspension has run before, so userspace can enter it again. - NMIs aimed at a higher plane fold back to plane 0 for delivery after the VTL returns; the higher plane only sees the forced entry. A real NMI inside VTL1 bugchecks the secure kernel (0x18B). - VTL1 exempted from the reserved-bits check on Hyper-V vAPIC MSR writes: the secure kernel writes HV_X64_MSR_EOI with stale upper bits, and the reference host truncates for VTL1. - KVM_REQ_HV_TLB_FLUSH made per-plane: a shared request could be consumed by the running twin, leaving the targeted plane's TLB stale. - Lower-twin kicks on the stimer retry (vcpu_kick=false) paths. Signed-off-by: David Woodhouse <[email protected]> Assisted-by: LLM --- arch/x86/include/asm/kvm_host.h | 12 ++++- arch/x86/kvm/hyperv.c | 3 +- arch/x86/kvm/lapic.c | 39 ++++++++++++++- arch/x86/kvm/x86.c | 89 +++++++++++++++++++++++++++++---- include/linux/kvm_host.h | 1 + virt/kvm/kvm_main.c | 5 ++ 6 files changed, 135 insertions(+), 14 deletions(-) diff --git a/arch/x86/include/asm/kvm_host.h b/arch/x86/include/asm/kvm_host.h index c54f47c52a59..834597791fe1 100644 --- a/arch/x86/include/asm/kvm_host.h +++ b/arch/x86/include/asm/kvm_host.h @@ -127,7 +127,8 @@ #define KVM_REQ_MMU_FREE_OBSOLETE_ROOTS \ KVM_ARCH_REQ_FLAGS(31, KVM_REQUEST_WAIT | KVM_REQUEST_NO_WAKEUP) #define KVM_REQ_HV_TLB_FLUSH \ - KVM_ARCH_REQ_FLAGS(32, KVM_REQUEST_WAIT | KVM_REQUEST_NO_WAKEUP) + KVM_ARCH_REQ_FLAGS(32, KVM_REQUEST_WAIT | KVM_REQUEST_NO_WAKEUP | \ + KVM_REQUEST_PER_PLANE) #define KVM_REQ_UPDATE_PROTECTED_GUEST_STATE \ KVM_ARCH_REQ_FLAGS(34, KVM_REQUEST_WAIT | KVM_REQUEST_PER_PLANE) @@ -854,6 +855,15 @@ int kvm_arch_vcpu_common_init(struct kvm_vcpu_common *common); void kvm_arch_vcpu_common_destroy(struct kvm_vcpu_common *common); struct kvm_vcpu_arch { + /* + * Hyper-V VSM: an APIC event (fixed vector or NMI) was delivered to + * this plane while it was suspended. Set at delivery, cleared when + * the plane next runs; a lower plane's run loop uses it to force a + * VTL entry even when the vector is currently masked by this + * plane's TPR (matching the reference host's apic_pending_vtls). + */ + bool hv_apic_event_pending; + /* * rip and regs accesses must go through * kvm_{register,rip}_{read,write} functions. diff --git a/arch/x86/kvm/hyperv.c b/arch/x86/kvm/hyperv.c index d4e44ae3f6e8..0d239f054856 100644 --- a/arch/x86/kvm/hyperv.c +++ b/arch/x86/kvm/hyperv.c @@ -605,8 +605,9 @@ static void stimer_mark_pending(struct kvm_vcpu_hv_stimer *stimer, * A pending stimer on a suspended higher plane is only processed * once that plane runs; kick the lower twins so the running one * re-evaluates pending higher-plane events and forces entry. + * This applies to retry paths too (vcpu_kick == false). */ - if (vcpu_kick && vcpu->plane_level) { + if (vcpu->plane_level) { struct kvm_vcpu *twin; int p; diff --git a/arch/x86/kvm/lapic.c b/arch/x86/kvm/lapic.c index f89156c26972..32cace3c89f4 100644 --- a/arch/x86/kvm/lapic.c +++ b/arch/x86/kvm/lapic.c @@ -1448,6 +1448,7 @@ static int __apic_accept_irq(struct kvm_lapic *apic, int delivery_mode, struct kvm_vcpu *twin; int p; + WRITE_ONCE(vcpu->arch.hv_apic_event_pending, true); for (p = 0; p < vcpu->plane_level; p++) { twin = vcpu->common->vcpus[p]; if (twin) { @@ -1474,8 +1475,31 @@ static int __apic_accept_irq(struct kvm_lapic *apic, int delivery_mode, case APIC_DM_NMI: result = 1; - kvm_inject_nmi(vcpu); - kvm_vcpu_kick(vcpu); + /* + * An NMI aimed at a higher plane forces entry into that + * plane (reason INTERRUPT, below) but the NMI itself is + * folded back to plane 0: the reference host never delivers + * an NMI inside a higher VTL, it re-queues it to VTL0's + * context for delivery after the VTL returns. + */ + if (vcpu->plane_level) { + struct kvm_vcpu *twin; + int p; + + WRITE_ONCE(vcpu->arch.hv_apic_event_pending, true); + if (vcpu->common->vcpus[0]) + kvm_inject_nmi(vcpu->common->vcpus[0]); + for (p = 0; p < vcpu->plane_level; p++) { + twin = vcpu->common->vcpus[p]; + if (twin) { + kvm_make_request(KVM_REQ_UNBLOCK, twin); + kvm_vcpu_kick(twin); + } + } + } else { + kvm_inject_nmi(vcpu); + kvm_vcpu_kick(vcpu); + } break; case APIC_DM_INIT: @@ -3535,6 +3559,17 @@ int kvm_hv_vapic_msr_write(struct kvm_vcpu *vcpu, u32 reg, u64 data) if (!lapic_in_kernel(vcpu)) return 1; + /* + * Bits 63:32 are reserved in non-ICR registers, but the Windows + * secure kernel leaves stale data in the high bits when writing + * the Hyper-V vAPIC MSRs from a higher VTL. The reference host + * exempts those writes and truncates; a #GP here is a VTL1 + * bugcheck. The exemption covers only this PV path: a native + * x2APIC WRMSR takes the architectural #GP whatever the plane. + */ + if (vcpu->plane_level && reg != APIC_ICR) + data = (u32)data; + return kvm_lapic_msr_write(vcpu->arch.apic, reg, data); } diff --git a/arch/x86/kvm/x86.c b/arch/x86/kvm/x86.c index 10e14ab3e2ca..3e2465b2fa2b 100644 --- a/arch/x86/kvm/x86.c +++ b/arch/x86/kvm/x86.c @@ -11229,18 +11229,26 @@ static struct kvm_vcpu *kvm_higher_plane_with_event(struct kvm_vcpu *vcpu) continue; #endif /* - * Test raw pending-ness, not injectability: - * kvm_cpu_has_injectable_intr() reports false on an - * APICv-active vCPU, and a suspended plane may carry a - * stale apicv_active until it next runs. A pending - * synthetic-timer request counts as well: it can only be - * processed (and its interrupt delivered) once the higher - * plane runs, so it would otherwise never fire while the - * plane is suspended. + * Forced entry is edge-triggered on the posting of an + * event, not level-triggered on the target's IRR, matching + * the reference host's apic_pending_vtls bitmap: one entry + * attempt per posted interrupt, via the hv_apic_event_pending + * marker __apic_accept_irq sets. A vector that stays + * latched after a refused attempt (no VP assist page yet) + * or that was masked by the plane's TPR waits for the next + * cooperative entry or the next posted event; scanning the + * IRR here instead would replay it the moment the assist + * page is re-enabled. A pending synthetic-timer request + * counts as well: it can only be processed (and its + * interrupt delivered) once the higher plane runs, so it + * would otherwise never fire while the plane is suspended. */ - if (kvm_cpu_has_extint(higher) || - kvm_apic_has_interrupt(higher) != -1 || + if (READ_ONCE(higher->arch.hv_apic_event_pending) || + kvm_cpu_has_extint(higher) || kvm_test_request(KVM_REQ_HV_STIMER, higher) || + kvm_test_request(KVM_REQ_NMI, higher) || + atomic_read(&higher->arch.nmi_queued) || + higher->arch.nmi_pending || (higher->arch.apic && atomic_read(&higher->arch.apic->lapic_timer.pending))) return higher; @@ -11265,6 +11273,14 @@ static bool kvm_hv_vtl_interrupt_exit(struct kvm_vcpu *vcpu) if (!higher) return false; + /* + * Single-shot, as in the reference host: if userspace refuses + * the entry (e.g. the target VTL has no VP assist page yet), + * the event waits in the target's IRR for a cooperative entry + * rather than re-triggering an exit storm. + */ + WRITE_ONCE(higher->arch.hv_apic_event_pending, false); + vcpu->run->exit_reason = KVM_EXIT_HYPERV; vcpu->run->hyperv.type = KVM_EXIT_HYPERV_VTL_INTERRUPT; vcpu->run->hyperv.plane = higher->plane_level; @@ -11286,6 +11302,50 @@ static inline bool kvm_check_plane0_events(struct kvm_vcpu *vcpu) vcpu_plane0->arch.smi_pending; } +/* + * A higher plane that suspends with an interrupt still latched in its IRR + * must keep its hv_apic_event_pending marker armed. The running plane + * consumes the marker in vcpu_enter_guest() before evaluating events, but + * evaluating does not imply delivering: a plane that keeps IF clear until + * it suspends (e.g. the secure kernel finishing a VTL round trip), or + * whose PPR masks the vector for the whole run, parks the vector in its + * IRR with the marker already consumed. Nothing re-posts the interrupt, + * so the deliverability-gated forced entry in kvm_higher_plane_with_event() + * would never fire again and the plane would never see the vector; a lower + * VTL spinning on the suspended VTL's response then wedges the VM. + * + * Re-arming cannot re-create the refused-entry exit storm the single-shot + * marker guards against: the forced-entry gate still requires the vector + * to be deliverable by the plane's own priority state. + */ +void kvm_arch_vcpu_plane_stopped(struct kvm_vcpu *vcpu) +{ + /* + * The hv_apic_event_pending marker drives the Hyper-V VSM forced + * entry (see kvm_higher_plane_with_event()). This hook also runs + * from __sev_snp_run_vmpl() for guest-architectural VMPL switches: + * without the Hyper-V gate, every SNP VMPL switch away from a + * plane with a latched IRR would feed VSM-flavoured wakeups and + * exits to a VM that has no VSM at all. + */ + if (!vcpu->arch.hyperv_enabled) + return; + + /* + * A plane that reached suspension normally has run before; the + * exception is userspace parking a never-run plane it had already + * injected an interrupt into. Forced entry into a never-run + * plane is always refused, so don't arm an attempt that cannot + * succeed; the IRR is evaluated on the plane's first real entry. + */ + if (vcpu->arch.last_vmentry_cpu == -1) + return; + + if (vcpu->plane_level && lapic_in_kernel(vcpu) && + kvm_lapic_find_highest_irr(vcpu) != -1) + WRITE_ONCE(vcpu->arch.hv_apic_event_pending, true); +} + static void process_nmi(struct kvm_vcpu *vcpu) { unsigned int limit; @@ -11771,6 +11831,15 @@ static int vcpu_enter_guest(struct kvm_vcpu *vcpu) goto out; } + /* + * This plane is running and about to evaluate its events: its + * pended-while-suspended marker is consumed. The clear must come + * after the VTL-exit and plane-reschedule bail-outs above: a run + * aborted there never evaluates the pended event, and nothing + * re-arms the marker for a TPR-masked vector. + */ + WRITE_ONCE(vcpu->arch.hv_apic_event_pending, false); + if (kvm_check_request(KVM_REQ_EVENT, vcpu) || req_int_win || kvm_xen_has_interrupt(vcpu)) { ++vcpu->stat.req_event; diff --git a/include/linux/kvm_host.h b/include/linux/kvm_host.h index cb8c4798df12..30dbc28885fb 100644 --- a/include/linux/kvm_host.h +++ b/include/linux/kvm_host.h @@ -493,6 +493,7 @@ static inline void kvm_vcpu_cancel_vtl_fpu_carry(struct kvm_vcpu_common *common) void kvm_vcpu_set_plane_runnable(struct kvm_vcpu *vcpu); void kvm_vcpu_set_plane_stopped(struct kvm_vcpu *vcpu); +void kvm_arch_vcpu_plane_stopped(struct kvm_vcpu *vcpu); struct kvm_vcpu *kvm_vcpu_select_plane(struct kvm_vcpu *vcpu); static inline bool kvm_vcpu_wants_to_run(struct kvm_vcpu *vcpu) diff --git a/virt/kvm/kvm_main.c b/virt/kvm/kvm_main.c index 5856672bbf6d..fac30b8b6518 100644 --- a/virt/kvm/kvm_main.c +++ b/virt/kvm/kvm_main.c @@ -5347,9 +5347,14 @@ void kvm_vcpu_set_plane_runnable(struct kvm_vcpu *vcpu) } EXPORT_SYMBOL_FOR_KVM_INTERNAL(kvm_vcpu_set_plane_runnable); +void __weak kvm_arch_vcpu_plane_stopped(struct kvm_vcpu *vcpu) +{ +} + void kvm_vcpu_set_plane_stopped(struct kvm_vcpu *vcpu) { vcpu->plane_state = STOPPED; + kvm_arch_vcpu_plane_stopped(vcpu); kvm_make_request(KVM_REQ_PLANE_RESCHED, vcpu); } EXPORT_SYMBOL_FOR_KVM_INTERNAL(kvm_vcpu_set_plane_stopped); -- 2.50.1
