Implement standalone Caretaker EL2 exception decoding, system register trap handling, and stage-2 non-RAM IPA yield in arch/arm64/kvm/caretaker.c.
Signed-off-by: Pasha Tatashin <[email protected]> --- arch/arm64/kvm/caretaker.c | 408 ++++++++++++++++++++++++++++++++++++- 1 file changed, 405 insertions(+), 3 deletions(-) diff --git a/arch/arm64/kvm/caretaker.c b/arch/arm64/kvm/caretaker.c index 65c11fe8c2fa..6980c1f6b998 100644 --- a/arch/arm64/kvm/caretaker.c +++ b/arch/arm64/kvm/caretaker.c @@ -60,10 +60,8 @@ arm64_caretaker_handle_invalid(u64 elr, u64 esr, u64 far) } while (1) { - if (cpu_preserved_should_exit(cpu)) { - cpu_preserved_set_dead(cpu); + if (cpu_preserved_should_exit(cpu)) arch_cpu_preserved_park_finish(cpu); - } arch_cpu_preserved_park_wait(); } } @@ -519,3 +517,407 @@ __caretaker_text static void caretaker_vgic_v3_save(struct vgic_v3_cpu_if *cpu_i isb(); } +__caretaker_text static void +caretaker_arm64_inject_sgi(struct caretaker_arm64_page *target_cap, u32 sgi) +{ + int slot = -1; + int i; + + if (!target_cap) + return; + + cpu_preserved_inval(target_cap); + + /* Check if the SGI is already pending or active */ + for (i = 0; i < target_cap->ctx.vgic_v3.used_lrs; i++) { + u64 lr = target_cap->ctx.vgic_v3.vgic_lr[i]; + + if ((lr & ICH_LR_VIRTUAL_ID_MASK) == (sgi & 0xf) && (lr & ICH_LR_STATE)) + return; + if ((lr & ICH_LR_STATE) == 0 && slot < 0) + slot = i; + } + + if (slot < 0 && target_cap->ctx.vgic_v3.used_lrs < VGIC_V3_MAX_LRS) { + slot = target_cap->ctx.vgic_v3.used_lrs; + target_cap->ctx.vgic_v3.used_lrs++; + } + + if (slot >= 0) { + target_cap->ctx.vgic_v3.vgic_lr[slot] = + ((u64)sgi & 0xf) | + ICH_LR_PENDING_BIT | + ICH_LR_GROUP | + (GIC_DEFAULT_SGI_PRIO << ICH_LR_PRIORITY_SHIFT); + } else { + target_cap->ctx.pending_sgis |= BIT(sgi & 0xf); + } + + cpu_preserved_clean(target_cap); + + /* If target vCPU is running on a remote physical CPU, kick it */ + if (target_cap->abi.cb.pcpu_id >= 0 && + target_cap->abi.cb.pcpu_id != arm64_caretaker_get_pcpu()) { + arch_cpu_preserved_kick(target_cap->abi.cb.pcpu_id); + } +} + +__caretaker_text static void +caretaker_arm64_handle_sgi(struct caretaker_arm64_page *src_cap, u64 reg) +{ + struct caretaker_arm64_page *target; + u32 sgi = FIELD_GET(ICC_SGI1R_SGI_ID_MASK, reg); + unsigned int i; + + if (!src_cap || !src_cap->next_vcpu) + return; + + if (reg & BIT_ULL(ICC_SGI1R_IRQ_ROUTING_MODE_BIT)) { + /* Broadcast to all other vCPUs */ + for (target = src_cap->next_vcpu; + target && target != src_cap; + target = target->next_vcpu) { + cpu_preserved_inval(target); + caretaker_arm64_inject_sgi(target, sgi); + } + } else { + u64 aff3 = FIELD_GET(ICC_SGI1R_AFFINITY_3_MASK, reg); + u64 aff2 = FIELD_GET(ICC_SGI1R_AFFINITY_2_MASK, reg); + u64 aff1 = FIELD_GET(ICC_SGI1R_AFFINITY_1_MASK, reg); + u64 rs = FIELD_GET(ICC_SGI1R_RS_MASK, reg); + u64 cluster_mpidr = (aff3 << MPIDR_LEVEL_SHIFT(3)) | + (aff2 << MPIDR_LEVEL_SHIFT(2)) | + (aff1 << MPIDR_LEVEL_SHIFT(1)); + u64 target_list = FIELD_GET(ICC_SGI1R_TARGET_LIST_MASK, reg); + + for (i = 0; i < 16; i++) { + u64 target_mpidr; + + if (!(target_list & BIT(i))) + continue; + + target_mpidr = cluster_mpidr | + ((rs * 16 + i) << MPIDR_LEVEL_SHIFT(0)); + + target = src_cap; + do { + cpu_preserved_inval(target); + if ((ctxt_sys_reg(&target->ctx.ctxt, MPIDR_EL1) & + MPIDR_HWID_BITMASK) == target_mpidr) { + caretaker_arm64_inject_sgi(target, sgi); + break; + } + target = target->next_vcpu; + } while (target && target != src_cap); + } + } +} + +static __caretaker_text int arm64_caretaker_op_enter(void *data) +{ + struct caretaker_arm64_page *cap = data; + u64 guest_hcr; + + gicv3_caretaker_clear_active_priorities(); + write_sysreg_s(ICC_PMR_EL1_MASK, SYS_ICC_PMR_EL1); + write_sysreg_s(ICC_CTLR_EL1_EOImode_drop, SYS_ICC_CTLR_EL1); + write_sysreg_s(ICC_IGRPEN1_EL1_MASK, SYS_ICC_IGRPEN1_EL1); + pmr_sync(); + + guest_hcr = (cap->ctx.hcr_el2 | HCR_AMO | HCR_IMO | HCR_FMO | HCR_E2H) & ~HCR_TGE; + write_sysreg_hcr(guest_hcr); + isb(); + + cap->last_ret = caretaker_guest_enter(&cap->ctx); + + write_sysreg_hcr(HCR_HOST_VHE_FLAGS); + isb(); + + return 0; +} + +static __caretaker_text void +arm64_caretaker_op_arm_timer(void *data, u64 deadline_ticks) +{ + if (deadline_ticks) { + write_sysreg_s(deadline_ticks, SYS_CNTHP_CVAL_EL2); + isb(); + write_sysreg_s(1, SYS_CNTHP_CTL_EL2); + } else { + write_sysreg_s(0, SYS_CNTHP_CTL_EL2); + } + isb(); +} + +static __caretaker_text void +arm64_caretaker_op_disarm_timer(void *data) +{ + write_sysreg_s(0, SYS_CNTHP_CTL_EL2); + isb(); +} + +static __caretaker_text void +arm64_caretaker_op_decode_exit(void *data, struct kvm_caretaker_exit *exit) +{ + struct caretaker_arm64_page *cap = data; + u64 ret = cap->last_ret; + + exit->rip = cap->ctx.ctxt.regs.pc; + exit->insn_len = 0; + exit->type = KVM_CARETAKER_EXIT_UNKNOWN; + + if (ARM_EXCEPTION_CODE(ret) == ARM_EXCEPTION_IRQ) { + caretaker_gic_drain_iar(); + gicv3_caretaker_clear_active_priorities(); + dsb(sy); + isb(); + + exit->type = KVM_CARETAKER_EXIT_PREEMPT_TIMER; + return; + } + + if (ARM_EXCEPTION_IS_TRAP(ret)) { + u64 esr = cap->ctx.fault.esr_el2; + u8 ec = ESR_ELx_EC(esr); + + exit->insn_len = 4; + + if (ec == ESR_ELx_EC_WFx) { + exit->type = KVM_CARETAKER_EXIT_IDLE; + return; + } + + if (ec == ESR_ELx_EC_SYS64) { + u32 iss = ESR_ELx_ISS(esr); + u32 sys_op = iss & ESR_ELx_SYS64_ISS_SYS_OP_MASK; + + if (sys_op == ESR_ELx_SYS64_ISS_SYS_ICC_SGI1R_EL1 || + sys_op == ESR_ELx_SYS64_ISS_SYS_ICC_ASGI1R_EL1 || + sys_op == ESR_ELx_SYS64_ISS_SYS_ICC_SGI0R_EL1) { + u32 rt = ESR_ELx_SYS64_ISS_RT(esr); + u64 val = (rt < 31) ? cap->ctx.ctxt.regs.regs[rt] : 0; + + exit->type = KVM_CARETAKER_EXIT_CROSS_VCPU; + exit->sgi.sgi_id = FIELD_GET(ICC_SGI1R_SGI_ID_MASK, val); + exit->sgi.target_mask = val; + return; + } + } + + if (ec == ESR_ELx_EC_DABT_LOW || ec == ESR_ELx_EC_IABT_LOW) { + /* + * Stage-2 abort on a non-RAM IPA (e.g. MMIO device). + * Do not advance PC: yield this vCPU out of guest mode + * so the incoming kernel's real KVM + VMM handles the + * fault on re-attachment. + */ + exit->type = KVM_CARETAKER_EXIT_IDLE; + exit->insn_len = 0; + return; + } + + exit->type = KVM_CARETAKER_EXIT_ARCH; + exit->raw_reason = esr; + return; + } + + exit->type = KVM_CARETAKER_EXIT_ARCH; +} + +static __caretaker_text void +arm64_caretaker_op_advance_rip(void *data, u64 next_rip) +{ + struct caretaker_arm64_page *cap = data; + + cap->ctx.ctxt.regs.pc = next_rip; +} + +static __caretaker_text bool +arm64_caretaker_op_handle_exit(void *data, struct kvm_caretaker_exit *exit) +{ + struct caretaker_arm64_page *cap = data; + + if (exit->type == KVM_CARETAKER_EXIT_CROSS_VCPU) { + caretaker_arm64_handle_sgi(cap, exit->sgi.target_mask); + exit->rip += exit->insn_len; + return true; + } + + /* + * KVM_CARETAKER_EXIT_ARCH is the fallback type assigned by + * arm64_caretaker_op_decode_exit() to every trap it did not decode, + * including system register accesses. Skipping the instruction here + * would leave the destination register holding whatever it held + * before the trap and bypass KVM's ID register sanitisation, with the + * guest none the wiser. + * + * Leave PC pointing at the trapping instruction and report the exit + * as unhandled so the vCPU parks until the incoming kernel reclaims + * it and full KVM handles the trap. + */ + return false; +} + +static __caretaker_text inline u64 arm64_sysreg_to_uapi_id(u32 reg) +{ + if (reg == SYS_CNTV_CVAL_EL0) + return KVM_REG_ARM_TIMER_CVAL; + return (KVM_REG_ARM64 | KVM_REG_SIZE_U64 | + KVM_REG_ARM64_SYSREG | + ((u64)sys_reg_Op0(reg) << KVM_REG_ARM64_SYSREG_OP0_SHIFT) | + ((u64)sys_reg_Op1(reg) << KVM_REG_ARM64_SYSREG_OP1_SHIFT) | + ((u64)sys_reg_CRn(reg) << KVM_REG_ARM64_SYSREG_CRN_SHIFT) | + ((u64)sys_reg_CRm(reg) << KVM_REG_ARM64_SYSREG_CRM_SHIFT) | + ((u64)sys_reg_Op2(reg) << KVM_REG_ARM64_SYSREG_OP2_SHIFT)); +} + +static __caretaker_text void +arm64_caretaker_update_sysreg(struct kvm_vcpu_arch_ser *state, + u32 reg, u64 val) +{ + u64 id = arm64_sysreg_to_uapi_id(reg); + u32 i; + + for (i = 0; i < state->num_sysregs; i++) { + if (state->sysregs[i].id == id) { + state->sysregs[i].addr = val; + return; + } + } +} + +static __caretaker_text void +arm64_caretaker_detach_serialize(struct caretaker_arm64_page *cap) +{ + cap->abi.vgic_initialized = cap->ctx.vgic_initialized ? 1 : 0; + cap->abi.cntvoff_el2 = cap->ctx.cntvoff_el2; + cap->abi.hcr_el2 = cap->ctx.hcr_el2; + cap->abi.mdcr_el2 = cap->ctx.mdcr_el2; + cap->abi.cflags = (u32)cap->ctx.cflags; + if (cap->ctx.vgic_initialized) { + int i; + + cap->abi.used_lrs = cap->ctx.vgic_v3.used_lrs; + cap->abi.vgic_hcr = cap->ctx.vgic_v3.vgic_hcr; + cap->abi.vgic_vmcr = cap->ctx.vgic_v3.vgic_vmcr; + for (i = 0; i < 4; i++) { + cap->abi.vgic_ap0r[i] = cap->ctx.vgic_v3.vgic_ap0r[i]; + cap->abi.vgic_ap1r[i] = cap->ctx.vgic_v3.vgic_ap1r[i]; + } + for (i = 0; i < 16; i++) + cap->abi.vgic_lr[i] = cap->ctx.vgic_v3.vgic_lr[i]; + } + + if (cap->arch_state) { + struct kvm_vcpu_arch_ser *state = cap->arch_state; + + cpu_preserved_memcpy(&state->regs.regs, &cap->ctx.ctxt.regs, + sizeof(state->regs.regs)); + state->regs.sp_el1 = ctxt_sys_reg(&cap->ctx.ctxt, SP_EL1); + state->regs.elr_el1 = ctxt_sys_reg(&cap->ctx.ctxt, ELR_EL1); + state->regs.spsr[KVM_SPSR_EL1] = ctxt_sys_reg(&cap->ctx.ctxt, SPSR_EL1); + state->regs.spsr[KVM_SPSR_ABT] = cap->ctx.ctxt.spsr_abt; + state->regs.spsr[KVM_SPSR_UND] = cap->ctx.ctxt.spsr_und; + state->regs.spsr[KVM_SPSR_IRQ] = cap->ctx.ctxt.spsr_irq; + state->regs.spsr[KVM_SPSR_FIQ] = cap->ctx.ctxt.spsr_fiq; + cpu_preserved_memcpy(&state->regs.fp_regs, &cap->ctx.ctxt.fp_regs, + sizeof(state->regs.fp_regs)); + + arm64_caretaker_update_sysreg(state, SYS_SCTLR_EL1, + ctxt_sys_reg(&cap->ctx.ctxt, SCTLR_EL1)); + arm64_caretaker_update_sysreg(state, SYS_CPACR_EL1, + ctxt_sys_reg(&cap->ctx.ctxt, CPACR_EL1)); + arm64_caretaker_update_sysreg(state, SYS_TTBR0_EL1, + ctxt_sys_reg(&cap->ctx.ctxt, TTBR0_EL1)); + arm64_caretaker_update_sysreg(state, SYS_TTBR1_EL1, + ctxt_sys_reg(&cap->ctx.ctxt, TTBR1_EL1)); + arm64_caretaker_update_sysreg(state, SYS_TCR_EL1, + ctxt_sys_reg(&cap->ctx.ctxt, TCR_EL1)); + arm64_caretaker_update_sysreg(state, SYS_ESR_EL1, + ctxt_sys_reg(&cap->ctx.ctxt, ESR_EL1)); + arm64_caretaker_update_sysreg(state, SYS_AFSR0_EL1, + ctxt_sys_reg(&cap->ctx.ctxt, AFSR0_EL1)); + arm64_caretaker_update_sysreg(state, SYS_AFSR1_EL1, + ctxt_sys_reg(&cap->ctx.ctxt, AFSR1_EL1)); + arm64_caretaker_update_sysreg(state, SYS_FAR_EL1, + ctxt_sys_reg(&cap->ctx.ctxt, FAR_EL1)); + arm64_caretaker_update_sysreg(state, SYS_MAIR_EL1, + ctxt_sys_reg(&cap->ctx.ctxt, MAIR_EL1)); + arm64_caretaker_update_sysreg(state, SYS_VBAR_EL1, + ctxt_sys_reg(&cap->ctx.ctxt, VBAR_EL1)); + arm64_caretaker_update_sysreg(state, SYS_CONTEXTIDR_EL1, + ctxt_sys_reg(&cap->ctx.ctxt, CONTEXTIDR_EL1)); + arm64_caretaker_update_sysreg(state, SYS_AMAIR_EL1, + ctxt_sys_reg(&cap->ctx.ctxt, AMAIR_EL1)); + arm64_caretaker_update_sysreg(state, SYS_CNTKCTL_EL1, + ctxt_sys_reg(&cap->ctx.ctxt, CNTKCTL_EL1)); + arm64_caretaker_update_sysreg(state, SYS_PAR_EL1, + ctxt_sys_reg(&cap->ctx.ctxt, PAR_EL1)); + arm64_caretaker_update_sysreg(state, SYS_TPIDR_EL1, + ctxt_sys_reg(&cap->ctx.ctxt, TPIDR_EL1)); + arm64_caretaker_update_sysreg(state, SYS_TPIDR_EL0, + ctxt_sys_reg(&cap->ctx.ctxt, TPIDR_EL0)); + arm64_caretaker_update_sysreg(state, SYS_TPIDRRO_EL0, + ctxt_sys_reg(&cap->ctx.ctxt, TPIDRRO_EL0)); + arm64_caretaker_update_sysreg(state, SYS_SP_EL1, + ctxt_sys_reg(&cap->ctx.ctxt, SP_EL1)); + arm64_caretaker_update_sysreg(state, SYS_ELR_EL1, + ctxt_sys_reg(&cap->ctx.ctxt, ELR_EL1)); + arm64_caretaker_update_sysreg(state, SYS_SPSR_EL1, + ctxt_sys_reg(&cap->ctx.ctxt, SPSR_EL1)); + arm64_caretaker_update_sysreg(state, SYS_MDSCR_EL1, + ctxt_sys_reg(&cap->ctx.ctxt, MDSCR_EL1)); + arm64_caretaker_update_sysreg(state, SYS_CNTV_CVAL_EL0, + cap->ctx.cntv_cval_el0); + arm64_caretaker_update_sysreg(state, SYS_CNTV_CTL_EL0, + cap->ctx.cntv_ctl_el0); + cpu_preserved_clean_sz(state, + struct_size(state, sysregs, state->num_sysregs)); + } + + cpu_preserved_clean(&cap->abi); +} + +static void arm64_caretaker_sync_vcpu(struct kvm_vcpu *vcpu, + void *data) +{ + struct arch_timer_context *vtimer = vcpu_vtimer(vcpu); + struct kvm_caretaker_arch_ser *abi = data; + u64 cval, ctl; + int t; + + /* Sync handover ABI prefix back into incoming vcpu */ + vcpu->arch.hcr_el2 = abi->hcr_el2; + vcpu->arch.mdcr_el2 = abi->mdcr_el2; + vcpu->arch.cflags = abi->cflags; + + if (abi->vgic_initialized) { + int i; + + vcpu->arch.vgic_cpu.vgic_v3.used_lrs = abi->used_lrs; + vcpu->arch.vgic_cpu.vgic_v3.vgic_hcr = abi->vgic_hcr; + vcpu->arch.vgic_cpu.vgic_v3.vgic_vmcr = abi->vgic_vmcr; + for (i = 0; i < 4; i++) { + vcpu->arch.vgic_cpu.vgic_v3.vgic_ap0r[i] = abi->vgic_ap0r[i]; + vcpu->arch.vgic_cpu.vgic_v3.vgic_ap1r[i] = abi->vgic_ap1r[i]; + } + for (i = 0; i < 16; i++) + vcpu->arch.vgic_cpu.vgic_v3.vgic_lr[i] = abi->vgic_lr[i]; + } + + cval = __vcpu_sys_reg(vcpu, CNTV_CVAL_EL0); + ctl = __vcpu_sys_reg(vcpu, CNTV_CTL_EL0); + timer_set_offset(vtimer, abi->cntvoff_el2); + write_sysreg(abi->cntvoff_el2, cntvoff_el2); + write_sysreg_el0(cval, SYS_CNTV_CVAL); + write_sysreg_el0(ctl, SYS_CNTV_CTL); + isb(); + + vcpu_set_flag(vcpu, VCPU_INITIALIZED); + + for (t = 0; t < NR_KVM_TIMERS; t++) + vcpu->arch.timer_cpu.timers[t].loaded = false; + + kvm_make_request(KVM_REQ_IRQ_PENDING, vcpu); +} + -- 2.55.0.1082.g2b9226bbc0-goog
