1 // SPDX-License-Identifier: GPL-2.0-only
3 * Copyright (C) 2016 - Linaro and Columbia University
4 * Author: Jintack Lim <jintack.lim@linaro.org>
8 #include <linux/kvm_host.h>
10 #include <asm/kvm_emulate.h>
11 #include <asm/kvm_nested.h>
13 #include "hyp/include/hyp/adjust_pc.h"
17 static u64 kvm_check_illegal_exception_return(struct kvm_vcpu *vcpu, u64 spsr)
19 u64 mode = spsr & PSR_MODE_MASK;
22 * Possible causes for an Illegal Exception Return from EL2:
23 * - trying to return to EL3
24 * - trying to return to an illegal M value
25 * - trying to return to a 32bit EL
26 * - trying to return to EL1 with HCR_EL2.TGE set
28 if (mode == PSR_MODE_EL3t || mode == PSR_MODE_EL3h ||
29 mode == 0b00001 || (mode & BIT(1)) ||
30 (spsr & PSR_MODE32_BIT) ||
31 (vcpu_el2_tge_is_set(vcpu) && (mode == PSR_MODE_EL1t ||
32 mode == PSR_MODE_EL1h))) {
34 * The guest is playing with our nerves. Preserve EL, SP,
35 * masks, flags from the existing PSTATE, and set IL.
36 * The HW will then generate an Illegal State Exception
37 * immediately after ERET.
39 spsr = *vcpu_cpsr(vcpu);
41 spsr &= (PSR_D_BIT | PSR_A_BIT | PSR_I_BIT | PSR_F_BIT |
42 PSR_N_BIT | PSR_Z_BIT | PSR_C_BIT | PSR_V_BIT |
43 PSR_MODE_MASK | PSR_MODE32_BIT);
50 void kvm_emulate_nested_eret(struct kvm_vcpu *vcpu)
56 * Going through the whole put/load motions is a waste of time
57 * if this is a VHE guest hypervisor returning to its own
58 * userspace, or the hypervisor performing a local exception
59 * return. No need to save/restore registers, no need to
60 * switch S2 MMU. Just do the canonical ERET.
62 spsr = vcpu_read_sys_reg(vcpu, SPSR_EL2);
63 spsr = kvm_check_illegal_exception_return(vcpu, spsr);
65 mode = spsr & (PSR_MODE_MASK | PSR_MODE32_BIT);
67 direct_eret = (mode == PSR_MODE_EL0t &&
68 vcpu_el2_e2h_is_set(vcpu) &&
69 vcpu_el2_tge_is_set(vcpu));
70 direct_eret |= (mode == PSR_MODE_EL2h || mode == PSR_MODE_EL2t);
73 *vcpu_pc(vcpu) = vcpu_read_sys_reg(vcpu, ELR_EL2);
74 *vcpu_cpsr(vcpu) = spsr;
75 trace_kvm_nested_eret(vcpu, *vcpu_pc(vcpu), spsr);
80 kvm_arch_vcpu_put(vcpu);
82 elr = __vcpu_sys_reg(vcpu, ELR_EL2);
84 trace_kvm_nested_eret(vcpu, elr, spsr);
87 * Note that the current exception level is always the virtual EL2,
88 * since we set HCR_EL2.NV bit only when entering the virtual EL2.
91 *vcpu_cpsr(vcpu) = spsr;
93 kvm_arch_vcpu_load(vcpu, smp_processor_id());
97 static void kvm_inject_el2_exception(struct kvm_vcpu *vcpu, u64 esr_el2,
98 enum exception_type type)
100 trace_kvm_inject_nested_exception(vcpu, esr_el2, type);
103 case except_type_sync:
104 kvm_pend_exception(vcpu, EXCEPT_AA64_EL2_SYNC);
105 vcpu_write_sys_reg(vcpu, esr_el2, ESR_EL2);
107 case except_type_irq:
108 kvm_pend_exception(vcpu, EXCEPT_AA64_EL2_IRQ);
111 WARN_ONCE(1, "Unsupported EL2 exception injection %d\n", type);
116 * Emulate taking an exception to EL2.
117 * See ARM ARM J8.1.2 AArch64.TakeException()
119 static int kvm_inject_nested(struct kvm_vcpu *vcpu, u64 esr_el2,
120 enum exception_type type)
125 if (!vcpu_has_nv(vcpu)) {
126 kvm_err("Unexpected call to %s for the non-nesting configuration\n",
132 * As for ERET, we can avoid doing too much on the injection path by
133 * checking that we either took the exception from a VHE host
134 * userspace or from vEL2. In these cases, there is no change in
135 * translation regime (or anything else), so let's do as little as
138 pstate = *vcpu_cpsr(vcpu);
139 mode = pstate & (PSR_MODE_MASK | PSR_MODE32_BIT);
141 direct_inject = (mode == PSR_MODE_EL0t &&
142 vcpu_el2_e2h_is_set(vcpu) &&
143 vcpu_el2_tge_is_set(vcpu));
144 direct_inject |= (mode == PSR_MODE_EL2h || mode == PSR_MODE_EL2t);
147 kvm_inject_el2_exception(vcpu, esr_el2, type);
154 * We may have an exception or PC update in the EL0/EL1 context.
155 * Commit it before entering EL2.
157 __kvm_adjust_pc(vcpu);
159 kvm_arch_vcpu_put(vcpu);
161 kvm_inject_el2_exception(vcpu, esr_el2, type);
164 * A hard requirement is that a switch between EL1 and EL2
165 * contexts has to happen between a put/load, so that we can
166 * pick the correct timer and interrupt configuration, among
169 * Make sure the exception actually took place before we load
172 __kvm_adjust_pc(vcpu);
174 kvm_arch_vcpu_load(vcpu, smp_processor_id());
180 int kvm_inject_nested_sync(struct kvm_vcpu *vcpu, u64 esr_el2)
182 return kvm_inject_nested(vcpu, esr_el2, except_type_sync);
185 int kvm_inject_nested_irq(struct kvm_vcpu *vcpu)
188 * Do not inject an irq if the:
189 * - Current exception level is EL2, and
190 * - virtual HCR_EL2.TGE == 0
191 * - virtual HCR_EL2.IMO == 0
193 * See Table D1-17 "Physical interrupt target and masking when EL3 is
194 * not implemented and EL2 is implemented" in ARM DDI 0487C.a.
197 if (vcpu_is_el2(vcpu) && !vcpu_el2_tge_is_set(vcpu) &&
198 !(__vcpu_sys_reg(vcpu, HCR_EL2) & HCR_IMO))
201 /* esr_el2 value doesn't matter for exits due to irqs. */
202 return kvm_inject_nested(vcpu, 0, except_type_irq);