arch/arm64/kvm/hyp/switch.c

   1 /*
   2  * Copyright (C) 2015 - ARM Ltd
   3  * Author: Marc Zyngier <marc.zyngier@arm.com>
   4  *
   5  * This program is free software; you can redistribute it and/or modify
   6  * it under the terms of the GNU General Public License version 2 as
   7  * published by the Free Software Foundation.
   8  *
   9  * This program is distributed in the hope that it will be useful,
  10  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  11  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  12  * GNU General Public License for more details.
  13  *
  14  * You should have received a copy of the GNU General Public License
  15  * along with this program.  If not, see <http://www.gnu.org/licenses/>.
  16  */
  17
  18 #include <linux/arm-smccc.h>
  19 #include <linux/types.h>
  20 #include <linux/jump_label.h>
  21 #include <uapi/linux/psci.h>
  22
  23 #include <kvm/arm_psci.h>
  24
  25 #include <asm/cpufeature.h>
  26 #include <asm/kvm_asm.h>
  27 #include <asm/kvm_emulate.h>
  28 #include <asm/kvm_host.h>
  29 #include <asm/kvm_hyp.h>
  30 #include <asm/kvm_mmu.h>
  31 #include <asm/fpsimd.h>
  32 #include <asm/debug-monitors.h>
  33 #include <asm/processor.h>
  34 #include <asm/thread_info.h>
  35
  36 /* Check whether the FP regs were dirtied while in the host-side run loop: */
  37 static bool __hyp_text update_fp_enabled(struct kvm_vcpu *vcpu)
  38 {
  39         if (vcpu->arch.host_thread_info->flags & _TIF_FOREIGN_FPSTATE)
  40                 vcpu->arch.flags &= ~(KVM_ARM64_FP_ENABLED |
  41                                       KVM_ARM64_FP_HOST);
  42
  43         return !!(vcpu->arch.flags & KVM_ARM64_FP_ENABLED);
  44 }
  45
  46 /* Save the 32-bit only FPSIMD system register state */
  47 static void __hyp_text __fpsimd_save_fpexc32(struct kvm_vcpu *vcpu)
  48 {
  49         if (!vcpu_el1_is_32bit(vcpu))
  50                 return;
  51
  52         vcpu->arch.ctxt.sys_regs[FPEXC32_EL2] = read_sysreg(fpexc32_el2);
  53 }
  54
  55 static void __hyp_text __activate_traps_fpsimd32(struct kvm_vcpu *vcpu)
  56 {
  57         /*
  58          * We are about to set CPTR_EL2.TFP to trap all floating point
  59          * register accesses to EL2, however, the ARM ARM clearly states that
  60          * traps are only taken to EL2 if the operation would not otherwise
  61          * trap to EL1.  Therefore, always make sure that for 32-bit guests,
  62          * we set FPEXC.EN to prevent traps to EL1, when setting the TFP bit.
  63          * If FP/ASIMD is not implemented, FPEXC is UNDEFINED and any access to
  64          * it will cause an exception.
  65          */
  66         if (vcpu_el1_is_32bit(vcpu) && system_supports_fpsimd()) {
  67                 write_sysreg(1 << 30, fpexc32_el2);
  68                 isb();
  69         }
  70 }
  71
  72 static void __hyp_text __activate_traps_common(struct kvm_vcpu *vcpu)
  73 {
  74         /* Trap on AArch32 cp15 c15 (impdef sysregs) accesses (EL1 or EL0) */
  75         write_sysreg(1 << 15, hstr_el2);
  76
  77         /*
  78          * Make sure we trap PMU access from EL0 to EL2. Also sanitize
  79          * PMSELR_EL0 to make sure it never contains the cycle
  80          * counter, which could make a PMXEVCNTR_EL0 access UNDEF at
  81          * EL1 instead of being trapped to EL2.
  82          */
  83         write_sysreg(0, pmselr_el0);
  84         write_sysreg(ARMV8_PMU_USERENR_MASK, pmuserenr_el0);
  85         write_sysreg(vcpu->arch.mdcr_el2, mdcr_el2);
  86 }
  87
  88 static void __hyp_text __deactivate_traps_common(void)
  89 {
  90         write_sysreg(0, hstr_el2);
  91         write_sysreg(0, pmuserenr_el0);
  92 }
  93
  94 static void activate_traps_vhe(struct kvm_vcpu *vcpu)
  95 {
  96         u64 val;
  97
  98         val = read_sysreg(cpacr_el1);
  99         val |= CPACR_EL1_TTA;
 100         val &= ~CPACR_EL1_ZEN;
 101         if (!update_fp_enabled(vcpu))
 102                 val &= ~CPACR_EL1_FPEN;
 103
 104         write_sysreg(val, cpacr_el1);
 105
 106         write_sysreg(kvm_get_hyp_vector(), vbar_el1);
 107 }
 108
 109 static void __hyp_text __activate_traps_nvhe(struct kvm_vcpu *vcpu)
 110 {
 111         u64 val;
 112
 113         __activate_traps_common(vcpu);
 114
 115         val = CPTR_EL2_DEFAULT;
 116         val |= CPTR_EL2_TTA | CPTR_EL2_TZ;
 117         if (!update_fp_enabled(vcpu))
 118                 val |= CPTR_EL2_TFP;
 119
 120         write_sysreg(val, cptr_el2);
 121 }
 122
 123 static void __hyp_text __activate_traps(struct kvm_vcpu *vcpu)
 124 {
 125         u64 hcr = vcpu->arch.hcr_el2;
 126
 127         write_sysreg(hcr, hcr_el2);
 128
 129         if (cpus_have_const_cap(ARM64_HAS_RAS_EXTN) && (hcr & HCR_VSE))
 130                 write_sysreg_s(vcpu->arch.vsesr_el2, SYS_VSESR_EL2);
 131
 132         __activate_traps_fpsimd32(vcpu);
 133         if (has_vhe())
 134                 activate_traps_vhe(vcpu);
 135         else
 136                 __activate_traps_nvhe(vcpu);
 137 }
 138
 139 static void deactivate_traps_vhe(void)
 140 {
 141         extern char vectors[];  /* kernel exception vectors */
 142         write_sysreg(HCR_HOST_VHE_FLAGS, hcr_el2);
 143         write_sysreg(CPACR_EL1_DEFAULT, cpacr_el1);
 144         write_sysreg(vectors, vbar_el1);
 145 }
 146
 147 static void __hyp_text __deactivate_traps_nvhe(void)
 148 {
 149         u64 mdcr_el2 = read_sysreg(mdcr_el2);
 150
 151         __deactivate_traps_common();
 152
 153         mdcr_el2 &= MDCR_EL2_HPMN_MASK;
 154         mdcr_el2 |= MDCR_EL2_E2PB_MASK << MDCR_EL2_E2PB_SHIFT;
 155
 156         write_sysreg(mdcr_el2, mdcr_el2);
 157         write_sysreg(HCR_RW, hcr_el2);
 158         write_sysreg(CPTR_EL2_DEFAULT, cptr_el2);
 159 }
 160
 161 static void __hyp_text __deactivate_traps(struct kvm_vcpu *vcpu)
 162 {
 163         /*
 164          * If we pended a virtual abort, preserve it until it gets
 165          * cleared. See D1.14.3 (Virtual Interrupts) for details, but
 166          * the crucial bit is "On taking a vSError interrupt,
 167          * HCR_EL2.VSE is cleared to 0."
 168          */
 169         if (vcpu->arch.hcr_el2 & HCR_VSE)
 170                 vcpu->arch.hcr_el2 = read_sysreg(hcr_el2);
 171
 172         if (has_vhe())
 173                 deactivate_traps_vhe();
 174         else
 175                 __deactivate_traps_nvhe();
 176 }
 177
 178 void activate_traps_vhe_load(struct kvm_vcpu *vcpu)
 179 {
 180         __activate_traps_common(vcpu);
 181 }
 182
 183 void deactivate_traps_vhe_put(void)
 184 {
 185         u64 mdcr_el2 = read_sysreg(mdcr_el2);
 186
 187         mdcr_el2 &= MDCR_EL2_HPMN_MASK |
 188                     MDCR_EL2_E2PB_MASK << MDCR_EL2_E2PB_SHIFT |
 189                     MDCR_EL2_TPMS;
 190
 191         write_sysreg(mdcr_el2, mdcr_el2);
 192
 193         __deactivate_traps_common();
 194 }
 195
 196 static void __hyp_text __activate_vm(struct kvm *kvm)
 197 {
 198         write_sysreg(kvm->arch.vttbr, vttbr_el2);
 199 }
 200
 201 static void __hyp_text __deactivate_vm(struct kvm_vcpu *vcpu)
 202 {
 203         write_sysreg(0, vttbr_el2);
 204 }
 205
 206 /* Save VGICv3 state on non-VHE systems */
 207 static void __hyp_text __hyp_vgic_save_state(struct kvm_vcpu *vcpu)
 208 {
 209         if (static_branch_unlikely(&kvm_vgic_global_state.gicv3_cpuif)) {
 210                 __vgic_v3_save_state(vcpu);
 211                 __vgic_v3_deactivate_traps(vcpu);
 212         }
 213 }
 214
 215 /* Restore VGICv3 state on non_VEH systems */
 216 static void __hyp_text __hyp_vgic_restore_state(struct kvm_vcpu *vcpu)
 217 {
 218         if (static_branch_unlikely(&kvm_vgic_global_state.gicv3_cpuif)) {
 219                 __vgic_v3_activate_traps(vcpu);
 220                 __vgic_v3_restore_state(vcpu);
 221         }
 222 }
 223
 224 static bool __hyp_text __true_value(void)
 225 {
 226         return true;
 227 }
 228
 229 static bool __hyp_text __false_value(void)
 230 {
 231         return false;
 232 }
 233
 234 static hyp_alternate_select(__check_arm_834220,
 235                             __false_value, __true_value,
 236                             ARM64_WORKAROUND_834220);
 237
 238 static bool __hyp_text __translate_far_to_hpfar(u64 far, u64 *hpfar)
 239 {
 240         u64 par, tmp;
 241
 242         /*
 243          * Resolve the IPA the hard way using the guest VA.
 244          *
 245          * Stage-1 translation already validated the memory access
 246          * rights. As such, we can use the EL1 translation regime, and
 247          * don't have to distinguish between EL0 and EL1 access.
 248          *
 249          * We do need to save/restore PAR_EL1 though, as we haven't
 250          * saved the guest context yet, and we may return early...
 251          */
 252         par = read_sysreg(par_el1);
 253         asm volatile("at s1e1r, %0" : : "r" (far));
 254         isb();
 255
 256         tmp = read_sysreg(par_el1);
 257         write_sysreg(par, par_el1);
 258
 259         if (unlikely(tmp & 1))
 260                 return false; /* Translation failed, back to guest */
 261
 262         /* Convert PAR to HPFAR format */
 263         *hpfar = ((tmp >> 12) & ((1UL << 36) - 1)) << 4;
 264         return true;
 265 }
 266
 267 static bool __hyp_text __populate_fault_info(struct kvm_vcpu *vcpu)
 268 {
 269         u8 ec;
 270         u64 esr;
 271         u64 hpfar, far;
 272
 273         esr = vcpu->arch.fault.esr_el2;
 274         ec = ESR_ELx_EC(esr);
 275
 276         if (ec != ESR_ELx_EC_DABT_LOW && ec != ESR_ELx_EC_IABT_LOW)
 277                 return true;
 278
 279         far = read_sysreg_el2(far);
 280
 281         /*
 282          * The HPFAR can be invalid if the stage 2 fault did not
 283          * happen during a stage 1 page table walk (the ESR_EL2.S1PTW
 284          * bit is clear) and one of the two following cases are true:
 285          *   1. The fault was due to a permission fault
 286          *   2. The processor carries errata 834220
 287          *
 288          * Therefore, for all non S1PTW faults where we either have a
 289          * permission fault or the errata workaround is enabled, we
 290          * resolve the IPA using the AT instruction.
 291          */
 292         if (!(esr & ESR_ELx_S1PTW) &&
 293             (__check_arm_834220()() || (esr & ESR_ELx_FSC_TYPE) == FSC_PERM)) {
 294                 if (!__translate_far_to_hpfar(far, &hpfar))
 295                         return false;
 296         } else {
 297                 hpfar = read_sysreg(hpfar_el2);
 298         }
 299
 300         vcpu->arch.fault.far_el2 = far;
 301         vcpu->arch.fault.hpfar_el2 = hpfar;
 302         return true;
 303 }
 304
 305 /* Skip an instruction which has been emulated. Returns true if
 306  * execution can continue or false if we need to exit hyp mode because
 307  * single-step was in effect.
 308  */
 309 static bool __hyp_text __skip_instr(struct kvm_vcpu *vcpu)
 310 {
 311         *vcpu_pc(vcpu) = read_sysreg_el2(elr);
 312
 313         if (vcpu_mode_is_32bit(vcpu)) {
 314                 vcpu->arch.ctxt.gp_regs.regs.pstate = read_sysreg_el2(spsr);
 315                 kvm_skip_instr32(vcpu, kvm_vcpu_trap_il_is32bit(vcpu));
 316                 write_sysreg_el2(vcpu->arch.ctxt.gp_regs.regs.pstate, spsr);
 317         } else {
 318                 *vcpu_pc(vcpu) += 4;
 319         }
 320
 321         write_sysreg_el2(*vcpu_pc(vcpu), elr);
 322
 323         if (vcpu->guest_debug & KVM_GUESTDBG_SINGLESTEP) {
 324                 vcpu->arch.fault.esr_el2 =
 325                         (ESR_ELx_EC_SOFTSTP_LOW << ESR_ELx_EC_SHIFT) | 0x22;
 326                 return false;
 327         } else {
 328                 return true;
 329         }
 330 }
 331
 332 static bool __hyp_text __hyp_switch_fpsimd(struct kvm_vcpu *vcpu)
 333 {
 334         struct user_fpsimd_state *host_fpsimd = vcpu->arch.host_fpsimd_state;
 335
 336         if (has_vhe())
 337                 write_sysreg(read_sysreg(cpacr_el1) | CPACR_EL1_FPEN,
 338                              cpacr_el1);
 339         else
 340                 write_sysreg(read_sysreg(cptr_el2) & ~(u64)CPTR_EL2_TFP,
 341                              cptr_el2);
 342
 343         isb();
 344
 345         if (vcpu->arch.flags & KVM_ARM64_FP_HOST) {
 346                 /*
 347                  * In the SVE case, VHE is assumed: it is enforced by
 348                  * Kconfig and kvm_arch_init().
 349                  */
 350                 if (system_supports_sve() &&
 351                     (vcpu->arch.flags & KVM_ARM64_HOST_SVE_IN_USE)) {
 352                         struct thread_struct *thread = container_of(
 353                                 host_fpsimd,
 354                                 struct thread_struct, uw.fpsimd_state);
 355
 356                         sve_save_state(sve_pffr(thread), &host_fpsimd->fpsr);
 357                 } else {
 358                         __fpsimd_save_state(host_fpsimd);
 359                 }
 360
 361                 vcpu->arch.flags &= ~KVM_ARM64_FP_HOST;
 362         }
 363
 364         __fpsimd_restore_state(&vcpu->arch.ctxt.gp_regs.fp_regs);
 365
 366         /* Skip restoring fpexc32 for AArch64 guests */
 367         if (!(read_sysreg(hcr_el2) & HCR_RW))
 368                 write_sysreg(vcpu->arch.ctxt.sys_regs[FPEXC32_EL2],
 369                              fpexc32_el2);
 370
 371         vcpu->arch.flags |= KVM_ARM64_FP_ENABLED;
 372
 373         return true;
 374 }
 375
 376 /*
 377  * Return true when we were able to fixup the guest exit and should return to
 378  * the guest, false when we should restore the host state and return to the
 379  * main run loop.
 380  */
 381 static bool __hyp_text fixup_guest_exit(struct kvm_vcpu *vcpu, u64 *exit_code)
 382 {
 383         if (ARM_EXCEPTION_CODE(*exit_code) != ARM_EXCEPTION_IRQ)
 384                 vcpu->arch.fault.esr_el2 = read_sysreg_el2(esr);
 385
 386         /*
 387          * We're using the raw exception code in order to only process
 388          * the trap if no SError is pending. We will come back to the
 389          * same PC once the SError has been injected, and replay the
 390          * trapping instruction.
 391          */
 392         if (*exit_code != ARM_EXCEPTION_TRAP)
 393                 goto exit;
 394
 395         /*
 396          * We trap the first access to the FP/SIMD to save the host context
 397          * and restore the guest context lazily.
 398          * If FP/SIMD is not implemented, handle the trap and inject an
 399          * undefined instruction exception to the guest.
 400          */
 401         if (system_supports_fpsimd() &&
 402             kvm_vcpu_trap_get_class(vcpu) == ESR_ELx_EC_FP_ASIMD)
 403                 return __hyp_switch_fpsimd(vcpu);
 404
 405         if (!__populate_fault_info(vcpu))
 406                 return true;
 407
 408         if (static_branch_unlikely(&vgic_v2_cpuif_trap)) {
 409                 bool valid;
 410
 411                 valid = kvm_vcpu_trap_get_class(vcpu) == ESR_ELx_EC_DABT_LOW &&
 412                         kvm_vcpu_trap_get_fault_type(vcpu) == FSC_FAULT &&
 413                         kvm_vcpu_dabt_isvalid(vcpu) &&
 414                         !kvm_vcpu_dabt_isextabt(vcpu) &&
 415                         !kvm_vcpu_dabt_iss1tw(vcpu);
 416
 417                 if (valid) {
 418                         int ret = __vgic_v2_perform_cpuif_access(vcpu);
 419
 420                         if (ret ==  1 && __skip_instr(vcpu))
 421                                 return true;
 422
 423                         if (ret == -1) {
 424                                 /* Promote an illegal access to an
 425                                  * SError. If we would be returning
 426                                  * due to single-step clear the SS
 427                                  * bit so handle_exit knows what to
 428                                  * do after dealing with the error.
 429                                  */
 430                                 if (!__skip_instr(vcpu))
 431                                         *vcpu_cpsr(vcpu) &= ~DBG_SPSR_SS;
 432                                 *exit_code = ARM_EXCEPTION_EL1_SERROR;
 433                         }
 434
 435                         goto exit;
 436                 }
 437         }
 438
 439         if (static_branch_unlikely(&vgic_v3_cpuif_trap) &&
 440             (kvm_vcpu_trap_get_class(vcpu) == ESR_ELx_EC_SYS64 ||
 441              kvm_vcpu_trap_get_class(vcpu) == ESR_ELx_EC_CP15_32)) {
 442                 int ret = __vgic_v3_perform_cpuif_access(vcpu);
 443
 444                 if (ret == 1 && __skip_instr(vcpu))
 445                         return true;
 446         }
 447
 448 exit:
 449         /* Return to the host kernel and handle the exit */
 450         return false;
 451 }
 452
 453 static inline bool __hyp_text __needs_ssbd_off(struct kvm_vcpu *vcpu)
 454 {
 455         if (!cpus_have_const_cap(ARM64_SSBD))
 456                 return false;
 457
 458         return !(vcpu->arch.workaround_flags & VCPU_WORKAROUND_2_FLAG);
 459 }
 460
 461 static void __hyp_text __set_guest_arch_workaround_state(struct kvm_vcpu *vcpu)
 462 {
 463 #ifdef CONFIG_ARM64_SSBD
 464         /*
 465          * The host runs with the workaround always present. If the
 466          * guest wants it disabled, so be it...
 467          */
 468         if (__needs_ssbd_off(vcpu) &&
 469             __hyp_this_cpu_read(arm64_ssbd_callback_required))
 470                 arm_smccc_1_1_smc(ARM_SMCCC_ARCH_WORKAROUND_2, 0, NULL);
 471 #endif
 472 }
 473
 474 static void __hyp_text __set_host_arch_workaround_state(struct kvm_vcpu *vcpu)
 475 {
 476 #ifdef CONFIG_ARM64_SSBD
 477         /*
 478          * If the guest has disabled the workaround, bring it back on.
 479          */
 480         if (__needs_ssbd_off(vcpu) &&
 481             __hyp_this_cpu_read(arm64_ssbd_callback_required))
 482                 arm_smccc_1_1_smc(ARM_SMCCC_ARCH_WORKAROUND_2, 1, NULL);
 483 #endif
 484 }
 485
 486 /* Switch to the guest for VHE systems running in EL2 */
 487 int kvm_vcpu_run_vhe(struct kvm_vcpu *vcpu)
 488 {
 489         struct kvm_cpu_context *host_ctxt;
 490         struct kvm_cpu_context *guest_ctxt;
 491         u64 exit_code;
 492
 493         host_ctxt = vcpu->arch.host_cpu_context;
 494         host_ctxt->__hyp_running_vcpu = vcpu;
 495         guest_ctxt = &vcpu->arch.ctxt;
 496
 497         sysreg_save_host_state_vhe(host_ctxt);
 498
 499         __activate_traps(vcpu);
 500         __activate_vm(vcpu->kvm);
 501
 502         sysreg_restore_guest_state_vhe(guest_ctxt);
 503         __debug_switch_to_guest(vcpu);
 504
 505         __set_guest_arch_workaround_state(vcpu);
 506
 507         do {
 508                 /* Jump in the fire! */
 509                 exit_code = __guest_enter(vcpu, host_ctxt);
 510
 511                 /* And we're baaack! */
 512         } while (fixup_guest_exit(vcpu, &exit_code));
 513
 514         __set_host_arch_workaround_state(vcpu);
 515
 516         sysreg_save_guest_state_vhe(guest_ctxt);
 517
 518         __deactivate_traps(vcpu);
 519
 520         sysreg_restore_host_state_vhe(host_ctxt);
 521
 522         if (vcpu->arch.flags & KVM_ARM64_FP_ENABLED)
 523                 __fpsimd_save_fpexc32(vcpu);
 524
 525         __debug_switch_to_host(vcpu);
 526
 527         return exit_code;
 528 }
 529
 530 /* Switch to the guest for legacy non-VHE systems */
 531 int __hyp_text __kvm_vcpu_run_nvhe(struct kvm_vcpu *vcpu)
 532 {
 533         struct kvm_cpu_context *host_ctxt;
 534         struct kvm_cpu_context *guest_ctxt;
 535         u64 exit_code;
 536
 537         vcpu = kern_hyp_va(vcpu);
 538
 539         host_ctxt = kern_hyp_va(vcpu->arch.host_cpu_context);
 540         host_ctxt->__hyp_running_vcpu = vcpu;
 541         guest_ctxt = &vcpu->arch.ctxt;
 542
 543         __sysreg_save_state_nvhe(host_ctxt);
 544
 545         __activate_traps(vcpu);
 546         __activate_vm(kern_hyp_va(vcpu->kvm));
 547
 548         __hyp_vgic_restore_state(vcpu);
 549         __timer_enable_traps(vcpu);
 550
 551         /*
 552          * We must restore the 32-bit state before the sysregs, thanks
 553          * to erratum #852523 (Cortex-A57) or #853709 (Cortex-A72).
 554          */
 555         __sysreg32_restore_state(vcpu);
 556         __sysreg_restore_state_nvhe(guest_ctxt);
 557         __debug_switch_to_guest(vcpu);
 558
 559         __set_guest_arch_workaround_state(vcpu);
 560
 561         do {
 562                 /* Jump in the fire! */
 563                 exit_code = __guest_enter(vcpu, host_ctxt);
 564
 565                 /* And we're baaack! */
 566         } while (fixup_guest_exit(vcpu, &exit_code));
 567
 568         __set_host_arch_workaround_state(vcpu);
 569
 570         __sysreg_save_state_nvhe(guest_ctxt);
 571         __sysreg32_save_state(vcpu);
 572         __timer_disable_traps(vcpu);
 573         __hyp_vgic_save_state(vcpu);
 574
 575         __deactivate_traps(vcpu);
 576         __deactivate_vm(vcpu);
 577
 578         __sysreg_restore_state_nvhe(host_ctxt);
 579
 580         if (vcpu->arch.flags & KVM_ARM64_FP_ENABLED)
 581                 __fpsimd_save_fpexc32(vcpu);
 582
 583         /*
 584          * This must come after restoring the host sysregs, since a non-VHE
 585          * system may enable SPE here and make use of the TTBRs.
 586          */
 587         __debug_switch_to_host(vcpu);
 588
 589         return exit_code;
 590 }
 591
 592 static const char __hyp_panic_string[] = "HYP panic:\nPS:%08llx PC:%016llx ESR:%08llx\nFAR:%016llx HPFAR:%016llx PAR:%016llx\nVCPU:%p\n";
 593
 594 static void __hyp_text __hyp_call_panic_nvhe(u64 spsr, u64 elr, u64 par,
 595                                              struct kvm_cpu_context *__host_ctxt)
 596 {
 597         struct kvm_vcpu *vcpu;
 598         unsigned long str_va;
 599
 600         vcpu = __host_ctxt->__hyp_running_vcpu;
 601
 602         if (read_sysreg(vttbr_el2)) {
 603                 __timer_disable_traps(vcpu);
 604                 __deactivate_traps(vcpu);
 605                 __deactivate_vm(vcpu);
 606                 __sysreg_restore_state_nvhe(__host_ctxt);
 607         }
 608
 609         /*
 610          * Force the panic string to be loaded from the literal pool,
 611          * making sure it is a kernel address and not a PC-relative
 612          * reference.
 613          */
 614         asm volatile("ldr %0, =__hyp_panic_string" : "=r" (str_va));
 615
 616         __hyp_do_panic(str_va,
 617                        spsr,  elr,
 618                        read_sysreg(esr_el2),   read_sysreg_el2(far),
 619                        read_sysreg(hpfar_el2), par, vcpu);
 620 }
 621
 622 static void __hyp_call_panic_vhe(u64 spsr, u64 elr, u64 par,
 623                                  struct kvm_cpu_context *host_ctxt)
 624 {
 625         struct kvm_vcpu *vcpu;
 626         vcpu = host_ctxt->__hyp_running_vcpu;
 627
 628         __deactivate_traps(vcpu);
 629         sysreg_restore_host_state_vhe(host_ctxt);
 630
 631         panic(__hyp_panic_string,
 632               spsr,  elr,
 633               read_sysreg_el2(esr),   read_sysreg_el2(far),
 634               read_sysreg(hpfar_el2), par, vcpu);
 635 }
 636
 637 void __hyp_text __noreturn hyp_panic(struct kvm_cpu_context *host_ctxt)
 638 {
 639         u64 spsr = read_sysreg_el2(spsr);
 640         u64 elr = read_sysreg_el2(elr);
 641         u64 par = read_sysreg(par_el1);
 642
 643         if (!has_vhe())
 644                 __hyp_call_panic_nvhe(spsr, elr, par, host_ctxt);
 645         else
 646                 __hyp_call_panic_vhe(spsr, elr, par, host_ctxt);
 647
 648         unreachable();
 649 }