KVM: arm64: Make struct kvm_regs userspace-only
authorMarc Zyngier <maz@kernel.org>
Fri, 28 Jun 2019 21:40:58 +0000 (22:40 +0100)
committerMarc Zyngier <maz@kernel.org>
Tue, 7 Jul 2020 08:28:38 +0000 (09:28 +0100)
struct kvm_regs is used by userspace to indicate which register gets
accessed by the {GET,SET}_ONE_REG API. But as we're about to refactor
the layout of the in-kernel register structures, we need the kernel to
move away from it.

Let's make kvm_regs userspace only, and let the kernel map it to its own
internal representation.

Reviewed-by: James Morse <james.morse@arm.com>
Signed-off-by: Marc Zyngier <maz@kernel.org>
arch/arm64/include/asm/kvm_emulate.h
arch/arm64/include/asm/kvm_host.h
arch/arm64/kernel/asm-offsets.c
arch/arm64/kvm/fpsimd.c
arch/arm64/kvm/guest.c
arch/arm64/kvm/hyp/entry.S
arch/arm64/kvm/hyp/include/hyp/switch.h
arch/arm64/kvm/hyp/include/hyp/sysreg-sr.h
arch/arm64/kvm/regmap.c
arch/arm64/kvm/reset.c

index 269a76cd51fffaffbcce19fea12a716b5a83a54d..cd607999abc2e3c0b87c09681c460ac44c4c1845 100644 (file)
@@ -124,12 +124,12 @@ static inline void vcpu_set_vsesr(struct kvm_vcpu *vcpu, u64 vsesr)
 
 static __always_inline unsigned long *vcpu_pc(const struct kvm_vcpu *vcpu)
 {
-       return (unsigned long *)&vcpu_gp_regs(vcpu)->regs.pc;
+       return (unsigned long *)&vcpu_gp_regs(vcpu)->pc;
 }
 
 static inline unsigned long *__vcpu_elr_el1(const struct kvm_vcpu *vcpu)
 {
-       return (unsigned long *)&vcpu_gp_regs(vcpu)->elr_el1;
+       return (unsigned long *)&vcpu->arch.ctxt.elr_el1;
 }
 
 static inline unsigned long vcpu_read_elr_el1(const struct kvm_vcpu *vcpu)
@@ -150,7 +150,7 @@ static inline void vcpu_write_elr_el1(const struct kvm_vcpu *vcpu, unsigned long
 
 static __always_inline unsigned long *vcpu_cpsr(const struct kvm_vcpu *vcpu)
 {
-       return (unsigned long *)&vcpu_gp_regs(vcpu)->regs.pstate;
+       return (unsigned long *)&vcpu_gp_regs(vcpu)->pstate;
 }
 
 static __always_inline bool vcpu_mode_is_32bit(const struct kvm_vcpu *vcpu)
@@ -179,14 +179,14 @@ static inline void vcpu_set_thumb(struct kvm_vcpu *vcpu)
 static __always_inline unsigned long vcpu_get_reg(const struct kvm_vcpu *vcpu,
                                         u8 reg_num)
 {
-       return (reg_num == 31) ? 0 : vcpu_gp_regs(vcpu)->regs.regs[reg_num];
+       return (reg_num == 31) ? 0 : vcpu_gp_regs(vcpu)->regs[reg_num];
 }
 
 static __always_inline void vcpu_set_reg(struct kvm_vcpu *vcpu, u8 reg_num,
                                unsigned long val)
 {
        if (reg_num != 31)
-               vcpu_gp_regs(vcpu)->regs.regs[reg_num] = val;
+               vcpu_gp_regs(vcpu)->regs[reg_num] = val;
 }
 
 static inline unsigned long vcpu_read_spsr(const struct kvm_vcpu *vcpu)
@@ -197,7 +197,7 @@ static inline unsigned long vcpu_read_spsr(const struct kvm_vcpu *vcpu)
        if (vcpu->arch.sysregs_loaded_on_cpu)
                return read_sysreg_el1(SYS_SPSR);
        else
-               return vcpu_gp_regs(vcpu)->spsr[KVM_SPSR_EL1];
+               return vcpu->arch.ctxt.spsr[KVM_SPSR_EL1];
 }
 
 static inline void vcpu_write_spsr(struct kvm_vcpu *vcpu, unsigned long v)
@@ -210,7 +210,7 @@ static inline void vcpu_write_spsr(struct kvm_vcpu *vcpu, unsigned long v)
        if (vcpu->arch.sysregs_loaded_on_cpu)
                write_sysreg_el1(v, SYS_SPSR);
        else
-               vcpu_gp_regs(vcpu)->spsr[KVM_SPSR_EL1] = v;
+               vcpu->arch.ctxt.spsr[KVM_SPSR_EL1] = v;
 }
 
 /*
@@ -519,11 +519,11 @@ static __always_inline void kvm_skip_instr(struct kvm_vcpu *vcpu, bool is_wide_i
 static __always_inline void __kvm_skip_instr(struct kvm_vcpu *vcpu)
 {
        *vcpu_pc(vcpu) = read_sysreg_el2(SYS_ELR);
-       vcpu->arch.ctxt.gp_regs.regs.pstate = read_sysreg_el2(SYS_SPSR);
+       vcpu_gp_regs(vcpu)->pstate = read_sysreg_el2(SYS_SPSR);
 
        kvm_skip_instr(vcpu, kvm_vcpu_trap_il_is32bit(vcpu));
 
-       write_sysreg_el2(vcpu->arch.ctxt.gp_regs.regs.pstate, SYS_SPSR);
+       write_sysreg_el2(vcpu_gp_regs(vcpu)->pstate, SYS_SPSR);
        write_sysreg_el2(*vcpu_pc(vcpu), SYS_ELR);
 }
 
index d9360f91ef4403a029c888b439035b79640b7e58..bc1e91573d00bd3dff71bcb09af6dab4405c81f7 100644 (file)
@@ -236,7 +236,15 @@ enum vcpu_sysreg {
 #define NR_COPRO_REGS  (NR_SYS_REGS * 2)
 
 struct kvm_cpu_context {
-       struct kvm_regs gp_regs;
+       struct user_pt_regs regs;       /* sp = sp_el0 */
+
+       u64     sp_el1;
+       u64     elr_el1;
+
+       u64     spsr[KVM_NR_SPSR];
+
+       struct user_fpsimd_state fp_regs;
+
        union {
                u64 sys_regs[NR_SYS_REGS];
                u32 copro[NR_COPRO_REGS];
@@ -402,7 +410,7 @@ struct kvm_vcpu_arch {
                                  system_supports_generic_auth()) && \
                                 ((vcpu)->arch.flags & KVM_ARM64_GUEST_HAS_PTRAUTH))
 
-#define vcpu_gp_regs(v)                (&(v)->arch.ctxt.gp_regs)
+#define vcpu_gp_regs(v)                (&(v)->arch.ctxt.regs)
 
 /*
  * Only use __vcpu_sys_reg/ctxt_sys_reg if you know you want the
index 0577e21422845fa3d5bbe5c5e8bd6fd784e8bbec..7d32fc959b1a071a0eefb2efe666b2dd1dbbe057 100644 (file)
@@ -102,13 +102,12 @@ int main(void)
   DEFINE(VCPU_FAULT_DISR,      offsetof(struct kvm_vcpu, arch.fault.disr_el1));
   DEFINE(VCPU_WORKAROUND_FLAGS,        offsetof(struct kvm_vcpu, arch.workaround_flags));
   DEFINE(VCPU_HCR_EL2,         offsetof(struct kvm_vcpu, arch.hcr_el2));
-  DEFINE(CPU_GP_REGS,          offsetof(struct kvm_cpu_context, gp_regs));
+  DEFINE(CPU_USER_PT_REGS,     offsetof(struct kvm_cpu_context, regs));
   DEFINE(CPU_APIAKEYLO_EL1,    offsetof(struct kvm_cpu_context, sys_regs[APIAKEYLO_EL1]));
   DEFINE(CPU_APIBKEYLO_EL1,    offsetof(struct kvm_cpu_context, sys_regs[APIBKEYLO_EL1]));
   DEFINE(CPU_APDAKEYLO_EL1,    offsetof(struct kvm_cpu_context, sys_regs[APDAKEYLO_EL1]));
   DEFINE(CPU_APDBKEYLO_EL1,    offsetof(struct kvm_cpu_context, sys_regs[APDBKEYLO_EL1]));
   DEFINE(CPU_APGAKEYLO_EL1,    offsetof(struct kvm_cpu_context, sys_regs[APGAKEYLO_EL1]));
-  DEFINE(CPU_USER_PT_REGS,     offsetof(struct kvm_regs, regs));
   DEFINE(HOST_CONTEXT_VCPU,    offsetof(struct kvm_cpu_context, __hyp_running_vcpu));
   DEFINE(HOST_DATA_CONTEXT,    offsetof(struct kvm_host_data, host_ctxt));
 #endif
index e503caff14d1e59fa774a5372d43dce9b97b8214..3e081d556e81092c931c4981ffe6d22bc03e5386 100644 (file)
@@ -85,7 +85,7 @@ void kvm_arch_vcpu_ctxsync_fp(struct kvm_vcpu *vcpu)
        WARN_ON_ONCE(!irqs_disabled());
 
        if (vcpu->arch.flags & KVM_ARM64_FP_ENABLED) {
-               fpsimd_bind_state_to_cpu(&vcpu->arch.ctxt.gp_regs.fp_regs,
+               fpsimd_bind_state_to_cpu(&vcpu->arch.ctxt.fp_regs,
                                         vcpu->arch.sve_state,
                                         vcpu->arch.sve_max_vl);
 
index aea43ec60f3794eab324c50bbe4c54505ce67723..9dd5bbeefae60099feb32b8cc0f5d345b85dc2a2 100644 (file)
@@ -101,19 +101,60 @@ static int core_reg_size_from_offset(const struct kvm_vcpu *vcpu, u64 off)
        return size;
 }
 
-static int validate_core_offset(const struct kvm_vcpu *vcpu,
-                               const struct kvm_one_reg *reg)
+static void *core_reg_addr(struct kvm_vcpu *vcpu, const struct kvm_one_reg *reg)
 {
        u64 off = core_reg_offset_from_id(reg->id);
        int size = core_reg_size_from_offset(vcpu, off);
 
        if (size < 0)
-               return -EINVAL;
+               return NULL;
 
        if (KVM_REG_SIZE(reg->id) != size)
-               return -EINVAL;
+               return NULL;
 
-       return 0;
+       switch (off) {
+       case KVM_REG_ARM_CORE_REG(regs.regs[0]) ...
+            KVM_REG_ARM_CORE_REG(regs.regs[30]):
+               off -= KVM_REG_ARM_CORE_REG(regs.regs[0]);
+               off /= 2;
+               return &vcpu->arch.ctxt.regs.regs[off];
+
+       case KVM_REG_ARM_CORE_REG(regs.sp):
+               return &vcpu->arch.ctxt.regs.sp;
+
+       case KVM_REG_ARM_CORE_REG(regs.pc):
+               return &vcpu->arch.ctxt.regs.pc;
+
+       case KVM_REG_ARM_CORE_REG(regs.pstate):
+               return &vcpu->arch.ctxt.regs.pstate;
+
+       case KVM_REG_ARM_CORE_REG(sp_el1):
+               return &vcpu->arch.ctxt.sp_el1;
+
+       case KVM_REG_ARM_CORE_REG(elr_el1):
+               return &vcpu->arch.ctxt.elr_el1;
+
+       case KVM_REG_ARM_CORE_REG(spsr[0]) ...
+            KVM_REG_ARM_CORE_REG(spsr[KVM_NR_SPSR - 1]):
+               off -= KVM_REG_ARM_CORE_REG(spsr[0]);
+               off /= 2;
+               return &vcpu->arch.ctxt.spsr[off];
+
+       case KVM_REG_ARM_CORE_REG(fp_regs.vregs[0]) ...
+            KVM_REG_ARM_CORE_REG(fp_regs.vregs[31]):
+               off -= KVM_REG_ARM_CORE_REG(fp_regs.vregs[0]);
+               off /= 4;
+               return &vcpu->arch.ctxt.fp_regs.vregs[off];
+
+       case KVM_REG_ARM_CORE_REG(fp_regs.fpsr):
+               return &vcpu->arch.ctxt.fp_regs.fpsr;
+
+       case KVM_REG_ARM_CORE_REG(fp_regs.fpcr):
+               return &vcpu->arch.ctxt.fp_regs.fpcr;
+
+       default:
+               return NULL;
+       }
 }
 
 static int get_core_reg(struct kvm_vcpu *vcpu, const struct kvm_one_reg *reg)
@@ -125,8 +166,8 @@ static int get_core_reg(struct kvm_vcpu *vcpu, const struct kvm_one_reg *reg)
         * off the index in the "array".
         */
        __u32 __user *uaddr = (__u32 __user *)(unsigned long)reg->addr;
-       struct kvm_regs *regs = vcpu_gp_regs(vcpu);
-       int nr_regs = sizeof(*regs) / sizeof(__u32);
+       int nr_regs = sizeof(struct kvm_regs) / sizeof(__u32);
+       void *addr;
        u32 off;
 
        /* Our ID is an index into the kvm_regs struct. */
@@ -135,10 +176,11 @@ static int get_core_reg(struct kvm_vcpu *vcpu, const struct kvm_one_reg *reg)
            (off + (KVM_REG_SIZE(reg->id) / sizeof(__u32))) >= nr_regs)
                return -ENOENT;
 
-       if (validate_core_offset(vcpu, reg))
+       addr = core_reg_addr(vcpu, reg);
+       if (!addr)
                return -EINVAL;
 
-       if (copy_to_user(uaddr, ((u32 *)regs) + off, KVM_REG_SIZE(reg->id)))
+       if (copy_to_user(uaddr, addr, KVM_REG_SIZE(reg->id)))
                return -EFAULT;
 
        return 0;
@@ -147,10 +189,9 @@ static int get_core_reg(struct kvm_vcpu *vcpu, const struct kvm_one_reg *reg)
 static int set_core_reg(struct kvm_vcpu *vcpu, const struct kvm_one_reg *reg)
 {
        __u32 __user *uaddr = (__u32 __user *)(unsigned long)reg->addr;
-       struct kvm_regs *regs = vcpu_gp_regs(vcpu);
-       int nr_regs = sizeof(*regs) / sizeof(__u32);
+       int nr_regs = sizeof(struct kvm_regs) / sizeof(__u32);
        __uint128_t tmp;
-       void *valp = &tmp;
+       void *valp = &tmp, *addr;
        u64 off;
        int err = 0;
 
@@ -160,7 +201,8 @@ static int set_core_reg(struct kvm_vcpu *vcpu, const struct kvm_one_reg *reg)
            (off + (KVM_REG_SIZE(reg->id) / sizeof(__u32))) >= nr_regs)
                return -ENOENT;
 
-       if (validate_core_offset(vcpu, reg))
+       addr = core_reg_addr(vcpu, reg);
+       if (!addr)
                return -EINVAL;
 
        if (KVM_REG_SIZE(reg->id) > sizeof(tmp))
@@ -198,7 +240,7 @@ static int set_core_reg(struct kvm_vcpu *vcpu, const struct kvm_one_reg *reg)
                }
        }
 
-       memcpy((u32 *)regs + off, valp, KVM_REG_SIZE(reg->id));
+       memcpy(addr, valp, KVM_REG_SIZE(reg->id));
 
        if (*vcpu_cpsr(vcpu) & PSR_MODE32_BIT) {
                int i;
index dfb4e6d359ab1448cebbbee24af12737dfdd8aa0..ee32a77433898da3e24bafcbee5d8c64ea2ab4d4 100644 (file)
@@ -16,8 +16,7 @@
 #include <asm/kvm_mmu.h>
 #include <asm/kvm_ptrauth.h>
 
-#define CPU_GP_REG_OFFSET(x)   (CPU_GP_REGS + x)
-#define CPU_XREG_OFFSET(x)     CPU_GP_REG_OFFSET(CPU_USER_PT_REGS + 8*x)
+#define CPU_XREG_OFFSET(x)     (CPU_USER_PT_REGS + 8*x)
 #define CPU_SP_EL0_OFFSET      (CPU_XREG_OFFSET(30) + 8)
 
        .text
index 70367699d69a1540b3f250fd82570c8ecc1cecf4..784a581071fcb3a697f14d61275ea1d6436bab71 100644 (file)
@@ -266,11 +266,11 @@ static inline bool __hyp_handle_fpsimd(struct kvm_vcpu *vcpu)
 
        if (sve_guest) {
                sve_load_state(vcpu_sve_pffr(vcpu),
-                              &vcpu->arch.ctxt.gp_regs.fp_regs.fpsr,
+                              &vcpu->arch.ctxt.fp_regs.fpsr,
                               sve_vq_from_vl(vcpu->arch.sve_max_vl) - 1);
                write_sysreg_s(__vcpu_sys_reg(vcpu, ZCR_EL1), SYS_ZCR_EL12);
        } else {
-               __fpsimd_restore_state(&vcpu->arch.ctxt.gp_regs.fp_regs);
+               __fpsimd_restore_state(&vcpu->arch.ctxt.fp_regs);
        }
 
        /* Skip restoring fpexc32 for AArch64 guests */
index 37ef3e2cdbef84ae8beb6d16995bc9b413272bdc..50938093cc5d19b1bdeb3050592f1a118425cd65 100644 (file)
@@ -46,15 +46,15 @@ static inline void __sysreg_save_el1_state(struct kvm_cpu_context *ctxt)
        ctxt_sys_reg(ctxt, PAR_EL1)     = read_sysreg(par_el1);
        ctxt_sys_reg(ctxt, TPIDR_EL1)   = read_sysreg(tpidr_el1);
 
-       ctxt->gp_regs.sp_el1            = read_sysreg(sp_el1);
-       ctxt->gp_regs.elr_el1           = read_sysreg_el1(SYS_ELR);
-       ctxt->gp_regs.spsr[KVM_SPSR_EL1]= read_sysreg_el1(SYS_SPSR);
+       ctxt->sp_el1                    = read_sysreg(sp_el1);
+       ctxt->elr_el1                   = read_sysreg_el1(SYS_ELR);
+       ctxt->spsr[KVM_SPSR_EL1]        = read_sysreg_el1(SYS_SPSR);
 }
 
 static inline void __sysreg_save_el2_return_state(struct kvm_cpu_context *ctxt)
 {
-       ctxt->gp_regs.regs.pc           = read_sysreg_el2(SYS_ELR);
-       ctxt->gp_regs.regs.pstate       = read_sysreg_el2(SYS_SPSR);
+       ctxt->regs.pc                   = read_sysreg_el2(SYS_ELR);
+       ctxt->regs.pstate               = read_sysreg_el2(SYS_SPSR);
 
        if (cpus_have_final_cap(ARM64_HAS_RAS_EXTN))
                ctxt_sys_reg(ctxt, DISR_EL1) = read_sysreg_s(SYS_VDISR_EL2);
@@ -125,14 +125,14 @@ static inline void __sysreg_restore_el1_state(struct kvm_cpu_context *ctxt)
                write_sysreg_el1(ctxt_sys_reg(ctxt, TCR_EL1),   SYS_TCR);
        }
 
-       write_sysreg(ctxt->gp_regs.sp_el1,              sp_el1);
-       write_sysreg_el1(ctxt->gp_regs.elr_el1,         SYS_ELR);
-       write_sysreg_el1(ctxt->gp_regs.spsr[KVM_SPSR_EL1],SYS_SPSR);
+       write_sysreg(ctxt->sp_el1,                      sp_el1);
+       write_sysreg_el1(ctxt->elr_el1,                 SYS_ELR);
+       write_sysreg_el1(ctxt->spsr[KVM_SPSR_EL1],      SYS_SPSR);
 }
 
 static inline void __sysreg_restore_el2_return_state(struct kvm_cpu_context *ctxt)
 {
-       u64 pstate = ctxt->gp_regs.regs.pstate;
+       u64 pstate = ctxt->regs.pstate;
        u64 mode = pstate & PSR_AA32_MODE_MASK;
 
        /*
@@ -149,7 +149,7 @@ static inline void __sysreg_restore_el2_return_state(struct kvm_cpu_context *ctx
        if (!(mode & PSR_MODE32_BIT) && mode >= PSR_MODE_EL2t)
                pstate = PSR_MODE_EL2h | PSR_IL_BIT;
 
-       write_sysreg_el2(ctxt->gp_regs.regs.pc,         SYS_ELR);
+       write_sysreg_el2(ctxt->regs.pc,                 SYS_ELR);
        write_sysreg_el2(pstate,                        SYS_SPSR);
 
        if (cpus_have_final_cap(ARM64_HAS_RAS_EXTN))
@@ -163,7 +163,7 @@ static inline void __sysreg32_save_state(struct kvm_vcpu *vcpu)
        if (!vcpu_el1_is_32bit(vcpu))
                return;
 
-       spsr = vcpu->arch.ctxt.gp_regs.spsr;
+       spsr = vcpu->arch.ctxt.spsr;
 
        spsr[KVM_SPSR_ABT] = read_sysreg(spsr_abt);
        spsr[KVM_SPSR_UND] = read_sysreg(spsr_und);
@@ -184,7 +184,7 @@ static inline void __sysreg32_restore_state(struct kvm_vcpu *vcpu)
        if (!vcpu_el1_is_32bit(vcpu))
                return;
 
-       spsr = vcpu->arch.ctxt.gp_regs.spsr;
+       spsr = vcpu->arch.ctxt.spsr;
 
        write_sysreg(spsr[KVM_SPSR_ABT], spsr_abt);
        write_sysreg(spsr[KVM_SPSR_UND], spsr_und);
index a900181e386786a9ae09a7bc49eaa9e5e39db84a..b1596f314087b5ebd694047a514cdaedece46418 100644 (file)
@@ -100,7 +100,7 @@ static const unsigned long vcpu_reg_offsets[VCPU_NR_MODES][16] = {
  */
 unsigned long *vcpu_reg32(const struct kvm_vcpu *vcpu, u8 reg_num)
 {
-       unsigned long *reg_array = (unsigned long *)&vcpu->arch.ctxt.gp_regs.regs;
+       unsigned long *reg_array = (unsigned long *)&vcpu->arch.ctxt.regs;
        unsigned long mode = *vcpu_cpsr(vcpu) & PSR_AA32_MODE_MASK;
 
        switch (mode) {
@@ -148,7 +148,7 @@ unsigned long vcpu_read_spsr32(const struct kvm_vcpu *vcpu)
        int spsr_idx = vcpu_spsr32_mode(vcpu);
 
        if (!vcpu->arch.sysregs_loaded_on_cpu)
-               return vcpu_gp_regs(vcpu)->spsr[spsr_idx];
+               return vcpu->arch.ctxt.spsr[spsr_idx];
 
        switch (spsr_idx) {
        case KVM_SPSR_SVC:
@@ -171,7 +171,7 @@ void vcpu_write_spsr32(struct kvm_vcpu *vcpu, unsigned long v)
        int spsr_idx = vcpu_spsr32_mode(vcpu);
 
        if (!vcpu->arch.sysregs_loaded_on_cpu) {
-               vcpu_gp_regs(vcpu)->spsr[spsr_idx] = v;
+               vcpu->arch.ctxt.spsr[spsr_idx] = v;
                return;
        }
 
index d3b2090237274f8ffe8f4d89de92074390d4075c..8ca8607f5a9fbe5bf385c24a69dac96ee04e446b 100644 (file)
@@ -288,7 +288,7 @@ int kvm_reset_vcpu(struct kvm_vcpu *vcpu)
 
        /* Reset core registers */
        memset(vcpu_gp_regs(vcpu), 0, sizeof(*vcpu_gp_regs(vcpu)));
-       vcpu_gp_regs(vcpu)->regs.pstate = pstate;
+       vcpu_gp_regs(vcpu)->pstate = pstate;
 
        /* Reset system registers */
        kvm_reset_sys_regs(vcpu);