Merge tag 'for-linus' of git://git.kernel.org/pub/scm/virt/kvm/kvm
authorLinus Torvalds <torvalds@linux-foundation.org>
Mon, 13 Apr 2015 16:47:01 +0000 (09:47 -0700)
committerLinus Torvalds <torvalds@linux-foundation.org>
Mon, 13 Apr 2015 16:47:01 +0000 (09:47 -0700)
Pull KVM updates from Paolo Bonzini:
 "First batch of KVM changes for 4.1

  The most interesting bit here is irqfd/ioeventfd support for ARM and
  ARM64.

  Summary:

  ARM/ARM64:
     fixes for live migration, irqfd and ioeventfd support (enabling
     vhost, too), page aging

  s390:
     interrupt handling rework, allowing to inject all local interrupts
     via new ioctl and to get/set the full local irq state for migration
     and introspection.  New ioctls to access memory by virtual address,
     and to get/set the guest storage keys.  SIMD support.

  MIPS:
     FPU and MIPS SIMD Architecture (MSA) support.  Includes some
     patches from Ralf Baechle's MIPS tree.

  x86:
     bugfixes (notably for pvclock, the others are small) and cleanups.
     Another small latency improvement for the TSC deadline timer"

* tag 'for-linus' of git://git.kernel.org/pub/scm/virt/kvm/kvm: (146 commits)
  KVM: use slowpath for cross page cached accesses
  kvm: mmu: lazy collapse small sptes into large sptes
  KVM: x86: Clear CR2 on VCPU reset
  KVM: x86: DR0-DR3 are not clear on reset
  KVM: x86: BSP in MSR_IA32_APICBASE is writable
  KVM: x86: simplify kvm_apic_map
  KVM: x86: avoid logical_map when it is invalid
  KVM: x86: fix mixed APIC mode broadcast
  KVM: x86: use MDA for interrupt matching
  kvm/ppc/mpic: drop unused IRQ_testbit
  KVM: nVMX: remove unnecessary double caching of MAXPHYADDR
  KVM: nVMX: checks for address bits beyond MAXPHYADDR on VM-entry
  KVM: x86: cache maxphyaddr CPUID leaf in struct kvm_vcpu
  KVM: vmx: pass error code with internal error #2
  x86: vdso: fix pvclock races with task migration
  KVM: remove kvm_read_hva and kvm_read_hva_atomic
  KVM: x86: optimize delivery of TSC deadline timer interrupt
  KVM: x86: extract blocking logic from __vcpu_run
  kvm: x86: fix x86 eflags fixed bit
  KVM: s390: migrate vcpu interrupt state
  ...

99 files changed:
Documentation/virtual/kvm/api.txt
Documentation/virtual/kvm/devices/s390_flic.txt
MAINTAINERS
arch/arm/include/asm/kvm_arm.h
arch/arm/include/asm/kvm_host.h
arch/arm/include/asm/kvm_mmio.h
arch/arm/include/uapi/asm/kvm.h
arch/arm/kernel/asm-offsets.c
arch/arm/kvm/Kconfig
arch/arm/kvm/Makefile
arch/arm/kvm/arm.c
arch/arm/kvm/guest.c
arch/arm/kvm/interrupts_head.S
arch/arm/kvm/mmio.c
arch/arm/kvm/mmu.c
arch/arm/kvm/trace.h
arch/arm64/include/asm/esr.h
arch/arm64/include/asm/kvm_arm.h
arch/arm64/include/asm/kvm_host.h
arch/arm64/include/asm/kvm_mmio.h
arch/arm64/include/uapi/asm/kvm.h
arch/arm64/kvm/Kconfig
arch/arm64/kvm/Makefile
arch/mips/include/asm/asmmacro-32.h
arch/mips/include/asm/asmmacro.h
arch/mips/include/asm/fpu.h
arch/mips/include/asm/kdebug.h
arch/mips/include/asm/kvm_host.h
arch/mips/include/asm/processor.h
arch/mips/include/uapi/asm/kvm.h
arch/mips/kernel/asm-offsets.c
arch/mips/kernel/genex.S
arch/mips/kernel/ptrace.c
arch/mips/kernel/r4k_fpu.S
arch/mips/kernel/traps.c
arch/mips/kvm/Makefile
arch/mips/kvm/emulate.c
arch/mips/kvm/fpu.S [new file with mode: 0644]
arch/mips/kvm/locore.S
arch/mips/kvm/mips.c
arch/mips/kvm/msa.S [new file with mode: 0644]
arch/mips/kvm/stats.c
arch/mips/kvm/tlb.c
arch/mips/kvm/trap_emul.c
arch/powerpc/kvm/mpic.c
arch/powerpc/kvm/powerpc.c
arch/s390/include/asm/kvm_host.h
arch/s390/include/uapi/asm/kvm.h
arch/s390/include/uapi/asm/sie.h
arch/s390/kernel/asm-offsets.c
arch/s390/kvm/diag.c
arch/s390/kvm/gaccess.c
arch/s390/kvm/gaccess.h
arch/s390/kvm/guestdbg.c
arch/s390/kvm/intercept.c
arch/s390/kvm/interrupt.c
arch/s390/kvm/kvm-s390.c
arch/s390/kvm/kvm-s390.h
arch/s390/kvm/priv.c
arch/s390/kvm/sigp.c
arch/x86/include/asm/kvm_host.h
arch/x86/include/asm/kvm_para.h
arch/x86/include/asm/pvclock.h
arch/x86/include/uapi/asm/vmx.h
arch/x86/kernel/pvclock.c
arch/x86/kvm/Makefile
arch/x86/kvm/cpuid.c
arch/x86/kvm/cpuid.h
arch/x86/kvm/emulate.c
arch/x86/kvm/i8254.c
arch/x86/kvm/i8254.h
arch/x86/kvm/i8259.c
arch/x86/kvm/ioapic.c
arch/x86/kvm/ioapic.h
arch/x86/kvm/irq.h
arch/x86/kvm/lapic.c
arch/x86/kvm/lapic.h
arch/x86/kvm/mmu.c
arch/x86/kvm/pmu.c
arch/x86/kvm/svm.c
arch/x86/kvm/vmx.c
arch/x86/kvm/x86.c
arch/x86/vdso/vclock_gettime.c
include/kvm/arm_arch_timer.h
include/kvm/arm_vgic.h
include/kvm/iodev.h [moved from virt/kvm/iodev.h with 66% similarity]
include/linux/kvm_host.h
include/linux/sched.h
include/uapi/linux/kvm.h
kernel/sched/core.c
virt/kvm/arm/arch_timer.c
virt/kvm/arm/vgic-v2-emul.c
virt/kvm/arm/vgic-v3-emul.c
virt/kvm/arm/vgic.c
virt/kvm/arm/vgic.h
virt/kvm/coalesced_mmio.c
virt/kvm/eventfd.c
virt/kvm/irqchip.c
virt/kvm/kvm_main.c

index b112efc816f155093ef80815db788a2cdddf0ecc..bc9f6fe44e27614c2f26f155385fe0d5d13d2e3b 100644 (file)
@@ -997,7 +997,7 @@ for vm-wide capabilities.
 4.38 KVM_GET_MP_STATE
 
 Capability: KVM_CAP_MP_STATE
-Architectures: x86, s390
+Architectures: x86, s390, arm, arm64
 Type: vcpu ioctl
 Parameters: struct kvm_mp_state (out)
 Returns: 0 on success; -1 on error
@@ -1011,7 +1011,7 @@ uniprocessor guests).
 
 Possible values are:
 
- - KVM_MP_STATE_RUNNABLE:        the vcpu is currently running [x86]
+ - KVM_MP_STATE_RUNNABLE:        the vcpu is currently running [x86,arm/arm64]
  - KVM_MP_STATE_UNINITIALIZED:   the vcpu is an application processor (AP)
                                  which has not yet received an INIT signal [x86]
  - KVM_MP_STATE_INIT_RECEIVED:   the vcpu has received an INIT signal, and is
@@ -1020,7 +1020,7 @@ Possible values are:
                                  is waiting for an interrupt [x86]
  - KVM_MP_STATE_SIPI_RECEIVED:   the vcpu has just received a SIPI (vector
                                  accessible via KVM_GET_VCPU_EVENTS) [x86]
- - KVM_MP_STATE_STOPPED:         the vcpu is stopped [s390]
+ - KVM_MP_STATE_STOPPED:         the vcpu is stopped [s390,arm/arm64]
  - KVM_MP_STATE_CHECK_STOP:      the vcpu is in a special error state [s390]
  - KVM_MP_STATE_OPERATING:       the vcpu is operating (running or halted)
                                  [s390]
@@ -1031,11 +1031,15 @@ On x86, this ioctl is only useful after KVM_CREATE_IRQCHIP. Without an
 in-kernel irqchip, the multiprocessing state must be maintained by userspace on
 these architectures.
 
+For arm/arm64:
+
+The only states that are valid are KVM_MP_STATE_STOPPED and
+KVM_MP_STATE_RUNNABLE which reflect if the vcpu is paused or not.
 
 4.39 KVM_SET_MP_STATE
 
 Capability: KVM_CAP_MP_STATE
-Architectures: x86, s390
+Architectures: x86, s390, arm, arm64
 Type: vcpu ioctl
 Parameters: struct kvm_mp_state (in)
 Returns: 0 on success; -1 on error
@@ -1047,6 +1051,10 @@ On x86, this ioctl is only useful after KVM_CREATE_IRQCHIP. Without an
 in-kernel irqchip, the multiprocessing state must be maintained by userspace on
 these architectures.
 
+For arm/arm64:
+
+The only states that are valid are KVM_MP_STATE_STOPPED and
+KVM_MP_STATE_RUNNABLE which reflect if the vcpu should be paused or not.
 
 4.40 KVM_SET_IDENTITY_MAP_ADDR
 
@@ -1967,15 +1975,25 @@ registers, find a list below:
   MIPS  | KVM_REG_MIPS_CP0_STATUS       | 32
   MIPS  | KVM_REG_MIPS_CP0_CAUSE        | 32
   MIPS  | KVM_REG_MIPS_CP0_EPC          | 64
+  MIPS  | KVM_REG_MIPS_CP0_PRID         | 32
   MIPS  | KVM_REG_MIPS_CP0_CONFIG       | 32
   MIPS  | KVM_REG_MIPS_CP0_CONFIG1      | 32
   MIPS  | KVM_REG_MIPS_CP0_CONFIG2      | 32
   MIPS  | KVM_REG_MIPS_CP0_CONFIG3      | 32
+  MIPS  | KVM_REG_MIPS_CP0_CONFIG4      | 32
+  MIPS  | KVM_REG_MIPS_CP0_CONFIG5      | 32
   MIPS  | KVM_REG_MIPS_CP0_CONFIG7      | 32
   MIPS  | KVM_REG_MIPS_CP0_ERROREPC     | 64
   MIPS  | KVM_REG_MIPS_COUNT_CTL        | 64
   MIPS  | KVM_REG_MIPS_COUNT_RESUME     | 64
   MIPS  | KVM_REG_MIPS_COUNT_HZ         | 64
+  MIPS  | KVM_REG_MIPS_FPR_32(0..31)    | 32
+  MIPS  | KVM_REG_MIPS_FPR_64(0..31)    | 64
+  MIPS  | KVM_REG_MIPS_VEC_128(0..31)   | 128
+  MIPS  | KVM_REG_MIPS_FCR_IR           | 32
+  MIPS  | KVM_REG_MIPS_FCR_CSR          | 32
+  MIPS  | KVM_REG_MIPS_MSA_IR           | 32
+  MIPS  | KVM_REG_MIPS_MSA_CSR          | 32
 
 ARM registers are mapped using the lower 32 bits.  The upper 16 of that
 is the register group type, or coprocessor number:
@@ -2029,6 +2047,25 @@ patterns depending on whether they're 32-bit or 64-bit registers:
 MIPS KVM control registers (see above) have the following id bit patterns:
   0x7030 0000 0002 <reg:16>
 
+MIPS FPU registers (see KVM_REG_MIPS_FPR_{32,64}() above) have the following
+id bit patterns depending on the size of the register being accessed. They are
+always accessed according to the current guest FPU mode (Status.FR and
+Config5.FRE), i.e. as the guest would see them, and they become unpredictable
+if the guest FPU mode is changed. MIPS SIMD Architecture (MSA) vector
+registers (see KVM_REG_MIPS_VEC_128() above) have similar patterns as they
+overlap the FPU registers:
+  0x7020 0000 0003 00 <0:3> <reg:5> (32-bit FPU registers)
+  0x7030 0000 0003 00 <0:3> <reg:5> (64-bit FPU registers)
+  0x7040 0000 0003 00 <0:3> <reg:5> (128-bit MSA vector registers)
+
+MIPS FPU control registers (see KVM_REG_MIPS_FCR_{IR,CSR} above) have the
+following id bit patterns:
+  0x7020 0000 0003 01 <0:3> <reg:5>
+
+MIPS MSA control registers (see KVM_REG_MIPS_MSA_{IR,CSR} above) have the
+following id bit patterns:
+  0x7020 0000 0003 02 <0:3> <reg:5>
+
 
 4.69 KVM_GET_ONE_REG
 
@@ -2234,7 +2271,7 @@ into the hash PTE second double word).
 4.75 KVM_IRQFD
 
 Capability: KVM_CAP_IRQFD
-Architectures: x86 s390
+Architectures: x86 s390 arm arm64
 Type: vm ioctl
 Parameters: struct kvm_irqfd (in)
 Returns: 0 on success, -1 on error
@@ -2260,6 +2297,10 @@ Note that closing the resamplefd is not sufficient to disable the
 irqfd.  The KVM_IRQFD_FLAG_RESAMPLE is only necessary on assignment
 and need not be specified with KVM_IRQFD_FLAG_DEASSIGN.
 
+On ARM/ARM64, the gsi field in the kvm_irqfd struct specifies the Shared
+Peripheral Interrupt (SPI) index, such that the GIC interrupt ID is
+given by gsi + 32.
+
 4.76 KVM_PPC_ALLOCATE_HTAB
 
 Capability: KVM_CAP_PPC_ALLOC_HTAB
@@ -2716,6 +2757,227 @@ The fields in each entry are defined as follows:
    eax, ebx, ecx, edx: the values returned by the cpuid instruction for
          this function/index combination
 
+4.89 KVM_S390_MEM_OP
+
+Capability: KVM_CAP_S390_MEM_OP
+Architectures: s390
+Type: vcpu ioctl
+Parameters: struct kvm_s390_mem_op (in)
+Returns: = 0 on success,
+         < 0 on generic error (e.g. -EFAULT or -ENOMEM),
+         > 0 if an exception occurred while walking the page tables
+
+Read or write data from/to the logical (virtual) memory of a VPCU.
+
+Parameters are specified via the following structure:
+
+struct kvm_s390_mem_op {
+       __u64 gaddr;            /* the guest address */
+       __u64 flags;            /* flags */
+       __u32 size;             /* amount of bytes */
+       __u32 op;               /* type of operation */
+       __u64 buf;              /* buffer in userspace */
+       __u8 ar;                /* the access register number */
+       __u8 reserved[31];      /* should be set to 0 */
+};
+
+The type of operation is specified in the "op" field. It is either
+KVM_S390_MEMOP_LOGICAL_READ for reading from logical memory space or
+KVM_S390_MEMOP_LOGICAL_WRITE for writing to logical memory space. The
+KVM_S390_MEMOP_F_CHECK_ONLY flag can be set in the "flags" field to check
+whether the corresponding memory access would create an access exception
+(without touching the data in the memory at the destination). In case an
+access exception occurred while walking the MMU tables of the guest, the
+ioctl returns a positive error number to indicate the type of exception.
+This exception is also raised directly at the corresponding VCPU if the
+flag KVM_S390_MEMOP_F_INJECT_EXCEPTION is set in the "flags" field.
+
+The start address of the memory region has to be specified in the "gaddr"
+field, and the length of the region in the "size" field. "buf" is the buffer
+supplied by the userspace application where the read data should be written
+to for KVM_S390_MEMOP_LOGICAL_READ, or where the data that should be written
+is stored for a KVM_S390_MEMOP_LOGICAL_WRITE. "buf" is unused and can be NULL
+when KVM_S390_MEMOP_F_CHECK_ONLY is specified. "ar" designates the access
+register number to be used.
+
+The "reserved" field is meant for future extensions. It is not used by
+KVM with the currently defined set of flags.
+
+4.90 KVM_S390_GET_SKEYS
+
+Capability: KVM_CAP_S390_SKEYS
+Architectures: s390
+Type: vm ioctl
+Parameters: struct kvm_s390_skeys
+Returns: 0 on success, KVM_S390_GET_KEYS_NONE if guest is not using storage
+         keys, negative value on error
+
+This ioctl is used to get guest storage key values on the s390
+architecture. The ioctl takes parameters via the kvm_s390_skeys struct.
+
+struct kvm_s390_skeys {
+       __u64 start_gfn;
+       __u64 count;
+       __u64 skeydata_addr;
+       __u32 flags;
+       __u32 reserved[9];
+};
+
+The start_gfn field is the number of the first guest frame whose storage keys
+you want to get.
+
+The count field is the number of consecutive frames (starting from start_gfn)
+whose storage keys to get. The count field must be at least 1 and the maximum
+allowed value is defined as KVM_S390_SKEYS_ALLOC_MAX. Values outside this range
+will cause the ioctl to return -EINVAL.
+
+The skeydata_addr field is the address to a buffer large enough to hold count
+bytes. This buffer will be filled with storage key data by the ioctl.
+
+4.91 KVM_S390_SET_SKEYS
+
+Capability: KVM_CAP_S390_SKEYS
+Architectures: s390
+Type: vm ioctl
+Parameters: struct kvm_s390_skeys
+Returns: 0 on success, negative value on error
+
+This ioctl is used to set guest storage key values on the s390
+architecture. The ioctl takes parameters via the kvm_s390_skeys struct.
+See section on KVM_S390_GET_SKEYS for struct definition.
+
+The start_gfn field is the number of the first guest frame whose storage keys
+you want to set.
+
+The count field is the number of consecutive frames (starting from start_gfn)
+whose storage keys to get. The count field must be at least 1 and the maximum
+allowed value is defined as KVM_S390_SKEYS_ALLOC_MAX. Values outside this range
+will cause the ioctl to return -EINVAL.
+
+The skeydata_addr field is the address to a buffer containing count bytes of
+storage keys. Each byte in the buffer will be set as the storage key for a
+single frame starting at start_gfn for count frames.
+
+Note: If any architecturally invalid key value is found in the given data then
+the ioctl will return -EINVAL.
+
+4.92 KVM_S390_IRQ
+
+Capability: KVM_CAP_S390_INJECT_IRQ
+Architectures: s390
+Type: vcpu ioctl
+Parameters: struct kvm_s390_irq (in)
+Returns: 0 on success, -1 on error
+Errors:
+  EINVAL: interrupt type is invalid
+          type is KVM_S390_SIGP_STOP and flag parameter is invalid value
+          type is KVM_S390_INT_EXTERNAL_CALL and code is bigger
+            than the maximum of VCPUs
+  EBUSY:  type is KVM_S390_SIGP_SET_PREFIX and vcpu is not stopped
+          type is KVM_S390_SIGP_STOP and a stop irq is already pending
+          type is KVM_S390_INT_EXTERNAL_CALL and an external call interrupt
+            is already pending
+
+Allows to inject an interrupt to the guest.
+
+Using struct kvm_s390_irq as a parameter allows
+to inject additional payload which is not
+possible via KVM_S390_INTERRUPT.
+
+Interrupt parameters are passed via kvm_s390_irq:
+
+struct kvm_s390_irq {
+       __u64 type;
+       union {
+               struct kvm_s390_io_info io;
+               struct kvm_s390_ext_info ext;
+               struct kvm_s390_pgm_info pgm;
+               struct kvm_s390_emerg_info emerg;
+               struct kvm_s390_extcall_info extcall;
+               struct kvm_s390_prefix_info prefix;
+               struct kvm_s390_stop_info stop;
+               struct kvm_s390_mchk_info mchk;
+               char reserved[64];
+       } u;
+};
+
+type can be one of the following:
+
+KVM_S390_SIGP_STOP - sigp stop; parameter in .stop
+KVM_S390_PROGRAM_INT - program check; parameters in .pgm
+KVM_S390_SIGP_SET_PREFIX - sigp set prefix; parameters in .prefix
+KVM_S390_RESTART - restart; no parameters
+KVM_S390_INT_CLOCK_COMP - clock comparator interrupt; no parameters
+KVM_S390_INT_CPU_TIMER - CPU timer interrupt; no parameters
+KVM_S390_INT_EMERGENCY - sigp emergency; parameters in .emerg
+KVM_S390_INT_EXTERNAL_CALL - sigp external call; parameters in .extcall
+KVM_S390_MCHK - machine check interrupt; parameters in .mchk
+
+
+Note that the vcpu ioctl is asynchronous to vcpu execution.
+
+4.94 KVM_S390_GET_IRQ_STATE
+
+Capability: KVM_CAP_S390_IRQ_STATE
+Architectures: s390
+Type: vcpu ioctl
+Parameters: struct kvm_s390_irq_state (out)
+Returns: >= number of bytes copied into buffer,
+         -EINVAL if buffer size is 0,
+         -ENOBUFS if buffer size is too small to fit all pending interrupts,
+         -EFAULT if the buffer address was invalid
+
+This ioctl allows userspace to retrieve the complete state of all currently
+pending interrupts in a single buffer. Use cases include migration
+and introspection. The parameter structure contains the address of a
+userspace buffer and its length:
+
+struct kvm_s390_irq_state {
+       __u64 buf;
+       __u32 flags;
+       __u32 len;
+       __u32 reserved[4];
+};
+
+Userspace passes in the above struct and for each pending interrupt a
+struct kvm_s390_irq is copied to the provided buffer.
+
+If -ENOBUFS is returned the buffer provided was too small and userspace
+may retry with a bigger buffer.
+
+4.95 KVM_S390_SET_IRQ_STATE
+
+Capability: KVM_CAP_S390_IRQ_STATE
+Architectures: s390
+Type: vcpu ioctl
+Parameters: struct kvm_s390_irq_state (in)
+Returns: 0 on success,
+         -EFAULT if the buffer address was invalid,
+         -EINVAL for an invalid buffer length (see below),
+         -EBUSY if there were already interrupts pending,
+         errors occurring when actually injecting the
+          interrupt. See KVM_S390_IRQ.
+
+This ioctl allows userspace to set the complete state of all cpu-local
+interrupts currently pending for the vcpu. It is intended for restoring
+interrupt state after a migration. The input parameter is a userspace buffer
+containing a struct kvm_s390_irq_state:
+
+struct kvm_s390_irq_state {
+       __u64 buf;
+       __u32 len;
+       __u32 pad;
+};
+
+The userspace memory referenced by buf contains a struct kvm_s390_irq
+for each interrupt to be injected into the guest.
+If one of the interrupts could not be injected for some reason the
+ioctl aborts.
+
+len must be a multiple of sizeof(struct kvm_s390_irq). It must be > 0
+and it must not exceed (max_vcpus + 32) * sizeof(struct kvm_s390_irq),
+which is the maximum number of possibly pending cpu-local interrupts.
+
 5. The kvm_run structure
 ------------------------
 
@@ -3189,6 +3451,31 @@ Parameters: none
 This capability enables the in-kernel irqchip for s390. Please refer to
 "4.24 KVM_CREATE_IRQCHIP" for details.
 
+6.9 KVM_CAP_MIPS_FPU
+
+Architectures: mips
+Target: vcpu
+Parameters: args[0] is reserved for future use (should be 0).
+
+This capability allows the use of the host Floating Point Unit by the guest. It
+allows the Config1.FP bit to be set to enable the FPU in the guest. Once this is
+done the KVM_REG_MIPS_FPR_* and KVM_REG_MIPS_FCR_* registers can be accessed
+(depending on the current guest FPU register mode), and the Status.FR,
+Config5.FRE bits are accessible via the KVM API and also from the guest,
+depending on them being supported by the FPU.
+
+6.10 KVM_CAP_MIPS_MSA
+
+Architectures: mips
+Target: vcpu
+Parameters: args[0] is reserved for future use (should be 0).
+
+This capability allows the use of the MIPS SIMD Architecture (MSA) by the guest.
+It allows the Config3.MSAP bit to be set to enable the use of MSA by the guest.
+Once this is done the KVM_REG_MIPS_VEC_* and KVM_REG_MIPS_MSA_* registers can be
+accessed, and the Config5.MSAEn bit is accessible via the KVM API and also from
+the guest.
+
 7. Capabilities that can be enabled on VMs
 ------------------------------------------
 
@@ -3248,3 +3535,41 @@ All other orders will be handled completely in user space.
 Only privileged operation exceptions will be checked for in the kernel (or even
 in the hardware prior to interception). If this capability is not enabled, the
 old way of handling SIGP orders is used (partially in kernel and user space).
+
+7.3 KVM_CAP_S390_VECTOR_REGISTERS
+
+Architectures: s390
+Parameters: none
+Returns: 0 on success, negative value on error
+
+Allows use of the vector registers introduced with z13 processor, and
+provides for the synchronization between host and user space.  Will
+return -EINVAL if the machine does not support vectors.
+
+7.4 KVM_CAP_S390_USER_STSI
+
+Architectures: s390
+Parameters: none
+
+This capability allows post-handlers for the STSI instruction. After
+initial handling in the kernel, KVM exits to user space with
+KVM_EXIT_S390_STSI to allow user space to insert further data.
+
+Before exiting to userspace, kvm handlers should fill in s390_stsi field of
+vcpu->run:
+struct {
+       __u64 addr;
+       __u8 ar;
+       __u8 reserved;
+       __u8 fc;
+       __u8 sel1;
+       __u16 sel2;
+} s390_stsi;
+
+@addr - guest address of STSI SYSIB
+@fc   - function code
+@sel1 - selector 1
+@sel2 - selector 2
+@ar   - access register number
+
+KVM handlers should exit to userspace with rc = -EREMOTE.
index 4ceef53164b0289237238c3cf29d83e38e5d34de..d1ad9d5cae467ceb2c1169ce8b53d70078aedf27 100644 (file)
@@ -27,6 +27,9 @@ Groups:
     Copies all floating interrupts into a buffer provided by userspace.
     When the buffer is too small it returns -ENOMEM, which is the indication
     for userspace to try again with a bigger buffer.
+    -ENOBUFS is returned when the allocation of a kernelspace buffer has
+    failed.
+    -EFAULT is returned when copying data to userspace failed.
     All interrupts remain pending, i.e. are not deleted from the list of
     currently pending interrupts.
     attr->addr contains the userspace address of the buffer into which all
index e8bdf1b17cdb09be255874c81eacc40d0f554c62..b84686826b23cca2bb9699d6fae5d9b8cae35afa 100644 (file)
@@ -5591,6 +5591,8 @@ S:        Supported
 F:     Documentation/*/kvm*.txt
 F:     Documentation/virtual/kvm/
 F:     arch/*/kvm/
+F:     arch/x86/kernel/kvm.c
+F:     arch/x86/kernel/kvmclock.c
 F:     arch/*/include/asm/kvm*
 F:     include/linux/kvm*
 F:     include/uapi/linux/kvm*
index 816db0bf2dd8addbd9844488b5a72d4495be72c7..d995821f1698c67bc3e57e2073af9cdc3353fab4 100644 (file)
 #define HSR_COND       (0xfU << HSR_COND_SHIFT)
 
 #define FSC_FAULT      (0x04)
+#define FSC_ACCESS     (0x08)
 #define FSC_PERM       (0x0c)
 
 /* Hyp Prefetch Fault Address Register (HPFAR/HDFAR) */
index 41008cd7c53f6b39d1476d5a46dc610c578e8d5c..d71607c16601b6b1e1a595e32562195ccd63f5b1 100644 (file)
@@ -27,6 +27,8 @@
 #include <asm/fpstate.h>
 #include <kvm/arm_arch_timer.h>
 
+#define __KVM_HAVE_ARCH_INTC_INITIALIZED
+
 #if defined(CONFIG_KVM_ARM_MAX_VCPUS)
 #define KVM_MAX_VCPUS CONFIG_KVM_ARM_MAX_VCPUS
 #else
@@ -165,19 +167,10 @@ void kvm_set_spte_hva(struct kvm *kvm, unsigned long hva, pte_t pte);
 
 unsigned long kvm_arm_num_regs(struct kvm_vcpu *vcpu);
 int kvm_arm_copy_reg_indices(struct kvm_vcpu *vcpu, u64 __user *indices);
+int kvm_age_hva(struct kvm *kvm, unsigned long start, unsigned long end);
+int kvm_test_age_hva(struct kvm *kvm, unsigned long hva);
 
 /* We do not have shadow page tables, hence the empty hooks */
-static inline int kvm_age_hva(struct kvm *kvm, unsigned long start,
-                             unsigned long end)
-{
-       return 0;
-}
-
-static inline int kvm_test_age_hva(struct kvm *kvm, unsigned long hva)
-{
-       return 0;
-}
-
 static inline void kvm_arch_mmu_notifier_invalidate_page(struct kvm *kvm,
                                                         unsigned long address)
 {
index 3f83db2f6cf053cb083c15b5fa910b5d7ae7ee61..d8e90c8cb5fa0ab4c0486cb6d9fd53d576456291 100644 (file)
@@ -28,28 +28,6 @@ struct kvm_decode {
        bool sign_extend;
 };
 
-/*
- * The in-kernel MMIO emulation code wants to use a copy of run->mmio,
- * which is an anonymous type. Use our own type instead.
- */
-struct kvm_exit_mmio {
-       phys_addr_t     phys_addr;
-       u8              data[8];
-       u32             len;
-       bool            is_write;
-       void            *private;
-};
-
-static inline void kvm_prepare_mmio(struct kvm_run *run,
-                                   struct kvm_exit_mmio *mmio)
-{
-       run->mmio.phys_addr     = mmio->phys_addr;
-       run->mmio.len           = mmio->len;
-       run->mmio.is_write      = mmio->is_write;
-       memcpy(run->mmio.data, mmio->data, mmio->len);
-       run->exit_reason        = KVM_EXIT_MMIO;
-}
-
 int kvm_handle_mmio_return(struct kvm_vcpu *vcpu, struct kvm_run *run);
 int io_mem_abort(struct kvm_vcpu *vcpu, struct kvm_run *run,
                 phys_addr_t fault_ipa);
index 0db25bc328643de55ded82f3b9583a748038d97d..2499867dd0d86d79477e85ac0933228b9bbcf6a2 100644 (file)
@@ -198,6 +198,9 @@ struct kvm_arch_memory_slot {
 /* Highest supported SPI, from VGIC_NR_IRQS */
 #define KVM_ARM_IRQ_GIC_MAX            127
 
+/* One single KVM irqchip, ie. the VGIC */
+#define KVM_NR_IRQCHIPS          1
+
 /* PSCI interface */
 #define KVM_PSCI_FN_BASE               0x95c1ba5e
 #define KVM_PSCI_FN(n)                 (KVM_PSCI_FN_BASE + (n))
index 2d2d6087b9b105d5dadcd66f9821deefe50d1e66..488eaac56028f59ed08fc4030de25dd7d6a59ccb 100644 (file)
@@ -190,7 +190,6 @@ int main(void)
   DEFINE(VCPU_HxFAR,           offsetof(struct kvm_vcpu, arch.fault.hxfar));
   DEFINE(VCPU_HPFAR,           offsetof(struct kvm_vcpu, arch.fault.hpfar));
   DEFINE(VCPU_HYP_PC,          offsetof(struct kvm_vcpu, arch.fault.hyp_pc));
-#ifdef CONFIG_KVM_ARM_VGIC
   DEFINE(VCPU_VGIC_CPU,                offsetof(struct kvm_vcpu, arch.vgic_cpu));
   DEFINE(VGIC_V2_CPU_HCR,      offsetof(struct vgic_cpu, vgic_v2.vgic_hcr));
   DEFINE(VGIC_V2_CPU_VMCR,     offsetof(struct vgic_cpu, vgic_v2.vgic_vmcr));
@@ -200,14 +199,11 @@ int main(void)
   DEFINE(VGIC_V2_CPU_APR,      offsetof(struct vgic_cpu, vgic_v2.vgic_apr));
   DEFINE(VGIC_V2_CPU_LR,       offsetof(struct vgic_cpu, vgic_v2.vgic_lr));
   DEFINE(VGIC_CPU_NR_LR,       offsetof(struct vgic_cpu, nr_lr));
-#ifdef CONFIG_KVM_ARM_TIMER
   DEFINE(VCPU_TIMER_CNTV_CTL,  offsetof(struct kvm_vcpu, arch.timer_cpu.cntv_ctl));
   DEFINE(VCPU_TIMER_CNTV_CVAL, offsetof(struct kvm_vcpu, arch.timer_cpu.cntv_cval));
   DEFINE(KVM_TIMER_CNTVOFF,    offsetof(struct kvm, arch.timer.cntvoff));
   DEFINE(KVM_TIMER_ENABLED,    offsetof(struct kvm, arch.timer.enabled));
-#endif
   DEFINE(KVM_VGIC_VCTRL,       offsetof(struct kvm, arch.vgic.vctrl_base));
-#endif
   DEFINE(KVM_VTTBR,            offsetof(struct kvm, arch.vttbr));
 #endif
   return 0; 
index 338ace78ed18611bcb4aea64baeb490fcf2bca05..f1f79d1043096093a780e46e4a68cf3796a153ac 100644 (file)
@@ -18,6 +18,7 @@ if VIRTUALIZATION
 
 config KVM
        bool "Kernel-based Virtual Machine (KVM) support"
+       depends on MMU && OF
        select PREEMPT_NOTIFIERS
        select ANON_INODES
        select HAVE_KVM_CPU_RELAX_INTERCEPT
@@ -26,10 +27,12 @@ config KVM
        select KVM_ARM_HOST
        select KVM_GENERIC_DIRTYLOG_READ_PROTECT
        select SRCU
-       depends on ARM_VIRT_EXT && ARM_LPAE
+       select MMU_NOTIFIER
+       select HAVE_KVM_EVENTFD
+       select HAVE_KVM_IRQFD
+       depends on ARM_VIRT_EXT && ARM_LPAE && ARM_ARCH_TIMER
        ---help---
-         Support hosting virtualized guest machines. You will also
-         need to select one or more of the processor modules below.
+         Support hosting virtualized guest machines.
 
          This module provides access to the hardware capabilities through
          a character device node named /dev/kvm.
@@ -37,10 +40,7 @@ config KVM
          If unsure, say N.
 
 config KVM_ARM_HOST
-       bool "KVM host support for ARM cpus."
-       depends on KVM
-       depends on MMU
-       select  MMU_NOTIFIER
+       bool
        ---help---
          Provides host support for ARM processors.
 
@@ -55,20 +55,4 @@ config KVM_ARM_MAX_VCPUS
          large, so only choose a reasonable number that you expect to
          actually use.
 
-config KVM_ARM_VGIC
-       bool "KVM support for Virtual GIC"
-       depends on KVM_ARM_HOST && OF
-       select HAVE_KVM_IRQCHIP
-       default y
-       ---help---
-         Adds support for a hardware assisted, in-kernel GIC emulation.
-
-config KVM_ARM_TIMER
-       bool "KVM support for Architected Timers"
-       depends on KVM_ARM_VGIC && ARM_ARCH_TIMER
-       select HAVE_KVM_IRQCHIP
-       default y
-       ---help---
-         Adds support for the Architected Timers in virtual machines
-
 endif # VIRTUALIZATION
index 443b8bea43e93e862653f5f153d3ce7759da5528..139e46c08b6ec5daff4a3692a569027cf23c0347 100644 (file)
@@ -7,7 +7,7 @@ ifeq ($(plus_virt),+virt)
        plus_virt_def := -DREQUIRES_VIRT=1
 endif
 
-ccflags-y += -Ivirt/kvm -Iarch/arm/kvm
+ccflags-y += -Iarch/arm/kvm
 CFLAGS_arm.o := -I. $(plus_virt_def)
 CFLAGS_mmu.o := -I.
 
@@ -15,12 +15,12 @@ AFLAGS_init.o := -Wa,-march=armv7-a$(plus_virt)
 AFLAGS_interrupts.o := -Wa,-march=armv7-a$(plus_virt)
 
 KVM := ../../../virt/kvm
-kvm-arm-y = $(KVM)/kvm_main.o $(KVM)/coalesced_mmio.o
+kvm-arm-y = $(KVM)/kvm_main.o $(KVM)/coalesced_mmio.o $(KVM)/eventfd.o
 
 obj-y += kvm-arm.o init.o interrupts.o
 obj-y += arm.o handle_exit.o guest.o mmu.o emulate.o reset.o
 obj-y += coproc.o coproc_a15.o coproc_a7.o mmio.o psci.o perf.o
-obj-$(CONFIG_KVM_ARM_VGIC) += $(KVM)/arm/vgic.o
-obj-$(CONFIG_KVM_ARM_VGIC) += $(KVM)/arm/vgic-v2.o
-obj-$(CONFIG_KVM_ARM_VGIC) += $(KVM)/arm/vgic-v2-emul.o
-obj-$(CONFIG_KVM_ARM_TIMER) += $(KVM)/arm/arch_timer.o
+obj-y += $(KVM)/arm/vgic.o
+obj-y += $(KVM)/arm/vgic-v2.o
+obj-y += $(KVM)/arm/vgic-v2-emul.o
+obj-y += $(KVM)/arm/arch_timer.o
index 5560f74f9eeef1e3e4d2c9c39fc672e539eee93f..6f536451ab784e99966a308c0892e8614214591f 100644 (file)
@@ -61,8 +61,6 @@ static atomic64_t kvm_vmid_gen = ATOMIC64_INIT(1);
 static u8 kvm_next_vmid;
 static DEFINE_SPINLOCK(kvm_vmid_lock);
 
-static bool vgic_present;
-
 static void kvm_arm_set_running_vcpu(struct kvm_vcpu *vcpu)
 {
        BUG_ON(preemptible());
@@ -173,8 +171,8 @@ int kvm_vm_ioctl_check_extension(struct kvm *kvm, long ext)
        int r;
        switch (ext) {
        case KVM_CAP_IRQCHIP:
-               r = vgic_present;
-               break;
+       case KVM_CAP_IRQFD:
+       case KVM_CAP_IOEVENTFD:
        case KVM_CAP_DEVICE_CTRL:
        case KVM_CAP_USER_MEMORY:
        case KVM_CAP_SYNC_MMU:
@@ -183,6 +181,7 @@ int kvm_vm_ioctl_check_extension(struct kvm *kvm, long ext)
        case KVM_CAP_ARM_PSCI:
        case KVM_CAP_ARM_PSCI_0_2:
        case KVM_CAP_READONLY_MEM:
+       case KVM_CAP_MP_STATE:
                r = 1;
                break;
        case KVM_CAP_COALESCED_MMIO:
@@ -268,7 +267,7 @@ void kvm_arch_vcpu_destroy(struct kvm_vcpu *vcpu)
 
 int kvm_cpu_has_pending_timer(struct kvm_vcpu *vcpu)
 {
-       return 0;
+       return kvm_timer_should_fire(vcpu);
 }
 
 int kvm_arch_vcpu_init(struct kvm_vcpu *vcpu)
@@ -313,13 +312,29 @@ int kvm_arch_vcpu_ioctl_set_guest_debug(struct kvm_vcpu *vcpu,
 int kvm_arch_vcpu_ioctl_get_mpstate(struct kvm_vcpu *vcpu,
                                    struct kvm_mp_state *mp_state)
 {
-       return -EINVAL;
+       if (vcpu->arch.pause)
+               mp_state->mp_state = KVM_MP_STATE_STOPPED;
+       else
+               mp_state->mp_state = KVM_MP_STATE_RUNNABLE;
+
+       return 0;
 }
 
 int kvm_arch_vcpu_ioctl_set_mpstate(struct kvm_vcpu *vcpu,
                                    struct kvm_mp_state *mp_state)
 {
-       return -EINVAL;
+       switch (mp_state->mp_state) {
+       case KVM_MP_STATE_RUNNABLE:
+               vcpu->arch.pause = false;
+               break;
+       case KVM_MP_STATE_STOPPED:
+               vcpu->arch.pause = true;
+               break;
+       default:
+               return -EINVAL;
+       }
+
+       return 0;
 }
 
 /**
@@ -452,6 +467,11 @@ static int kvm_vcpu_first_run_init(struct kvm_vcpu *vcpu)
        return 0;
 }
 
+bool kvm_arch_intc_initialized(struct kvm *kvm)
+{
+       return vgic_initialized(kvm);
+}
+
 static void vcpu_pause(struct kvm_vcpu *vcpu)
 {
        wait_queue_head_t *wq = kvm_arch_vcpu_wq(vcpu);
@@ -831,8 +851,6 @@ static int kvm_vm_ioctl_set_device_addr(struct kvm *kvm,
 
        switch (dev_id) {
        case KVM_ARM_DEVICE_VGIC_V2:
-               if (!vgic_present)
-                       return -ENXIO;
                return kvm_vgic_addr(kvm, type, &dev_addr->addr, true);
        default:
                return -ENODEV;
@@ -847,10 +865,7 @@ long kvm_arch_vm_ioctl(struct file *filp,
 
        switch (ioctl) {
        case KVM_CREATE_IRQCHIP: {
-               if (vgic_present)
-                       return kvm_vgic_create(kvm, KVM_DEV_TYPE_ARM_VGIC_V2);
-               else
-                       return -ENXIO;
+               return kvm_vgic_create(kvm, KVM_DEV_TYPE_ARM_VGIC_V2);
        }
        case KVM_ARM_SET_DEVICE_ADDR: {
                struct kvm_arm_device_addr dev_addr;
@@ -1035,10 +1050,6 @@ static int init_hyp_mode(void)
        if (err)
                goto out_free_context;
 
-#ifdef CONFIG_KVM_ARM_VGIC
-               vgic_present = true;
-#endif
-
        /*
         * Init HYP architected timer support
         */
index 384bab67c4629a9bece251d5577c9bf908f8348f..d503fbb787d362752b9b6b688b2829e19b675095 100644 (file)
@@ -109,22 +109,6 @@ int kvm_arch_vcpu_ioctl_set_regs(struct kvm_vcpu *vcpu, struct kvm_regs *regs)
        return -EINVAL;
 }
 
-#ifndef CONFIG_KVM_ARM_TIMER
-
-#define NUM_TIMER_REGS 0
-
-static int copy_timer_indices(struct kvm_vcpu *vcpu, u64 __user *uindices)
-{
-       return 0;
-}
-
-static bool is_timer_reg(u64 index)
-{
-       return false;
-}
-
-#else
-
 #define NUM_TIMER_REGS 3
 
 static bool is_timer_reg(u64 index)
@@ -152,8 +136,6 @@ static int copy_timer_indices(struct kvm_vcpu *vcpu, u64 __user *uindices)
        return 0;
 }
 
-#endif
-
 static int set_timer_reg(struct kvm_vcpu *vcpu, const struct kvm_one_reg *reg)
 {
        void __user *uaddr = (void __user *)(long)reg->addr;
index 14d488388480ea50a80d24b18bbed9636c8d1c25..35e4a3a0c476cd9730afa52192bea5b3ff6f20fd 100644 (file)
@@ -402,7 +402,6 @@ vcpu        .req    r0              @ vcpu pointer always in r0
  * Assumes vcpu pointer in vcpu reg
  */
 .macro save_vgic_state
-#ifdef CONFIG_KVM_ARM_VGIC
        /* Get VGIC VCTRL base into r2 */
        ldr     r2, [vcpu, #VCPU_KVM]
        ldr     r2, [r2, #KVM_VGIC_VCTRL]
@@ -460,7 +459,6 @@ ARM_BE8(rev r6, r6  )
        subs    r4, r4, #1
        bne     1b
 2:
-#endif
 .endm
 
 /*
@@ -469,7 +467,6 @@ ARM_BE8(rev r6, r6  )
  * Assumes vcpu pointer in vcpu reg
  */
 .macro restore_vgic_state
-#ifdef CONFIG_KVM_ARM_VGIC
        /* Get VGIC VCTRL base into r2 */
        ldr     r2, [vcpu, #VCPU_KVM]
        ldr     r2, [r2, #KVM_VGIC_VCTRL]
@@ -501,7 +498,6 @@ ARM_BE8(rev r6, r6  )
        subs    r4, r4, #1
        bne     1b
 2:
-#endif
 .endm
 
 #define CNTHCTL_PL1PCTEN       (1 << 0)
@@ -515,7 +511,6 @@ ARM_BE8(rev r6, r6  )
  * Clobbers r2-r5
  */
 .macro save_timer_state
-#ifdef CONFIG_KVM_ARM_TIMER
        ldr     r4, [vcpu, #VCPU_KVM]
        ldr     r2, [r4, #KVM_TIMER_ENABLED]
        cmp     r2, #0
@@ -537,7 +532,6 @@ ARM_BE8(rev r6, r6  )
        mcrr    p15, 4, r2, r2, c14     @ CNTVOFF
 
 1:
-#endif
        @ Allow physical timer/counter access for the host
        mrc     p15, 4, r2, c14, c1, 0  @ CNTHCTL
        orr     r2, r2, #(CNTHCTL_PL1PCEN | CNTHCTL_PL1PCTEN)
@@ -559,7 +553,6 @@ ARM_BE8(rev r6, r6  )
        bic     r2, r2, #CNTHCTL_PL1PCEN
        mcr     p15, 4, r2, c14, c1, 0  @ CNTHCTL
 
-#ifdef CONFIG_KVM_ARM_TIMER
        ldr     r4, [vcpu, #VCPU_KVM]
        ldr     r2, [r4, #KVM_TIMER_ENABLED]
        cmp     r2, #0
@@ -579,7 +572,6 @@ ARM_BE8(rev r6, r6  )
        and     r2, r2, #3
        mcr     p15, 0, r2, c14, c3, 1  @ CNTV_CTL
 1:
-#endif
 .endm
 
 .equ vmentry,  0
index 5d3bfc0eb3f000cb41cb217eb7fdc2611da85fc9..974b1c606d044c239bfa14ffbdf66f0fc982c4fb 100644 (file)
@@ -121,12 +121,11 @@ int kvm_handle_mmio_return(struct kvm_vcpu *vcpu, struct kvm_run *run)
        return 0;
 }
 
-static int decode_hsr(struct kvm_vcpu *vcpu, phys_addr_t fault_ipa,
-                     struct kvm_exit_mmio *mmio)
+static int decode_hsr(struct kvm_vcpu *vcpu, bool *is_write, int *len)
 {
        unsigned long rt;
-       int len;
-       bool is_write, sign_extend;
+       int access_size;
+       bool sign_extend;
 
        if (kvm_vcpu_dabt_isextabt(vcpu)) {
                /* cache operation on I/O addr, tell guest unsupported */
@@ -140,17 +139,15 @@ static int decode_hsr(struct kvm_vcpu *vcpu, phys_addr_t fault_ipa,
                return 1;
        }
 
-       len = kvm_vcpu_dabt_get_as(vcpu);
-       if (unlikely(len < 0))
-               return len;
+       access_size = kvm_vcpu_dabt_get_as(vcpu);
+       if (unlikely(access_size < 0))
+               return access_size;
 
-       is_write = kvm_vcpu_dabt_iswrite(vcpu);
+       *is_write = kvm_vcpu_dabt_iswrite(vcpu);
        sign_extend = kvm_vcpu_dabt_issext(vcpu);
        rt = kvm_vcpu_dabt_get_rd(vcpu);
 
-       mmio->is_write = is_write;
-       mmio->phys_addr = fault_ipa;
-       mmio->len = len;
+       *len = access_size;
        vcpu->arch.mmio_decode.sign_extend = sign_extend;
        vcpu->arch.mmio_decode.rt = rt;
 
@@ -165,20 +162,20 @@ static int decode_hsr(struct kvm_vcpu *vcpu, phys_addr_t fault_ipa,
 int io_mem_abort(struct kvm_vcpu *vcpu, struct kvm_run *run,
                 phys_addr_t fault_ipa)
 {
-       struct kvm_exit_mmio mmio;
        unsigned long data;
        unsigned long rt;
        int ret;
+       bool is_write;
+       int len;
+       u8 data_buf[8];
 
        /*
-        * Prepare MMIO operation. First stash it in a private
-        * structure that we can use for in-kernel emulation. If the
-        * kernel can't handle it, copy it into run->mmio and let user
-        * space do its magic.
+        * Prepare MMIO operation. First decode the syndrome data we get
+        * from the CPU. Then try if some in-kernel emulation feels
+        * responsible, otherwise let user space do its magic.
         */
-
        if (kvm_vcpu_dabt_isvalid(vcpu)) {
-               ret = decode_hsr(vcpu, fault_ipa, &mmio);
+               ret = decode_hsr(vcpu, &is_write, &len);
                if (ret)
                        return ret;
        } else {
@@ -188,21 +185,34 @@ int io_mem_abort(struct kvm_vcpu *vcpu, struct kvm_run *run,
 
        rt = vcpu->arch.mmio_decode.rt;
 
-       if (mmio.is_write) {
-               data = vcpu_data_guest_to_host(vcpu, *vcpu_reg(vcpu, rt),
-                                              mmio.len);
+       if (is_write) {
+               data = vcpu_data_guest_to_host(vcpu, *vcpu_reg(vcpu, rt), len);
+
+               trace_kvm_mmio(KVM_TRACE_MMIO_WRITE, len, fault_ipa, data);
+               mmio_write_buf(data_buf, len, data);
 
-               trace_kvm_mmio(KVM_TRACE_MMIO_WRITE, mmio.len,
-                              fault_ipa, data);
-               mmio_write_buf(mmio.data, mmio.len, data);
+               ret = kvm_io_bus_write(vcpu, KVM_MMIO_BUS, fault_ipa, len,
+                                      data_buf);
        } else {
-               trace_kvm_mmio(KVM_TRACE_MMIO_READ_UNSATISFIED, mmio.len,
+               trace_kvm_mmio(KVM_TRACE_MMIO_READ_UNSATISFIED, len,
                               fault_ipa, 0);
+
+               ret = kvm_io_bus_read(vcpu, KVM_MMIO_BUS, fault_ipa, len,
+                                     data_buf);
        }
 
-       if (vgic_handle_mmio(vcpu, run, &mmio))
+       /* Now prepare kvm_run for the potential return to userland. */
+       run->mmio.is_write      = is_write;
+       run->mmio.phys_addr     = fault_ipa;
+       run->mmio.len           = len;
+       memcpy(run->mmio.data, data_buf, len);
+
+       if (!ret) {
+               /* We handled the access successfully in the kernel. */
+               kvm_handle_mmio_return(vcpu, run);
                return 1;
+       }
 
-       kvm_prepare_mmio(run, &mmio);
+       run->exit_reason        = KVM_EXIT_MMIO;
        return 0;
 }
index 5656d79c5a44f4d2ca816e15b647abf29a114e0b..15b050d46fc968afdc53029ada4b7d945ee23515 100644 (file)
@@ -1330,10 +1330,51 @@ static int user_mem_abort(struct kvm_vcpu *vcpu, phys_addr_t fault_ipa,
 
 out_unlock:
        spin_unlock(&kvm->mmu_lock);
+       kvm_set_pfn_accessed(pfn);
        kvm_release_pfn_clean(pfn);
        return ret;
 }
 
+/*
+ * Resolve the access fault by making the page young again.
+ * Note that because the faulting entry is guaranteed not to be
+ * cached in the TLB, we don't need to invalidate anything.
+ */
+static void handle_access_fault(struct kvm_vcpu *vcpu, phys_addr_t fault_ipa)
+{
+       pmd_t *pmd;
+       pte_t *pte;
+       pfn_t pfn;
+       bool pfn_valid = false;
+
+       trace_kvm_access_fault(fault_ipa);
+
+       spin_lock(&vcpu->kvm->mmu_lock);
+
+       pmd = stage2_get_pmd(vcpu->kvm, NULL, fault_ipa);
+       if (!pmd || pmd_none(*pmd))     /* Nothing there */
+               goto out;
+
+       if (kvm_pmd_huge(*pmd)) {       /* THP, HugeTLB */
+               *pmd = pmd_mkyoung(*pmd);
+               pfn = pmd_pfn(*pmd);
+               pfn_valid = true;
+               goto out;
+       }
+
+       pte = pte_offset_kernel(pmd, fault_ipa);
+       if (pte_none(*pte))             /* Nothing there either */
+               goto out;
+
+       *pte = pte_mkyoung(*pte);       /* Just a page... */
+       pfn = pte_pfn(*pte);
+       pfn_valid = true;
+out:
+       spin_unlock(&vcpu->kvm->mmu_lock);
+       if (pfn_valid)
+               kvm_set_pfn_accessed(pfn);
+}
+
 /**
  * kvm_handle_guest_abort - handles all 2nd stage aborts
  * @vcpu:      the VCPU pointer
@@ -1364,7 +1405,8 @@ int kvm_handle_guest_abort(struct kvm_vcpu *vcpu, struct kvm_run *run)
 
        /* Check the stage-2 fault is trans. fault or write fault */
        fault_status = kvm_vcpu_trap_get_fault_type(vcpu);
-       if (fault_status != FSC_FAULT && fault_status != FSC_PERM) {
+       if (fault_status != FSC_FAULT && fault_status != FSC_PERM &&
+           fault_status != FSC_ACCESS) {
                kvm_err("Unsupported FSC: EC=%#x xFSC=%#lx ESR_EL2=%#lx\n",
                        kvm_vcpu_trap_get_class(vcpu),
                        (unsigned long)kvm_vcpu_trap_get_fault(vcpu),
@@ -1400,6 +1442,12 @@ int kvm_handle_guest_abort(struct kvm_vcpu *vcpu, struct kvm_run *run)
        /* Userspace should not be able to register out-of-bounds IPAs */
        VM_BUG_ON(fault_ipa >= KVM_PHYS_SIZE);
 
+       if (fault_status == FSC_ACCESS) {
+               handle_access_fault(vcpu, fault_ipa);
+               ret = 1;
+               goto out_unlock;
+       }
+
        ret = user_mem_abort(vcpu, fault_ipa, memslot, hva, fault_status);
        if (ret == 0)
                ret = 1;
@@ -1408,15 +1456,16 @@ out_unlock:
        return ret;
 }
 
-static void handle_hva_to_gpa(struct kvm *kvm,
-                             unsigned long start,
-                             unsigned long end,
-                             void (*handler)(struct kvm *kvm,
-                                             gpa_t gpa, void *data),
-                             void *data)
+static int handle_hva_to_gpa(struct kvm *kvm,
+                            unsigned long start,
+                            unsigned long end,
+                            int (*handler)(struct kvm *kvm,
+                                           gpa_t gpa, void *data),
+                            void *data)
 {
        struct kvm_memslots *slots;
        struct kvm_memory_slot *memslot;
+       int ret = 0;
 
        slots = kvm_memslots(kvm);
 
@@ -1440,14 +1489,17 @@ static void handle_hva_to_gpa(struct kvm *kvm,
 
                for (; gfn < gfn_end; ++gfn) {
                        gpa_t gpa = gfn << PAGE_SHIFT;
-                       handler(kvm, gpa, data);
+                       ret |= handler(kvm, gpa, data);
                }
        }
+
+       return ret;
 }
 
-static void kvm_unmap_hva_handler(struct kvm *kvm, gpa_t gpa, void *data)
+static int kvm_unmap_hva_handler(struct kvm *kvm, gpa_t gpa, void *data)
 {
        unmap_stage2_range(kvm, gpa, PAGE_SIZE);
+       return 0;
 }
 
 int kvm_unmap_hva(struct kvm *kvm, unsigned long hva)
@@ -1473,7 +1525,7 @@ int kvm_unmap_hva_range(struct kvm *kvm,
        return 0;
 }
 
-static void kvm_set_spte_handler(struct kvm *kvm, gpa_t gpa, void *data)
+static int kvm_set_spte_handler(struct kvm *kvm, gpa_t gpa, void *data)
 {
        pte_t *pte = (pte_t *)data;
 
@@ -1485,6 +1537,7 @@ static void kvm_set_spte_handler(struct kvm *kvm, gpa_t gpa, void *data)
         * through this calling path.
         */
        stage2_set_pte(kvm, NULL, gpa, pte, 0);
+       return 0;
 }
 
 
@@ -1501,6 +1554,67 @@ void kvm_set_spte_hva(struct kvm *kvm, unsigned long hva, pte_t pte)
        handle_hva_to_gpa(kvm, hva, end, &kvm_set_spte_handler, &stage2_pte);
 }
 
+static int kvm_age_hva_handler(struct kvm *kvm, gpa_t gpa, void *data)
+{
+       pmd_t *pmd;
+       pte_t *pte;
+
+       pmd = stage2_get_pmd(kvm, NULL, gpa);
+       if (!pmd || pmd_none(*pmd))     /* Nothing there */
+               return 0;
+
+       if (kvm_pmd_huge(*pmd)) {       /* THP, HugeTLB */
+               if (pmd_young(*pmd)) {
+                       *pmd = pmd_mkold(*pmd);
+                       return 1;
+               }
+
+               return 0;
+       }
+
+       pte = pte_offset_kernel(pmd, gpa);
+       if (pte_none(*pte))
+               return 0;
+
+       if (pte_young(*pte)) {
+               *pte = pte_mkold(*pte); /* Just a page... */
+               return 1;
+       }
+
+       return 0;
+}
+
+static int kvm_test_age_hva_handler(struct kvm *kvm, gpa_t gpa, void *data)
+{
+       pmd_t *pmd;
+       pte_t *pte;
+
+       pmd = stage2_get_pmd(kvm, NULL, gpa);
+       if (!pmd || pmd_none(*pmd))     /* Nothing there */
+               return 0;
+
+       if (kvm_pmd_huge(*pmd))         /* THP, HugeTLB */
+               return pmd_young(*pmd);
+
+       pte = pte_offset_kernel(pmd, gpa);
+       if (!pte_none(*pte))            /* Just a page... */
+               return pte_young(*pte);
+
+       return 0;
+}
+
+int kvm_age_hva(struct kvm *kvm, unsigned long start, unsigned long end)
+{
+       trace_kvm_age_hva(start, end);
+       return handle_hva_to_gpa(kvm, start, end, kvm_age_hva_handler, NULL);
+}
+
+int kvm_test_age_hva(struct kvm *kvm, unsigned long hva)
+{
+       trace_kvm_test_age_hva(hva);
+       return handle_hva_to_gpa(kvm, hva, hva, kvm_test_age_hva_handler, NULL);
+}
+
 void kvm_mmu_free_memory_caches(struct kvm_vcpu *vcpu)
 {
        mmu_free_memory_cache(&vcpu->arch.mmu_page_cache);
index 6817664b46b80419047066686a47a8bc7953ebeb..0ec35392d2083ac4d8df391de0670d6ea6be6378 100644 (file)
@@ -68,6 +68,21 @@ TRACE_EVENT(kvm_guest_fault,
                  __entry->hxfar, __entry->vcpu_pc)
 );
 
+TRACE_EVENT(kvm_access_fault,
+       TP_PROTO(unsigned long ipa),
+       TP_ARGS(ipa),
+
+       TP_STRUCT__entry(
+               __field(        unsigned long,  ipa             )
+       ),
+
+       TP_fast_assign(
+               __entry->ipa            = ipa;
+       ),
+
+       TP_printk("IPA: %lx", __entry->ipa)
+);
+
 TRACE_EVENT(kvm_irq_line,
        TP_PROTO(unsigned int type, int vcpu_idx, int irq_num, int level),
        TP_ARGS(type, vcpu_idx, irq_num, level),
@@ -210,6 +225,39 @@ TRACE_EVENT(kvm_set_spte_hva,
        TP_printk("mmu notifier set pte hva: %#08lx", __entry->hva)
 );
 
+TRACE_EVENT(kvm_age_hva,
+       TP_PROTO(unsigned long start, unsigned long end),
+       TP_ARGS(start, end),
+
+       TP_STRUCT__entry(
+               __field(        unsigned long,  start           )
+               __field(        unsigned long,  end             )
+       ),
+
+       TP_fast_assign(
+               __entry->start          = start;
+               __entry->end            = end;
+       ),
+
+       TP_printk("mmu notifier age hva: %#08lx -- %#08lx",
+                 __entry->start, __entry->end)
+);
+
+TRACE_EVENT(kvm_test_age_hva,
+       TP_PROTO(unsigned long hva),
+       TP_ARGS(hva),
+
+       TP_STRUCT__entry(
+               __field(        unsigned long,  hva             )
+       ),
+
+       TP_fast_assign(
+               __entry->hva            = hva;
+       ),
+
+       TP_printk("mmu notifier test age hva: %#08lx", __entry->hva)
+);
+
 TRACE_EVENT(kvm_hvc,
        TP_PROTO(unsigned long vcpu_pc, unsigned long r0, unsigned long imm),
        TP_ARGS(vcpu_pc, r0, imm),
index 92bbae38159821cb6ce45ab3dab456c7eedced15..70522450ca2342a66b3ec27167bb94e08590f6d1 100644 (file)
@@ -90,6 +90,7 @@
 #define ESR_ELx_FSC            (0x3F)
 #define ESR_ELx_FSC_TYPE       (0x3C)
 #define ESR_ELx_FSC_EXTABT     (0x10)
+#define ESR_ELx_FSC_ACCESS     (0x08)
 #define ESR_ELx_FSC_FAULT      (0x04)
 #define ESR_ELx_FSC_PERM       (0x0C)
 #define ESR_ELx_CV             (UL(1) << 24)
index 54bb4ba974417e269656d50adb524654851fbbd2..ac6fafb95fe71e48048fe3831f226853f2f4914d 100644 (file)
 
 /* For compatibility with fault code shared with 32-bit */
 #define FSC_FAULT      ESR_ELx_FSC_FAULT
+#define FSC_ACCESS     ESR_ELx_FSC_ACCESS
 #define FSC_PERM       ESR_ELx_FSC_PERM
 
 /* Hyp Prefetch Fault Address Register (HPFAR/HDFAR) */
index 8ac3c70fe3c6ae7f234c5505a4a3c0e5bbd6c4ff..f0f58c9beec0e87c8c7eefa7a8ba52ba518e07c3 100644 (file)
@@ -28,6 +28,8 @@
 #include <asm/kvm_asm.h>
 #include <asm/kvm_mmio.h>
 
+#define __KVM_HAVE_ARCH_INTC_INITIALIZED
+
 #if defined(CONFIG_KVM_ARM_MAX_VCPUS)
 #define KVM_MAX_VCPUS CONFIG_KVM_ARM_MAX_VCPUS
 #else
@@ -177,19 +179,10 @@ int kvm_unmap_hva(struct kvm *kvm, unsigned long hva);
 int kvm_unmap_hva_range(struct kvm *kvm,
                        unsigned long start, unsigned long end);
 void kvm_set_spte_hva(struct kvm *kvm, unsigned long hva, pte_t pte);
+int kvm_age_hva(struct kvm *kvm, unsigned long start, unsigned long end);
+int kvm_test_age_hva(struct kvm *kvm, unsigned long hva);
 
 /* We do not have shadow page tables, hence the empty hooks */
-static inline int kvm_age_hva(struct kvm *kvm, unsigned long start,
-                             unsigned long end)
-{
-       return 0;
-}
-
-static inline int kvm_test_age_hva(struct kvm *kvm, unsigned long hva)
-{
-       return 0;
-}
-
 static inline void kvm_arch_mmu_notifier_invalidate_page(struct kvm *kvm,
                                                         unsigned long address)
 {
index 9f52beb7cb1355e1ebf129b899830b7b463d67b2..889c908ee631b526594b5dfc32ef5dfde15480df 100644 (file)
@@ -31,28 +31,6 @@ struct kvm_decode {
        bool sign_extend;
 };
 
-/*
- * The in-kernel MMIO emulation code wants to use a copy of run->mmio,
- * which is an anonymous type. Use our own type instead.
- */
-struct kvm_exit_mmio {
-       phys_addr_t     phys_addr;
-       u8              data[8];
-       u32             len;
-       bool            is_write;
-       void            *private;
-};
-
-static inline void kvm_prepare_mmio(struct kvm_run *run,
-                                   struct kvm_exit_mmio *mmio)
-{
-       run->mmio.phys_addr     = mmio->phys_addr;
-       run->mmio.len           = mmio->len;
-       run->mmio.is_write      = mmio->is_write;
-       memcpy(run->mmio.data, mmio->data, mmio->len);
-       run->exit_reason        = KVM_EXIT_MMIO;
-}
-
 int kvm_handle_mmio_return(struct kvm_vcpu *vcpu, struct kvm_run *run);
 int io_mem_abort(struct kvm_vcpu *vcpu, struct kvm_run *run,
                 phys_addr_t fault_ipa);
index 3ef77a4660187ace735216639ffc1e8f8d11a38b..c154c0b7eb604ae09214beed276063f222af69ae 100644 (file)
@@ -191,6 +191,9 @@ struct kvm_arch_memory_slot {
 /* Highest supported SPI, from VGIC_NR_IRQS */
 #define KVM_ARM_IRQ_GIC_MAX            127
 
+/* One single KVM irqchip, ie. the VGIC */
+#define KVM_NR_IRQCHIPS          1
+
 /* PSCI interface */
 #define KVM_PSCI_FN_BASE               0x95c1ba5e
 #define KVM_PSCI_FN(n)                 (KVM_PSCI_FN_BASE + (n))
index f5590c81d95f9e494bd82b5dda127762d5ea01f7..5105e297ed5fef43509f264a001138d299fb5cba 100644 (file)
@@ -18,6 +18,7 @@ if VIRTUALIZATION
 
 config KVM
        bool "Kernel-based Virtual Machine (KVM) support"
+       depends on OF
        select MMU_NOTIFIER
        select PREEMPT_NOTIFIERS
        select ANON_INODES
@@ -25,10 +26,10 @@ config KVM
        select HAVE_KVM_ARCH_TLB_FLUSH_ALL
        select KVM_MMIO
        select KVM_ARM_HOST
-       select KVM_ARM_VGIC
-       select KVM_ARM_TIMER
        select KVM_GENERIC_DIRTYLOG_READ_PROTECT
        select SRCU
+       select HAVE_KVM_EVENTFD
+       select HAVE_KVM_IRQFD
        ---help---
          Support hosting virtualized guest machines.
 
@@ -50,17 +51,4 @@ config KVM_ARM_MAX_VCPUS
          large, so only choose a reasonable number that you expect to
          actually use.
 
-config KVM_ARM_VGIC
-       bool
-       depends on KVM_ARM_HOST && OF
-       select HAVE_KVM_IRQCHIP
-       ---help---
-         Adds support for a hardware assisted, in-kernel GIC emulation.
-
-config KVM_ARM_TIMER
-       bool
-       depends on KVM_ARM_VGIC
-       ---help---
-         Adds support for the Architected Timers in virtual machines.
-
 endif # VIRTUALIZATION
index 4e6e09ee4033503088d686af976a6f7b0f3ee46a..d5904f876cdb535a373c6299beeec09bb5538331 100644 (file)
@@ -2,7 +2,7 @@
 # Makefile for Kernel-based Virtual Machine module
 #
 
-ccflags-y += -Ivirt/kvm -Iarch/arm64/kvm
+ccflags-y += -Iarch/arm64/kvm
 CFLAGS_arm.o := -I.
 CFLAGS_mmu.o := -I.
 
@@ -11,7 +11,7 @@ ARM=../../../arch/arm/kvm
 
 obj-$(CONFIG_KVM_ARM_HOST) += kvm.o
 
-kvm-$(CONFIG_KVM_ARM_HOST) += $(KVM)/kvm_main.o $(KVM)/coalesced_mmio.o
+kvm-$(CONFIG_KVM_ARM_HOST) += $(KVM)/kvm_main.o $(KVM)/coalesced_mmio.o $(KVM)/eventfd.o
 kvm-$(CONFIG_KVM_ARM_HOST) += $(ARM)/arm.o $(ARM)/mmu.o $(ARM)/mmio.o
 kvm-$(CONFIG_KVM_ARM_HOST) += $(ARM)/psci.o $(ARM)/perf.o
 
@@ -19,11 +19,11 @@ kvm-$(CONFIG_KVM_ARM_HOST) += emulate.o inject_fault.o regmap.o
 kvm-$(CONFIG_KVM_ARM_HOST) += hyp.o hyp-init.o handle_exit.o
 kvm-$(CONFIG_KVM_ARM_HOST) += guest.o reset.o sys_regs.o sys_regs_generic_v8.o
 
-kvm-$(CONFIG_KVM_ARM_VGIC) += $(KVM)/arm/vgic.o
-kvm-$(CONFIG_KVM_ARM_VGIC) += $(KVM)/arm/vgic-v2.o
-kvm-$(CONFIG_KVM_ARM_VGIC) += $(KVM)/arm/vgic-v2-emul.o
-kvm-$(CONFIG_KVM_ARM_VGIC) += vgic-v2-switch.o
-kvm-$(CONFIG_KVM_ARM_VGIC) += $(KVM)/arm/vgic-v3.o
-kvm-$(CONFIG_KVM_ARM_VGIC) += $(KVM)/arm/vgic-v3-emul.o
-kvm-$(CONFIG_KVM_ARM_VGIC) += vgic-v3-switch.o
-kvm-$(CONFIG_KVM_ARM_TIMER) += $(KVM)/arm/arch_timer.o
+kvm-$(CONFIG_KVM_ARM_HOST) += $(KVM)/arm/vgic.o
+kvm-$(CONFIG_KVM_ARM_HOST) += $(KVM)/arm/vgic-v2.o
+kvm-$(CONFIG_KVM_ARM_HOST) += $(KVM)/arm/vgic-v2-emul.o
+kvm-$(CONFIG_KVM_ARM_HOST) += vgic-v2-switch.o
+kvm-$(CONFIG_KVM_ARM_HOST) += $(KVM)/arm/vgic-v3.o
+kvm-$(CONFIG_KVM_ARM_HOST) += $(KVM)/arm/vgic-v3-emul.o
+kvm-$(CONFIG_KVM_ARM_HOST) += vgic-v3-switch.o
+kvm-$(CONFIG_KVM_ARM_HOST) += $(KVM)/arm/arch_timer.o
index cdac7b3eeaf7fa6524b8f6ffb92edc7209eef4c5..80386470d3a4414da1a6ff17d14b624ddd417ad7 100644 (file)
        .set push
        SET_HARDFLOAT
        cfc1    \tmp,  fcr31
-       swc1    $f0,  THREAD_FPR0_LS64(\thread)
-       swc1    $f1,  THREAD_FPR1_LS64(\thread)
-       swc1    $f2,  THREAD_FPR2_LS64(\thread)
-       swc1    $f3,  THREAD_FPR3_LS64(\thread)
-       swc1    $f4,  THREAD_FPR4_LS64(\thread)
-       swc1    $f5,  THREAD_FPR5_LS64(\thread)
-       swc1    $f6,  THREAD_FPR6_LS64(\thread)
-       swc1    $f7,  THREAD_FPR7_LS64(\thread)
-       swc1    $f8,  THREAD_FPR8_LS64(\thread)
-       swc1    $f9,  THREAD_FPR9_LS64(\thread)
-       swc1    $f10, THREAD_FPR10_LS64(\thread)
-       swc1    $f11, THREAD_FPR11_LS64(\thread)
-       swc1    $f12, THREAD_FPR12_LS64(\thread)
-       swc1    $f13, THREAD_FPR13_LS64(\thread)
-       swc1    $f14, THREAD_FPR14_LS64(\thread)
-       swc1    $f15, THREAD_FPR15_LS64(\thread)
-       swc1    $f16, THREAD_FPR16_LS64(\thread)
-       swc1    $f17, THREAD_FPR17_LS64(\thread)
-       swc1    $f18, THREAD_FPR18_LS64(\thread)
-       swc1    $f19, THREAD_FPR19_LS64(\thread)
-       swc1    $f20, THREAD_FPR20_LS64(\thread)
-       swc1    $f21, THREAD_FPR21_LS64(\thread)
-       swc1    $f22, THREAD_FPR22_LS64(\thread)
-       swc1    $f23, THREAD_FPR23_LS64(\thread)
-       swc1    $f24, THREAD_FPR24_LS64(\thread)
-       swc1    $f25, THREAD_FPR25_LS64(\thread)
-       swc1    $f26, THREAD_FPR26_LS64(\thread)
-       swc1    $f27, THREAD_FPR27_LS64(\thread)
-       swc1    $f28, THREAD_FPR28_LS64(\thread)
-       swc1    $f29, THREAD_FPR29_LS64(\thread)
-       swc1    $f30, THREAD_FPR30_LS64(\thread)
-       swc1    $f31, THREAD_FPR31_LS64(\thread)
+       swc1    $f0,  THREAD_FPR0(\thread)
+       swc1    $f1,  THREAD_FPR1(\thread)
+       swc1    $f2,  THREAD_FPR2(\thread)
+       swc1    $f3,  THREAD_FPR3(\thread)
+       swc1    $f4,  THREAD_FPR4(\thread)
+       swc1    $f5,  THREAD_FPR5(\thread)
+       swc1    $f6,  THREAD_FPR6(\thread)
+       swc1    $f7,  THREAD_FPR7(\thread)
+       swc1    $f8,  THREAD_FPR8(\thread)
+       swc1    $f9,  THREAD_FPR9(\thread)
+       swc1    $f10, THREAD_FPR10(\thread)
+       swc1    $f11, THREAD_FPR11(\thread)
+       swc1    $f12, THREAD_FPR12(\thread)
+       swc1    $f13, THREAD_FPR13(\thread)
+       swc1    $f14, THREAD_FPR14(\thread)
+       swc1    $f15, THREAD_FPR15(\thread)
+       swc1    $f16, THREAD_FPR16(\thread)
+       swc1    $f17, THREAD_FPR17(\thread)
+       swc1    $f18, THREAD_FPR18(\thread)
+       swc1    $f19, THREAD_FPR19(\thread)
+       swc1    $f20, THREAD_FPR20(\thread)
+       swc1    $f21, THREAD_FPR21(\thread)
+       swc1    $f22, THREAD_FPR22(\thread)
+       swc1    $f23, THREAD_FPR23(\thread)
+       swc1    $f24, THREAD_FPR24(\thread)
+       swc1    $f25, THREAD_FPR25(\thread)
+       swc1    $f26, THREAD_FPR26(\thread)
+       swc1    $f27, THREAD_FPR27(\thread)
+       swc1    $f28, THREAD_FPR28(\thread)
+       swc1    $f29, THREAD_FPR29(\thread)
+       swc1    $f30, THREAD_FPR30(\thread)
+       swc1    $f31, THREAD_FPR31(\thread)
        sw      \tmp, THREAD_FCR31(\thread)
        .set pop
        .endm
        .set push
        SET_HARDFLOAT
        lw      \tmp, THREAD_FCR31(\thread)
-       lwc1    $f0,  THREAD_FPR0_LS64(\thread)
-       lwc1    $f1,  THREAD_FPR1_LS64(\thread)
-       lwc1    $f2,  THREAD_FPR2_LS64(\thread)
-       lwc1    $f3,  THREAD_FPR3_LS64(\thread)
-       lwc1    $f4,  THREAD_FPR4_LS64(\thread)
-       lwc1    $f5,  THREAD_FPR5_LS64(\thread)
-       lwc1    $f6,  THREAD_FPR6_LS64(\thread)
-       lwc1    $f7,  THREAD_FPR7_LS64(\thread)
-       lwc1    $f8,  THREAD_FPR8_LS64(\thread)
-       lwc1    $f9,  THREAD_FPR9_LS64(\thread)
-       lwc1    $f10, THREAD_FPR10_LS64(\thread)
-       lwc1    $f11, THREAD_FPR11_LS64(\thread)
-       lwc1    $f12, THREAD_FPR12_LS64(\thread)
-       lwc1    $f13, THREAD_FPR13_LS64(\thread)
-       lwc1    $f14, THREAD_FPR14_LS64(\thread)
-       lwc1    $f15, THREAD_FPR15_LS64(\thread)
-       lwc1    $f16, THREAD_FPR16_LS64(\thread)
-       lwc1    $f17, THREAD_FPR17_LS64(\thread)
-       lwc1    $f18, THREAD_FPR18_LS64(\thread)
-       lwc1    $f19, THREAD_FPR19_LS64(\thread)
-       lwc1    $f20, THREAD_FPR20_LS64(\thread)
-       lwc1    $f21, THREAD_FPR21_LS64(\thread)
-       lwc1    $f22, THREAD_FPR22_LS64(\thread)
-       lwc1    $f23, THREAD_FPR23_LS64(\thread)
-       lwc1    $f24, THREAD_FPR24_LS64(\thread)
-       lwc1    $f25, THREAD_FPR25_LS64(\thread)
-       lwc1    $f26, THREAD_FPR26_LS64(\thread)
-       lwc1    $f27, THREAD_FPR27_LS64(\thread)
-       lwc1    $f28, THREAD_FPR28_LS64(\thread)
-       lwc1    $f29, THREAD_FPR29_LS64(\thread)
-       lwc1    $f30, THREAD_FPR30_LS64(\thread)
-       lwc1    $f31, THREAD_FPR31_LS64(\thread)
+       lwc1    $f0,  THREAD_FPR0(\thread)
+       lwc1    $f1,  THREAD_FPR1(\thread)
+       lwc1    $f2,  THREAD_FPR2(\thread)
+       lwc1    $f3,  THREAD_FPR3(\thread)
+       lwc1    $f4,  THREAD_FPR4(\thread)
+       lwc1    $f5,  THREAD_FPR5(\thread)
+       lwc1    $f6,  THREAD_FPR6(\thread)
+       lwc1    $f7,  THREAD_FPR7(\thread)
+       lwc1    $f8,  THREAD_FPR8(\thread)
+       lwc1    $f9,  THREAD_FPR9(\thread)
+       lwc1    $f10, THREAD_FPR10(\thread)
+       lwc1    $f11, THREAD_FPR11(\thread)
+       lwc1    $f12, THREAD_FPR12(\thread)
+       lwc1    $f13, THREAD_FPR13(\thread)
+       lwc1    $f14, THREAD_FPR14(\thread)
+       lwc1    $f15, THREAD_FPR15(\thread)
+       lwc1    $f16, THREAD_FPR16(\thread)
+       lwc1    $f17, THREAD_FPR17(\thread)
+       lwc1    $f18, THREAD_FPR18(\thread)
+       lwc1    $f19, THREAD_FPR19(\thread)
+       lwc1    $f20, THREAD_FPR20(\thread)
+       lwc1    $f21, THREAD_FPR21(\thread)
+       lwc1    $f22, THREAD_FPR22(\thread)
+       lwc1    $f23, THREAD_FPR23(\thread)
+       lwc1    $f24, THREAD_FPR24(\thread)
+       lwc1    $f25, THREAD_FPR25(\thread)
+       lwc1    $f26, THREAD_FPR26(\thread)
+       lwc1    $f27, THREAD_FPR27(\thread)
+       lwc1    $f28, THREAD_FPR28(\thread)
+       lwc1    $f29, THREAD_FPR29(\thread)
+       lwc1    $f30, THREAD_FPR30(\thread)
+       lwc1    $f31, THREAD_FPR31(\thread)
        ctc1    \tmp, fcr31
        .set pop
        .endm
index 0cae4595e985bbc3d8043b3bb85aef66c582615b..6156ac8c4cfb9a854bf3ed3a5546606216161118 100644 (file)
        .set    push
        SET_HARDFLOAT
        cfc1    \tmp, fcr31
-       sdc1    $f0,  THREAD_FPR0_LS64(\thread)
-       sdc1    $f2,  THREAD_FPR2_LS64(\thread)
-       sdc1    $f4,  THREAD_FPR4_LS64(\thread)
-       sdc1    $f6,  THREAD_FPR6_LS64(\thread)
-       sdc1    $f8,  THREAD_FPR8_LS64(\thread)
-       sdc1    $f10, THREAD_FPR10_LS64(\thread)
-       sdc1    $f12, THREAD_FPR12_LS64(\thread)
-       sdc1    $f14, THREAD_FPR14_LS64(\thread)
-       sdc1    $f16, THREAD_FPR16_LS64(\thread)
-       sdc1    $f18, THREAD_FPR18_LS64(\thread)
-       sdc1    $f20, THREAD_FPR20_LS64(\thread)
-       sdc1    $f22, THREAD_FPR22_LS64(\thread)
-       sdc1    $f24, THREAD_FPR24_LS64(\thread)
-       sdc1    $f26, THREAD_FPR26_LS64(\thread)
-       sdc1    $f28, THREAD_FPR28_LS64(\thread)
-       sdc1    $f30, THREAD_FPR30_LS64(\thread)
+       sdc1    $f0,  THREAD_FPR0(\thread)
+       sdc1    $f2,  THREAD_FPR2(\thread)
+       sdc1    $f4,  THREAD_FPR4(\thread)
+       sdc1    $f6,  THREAD_FPR6(\thread)
+       sdc1    $f8,  THREAD_FPR8(\thread)
+       sdc1    $f10, THREAD_FPR10(\thread)
+       sdc1    $f12, THREAD_FPR12(\thread)
+       sdc1    $f14, THREAD_FPR14(\thread)
+       sdc1    $f16, THREAD_FPR16(\thread)
+       sdc1    $f18, THREAD_FPR18(\thread)
+       sdc1    $f20, THREAD_FPR20(\thread)
+       sdc1    $f22, THREAD_FPR22(\thread)
+       sdc1    $f24, THREAD_FPR24(\thread)
+       sdc1    $f26, THREAD_FPR26(\thread)
+       sdc1    $f28, THREAD_FPR28(\thread)
+       sdc1    $f30, THREAD_FPR30(\thread)
        sw      \tmp, THREAD_FCR31(\thread)
        .set    pop
        .endm
        .set    push
        .set    mips64r2
        SET_HARDFLOAT
-       sdc1    $f1,  THREAD_FPR1_LS64(\thread)
-       sdc1    $f3,  THREAD_FPR3_LS64(\thread)
-       sdc1    $f5,  THREAD_FPR5_LS64(\thread)
-       sdc1    $f7,  THREAD_FPR7_LS64(\thread)
-       sdc1    $f9,  THREAD_FPR9_LS64(\thread)
-       sdc1    $f11, THREAD_FPR11_LS64(\thread)
-       sdc1    $f13, THREAD_FPR13_LS64(\thread)
-       sdc1    $f15, THREAD_FPR15_LS64(\thread)
-       sdc1    $f17, THREAD_FPR17_LS64(\thread)
-       sdc1    $f19, THREAD_FPR19_LS64(\thread)
-       sdc1    $f21, THREAD_FPR21_LS64(\thread)
-       sdc1    $f23, THREAD_FPR23_LS64(\thread)
-       sdc1    $f25, THREAD_FPR25_LS64(\thread)
-       sdc1    $f27, THREAD_FPR27_LS64(\thread)
-       sdc1    $f29, THREAD_FPR29_LS64(\thread)
-       sdc1    $f31, THREAD_FPR31_LS64(\thread)
+       sdc1    $f1,  THREAD_FPR1(\thread)
+       sdc1    $f3,  THREAD_FPR3(\thread)
+       sdc1    $f5,  THREAD_FPR5(\thread)
+       sdc1    $f7,  THREAD_FPR7(\thread)
+       sdc1    $f9,  THREAD_FPR9(\thread)
+       sdc1    $f11, THREAD_FPR11(\thread)
+       sdc1    $f13, THREAD_FPR13(\thread)
+       sdc1    $f15, THREAD_FPR15(\thread)
+       sdc1    $f17, THREAD_FPR17(\thread)
+       sdc1    $f19, THREAD_FPR19(\thread)
+       sdc1    $f21, THREAD_FPR21(\thread)
+       sdc1    $f23, THREAD_FPR23(\thread)
+       sdc1    $f25, THREAD_FPR25(\thread)
+       sdc1    $f27, THREAD_FPR27(\thread)
+       sdc1    $f29, THREAD_FPR29(\thread)
+       sdc1    $f31, THREAD_FPR31(\thread)
        .set    pop
        .endm
 
        .set    push
        SET_HARDFLOAT
        lw      \tmp, THREAD_FCR31(\thread)
-       ldc1    $f0,  THREAD_FPR0_LS64(\thread)
-       ldc1    $f2,  THREAD_FPR2_LS64(\thread)
-       ldc1    $f4,  THREAD_FPR4_LS64(\thread)
-       ldc1    $f6,  THREAD_FPR6_LS64(\thread)
-       ldc1    $f8,  THREAD_FPR8_LS64(\thread)
-       ldc1    $f10, THREAD_FPR10_LS64(\thread)
-       ldc1    $f12, THREAD_FPR12_LS64(\thread)
-       ldc1    $f14, THREAD_FPR14_LS64(\thread)
-       ldc1    $f16, THREAD_FPR16_LS64(\thread)
-       ldc1    $f18, THREAD_FPR18_LS64(\thread)
-       ldc1    $f20, THREAD_FPR20_LS64(\thread)
-       ldc1    $f22, THREAD_FPR22_LS64(\thread)
-       ldc1    $f24, THREAD_FPR24_LS64(\thread)
-       ldc1    $f26, THREAD_FPR26_LS64(\thread)
-       ldc1    $f28, THREAD_FPR28_LS64(\thread)
-       ldc1    $f30, THREAD_FPR30_LS64(\thread)
+       ldc1    $f0,  THREAD_FPR0(\thread)
+       ldc1    $f2,  THREAD_FPR2(\thread)
+       ldc1    $f4,  THREAD_FPR4(\thread)
+       ldc1    $f6,  THREAD_FPR6(\thread)
+       ldc1    $f8,  THREAD_FPR8(\thread)
+       ldc1    $f10, THREAD_FPR10(\thread)
+       ldc1    $f12, THREAD_FPR12(\thread)
+       ldc1    $f14, THREAD_FPR14(\thread)
+       ldc1    $f16, THREAD_FPR16(\thread)
+       ldc1    $f18, THREAD_FPR18(\thread)
+       ldc1    $f20, THREAD_FPR20(\thread)
+       ldc1    $f22, THREAD_FPR22(\thread)
+       ldc1    $f24, THREAD_FPR24(\thread)
+       ldc1    $f26, THREAD_FPR26(\thread)
+       ldc1    $f28, THREAD_FPR28(\thread)
+       ldc1    $f30, THREAD_FPR30(\thread)
        ctc1    \tmp, fcr31
        .endm
 
        .set    push
        .set    mips64r2
        SET_HARDFLOAT
-       ldc1    $f1,  THREAD_FPR1_LS64(\thread)
-       ldc1    $f3,  THREAD_FPR3_LS64(\thread)
-       ldc1    $f5,  THREAD_FPR5_LS64(\thread)
-       ldc1    $f7,  THREAD_FPR7_LS64(\thread)
-       ldc1    $f9,  THREAD_FPR9_LS64(\thread)
-       ldc1    $f11, THREAD_FPR11_LS64(\thread)
-       ldc1    $f13, THREAD_FPR13_LS64(\thread)
-       ldc1    $f15, THREAD_FPR15_LS64(\thread)
-       ldc1    $f17, THREAD_FPR17_LS64(\thread)
-       ldc1    $f19, THREAD_FPR19_LS64(\thread)
-       ldc1    $f21, THREAD_FPR21_LS64(\thread)
-       ldc1    $f23, THREAD_FPR23_LS64(\thread)
-       ldc1    $f25, THREAD_FPR25_LS64(\thread)
-       ldc1    $f27, THREAD_FPR27_LS64(\thread)
-       ldc1    $f29, THREAD_FPR29_LS64(\thread)
-       ldc1    $f31, THREAD_FPR31_LS64(\thread)
+       ldc1    $f1,  THREAD_FPR1(\thread)
+       ldc1    $f3,  THREAD_FPR3(\thread)
+       ldc1    $f5,  THREAD_FPR5(\thread)
+       ldc1    $f7,  THREAD_FPR7(\thread)
+       ldc1    $f9,  THREAD_FPR9(\thread)
+       ldc1    $f11, THREAD_FPR11(\thread)
+       ldc1    $f13, THREAD_FPR13(\thread)
+       ldc1    $f15, THREAD_FPR15(\thread)
+       ldc1    $f17, THREAD_FPR17(\thread)
+       ldc1    $f19, THREAD_FPR19(\thread)
+       ldc1    $f21, THREAD_FPR21(\thread)
+       ldc1    $f23, THREAD_FPR23(\thread)
+       ldc1    $f25, THREAD_FPR25(\thread)
+       ldc1    $f27, THREAD_FPR27(\thread)
+       ldc1    $f29, THREAD_FPR29(\thread)
+       ldc1    $f31, THREAD_FPR31(\thread)
        .set    pop
        .endm
 
        .endm
 
 #ifdef TOOLCHAIN_SUPPORTS_MSA
+       .macro  _cfcmsa rd, cs
+       .set    push
+       .set    mips32r2
+       .set    msa
+       cfcmsa  \rd, $\cs
+       .set    pop
+       .endm
+
+       .macro  _ctcmsa cd, rs
+       .set    push
+       .set    mips32r2
+       .set    msa
+       ctcmsa  $\cd, \rs
+       .set    pop
+       .endm
+
        .macro  ld_d    wd, off, base
        .set    push
        .set    mips32r2
        .set    pop
        .endm
 
-       .macro  copy_u_w        rd, ws, n
+       .macro  copy_u_w        ws, n
        .set    push
        .set    mips32r2
        .set    msa
-       copy_u.w \rd, $w\ws[\n]
+       copy_u.w $1, $w\ws[\n]
        .set    pop
        .endm
 
-       .macro  copy_u_d        rd, ws, n
+       .macro  copy_u_d        ws, n
        .set    push
        .set    mips64r2
        .set    msa
-       copy_u.d \rd, $w\ws[\n]
+       copy_u.d $1, $w\ws[\n]
        .set    pop
        .endm
 
-       .macro  insert_w        wd, n, rs
+       .macro  insert_w        wd, n
        .set    push
        .set    mips32r2
        .set    msa
-       insert.w $w\wd[\n], \rs
+       insert.w $w\wd[\n], $1
        .set    pop
        .endm
 
-       .macro  insert_d        wd, n, rs
+       .macro  insert_d        wd, n
        .set    push
        .set    mips64r2
        .set    msa
-       insert.d $w\wd[\n], \rs
+       insert.d $w\wd[\n], $1
        .set    pop
        .endm
 #else
        /*
         * Temporary until all toolchains in use include MSA support.
         */
-       .macro  cfcmsa  rd, cs
+       .macro  _cfcmsa rd, cs
        .set    push
        .set    noat
        SET_HARDFLOAT
        .set    pop
        .endm
 
-       .macro  ctcmsa  cd, rs
+       .macro  _ctcmsa cd, rs
        .set    push
        .set    noat
        SET_HARDFLOAT
        .set    pop
        .endm
 
-       .macro  copy_u_w        rd, ws, n
+       .macro  copy_u_w        ws, n
        .set    push
        .set    noat
        SET_HARDFLOAT
        .insn
        .word   COPY_UW_MSA_INSN | (\n << 16) | (\ws << 11)
-       /* move triggers an assembler bug... */
-       or      \rd, $1, zero
        .set    pop
        .endm
 
-       .macro  copy_u_d        rd, ws, n
+       .macro  copy_u_d        ws, n
        .set    push
        .set    noat
        SET_HARDFLOAT
        .insn
        .word   COPY_UD_MSA_INSN | (\n << 16) | (\ws << 11)
-       /* move triggers an assembler bug... */
-       or      \rd, $1, zero
        .set    pop
        .endm
 
-       .macro  insert_w        wd, n, rs
+       .macro  insert_w        wd, n
        .set    push
        .set    noat
        SET_HARDFLOAT
-       /* move triggers an assembler bug... */
-       or      $1, \rs, zero
        .word   INSERT_W_MSA_INSN | (\n << 16) | (\wd << 6)
        .set    pop
        .endm
 
-       .macro  insert_d        wd, n, rs
+       .macro  insert_d        wd, n
        .set    push
        .set    noat
        SET_HARDFLOAT
-       /* move triggers an assembler bug... */
-       or      $1, \rs, zero
        .word   INSERT_D_MSA_INSN | (\n << 16) | (\wd << 6)
        .set    pop
        .endm
        .set    push
        .set    noat
        SET_HARDFLOAT
-       cfcmsa  $1, MSA_CSR
+       _cfcmsa $1, MSA_CSR
        sw      $1, THREAD_MSA_CSR(\thread)
        .set    pop
        .endm
        .set    noat
        SET_HARDFLOAT
        lw      $1, THREAD_MSA_CSR(\thread)
-       ctcmsa  MSA_CSR, $1
+       _ctcmsa MSA_CSR, $1
        .set    pop
        ld_d    0, THREAD_FPR0, \thread
        ld_d    1, THREAD_FPR1, \thread
        insert_w \wd, 2
        insert_w \wd, 3
 #endif
-       .if     31-\wd
-       msa_init_upper  (\wd+1)
-       .endif
        .endm
 
        .macro  msa_init_all_upper
        SET_HARDFLOAT
        not     $1, zero
        msa_init_upper  0
+       msa_init_upper  1
+       msa_init_upper  2
+       msa_init_upper  3
+       msa_init_upper  4
+       msa_init_upper  5
+       msa_init_upper  6
+       msa_init_upper  7
+       msa_init_upper  8
+       msa_init_upper  9
+       msa_init_upper  10
+       msa_init_upper  11
+       msa_init_upper  12
+       msa_init_upper  13
+       msa_init_upper  14
+       msa_init_upper  15
+       msa_init_upper  16
+       msa_init_upper  17
+       msa_init_upper  18
+       msa_init_upper  19
+       msa_init_upper  20
+       msa_init_upper  21
+       msa_init_upper  22
+       msa_init_upper  23
+       msa_init_upper  24
+       msa_init_upper  25
+       msa_init_upper  26
+       msa_init_upper  27
+       msa_init_upper  28
+       msa_init_upper  29
+       msa_init_upper  30
+       msa_init_upper  31
        .set    pop
        .endm
 
index dd083e999b08a14ffdbef46d5f5f4a0731e9f18e..b104ad9d655f2da157544fcf783a225377cb996d 100644 (file)
@@ -48,6 +48,12 @@ enum fpu_mode {
 #define FPU_FR_MASK            0x1
 };
 
+#define __disable_fpu()                                                        \
+do {                                                                   \
+       clear_c0_status(ST0_CU1);                                       \
+       disable_fpu_hazard();                                           \
+} while (0)
+
 static inline int __enable_fpu(enum fpu_mode mode)
 {
        int fr;
@@ -86,7 +92,12 @@ fr_common:
                enable_fpu_hazard();
 
                /* check FR has the desired value */
-               return (!!(read_c0_status() & ST0_FR) == !!fr) ? 0 : SIGFPE;
+               if (!!(read_c0_status() & ST0_FR) == !!fr)
+                       return 0;
+
+               /* unsupported FR value */
+               __disable_fpu();
+               return SIGFPE;
 
        default:
                BUG();
@@ -95,12 +106,6 @@ fr_common:
        return SIGFPE;
 }
 
-#define __disable_fpu()                                                        \
-do {                                                                   \
-       clear_c0_status(ST0_CU1);                                       \
-       disable_fpu_hazard();                                           \
-} while (0)
-
 #define clear_fpu_owner()      clear_thread_flag(TIF_USEDFPU)
 
 static inline int __is_fpu_owner(void)
@@ -170,6 +175,7 @@ static inline void lose_fpu(int save)
                }
                disable_msa();
                clear_thread_flag(TIF_USEDMSA);
+               __disable_fpu();
        } else if (is_fpu_owner()) {
                if (save)
                        _save_fp(current);
index 6a9af5fcb5d72ef7878dc9581f366b07568a1c0f..cba22ab7ad4d5fd9087aeb8181c6c45ba3398a8c 100644 (file)
@@ -10,7 +10,8 @@ enum die_val {
        DIE_RI,
        DIE_PAGE_FAULT,
        DIE_BREAK,
-       DIE_SSTEPBP
+       DIE_SSTEPBP,
+       DIE_MSAFP
 };
 
 #endif /* _ASM_MIPS_KDEBUG_H */
index ac4fc716062b791003c76f5572d56863bcdcb2cd..4c25823563fe16dfe8f4008351c111eb0dd5c4ad 100644 (file)
 
 /* MIPS KVM register ids */
 #define MIPS_CP0_32(_R, _S)                                    \
-       (KVM_REG_MIPS | KVM_REG_SIZE_U32 | 0x10000 | (8 * (_R) + (_S)))
+       (KVM_REG_MIPS_CP0 | KVM_REG_SIZE_U32 | (8 * (_R) + (_S)))
 
 #define MIPS_CP0_64(_R, _S)                                    \
-       (KVM_REG_MIPS | KVM_REG_SIZE_U64 | 0x10000 | (8 * (_R) + (_S)))
+       (KVM_REG_MIPS_CP0 | KVM_REG_SIZE_U64 | (8 * (_R) + (_S)))
 
 #define KVM_REG_MIPS_CP0_INDEX         MIPS_CP0_32(0, 0)
 #define KVM_REG_MIPS_CP0_ENTRYLO0      MIPS_CP0_64(2, 0)
 #define KVM_REG_MIPS_CP0_STATUS                MIPS_CP0_32(12, 0)
 #define KVM_REG_MIPS_CP0_CAUSE         MIPS_CP0_32(13, 0)
 #define KVM_REG_MIPS_CP0_EPC           MIPS_CP0_64(14, 0)
+#define KVM_REG_MIPS_CP0_PRID          MIPS_CP0_32(15, 0)
 #define KVM_REG_MIPS_CP0_EBASE         MIPS_CP0_64(15, 1)
 #define KVM_REG_MIPS_CP0_CONFIG                MIPS_CP0_32(16, 0)
 #define KVM_REG_MIPS_CP0_CONFIG1       MIPS_CP0_32(16, 1)
 #define KVM_REG_MIPS_CP0_CONFIG2       MIPS_CP0_32(16, 2)
 #define KVM_REG_MIPS_CP0_CONFIG3       MIPS_CP0_32(16, 3)
+#define KVM_REG_MIPS_CP0_CONFIG4       MIPS_CP0_32(16, 4)
+#define KVM_REG_MIPS_CP0_CONFIG5       MIPS_CP0_32(16, 5)
 #define KVM_REG_MIPS_CP0_CONFIG7       MIPS_CP0_32(16, 7)
 #define KVM_REG_MIPS_CP0_XCONTEXT      MIPS_CP0_64(20, 0)
 #define KVM_REG_MIPS_CP0_ERROREPC      MIPS_CP0_64(30, 0)
@@ -119,6 +122,10 @@ struct kvm_vcpu_stat {
        u32 syscall_exits;
        u32 resvd_inst_exits;
        u32 break_inst_exits;
+       u32 trap_inst_exits;
+       u32 msa_fpe_exits;
+       u32 fpe_exits;
+       u32 msa_disabled_exits;
        u32 flush_dcache_exits;
        u32 halt_successful_poll;
        u32 halt_wakeup;
@@ -138,6 +145,10 @@ enum kvm_mips_exit_types {
        SYSCALL_EXITS,
        RESVD_INST_EXITS,
        BREAK_INST_EXITS,
+       TRAP_INST_EXITS,
+       MSA_FPE_EXITS,
+       FPE_EXITS,
+       MSA_DISABLED_EXITS,
        FLUSH_DCACHE_EXITS,
        MAX_KVM_MIPS_EXIT_TYPES
 };
@@ -206,6 +217,8 @@ struct mips_coproc {
 #define MIPS_CP0_CONFIG1_SEL   1
 #define MIPS_CP0_CONFIG2_SEL   2
 #define MIPS_CP0_CONFIG3_SEL   3
+#define MIPS_CP0_CONFIG4_SEL   4
+#define MIPS_CP0_CONFIG5_SEL   5
 
 /* Config0 register bits */
 #define CP0C0_M                        31
@@ -262,31 +275,6 @@ struct mips_coproc {
 #define CP0C3_SM               1
 #define CP0C3_TL               0
 
-/* Have config1, Cacheable, noncoherent, write-back, write allocate*/
-#define MIPS_CONFIG0                                           \
-  ((1 << CP0C0_M) | (0x3 << CP0C0_K0))
-
-/* Have config2, no coprocessor2 attached, no MDMX support attached,
-   no performance counters, watch registers present,
-   no code compression, EJTAG present, no FPU, no watch registers */
-#define MIPS_CONFIG1                                           \
-((1 << CP0C1_M) |                                              \
- (0 << CP0C1_C2) | (0 << CP0C1_MD) | (0 << CP0C1_PC) |         \
- (0 << CP0C1_WR) | (0 << CP0C1_CA) | (1 << CP0C1_EP) |         \
- (0 << CP0C1_FP))
-
-/* Have config3, no tertiary/secondary caches implemented */
-#define MIPS_CONFIG2                                           \
-((1 << CP0C2_M))
-
-/* No config4, no DSP ASE, no large physaddr (PABITS),
-   no external interrupt controller, no vectored interrupts,
-   no 1kb pages, no SmartMIPS ASE, no trace logic */
-#define MIPS_CONFIG3                                           \
-((0 << CP0C3_M) | (0 << CP0C3_DSPP) | (0 << CP0C3_LPA) |       \
- (0 << CP0C3_VEIC) | (0 << CP0C3_VInt) | (0 << CP0C3_SP) |     \
- (0 << CP0C3_SM) | (0 << CP0C3_TL))
-
 /* MMU types, the first four entries have the same layout as the
    CP0C0_MT field.  */
 enum mips_mmu_types {
@@ -321,7 +309,9 @@ enum mips_mmu_types {
  */
 #define T_TRAP                 13      /* Trap instruction */
 #define T_VCEI                 14      /* Virtual coherency exception */
+#define T_MSAFPE               14      /* MSA floating point exception */
 #define T_FPE                  15      /* Floating point exception */
+#define T_MSADIS               21      /* MSA disabled exception */
 #define T_WATCH                        23      /* Watch address reference */
 #define T_VCED                 31      /* Virtual coherency data */
 
@@ -374,6 +364,9 @@ struct kvm_mips_tlb {
        long tlb_lo1;
 };
 
+#define KVM_MIPS_FPU_FPU       0x1
+#define KVM_MIPS_FPU_MSA       0x2
+
 #define KVM_MIPS_GUEST_TLB_SIZE        64
 struct kvm_vcpu_arch {
        void *host_ebase, *guest_ebase;
@@ -395,6 +388,8 @@ struct kvm_vcpu_arch {
 
        /* FPU State */
        struct mips_fpu_struct fpu;
+       /* Which FPU state is loaded (KVM_MIPS_FPU_*) */
+       unsigned int fpu_inuse;
 
        /* COP0 State */
        struct mips_coproc *cop0;
@@ -441,6 +436,9 @@ struct kvm_vcpu_arch {
 
        /* WAIT executed */
        int wait;
+
+       u8 fpu_enabled;
+       u8 msa_enabled;
 };
 
 
@@ -482,11 +480,15 @@ struct kvm_vcpu_arch {
 #define kvm_read_c0_guest_config1(cop0)                (cop0->reg[MIPS_CP0_CONFIG][1])
 #define kvm_read_c0_guest_config2(cop0)                (cop0->reg[MIPS_CP0_CONFIG][2])
 #define kvm_read_c0_guest_config3(cop0)                (cop0->reg[MIPS_CP0_CONFIG][3])
+#define kvm_read_c0_guest_config4(cop0)                (cop0->reg[MIPS_CP0_CONFIG][4])
+#define kvm_read_c0_guest_config5(cop0)                (cop0->reg[MIPS_CP0_CONFIG][5])
 #define kvm_read_c0_guest_config7(cop0)                (cop0->reg[MIPS_CP0_CONFIG][7])
 #define kvm_write_c0_guest_config(cop0, val)   (cop0->reg[MIPS_CP0_CONFIG][0] = (val))
 #define kvm_write_c0_guest_config1(cop0, val)  (cop0->reg[MIPS_CP0_CONFIG][1] = (val))
 #define kvm_write_c0_guest_config2(cop0, val)  (cop0->reg[MIPS_CP0_CONFIG][2] = (val))
 #define kvm_write_c0_guest_config3(cop0, val)  (cop0->reg[MIPS_CP0_CONFIG][3] = (val))
+#define kvm_write_c0_guest_config4(cop0, val)  (cop0->reg[MIPS_CP0_CONFIG][4] = (val))
+#define kvm_write_c0_guest_config5(cop0, val)  (cop0->reg[MIPS_CP0_CONFIG][5] = (val))
 #define kvm_write_c0_guest_config7(cop0, val)  (cop0->reg[MIPS_CP0_CONFIG][7] = (val))
 #define kvm_read_c0_guest_errorepc(cop0)       (cop0->reg[MIPS_CP0_ERROR_PC][0])
 #define kvm_write_c0_guest_errorepc(cop0, val) (cop0->reg[MIPS_CP0_ERROR_PC][0] = (val))
@@ -567,6 +569,31 @@ static inline void _kvm_atomic_change_c0_guest_reg(unsigned long *reg,
        kvm_set_c0_guest_ebase(cop0, ((val) & (change)));               \
 }
 
+/* Helpers */
+
+static inline bool kvm_mips_guest_can_have_fpu(struct kvm_vcpu_arch *vcpu)
+{
+       return (!__builtin_constant_p(cpu_has_fpu) || cpu_has_fpu) &&
+               vcpu->fpu_enabled;
+}
+
+static inline bool kvm_mips_guest_has_fpu(struct kvm_vcpu_arch *vcpu)
+{
+       return kvm_mips_guest_can_have_fpu(vcpu) &&
+               kvm_read_c0_guest_config1(vcpu->cop0) & MIPS_CONF1_FP;
+}
+
+static inline bool kvm_mips_guest_can_have_msa(struct kvm_vcpu_arch *vcpu)
+{
+       return (!__builtin_constant_p(cpu_has_msa) || cpu_has_msa) &&
+               vcpu->msa_enabled;
+}
+
+static inline bool kvm_mips_guest_has_msa(struct kvm_vcpu_arch *vcpu)
+{
+       return kvm_mips_guest_can_have_msa(vcpu) &&
+               kvm_read_c0_guest_config3(vcpu->cop0) & MIPS_CONF3_MSA;
+}
 
 struct kvm_mips_callbacks {
        int (*handle_cop_unusable)(struct kvm_vcpu *vcpu);
@@ -578,6 +605,10 @@ struct kvm_mips_callbacks {
        int (*handle_syscall)(struct kvm_vcpu *vcpu);
        int (*handle_res_inst)(struct kvm_vcpu *vcpu);
        int (*handle_break)(struct kvm_vcpu *vcpu);
+       int (*handle_trap)(struct kvm_vcpu *vcpu);
+       int (*handle_msa_fpe)(struct kvm_vcpu *vcpu);
+       int (*handle_fpe)(struct kvm_vcpu *vcpu);
+       int (*handle_msa_disabled)(struct kvm_vcpu *vcpu);
        int (*vm_init)(struct kvm *kvm);
        int (*vcpu_init)(struct kvm_vcpu *vcpu);
        int (*vcpu_setup)(struct kvm_vcpu *vcpu);
@@ -596,6 +627,8 @@ struct kvm_mips_callbacks {
                           const struct kvm_one_reg *reg, s64 *v);
        int (*set_one_reg)(struct kvm_vcpu *vcpu,
                           const struct kvm_one_reg *reg, s64 v);
+       int (*vcpu_get_regs)(struct kvm_vcpu *vcpu);
+       int (*vcpu_set_regs)(struct kvm_vcpu *vcpu);
 };
 extern struct kvm_mips_callbacks *kvm_mips_callbacks;
 int kvm_mips_emulation_init(struct kvm_mips_callbacks **install_callbacks);
@@ -606,6 +639,19 @@ int kvm_arch_vcpu_dump_regs(struct kvm_vcpu *vcpu);
 /* Trampoline ASM routine to start running in "Guest" context */
 extern int __kvm_mips_vcpu_run(struct kvm_run *run, struct kvm_vcpu *vcpu);
 
+/* FPU/MSA context management */
+void __kvm_save_fpu(struct kvm_vcpu_arch *vcpu);
+void __kvm_restore_fpu(struct kvm_vcpu_arch *vcpu);
+void __kvm_restore_fcsr(struct kvm_vcpu_arch *vcpu);
+void __kvm_save_msa(struct kvm_vcpu_arch *vcpu);
+void __kvm_restore_msa(struct kvm_vcpu_arch *vcpu);
+void __kvm_restore_msa_upper(struct kvm_vcpu_arch *vcpu);
+void __kvm_restore_msacsr(struct kvm_vcpu_arch *vcpu);
+void kvm_own_fpu(struct kvm_vcpu *vcpu);
+void kvm_own_msa(struct kvm_vcpu *vcpu);
+void kvm_drop_fpu(struct kvm_vcpu *vcpu);
+void kvm_lose_fpu(struct kvm_vcpu *vcpu);
+
 /* TLB handling */
 uint32_t kvm_get_kernel_asid(struct kvm_vcpu *vcpu);
 
@@ -711,6 +757,26 @@ extern enum emulation_result kvm_mips_emulate_bp_exc(unsigned long cause,
                                                     struct kvm_run *run,
                                                     struct kvm_vcpu *vcpu);
 
+extern enum emulation_result kvm_mips_emulate_trap_exc(unsigned long cause,
+                                                      uint32_t *opc,
+                                                      struct kvm_run *run,
+                                                      struct kvm_vcpu *vcpu);
+
+extern enum emulation_result kvm_mips_emulate_msafpe_exc(unsigned long cause,
+                                                        uint32_t *opc,
+                                                        struct kvm_run *run,
+                                                        struct kvm_vcpu *vcpu);
+
+extern enum emulation_result kvm_mips_emulate_fpe_exc(unsigned long cause,
+                                                     uint32_t *opc,
+                                                     struct kvm_run *run,
+                                                     struct kvm_vcpu *vcpu);
+
+extern enum emulation_result kvm_mips_emulate_msadis_exc(unsigned long cause,
+                                                        uint32_t *opc,
+                                                        struct kvm_run *run,
+                                                        struct kvm_vcpu *vcpu);
+
 extern enum emulation_result kvm_mips_complete_mmio_load(struct kvm_vcpu *vcpu,
                                                         struct kvm_run *run);
 
@@ -749,6 +815,11 @@ enum emulation_result kvm_mips_emulate_load(uint32_t inst,
                                            struct kvm_run *run,
                                            struct kvm_vcpu *vcpu);
 
+unsigned int kvm_mips_config1_wrmask(struct kvm_vcpu *vcpu);
+unsigned int kvm_mips_config3_wrmask(struct kvm_vcpu *vcpu);
+unsigned int kvm_mips_config4_wrmask(struct kvm_vcpu *vcpu);
+unsigned int kvm_mips_config5_wrmask(struct kvm_vcpu *vcpu);
+
 /* Dynamic binary translation */
 extern int kvm_mips_trans_cache_index(uint32_t inst, uint32_t *opc,
                                      struct kvm_vcpu *vcpu);
index b5dcbee01fd7a52641584cbbf8b80848f7c6f4b9..9b3b48e21c221ffdcfc04ec9cae6165576043225 100644 (file)
@@ -105,7 +105,7 @@ union fpureg {
 #ifdef CONFIG_CPU_LITTLE_ENDIAN
 # define FPR_IDX(width, idx)   (idx)
 #else
-# define FPR_IDX(width, idx)   ((FPU_REG_WIDTH / (width)) - 1 - (idx))
+# define FPR_IDX(width, idx)   ((idx) ^ ((64 / (width)) - 1))
 #endif
 
 #define BUILD_FPR_ACCESS(width) \
index 2c04b6d9ff85380de722745e934944411a5e33d3..6985eb59b08534581f7b4316655367182cb6f64f 100644 (file)
@@ -36,77 +36,85 @@ struct kvm_regs {
 
 /*
  * for KVM_GET_FPU and KVM_SET_FPU
- *
- * If Status[FR] is zero (32-bit FPU), the upper 32-bits of the FPRs
- * are zero filled.
  */
 struct kvm_fpu {
-       __u64 fpr[32];
-       __u32 fir;
-       __u32 fccr;
-       __u32 fexr;
-       __u32 fenr;
-       __u32 fcsr;
-       __u32 pad;
 };
 
 
 /*
- * For MIPS, we use KVM_SET_ONE_REG and KVM_GET_ONE_REG to access CP0
+ * For MIPS, we use KVM_SET_ONE_REG and KVM_GET_ONE_REG to access various
  * registers.  The id field is broken down as follows:
  *
- *  bits[2..0]   - Register 'sel' index.
- *  bits[7..3]   - Register 'rd'  index.
- *  bits[15..8]  - Must be zero.
- *  bits[31..16] - 1 -> CP0 registers.
- *  bits[51..32] - Must be zero.
  *  bits[63..52] - As per linux/kvm.h
+ *  bits[51..32] - Must be zero.
+ *  bits[31..16] - Register set.
+ *
+ * Register set = 0: GP registers from kvm_regs (see definitions below).
+ *
+ * Register set = 1: CP0 registers.
+ *  bits[15..8]  - Must be zero.
+ *  bits[7..3]   - Register 'rd'  index.
+ *  bits[2..0]   - Register 'sel' index.
+ *
+ * Register set = 2: KVM specific registers (see definitions below).
+ *
+ * Register set = 3: FPU / MSA registers (see definitions below).
  *
  * Other sets registers may be added in the future.  Each set would
  * have its own identifier in bits[31..16].
- *
- * The registers defined in struct kvm_regs are also accessible, the
- * id values for these are below.
  */
 
-#define KVM_REG_MIPS_R0 (KVM_REG_MIPS | KVM_REG_SIZE_U64 | 0)
-#define KVM_REG_MIPS_R1 (KVM_REG_MIPS | KVM_REG_SIZE_U64 | 1)
-#define KVM_REG_MIPS_R2 (KVM_REG_MIPS | KVM_REG_SIZE_U64 | 2)
-#define KVM_REG_MIPS_R3 (KVM_REG_MIPS | KVM_REG_SIZE_U64 | 3)
-#define KVM_REG_MIPS_R4 (KVM_REG_MIPS | KVM_REG_SIZE_U64 | 4)
-#define KVM_REG_MIPS_R5 (KVM_REG_MIPS | KVM_REG_SIZE_U64 | 5)
-#define KVM_REG_MIPS_R6 (KVM_REG_MIPS | KVM_REG_SIZE_U64 | 6)
-#define KVM_REG_MIPS_R7 (KVM_REG_MIPS | KVM_REG_SIZE_U64 | 7)
-#define KVM_REG_MIPS_R8 (KVM_REG_MIPS | KVM_REG_SIZE_U64 | 8)
-#define KVM_REG_MIPS_R9 (KVM_REG_MIPS | KVM_REG_SIZE_U64 | 9)
-#define KVM_REG_MIPS_R10 (KVM_REG_MIPS | KVM_REG_SIZE_U64 | 10)
-#define KVM_REG_MIPS_R11 (KVM_REG_MIPS | KVM_REG_SIZE_U64 | 11)
-#define KVM_REG_MIPS_R12 (KVM_REG_MIPS | KVM_REG_SIZE_U64 | 12)
-#define KVM_REG_MIPS_R13 (KVM_REG_MIPS | KVM_REG_SIZE_U64 | 13)
-#define KVM_REG_MIPS_R14 (KVM_REG_MIPS | KVM_REG_SIZE_U64 | 14)
-#define KVM_REG_MIPS_R15 (KVM_REG_MIPS | KVM_REG_SIZE_U64 | 15)
-#define KVM_REG_MIPS_R16 (KVM_REG_MIPS | KVM_REG_SIZE_U64 | 16)
-#define KVM_REG_MIPS_R17 (KVM_REG_MIPS | KVM_REG_SIZE_U64 | 17)
-#define KVM_REG_MIPS_R18 (KVM_REG_MIPS | KVM_REG_SIZE_U64 | 18)
-#define KVM_REG_MIPS_R19 (KVM_REG_MIPS | KVM_REG_SIZE_U64 | 19)
-#define KVM_REG_MIPS_R20 (KVM_REG_MIPS | KVM_REG_SIZE_U64 | 20)
-#define KVM_REG_MIPS_R21 (KVM_REG_MIPS | KVM_REG_SIZE_U64 | 21)
-#define KVM_REG_MIPS_R22 (KVM_REG_MIPS | KVM_REG_SIZE_U64 | 22)
-#define KVM_REG_MIPS_R23 (KVM_REG_MIPS | KVM_REG_SIZE_U64 | 23)
-#define KVM_REG_MIPS_R24 (KVM_REG_MIPS | KVM_REG_SIZE_U64 | 24)
-#define KVM_REG_MIPS_R25 (KVM_REG_MIPS | KVM_REG_SIZE_U64 | 25)
-#define KVM_REG_MIPS_R26 (KVM_REG_MIPS | KVM_REG_SIZE_U64 | 26)
-#define KVM_REG_MIPS_R27 (KVM_REG_MIPS | KVM_REG_SIZE_U64 | 27)
-#define KVM_REG_MIPS_R28 (KVM_REG_MIPS | KVM_REG_SIZE_U64 | 28)
-#define KVM_REG_MIPS_R29 (KVM_REG_MIPS | KVM_REG_SIZE_U64 | 29)
-#define KVM_REG_MIPS_R30 (KVM_REG_MIPS | KVM_REG_SIZE_U64 | 30)
-#define KVM_REG_MIPS_R31 (KVM_REG_MIPS | KVM_REG_SIZE_U64 | 31)
-
-#define KVM_REG_MIPS_HI (KVM_REG_MIPS | KVM_REG_SIZE_U64 | 32)
-#define KVM_REG_MIPS_LO (KVM_REG_MIPS | KVM_REG_SIZE_U64 | 33)
-#define KVM_REG_MIPS_PC (KVM_REG_MIPS | KVM_REG_SIZE_U64 | 34)
-
-/* KVM specific control registers */
+#define KVM_REG_MIPS_GP                (KVM_REG_MIPS | 0x0000000000000000ULL)
+#define KVM_REG_MIPS_CP0       (KVM_REG_MIPS | 0x0000000000010000ULL)
+#define KVM_REG_MIPS_KVM       (KVM_REG_MIPS | 0x0000000000020000ULL)
+#define KVM_REG_MIPS_FPU       (KVM_REG_MIPS | 0x0000000000030000ULL)
+
+
+/*
+ * KVM_REG_MIPS_GP - General purpose registers from kvm_regs.
+ */
+
+#define KVM_REG_MIPS_R0                (KVM_REG_MIPS_GP | KVM_REG_SIZE_U64 |  0)
+#define KVM_REG_MIPS_R1                (KVM_REG_MIPS_GP | KVM_REG_SIZE_U64 |  1)
+#define KVM_REG_MIPS_R2                (KVM_REG_MIPS_GP | KVM_REG_SIZE_U64 |  2)
+#define KVM_REG_MIPS_R3                (KVM_REG_MIPS_GP | KVM_REG_SIZE_U64 |  3)
+#define KVM_REG_MIPS_R4                (KVM_REG_MIPS_GP | KVM_REG_SIZE_U64 |  4)
+#define KVM_REG_MIPS_R5                (KVM_REG_MIPS_GP | KVM_REG_SIZE_U64 |  5)
+#define KVM_REG_MIPS_R6                (KVM_REG_MIPS_GP | KVM_REG_SIZE_U64 |  6)
+#define KVM_REG_MIPS_R7                (KVM_REG_MIPS_GP | KVM_REG_SIZE_U64 |  7)
+#define KVM_REG_MIPS_R8                (KVM_REG_MIPS_GP | KVM_REG_SIZE_U64 |  8)
+#define KVM_REG_MIPS_R9                (KVM_REG_MIPS_GP | KVM_REG_SIZE_U64 |  9)
+#define KVM_REG_MIPS_R10       (KVM_REG_MIPS_GP | KVM_REG_SIZE_U64 | 10)
+#define KVM_REG_MIPS_R11       (KVM_REG_MIPS_GP | KVM_REG_SIZE_U64 | 11)
+#define KVM_REG_MIPS_R12       (KVM_REG_MIPS_GP | KVM_REG_SIZE_U64 | 12)
+#define KVM_REG_MIPS_R13       (KVM_REG_MIPS_GP | KVM_REG_SIZE_U64 | 13)
+#define KVM_REG_MIPS_R14       (KVM_REG_MIPS_GP | KVM_REG_SIZE_U64 | 14)
+#define KVM_REG_MIPS_R15       (KVM_REG_MIPS_GP | KVM_REG_SIZE_U64 | 15)
+#define KVM_REG_MIPS_R16       (KVM_REG_MIPS_GP | KVM_REG_SIZE_U64 | 16)
+#define KVM_REG_MIPS_R17       (KVM_REG_MIPS_GP | KVM_REG_SIZE_U64 | 17)
+#define KVM_REG_MIPS_R18       (KVM_REG_MIPS_GP | KVM_REG_SIZE_U64 | 18)
+#define KVM_REG_MIPS_R19       (KVM_REG_MIPS_GP | KVM_REG_SIZE_U64 | 19)
+#define KVM_REG_MIPS_R20       (KVM_REG_MIPS_GP | KVM_REG_SIZE_U64 | 20)
+#define KVM_REG_MIPS_R21       (KVM_REG_MIPS_GP | KVM_REG_SIZE_U64 | 21)
+#define KVM_REG_MIPS_R22       (KVM_REG_MIPS_GP | KVM_REG_SIZE_U64 | 22)
+#define KVM_REG_MIPS_R23       (KVM_REG_MIPS_GP | KVM_REG_SIZE_U64 | 23)
+#define KVM_REG_MIPS_R24       (KVM_REG_MIPS_GP | KVM_REG_SIZE_U64 | 24)
+#define KVM_REG_MIPS_R25       (KVM_REG_MIPS_GP | KVM_REG_SIZE_U64 | 25)
+#define KVM_REG_MIPS_R26       (KVM_REG_MIPS_GP | KVM_REG_SIZE_U64 | 26)
+#define KVM_REG_MIPS_R27       (KVM_REG_MIPS_GP | KVM_REG_SIZE_U64 | 27)
+#define KVM_REG_MIPS_R28       (KVM_REG_MIPS_GP | KVM_REG_SIZE_U64 | 28)
+#define KVM_REG_MIPS_R29       (KVM_REG_MIPS_GP | KVM_REG_SIZE_U64 | 29)
+#define KVM_REG_MIPS_R30       (KVM_REG_MIPS_GP | KVM_REG_SIZE_U64 | 30)
+#define KVM_REG_MIPS_R31       (KVM_REG_MIPS_GP | KVM_REG_SIZE_U64 | 31)
+
+#define KVM_REG_MIPS_HI                (KVM_REG_MIPS_GP | KVM_REG_SIZE_U64 | 32)
+#define KVM_REG_MIPS_LO                (KVM_REG_MIPS_GP | KVM_REG_SIZE_U64 | 33)
+#define KVM_REG_MIPS_PC                (KVM_REG_MIPS_GP | KVM_REG_SIZE_U64 | 34)
+
+
+/*
+ * KVM_REG_MIPS_KVM - KVM specific control registers.
+ */
 
 /*
  * CP0_Count control
@@ -118,8 +126,7 @@ struct kvm_fpu {
  *        safely without losing time or guest timer interrupts.
  * Other: Reserved, do not change.
  */
-#define KVM_REG_MIPS_COUNT_CTL         (KVM_REG_MIPS | KVM_REG_SIZE_U64 | \
-                                        0x20000 | 0)
+#define KVM_REG_MIPS_COUNT_CTL     (KVM_REG_MIPS_KVM | KVM_REG_SIZE_U64 | 0)
 #define KVM_REG_MIPS_COUNT_CTL_DC      0x00000001
 
 /*
@@ -131,15 +138,46 @@ struct kvm_fpu {
  * emulated.
  * Modifications to times in the future are rejected.
  */
-#define KVM_REG_MIPS_COUNT_RESUME      (KVM_REG_MIPS | KVM_REG_SIZE_U64 | \
-                                        0x20000 | 1)
+#define KVM_REG_MIPS_COUNT_RESUME   (KVM_REG_MIPS_KVM | KVM_REG_SIZE_U64 | 1)
 /*
  * CP0_Count rate in Hz
  * Specifies the rate of the CP0_Count timer in Hz. Modifications occur without
  * discontinuities in CP0_Count.
  */
-#define KVM_REG_MIPS_COUNT_HZ          (KVM_REG_MIPS | KVM_REG_SIZE_U64 | \
-                                        0x20000 | 2)
+#define KVM_REG_MIPS_COUNT_HZ      (KVM_REG_MIPS_KVM | KVM_REG_SIZE_U64 | 2)
+
+
+/*
+ * KVM_REG_MIPS_FPU - Floating Point and MIPS SIMD Architecture (MSA) registers.
+ *
+ *  bits[15..8]  - Register subset (see definitions below).
+ *  bits[7..5]   - Must be zero.
+ *  bits[4..0]   - Register number within register subset.
+ */
+
+#define KVM_REG_MIPS_FPR       (KVM_REG_MIPS_FPU | 0x0000000000000000ULL)
+#define KVM_REG_MIPS_FCR       (KVM_REG_MIPS_FPU | 0x0000000000000100ULL)
+#define KVM_REG_MIPS_MSACR     (KVM_REG_MIPS_FPU | 0x0000000000000200ULL)
+
+/*
+ * KVM_REG_MIPS_FPR - Floating point / Vector registers.
+ */
+#define KVM_REG_MIPS_FPR_32(n) (KVM_REG_MIPS_FPR | KVM_REG_SIZE_U32  | (n))
+#define KVM_REG_MIPS_FPR_64(n) (KVM_REG_MIPS_FPR | KVM_REG_SIZE_U64  | (n))
+#define KVM_REG_MIPS_VEC_128(n)        (KVM_REG_MIPS_FPR | KVM_REG_SIZE_U128 | (n))
+
+/*
+ * KVM_REG_MIPS_FCR - Floating point control registers.
+ */
+#define KVM_REG_MIPS_FCR_IR    (KVM_REG_MIPS_FCR | KVM_REG_SIZE_U32 |  0)
+#define KVM_REG_MIPS_FCR_CSR   (KVM_REG_MIPS_FCR | KVM_REG_SIZE_U32 | 31)
+
+/*
+ * KVM_REG_MIPS_MSACR - MIPS SIMD Architecture (MSA) control registers.
+ */
+#define KVM_REG_MIPS_MSA_IR     (KVM_REG_MIPS_MSACR | KVM_REG_SIZE_U32 |  0)
+#define KVM_REG_MIPS_MSA_CSR    (KVM_REG_MIPS_MSACR | KVM_REG_SIZE_U32 |  1)
+
 
 /*
  * KVM MIPS specific structures and definitions
index 750d67ac41e9b19affe066d5be8d1f56f7363041..e59fd7cfac9e35b2eeb90912c58504b9ed435606 100644 (file)
@@ -167,72 +167,6 @@ void output_thread_fpu_defines(void)
        OFFSET(THREAD_FPR30, task_struct, thread.fpu.fpr[30]);
        OFFSET(THREAD_FPR31, task_struct, thread.fpu.fpr[31]);
 
-       /* the least significant 64 bits of each FP register */
-       OFFSET(THREAD_FPR0_LS64, task_struct,
-              thread.fpu.fpr[0].val64[FPR_IDX(64, 0)]);
-       OFFSET(THREAD_FPR1_LS64, task_struct,
-              thread.fpu.fpr[1].val64[FPR_IDX(64, 0)]);
-       OFFSET(THREAD_FPR2_LS64, task_struct,
-              thread.fpu.fpr[2].val64[FPR_IDX(64, 0)]);
-       OFFSET(THREAD_FPR3_LS64, task_struct,
-              thread.fpu.fpr[3].val64[FPR_IDX(64, 0)]);
-       OFFSET(THREAD_FPR4_LS64, task_struct,
-              thread.fpu.fpr[4].val64[FPR_IDX(64, 0)]);
-       OFFSET(THREAD_FPR5_LS64, task_struct,
-              thread.fpu.fpr[5].val64[FPR_IDX(64, 0)]);
-       OFFSET(THREAD_FPR6_LS64, task_struct,
-              thread.fpu.fpr[6].val64[FPR_IDX(64, 0)]);
-       OFFSET(THREAD_FPR7_LS64, task_struct,
-              thread.fpu.fpr[7].val64[FPR_IDX(64, 0)]);
-       OFFSET(THREAD_FPR8_LS64, task_struct,
-              thread.fpu.fpr[8].val64[FPR_IDX(64, 0)]);
-       OFFSET(THREAD_FPR9_LS64, task_struct,
-              thread.fpu.fpr[9].val64[FPR_IDX(64, 0)]);
-       OFFSET(THREAD_FPR10_LS64, task_struct,
-              thread.fpu.fpr[10].val64[FPR_IDX(64, 0)]);
-       OFFSET(THREAD_FPR11_LS64, task_struct,
-              thread.fpu.fpr[11].val64[FPR_IDX(64, 0)]);
-       OFFSET(THREAD_FPR12_LS64, task_struct,
-              thread.fpu.fpr[12].val64[FPR_IDX(64, 0)]);
-       OFFSET(THREAD_FPR13_LS64, task_struct,
-              thread.fpu.fpr[13].val64[FPR_IDX(64, 0)]);
-       OFFSET(THREAD_FPR14_LS64, task_struct,
-              thread.fpu.fpr[14].val64[FPR_IDX(64, 0)]);
-       OFFSET(THREAD_FPR15_LS64, task_struct,
-              thread.fpu.fpr[15].val64[FPR_IDX(64, 0)]);
-       OFFSET(THREAD_FPR16_LS64, task_struct,
-              thread.fpu.fpr[16].val64[FPR_IDX(64, 0)]);
-       OFFSET(THREAD_FPR17_LS64, task_struct,
-              thread.fpu.fpr[17].val64[FPR_IDX(64, 0)]);
-       OFFSET(THREAD_FPR18_LS64, task_struct,
-              thread.fpu.fpr[18].val64[FPR_IDX(64, 0)]);
-       OFFSET(THREAD_FPR19_LS64, task_struct,
-              thread.fpu.fpr[19].val64[FPR_IDX(64, 0)]);
-       OFFSET(THREAD_FPR20_LS64, task_struct,
-              thread.fpu.fpr[20].val64[FPR_IDX(64, 0)]);
-       OFFSET(THREAD_FPR21_LS64, task_struct,
-              thread.fpu.fpr[21].val64[FPR_IDX(64, 0)]);
-       OFFSET(THREAD_FPR22_LS64, task_struct,
-              thread.fpu.fpr[22].val64[FPR_IDX(64, 0)]);
-       OFFSET(THREAD_FPR23_LS64, task_struct,
-              thread.fpu.fpr[23].val64[FPR_IDX(64, 0)]);
-       OFFSET(THREAD_FPR24_LS64, task_struct,
-              thread.fpu.fpr[24].val64[FPR_IDX(64, 0)]);
-       OFFSET(THREAD_FPR25_LS64, task_struct,
-              thread.fpu.fpr[25].val64[FPR_IDX(64, 0)]);
-       OFFSET(THREAD_FPR26_LS64, task_struct,
-              thread.fpu.fpr[26].val64[FPR_IDX(64, 0)]);
-       OFFSET(THREAD_FPR27_LS64, task_struct,
-              thread.fpu.fpr[27].val64[FPR_IDX(64, 0)]);
-       OFFSET(THREAD_FPR28_LS64, task_struct,
-              thread.fpu.fpr[28].val64[FPR_IDX(64, 0)]);
-       OFFSET(THREAD_FPR29_LS64, task_struct,
-              thread.fpu.fpr[29].val64[FPR_IDX(64, 0)]);
-       OFFSET(THREAD_FPR30_LS64, task_struct,
-              thread.fpu.fpr[30].val64[FPR_IDX(64, 0)]);
-       OFFSET(THREAD_FPR31_LS64, task_struct,
-              thread.fpu.fpr[31].val64[FPR_IDX(64, 0)]);
-
        OFFSET(THREAD_FCR31, task_struct, thread.fpu.fcr31);
        OFFSET(THREAD_MSA_CSR, task_struct, thread.fpu.msacsr);
        BLANK();
@@ -470,6 +404,45 @@ void output_kvm_defines(void)
        OFFSET(VCPU_LO, kvm_vcpu_arch, lo);
        OFFSET(VCPU_HI, kvm_vcpu_arch, hi);
        OFFSET(VCPU_PC, kvm_vcpu_arch, pc);
+       BLANK();
+
+       OFFSET(VCPU_FPR0, kvm_vcpu_arch, fpu.fpr[0]);
+       OFFSET(VCPU_FPR1, kvm_vcpu_arch, fpu.fpr[1]);
+       OFFSET(VCPU_FPR2, kvm_vcpu_arch, fpu.fpr[2]);
+       OFFSET(VCPU_FPR3, kvm_vcpu_arch, fpu.fpr[3]);
+       OFFSET(VCPU_FPR4, kvm_vcpu_arch, fpu.fpr[4]);
+       OFFSET(VCPU_FPR5, kvm_vcpu_arch, fpu.fpr[5]);
+       OFFSET(VCPU_FPR6, kvm_vcpu_arch, fpu.fpr[6]);
+       OFFSET(VCPU_FPR7, kvm_vcpu_arch, fpu.fpr[7]);
+       OFFSET(VCPU_FPR8, kvm_vcpu_arch, fpu.fpr[8]);
+       OFFSET(VCPU_FPR9, kvm_vcpu_arch, fpu.fpr[9]);
+       OFFSET(VCPU_FPR10, kvm_vcpu_arch, fpu.fpr[10]);
+       OFFSET(VCPU_FPR11, kvm_vcpu_arch, fpu.fpr[11]);
+       OFFSET(VCPU_FPR12, kvm_vcpu_arch, fpu.fpr[12]);
+       OFFSET(VCPU_FPR13, kvm_vcpu_arch, fpu.fpr[13]);
+       OFFSET(VCPU_FPR14, kvm_vcpu_arch, fpu.fpr[14]);
+       OFFSET(VCPU_FPR15, kvm_vcpu_arch, fpu.fpr[15]);
+       OFFSET(VCPU_FPR16, kvm_vcpu_arch, fpu.fpr[16]);
+       OFFSET(VCPU_FPR17, kvm_vcpu_arch, fpu.fpr[17]);
+       OFFSET(VCPU_FPR18, kvm_vcpu_arch, fpu.fpr[18]);
+       OFFSET(VCPU_FPR19, kvm_vcpu_arch, fpu.fpr[19]);
+       OFFSET(VCPU_FPR20, kvm_vcpu_arch, fpu.fpr[20]);
+       OFFSET(VCPU_FPR21, kvm_vcpu_arch, fpu.fpr[21]);
+       OFFSET(VCPU_FPR22, kvm_vcpu_arch, fpu.fpr[22]);
+       OFFSET(VCPU_FPR23, kvm_vcpu_arch, fpu.fpr[23]);
+       OFFSET(VCPU_FPR24, kvm_vcpu_arch, fpu.fpr[24]);
+       OFFSET(VCPU_FPR25, kvm_vcpu_arch, fpu.fpr[25]);
+       OFFSET(VCPU_FPR26, kvm_vcpu_arch, fpu.fpr[26]);
+       OFFSET(VCPU_FPR27, kvm_vcpu_arch, fpu.fpr[27]);
+       OFFSET(VCPU_FPR28, kvm_vcpu_arch, fpu.fpr[28]);
+       OFFSET(VCPU_FPR29, kvm_vcpu_arch, fpu.fpr[29]);
+       OFFSET(VCPU_FPR30, kvm_vcpu_arch, fpu.fpr[30]);
+       OFFSET(VCPU_FPR31, kvm_vcpu_arch, fpu.fpr[31]);
+
+       OFFSET(VCPU_FCR31, kvm_vcpu_arch, fpu.fcr31);
+       OFFSET(VCPU_MSA_CSR, kvm_vcpu_arch, fpu.msacsr);
+       BLANK();
+
        OFFSET(VCPU_COP0, kvm_vcpu_arch, cop0);
        OFFSET(VCPU_GUEST_KERNEL_ASID, kvm_vcpu_arch, guest_kernel_asid);
        OFFSET(VCPU_GUEST_USER_ASID, kvm_vcpu_arch, guest_user_asid);
index 2ebaabe3af1513269e100d8bcffa9e8e9cb1f2c8..af42e7003f12d025cd31e2a5d167f2f4b158d37a 100644 (file)
@@ -360,12 +360,15 @@ NESTED(nmi_handler, PT_SIZE, sp)
        .set    mips1
        SET_HARDFLOAT
        cfc1    a1, fcr31
-       li      a2, ~(0x3f << 12)
-       and     a2, a1
-       ctc1    a2, fcr31
        .set    pop
-       TRACE_IRQS_ON
-       STI
+       CLI
+       TRACE_IRQS_OFF
+       .endm
+
+       .macro  __build_clear_msa_fpe
+       _cfcmsa a1, MSA_CSR
+       CLI
+       TRACE_IRQS_OFF
        .endm
 
        .macro  __build_clear_ade
@@ -426,7 +429,7 @@ NESTED(nmi_handler, PT_SIZE, sp)
        BUILD_HANDLER cpu cpu sti silent                /* #11 */
        BUILD_HANDLER ov ov sti silent                  /* #12 */
        BUILD_HANDLER tr tr sti silent                  /* #13 */
-       BUILD_HANDLER msa_fpe msa_fpe sti silent        /* #14 */
+       BUILD_HANDLER msa_fpe msa_fpe msa_fpe silent    /* #14 */
        BUILD_HANDLER fpe fpe fpe silent                /* #15 */
        BUILD_HANDLER ftlb ftlb none silent             /* #16 */
        BUILD_HANDLER msa msa sti silent                /* #21 */
index 51045281259403c55fcefac09d510f874a3047bb..7da6e324dd354a77991c4fe2dd998c07b998d956 100644 (file)
 #define CREATE_TRACE_POINTS
 #include <trace/events/syscalls.h>
 
+static void init_fp_ctx(struct task_struct *target)
+{
+       /* If FP has been used then the target already has context */
+       if (tsk_used_math(target))
+               return;
+
+       /* Begin with data registers set to all 1s... */
+       memset(&target->thread.fpu.fpr, ~0, sizeof(target->thread.fpu.fpr));
+
+       /* ...and FCSR zeroed */
+       target->thread.fpu.fcr31 = 0;
+
+       /*
+        * Record that the target has "used" math, such that the context
+        * just initialised, and any modifications made by the caller,
+        * aren't discarded.
+        */
+       set_stopped_child_used_math(target);
+}
+
 /*
  * Called by kernel/ptrace.c when detaching..
  *
@@ -142,6 +162,7 @@ int ptrace_setfpregs(struct task_struct *child, __u32 __user *data)
        if (!access_ok(VERIFY_READ, data, 33 * 8))
                return -EIO;
 
+       init_fp_ctx(child);
        fregs = get_fpu_regs(child);
 
        for (i = 0; i < 32; i++) {
@@ -439,6 +460,8 @@ static int fpr_set(struct task_struct *target,
 
        /* XXX fcr31  */
 
+       init_fp_ctx(target);
+
        if (sizeof(target->thread.fpu.fpr[i]) == sizeof(elf_fpreg_t))
                return user_regset_copyin(&pos, &count, &kbuf, &ubuf,
                                          &target->thread.fpu,
@@ -660,12 +683,7 @@ long arch_ptrace(struct task_struct *child, long request,
                case FPR_BASE ... FPR_BASE + 31: {
                        union fpureg *fregs = get_fpu_regs(child);
 
-                       if (!tsk_used_math(child)) {
-                               /* FP not yet used  */
-                               memset(&child->thread.fpu, ~0,
-                                      sizeof(child->thread.fpu));
-                               child->thread.fpu.fcr31 = 0;
-                       }
+                       init_fp_ctx(child);
 #ifdef CONFIG_32BIT
                        if (test_thread_flag(TIF_32BIT_FPREGS)) {
                                /*
index 676c5030a953bf9cca5ad038a7526d3b94ce372d..1d88af26ba82a0c3ee58ff8ff3b2b1661dad9455 100644 (file)
@@ -34,7 +34,6 @@
        .endm
 
        .set    noreorder
-       .set    MIPS_ISA_ARCH_LEVEL_RAW
 
 LEAF(_save_fp_context)
        .set    push
@@ -103,6 +102,7 @@ LEAF(_save_fp_context)
        /* Save 32-bit process floating point context */
 LEAF(_save_fp_context32)
        .set push
+       .set MIPS_ISA_ARCH_LEVEL_RAW
        SET_HARDFLOAT
        cfc1    t1, fcr31
 
index 33984c04b60b710516f1b0bfb88aa52aaa04629f..5b4d711f878da251a101526671a80283503736a4 100644 (file)
@@ -701,6 +701,13 @@ asmlinkage void do_ov(struct pt_regs *regs)
 
 int process_fpemu_return(int sig, void __user *fault_addr)
 {
+       /*
+        * We can't allow the emulated instruction to leave any of the cause
+        * bits set in FCSR. If they were then the kernel would take an FP
+        * exception when restoring FP context.
+        */
+       current->thread.fpu.fcr31 &= ~FPU_CSR_ALL_X;
+
        if (sig == SIGSEGV || sig == SIGBUS) {
                struct siginfo si = {0};
                si.si_addr = fault_addr;
@@ -781,6 +788,11 @@ asmlinkage void do_fpe(struct pt_regs *regs, unsigned long fcr31)
        if (notify_die(DIE_FP, "FP exception", regs, 0, regs_to_trapnr(regs),
                       SIGFPE) == NOTIFY_STOP)
                goto out;
+
+       /* Clear FCSR.Cause before enabling interrupts */
+       write_32bit_cp1_register(CP1_STATUS, fcr31 & ~FPU_CSR_ALL_X);
+       local_irq_enable();
+
        die_if_kernel("FP exception in kernel code", regs);
 
        if (fcr31 & FPU_CSR_UNI_X) {
@@ -804,18 +816,12 @@ asmlinkage void do_fpe(struct pt_regs *regs, unsigned long fcr31)
                sig = fpu_emulator_cop1Handler(regs, &current->thread.fpu, 1,
                                               &fault_addr);
 
-               /*
-                * We can't allow the emulated instruction to leave any of
-                * the cause bit set in $fcr31.
-                */
-               current->thread.fpu.fcr31 &= ~FPU_CSR_ALL_X;
+               /* If something went wrong, signal */
+               process_fpemu_return(sig, fault_addr);
 
                /* Restore the hardware register state */
                own_fpu(1);     /* Using the FPU again.  */
 
-               /* If something went wrong, signal */
-               process_fpemu_return(sig, fault_addr);
-
                goto out;
        } else if (fcr31 & FPU_CSR_INV_X)
                info.si_code = FPE_FLTINV;
@@ -1392,13 +1398,22 @@ out:
        exception_exit(prev_state);
 }
 
-asmlinkage void do_msa_fpe(struct pt_regs *regs)
+asmlinkage void do_msa_fpe(struct pt_regs *regs, unsigned int msacsr)
 {
        enum ctx_state prev_state;
 
        prev_state = exception_enter();
+       if (notify_die(DIE_MSAFP, "MSA FP exception", regs, 0,
+                      regs_to_trapnr(regs), SIGFPE) == NOTIFY_STOP)
+               goto out;
+
+       /* Clear MSACSR.Cause before enabling interrupts */
+       write_msa_csr(msacsr & ~MSA_CSR_CAUSEF);
+       local_irq_enable();
+
        die_if_kernel("do_msa_fpe invoked from kernel context!", regs);
        force_sig(SIGFPE, current);
+out:
        exception_exit(prev_state);
 }
 
index 401fe027c2612cf774fa53173420f93c072ea3c7..637ebbebd549701c1a0a67e7e3a8fbb9d2cb61f0 100644 (file)
@@ -1,13 +1,15 @@
 # Makefile for KVM support for MIPS
 #
 
-common-objs = $(addprefix ../../../virt/kvm/, kvm_main.o coalesced_mmio.o)
+common-objs-y = $(addprefix ../../../virt/kvm/, kvm_main.o coalesced_mmio.o)
 
 EXTRA_CFLAGS += -Ivirt/kvm -Iarch/mips/kvm
 
-kvm-objs := $(common-objs) mips.o emulate.o locore.o \
+common-objs-$(CONFIG_CPU_HAS_MSA) += msa.o
+
+kvm-objs := $(common-objs-y) mips.o emulate.o locore.o \
            interrupt.o stats.o commpage.o \
-           dyntrans.o trap_emul.o
+           dyntrans.o trap_emul.o fpu.o
 
 obj-$(CONFIG_KVM)      += kvm.o
 obj-y                  += callback.o tlb.o
index fb3e8dfd1ff647263c0ca93a1d2ba6c5fdbd906b..6230f376a44e7ab6f09041c4b805e54e59468489 100644 (file)
@@ -884,6 +884,84 @@ enum emulation_result kvm_mips_emul_tlbp(struct kvm_vcpu *vcpu)
        return EMULATE_DONE;
 }
 
+/**
+ * kvm_mips_config1_wrmask() - Find mask of writable bits in guest Config1
+ * @vcpu:      Virtual CPU.
+ *
+ * Finds the mask of bits which are writable in the guest's Config1 CP0
+ * register, by userland (currently read-only to the guest).
+ */
+unsigned int kvm_mips_config1_wrmask(struct kvm_vcpu *vcpu)
+{
+       unsigned int mask = 0;
+
+       /* Permit FPU to be present if FPU is supported */
+       if (kvm_mips_guest_can_have_fpu(&vcpu->arch))
+               mask |= MIPS_CONF1_FP;
+
+       return mask;
+}
+
+/**
+ * kvm_mips_config3_wrmask() - Find mask of writable bits in guest Config3
+ * @vcpu:      Virtual CPU.
+ *
+ * Finds the mask of bits which are writable in the guest's Config3 CP0
+ * register, by userland (currently read-only to the guest).
+ */
+unsigned int kvm_mips_config3_wrmask(struct kvm_vcpu *vcpu)
+{
+       /* Config4 is optional */
+       unsigned int mask = MIPS_CONF_M;
+
+       /* Permit MSA to be present if MSA is supported */
+       if (kvm_mips_guest_can_have_msa(&vcpu->arch))
+               mask |= MIPS_CONF3_MSA;
+
+       return mask;
+}
+
+/**
+ * kvm_mips_config4_wrmask() - Find mask of writable bits in guest Config4
+ * @vcpu:      Virtual CPU.
+ *
+ * Finds the mask of bits which are writable in the guest's Config4 CP0
+ * register, by userland (currently read-only to the guest).
+ */
+unsigned int kvm_mips_config4_wrmask(struct kvm_vcpu *vcpu)
+{
+       /* Config5 is optional */
+       return MIPS_CONF_M;
+}
+
+/**
+ * kvm_mips_config5_wrmask() - Find mask of writable bits in guest Config5
+ * @vcpu:      Virtual CPU.
+ *
+ * Finds the mask of bits which are writable in the guest's Config5 CP0
+ * register, by the guest itself.
+ */
+unsigned int kvm_mips_config5_wrmask(struct kvm_vcpu *vcpu)
+{
+       unsigned int mask = 0;
+
+       /* Permit MSAEn changes if MSA supported and enabled */
+       if (kvm_mips_guest_has_msa(&vcpu->arch))
+               mask |= MIPS_CONF5_MSAEN;
+
+       /*
+        * Permit guest FPU mode changes if FPU is enabled and the relevant
+        * feature exists according to FIR register.
+        */
+       if (kvm_mips_guest_has_fpu(&vcpu->arch)) {
+               if (cpu_has_fre)
+                       mask |= MIPS_CONF5_FRE;
+               /* We don't support UFR or UFE */
+       }
+
+       return mask;
+}
+
 enum emulation_result kvm_mips_emulate_CP0(uint32_t inst, uint32_t *opc,
                                           uint32_t cause, struct kvm_run *run,
                                           struct kvm_vcpu *vcpu)
@@ -1021,18 +1099,114 @@ enum emulation_result kvm_mips_emulate_CP0(uint32_t inst, uint32_t *opc,
                                kvm_mips_write_compare(vcpu,
                                                       vcpu->arch.gprs[rt]);
                        } else if ((rd == MIPS_CP0_STATUS) && (sel == 0)) {
-                               kvm_write_c0_guest_status(cop0,
-                                                         vcpu->arch.gprs[rt]);
+                               unsigned int old_val, val, change;
+
+                               old_val = kvm_read_c0_guest_status(cop0);
+                               val = vcpu->arch.gprs[rt];
+                               change = val ^ old_val;
+
+                               /* Make sure that the NMI bit is never set */
+                               val &= ~ST0_NMI;
+
+                               /*
+                                * Don't allow CU1 or FR to be set unless FPU
+                                * capability enabled and exists in guest
+                                * configuration.
+                                */
+                               if (!kvm_mips_guest_has_fpu(&vcpu->arch))
+                                       val &= ~(ST0_CU1 | ST0_FR);
+
+                               /*
+                                * Also don't allow FR to be set if host doesn't
+                                * support it.
+                                */
+                               if (!(current_cpu_data.fpu_id & MIPS_FPIR_F64))
+                                       val &= ~ST0_FR;
+
+
+                               /* Handle changes in FPU mode */
+                               preempt_disable();
+
+                               /*
+                                * FPU and Vector register state is made
+                                * UNPREDICTABLE by a change of FR, so don't
+                                * even bother saving it.
+                                */
+                               if (change & ST0_FR)
+                                       kvm_drop_fpu(vcpu);
+
+                               /*
+                                * If MSA state is already live, it is undefined
+                                * how it interacts with FR=0 FPU state, and we
+                                * don't want to hit reserved instruction
+                                * exceptions trying to save the MSA state later
+                                * when CU=1 && FR=1, so play it safe and save
+                                * it first.
+                                */
+                               if (change & ST0_CU1 && !(val & ST0_FR) &&
+                                   vcpu->arch.fpu_inuse & KVM_MIPS_FPU_MSA)
+                                       kvm_lose_fpu(vcpu);
+
                                /*
-                                * Make sure that CU1 and NMI bits are
-                                * never set
+                                * Propagate CU1 (FPU enable) changes
+                                * immediately if the FPU context is already
+                                * loaded. When disabling we leave the context
+                                * loaded so it can be quickly enabled again in
+                                * the near future.
                                 */
-                               kvm_clear_c0_guest_status(cop0,
-                                                         (ST0_CU1 | ST0_NMI));
+                               if (change & ST0_CU1 &&
+                                   vcpu->arch.fpu_inuse & KVM_MIPS_FPU_FPU)
+                                       change_c0_status(ST0_CU1, val);
+
+                               preempt_enable();
+
+                               kvm_write_c0_guest_status(cop0, val);
 
 #ifdef CONFIG_KVM_MIPS_DYN_TRANS
-                               kvm_mips_trans_mtc0(inst, opc, vcpu);
+                               /*
+                                * If FPU present, we need CU1/FR bits to take
+                                * effect fairly soon.
+                                */
+                               if (!kvm_mips_guest_has_fpu(&vcpu->arch))
+                                       kvm_mips_trans_mtc0(inst, opc, vcpu);
 #endif
+                       } else if ((rd == MIPS_CP0_CONFIG) && (sel == 5)) {
+                               unsigned int old_val, val, change, wrmask;
+
+                               old_val = kvm_read_c0_guest_config5(cop0);
+                               val = vcpu->arch.gprs[rt];
+
+                               /* Only a few bits are writable in Config5 */
+                               wrmask = kvm_mips_config5_wrmask(vcpu);
+                               change = (val ^ old_val) & wrmask;
+                               val = old_val ^ change;
+
+
+                               /* Handle changes in FPU/MSA modes */
+                               preempt_disable();
+
+                               /*
+                                * Propagate FRE changes immediately if the FPU
+                                * context is already loaded.
+                                */
+                               if (change & MIPS_CONF5_FRE &&
+                                   vcpu->arch.fpu_inuse & KVM_MIPS_FPU_FPU)
+                                       change_c0_config5(MIPS_CONF5_FRE, val);
+
+                               /*
+                                * Propagate MSAEn changes immediately if the
+                                * MSA context is already loaded. When disabling
+                                * we leave the context loaded so it can be
+                                * quickly enabled again in the near future.
+                                */
+                               if (change & MIPS_CONF5_MSAEN &&
+                                   vcpu->arch.fpu_inuse & KVM_MIPS_FPU_MSA)
+                                       change_c0_config5(MIPS_CONF5_MSAEN,
+                                                         val);
+
+                               preempt_enable();
+
+                               kvm_write_c0_guest_config5(cop0, val);
                        } else if ((rd == MIPS_CP0_CAUSE) && (sel == 0)) {
                                uint32_t old_cause, new_cause;
 
@@ -1970,6 +2144,146 @@ enum emulation_result kvm_mips_emulate_bp_exc(unsigned long cause,
        return er;
 }
 
+enum emulation_result kvm_mips_emulate_trap_exc(unsigned long cause,
+                                               uint32_t *opc,
+                                               struct kvm_run *run,
+                                               struct kvm_vcpu *vcpu)
+{
+       struct mips_coproc *cop0 = vcpu->arch.cop0;
+       struct kvm_vcpu_arch *arch = &vcpu->arch;
+       enum emulation_result er = EMULATE_DONE;
+
+       if ((kvm_read_c0_guest_status(cop0) & ST0_EXL) == 0) {
+               /* save old pc */
+               kvm_write_c0_guest_epc(cop0, arch->pc);
+               kvm_set_c0_guest_status(cop0, ST0_EXL);
+
+               if (cause & CAUSEF_BD)
+                       kvm_set_c0_guest_cause(cop0, CAUSEF_BD);
+               else
+                       kvm_clear_c0_guest_cause(cop0, CAUSEF_BD);
+
+               kvm_debug("Delivering TRAP @ pc %#lx\n", arch->pc);
+
+               kvm_change_c0_guest_cause(cop0, (0xff),
+                                         (T_TRAP << CAUSEB_EXCCODE));
+
+               /* Set PC to the exception entry point */
+               arch->pc = KVM_GUEST_KSEG0 + 0x180;
+
+       } else {
+               kvm_err("Trying to deliver TRAP when EXL is already set\n");
+               er = EMULATE_FAIL;
+       }
+
+       return er;
+}
+
+enum emulation_result kvm_mips_emulate_msafpe_exc(unsigned long cause,
+                                                 uint32_t *opc,
+                                                 struct kvm_run *run,
+                                                 struct kvm_vcpu *vcpu)
+{
+       struct mips_coproc *cop0 = vcpu->arch.cop0;
+       struct kvm_vcpu_arch *arch = &vcpu->arch;
+       enum emulation_result er = EMULATE_DONE;
+
+       if ((kvm_read_c0_guest_status(cop0) & ST0_EXL) == 0) {
+               /* save old pc */
+               kvm_write_c0_guest_epc(cop0, arch->pc);
+               kvm_set_c0_guest_status(cop0, ST0_EXL);
+
+               if (cause & CAUSEF_BD)
+                       kvm_set_c0_guest_cause(cop0, CAUSEF_BD);
+               else
+                       kvm_clear_c0_guest_cause(cop0, CAUSEF_BD);
+
+               kvm_debug("Delivering MSAFPE @ pc %#lx\n", arch->pc);
+
+               kvm_change_c0_guest_cause(cop0, (0xff),
+                                         (T_MSAFPE << CAUSEB_EXCCODE));
+
+               /* Set PC to the exception entry point */
+               arch->pc = KVM_GUEST_KSEG0 + 0x180;
+
+       } else {
+               kvm_err("Trying to deliver MSAFPE when EXL is already set\n");
+               er = EMULATE_FAIL;
+       }
+
+       return er;
+}
+
+enum emulation_result kvm_mips_emulate_fpe_exc(unsigned long cause,
+                                              uint32_t *opc,
+                                              struct kvm_run *run,
+                                              struct kvm_vcpu *vcpu)
+{
+       struct mips_coproc *cop0 = vcpu->arch.cop0;
+       struct kvm_vcpu_arch *arch = &vcpu->arch;
+       enum emulation_result er = EMULATE_DONE;
+
+       if ((kvm_read_c0_guest_status(cop0) & ST0_EXL) == 0) {
+               /* save old pc */
+               kvm_write_c0_guest_epc(cop0, arch->pc);
+               kvm_set_c0_guest_status(cop0, ST0_EXL);
+
+               if (cause & CAUSEF_BD)
+                       kvm_set_c0_guest_cause(cop0, CAUSEF_BD);
+               else
+                       kvm_clear_c0_guest_cause(cop0, CAUSEF_BD);
+
+               kvm_debug("Delivering FPE @ pc %#lx\n", arch->pc);
+
+               kvm_change_c0_guest_cause(cop0, (0xff),
+                                         (T_FPE << CAUSEB_EXCCODE));
+
+               /* Set PC to the exception entry point */
+               arch->pc = KVM_GUEST_KSEG0 + 0x180;
+
+       } else {
+               kvm_err("Trying to deliver FPE when EXL is already set\n");
+               er = EMULATE_FAIL;
+       }
+
+       return er;
+}
+
+enum emulation_result kvm_mips_emulate_msadis_exc(unsigned long cause,
+                                                 uint32_t *opc,
+                                                 struct kvm_run *run,
+                                                 struct kvm_vcpu *vcpu)
+{
+       struct mips_coproc *cop0 = vcpu->arch.cop0;
+       struct kvm_vcpu_arch *arch = &vcpu->arch;
+       enum emulation_result er = EMULATE_DONE;
+
+       if ((kvm_read_c0_guest_status(cop0) & ST0_EXL) == 0) {
+               /* save old pc */
+               kvm_write_c0_guest_epc(cop0, arch->pc);
+               kvm_set_c0_guest_status(cop0, ST0_EXL);
+
+               if (cause & CAUSEF_BD)
+                       kvm_set_c0_guest_cause(cop0, CAUSEF_BD);
+               else
+                       kvm_clear_c0_guest_cause(cop0, CAUSEF_BD);
+
+               kvm_debug("Delivering MSADIS @ pc %#lx\n", arch->pc);
+
+               kvm_change_c0_guest_cause(cop0, (0xff),
+                                         (T_MSADIS << CAUSEB_EXCCODE));
+
+               /* Set PC to the exception entry point */
+               arch->pc = KVM_GUEST_KSEG0 + 0x180;
+
+       } else {
+               kvm_err("Trying to deliver MSADIS when EXL is already set\n");
+               er = EMULATE_FAIL;
+       }
+
+       return er;
+}
+
 /* ll/sc, rdhwr, sync emulation */
 
 #define OPCODE 0xfc000000
@@ -2176,6 +2490,10 @@ enum emulation_result kvm_mips_check_privilege(unsigned long cause,
                case T_SYSCALL:
                case T_BREAK:
                case T_RES_INST:
+               case T_TRAP:
+               case T_MSAFPE:
+               case T_FPE:
+               case T_MSADIS:
                        break;
 
                case T_COP_UNUSABLE:
diff --git a/arch/mips/kvm/fpu.S b/arch/mips/kvm/fpu.S
new file mode 100644 (file)
index 0000000..531fbf5
--- /dev/null
@@ -0,0 +1,122 @@
+/*
+ * This file is subject to the terms and conditions of the GNU General Public
+ * License.  See the file "COPYING" in the main directory of this archive
+ * for more details.
+ *
+ * FPU context handling code for KVM.
+ *
+ * Copyright (C) 2015 Imagination Technologies Ltd.
+ */
+
+#include <asm/asm.h>
+#include <asm/asm-offsets.h>
+#include <asm/fpregdef.h>
+#include <asm/mipsregs.h>
+#include <asm/regdef.h>
+
+       .set    noreorder
+       .set    noat
+
+LEAF(__kvm_save_fpu)
+       .set    push
+       .set    mips64r2
+       SET_HARDFLOAT
+       mfc0    t0, CP0_STATUS
+       sll     t0, t0, 5                       # is Status.FR set?
+       bgez    t0, 1f                          # no: skip odd doubles
+        nop
+       sdc1    $f1,  VCPU_FPR1(a0)
+       sdc1    $f3,  VCPU_FPR3(a0)
+       sdc1    $f5,  VCPU_FPR5(a0)
+       sdc1    $f7,  VCPU_FPR7(a0)
+       sdc1    $f9,  VCPU_FPR9(a0)
+       sdc1    $f11, VCPU_FPR11(a0)
+       sdc1    $f13, VCPU_FPR13(a0)
+       sdc1    $f15, VCPU_FPR15(a0)
+       sdc1    $f17, VCPU_FPR17(a0)
+       sdc1    $f19, VCPU_FPR19(a0)
+       sdc1    $f21, VCPU_FPR21(a0)
+       sdc1    $f23, VCPU_FPR23(a0)
+       sdc1    $f25, VCPU_FPR25(a0)
+       sdc1    $f27, VCPU_FPR27(a0)
+       sdc1    $f29, VCPU_FPR29(a0)
+       sdc1    $f31, VCPU_FPR31(a0)
+1:     sdc1    $f0,  VCPU_FPR0(a0)
+       sdc1    $f2,  VCPU_FPR2(a0)
+       sdc1    $f4,  VCPU_FPR4(a0)
+       sdc1    $f6,  VCPU_FPR6(a0)
+       sdc1    $f8,  VCPU_FPR8(a0)
+       sdc1    $f10, VCPU_FPR10(a0)
+       sdc1    $f12, VCPU_FPR12(a0)
+       sdc1    $f14, VCPU_FPR14(a0)
+       sdc1    $f16, VCPU_FPR16(a0)
+       sdc1    $f18, VCPU_FPR18(a0)
+       sdc1    $f20, VCPU_FPR20(a0)
+       sdc1    $f22, VCPU_FPR22(a0)
+       sdc1    $f24, VCPU_FPR24(a0)
+       sdc1    $f26, VCPU_FPR26(a0)
+       sdc1    $f28, VCPU_FPR28(a0)
+       jr      ra
+        sdc1   $f30, VCPU_FPR30(a0)
+       .set    pop
+       END(__kvm_save_fpu)
+
+LEAF(__kvm_restore_fpu)
+       .set    push
+       .set    mips64r2
+       SET_HARDFLOAT
+       mfc0    t0, CP0_STATUS
+       sll     t0, t0, 5                       # is Status.FR set?
+       bgez    t0, 1f                          # no: skip odd doubles
+        nop
+       ldc1    $f1,  VCPU_FPR1(a0)
+       ldc1    $f3,  VCPU_FPR3(a0)
+       ldc1    $f5,  VCPU_FPR5(a0)
+       ldc1    $f7,  VCPU_FPR7(a0)
+       ldc1    $f9,  VCPU_FPR9(a0)
+       ldc1    $f11, VCPU_FPR11(a0)
+       ldc1    $f13, VCPU_FPR13(a0)
+       ldc1    $f15, VCPU_FPR15(a0)
+       ldc1    $f17, VCPU_FPR17(a0)
+       ldc1    $f19, VCPU_FPR19(a0)
+       ldc1    $f21, VCPU_FPR21(a0)
+       ldc1    $f23, VCPU_FPR23(a0)
+       ldc1    $f25, VCPU_FPR25(a0)
+       ldc1    $f27, VCPU_FPR27(a0)
+       ldc1    $f29, VCPU_FPR29(a0)
+       ldc1    $f31, VCPU_FPR31(a0)
+1:     ldc1    $f0,  VCPU_FPR0(a0)
+       ldc1    $f2,  VCPU_FPR2(a0)
+       ldc1    $f4,  VCPU_FPR4(a0)
+       ldc1    $f6,  VCPU_FPR6(a0)
+       ldc1    $f8,  VCPU_FPR8(a0)
+       ldc1    $f10, VCPU_FPR10(a0)
+       ldc1    $f12, VCPU_FPR12(a0)
+       ldc1    $f14, VCPU_FPR14(a0)
+       ldc1    $f16, VCPU_FPR16(a0)
+       ldc1    $f18, VCPU_FPR18(a0)
+       ldc1    $f20, VCPU_FPR20(a0)
+       ldc1    $f22, VCPU_FPR22(a0)
+       ldc1    $f24, VCPU_FPR24(a0)
+       ldc1    $f26, VCPU_FPR26(a0)
+       ldc1    $f28, VCPU_FPR28(a0)
+       jr      ra
+        ldc1   $f30, VCPU_FPR30(a0)
+       .set    pop
+       END(__kvm_restore_fpu)
+
+LEAF(__kvm_restore_fcsr)
+       .set    push
+       SET_HARDFLOAT
+       lw      t0, VCPU_FCR31(a0)
+       /*
+        * The ctc1 must stay at this offset in __kvm_restore_fcsr.
+        * See kvm_mips_csr_die_notify() which handles t0 containing a value
+        * which triggers an FP Exception, which must be stepped over and
+        * ignored since the set cause bits must remain there for the guest.
+        */
+       ctc1    t0, fcr31
+       jr      ra
+        nop
+       .set    pop
+       END(__kvm_restore_fcsr)
index 4a68b176d6e4f8dff9680a2c22a41019be1d574f..c567240386a0f10818b0e06433f4b33b1bf5e6dc 100644 (file)
@@ -36,6 +36,8 @@
 #define PT_HOST_USERLOCAL   PT_EPC
 
 #define CP0_DDATA_LO        $28,3
+#define CP0_CONFIG3         $16,3
+#define CP0_CONFIG5         $16,5
 #define CP0_EBASE           $15,1
 
 #define CP0_INTCTL          $12,1
@@ -353,6 +355,42 @@ NESTED (MIPSX(GuestException), CALLFRAME_SIZ, ra)
        LONG_L  k0, VCPU_HOST_EBASE(k1)
        mtc0    k0,CP0_EBASE
 
+       /*
+        * If FPU is enabled, save FCR31 and clear it so that later ctc1's don't
+        * trigger FPE for pending exceptions.
+        */
+       .set    at
+       and     v1, v0, ST0_CU1
+       beqz    v1, 1f
+        nop
+       .set    push
+       SET_HARDFLOAT
+       cfc1    t0, fcr31
+       sw      t0, VCPU_FCR31(k1)
+       ctc1    zero,fcr31
+       .set    pop
+       .set    noat
+1:
+
+#ifdef CONFIG_CPU_HAS_MSA
+       /*
+        * If MSA is enabled, save MSACSR and clear it so that later
+        * instructions don't trigger MSAFPE for pending exceptions.
+        */
+       mfc0    t0, CP0_CONFIG3
+       ext     t0, t0, 28, 1 /* MIPS_CONF3_MSAP */
+       beqz    t0, 1f
+        nop
+       mfc0    t0, CP0_CONFIG5
+       ext     t0, t0, 27, 1 /* MIPS_CONF5_MSAEN */
+       beqz    t0, 1f
+        nop
+       _cfcmsa t0, MSA_CSR
+       sw      t0, VCPU_MSA_CSR(k1)
+       _ctcmsa MSA_CSR, zero
+1:
+#endif
+
        /* Now that the new EBASE has been loaded, unset BEV and KSU_USER */
        .set    at
        and     v0, v0, ~(ST0_EXL | KSU_USER | ST0_IE)
index c9eccf5df912037e2b71bbb4a7dddd2a1d2d866e..bb68e8d520e83b5a30b74b22ae3292b1dd1469e1 100644 (file)
@@ -11,6 +11,7 @@
 
 #include <linux/errno.h>
 #include <linux/err.h>
+#include <linux/kdebug.h>
 #include <linux/module.h>
 #include <linux/vmalloc.h>
 #include <linux/fs.h>
@@ -48,6 +49,10 @@ struct kvm_stats_debugfs_item debugfs_entries[] = {
        { "syscall",      VCPU_STAT(syscall_exits),      KVM_STAT_VCPU },
        { "resvd_inst",   VCPU_STAT(resvd_inst_exits),   KVM_STAT_VCPU },
        { "break_inst",   VCPU_STAT(break_inst_exits),   KVM_STAT_VCPU },
+       { "trap_inst",    VCPU_STAT(trap_inst_exits),    KVM_STAT_VCPU },
+       { "msa_fpe",      VCPU_STAT(msa_fpe_exits),      KVM_STAT_VCPU },
+       { "fpe",          VCPU_STAT(fpe_exits),          KVM_STAT_VCPU },
+       { "msa_disabled", VCPU_STAT(msa_disabled_exits), KVM_STAT_VCPU },
        { "flush_dcache", VCPU_STAT(flush_dcache_exits), KVM_STAT_VCPU },
        { "halt_successful_poll", VCPU_STAT(halt_successful_poll), KVM_STAT_VCPU },
        { "halt_wakeup",  VCPU_STAT(halt_wakeup),        KVM_STAT_VCPU },
@@ -504,10 +509,13 @@ static u64 kvm_mips_get_one_regs[] = {
        KVM_REG_MIPS_CP0_STATUS,
        KVM_REG_MIPS_CP0_CAUSE,
        KVM_REG_MIPS_CP0_EPC,
+       KVM_REG_MIPS_CP0_PRID,
        KVM_REG_MIPS_CP0_CONFIG,
        KVM_REG_MIPS_CP0_CONFIG1,
        KVM_REG_MIPS_CP0_CONFIG2,
        KVM_REG_MIPS_CP0_CONFIG3,
+       KVM_REG_MIPS_CP0_CONFIG4,
+       KVM_REG_MIPS_CP0_CONFIG5,
        KVM_REG_MIPS_CP0_CONFIG7,
        KVM_REG_MIPS_CP0_ERROREPC,
 
@@ -520,10 +528,14 @@ static int kvm_mips_get_reg(struct kvm_vcpu *vcpu,
                            const struct kvm_one_reg *reg)
 {
        struct mips_coproc *cop0 = vcpu->arch.cop0;
+       struct mips_fpu_struct *fpu = &vcpu->arch.fpu;
        int ret;
        s64 v;
+       s64 vs[2];
+       unsigned int idx;
 
        switch (reg->id) {
+       /* General purpose registers */
        case KVM_REG_MIPS_R0 ... KVM_REG_MIPS_R31:
                v = (long)vcpu->arch.gprs[reg->id - KVM_REG_MIPS_R0];
                break;
@@ -537,6 +549,67 @@ static int kvm_mips_get_reg(struct kvm_vcpu *vcpu,
                v = (long)vcpu->arch.pc;
                break;
 
+       /* Floating point registers */
+       case KVM_REG_MIPS_FPR_32(0) ... KVM_REG_MIPS_FPR_32(31):
+               if (!kvm_mips_guest_has_fpu(&vcpu->arch))
+                       return -EINVAL;
+               idx = reg->id - KVM_REG_MIPS_FPR_32(0);
+               /* Odd singles in top of even double when FR=0 */
+               if (kvm_read_c0_guest_status(cop0) & ST0_FR)
+                       v = get_fpr32(&fpu->fpr[idx], 0);
+               else
+                       v = get_fpr32(&fpu->fpr[idx & ~1], idx & 1);
+               break;
+       case KVM_REG_MIPS_FPR_64(0) ... KVM_REG_MIPS_FPR_64(31):
+               if (!kvm_mips_guest_has_fpu(&vcpu->arch))
+                       return -EINVAL;
+               idx = reg->id - KVM_REG_MIPS_FPR_64(0);
+               /* Can't access odd doubles in FR=0 mode */
+               if (idx & 1 && !(kvm_read_c0_guest_status(cop0) & ST0_FR))
+                       return -EINVAL;
+               v = get_fpr64(&fpu->fpr[idx], 0);
+               break;
+       case KVM_REG_MIPS_FCR_IR:
+               if (!kvm_mips_guest_has_fpu(&vcpu->arch))
+                       return -EINVAL;
+               v = boot_cpu_data.fpu_id;
+               break;
+       case KVM_REG_MIPS_FCR_CSR:
+               if (!kvm_mips_guest_has_fpu(&vcpu->arch))
+                       return -EINVAL;
+               v = fpu->fcr31;
+               break;
+
+       /* MIPS SIMD Architecture (MSA) registers */
+       case KVM_REG_MIPS_VEC_128(0) ... KVM_REG_MIPS_VEC_128(31):
+               if (!kvm_mips_guest_has_msa(&vcpu->arch))
+                       return -EINVAL;
+               /* Can't access MSA registers in FR=0 mode */
+               if (!(kvm_read_c0_guest_status(cop0) & ST0_FR))
+                       return -EINVAL;
+               idx = reg->id - KVM_REG_MIPS_VEC_128(0);
+#ifdef CONFIG_CPU_LITTLE_ENDIAN
+               /* least significant byte first */
+               vs[0] = get_fpr64(&fpu->fpr[idx], 0);
+               vs[1] = get_fpr64(&fpu->fpr[idx], 1);
+#else
+               /* most significant byte first */
+               vs[0] = get_fpr64(&fpu->fpr[idx], 1);
+               vs[1] = get_fpr64(&fpu->fpr[idx], 0);
+#endif
+               break;
+       case KVM_REG_MIPS_MSA_IR:
+               if (!kvm_mips_guest_has_msa(&vcpu->arch))
+                       return -EINVAL;
+               v = boot_cpu_data.msa_id;
+               break;
+       case KVM_REG_MIPS_MSA_CSR:
+               if (!kvm_mips_guest_has_msa(&vcpu->arch))
+                       return -EINVAL;
+               v = fpu->msacsr;
+               break;
+
+       /* Co-processor 0 registers */
        case KVM_REG_MIPS_CP0_INDEX:
                v = (long)kvm_read_c0_guest_index(cop0);
                break;
@@ -573,8 +646,8 @@ static int kvm_mips_get_reg(struct kvm_vcpu *vcpu,
        case KVM_REG_MIPS_CP0_EPC:
                v = (long)kvm_read_c0_guest_epc(cop0);
                break;
-       case KVM_REG_MIPS_CP0_ERROREPC:
-               v = (long)kvm_read_c0_guest_errorepc(cop0);
+       case KVM_REG_MIPS_CP0_PRID:
+               v = (long)kvm_read_c0_guest_prid(cop0);
                break;
        case KVM_REG_MIPS_CP0_CONFIG:
                v = (long)kvm_read_c0_guest_config(cop0);
@@ -588,9 +661,18 @@ static int kvm_mips_get_reg(struct kvm_vcpu *vcpu,
        case KVM_REG_MIPS_CP0_CONFIG3:
                v = (long)kvm_read_c0_guest_config3(cop0);
                break;
+       case KVM_REG_MIPS_CP0_CONFIG4:
+               v = (long)kvm_read_c0_guest_config4(cop0);
+               break;
+       case KVM_REG_MIPS_CP0_CONFIG5:
+               v = (long)kvm_read_c0_guest_config5(cop0);
+               break;
        case KVM_REG_MIPS_CP0_CONFIG7:
                v = (long)kvm_read_c0_guest_config7(cop0);
                break;
+       case KVM_REG_MIPS_CP0_ERROREPC:
+               v = (long)kvm_read_c0_guest_errorepc(cop0);
+               break;
        /* registers to be handled specially */
        case KVM_REG_MIPS_CP0_COUNT:
        case KVM_REG_MIPS_COUNT_CTL:
@@ -612,6 +694,10 @@ static int kvm_mips_get_reg(struct kvm_vcpu *vcpu,
                u32 v32 = (u32)v;
 
                return put_user(v32, uaddr32);
+       } else if ((reg->id & KVM_REG_SIZE_MASK) == KVM_REG_SIZE_U128) {
+               void __user *uaddr = (void __user *)(long)reg->addr;
+
+               return copy_to_user(uaddr, vs, 16);
        } else {
                return -EINVAL;
        }
@@ -621,7 +707,10 @@ static int kvm_mips_set_reg(struct kvm_vcpu *vcpu,
                            const struct kvm_one_reg *reg)
 {
        struct mips_coproc *cop0 = vcpu->arch.cop0;
-       u64 v;
+       struct mips_fpu_struct *fpu = &vcpu->arch.fpu;
+       s64 v;
+       s64 vs[2];
+       unsigned int idx;
 
        if ((reg->id & KVM_REG_SIZE_MASK) == KVM_REG_SIZE_U64) {
                u64 __user *uaddr64 = (u64 __user *)(long)reg->addr;
@@ -635,11 +724,16 @@ static int kvm_mips_set_reg(struct kvm_vcpu *vcpu,
                if (get_user(v32, uaddr32) != 0)
                        return -EFAULT;
                v = (s64)v32;
+       } else if ((reg->id & KVM_REG_SIZE_MASK) == KVM_REG_SIZE_U128) {
+               void __user *uaddr = (void __user *)(long)reg->addr;
+
+               return copy_from_user(vs, uaddr, 16);
        } else {
                return -EINVAL;
        }
 
        switch (reg->id) {
+       /* General purpose registers */
        case KVM_REG_MIPS_R0:
                /* Silently ignore requests to set $0 */
                break;
@@ -656,6 +750,64 @@ static int kvm_mips_set_reg(struct kvm_vcpu *vcpu,
                vcpu->arch.pc = v;
                break;
 
+       /* Floating point registers */
+       case KVM_REG_MIPS_FPR_32(0) ... KVM_REG_MIPS_FPR_32(31):
+               if (!kvm_mips_guest_has_fpu(&vcpu->arch))
+                       return -EINVAL;
+               idx = reg->id - KVM_REG_MIPS_FPR_32(0);
+               /* Odd singles in top of even double when FR=0 */
+               if (kvm_read_c0_guest_status(cop0) & ST0_FR)
+                       set_fpr32(&fpu->fpr[idx], 0, v);
+               else
+                       set_fpr32(&fpu->fpr[idx & ~1], idx & 1, v);
+               break;
+       case KVM_REG_MIPS_FPR_64(0) ... KVM_REG_MIPS_FPR_64(31):
+               if (!kvm_mips_guest_has_fpu(&vcpu->arch))
+                       return -EINVAL;
+               idx = reg->id - KVM_REG_MIPS_FPR_64(0);
+               /* Can't access odd doubles in FR=0 mode */
+               if (idx & 1 && !(kvm_read_c0_guest_status(cop0) & ST0_FR))
+                       return -EINVAL;
+               set_fpr64(&fpu->fpr[idx], 0, v);
+               break;
+       case KVM_REG_MIPS_FCR_IR:
+               if (!kvm_mips_guest_has_fpu(&vcpu->arch))
+                       return -EINVAL;
+               /* Read-only */
+               break;
+       case KVM_REG_MIPS_FCR_CSR:
+               if (!kvm_mips_guest_has_fpu(&vcpu->arch))
+                       return -EINVAL;
+               fpu->fcr31 = v;
+               break;
+
+       /* MIPS SIMD Architecture (MSA) registers */
+       case KVM_REG_MIPS_VEC_128(0) ... KVM_REG_MIPS_VEC_128(31):
+               if (!kvm_mips_guest_has_msa(&vcpu->arch))
+                       return -EINVAL;
+               idx = reg->id - KVM_REG_MIPS_VEC_128(0);
+#ifdef CONFIG_CPU_LITTLE_ENDIAN
+               /* least significant byte first */
+               set_fpr64(&fpu->fpr[idx], 0, vs[0]);
+               set_fpr64(&fpu->fpr[idx], 1, vs[1]);
+#else
+               /* most significant byte first */
+               set_fpr64(&fpu->fpr[idx], 1, vs[0]);
+               set_fpr64(&fpu->fpr[idx], 0, vs[1]);
+#endif
+               break;
+       case KVM_REG_MIPS_MSA_IR:
+               if (!kvm_mips_guest_has_msa(&vcpu->arch))
+                       return -EINVAL;
+               /* Read-only */
+               break;
+       case KVM_REG_MIPS_MSA_CSR:
+               if (!kvm_mips_guest_has_msa(&vcpu->arch))
+                       return -EINVAL;
+               fpu->msacsr = v;
+               break;
+
+       /* Co-processor 0 registers */
        case KVM_REG_MIPS_CP0_INDEX:
                kvm_write_c0_guest_index(cop0, v);
                break;
@@ -686,6 +838,9 @@ static int kvm_mips_set_reg(struct kvm_vcpu *vcpu,
        case KVM_REG_MIPS_CP0_EPC:
                kvm_write_c0_guest_epc(cop0, v);
                break;
+       case KVM_REG_MIPS_CP0_PRID:
+               kvm_write_c0_guest_prid(cop0, v);
+               break;
        case KVM_REG_MIPS_CP0_ERROREPC:
                kvm_write_c0_guest_errorepc(cop0, v);
                break;
@@ -693,6 +848,12 @@ static int kvm_mips_set_reg(struct kvm_vcpu *vcpu,
        case KVM_REG_MIPS_CP0_COUNT:
        case KVM_REG_MIPS_CP0_COMPARE:
        case KVM_REG_MIPS_CP0_CAUSE:
+       case KVM_REG_MIPS_CP0_CONFIG:
+       case KVM_REG_MIPS_CP0_CONFIG1:
+       case KVM_REG_MIPS_CP0_CONFIG2:
+       case KVM_REG_MIPS_CP0_CONFIG3:
+       case KVM_REG_MIPS_CP0_CONFIG4:
+       case KVM_REG_MIPS_CP0_CONFIG5:
        case KVM_REG_MIPS_COUNT_CTL:
        case KVM_REG_MIPS_COUNT_RESUME:
        case KVM_REG_MIPS_COUNT_HZ:
@@ -703,6 +864,33 @@ static int kvm_mips_set_reg(struct kvm_vcpu *vcpu,
        return 0;
 }
 
+static int kvm_vcpu_ioctl_enable_cap(struct kvm_vcpu *vcpu,
+                                    struct kvm_enable_cap *cap)
+{
+       int r = 0;
+
+       if (!kvm_vm_ioctl_check_extension(vcpu->kvm, cap->cap))
+               return -EINVAL;
+       if (cap->flags)
+               return -EINVAL;
+       if (cap->args[0])
+               return -EINVAL;
+
+       switch (cap->cap) {
+       case KVM_CAP_MIPS_FPU:
+               vcpu->arch.fpu_enabled = true;
+               break;
+       case KVM_CAP_MIPS_MSA:
+               vcpu->arch.msa_enabled = true;
+               break;
+       default:
+               r = -EINVAL;
+               break;
+       }
+
+       return r;
+}
+
 long kvm_arch_vcpu_ioctl(struct file *filp, unsigned int ioctl,
                         unsigned long arg)
 {
@@ -760,6 +948,15 @@ long kvm_arch_vcpu_ioctl(struct file *filp, unsigned int ioctl,
                        r = kvm_vcpu_ioctl_interrupt(vcpu, &irq);
                        break;
                }
+       case KVM_ENABLE_CAP: {
+               struct kvm_enable_cap cap;
+
+               r = -EFAULT;
+               if (copy_from_user(&cap, argp, sizeof(cap)))
+                       goto out;
+               r = kvm_vcpu_ioctl_enable_cap(vcpu, &cap);
+               break;
+       }
        default:
                r = -ENOIOCTLCMD;
        }
@@ -868,11 +1065,30 @@ int kvm_vm_ioctl_check_extension(struct kvm *kvm, long ext)
 
        switch (ext) {
        case KVM_CAP_ONE_REG:
+       case KVM_CAP_ENABLE_CAP:
                r = 1;
                break;
        case KVM_CAP_COALESCED_MMIO:
                r = KVM_COALESCED_MMIO_PAGE_OFFSET;
                break;
+       case KVM_CAP_MIPS_FPU:
+               r = !!cpu_has_fpu;
+               break;
+       case KVM_CAP_MIPS_MSA:
+               /*
+                * We don't support MSA vector partitioning yet:
+                * 1) It would require explicit support which can't be tested
+                *    yet due to lack of support in current hardware.
+                * 2) It extends the state that would need to be saved/restored
+                *    by e.g. QEMU for migration.
+                *
+                * When vector partitioning hardware becomes available, support
+                * could be added by requiring a flag when enabling
+                * KVM_CAP_MIPS_MSA capability to indicate that userland knows
+                * to save/restore the appropriate extra state.
+                */
+               r = cpu_has_msa && !(boot_cpu_data.msa_id & MSA_IR_WRPF);
+               break;
        default:
                r = 0;
                break;
@@ -1119,6 +1335,30 @@ int kvm_mips_handle_exit(struct kvm_run *run, struct kvm_vcpu *vcpu)
                ret = kvm_mips_callbacks->handle_break(vcpu);
                break;
 
+       case T_TRAP:
+               ++vcpu->stat.trap_inst_exits;
+               trace_kvm_exit(vcpu, TRAP_INST_EXITS);
+               ret = kvm_mips_callbacks->handle_trap(vcpu);
+               break;
+
+       case T_MSAFPE:
+               ++vcpu->stat.msa_fpe_exits;
+               trace_kvm_exit(vcpu, MSA_FPE_EXITS);
+               ret = kvm_mips_callbacks->handle_msa_fpe(vcpu);
+               break;
+
+       case T_FPE:
+               ++vcpu->stat.fpe_exits;
+               trace_kvm_exit(vcpu, FPE_EXITS);
+               ret = kvm_mips_callbacks->handle_fpe(vcpu);
+               break;
+
+       case T_MSADIS:
+               ++vcpu->stat.msa_disabled_exits;
+               trace_kvm_exit(vcpu, MSA_DISABLED_EXITS);
+               ret = kvm_mips_callbacks->handle_msa_disabled(vcpu);
+               break;
+
        default:
                kvm_err("Exception Code: %d, not yet handled, @ PC: %p, inst: 0x%08x  BadVaddr: %#lx Status: %#lx\n",
                        exccode, opc, kvm_get_inst(opc, vcpu), badvaddr,
@@ -1146,12 +1386,233 @@ skip_emul:
                }
        }
 
+       if (ret == RESUME_GUEST) {
+               /*
+                * If FPU / MSA are enabled (i.e. the guest's FPU / MSA context
+                * is live), restore FCR31 / MSACSR.
+                *
+                * This should be before returning to the guest exception
+                * vector, as it may well cause an [MSA] FP exception if there
+                * are pending exception bits unmasked. (see
+                * kvm_mips_csr_die_notifier() for how that is handled).
+                */
+               if (kvm_mips_guest_has_fpu(&vcpu->arch) &&
+                   read_c0_status() & ST0_CU1)
+                       __kvm_restore_fcsr(&vcpu->arch);
+
+               if (kvm_mips_guest_has_msa(&vcpu->arch) &&
+                   read_c0_config5() & MIPS_CONF5_MSAEN)
+                       __kvm_restore_msacsr(&vcpu->arch);
+       }
+
        /* Disable HTW before returning to guest or host */
        htw_stop();
 
        return ret;
 }
 
+/* Enable FPU for guest and restore context */
+void kvm_own_fpu(struct kvm_vcpu *vcpu)
+{
+       struct mips_coproc *cop0 = vcpu->arch.cop0;
+       unsigned int sr, cfg5;
+
+       preempt_disable();
+
+       sr = kvm_read_c0_guest_status(cop0);
+
+       /*
+        * If MSA state is already live, it is undefined how it interacts with
+        * FR=0 FPU state, and we don't want to hit reserved instruction
+        * exceptions trying to save the MSA state later when CU=1 && FR=1, so
+        * play it safe and save it first.
+        *
+        * In theory we shouldn't ever hit this case since kvm_lose_fpu() should
+        * get called when guest CU1 is set, however we can't trust the guest
+        * not to clobber the status register directly via the commpage.
+        */
+       if (cpu_has_msa && sr & ST0_CU1 && !(sr & ST0_FR) &&
+           vcpu->arch.fpu_inuse & KVM_MIPS_FPU_MSA)
+               kvm_lose_fpu(vcpu);
+
+       /*
+        * Enable FPU for guest
+        * We set FR and FRE according to guest context
+        */
+       change_c0_status(ST0_CU1 | ST0_FR, sr);
+       if (cpu_has_fre) {
+               cfg5 = kvm_read_c0_guest_config5(cop0);
+               change_c0_config5(MIPS_CONF5_FRE, cfg5);
+       }
+       enable_fpu_hazard();
+
+       /* If guest FPU state not active, restore it now */
+       if (!(vcpu->arch.fpu_inuse & KVM_MIPS_FPU_FPU)) {
+               __kvm_restore_fpu(&vcpu->arch);
+               vcpu->arch.fpu_inuse |= KVM_MIPS_FPU_FPU;
+       }
+
+       preempt_enable();
+}
+
+#ifdef CONFIG_CPU_HAS_MSA
+/* Enable MSA for guest and restore context */
+void kvm_own_msa(struct kvm_vcpu *vcpu)
+{
+       struct mips_coproc *cop0 = vcpu->arch.cop0;
+       unsigned int sr, cfg5;
+
+       preempt_disable();
+
+       /*
+        * Enable FPU if enabled in guest, since we're restoring FPU context
+        * anyway. We set FR and FRE according to guest context.
+        */
+       if (kvm_mips_guest_has_fpu(&vcpu->arch)) {
+               sr = kvm_read_c0_guest_status(cop0);
+
+               /*
+                * If FR=0 FPU state is already live, it is undefined how it
+                * interacts with MSA state, so play it safe and save it first.
+                */
+               if (!(sr & ST0_FR) &&
+                   (vcpu->arch.fpu_inuse & (KVM_MIPS_FPU_FPU |
+                               KVM_MIPS_FPU_MSA)) == KVM_MIPS_FPU_FPU)
+                       kvm_lose_fpu(vcpu);
+
+               change_c0_status(ST0_CU1 | ST0_FR, sr);
+               if (sr & ST0_CU1 && cpu_has_fre) {
+                       cfg5 = kvm_read_c0_guest_config5(cop0);
+                       change_c0_config5(MIPS_CONF5_FRE, cfg5);
+               }
+       }
+
+       /* Enable MSA for guest */
+       set_c0_config5(MIPS_CONF5_MSAEN);
+       enable_fpu_hazard();
+
+       switch (vcpu->arch.fpu_inuse & (KVM_MIPS_FPU_FPU | KVM_MIPS_FPU_MSA)) {
+       case KVM_MIPS_FPU_FPU:
+               /*
+                * Guest FPU state already loaded, only restore upper MSA state
+                */
+               __kvm_restore_msa_upper(&vcpu->arch);
+               vcpu->arch.fpu_inuse |= KVM_MIPS_FPU_MSA;
+               break;
+       case 0:
+               /* Neither FPU or MSA already active, restore full MSA state */
+               __kvm_restore_msa(&vcpu->arch);
+               vcpu->arch.fpu_inuse |= KVM_MIPS_FPU_MSA;
+               if (kvm_mips_guest_has_fpu(&vcpu->arch))
+                       vcpu->arch.fpu_inuse |= KVM_MIPS_FPU_FPU;
+               break;
+       default:
+               break;
+       }
+
+       preempt_enable();
+}
+#endif
+
+/* Drop FPU & MSA without saving it */
+void kvm_drop_fpu(struct kvm_vcpu *vcpu)
+{
+       preempt_disable();
+       if (cpu_has_msa && vcpu->arch.fpu_inuse & KVM_MIPS_FPU_MSA) {
+               disable_msa();
+               vcpu->arch.fpu_inuse &= ~KVM_MIPS_FPU_MSA;
+       }
+       if (vcpu->arch.fpu_inuse & KVM_MIPS_FPU_FPU) {
+               clear_c0_status(ST0_CU1 | ST0_FR);
+               vcpu->arch.fpu_inuse &= ~KVM_MIPS_FPU_FPU;
+       }
+       preempt_enable();
+}
+
+/* Save and disable FPU & MSA */
+void kvm_lose_fpu(struct kvm_vcpu *vcpu)
+{
+       /*
+        * FPU & MSA get disabled in root context (hardware) when it is disabled
+        * in guest context (software), but the register state in the hardware
+        * may still be in use. This is why we explicitly re-enable the hardware
+        * before saving.
+        */
+
+       preempt_disable();
+       if (cpu_has_msa && vcpu->arch.fpu_inuse & KVM_MIPS_FPU_MSA) {
+               set_c0_config5(MIPS_CONF5_MSAEN);
+               enable_fpu_hazard();
+
+               __kvm_save_msa(&vcpu->arch);
+
+               /* Disable MSA & FPU */
+               disable_msa();
+               if (vcpu->arch.fpu_inuse & KVM_MIPS_FPU_FPU)
+                       clear_c0_status(ST0_CU1 | ST0_FR);
+               vcpu->arch.fpu_inuse &= ~(KVM_MIPS_FPU_FPU | KVM_MIPS_FPU_MSA);
+       } else if (vcpu->arch.fpu_inuse & KVM_MIPS_FPU_FPU) {
+               set_c0_status(ST0_CU1);
+               enable_fpu_hazard();
+
+               __kvm_save_fpu(&vcpu->arch);
+               vcpu->arch.fpu_inuse &= ~KVM_MIPS_FPU_FPU;
+
+               /* Disable FPU */
+               clear_c0_status(ST0_CU1 | ST0_FR);
+       }
+       preempt_enable();
+}
+
+/*
+ * Step over a specific ctc1 to FCSR and a specific ctcmsa to MSACSR which are
+ * used to restore guest FCSR/MSACSR state and may trigger a "harmless" FP/MSAFP
+ * exception if cause bits are set in the value being written.
+ */
+static int kvm_mips_csr_die_notify(struct notifier_block *self,
+                                  unsigned long cmd, void *ptr)
+{
+       struct die_args *args = (struct die_args *)ptr;
+       struct pt_regs *regs = args->regs;
+       unsigned long pc;
+
+       /* Only interested in FPE and MSAFPE */
+       if (cmd != DIE_FP && cmd != DIE_MSAFP)
+               return NOTIFY_DONE;
+
+       /* Return immediately if guest context isn't active */
+       if (!(current->flags & PF_VCPU))
+               return NOTIFY_DONE;
+
+       /* Should never get here from user mode */
+       BUG_ON(user_mode(regs));
+
+       pc = instruction_pointer(regs);
+       switch (cmd) {
+       case DIE_FP:
+               /* match 2nd instruction in __kvm_restore_fcsr */
+               if (pc != (unsigned long)&__kvm_restore_fcsr + 4)
+                       return NOTIFY_DONE;
+               break;
+       case DIE_MSAFP:
+               /* match 2nd/3rd instruction in __kvm_restore_msacsr */
+               if (!cpu_has_msa ||
+                   pc < (unsigned long)&__kvm_restore_msacsr + 4 ||
+                   pc > (unsigned long)&__kvm_restore_msacsr + 8)
+                       return NOTIFY_DONE;
+               break;
+       }
+
+       /* Move PC forward a little and continue executing */
+       instruction_pointer(regs) += 4;
+
+       return NOTIFY_STOP;
+}
+
+static struct notifier_block kvm_mips_csr_die_notifier = {
+       .notifier_call = kvm_mips_csr_die_notify,
+};
+
 int __init kvm_mips_init(void)
 {
        int ret;
@@ -1161,6 +1622,8 @@ int __init kvm_mips_init(void)
        if (ret)
                return ret;
 
+       register_die_notifier(&kvm_mips_csr_die_notifier);
+
        /*
         * On MIPS, kernel modules are executed from "mapped space", which
         * requires TLBs. The TLB handling code is statically linked with
@@ -1173,7 +1636,6 @@ int __init kvm_mips_init(void)
        kvm_mips_release_pfn_clean = kvm_release_pfn_clean;
        kvm_mips_is_error_pfn = is_error_pfn;
 
-       pr_info("KVM/MIPS Initialized\n");
        return 0;
 }
 
@@ -1185,7 +1647,7 @@ void __exit kvm_mips_exit(void)
        kvm_mips_release_pfn_clean = NULL;
        kvm_mips_is_error_pfn = NULL;
 
-       pr_info("KVM/MIPS unloaded\n");
+       unregister_die_notifier(&kvm_mips_csr_die_notifier);
 }
 
 module_init(kvm_mips_init);
diff --git a/arch/mips/kvm/msa.S b/arch/mips/kvm/msa.S
new file mode 100644 (file)
index 0000000..d02f0c6
--- /dev/null
@@ -0,0 +1,161 @@
+/*
+ * This file is subject to the terms and conditions of the GNU General Public
+ * License.  See the file "COPYING" in the main directory of this archive
+ * for more details.
+ *
+ * MIPS SIMD Architecture (MSA) context handling code for KVM.
+ *
+ * Copyright (C) 2015 Imagination Technologies Ltd.
+ */
+
+#include <asm/asm.h>
+#include <asm/asm-offsets.h>
+#include <asm/asmmacro.h>
+#include <asm/regdef.h>
+
+       .set    noreorder
+       .set    noat
+
+LEAF(__kvm_save_msa)
+       st_d    0,  VCPU_FPR0,  a0
+       st_d    1,  VCPU_FPR1,  a0
+       st_d    2,  VCPU_FPR2,  a0
+       st_d    3,  VCPU_FPR3,  a0
+       st_d    4,  VCPU_FPR4,  a0
+       st_d    5,  VCPU_FPR5,  a0
+       st_d    6,  VCPU_FPR6,  a0
+       st_d    7,  VCPU_FPR7,  a0
+       st_d    8,  VCPU_FPR8,  a0
+       st_d    9,  VCPU_FPR9,  a0
+       st_d    10, VCPU_FPR10, a0
+       st_d    11, VCPU_FPR11, a0
+       st_d    12, VCPU_FPR12, a0
+       st_d    13, VCPU_FPR13, a0
+       st_d    14, VCPU_FPR14, a0
+       st_d    15, VCPU_FPR15, a0
+       st_d    16, VCPU_FPR16, a0
+       st_d    17, VCPU_FPR17, a0
+       st_d    18, VCPU_FPR18, a0
+       st_d    19, VCPU_FPR19, a0
+       st_d    20, VCPU_FPR20, a0
+       st_d    21, VCPU_FPR21, a0
+       st_d    22, VCPU_FPR22, a0
+       st_d    23, VCPU_FPR23, a0
+       st_d    24, VCPU_FPR24, a0
+       st_d    25, VCPU_FPR25, a0
+       st_d    26, VCPU_FPR26, a0
+       st_d    27, VCPU_FPR27, a0
+       st_d    28, VCPU_FPR28, a0
+       st_d    29, VCPU_FPR29, a0
+       st_d    30, VCPU_FPR30, a0
+       st_d    31, VCPU_FPR31, a0
+       jr      ra
+        nop
+       END(__kvm_save_msa)
+
+LEAF(__kvm_restore_msa)
+       ld_d    0,  VCPU_FPR0,  a0
+       ld_d    1,  VCPU_FPR1,  a0
+       ld_d    2,  VCPU_FPR2,  a0
+       ld_d    3,  VCPU_FPR3,  a0
+       ld_d    4,  VCPU_FPR4,  a0
+       ld_d    5,  VCPU_FPR5,  a0
+       ld_d    6,  VCPU_FPR6,  a0
+       ld_d    7,  VCPU_FPR7,  a0
+       ld_d    8,  VCPU_FPR8,  a0
+       ld_d    9,  VCPU_FPR9,  a0
+       ld_d    10, VCPU_FPR10, a0
+       ld_d    11, VCPU_FPR11, a0
+       ld_d    12, VCPU_FPR12, a0
+       ld_d    13, VCPU_FPR13, a0
+       ld_d    14, VCPU_FPR14, a0
+       ld_d    15, VCPU_FPR15, a0
+       ld_d    16, VCPU_FPR16, a0
+       ld_d    17, VCPU_FPR17, a0
+       ld_d    18, VCPU_FPR18, a0
+       ld_d    19, VCPU_FPR19, a0
+       ld_d    20, VCPU_FPR20, a0
+       ld_d    21, VCPU_FPR21, a0
+       ld_d    22, VCPU_FPR22, a0
+       ld_d    23, VCPU_FPR23, a0
+       ld_d    24, VCPU_FPR24, a0
+       ld_d    25, VCPU_FPR25, a0
+       ld_d    26, VCPU_FPR26, a0
+       ld_d    27, VCPU_FPR27, a0
+       ld_d    28, VCPU_FPR28, a0
+       ld_d    29, VCPU_FPR29, a0
+       ld_d    30, VCPU_FPR30, a0
+       ld_d    31, VCPU_FPR31, a0
+       jr      ra
+        nop
+       END(__kvm_restore_msa)
+
+       .macro  kvm_restore_msa_upper   wr, off, base
+       .set    push
+       .set    noat
+#ifdef CONFIG_64BIT
+       ld      $1, \off(\base)
+       insert_d \wr, 1
+#elif defined(CONFIG_CPU_LITTLE_ENDIAN)
+       lw      $1, \off(\base)
+       insert_w \wr, 2
+       lw      $1, (\off+4)(\base)
+       insert_w \wr, 3
+#else /* CONFIG_CPU_BIG_ENDIAN */
+       lw      $1, (\off+4)(\base)
+       insert_w \wr, 2
+       lw      $1, \off(\base)
+       insert_w \wr, 3
+#endif
+       .set    pop
+       .endm
+
+LEAF(__kvm_restore_msa_upper)
+       kvm_restore_msa_upper   0,  VCPU_FPR0 +8, a0
+       kvm_restore_msa_upper   1,  VCPU_FPR1 +8, a0
+       kvm_restore_msa_upper   2,  VCPU_FPR2 +8, a0
+       kvm_restore_msa_upper   3,  VCPU_FPR3 +8, a0
+       kvm_restore_msa_upper   4,  VCPU_FPR4 +8, a0
+       kvm_restore_msa_upper   5,  VCPU_FPR5 +8, a0
+       kvm_restore_msa_upper   6,  VCPU_FPR6 +8, a0
+       kvm_restore_msa_upper   7,  VCPU_FPR7 +8, a0
+       kvm_restore_msa_upper   8,  VCPU_FPR8 +8, a0
+       kvm_restore_msa_upper   9,  VCPU_FPR9 +8, a0
+       kvm_restore_msa_upper   10, VCPU_FPR10+8, a0
+       kvm_restore_msa_upper   11, VCPU_FPR11+8, a0
+       kvm_restore_msa_upper   12, VCPU_FPR12+8, a0
+       kvm_restore_msa_upper   13, VCPU_FPR13+8, a0
+       kvm_restore_msa_upper   14, VCPU_FPR14+8, a0
+       kvm_restore_msa_upper   15, VCPU_FPR15+8, a0
+       kvm_restore_msa_upper   16, VCPU_FPR16+8, a0
+       kvm_restore_msa_upper   17, VCPU_FPR17+8, a0
+       kvm_restore_msa_upper   18, VCPU_FPR18+8, a0
+       kvm_restore_msa_upper   19, VCPU_FPR19+8, a0
+       kvm_restore_msa_upper   20, VCPU_FPR20+8, a0
+       kvm_restore_msa_upper   21, VCPU_FPR21+8, a0
+       kvm_restore_msa_upper   22, VCPU_FPR22+8, a0
+       kvm_restore_msa_upper   23, VCPU_FPR23+8, a0
+       kvm_restore_msa_upper   24, VCPU_FPR24+8, a0
+       kvm_restore_msa_upper   25, VCPU_FPR25+8, a0
+       kvm_restore_msa_upper   26, VCPU_FPR26+8, a0
+       kvm_restore_msa_upper   27, VCPU_FPR27+8, a0
+       kvm_restore_msa_upper   28, VCPU_FPR28+8, a0
+       kvm_restore_msa_upper   29, VCPU_FPR29+8, a0
+       kvm_restore_msa_upper   30, VCPU_FPR30+8, a0
+       kvm_restore_msa_upper   31, VCPU_FPR31+8, a0
+       jr      ra
+        nop
+       END(__kvm_restore_msa_upper)
+
+LEAF(__kvm_restore_msacsr)
+       lw      t0, VCPU_MSA_CSR(a0)
+       /*
+        * The ctcmsa must stay at this offset in __kvm_restore_msacsr.
+        * See kvm_mips_csr_die_notify() which handles t0 containing a value
+        * which triggers an MSA FP Exception, which must be stepped over and
+        * ignored since the set cause bits must remain there for the guest.
+        */
+       _ctcmsa MSA_CSR, t0
+       jr      ra
+        nop
+       END(__kvm_restore_msacsr)
index a74d6024c5ad5f5c7e0701a309e246bd42532992..888bb67070ac6d1139a7f06f5958e180680511ef 100644 (file)
@@ -25,6 +25,10 @@ char *kvm_mips_exit_types_str[MAX_KVM_MIPS_EXIT_TYPES] = {
        "System Call",
        "Reserved Inst",
        "Break Inst",
+       "Trap Inst",
+       "MSA FPE",
+       "FPE",
+       "MSA Disabled",
        "D-Cache Flushes",
 };
 
index b6beb0e07b1b3b535f7625d61100e6c0087de00d..aed0ac2a4972cd1daf0f2992db6c100e9912fb70 100644 (file)
@@ -733,6 +733,9 @@ void kvm_arch_vcpu_load(struct kvm_vcpu *vcpu, int cpu)
                }
        }
 
+       /* restore guest state to registers */
+       kvm_mips_callbacks->vcpu_set_regs(vcpu);
+
        local_irq_restore(flags);
 
 }
@@ -751,6 +754,9 @@ void kvm_arch_vcpu_put(struct kvm_vcpu *vcpu)
        vcpu->arch.preempt_entryhi = read_c0_entryhi();
        vcpu->arch.last_sched_cpu = cpu;
 
+       /* save guest state in registers */
+       kvm_mips_callbacks->vcpu_get_regs(vcpu);
+
        if (((cpu_context(cpu, current->mm) ^ asid_cache(cpu)) &
             ASID_VERSION_MASK)) {
                kvm_debug("%s: Dropping MMU Context:  %#lx\n", __func__,
index fd7257b70e656fcb8c53d72b552f240f5b255ce6..d836ed5b0bc7ea38e36350304a6238a520e0d74d 100644 (file)
@@ -39,16 +39,30 @@ static gpa_t kvm_trap_emul_gva_to_gpa_cb(gva_t gva)
 
 static int kvm_trap_emul_handle_cop_unusable(struct kvm_vcpu *vcpu)
 {
+       struct mips_coproc *cop0 = vcpu->arch.cop0;
        struct kvm_run *run = vcpu->run;
        uint32_t __user *opc = (uint32_t __user *) vcpu->arch.pc;
        unsigned long cause = vcpu->arch.host_cp0_cause;
        enum emulation_result er = EMULATE_DONE;
        int ret = RESUME_GUEST;
 
-       if (((cause & CAUSEF_CE) >> CAUSEB_CE) == 1)
-               er = kvm_mips_emulate_fpu_exc(cause, opc, run, vcpu);
-       else
+       if (((cause & CAUSEF_CE) >> CAUSEB_CE) == 1) {
+               /* FPU Unusable */
+               if (!kvm_mips_guest_has_fpu(&vcpu->arch) ||
+                   (kvm_read_c0_guest_status(cop0) & ST0_CU1) == 0) {
+                       /*
+                        * Unusable/no FPU in guest:
+                        * deliver guest COP1 Unusable Exception
+                        */
+                       er = kvm_mips_emulate_fpu_exc(cause, opc, run, vcpu);
+               } else {
+                       /* Restore FPU state */
+                       kvm_own_fpu(vcpu);
+                       er = EMULATE_DONE;
+               }
+       } else {
                er = kvm_mips_emulate_inst(cause, opc, run, vcpu);
+       }
 
        switch (er) {
        case EMULATE_DONE:
@@ -330,6 +344,107 @@ static int kvm_trap_emul_handle_break(struct kvm_vcpu *vcpu)
        return ret;
 }
 
+static int kvm_trap_emul_handle_trap(struct kvm_vcpu *vcpu)
+{
+       struct kvm_run *run = vcpu->run;
+       uint32_t __user *opc = (uint32_t __user *)vcpu->arch.pc;
+       unsigned long cause = vcpu->arch.host_cp0_cause;
+       enum emulation_result er = EMULATE_DONE;
+       int ret = RESUME_GUEST;
+
+       er = kvm_mips_emulate_trap_exc(cause, opc, run, vcpu);
+       if (er == EMULATE_DONE) {
+               ret = RESUME_GUEST;
+       } else {
+               run->exit_reason = KVM_EXIT_INTERNAL_ERROR;
+               ret = RESUME_HOST;
+       }
+       return ret;
+}
+
+static int kvm_trap_emul_handle_msa_fpe(struct kvm_vcpu *vcpu)
+{
+       struct kvm_run *run = vcpu->run;
+       uint32_t __user *opc = (uint32_t __user *)vcpu->arch.pc;
+       unsigned long cause = vcpu->arch.host_cp0_cause;
+       enum emulation_result er = EMULATE_DONE;
+       int ret = RESUME_GUEST;
+
+       er = kvm_mips_emulate_msafpe_exc(cause, opc, run, vcpu);
+       if (er == EMULATE_DONE) {
+               ret = RESUME_GUEST;
+       } else {
+               run->exit_reason = KVM_EXIT_INTERNAL_ERROR;
+               ret = RESUME_HOST;
+       }
+       return ret;
+}
+
+static int kvm_trap_emul_handle_fpe(struct kvm_vcpu *vcpu)
+{
+       struct kvm_run *run = vcpu->run;
+       uint32_t __user *opc = (uint32_t __user *)vcpu->arch.pc;
+       unsigned long cause = vcpu->arch.host_cp0_cause;
+       enum emulation_result er = EMULATE_DONE;
+       int ret = RESUME_GUEST;
+
+       er = kvm_mips_emulate_fpe_exc(cause, opc, run, vcpu);
+       if (er == EMULATE_DONE) {
+               ret = RESUME_GUEST;
+       } else {
+               run->exit_reason = KVM_EXIT_INTERNAL_ERROR;
+               ret = RESUME_HOST;
+       }
+       return ret;
+}
+
+/**
+ * kvm_trap_emul_handle_msa_disabled() - Guest used MSA while disabled in root.
+ * @vcpu:      Virtual CPU context.
+ *
+ * Handle when the guest attempts to use MSA when it is disabled.
+ */
+static int kvm_trap_emul_handle_msa_disabled(struct kvm_vcpu *vcpu)
+{
+       struct mips_coproc *cop0 = vcpu->arch.cop0;
+       struct kvm_run *run = vcpu->run;
+       uint32_t __user *opc = (uint32_t __user *) vcpu->arch.pc;
+       unsigned long cause = vcpu->arch.host_cp0_cause;
+       enum emulation_result er = EMULATE_DONE;
+       int ret = RESUME_GUEST;
+
+       if (!kvm_mips_guest_has_msa(&vcpu->arch) ||
+           (kvm_read_c0_guest_status(cop0) & (ST0_CU1 | ST0_FR)) == ST0_CU1) {
+               /*
+                * No MSA in guest, or FPU enabled and not in FR=1 mode,
+                * guest reserved instruction exception
+                */
+               er = kvm_mips_emulate_ri_exc(cause, opc, run, vcpu);
+       } else if (!(kvm_read_c0_guest_config5(cop0) & MIPS_CONF5_MSAEN)) {
+               /* MSA disabled by guest, guest MSA disabled exception */
+               er = kvm_mips_emulate_msadis_exc(cause, opc, run, vcpu);
+       } else {
+               /* Restore MSA/FPU state */
+               kvm_own_msa(vcpu);
+               er = EMULATE_DONE;
+       }
+
+       switch (er) {
+       case EMULATE_DONE:
+               ret = RESUME_GUEST;
+               break;
+
+       case EMULATE_FAIL:
+               run->exit_reason = KVM_EXIT_INTERNAL_ERROR;
+               ret = RESUME_HOST;
+               break;
+
+       default:
+               BUG();
+       }
+       return ret;
+}
+
 static int kvm_trap_emul_vm_init(struct kvm *kvm)
 {
        return 0;
@@ -351,8 +466,9 @@ static int kvm_trap_emul_vcpu_setup(struct kvm_vcpu *vcpu)
         * guest will come up as expected, for now we simulate a MIPS 24kc
         */
        kvm_write_c0_guest_prid(cop0, 0x00019300);
-       kvm_write_c0_guest_config(cop0,
-                                 MIPS_CONFIG0 | (0x1 << CP0C0_AR) |
+       /* Have config1, Cacheable, noncoherent, write-back, write allocate */
+       kvm_write_c0_guest_config(cop0, MIPS_CONF_M | (0x3 << CP0C0_K0) |
+                                 (0x1 << CP0C0_AR) |
                                  (MMU_TYPE_R4000 << CP0C0_MT));
 
        /* Read the cache characteristics from the host Config1 Register */
@@ -368,10 +484,18 @@ static int kvm_trap_emul_vcpu_setup(struct kvm_vcpu *vcpu)
              (1 << CP0C1_WR) | (1 << CP0C1_CA));
        kvm_write_c0_guest_config1(cop0, config1);
 
-       kvm_write_c0_guest_config2(cop0, MIPS_CONFIG2);
-       /* MIPS_CONFIG2 | (read_c0_config2() & 0xfff) */
-       kvm_write_c0_guest_config3(cop0, MIPS_CONFIG3 | (0 << CP0C3_VInt) |
-                                        (1 << CP0C3_ULRI));
+       /* Have config3, no tertiary/secondary caches implemented */
+       kvm_write_c0_guest_config2(cop0, MIPS_CONF_M);
+       /* MIPS_CONF_M | (read_c0_config2() & 0xfff) */
+
+       /* Have config4, UserLocal */
+       kvm_write_c0_guest_config3(cop0, MIPS_CONF_M | MIPS_CONF3_ULRI);
+
+       /* Have config5 */
+       kvm_write_c0_guest_config4(cop0, MIPS_CONF_M);
+
+       /* No config6 */
+       kvm_write_c0_guest_config5(cop0, 0);
 
        /* Set Wait IE/IXMT Ignore in Config7, IAR, AR */
        kvm_write_c0_guest_config7(cop0, (MIPS_CONF7_WII) | (1 << 10));
@@ -416,6 +540,7 @@ static int kvm_trap_emul_set_one_reg(struct kvm_vcpu *vcpu,
 {
        struct mips_coproc *cop0 = vcpu->arch.cop0;
        int ret = 0;
+       unsigned int cur, change;
 
        switch (reg->id) {
        case KVM_REG_MIPS_CP0_COUNT:
@@ -444,6 +569,44 @@ static int kvm_trap_emul_set_one_reg(struct kvm_vcpu *vcpu,
                        kvm_write_c0_guest_cause(cop0, v);
                }
                break;
+       case KVM_REG_MIPS_CP0_CONFIG:
+               /* read-only for now */
+               break;
+       case KVM_REG_MIPS_CP0_CONFIG1:
+               cur = kvm_read_c0_guest_config1(cop0);
+               change = (cur ^ v) & kvm_mips_config1_wrmask(vcpu);
+               if (change) {
+                       v = cur ^ change;
+                       kvm_write_c0_guest_config1(cop0, v);
+               }
+               break;
+       case KVM_REG_MIPS_CP0_CONFIG2:
+               /* read-only for now */
+               break;
+       case KVM_REG_MIPS_CP0_CONFIG3:
+               cur = kvm_read_c0_guest_config3(cop0);
+               change = (cur ^ v) & kvm_mips_config3_wrmask(vcpu);
+               if (change) {
+                       v = cur ^ change;
+                       kvm_write_c0_guest_config3(cop0, v);
+               }
+               break;
+       case KVM_REG_MIPS_CP0_CONFIG4:
+               cur = kvm_read_c0_guest_config4(cop0);
+               change = (cur ^ v) & kvm_mips_config4_wrmask(vcpu);
+               if (change) {
+                       v = cur ^ change;
+                       kvm_write_c0_guest_config4(cop0, v);
+               }
+               break;
+       case KVM_REG_MIPS_CP0_CONFIG5:
+               cur = kvm_read_c0_guest_config5(cop0);
+               change = (cur ^ v) & kvm_mips_config5_wrmask(vcpu);
+               if (change) {
+                       v = cur ^ change;
+                       kvm_write_c0_guest_config5(cop0, v);
+               }
+               break;
        case KVM_REG_MIPS_COUNT_CTL:
                ret = kvm_mips_set_count_ctl(vcpu, v);
                break;
@@ -459,6 +622,18 @@ static int kvm_trap_emul_set_one_reg(struct kvm_vcpu *vcpu,
        return ret;
 }
 
+static int kvm_trap_emul_vcpu_get_regs(struct kvm_vcpu *vcpu)
+{
+       kvm_lose_fpu(vcpu);
+
+       return 0;
+}
+
+static int kvm_trap_emul_vcpu_set_regs(struct kvm_vcpu *vcpu)
+{
+       return 0;
+}
+
 static struct kvm_mips_callbacks kvm_trap_emul_callbacks = {
        /* exit handlers */
        .handle_cop_unusable = kvm_trap_emul_handle_cop_unusable,
@@ -470,6 +645,10 @@ static struct kvm_mips_callbacks kvm_trap_emul_callbacks = {
        .handle_syscall = kvm_trap_emul_handle_syscall,
        .handle_res_inst = kvm_trap_emul_handle_res_inst,
        .handle_break = kvm_trap_emul_handle_break,
+       .handle_trap = kvm_trap_emul_handle_trap,
+       .handle_msa_fpe = kvm_trap_emul_handle_msa_fpe,
+       .handle_fpe = kvm_trap_emul_handle_fpe,
+       .handle_msa_disabled = kvm_trap_emul_handle_msa_disabled,
 
        .vm_init = kvm_trap_emul_vm_init,
        .vcpu_init = kvm_trap_emul_vcpu_init,
@@ -483,6 +662,8 @@ static struct kvm_mips_callbacks kvm_trap_emul_callbacks = {
        .irq_clear = kvm_mips_irq_clear_cb,
        .get_one_reg = kvm_trap_emul_get_one_reg,
        .set_one_reg = kvm_trap_emul_set_one_reg,
+       .vcpu_get_regs = kvm_trap_emul_vcpu_get_regs,
+       .vcpu_set_regs = kvm_trap_emul_vcpu_set_regs,
 };
 
 int kvm_mips_emulation_init(struct kvm_mips_callbacks **install_callbacks)
index 39b3a8f816f28d0ecd61ca8a110d22bb99fb4507..6249cdc834d14977ffe5344a4b15857de01abe79 100644 (file)
@@ -34,7 +34,7 @@
 #include <asm/kvm_para.h>
 #include <asm/kvm_host.h>
 #include <asm/kvm_ppc.h>
-#include "iodev.h"
+#include <kvm/iodev.h>
 
 #define MAX_CPU     32
 #define MAX_SRC     256
@@ -289,11 +289,6 @@ static inline void IRQ_resetbit(struct irq_queue *q, int n_IRQ)
        clear_bit(n_IRQ, q->queue);
 }
 
-static inline int IRQ_testbit(struct irq_queue *q, int n_IRQ)
-{
-       return test_bit(n_IRQ, q->queue);
-}
-
 static void IRQ_check(struct openpic *opp, struct irq_queue *q)
 {
        int irq = -1;
@@ -1374,8 +1369,9 @@ static int kvm_mpic_write_internal(struct openpic *opp, gpa_t addr, u32 val)
        return -ENXIO;
 }
 
-static int kvm_mpic_read(struct kvm_io_device *this, gpa_t addr,
-                        int len, void *ptr)
+static int kvm_mpic_read(struct kvm_vcpu *vcpu,
+                        struct kvm_io_device *this,
+                        gpa_t addr, int len, void *ptr)
 {
        struct openpic *opp = container_of(this, struct openpic, mmio);
        int ret;
@@ -1415,8 +1411,9 @@ static int kvm_mpic_read(struct kvm_io_device *this, gpa_t addr,
        return ret;
 }
 
-static int kvm_mpic_write(struct kvm_io_device *this, gpa_t addr,
-                         int len, const void *ptr)
+static int kvm_mpic_write(struct kvm_vcpu *vcpu,
+                         struct kvm_io_device *this,
+                         gpa_t addr, int len, const void *ptr)
 {
        struct openpic *opp = container_of(this, struct openpic, mmio);
        int ret;
index 27c0face86f45cdac10ac74ba309de04094a8367..24bfe401373e44aad58268c95caf7e2a5e09198e 100644 (file)
@@ -807,7 +807,7 @@ int kvmppc_handle_load(struct kvm_run *run, struct kvm_vcpu *vcpu,
 
        idx = srcu_read_lock(&vcpu->kvm->srcu);
 
-       ret = kvm_io_bus_read(vcpu->kvm, KVM_MMIO_BUS, run->mmio.phys_addr,
+       ret = kvm_io_bus_read(vcpu, KVM_MMIO_BUS, run->mmio.phys_addr,
                              bytes, &run->mmio.data);
 
        srcu_read_unlock(&vcpu->kvm->srcu, idx);
@@ -880,7 +880,7 @@ int kvmppc_handle_store(struct kvm_run *run, struct kvm_vcpu *vcpu,
 
        idx = srcu_read_lock(&vcpu->kvm->srcu);
 
-       ret = kvm_io_bus_write(vcpu->kvm, KVM_MMIO_BUS, run->mmio.phys_addr,
+       ret = kvm_io_bus_write(vcpu, KVM_MMIO_BUS, run->mmio.phys_addr,
                               bytes, &run->mmio.data);
 
        srcu_read_unlock(&vcpu->kvm->srcu, idx);
index f407bbf5ee94ca5e2f6122951e52ce2d7db1a7ef..d01fc588b5c378fddc46eba49e28b4de4be1f1a9 100644 (file)
@@ -172,7 +172,9 @@ struct kvm_s390_sie_block {
        __u32   fac;                    /* 0x01a0 */
        __u8    reserved1a4[20];        /* 0x01a4 */
        __u64   cbrlo;                  /* 0x01b8 */
-       __u8    reserved1c0[30];        /* 0x01c0 */
+       __u8    reserved1c0[8];         /* 0x01c0 */
+       __u32   ecd;                    /* 0x01c8 */
+       __u8    reserved1cc[18];        /* 0x01cc */
        __u64   pp;                     /* 0x01de */
        __u8    reserved1e6[2];         /* 0x01e6 */
        __u64   itdba;                  /* 0x01e8 */
@@ -183,11 +185,17 @@ struct kvm_s390_itdb {
        __u8    data[256];
 } __packed;
 
+struct kvm_s390_vregs {
+       __vector128 vrs[32];
+       __u8    reserved200[512];       /* for future vector expansion */
+} __packed;
+
 struct sie_page {
        struct kvm_s390_sie_block sie_block;
        __u8 reserved200[1024];         /* 0x0200 */
        struct kvm_s390_itdb itdb;      /* 0x0600 */
-       __u8 reserved700[2304];         /* 0x0700 */
+       __u8 reserved700[1280];         /* 0x0700 */
+       struct kvm_s390_vregs vregs;    /* 0x0c00 */
 } __packed;
 
 struct kvm_vcpu_stat {
@@ -238,6 +246,7 @@ struct kvm_vcpu_stat {
        u32 instruction_sigp_stop;
        u32 instruction_sigp_stop_store_status;
        u32 instruction_sigp_store_status;
+       u32 instruction_sigp_store_adtl_status;
        u32 instruction_sigp_arch;
        u32 instruction_sigp_prefix;
        u32 instruction_sigp_restart;
@@ -270,6 +279,7 @@ struct kvm_vcpu_stat {
 #define PGM_SPECIAL_OPERATION          0x13
 #define PGM_OPERAND                    0x15
 #define PGM_TRACE_TABEL                        0x16
+#define PGM_VECTOR_PROCESSING          0x1b
 #define PGM_SPACE_SWITCH               0x1c
 #define PGM_HFP_SQUARE_ROOT            0x1d
 #define PGM_PC_TRANSLATION_SPEC                0x1f
@@ -334,6 +344,11 @@ enum irq_types {
        IRQ_PEND_COUNT
 };
 
+/* We have 2M for virtio device descriptor pages. Smallest amount of
+ * memory per page is 24 bytes (1 queue), so (2048*1024) / 24 = 87381
+ */
+#define KVM_S390_MAX_VIRTIO_IRQS 87381
+
 /*
  * Repressible (non-floating) machine check interrupts
  * subclass bits in MCIC
@@ -411,13 +426,32 @@ struct kvm_s390_local_interrupt {
        unsigned long pending_irqs;
 };
 
+#define FIRQ_LIST_IO_ISC_0 0
+#define FIRQ_LIST_IO_ISC_1 1
+#define FIRQ_LIST_IO_ISC_2 2
+#define FIRQ_LIST_IO_ISC_3 3
+#define FIRQ_LIST_IO_ISC_4 4
+#define FIRQ_LIST_IO_ISC_5 5
+#define FIRQ_LIST_IO_ISC_6 6
+#define FIRQ_LIST_IO_ISC_7 7
+#define FIRQ_LIST_PFAULT   8
+#define FIRQ_LIST_VIRTIO   9
+#define FIRQ_LIST_COUNT   10
+#define FIRQ_CNTR_IO       0
+#define FIRQ_CNTR_SERVICE  1
+#define FIRQ_CNTR_VIRTIO   2
+#define FIRQ_CNTR_PFAULT   3
+#define FIRQ_MAX_COUNT     4
+
 struct kvm_s390_float_interrupt {
+       unsigned long pending_irqs;
        spinlock_t lock;
-       struct list_head list;
-       atomic_t active;
+       struct list_head lists[FIRQ_LIST_COUNT];
+       int counters[FIRQ_MAX_COUNT];
+       struct kvm_s390_mchk_info mchk;
+       struct kvm_s390_ext_info srv_signal;
        int next_rr_cpu;
        unsigned long idle_mask[BITS_TO_LONGS(KVM_MAX_VCPUS)];
-       unsigned int irq_count;
 };
 
 struct kvm_hw_wp_info_arch {
@@ -465,6 +499,7 @@ struct kvm_vcpu_arch {
        s390_fp_regs      host_fpregs;
        unsigned int      host_acrs[NUM_ACRS];
        s390_fp_regs      guest_fpregs;
+       struct kvm_s390_vregs   *host_vregs;
        struct kvm_s390_local_interrupt local_int;
        struct hrtimer    ckc_timer;
        struct kvm_s390_pgm_info pgm;
@@ -553,6 +588,7 @@ struct kvm_arch{
        int use_cmma;
        int user_cpu_state_ctrl;
        int user_sigp;
+       int user_stsi;
        struct s390_io_adapter *adapters[MAX_S390_IO_ADAPTERS];
        wait_queue_head_t ipte_wq;
        int ipte_lock_count;
index 9c77e60b9a269a59bcde0d4fe0a5fb77348f1520..ef1a5fcc6c66bbf5705173b41371378c4b541483 100644 (file)
@@ -150,6 +150,7 @@ struct kvm_guest_debug_arch {
 #define KVM_SYNC_CRS    (1UL << 3)
 #define KVM_SYNC_ARCH0  (1UL << 4)
 #define KVM_SYNC_PFAULT (1UL << 5)
+#define KVM_SYNC_VRS    (1UL << 6)
 /* definition of registers in kvm_run */
 struct kvm_sync_regs {
        __u64 prefix;   /* prefix register */
@@ -164,6 +165,9 @@ struct kvm_sync_regs {
        __u64 pft;      /* pfault token [PFAULT] */
        __u64 pfs;      /* pfault select [PFAULT] */
        __u64 pfc;      /* pfault compare [PFAULT] */
+       __u64 vrs[32][2];       /* vector registers */
+       __u8  reserved[512];    /* for future vector expansion */
+       __u32 fpc;      /* only valid with vector registers */
 };
 
 #define KVM_REG_S390_TODPR     (KVM_REG_S390 | KVM_REG_SIZE_U32 | 0x1)
index d4096fdfc6ab45b02eda2f0a7258da5b98f46930..ee69c0854c8891067b67a0d8920e7f3cf671f308 100644 (file)
  * and returns a key, which can be used to find a mnemonic name
  * of the instruction in the icpt_insn_codes table.
  */
-#define icpt_insn_decoder(insn)                        \
+#define icpt_insn_decoder(insn) (              \
        INSN_DECODE_IPA0(0x01, insn, 48, 0xff)  \
        INSN_DECODE_IPA0(0xaa, insn, 48, 0x0f)  \
        INSN_DECODE_IPA0(0xb2, insn, 48, 0xff)  \
        INSN_DECODE_IPA0(0xe5, insn, 48, 0xff)  \
        INSN_DECODE_IPA0(0xeb, insn, 16, 0xff)  \
        INSN_DECODE_IPA0(0xc8, insn, 48, 0x0f)  \
-       INSN_DECODE(insn)
+       INSN_DECODE(insn))
 
 #endif /* _UAPI_ASM_S390_SIE_H */
index e07e91605353003084ff19511c1c520e8bca1dd7..8dc4db10d1608e81eafad41e5f87d1c92af7bbf5 100644 (file)
@@ -171,6 +171,7 @@ int main(void)
 #else /* CONFIG_32BIT */
        DEFINE(__LC_DATA_EXC_CODE, offsetof(struct _lowcore, data_exc_code));
        DEFINE(__LC_MCCK_FAIL_STOR_ADDR, offsetof(struct _lowcore, failing_storage_address));
+       DEFINE(__LC_VX_SAVE_AREA_ADDR, offsetof(struct _lowcore, vector_save_area_addr));
        DEFINE(__LC_EXT_PARAMS2, offsetof(struct _lowcore, ext_params2));
        DEFINE(SAVE_AREA_BASE, offsetof(struct _lowcore, floating_pt_save_area));
        DEFINE(__LC_PASTE, offsetof(struct _lowcore, paste));
index 9254afff250c968682db79c453d0542868bfdd9c..fc7ec95848c39c527c2a24ee723c9f45624e31a0 100644 (file)
@@ -77,7 +77,7 @@ static int __diag_page_ref_service(struct kvm_vcpu *vcpu)
 
        if (vcpu->run->s.regs.gprs[rx] & 7)
                return kvm_s390_inject_program_int(vcpu, PGM_SPECIFICATION);
-       rc = read_guest(vcpu, vcpu->run->s.regs.gprs[rx], &parm, sizeof(parm));
+       rc = read_guest(vcpu, vcpu->run->s.regs.gprs[rx], rx, &parm, sizeof(parm));
        if (rc)
                return kvm_s390_inject_prog_cond(vcpu, rc);
        if (parm.parm_version != 2 || parm.parm_len < 5 || parm.code != 0x258)
@@ -213,7 +213,7 @@ static int __diag_virtio_hypercall(struct kvm_vcpu *vcpu)
         * - gpr 3 contains the virtqueue index (passed as datamatch)
         * - gpr 4 contains the index on the bus (optionally)
         */
-       ret = kvm_io_bus_write_cookie(vcpu->kvm, KVM_VIRTIO_CCW_NOTIFY_BUS,
+       ret = kvm_io_bus_write_cookie(vcpu, KVM_VIRTIO_CCW_NOTIFY_BUS,
                                      vcpu->run->s.regs.gprs[2] & 0xffffffff,
                                      8, &vcpu->run->s.regs.gprs[3],
                                      vcpu->run->s.regs.gprs[4]);
@@ -230,7 +230,7 @@ static int __diag_virtio_hypercall(struct kvm_vcpu *vcpu)
 
 int kvm_s390_handle_diag(struct kvm_vcpu *vcpu)
 {
-       int code = kvm_s390_get_base_disp_rs(vcpu) & 0xffff;
+       int code = kvm_s390_get_base_disp_rs(vcpu, NULL) & 0xffff;
 
        if (vcpu->arch.sie_block->gpsw.mask & PSW_MASK_PSTATE)
                return kvm_s390_inject_program_int(vcpu, PGM_PRIVILEGED_OP);
index 267523cac6de7860cda644017e2b52ccfc7ea8d4..a7559f7207df3a0ac62d0fc16b199f3b4c6b6dac 100644 (file)
@@ -10,6 +10,7 @@
 #include <asm/pgtable.h>
 #include "kvm-s390.h"
 #include "gaccess.h"
+#include <asm/switch_to.h>
 
 union asce {
        unsigned long val;
@@ -207,6 +208,54 @@ union raddress {
        unsigned long pfra : 52; /* Page-Frame Real Address */
 };
 
+union alet {
+       u32 val;
+       struct {
+               u32 reserved : 7;
+               u32 p        : 1;
+               u32 alesn    : 8;
+               u32 alen     : 16;
+       };
+};
+
+union ald {
+       u32 val;
+       struct {
+               u32     : 1;
+               u32 alo : 24;
+               u32 all : 7;
+       };
+};
+
+struct ale {
+       unsigned long i      : 1; /* ALEN-Invalid Bit */
+       unsigned long        : 5;
+       unsigned long fo     : 1; /* Fetch-Only Bit */
+       unsigned long p      : 1; /* Private Bit */
+       unsigned long alesn  : 8; /* Access-List-Entry Sequence Number */
+       unsigned long aleax  : 16; /* Access-List-Entry Authorization Index */
+       unsigned long        : 32;
+       unsigned long        : 1;
+       unsigned long asteo  : 25; /* ASN-Second-Table-Entry Origin */
+       unsigned long        : 6;
+       unsigned long astesn : 32; /* ASTE Sequence Number */
+} __packed;
+
+struct aste {
+       unsigned long i      : 1; /* ASX-Invalid Bit */
+       unsigned long ato    : 29; /* Authority-Table Origin */
+       unsigned long        : 1;
+       unsigned long b      : 1; /* Base-Space Bit */
+       unsigned long ax     : 16; /* Authorization Index */
+       unsigned long atl    : 12; /* Authority-Table Length */
+       unsigned long        : 2;
+       unsigned long ca     : 1; /* Controlled-ASN Bit */
+       unsigned long ra     : 1; /* Reusable-ASN Bit */
+       unsigned long asce   : 64; /* Address-Space-Control Element */
+       unsigned long ald    : 32;
+       unsigned long astesn : 32;
+       /* .. more fields there */
+} __packed;
 
 int ipte_lock_held(struct kvm_vcpu *vcpu)
 {
@@ -307,15 +356,157 @@ void ipte_unlock(struct kvm_vcpu *vcpu)
                ipte_unlock_simple(vcpu);
 }
 
-static unsigned long get_vcpu_asce(struct kvm_vcpu *vcpu)
+static int ar_translation(struct kvm_vcpu *vcpu, union asce *asce, ar_t ar,
+                         int write)
+{
+       union alet alet;
+       struct ale ale;
+       struct aste aste;
+       unsigned long ald_addr, authority_table_addr;
+       union ald ald;
+       int eax, rc;
+       u8 authority_table;
+
+       if (ar >= NUM_ACRS)
+               return -EINVAL;
+
+       save_access_regs(vcpu->run->s.regs.acrs);
+       alet.val = vcpu->run->s.regs.acrs[ar];
+
+       if (ar == 0 || alet.val == 0) {
+               asce->val = vcpu->arch.sie_block->gcr[1];
+               return 0;
+       } else if (alet.val == 1) {
+               asce->val = vcpu->arch.sie_block->gcr[7];
+               return 0;
+       }
+
+       if (alet.reserved)
+               return PGM_ALET_SPECIFICATION;
+
+       if (alet.p)
+               ald_addr = vcpu->arch.sie_block->gcr[5];
+       else
+               ald_addr = vcpu->arch.sie_block->gcr[2];
+       ald_addr &= 0x7fffffc0;
+
+       rc = read_guest_real(vcpu, ald_addr + 16, &ald.val, sizeof(union ald));
+       if (rc)
+               return rc;
+
+       if (alet.alen / 8 > ald.all)
+               return PGM_ALEN_TRANSLATION;
+
+       if (0x7fffffff - ald.alo * 128 < alet.alen * 16)
+               return PGM_ADDRESSING;
+
+       rc = read_guest_real(vcpu, ald.alo * 128 + alet.alen * 16, &ale,
+                            sizeof(struct ale));
+       if (rc)
+               return rc;
+
+       if (ale.i == 1)
+               return PGM_ALEN_TRANSLATION;
+       if (ale.alesn != alet.alesn)
+               return PGM_ALE_SEQUENCE;
+
+       rc = read_guest_real(vcpu, ale.asteo * 64, &aste, sizeof(struct aste));
+       if (rc)
+               return rc;
+
+       if (aste.i)
+               return PGM_ASTE_VALIDITY;
+       if (aste.astesn != ale.astesn)
+               return PGM_ASTE_SEQUENCE;
+
+       if (ale.p == 1) {
+               eax = (vcpu->arch.sie_block->gcr[8] >> 16) & 0xffff;
+               if (ale.aleax != eax) {
+                       if (eax / 16 > aste.atl)
+                               return PGM_EXTENDED_AUTHORITY;
+
+                       authority_table_addr = aste.ato * 4 + eax / 4;
+
+                       rc = read_guest_real(vcpu, authority_table_addr,
+                                            &authority_table,
+                                            sizeof(u8));
+                       if (rc)
+                               return rc;
+
+                       if ((authority_table & (0x40 >> ((eax & 3) * 2))) == 0)
+                               return PGM_EXTENDED_AUTHORITY;
+               }
+       }
+
+       if (ale.fo == 1 && write)
+               return PGM_PROTECTION;
+
+       asce->val = aste.asce;
+       return 0;
+}
+
+struct trans_exc_code_bits {
+       unsigned long addr : 52; /* Translation-exception Address */
+       unsigned long fsi  : 2;  /* Access Exception Fetch/Store Indication */
+       unsigned long      : 6;
+       unsigned long b60  : 1;
+       unsigned long b61  : 1;
+       unsigned long as   : 2;  /* ASCE Identifier */
+};
+
+enum {
+       FSI_UNKNOWN = 0, /* Unknown wether fetch or store */
+       FSI_STORE   = 1, /* Exception was due to store operation */
+       FSI_FETCH   = 2  /* Exception was due to fetch operation */
+};
+
+static int get_vcpu_asce(struct kvm_vcpu *vcpu, union asce *asce,
+                        ar_t ar, int write)
 {
+       int rc;
+       psw_t *psw = &vcpu->arch.sie_block->gpsw;
+       struct kvm_s390_pgm_info *pgm = &vcpu->arch.pgm;
+       struct trans_exc_code_bits *tec_bits;
+
+       memset(pgm, 0, sizeof(*pgm));
+       tec_bits = (struct trans_exc_code_bits *)&pgm->trans_exc_code;
+       tec_bits->fsi = write ? FSI_STORE : FSI_FETCH;
+       tec_bits->as = psw_bits(*psw).as;
+
+       if (!psw_bits(*psw).t) {
+               asce->val = 0;
+               asce->r = 1;
+               return 0;
+       }
+
        switch (psw_bits(vcpu->arch.sie_block->gpsw).as) {
        case PSW_AS_PRIMARY:
-               return vcpu->arch.sie_block->gcr[1];
+               asce->val = vcpu->arch.sie_block->gcr[1];
+               return 0;
        case PSW_AS_SECONDARY:
-               return vcpu->arch.sie_block->gcr[7];
+               asce->val = vcpu->arch.sie_block->gcr[7];
+               return 0;
        case PSW_AS_HOME:
-               return vcpu->arch.sie_block->gcr[13];
+               asce->val = vcpu->arch.sie_block->gcr[13];
+               return 0;
+       case PSW_AS_ACCREG:
+               rc = ar_translation(vcpu, asce, ar, write);
+               switch (rc) {
+               case PGM_ALEN_TRANSLATION:
+               case PGM_ALE_SEQUENCE:
+               case PGM_ASTE_VALIDITY:
+               case PGM_ASTE_SEQUENCE:
+               case PGM_EXTENDED_AUTHORITY:
+                       vcpu->arch.pgm.exc_access_id = ar;
+                       break;
+               case PGM_PROTECTION:
+                       tec_bits->b60 = 1;
+                       tec_bits->b61 = 1;
+                       break;
+               }
+               if (rc > 0)
+                       pgm->code = rc;
+               return rc;
        }
        return 0;
 }
@@ -330,10 +521,11 @@ static int deref_table(struct kvm *kvm, unsigned long gpa, unsigned long *val)
  * @vcpu: virtual cpu
  * @gva: guest virtual address
  * @gpa: points to where guest physical (absolute) address should be stored
+ * @asce: effective asce
  * @write: indicates if access is a write access
  *
  * Translate a guest virtual address into a guest absolute address by means
- * of dynamic address translation as specified by the architecuture.
+ * of dynamic address translation as specified by the architecture.
  * If the resulting absolute address is not available in the configuration
  * an addressing exception is indicated and @gpa will not be changed.
  *
@@ -345,7 +537,8 @@ static int deref_table(struct kvm *kvm, unsigned long gpa, unsigned long *val)
  *           by the architecture
  */
 static unsigned long guest_translate(struct kvm_vcpu *vcpu, unsigned long gva,
-                                    unsigned long *gpa, int write)
+                                    unsigned long *gpa, const union asce asce,
+                                    int write)
 {
        union vaddress vaddr = {.addr = gva};
        union raddress raddr = {.addr = gva};
@@ -354,12 +547,10 @@ static unsigned long guest_translate(struct kvm_vcpu *vcpu, unsigned long gva,
        union ctlreg0 ctlreg0;
        unsigned long ptr;
        int edat1, edat2;
-       union asce asce;
 
        ctlreg0.val = vcpu->arch.sie_block->gcr[0];
        edat1 = ctlreg0.edat && test_kvm_facility(vcpu->kvm, 8);
        edat2 = edat1 && test_kvm_facility(vcpu->kvm, 78);
-       asce.val = get_vcpu_asce(vcpu);
        if (asce.r)
                goto real_address;
        ptr = asce.origin * 4096;
@@ -506,48 +697,30 @@ static inline int is_low_address(unsigned long ga)
        return (ga & ~0x11fful) == 0;
 }
 
-static int low_address_protection_enabled(struct kvm_vcpu *vcpu)
+static int low_address_protection_enabled(struct kvm_vcpu *vcpu,
+                                         const union asce asce)
 {
        union ctlreg0 ctlreg0 = {.val = vcpu->arch.sie_block->gcr[0]};
        psw_t *psw = &vcpu->arch.sie_block->gpsw;
-       union asce asce;
 
        if (!ctlreg0.lap)
                return 0;
-       asce.val = get_vcpu_asce(vcpu);
        if (psw_bits(*psw).t && asce.p)
                return 0;
        return 1;
 }
 
-struct trans_exc_code_bits {
-       unsigned long addr : 52; /* Translation-exception Address */
-       unsigned long fsi  : 2;  /* Access Exception Fetch/Store Indication */
-       unsigned long      : 7;
-       unsigned long b61  : 1;
-       unsigned long as   : 2;  /* ASCE Identifier */
-};
-
-enum {
-       FSI_UNKNOWN = 0, /* Unknown wether fetch or store */
-       FSI_STORE   = 1, /* Exception was due to store operation */
-       FSI_FETCH   = 2  /* Exception was due to fetch operation */
-};
-
 static int guest_page_range(struct kvm_vcpu *vcpu, unsigned long ga,
                            unsigned long *pages, unsigned long nr_pages,
-                           int write)
+                           const union asce asce, int write)
 {
        struct kvm_s390_pgm_info *pgm = &vcpu->arch.pgm;
        psw_t *psw = &vcpu->arch.sie_block->gpsw;
        struct trans_exc_code_bits *tec_bits;
        int lap_enabled, rc;
 
-       memset(pgm, 0, sizeof(*pgm));
        tec_bits = (struct trans_exc_code_bits *)&pgm->trans_exc_code;
-       tec_bits->fsi = write ? FSI_STORE : FSI_FETCH;
-       tec_bits->as = psw_bits(*psw).as;
-       lap_enabled = low_address_protection_enabled(vcpu);
+       lap_enabled = low_address_protection_enabled(vcpu, asce);
        while (nr_pages) {
                ga = kvm_s390_logical_to_effective(vcpu, ga);
                tec_bits->addr = ga >> PAGE_SHIFT;
@@ -557,7 +730,7 @@ static int guest_page_range(struct kvm_vcpu *vcpu, unsigned long ga,
                }
                ga &= PAGE_MASK;
                if (psw_bits(*psw).t) {
-                       rc = guest_translate(vcpu, ga, pages, write);
+                       rc = guest_translate(vcpu, ga, pages, asce, write);
                        if (rc < 0)
                                return rc;
                        if (rc == PGM_PROTECTION)
@@ -578,7 +751,7 @@ static int guest_page_range(struct kvm_vcpu *vcpu, unsigned long ga,
        return 0;
 }
 
-int access_guest(struct kvm_vcpu *vcpu, unsigned long ga, void *data,
+int access_guest(struct kvm_vcpu *vcpu, unsigned long ga, ar_t ar, void *data,
                 unsigned long len, int write)
 {
        psw_t *psw = &vcpu->arch.sie_block->gpsw;
@@ -591,20 +764,19 @@ int access_guest(struct kvm_vcpu *vcpu, unsigned long ga, void *data,
 
        if (!len)
                return 0;
-       /* Access register mode is not supported yet. */
-       if (psw_bits(*psw).t && psw_bits(*psw).as == PSW_AS_ACCREG)
-               return -EOPNOTSUPP;
+       rc = get_vcpu_asce(vcpu, &asce, ar, write);
+       if (rc)
+               return rc;
        nr_pages = (((ga & ~PAGE_MASK) + len - 1) >> PAGE_SHIFT) + 1;
        pages = pages_array;
        if (nr_pages > ARRAY_SIZE(pages_array))
                pages = vmalloc(nr_pages * sizeof(unsigned long));
        if (!pages)
                return -ENOMEM;
-       asce.val = get_vcpu_asce(vcpu);
        need_ipte_lock = psw_bits(*psw).t && !asce.r;
        if (need_ipte_lock)
                ipte_lock(vcpu);
-       rc = guest_page_range(vcpu, ga, pages, nr_pages, write);
+       rc = guest_page_range(vcpu, ga, pages, nr_pages, asce, write);
        for (idx = 0; idx < nr_pages && !rc; idx++) {
                gpa = *(pages + idx) + (ga & ~PAGE_MASK);
                _len = min(PAGE_SIZE - (gpa & ~PAGE_MASK), len);
@@ -652,7 +824,7 @@ int access_guest_real(struct kvm_vcpu *vcpu, unsigned long gra,
  * Note: The IPTE lock is not taken during this function, so the caller
  * has to take care of this.
  */
-int guest_translate_address(struct kvm_vcpu *vcpu, unsigned long gva,
+int guest_translate_address(struct kvm_vcpu *vcpu, unsigned long gva, ar_t ar,
                            unsigned long *gpa, int write)
 {
        struct kvm_s390_pgm_info *pgm = &vcpu->arch.pgm;
@@ -661,26 +833,21 @@ int guest_translate_address(struct kvm_vcpu *vcpu, unsigned long gva,
        union asce asce;
        int rc;
 
-       /* Access register mode is not supported yet. */
-       if (psw_bits(*psw).t && psw_bits(*psw).as == PSW_AS_ACCREG)
-               return -EOPNOTSUPP;
-
        gva = kvm_s390_logical_to_effective(vcpu, gva);
-       memset(pgm, 0, sizeof(*pgm));
        tec = (struct trans_exc_code_bits *)&pgm->trans_exc_code;
-       tec->as = psw_bits(*psw).as;
-       tec->fsi = write ? FSI_STORE : FSI_FETCH;
+       rc = get_vcpu_asce(vcpu, &asce, ar, write);
        tec->addr = gva >> PAGE_SHIFT;
-       if (is_low_address(gva) && low_address_protection_enabled(vcpu)) {
+       if (rc)
+               return rc;
+       if (is_low_address(gva) && low_address_protection_enabled(vcpu, asce)) {
                if (write) {
                        rc = pgm->code = PGM_PROTECTION;
                        return rc;
                }
        }
 
-       asce.val = get_vcpu_asce(vcpu);
        if (psw_bits(*psw).t && !asce.r) {      /* Use DAT? */
-               rc = guest_translate(vcpu, gva, gpa, write);
+               rc = guest_translate(vcpu, gva, gpa, asce, write);
                if (rc > 0) {
                        if (rc == PGM_PROTECTION)
                                tec->b61 = 1;
@@ -697,28 +864,51 @@ int guest_translate_address(struct kvm_vcpu *vcpu, unsigned long gva,
 }
 
 /**
- * kvm_s390_check_low_addr_protection - check for low-address protection
- * @ga: Guest address
+ * check_gva_range - test a range of guest virtual addresses for accessibility
+ */
+int check_gva_range(struct kvm_vcpu *vcpu, unsigned long gva, ar_t ar,
+                   unsigned long length, int is_write)
+{
+       unsigned long gpa;
+       unsigned long currlen;
+       int rc = 0;
+
+       ipte_lock(vcpu);
+       while (length > 0 && !rc) {
+               currlen = min(length, PAGE_SIZE - (gva % PAGE_SIZE));
+               rc = guest_translate_address(vcpu, gva, ar, &gpa, is_write);
+               gva += currlen;
+               length -= currlen;
+       }
+       ipte_unlock(vcpu);
+
+       return rc;
+}
+
+/**
+ * kvm_s390_check_low_addr_prot_real - check for low-address protection
+ * @gra: Guest real address
  *
  * Checks whether an address is subject to low-address protection and set
  * up vcpu->arch.pgm accordingly if necessary.
  *
  * Return: 0 if no protection exception, or PGM_PROTECTION if protected.
  */
-int kvm_s390_check_low_addr_protection(struct kvm_vcpu *vcpu, unsigned long ga)
+int kvm_s390_check_low_addr_prot_real(struct kvm_vcpu *vcpu, unsigned long gra)
 {
        struct kvm_s390_pgm_info *pgm = &vcpu->arch.pgm;
        psw_t *psw = &vcpu->arch.sie_block->gpsw;
        struct trans_exc_code_bits *tec_bits;
+       union ctlreg0 ctlreg0 = {.val = vcpu->arch.sie_block->gcr[0]};
 
-       if (!is_low_address(ga) || !low_address_protection_enabled(vcpu))
+       if (!ctlreg0.lap || !is_low_address(gra))
                return 0;
 
        memset(pgm, 0, sizeof(*pgm));
        tec_bits = (struct trans_exc_code_bits *)&pgm->trans_exc_code;
        tec_bits->fsi = FSI_STORE;
        tec_bits->as = psw_bits(*psw).as;
-       tec_bits->addr = ga >> PAGE_SHIFT;
+       tec_bits->addr = gra >> PAGE_SHIFT;
        pgm->code = PGM_PROTECTION;
 
        return pgm->code;
index 0149cf15058ab9e8d12918192353884ac4ce8f4f..ef03726cc6611acd1e52fb6970e2e802a1730cbd 100644 (file)
@@ -156,9 +156,11 @@ int read_guest_lc(struct kvm_vcpu *vcpu, unsigned long gra, void *data,
 }
 
 int guest_translate_address(struct kvm_vcpu *vcpu, unsigned long gva,
-                           unsigned long *gpa, int write);
+                           ar_t ar, unsigned long *gpa, int write);
+int check_gva_range(struct kvm_vcpu *vcpu, unsigned long gva, ar_t ar,
+                   unsigned long length, int is_write);
 
-int access_guest(struct kvm_vcpu *vcpu, unsigned long ga, void *data,
+int access_guest(struct kvm_vcpu *vcpu, unsigned long ga, ar_t ar, void *data,
                 unsigned long len, int write);
 
 int access_guest_real(struct kvm_vcpu *vcpu, unsigned long gra,
@@ -168,6 +170,7 @@ int access_guest_real(struct kvm_vcpu *vcpu, unsigned long gra,
  * write_guest - copy data from kernel space to guest space
  * @vcpu: virtual cpu
  * @ga: guest address
+ * @ar: access register
  * @data: source address in kernel space
  * @len: number of bytes to copy
  *
@@ -176,8 +179,7 @@ int access_guest_real(struct kvm_vcpu *vcpu, unsigned long gra,
  * If DAT is off data will be copied to guest real or absolute memory.
  * If DAT is on data will be copied to the address space as specified by
  * the address space bits of the PSW:
- * Primary, secondory or home space (access register mode is currently not
- * implemented).
+ * Primary, secondary, home space or access register mode.
  * The addressing mode of the PSW is also inspected, so that address wrap
  * around is taken into account for 24-, 31- and 64-bit addressing mode,
  * if the to be copied data crosses page boundaries in guest address space.
@@ -210,16 +212,17 @@ int access_guest_real(struct kvm_vcpu *vcpu, unsigned long gra,
  *      if data has been changed in guest space in case of an exception.
  */
 static inline __must_check
-int write_guest(struct kvm_vcpu *vcpu, unsigned long ga, void *data,
+int write_guest(struct kvm_vcpu *vcpu, unsigned long ga, ar_t ar, void *data,
                unsigned long len)
 {
-       return access_guest(vcpu, ga, data, len, 1);
+       return access_guest(vcpu, ga, ar, data, len, 1);
 }
 
 /**
  * read_guest - copy data from guest space to kernel space
  * @vcpu: virtual cpu
  * @ga: guest address
+ * @ar: access register
  * @data: destination address in kernel space
  * @len: number of bytes to copy
  *
@@ -229,10 +232,10 @@ int write_guest(struct kvm_vcpu *vcpu, unsigned long ga, void *data,
  * data will be copied from guest space to kernel space.
  */
 static inline __must_check
-int read_guest(struct kvm_vcpu *vcpu, unsigned long ga, void *data,
+int read_guest(struct kvm_vcpu *vcpu, unsigned long ga, ar_t ar, void *data,
               unsigned long len)
 {
-       return access_guest(vcpu, ga, data, len, 0);
+       return access_guest(vcpu, ga, ar, data, len, 0);
 }
 
 /**
@@ -330,6 +333,6 @@ int read_guest_real(struct kvm_vcpu *vcpu, unsigned long gra, void *data,
 void ipte_lock(struct kvm_vcpu *vcpu);
 void ipte_unlock(struct kvm_vcpu *vcpu);
 int ipte_lock_held(struct kvm_vcpu *vcpu);
-int kvm_s390_check_low_addr_protection(struct kvm_vcpu *vcpu, unsigned long ga);
+int kvm_s390_check_low_addr_prot_real(struct kvm_vcpu *vcpu, unsigned long gra);
 
 #endif /* __KVM_S390_GACCESS_H */
index 3e8d4092ce30f10a6bd4b9e486e926889a937b2e..e97b3455d7e6bfbb7b12c606a5224c5efeba37d4 100644 (file)
@@ -191,8 +191,8 @@ static int __import_wp_info(struct kvm_vcpu *vcpu,
        if (!wp_info->old_data)
                return -ENOMEM;
        /* try to backup the original value */
-       ret = read_guest(vcpu, wp_info->phys_addr, wp_info->old_data,
-                        wp_info->len);
+       ret = read_guest_abs(vcpu, wp_info->phys_addr, wp_info->old_data,
+                            wp_info->len);
        if (ret) {
                kfree(wp_info->old_data);
                wp_info->old_data = NULL;
@@ -362,8 +362,8 @@ static struct kvm_hw_wp_info_arch *any_wp_changed(struct kvm_vcpu *vcpu)
                        continue;
 
                /* refetch the wp data and compare it to the old value */
-               if (!read_guest(vcpu, wp_info->phys_addr, temp,
-                               wp_info->len)) {
+               if (!read_guest_abs(vcpu, wp_info->phys_addr, temp,
+                                   wp_info->len)) {
                        if (memcmp(temp, wp_info->old_data, wp_info->len)) {
                                kfree(temp);
                                return wp_info;
index bebd2157edd019448ff772c9ac75a778e7e124f4..9e3779e3e496314a4e3f15823c152867c3a3cadd 100644 (file)
@@ -165,6 +165,7 @@ static void __extract_prog_irq(struct kvm_vcpu *vcpu,
                pgm_info->mon_class_nr = vcpu->arch.sie_block->mcn;
                pgm_info->mon_code = vcpu->arch.sie_block->tecmc;
                break;
+       case PGM_VECTOR_PROCESSING:
        case PGM_DATA:
                pgm_info->data_exc_code = vcpu->arch.sie_block->dxc;
                break;
@@ -319,7 +320,7 @@ static int handle_mvpg_pei(struct kvm_vcpu *vcpu)
 
        /* Make sure that the source is paged-in */
        rc = guest_translate_address(vcpu, vcpu->run->s.regs.gprs[reg2],
-                                    &srcaddr, 0);
+                                    reg2, &srcaddr, 0);
        if (rc)
                return kvm_s390_inject_prog_cond(vcpu, rc);
        rc = kvm_arch_fault_in_page(vcpu, srcaddr, 0);
@@ -328,7 +329,7 @@ static int handle_mvpg_pei(struct kvm_vcpu *vcpu)
 
        /* Make sure that the destination is paged-in */
        rc = guest_translate_address(vcpu, vcpu->run->s.regs.gprs[reg1],
-                                    &dstaddr, 1);
+                                    reg1, &dstaddr, 1);
        if (rc)
                return kvm_s390_inject_prog_cond(vcpu, rc);
        rc = kvm_arch_fault_in_page(vcpu, dstaddr, 1);
index 073b5f387d1dd3484186dd69dcfc5aae63d90b21..9de47265ef73da07ffd7ef37337bf2e44e59bd46 100644 (file)
@@ -1,7 +1,7 @@
 /*
  * handling kvm guest interrupts
  *
- * Copyright IBM Corp. 2008,2014
+ * Copyright IBM Corp. 2008, 2015
  *
  * This program is free software; you can redistribute it and/or modify
  * it under the terms of the GNU General Public License (version 2 only)
 #include <linux/signal.h>
 #include <linux/slab.h>
 #include <linux/bitmap.h>
+#include <linux/vmalloc.h>
 #include <asm/asm-offsets.h>
+#include <asm/dis.h>
 #include <asm/uaccess.h>
 #include <asm/sclp.h>
+#include <asm/isc.h>
 #include "kvm-s390.h"
 #include "gaccess.h"
 #include "trace-s390.h"
 #define PFAULT_DONE 0x0680
 #define VIRTIO_PARAM 0x0d00
 
-static int is_ioint(u64 type)
-{
-       return ((type & 0xfffe0000u) != 0xfffe0000u);
-}
-
 int psw_extint_disabled(struct kvm_vcpu *vcpu)
 {
        return !(vcpu->arch.sie_block->gpsw.mask & PSW_MASK_EXT);
@@ -72,70 +70,45 @@ static int ckc_interrupts_enabled(struct kvm_vcpu *vcpu)
        return 1;
 }
 
-static u64 int_word_to_isc_bits(u32 int_word)
+static int ckc_irq_pending(struct kvm_vcpu *vcpu)
+{
+       if (!(vcpu->arch.sie_block->ckc <
+             get_tod_clock_fast() + vcpu->arch.sie_block->epoch))
+               return 0;
+       return ckc_interrupts_enabled(vcpu);
+}
+
+static int cpu_timer_interrupts_enabled(struct kvm_vcpu *vcpu)
+{
+       return !psw_extint_disabled(vcpu) &&
+              (vcpu->arch.sie_block->gcr[0] & 0x400ul);
+}
+
+static int cpu_timer_irq_pending(struct kvm_vcpu *vcpu)
+{
+       return (vcpu->arch.sie_block->cputm >> 63) &&
+              cpu_timer_interrupts_enabled(vcpu);
+}
+
+static inline int is_ioirq(unsigned long irq_type)
 {
-       u8 isc = (int_word & 0x38000000) >> 27;
+       return ((irq_type >= IRQ_PEND_IO_ISC_0) &&
+               (irq_type <= IRQ_PEND_IO_ISC_7));
+}
 
+static uint64_t isc_to_isc_bits(int isc)
+{
        return (0x80 >> isc) << 24;
 }
 
-static int __must_check __interrupt_is_deliverable(struct kvm_vcpu *vcpu,
-                                     struct kvm_s390_interrupt_info *inti)
+static inline u8 int_word_to_isc(u32 int_word)
 {
-       switch (inti->type) {
-       case KVM_S390_INT_EXTERNAL_CALL:
-               if (psw_extint_disabled(vcpu))
-                       return 0;
-               if (vcpu->arch.sie_block->gcr[0] & 0x2000ul)
-                       return 1;
-               return 0;
-       case KVM_S390_INT_EMERGENCY:
-               if (psw_extint_disabled(vcpu))
-                       return 0;
-               if (vcpu->arch.sie_block->gcr[0] & 0x4000ul)
-                       return 1;
-               return 0;
-       case KVM_S390_INT_CLOCK_COMP:
-               return ckc_interrupts_enabled(vcpu);
-       case KVM_S390_INT_CPU_TIMER:
-               if (psw_extint_disabled(vcpu))
-                       return 0;
-               if (vcpu->arch.sie_block->gcr[0] & 0x400ul)
-                       return 1;
-               return 0;
-       case KVM_S390_INT_SERVICE:
-       case KVM_S390_INT_PFAULT_INIT:
-       case KVM_S390_INT_PFAULT_DONE:
-       case KVM_S390_INT_VIRTIO:
-               if (psw_extint_disabled(vcpu))
-                       return 0;
-               if (vcpu->arch.sie_block->gcr[0] & 0x200ul)
-                       return 1;
-               return 0;
-       case KVM_S390_PROGRAM_INT:
-       case KVM_S390_SIGP_STOP:
-       case KVM_S390_SIGP_SET_PREFIX:
-       case KVM_S390_RESTART:
-               return 1;
-       case KVM_S390_MCHK:
-               if (psw_mchk_disabled(vcpu))
-                       return 0;
-               if (vcpu->arch.sie_block->gcr[14] & inti->mchk.cr14)
-                       return 1;
-               return 0;
-       case KVM_S390_INT_IO_MIN...KVM_S390_INT_IO_MAX:
-               if (psw_ioint_disabled(vcpu))
-                       return 0;
-               if (vcpu->arch.sie_block->gcr[6] &
-                   int_word_to_isc_bits(inti->io.io_int_word))
-                       return 1;
-               return 0;
-       default:
-               printk(KERN_WARNING "illegal interrupt type %llx\n",
-                      inti->type);
-               BUG();
-       }
-       return 0;
+       return (int_word & 0x38000000) >> 27;
+}
+
+static inline unsigned long pending_floating_irqs(struct kvm_vcpu *vcpu)
+{
+       return vcpu->kvm->arch.float_int.pending_irqs;
 }
 
 static inline unsigned long pending_local_irqs(struct kvm_vcpu *vcpu)
@@ -143,12 +116,31 @@ static inline unsigned long pending_local_irqs(struct kvm_vcpu *vcpu)
        return vcpu->arch.local_int.pending_irqs;
 }
 
-static unsigned long deliverable_local_irqs(struct kvm_vcpu *vcpu)
+static unsigned long disable_iscs(struct kvm_vcpu *vcpu,
+                                  unsigned long active_mask)
+{
+       int i;
+
+       for (i = 0; i <= MAX_ISC; i++)
+               if (!(vcpu->arch.sie_block->gcr[6] & isc_to_isc_bits(i)))
+                       active_mask &= ~(1UL << (IRQ_PEND_IO_ISC_0 + i));
+
+       return active_mask;
+}
+
+static unsigned long deliverable_irqs(struct kvm_vcpu *vcpu)
 {
-       unsigned long active_mask = pending_local_irqs(vcpu);
+       unsigned long active_mask;
+
+       active_mask = pending_local_irqs(vcpu);
+       active_mask |= pending_floating_irqs(vcpu);
 
        if (psw_extint_disabled(vcpu))
                active_mask &= ~IRQ_PEND_EXT_MASK;
+       if (psw_ioint_disabled(vcpu))
+               active_mask &= ~IRQ_PEND_IO_MASK;
+       else
+               active_mask = disable_iscs(vcpu, active_mask);
        if (!(vcpu->arch.sie_block->gcr[0] & 0x2000ul))
                __clear_bit(IRQ_PEND_EXT_EXTERNAL, &active_mask);
        if (!(vcpu->arch.sie_block->gcr[0] & 0x4000ul))
@@ -157,8 +149,13 @@ static unsigned long deliverable_local_irqs(struct kvm_vcpu *vcpu)
                __clear_bit(IRQ_PEND_EXT_CLOCK_COMP, &active_mask);
        if (!(vcpu->arch.sie_block->gcr[0] & 0x400ul))
                __clear_bit(IRQ_PEND_EXT_CPU_TIMER, &active_mask);
+       if (!(vcpu->arch.sie_block->gcr[0] & 0x200ul))
+               __clear_bit(IRQ_PEND_EXT_SERVICE, &active_mask);
        if (psw_mchk_disabled(vcpu))
                active_mask &= ~IRQ_PEND_MCHK_MASK;
+       if (!(vcpu->arch.sie_block->gcr[14] &
+             vcpu->kvm->arch.float_int.mchk.cr14))
+               __clear_bit(IRQ_PEND_MCHK_REP, &active_mask);
 
        /*
         * STOP irqs will never be actively delivered. They are triggered via
@@ -200,6 +197,16 @@ static void __set_cpuflag(struct kvm_vcpu *vcpu, u32 flag)
        atomic_set_mask(flag, &vcpu->arch.sie_block->cpuflags);
 }
 
+static void set_intercept_indicators_io(struct kvm_vcpu *vcpu)
+{
+       if (!(pending_floating_irqs(vcpu) & IRQ_PEND_IO_MASK))
+               return;
+       else if (psw_ioint_disabled(vcpu))
+               __set_cpuflag(vcpu, CPUSTAT_IO_INT);
+       else
+               vcpu->arch.sie_block->lctl |= LCTL_CR6;
+}
+
 static void set_intercept_indicators_ext(struct kvm_vcpu *vcpu)
 {
        if (!(pending_local_irqs(vcpu) & IRQ_PEND_EXT_MASK))
@@ -226,47 +233,17 @@ static void set_intercept_indicators_stop(struct kvm_vcpu *vcpu)
                __set_cpuflag(vcpu, CPUSTAT_STOP_INT);
 }
 
-/* Set interception request for non-deliverable local interrupts */
-static void set_intercept_indicators_local(struct kvm_vcpu *vcpu)
+/* Set interception request for non-deliverable interrupts */
+static void set_intercept_indicators(struct kvm_vcpu *vcpu)
 {
+       set_intercept_indicators_io(vcpu);
        set_intercept_indicators_ext(vcpu);
        set_intercept_indicators_mchk(vcpu);
        set_intercept_indicators_stop(vcpu);
 }
 
-static void __set_intercept_indicator(struct kvm_vcpu *vcpu,
-                                     struct kvm_s390_interrupt_info *inti)
-{
-       switch (inti->type) {
-       case KVM_S390_INT_SERVICE:
-       case KVM_S390_INT_PFAULT_DONE:
-       case KVM_S390_INT_VIRTIO:
-               if (psw_extint_disabled(vcpu))
-                       __set_cpuflag(vcpu, CPUSTAT_EXT_INT);
-               else
-                       vcpu->arch.sie_block->lctl |= LCTL_CR0;
-               break;
-       case KVM_S390_MCHK:
-               if (psw_mchk_disabled(vcpu))
-                       vcpu->arch.sie_block->ictl |= ICTL_LPSW;
-               else
-                       vcpu->arch.sie_block->lctl |= LCTL_CR14;
-               break;
-       case KVM_S390_INT_IO_MIN...KVM_S390_INT_IO_MAX:
-               if (psw_ioint_disabled(vcpu))
-                       __set_cpuflag(vcpu, CPUSTAT_IO_INT);
-               else
-                       vcpu->arch.sie_block->lctl |= LCTL_CR6;
-               break;
-       default:
-               BUG();
-       }
-}
-
 static u16 get_ilc(struct kvm_vcpu *vcpu)
 {
-       const unsigned short table[] = { 2, 4, 4, 6 };
-
        switch (vcpu->arch.sie_block->icptcode) {
        case ICPT_INST:
        case ICPT_INSTPROGI:
@@ -274,7 +251,7 @@ static u16 get_ilc(struct kvm_vcpu *vcpu)
        case ICPT_PARTEXEC:
        case ICPT_IOINST:
                /* last instruction only stored for these icptcodes */
-               return table[vcpu->arch.sie_block->ipa >> 14];
+               return insn_length(vcpu->arch.sie_block->ipa >> 8);
        case ICPT_PROGI:
                return vcpu->arch.sie_block->pgmilc;
        default:
@@ -350,38 +327,72 @@ static int __must_check __deliver_pfault_init(struct kvm_vcpu *vcpu)
 
 static int __must_check __deliver_machine_check(struct kvm_vcpu *vcpu)
 {
+       struct kvm_s390_float_interrupt *fi = &vcpu->kvm->arch.float_int;
        struct kvm_s390_local_interrupt *li = &vcpu->arch.local_int;
-       struct kvm_s390_mchk_info mchk;
-       int rc;
+       struct kvm_s390_mchk_info mchk = {};
+       unsigned long adtl_status_addr;
+       int deliver = 0;
+       int rc = 0;
 
+       spin_lock(&fi->lock);
        spin_lock(&li->lock);
-       mchk = li->irq.mchk;
+       if (test_bit(IRQ_PEND_MCHK_EX, &li->pending_irqs) ||
+           test_bit(IRQ_PEND_MCHK_REP, &li->pending_irqs)) {
+               /*
+                * If there was an exigent machine check pending, then any
+                * repressible machine checks that might have been pending
+                * are indicated along with it, so always clear bits for
+                * repressible and exigent interrupts
+                */
+               mchk = li->irq.mchk;
+               clear_bit(IRQ_PEND_MCHK_EX, &li->pending_irqs);
+               clear_bit(IRQ_PEND_MCHK_REP, &li->pending_irqs);
+               memset(&li->irq.mchk, 0, sizeof(mchk));
+               deliver = 1;
+       }
        /*
-        * If there was an exigent machine check pending, then any repressible
-        * machine checks that might have been pending are indicated along
-        * with it, so always clear both bits
+        * We indicate floating repressible conditions along with
+        * other pending conditions. Channel Report Pending and Channel
+        * Subsystem damage are the only two and and are indicated by
+        * bits in mcic and masked in cr14.
         */
-       clear_bit(IRQ_PEND_MCHK_EX, &li->pending_irqs);
-       clear_bit(IRQ_PEND_MCHK_REP, &li->pending_irqs);
-       memset(&li->irq.mchk, 0, sizeof(mchk));
+       if (test_and_clear_bit(IRQ_PEND_MCHK_REP, &fi->pending_irqs)) {
+               mchk.mcic |= fi->mchk.mcic;
+               mchk.cr14 |= fi->mchk.cr14;
+               memset(&fi->mchk, 0, sizeof(mchk));
+               deliver = 1;
+       }
        spin_unlock(&li->lock);
+       spin_unlock(&fi->lock);
 
-       VCPU_EVENT(vcpu, 4, "interrupt: machine check mcic=%llx",
-                  mchk.mcic);
-       trace_kvm_s390_deliver_interrupt(vcpu->vcpu_id, KVM_S390_MCHK,
-                                        mchk.cr14, mchk.mcic);
-
-       rc  = kvm_s390_vcpu_store_status(vcpu, KVM_S390_STORE_STATUS_PREFIXED);
-       rc |= put_guest_lc(vcpu, mchk.mcic,
-                          (u64 __user *) __LC_MCCK_CODE);
-       rc |= put_guest_lc(vcpu, mchk.failing_storage_address,
-                          (u64 __user *) __LC_MCCK_FAIL_STOR_ADDR);
-       rc |= write_guest_lc(vcpu, __LC_PSW_SAVE_AREA,
-                            &mchk.fixed_logout, sizeof(mchk.fixed_logout));
-       rc |= write_guest_lc(vcpu, __LC_MCK_OLD_PSW,
-                            &vcpu->arch.sie_block->gpsw, sizeof(psw_t));
-       rc |= read_guest_lc(vcpu, __LC_MCK_NEW_PSW,
-                           &vcpu->arch.sie_block->gpsw, sizeof(psw_t));
+       if (deliver) {
+               VCPU_EVENT(vcpu, 4, "interrupt: machine check mcic=%llx",
+                          mchk.mcic);
+               trace_kvm_s390_deliver_interrupt(vcpu->vcpu_id,
+                                                KVM_S390_MCHK,
+                                                mchk.cr14, mchk.mcic);
+
+               rc  = kvm_s390_vcpu_store_status(vcpu,
+                                                KVM_S390_STORE_STATUS_PREFIXED);
+               rc |= read_guest_lc(vcpu, __LC_VX_SAVE_AREA_ADDR,
+                                   &adtl_status_addr,
+                                   sizeof(unsigned long));
+               rc |= kvm_s390_vcpu_store_adtl_status(vcpu,
+                                                     adtl_status_addr);
+               rc |= put_guest_lc(vcpu, mchk.mcic,
+                                  (u64 __user *) __LC_MCCK_CODE);
+               rc |= put_guest_lc(vcpu, mchk.failing_storage_address,
+                                  (u64 __user *) __LC_MCCK_FAIL_STOR_ADDR);
+               rc |= write_guest_lc(vcpu, __LC_PSW_SAVE_AREA,
+                                    &mchk.fixed_logout,
+                                    sizeof(mchk.fixed_logout));
+               rc |= write_guest_lc(vcpu, __LC_MCK_OLD_PSW,
+                                    &vcpu->arch.sie_block->gpsw,
+                                    sizeof(psw_t));
+               rc |= read_guest_lc(vcpu, __LC_MCK_NEW_PSW,
+                                   &vcpu->arch.sie_block->gpsw,
+                                   sizeof(psw_t));
+       }
        return rc ? -EFAULT : 0;
 }
 
@@ -484,7 +495,7 @@ static int __must_check __deliver_prog(struct kvm_vcpu *vcpu)
 {
        struct kvm_s390_local_interrupt *li = &vcpu->arch.local_int;
        struct kvm_s390_pgm_info pgm_info;
-       int rc = 0;
+       int rc = 0, nullifying = false;
        u16 ilc = get_ilc(vcpu);
 
        spin_lock(&li->lock);
@@ -509,6 +520,8 @@ static int __must_check __deliver_prog(struct kvm_vcpu *vcpu)
        case PGM_LX_TRANSLATION:
        case PGM_PRIMARY_AUTHORITY:
        case PGM_SECONDARY_AUTHORITY:
+               nullifying = true;
+               /* fall through */
        case PGM_SPACE_SWITCH:
                rc = put_guest_lc(vcpu, pgm_info.trans_exc_code,
                                  (u64 *)__LC_TRANS_EXC_CODE);
@@ -521,6 +534,7 @@ static int __must_check __deliver_prog(struct kvm_vcpu *vcpu)
        case PGM_EXTENDED_AUTHORITY:
                rc = put_guest_lc(vcpu, pgm_info.exc_access_id,
                                  (u8 *)__LC_EXC_ACCESS_ID);
+               nullifying = true;
                break;
        case PGM_ASCE_TYPE:
        case PGM_PAGE_TRANSLATION:
@@ -534,6 +548,7 @@ static int __must_check __deliver_prog(struct kvm_vcpu *vcpu)
                                   (u8 *)__LC_EXC_ACCESS_ID);
                rc |= put_guest_lc(vcpu, pgm_info.op_access_id,
                                   (u8 *)__LC_OP_ACCESS_ID);
+               nullifying = true;
                break;
        case PGM_MONITOR:
                rc = put_guest_lc(vcpu, pgm_info.mon_class_nr,
@@ -541,6 +556,7 @@ static int __must_check __deliver_prog(struct kvm_vcpu *vcpu)
                rc |= put_guest_lc(vcpu, pgm_info.mon_code,
                                   (u64 *)__LC_MON_CODE);
                break;
+       case PGM_VECTOR_PROCESSING:
        case PGM_DATA:
                rc = put_guest_lc(vcpu, pgm_info.data_exc_code,
                                  (u32 *)__LC_DATA_EXC_CODE);
@@ -551,6 +567,15 @@ static int __must_check __deliver_prog(struct kvm_vcpu *vcpu)
                rc |= put_guest_lc(vcpu, pgm_info.exc_access_id,
                                   (u8 *)__LC_EXC_ACCESS_ID);
                break;
+       case PGM_STACK_FULL:
+       case PGM_STACK_EMPTY:
+       case PGM_STACK_SPECIFICATION:
+       case PGM_STACK_TYPE:
+       case PGM_STACK_OPERATION:
+       case PGM_TRACE_TABEL:
+       case PGM_CRYPTO_OPERATION:
+               nullifying = true;
+               break;
        }
 
        if (pgm_info.code & PGM_PER) {
@@ -564,7 +589,12 @@ static int __must_check __deliver_prog(struct kvm_vcpu *vcpu)
                                   (u8 *) __LC_PER_ACCESS_ID);
        }
 
+       if (nullifying && vcpu->arch.sie_block->icptcode == ICPT_INST)
+               kvm_s390_rewind_psw(vcpu, ilc);
+
        rc |= put_guest_lc(vcpu, ilc, (u16 *) __LC_PGM_ILC);
+       rc |= put_guest_lc(vcpu, vcpu->arch.sie_block->gbea,
+                                (u64 *) __LC_LAST_BREAK);
        rc |= put_guest_lc(vcpu, pgm_info.code,
                           (u16 *)__LC_PGM_INT_CODE);
        rc |= write_guest_lc(vcpu, __LC_PGM_OLD_PSW,
@@ -574,16 +604,27 @@ static int __must_check __deliver_prog(struct kvm_vcpu *vcpu)
        return rc ? -EFAULT : 0;
 }
 
-static int __must_check __deliver_service(struct kvm_vcpu *vcpu,
-                                         struct kvm_s390_interrupt_info *inti)
+static int __must_check __deliver_service(struct kvm_vcpu *vcpu)
 {
-       int rc;
+       struct kvm_s390_float_interrupt *fi = &vcpu->kvm->arch.float_int;
+       struct kvm_s390_ext_info ext;
+       int rc = 0;
+
+       spin_lock(&fi->lock);
+       if (!(test_bit(IRQ_PEND_EXT_SERVICE, &fi->pending_irqs))) {
+               spin_unlock(&fi->lock);
+               return 0;
+       }
+       ext = fi->srv_signal;
+       memset(&fi->srv_signal, 0, sizeof(ext));
+       clear_bit(IRQ_PEND_EXT_SERVICE, &fi->pending_irqs);
+       spin_unlock(&fi->lock);
 
        VCPU_EVENT(vcpu, 4, "interrupt: sclp parm:%x",
-                  inti->ext.ext_params);
+                  ext.ext_params);
        vcpu->stat.deliver_service_signal++;
-       trace_kvm_s390_deliver_interrupt(vcpu->vcpu_id, inti->type,
-                                        inti->ext.ext_params, 0);
+       trace_kvm_s390_deliver_interrupt(vcpu->vcpu_id, KVM_S390_INT_SERVICE,
+                                        ext.ext_params, 0);
 
        rc  = put_guest_lc(vcpu, EXT_IRQ_SERVICE_SIG, (u16 *)__LC_EXT_INT_CODE);
        rc |= put_guest_lc(vcpu, 0, (u16 *)__LC_EXT_CPU_ADDR);
@@ -591,106 +632,146 @@ static int __must_check __deliver_service(struct kvm_vcpu *vcpu,
                             &vcpu->arch.sie_block->gpsw, sizeof(psw_t));
        rc |= read_guest_lc(vcpu, __LC_EXT_NEW_PSW,
                            &vcpu->arch.sie_block->gpsw, sizeof(psw_t));
-       rc |= put_guest_lc(vcpu, inti->ext.ext_params,
+       rc |= put_guest_lc(vcpu, ext.ext_params,
                           (u32 *)__LC_EXT_PARAMS);
+
        return rc ? -EFAULT : 0;
 }
 
-static int __must_check __deliver_pfault_done(struct kvm_vcpu *vcpu,
-                                          struct kvm_s390_interrupt_info *inti)
+static int __must_check __deliver_pfault_done(struct kvm_vcpu *vcpu)
 {
-       int rc;
+       struct kvm_s390_float_interrupt *fi = &vcpu->kvm->arch.float_int;
+       struct kvm_s390_interrupt_info *inti;
+       int rc = 0;
 
-       trace_kvm_s390_deliver_interrupt(vcpu->vcpu_id,
-                                        KVM_S390_INT_PFAULT_DONE, 0,
-                                        inti->ext.ext_params2);
+       spin_lock(&fi->lock);
+       inti = list_first_entry_or_null(&fi->lists[FIRQ_LIST_PFAULT],
+                                       struct kvm_s390_interrupt_info,
+                                       list);
+       if (inti) {
+               trace_kvm_s390_deliver_interrupt(vcpu->vcpu_id,
+                               KVM_S390_INT_PFAULT_DONE, 0,
+                               inti->ext.ext_params2);
+               list_del(&inti->list);
+               fi->counters[FIRQ_CNTR_PFAULT] -= 1;
+       }
+       if (list_empty(&fi->lists[FIRQ_LIST_PFAULT]))
+               clear_bit(IRQ_PEND_PFAULT_DONE, &fi->pending_irqs);
+       spin_unlock(&fi->lock);
 
-       rc  = put_guest_lc(vcpu, EXT_IRQ_CP_SERVICE, (u16 *)__LC_EXT_INT_CODE);
-       rc |= put_guest_lc(vcpu, PFAULT_DONE, (u16 *)__LC_EXT_CPU_ADDR);
-       rc |= write_guest_lc(vcpu, __LC_EXT_OLD_PSW,
-                            &vcpu->arch.sie_block->gpsw, sizeof(psw_t));
-       rc |= read_guest_lc(vcpu, __LC_EXT_NEW_PSW,
-                           &vcpu->arch.sie_block->gpsw, sizeof(psw_t));
-       rc |= put_guest_lc(vcpu, inti->ext.ext_params2,
-                          (u64 *)__LC_EXT_PARAMS2);
+       if (inti) {
+               rc  = put_guest_lc(vcpu, EXT_IRQ_CP_SERVICE,
+                               (u16 *)__LC_EXT_INT_CODE);
+               rc |= put_guest_lc(vcpu, PFAULT_DONE,
+                               (u16 *)__LC_EXT_CPU_ADDR);
+               rc |= write_guest_lc(vcpu, __LC_EXT_OLD_PSW,
+                               &vcpu->arch.sie_block->gpsw,
+                               sizeof(psw_t));
+               rc |= read_guest_lc(vcpu, __LC_EXT_NEW_PSW,
+                               &vcpu->arch.sie_block->gpsw,
+                               sizeof(psw_t));
+               rc |= put_guest_lc(vcpu, inti->ext.ext_params2,
+                               (u64 *)__LC_EXT_PARAMS2);
+               kfree(inti);
+       }
        return rc ? -EFAULT : 0;
 }
 
-static int __must_check __deliver_virtio(struct kvm_vcpu *vcpu,
-                                        struct kvm_s390_interrupt_info *inti)
+static int __must_check __deliver_virtio(struct kvm_vcpu *vcpu)
 {
-       int rc;
+       struct kvm_s390_float_interrupt *fi = &vcpu->kvm->arch.float_int;
+       struct kvm_s390_interrupt_info *inti;
+       int rc = 0;
 
-       VCPU_EVENT(vcpu, 4, "interrupt: virtio parm:%x,parm64:%llx",
-                  inti->ext.ext_params, inti->ext.ext_params2);
-       vcpu->stat.deliver_virtio_interrupt++;
-       trace_kvm_s390_deliver_interrupt(vcpu->vcpu_id, inti->type,
-                                        inti->ext.ext_params,
-                                        inti->ext.ext_params2);
+       spin_lock(&fi->lock);
+       inti = list_first_entry_or_null(&fi->lists[FIRQ_LIST_VIRTIO],
+                                       struct kvm_s390_interrupt_info,
+                                       list);
+       if (inti) {
+               VCPU_EVENT(vcpu, 4,
+                          "interrupt: virtio parm:%x,parm64:%llx",
+                          inti->ext.ext_params, inti->ext.ext_params2);
+               vcpu->stat.deliver_virtio_interrupt++;
+               trace_kvm_s390_deliver_interrupt(vcpu->vcpu_id,
+                               inti->type,
+                               inti->ext.ext_params,
+                               inti->ext.ext_params2);
+               list_del(&inti->list);
+               fi->counters[FIRQ_CNTR_VIRTIO] -= 1;
+       }
+       if (list_empty(&fi->lists[FIRQ_LIST_VIRTIO]))
+               clear_bit(IRQ_PEND_VIRTIO, &fi->pending_irqs);
+       spin_unlock(&fi->lock);
 
-       rc  = put_guest_lc(vcpu, EXT_IRQ_CP_SERVICE, (u16 *)__LC_EXT_INT_CODE);
-       rc |= put_guest_lc(vcpu, VIRTIO_PARAM, (u16 *)__LC_EXT_CPU_ADDR);
-       rc |= write_guest_lc(vcpu, __LC_EXT_OLD_PSW,
-                            &vcpu->arch.sie_block->gpsw, sizeof(psw_t));
-       rc |= read_guest_lc(vcpu, __LC_EXT_NEW_PSW,
-                           &vcpu->arch.sie_block->gpsw, sizeof(psw_t));
-       rc |= put_guest_lc(vcpu, inti->ext.ext_params,
-                          (u32 *)__LC_EXT_PARAMS);
-       rc |= put_guest_lc(vcpu, inti->ext.ext_params2,
-                          (u64 *)__LC_EXT_PARAMS2);
+       if (inti) {
+               rc  = put_guest_lc(vcpu, EXT_IRQ_CP_SERVICE,
+                               (u16 *)__LC_EXT_INT_CODE);
+               rc |= put_guest_lc(vcpu, VIRTIO_PARAM,
+                               (u16 *)__LC_EXT_CPU_ADDR);
+               rc |= write_guest_lc(vcpu, __LC_EXT_OLD_PSW,
+                               &vcpu->arch.sie_block->gpsw,
+                               sizeof(psw_t));
+               rc |= read_guest_lc(vcpu, __LC_EXT_NEW_PSW,
+                               &vcpu->arch.sie_block->gpsw,
+                               sizeof(psw_t));
+               rc |= put_guest_lc(vcpu, inti->ext.ext_params,
+                               (u32 *)__LC_EXT_PARAMS);
+               rc |= put_guest_lc(vcpu, inti->ext.ext_params2,
+                               (u64 *)__LC_EXT_PARAMS2);
+               kfree(inti);
+       }
        return rc ? -EFAULT : 0;
 }
 
 static int __must_check __deliver_io(struct kvm_vcpu *vcpu,
-                                    struct kvm_s390_interrupt_info *inti)
+                                    unsigned long irq_type)
 {
-       int rc;
+       struct list_head *isc_list;
+       struct kvm_s390_float_interrupt *fi;
+       struct kvm_s390_interrupt_info *inti = NULL;
+       int rc = 0;
 
-       VCPU_EVENT(vcpu, 4, "interrupt: I/O %llx", inti->type);
-       vcpu->stat.deliver_io_int++;
-       trace_kvm_s390_deliver_interrupt(vcpu->vcpu_id, inti->type,
-                                        ((__u32)inti->io.subchannel_id << 16) |
-                                               inti->io.subchannel_nr,
-                                        ((__u64)inti->io.io_int_parm << 32) |
-                                               inti->io.io_int_word);
-
-       rc  = put_guest_lc(vcpu, inti->io.subchannel_id,
-                          (u16 *)__LC_SUBCHANNEL_ID);
-       rc |= put_guest_lc(vcpu, inti->io.subchannel_nr,
-                          (u16 *)__LC_SUBCHANNEL_NR);
-       rc |= put_guest_lc(vcpu, inti->io.io_int_parm,
-                          (u32 *)__LC_IO_INT_PARM);
-       rc |= put_guest_lc(vcpu, inti->io.io_int_word,
-                          (u32 *)__LC_IO_INT_WORD);
-       rc |= write_guest_lc(vcpu, __LC_IO_OLD_PSW,
-                            &vcpu->arch.sie_block->gpsw, sizeof(psw_t));
-       rc |= read_guest_lc(vcpu, __LC_IO_NEW_PSW,
-                           &vcpu->arch.sie_block->gpsw, sizeof(psw_t));
-       return rc ? -EFAULT : 0;
-}
+       fi = &vcpu->kvm->arch.float_int;
 
-static int __must_check __deliver_mchk_floating(struct kvm_vcpu *vcpu,
-                                          struct kvm_s390_interrupt_info *inti)
-{
-       struct kvm_s390_mchk_info *mchk = &inti->mchk;
-       int rc;
+       spin_lock(&fi->lock);
+       isc_list = &fi->lists[irq_type - IRQ_PEND_IO_ISC_0];
+       inti = list_first_entry_or_null(isc_list,
+                                       struct kvm_s390_interrupt_info,
+                                       list);
+       if (inti) {
+               VCPU_EVENT(vcpu, 4, "interrupt: I/O %llx", inti->type);
+               vcpu->stat.deliver_io_int++;
+               trace_kvm_s390_deliver_interrupt(vcpu->vcpu_id,
+                               inti->type,
+                               ((__u32)inti->io.subchannel_id << 16) |
+                               inti->io.subchannel_nr,
+                               ((__u64)inti->io.io_int_parm << 32) |
+                               inti->io.io_int_word);
+               list_del(&inti->list);
+               fi->counters[FIRQ_CNTR_IO] -= 1;
+       }
+       if (list_empty(isc_list))
+               clear_bit(irq_type, &fi->pending_irqs);
+       spin_unlock(&fi->lock);
+
+       if (inti) {
+               rc  = put_guest_lc(vcpu, inti->io.subchannel_id,
+                               (u16 *)__LC_SUBCHANNEL_ID);
+               rc |= put_guest_lc(vcpu, inti->io.subchannel_nr,
+                               (u16 *)__LC_SUBCHANNEL_NR);
+               rc |= put_guest_lc(vcpu, inti->io.io_int_parm,
+                               (u32 *)__LC_IO_INT_PARM);
+               rc |= put_guest_lc(vcpu, inti->io.io_int_word,
+                               (u32 *)__LC_IO_INT_WORD);
+               rc |= write_guest_lc(vcpu, __LC_IO_OLD_PSW,
+                               &vcpu->arch.sie_block->gpsw,
+                               sizeof(psw_t));
+               rc |= read_guest_lc(vcpu, __LC_IO_NEW_PSW,
+                               &vcpu->arch.sie_block->gpsw,
+                               sizeof(psw_t));
+               kfree(inti);
+       }
 
-       VCPU_EVENT(vcpu, 4, "interrupt: machine check mcic=%llx",
-                  mchk->mcic);
-       trace_kvm_s390_deliver_interrupt(vcpu->vcpu_id, KVM_S390_MCHK,
-                                        mchk->cr14, mchk->mcic);
-
-       rc  = kvm_s390_vcpu_store_status(vcpu, KVM_S390_STORE_STATUS_PREFIXED);
-       rc |= put_guest_lc(vcpu, mchk->mcic,
-                       (u64 __user *) __LC_MCCK_CODE);
-       rc |= put_guest_lc(vcpu, mchk->failing_storage_address,
-                       (u64 __user *) __LC_MCCK_FAIL_STOR_ADDR);
-       rc |= write_guest_lc(vcpu, __LC_PSW_SAVE_AREA,
-                            &mchk->fixed_logout, sizeof(mchk->fixed_logout));
-       rc |= write_guest_lc(vcpu, __LC_MCK_OLD_PSW,
-                            &vcpu->arch.sie_block->gpsw, sizeof(psw_t));
-       rc |= read_guest_lc(vcpu, __LC_MCK_NEW_PSW,
-                           &vcpu->arch.sie_block->gpsw, sizeof(psw_t));
        return rc ? -EFAULT : 0;
 }
 
@@ -698,6 +779,7 @@ typedef int (*deliver_irq_t)(struct kvm_vcpu *vcpu);
 
 static const deliver_irq_t deliver_irq_funcs[] = {
        [IRQ_PEND_MCHK_EX]        = __deliver_machine_check,
+       [IRQ_PEND_MCHK_REP]       = __deliver_machine_check,
        [IRQ_PEND_PROG]           = __deliver_prog,
        [IRQ_PEND_EXT_EMERGENCY]  = __deliver_emergency_signal,
        [IRQ_PEND_EXT_EXTERNAL]   = __deliver_external_call,
@@ -706,36 +788,11 @@ static const deliver_irq_t deliver_irq_funcs[] = {
        [IRQ_PEND_RESTART]        = __deliver_restart,
        [IRQ_PEND_SET_PREFIX]     = __deliver_set_prefix,
        [IRQ_PEND_PFAULT_INIT]    = __deliver_pfault_init,
+       [IRQ_PEND_EXT_SERVICE]    = __deliver_service,
+       [IRQ_PEND_PFAULT_DONE]    = __deliver_pfault_done,
+       [IRQ_PEND_VIRTIO]         = __deliver_virtio,
 };
 
-static int __must_check __deliver_floating_interrupt(struct kvm_vcpu *vcpu,
-                                          struct kvm_s390_interrupt_info *inti)
-{
-       int rc;
-
-       switch (inti->type) {
-       case KVM_S390_INT_SERVICE:
-               rc = __deliver_service(vcpu, inti);
-               break;
-       case KVM_S390_INT_PFAULT_DONE:
-               rc = __deliver_pfault_done(vcpu, inti);
-               break;
-       case KVM_S390_INT_VIRTIO:
-               rc = __deliver_virtio(vcpu, inti);
-               break;
-       case KVM_S390_MCHK:
-               rc = __deliver_mchk_floating(vcpu, inti);
-               break;
-       case KVM_S390_INT_IO_MIN...KVM_S390_INT_IO_MAX:
-               rc = __deliver_io(vcpu, inti);
-               break;
-       default:
-               BUG();
-       }
-
-       return rc;
-}
-
 /* Check whether an external call is pending (deliverable or not) */
 int kvm_s390_ext_call_pending(struct kvm_vcpu *vcpu)
 {
@@ -751,21 +808,9 @@ int kvm_s390_ext_call_pending(struct kvm_vcpu *vcpu)
 
 int kvm_s390_vcpu_has_irq(struct kvm_vcpu *vcpu, int exclude_stop)
 {
-       struct kvm_s390_float_interrupt *fi = vcpu->arch.local_int.float_int;
-       struct kvm_s390_interrupt_info  *inti;
        int rc;
 
-       rc = !!deliverable_local_irqs(vcpu);
-
-       if ((!rc) && atomic_read(&fi->active)) {
-               spin_lock(&fi->lock);
-               list_for_each_entry(inti, &fi->list, list)
-                       if (__interrupt_is_deliverable(vcpu, inti)) {
-                               rc = 1;
-                               break;
-                       }
-               spin_unlock(&fi->lock);
-       }
+       rc = !!deliverable_irqs(vcpu);
 
        if (!rc && kvm_cpu_has_pending_timer(vcpu))
                rc = 1;
@@ -784,12 +829,7 @@ int kvm_s390_vcpu_has_irq(struct kvm_vcpu *vcpu, int exclude_stop)
 
 int kvm_cpu_has_pending_timer(struct kvm_vcpu *vcpu)
 {
-       if (!(vcpu->arch.sie_block->ckc <
-             get_tod_clock_fast() + vcpu->arch.sie_block->epoch))
-               return 0;
-       if (!ckc_interrupts_enabled(vcpu))
-               return 0;
-       return 1;
+       return ckc_irq_pending(vcpu) || cpu_timer_irq_pending(vcpu);
 }
 
 int kvm_s390_handle_wait(struct kvm_vcpu *vcpu)
@@ -884,60 +924,45 @@ void kvm_s390_clear_local_irqs(struct kvm_vcpu *vcpu)
 int __must_check kvm_s390_deliver_pending_interrupts(struct kvm_vcpu *vcpu)
 {
        struct kvm_s390_local_interrupt *li = &vcpu->arch.local_int;
-       struct kvm_s390_float_interrupt *fi = vcpu->arch.local_int.float_int;
-       struct kvm_s390_interrupt_info  *n, *inti = NULL;
        deliver_irq_t func;
-       int deliver;
        int rc = 0;
        unsigned long irq_type;
-       unsigned long deliverable_irqs;
+       unsigned long irqs;
 
        __reset_intercept_indicators(vcpu);
 
        /* pending ckc conditions might have been invalidated */
        clear_bit(IRQ_PEND_EXT_CLOCK_COMP, &li->pending_irqs);
-       if (kvm_cpu_has_pending_timer(vcpu))
+       if (ckc_irq_pending(vcpu))
                set_bit(IRQ_PEND_EXT_CLOCK_COMP, &li->pending_irqs);
 
+       /* pending cpu timer conditions might have been invalidated */
+       clear_bit(IRQ_PEND_EXT_CPU_TIMER, &li->pending_irqs);
+       if (cpu_timer_irq_pending(vcpu))
+               set_bit(IRQ_PEND_EXT_CPU_TIMER, &li->pending_irqs);
+
        do {
-               deliverable_irqs = deliverable_local_irqs(vcpu);
+               irqs = deliverable_irqs(vcpu);
                /* bits are in the order of interrupt priority */
-               irq_type = find_first_bit(&deliverable_irqs, IRQ_PEND_COUNT);
+               irq_type = find_first_bit(&irqs, IRQ_PEND_COUNT);
                if (irq_type == IRQ_PEND_COUNT)
                        break;
-               func = deliver_irq_funcs[irq_type];
-               if (!func) {
-                       WARN_ON_ONCE(func == NULL);
-                       clear_bit(irq_type, &li->pending_irqs);
-                       continue;
+               if (is_ioirq(irq_type)) {
+                       rc = __deliver_io(vcpu, irq_type);
+               } else {
+                       func = deliver_irq_funcs[irq_type];
+                       if (!func) {
+                               WARN_ON_ONCE(func == NULL);
+                               clear_bit(irq_type, &li->pending_irqs);
+                               continue;
+                       }
+                       rc = func(vcpu);
                }
-               rc = func(vcpu);
-       } while (!rc && irq_type != IRQ_PEND_COUNT);
+               if (rc)
+                       break;
+       } while (!rc);
 
-       set_intercept_indicators_local(vcpu);
-
-       if (!rc && atomic_read(&fi->active)) {
-               do {
-                       deliver = 0;
-                       spin_lock(&fi->lock);
-                       list_for_each_entry_safe(inti, n, &fi->list, list) {
-                               if (__interrupt_is_deliverable(vcpu, inti)) {
-                                       list_del(&inti->list);
-                                       fi->irq_count--;
-                                       deliver = 1;
-                                       break;
-                               }
-                               __set_intercept_indicator(vcpu, inti);
-                       }
-                       if (list_empty(&fi->list))
-                               atomic_set(&fi->active, 0);
-                       spin_unlock(&fi->lock);
-                       if (deliver) {
-                               rc = __deliver_floating_interrupt(vcpu, inti);
-                               kfree(inti);
-                       }
-               } while (!rc && deliver);
-       }
+       set_intercept_indicators(vcpu);
 
        return rc;
 }
@@ -1172,80 +1197,182 @@ static int __inject_cpu_timer(struct kvm_vcpu *vcpu)
        return 0;
 }
 
+static struct kvm_s390_interrupt_info *get_io_int(struct kvm *kvm,
+                                                 int isc, u32 schid)
+{
+       struct kvm_s390_float_interrupt *fi = &kvm->arch.float_int;
+       struct list_head *isc_list = &fi->lists[FIRQ_LIST_IO_ISC_0 + isc];
+       struct kvm_s390_interrupt_info *iter;
+       u16 id = (schid & 0xffff0000U) >> 16;
+       u16 nr = schid & 0x0000ffffU;
 
+       spin_lock(&fi->lock);
+       list_for_each_entry(iter, isc_list, list) {
+               if (schid && (id != iter->io.subchannel_id ||
+                             nr != iter->io.subchannel_nr))
+                       continue;
+               /* found an appropriate entry */
+               list_del_init(&iter->list);
+               fi->counters[FIRQ_CNTR_IO] -= 1;
+               if (list_empty(isc_list))
+                       clear_bit(IRQ_PEND_IO_ISC_0 + isc, &fi->pending_irqs);
+               spin_unlock(&fi->lock);
+               return iter;
+       }
+       spin_unlock(&fi->lock);
+       return NULL;
+}
+
+/*
+ * Dequeue and return an I/O interrupt matching any of the interruption
+ * subclasses as designated by the isc mask in cr6 and the schid (if != 0).
+ */
 struct kvm_s390_interrupt_info *kvm_s390_get_io_int(struct kvm *kvm,
-                                                   u64 cr6, u64 schid)
+                                                   u64 isc_mask, u32 schid)
+{
+       struct kvm_s390_interrupt_info *inti = NULL;
+       int isc;
+
+       for (isc = 0; isc <= MAX_ISC && !inti; isc++) {
+               if (isc_mask & isc_to_isc_bits(isc))
+                       inti = get_io_int(kvm, isc, schid);
+       }
+       return inti;
+}
+
+#define SCCB_MASK 0xFFFFFFF8
+#define SCCB_EVENT_PENDING 0x3
+
+static int __inject_service(struct kvm *kvm,
+                            struct kvm_s390_interrupt_info *inti)
+{
+       struct kvm_s390_float_interrupt *fi = &kvm->arch.float_int;
+
+       spin_lock(&fi->lock);
+       fi->srv_signal.ext_params |= inti->ext.ext_params & SCCB_EVENT_PENDING;
+       /*
+        * Early versions of the QEMU s390 bios will inject several
+        * service interrupts after another without handling a
+        * condition code indicating busy.
+        * We will silently ignore those superfluous sccb values.
+        * A future version of QEMU will take care of serialization
+        * of servc requests
+        */
+       if (fi->srv_signal.ext_params & SCCB_MASK)
+               goto out;
+       fi->srv_signal.ext_params |= inti->ext.ext_params & SCCB_MASK;
+       set_bit(IRQ_PEND_EXT_SERVICE, &fi->pending_irqs);
+out:
+       spin_unlock(&fi->lock);
+       kfree(inti);
+       return 0;
+}
+
+static int __inject_virtio(struct kvm *kvm,
+                           struct kvm_s390_interrupt_info *inti)
+{
+       struct kvm_s390_float_interrupt *fi = &kvm->arch.float_int;
+
+       spin_lock(&fi->lock);
+       if (fi->counters[FIRQ_CNTR_VIRTIO] >= KVM_S390_MAX_VIRTIO_IRQS) {
+               spin_unlock(&fi->lock);
+               return -EBUSY;
+       }
+       fi->counters[FIRQ_CNTR_VIRTIO] += 1;
+       list_add_tail(&inti->list, &fi->lists[FIRQ_LIST_VIRTIO]);
+       set_bit(IRQ_PEND_VIRTIO, &fi->pending_irqs);
+       spin_unlock(&fi->lock);
+       return 0;
+}
+
+static int __inject_pfault_done(struct kvm *kvm,
+                                struct kvm_s390_interrupt_info *inti)
+{
+       struct kvm_s390_float_interrupt *fi = &kvm->arch.float_int;
+
+       spin_lock(&fi->lock);
+       if (fi->counters[FIRQ_CNTR_PFAULT] >=
+               (ASYNC_PF_PER_VCPU * KVM_MAX_VCPUS)) {
+               spin_unlock(&fi->lock);
+               return -EBUSY;
+       }
+       fi->counters[FIRQ_CNTR_PFAULT] += 1;
+       list_add_tail(&inti->list, &fi->lists[FIRQ_LIST_PFAULT]);
+       set_bit(IRQ_PEND_PFAULT_DONE, &fi->pending_irqs);
+       spin_unlock(&fi->lock);
+       return 0;
+}
+
+#define CR_PENDING_SUBCLASS 28
+static int __inject_float_mchk(struct kvm *kvm,
+                               struct kvm_s390_interrupt_info *inti)
+{
+       struct kvm_s390_float_interrupt *fi = &kvm->arch.float_int;
+
+       spin_lock(&fi->lock);
+       fi->mchk.cr14 |= inti->mchk.cr14 & (1UL << CR_PENDING_SUBCLASS);
+       fi->mchk.mcic |= inti->mchk.mcic;
+       set_bit(IRQ_PEND_MCHK_REP, &fi->pending_irqs);
+       spin_unlock(&fi->lock);
+       kfree(inti);
+       return 0;
+}
+
+static int __inject_io(struct kvm *kvm, struct kvm_s390_interrupt_info *inti)
 {
        struct kvm_s390_float_interrupt *fi;
-       struct kvm_s390_interrupt_info *inti, *iter;
+       struct list_head *list;
+       int isc;
 
-       if ((!schid && !cr6) || (schid && cr6))
-               return NULL;
        fi = &kvm->arch.float_int;
        spin_lock(&fi->lock);
-       inti = NULL;
-       list_for_each_entry(iter, &fi->list, list) {
-               if (!is_ioint(iter->type))
-                       continue;
-               if (cr6 &&
-                   ((cr6 & int_word_to_isc_bits(iter->io.io_int_word)) == 0))
-                       continue;
-               if (schid) {
-                       if (((schid & 0x00000000ffff0000) >> 16) !=
-                           iter->io.subchannel_id)
-                               continue;
-                       if ((schid & 0x000000000000ffff) !=
-                           iter->io.subchannel_nr)
-                               continue;
-               }
-               inti = iter;
-               break;
-       }
-       if (inti) {
-               list_del_init(&inti->list);
-               fi->irq_count--;
+       if (fi->counters[FIRQ_CNTR_IO] >= KVM_S390_MAX_FLOAT_IRQS) {
+               spin_unlock(&fi->lock);
+               return -EBUSY;
        }
-       if (list_empty(&fi->list))
-               atomic_set(&fi->active, 0);
+       fi->counters[FIRQ_CNTR_IO] += 1;
+
+       isc = int_word_to_isc(inti->io.io_int_word);
+       list = &fi->lists[FIRQ_LIST_IO_ISC_0 + isc];
+       list_add_tail(&inti->list, list);
+       set_bit(IRQ_PEND_IO_ISC_0 + isc, &fi->pending_irqs);
        spin_unlock(&fi->lock);
-       return inti;
+       return 0;
 }
 
 static int __inject_vm(struct kvm *kvm, struct kvm_s390_interrupt_info *inti)
 {
        struct kvm_s390_local_interrupt *li;
        struct kvm_s390_float_interrupt *fi;
-       struct kvm_s390_interrupt_info *iter;
        struct kvm_vcpu *dst_vcpu = NULL;
        int sigcpu;
-       int rc = 0;
+       u64 type = READ_ONCE(inti->type);
+       int rc;
 
        fi = &kvm->arch.float_int;
-       spin_lock(&fi->lock);
-       if (fi->irq_count >= KVM_S390_MAX_FLOAT_IRQS) {
+
+       switch (type) {
+       case KVM_S390_MCHK:
+               rc = __inject_float_mchk(kvm, inti);
+               break;
+       case KVM_S390_INT_VIRTIO:
+               rc = __inject_virtio(kvm, inti);
+               break;
+       case KVM_S390_INT_SERVICE:
+               rc = __inject_service(kvm, inti);
+               break;
+       case KVM_S390_INT_PFAULT_DONE:
+               rc = __inject_pfault_done(kvm, inti);
+               break;
+       case KVM_S390_INT_IO_MIN...KVM_S390_INT_IO_MAX:
+               rc = __inject_io(kvm, inti);
+               break;
+       default:
                rc = -EINVAL;
-               goto unlock_fi;
        }
-       fi->irq_count++;
-       if (!is_ioint(inti->type)) {
-               list_add_tail(&inti->list, &fi->list);
-       } else {
-               u64 isc_bits = int_word_to_isc_bits(inti->io.io_int_word);
+       if (rc)
+               return rc;
 
-               /* Keep I/O interrupts sorted in isc order. */
-               list_for_each_entry(iter, &fi->list, list) {
-                       if (!is_ioint(iter->type))
-                               continue;
-                       if (int_word_to_isc_bits(iter->io.io_int_word)
-                           <= isc_bits)
-                               continue;
-                       break;
-               }
-               list_add_tail(&inti->list, &iter->list);
-       }
-       atomic_set(&fi->active, 1);
-       if (atomic_read(&kvm->online_vcpus) == 0)
-               goto unlock_fi;
        sigcpu = find_first_bit(fi->idle_mask, KVM_MAX_VCPUS);
        if (sigcpu == KVM_MAX_VCPUS) {
                do {
@@ -1257,7 +1384,7 @@ static int __inject_vm(struct kvm *kvm, struct kvm_s390_interrupt_info *inti)
        dst_vcpu = kvm_get_vcpu(kvm, sigcpu);
        li = &dst_vcpu->arch.local_int;
        spin_lock(&li->lock);
-       switch (inti->type) {
+       switch (type) {
        case KVM_S390_MCHK:
                atomic_set_mask(CPUSTAT_STOP_INT, li->cpuflags);
                break;
@@ -1270,9 +1397,8 @@ static int __inject_vm(struct kvm *kvm, struct kvm_s390_interrupt_info *inti)
        }
        spin_unlock(&li->lock);
        kvm_s390_vcpu_wakeup(kvm_get_vcpu(kvm, sigcpu));
-unlock_fi:
-       spin_unlock(&fi->lock);
-       return rc;
+       return 0;
+
 }
 
 int kvm_s390_inject_vm(struct kvm *kvm,
@@ -1332,10 +1458,10 @@ int kvm_s390_inject_vm(struct kvm *kvm,
        return rc;
 }
 
-void kvm_s390_reinject_io_int(struct kvm *kvm,
+int kvm_s390_reinject_io_int(struct kvm *kvm,
                              struct kvm_s390_interrupt_info *inti)
 {
-       __inject_vm(kvm, inti);
+       return __inject_vm(kvm, inti);
 }
 
 int s390int_to_s390irq(struct kvm_s390_interrupt *s390int,
@@ -1388,12 +1514,10 @@ void kvm_s390_clear_stop_irq(struct kvm_vcpu *vcpu)
        spin_unlock(&li->lock);
 }
 
-int kvm_s390_inject_vcpu(struct kvm_vcpu *vcpu, struct kvm_s390_irq *irq)
+static int do_inject_vcpu(struct kvm_vcpu *vcpu, struct kvm_s390_irq *irq)
 {
-       struct kvm_s390_local_interrupt *li = &vcpu->arch.local_int;
        int rc;
 
-       spin_lock(&li->lock);
        switch (irq->type) {
        case KVM_S390_PROGRAM_INT:
                VCPU_EVENT(vcpu, 3, "inject: program check %d (from user)",
@@ -1433,83 +1557,130 @@ int kvm_s390_inject_vcpu(struct kvm_vcpu *vcpu, struct kvm_s390_irq *irq)
        default:
                rc = -EINVAL;
        }
+
+       return rc;
+}
+
+int kvm_s390_inject_vcpu(struct kvm_vcpu *vcpu, struct kvm_s390_irq *irq)
+{
+       struct kvm_s390_local_interrupt *li = &vcpu->arch.local_int;
+       int rc;
+
+       spin_lock(&li->lock);
+       rc = do_inject_vcpu(vcpu, irq);
        spin_unlock(&li->lock);
        if (!rc)
                kvm_s390_vcpu_wakeup(vcpu);
        return rc;
 }
 
-void kvm_s390_clear_float_irqs(struct kvm *kvm)
+static inline void clear_irq_list(struct list_head *_list)
 {
-       struct kvm_s390_float_interrupt *fi;
-       struct kvm_s390_interrupt_info  *n, *inti = NULL;
+       struct kvm_s390_interrupt_info *inti, *n;
 
-       fi = &kvm->arch.float_int;
-       spin_lock(&fi->lock);
-       list_for_each_entry_safe(inti, n, &fi->list, list) {
+       list_for_each_entry_safe(inti, n, _list, list) {
                list_del(&inti->list);
                kfree(inti);
        }
-       fi->irq_count = 0;
-       atomic_set(&fi->active, 0);
-       spin_unlock(&fi->lock);
 }
 
-static inline int copy_irq_to_user(s