arch/s390/kvm/kvm-s390.c

   1 /*
   2  * hosting zSeries kernel virtual machines
   3  *
   4  * Copyright IBM Corp. 2008, 2009
   5  *
   6  * This program is free software; you can redistribute it and/or modify
   7  * it under the terms of the GNU General Public License (version 2 only)
   8  * as published by the Free Software Foundation.
   9  *
  10  *    Author(s): Carsten Otte <cotte@de.ibm.com>
  11  *               Christian Borntraeger <borntraeger@de.ibm.com>
  12  *               Heiko Carstens <heiko.carstens@de.ibm.com>
  13  *               Christian Ehrhardt <ehrhardt@de.ibm.com>
  14  *               Jason J. Herne <jjherne@us.ibm.com>
  15  */
  16
  17 #include <linux/compiler.h>
  18 #include <linux/err.h>
  19 #include <linux/fs.h>
  20 #include <linux/hrtimer.h>
  21 #include <linux/init.h>
  22 #include <linux/kvm.h>
  23 #include <linux/kvm_host.h>
  24 #include <linux/module.h>
  25 #include <linux/random.h>
  26 #include <linux/slab.h>
  27 #include <linux/timer.h>
  28 #include <asm/asm-offsets.h>
  29 #include <asm/lowcore.h>
  30 #include <asm/pgtable.h>
  31 #include <asm/nmi.h>
  32 #include <asm/switch_to.h>
  33 #include <asm/sclp.h>
  34 #include "kvm-s390.h"
  35 #include "gaccess.h"
  36
  37 #define CREATE_TRACE_POINTS
  38 #include "trace.h"
  39 #include "trace-s390.h"
  40
  41 #define VCPU_STAT(x) offsetof(struct kvm_vcpu, stat.x), KVM_STAT_VCPU
  42
  43 struct kvm_stats_debugfs_item debugfs_entries[] = {
  44         { "userspace_handled", VCPU_STAT(exit_userspace) },
  45         { "exit_null", VCPU_STAT(exit_null) },
  46         { "exit_validity", VCPU_STAT(exit_validity) },
  47         { "exit_stop_request", VCPU_STAT(exit_stop_request) },
  48         { "exit_external_request", VCPU_STAT(exit_external_request) },
  49         { "exit_external_interrupt", VCPU_STAT(exit_external_interrupt) },
  50         { "exit_instruction", VCPU_STAT(exit_instruction) },
  51         { "exit_program_interruption", VCPU_STAT(exit_program_interruption) },
  52         { "exit_instr_and_program_int", VCPU_STAT(exit_instr_and_program) },
  53         { "halt_successful_poll", VCPU_STAT(halt_successful_poll) },
  54         { "halt_wakeup", VCPU_STAT(halt_wakeup) },
  55         { "instruction_lctlg", VCPU_STAT(instruction_lctlg) },
  56         { "instruction_lctl", VCPU_STAT(instruction_lctl) },
  57         { "instruction_stctl", VCPU_STAT(instruction_stctl) },
  58         { "instruction_stctg", VCPU_STAT(instruction_stctg) },
  59         { "deliver_emergency_signal", VCPU_STAT(deliver_emergency_signal) },
  60         { "deliver_external_call", VCPU_STAT(deliver_external_call) },
  61         { "deliver_service_signal", VCPU_STAT(deliver_service_signal) },
  62         { "deliver_virtio_interrupt", VCPU_STAT(deliver_virtio_interrupt) },
  63         { "deliver_stop_signal", VCPU_STAT(deliver_stop_signal) },
  64         { "deliver_prefix_signal", VCPU_STAT(deliver_prefix_signal) },
  65         { "deliver_restart_signal", VCPU_STAT(deliver_restart_signal) },
  66         { "deliver_program_interruption", VCPU_STAT(deliver_program_int) },
  67         { "exit_wait_state", VCPU_STAT(exit_wait_state) },
  68         { "instruction_pfmf", VCPU_STAT(instruction_pfmf) },
  69         { "instruction_stidp", VCPU_STAT(instruction_stidp) },
  70         { "instruction_spx", VCPU_STAT(instruction_spx) },
  71         { "instruction_stpx", VCPU_STAT(instruction_stpx) },
  72         { "instruction_stap", VCPU_STAT(instruction_stap) },
  73         { "instruction_storage_key", VCPU_STAT(instruction_storage_key) },
  74         { "instruction_ipte_interlock", VCPU_STAT(instruction_ipte_interlock) },
  75         { "instruction_stsch", VCPU_STAT(instruction_stsch) },
  76         { "instruction_chsc", VCPU_STAT(instruction_chsc) },
  77         { "instruction_essa", VCPU_STAT(instruction_essa) },
  78         { "instruction_stsi", VCPU_STAT(instruction_stsi) },
  79         { "instruction_stfl", VCPU_STAT(instruction_stfl) },
  80         { "instruction_tprot", VCPU_STAT(instruction_tprot) },
  81         { "instruction_sigp_sense", VCPU_STAT(instruction_sigp_sense) },
  82         { "instruction_sigp_sense_running", VCPU_STAT(instruction_sigp_sense_running) },
  83         { "instruction_sigp_external_call", VCPU_STAT(instruction_sigp_external_call) },
  84         { "instruction_sigp_emergency", VCPU_STAT(instruction_sigp_emergency) },
  85         { "instruction_sigp_cond_emergency", VCPU_STAT(instruction_sigp_cond_emergency) },
  86         { "instruction_sigp_start", VCPU_STAT(instruction_sigp_start) },
  87         { "instruction_sigp_stop", VCPU_STAT(instruction_sigp_stop) },
  88         { "instruction_sigp_stop_store_status", VCPU_STAT(instruction_sigp_stop_store_status) },
  89         { "instruction_sigp_store_status", VCPU_STAT(instruction_sigp_store_status) },
  90         { "instruction_sigp_set_arch", VCPU_STAT(instruction_sigp_arch) },
  91         { "instruction_sigp_set_prefix", VCPU_STAT(instruction_sigp_prefix) },
  92         { "instruction_sigp_restart", VCPU_STAT(instruction_sigp_restart) },
  93         { "instruction_sigp_cpu_reset", VCPU_STAT(instruction_sigp_cpu_reset) },
  94         { "instruction_sigp_init_cpu_reset", VCPU_STAT(instruction_sigp_init_cpu_reset) },
  95         { "instruction_sigp_unknown", VCPU_STAT(instruction_sigp_unknown) },
  96         { "diagnose_10", VCPU_STAT(diagnose_10) },
  97         { "diagnose_44", VCPU_STAT(diagnose_44) },
  98         { "diagnose_9c", VCPU_STAT(diagnose_9c) },
  99         { NULL }
 100 };
 101
 102 /* upper facilities limit for kvm */
 103 unsigned long kvm_s390_fac_list_mask[] = {
 104         0xff82fffbf4fc2000UL,
 105         0x005c000000000000UL,
 106 };
 107
 108 unsigned long kvm_s390_fac_list_mask_size(void)
 109 {
 110         BUILD_BUG_ON(ARRAY_SIZE(kvm_s390_fac_list_mask) > S390_ARCH_FAC_MASK_SIZE_U64);
 111         return ARRAY_SIZE(kvm_s390_fac_list_mask);
 112 }
 113
 114 static struct gmap_notifier gmap_notifier;
 115
 116 /* Section: not file related */
 117 int kvm_arch_hardware_enable(void)
 118 {
 119         /* every s390 is virtualization enabled ;-) */
 120         return 0;
 121 }
 122
 123 static void kvm_gmap_notifier(struct gmap *gmap, unsigned long address);
 124
 125 int kvm_arch_hardware_setup(void)
 126 {
 127         gmap_notifier.notifier_call = kvm_gmap_notifier;
 128         gmap_register_ipte_notifier(&gmap_notifier);
 129         return 0;
 130 }
 131
 132 void kvm_arch_hardware_unsetup(void)
 133 {
 134         gmap_unregister_ipte_notifier(&gmap_notifier);
 135 }
 136
 137 int kvm_arch_init(void *opaque)
 138 {
 139         /* Register floating interrupt controller interface. */
 140         return kvm_register_device_ops(&kvm_flic_ops, KVM_DEV_TYPE_FLIC);
 141 }
 142
 143 /* Section: device related */
 144 long kvm_arch_dev_ioctl(struct file *filp,
 145                         unsigned int ioctl, unsigned long arg)
 146 {
 147         if (ioctl == KVM_S390_ENABLE_SIE)
 148                 return s390_enable_sie();
 149         return -EINVAL;
 150 }
 151
 152 int kvm_vm_ioctl_check_extension(struct kvm *kvm, long ext)
 153 {
 154         int r;
 155
 156         switch (ext) {
 157         case KVM_CAP_S390_PSW:
 158         case KVM_CAP_S390_GMAP:
 159         case KVM_CAP_SYNC_MMU:
 160 #ifdef CONFIG_KVM_S390_UCONTROL
 161         case KVM_CAP_S390_UCONTROL:
 162 #endif
 163         case KVM_CAP_ASYNC_PF:
 164         case KVM_CAP_SYNC_REGS:
 165         case KVM_CAP_ONE_REG:
 166         case KVM_CAP_ENABLE_CAP:
 167         case KVM_CAP_S390_CSS_SUPPORT:
 168         case KVM_CAP_IOEVENTFD:
 169         case KVM_CAP_DEVICE_CTRL:
 170         case KVM_CAP_ENABLE_CAP_VM:
 171         case KVM_CAP_S390_IRQCHIP:
 172         case KVM_CAP_VM_ATTRIBUTES:
 173         case KVM_CAP_MP_STATE:
 174         case KVM_CAP_S390_USER_SIGP:
 175                 r = 1;
 176                 break;
 177         case KVM_CAP_NR_VCPUS:
 178         case KVM_CAP_MAX_VCPUS:
 179                 r = KVM_MAX_VCPUS;
 180                 break;
 181         case KVM_CAP_NR_MEMSLOTS:
 182                 r = KVM_USER_MEM_SLOTS;
 183                 break;
 184         case KVM_CAP_S390_COW:
 185                 r = MACHINE_HAS_ESOP;
 186                 break;
 187         default:
 188                 r = 0;
 189         }
 190         return r;
 191 }
 192
 193 static void kvm_s390_sync_dirty_log(struct kvm *kvm,
 194                                         struct kvm_memory_slot *memslot)
 195 {
 196         gfn_t cur_gfn, last_gfn;
 197         unsigned long address;
 198         struct gmap *gmap = kvm->arch.gmap;
 199
 200         down_read(&gmap->mm->mmap_sem);
 201         /* Loop over all guest pages */
 202         last_gfn = memslot->base_gfn + memslot->npages;
 203         for (cur_gfn = memslot->base_gfn; cur_gfn <= last_gfn; cur_gfn++) {
 204                 address = gfn_to_hva_memslot(memslot, cur_gfn);
 205
 206                 if (gmap_test_and_clear_dirty(address, gmap))
 207                         mark_page_dirty(kvm, cur_gfn);
 208         }
 209         up_read(&gmap->mm->mmap_sem);
 210 }
 211
 212 /* Section: vm related */
 213 /*
 214  * Get (and clear) the dirty memory log for a memory slot.
 215  */
 216 int kvm_vm_ioctl_get_dirty_log(struct kvm *kvm,
 217                                struct kvm_dirty_log *log)
 218 {
 219         int r;
 220         unsigned long n;
 221         struct kvm_memory_slot *memslot;
 222         int is_dirty = 0;
 223
 224         mutex_lock(&kvm->slots_lock);
 225
 226         r = -EINVAL;
 227         if (log->slot >= KVM_USER_MEM_SLOTS)
 228                 goto out;
 229
 230         memslot = id_to_memslot(kvm->memslots, log->slot);
 231         r = -ENOENT;
 232         if (!memslot->dirty_bitmap)
 233                 goto out;
 234
 235         kvm_s390_sync_dirty_log(kvm, memslot);
 236         r = kvm_get_dirty_log(kvm, log, &is_dirty);
 237         if (r)
 238                 goto out;
 239
 240         /* Clear the dirty log */
 241         if (is_dirty) {
 242                 n = kvm_dirty_bitmap_bytes(memslot);
 243                 memset(memslot->dirty_bitmap, 0, n);
 244         }
 245         r = 0;
 246 out:
 247         mutex_unlock(&kvm->slots_lock);
 248         return r;
 249 }
 250
 251 static int kvm_vm_ioctl_enable_cap(struct kvm *kvm, struct kvm_enable_cap *cap)
 252 {
 253         int r;
 254
 255         if (cap->flags)
 256                 return -EINVAL;
 257
 258         switch (cap->cap) {
 259         case KVM_CAP_S390_IRQCHIP:
 260                 kvm->arch.use_irqchip = 1;
 261                 r = 0;
 262                 break;
 263         case KVM_CAP_S390_USER_SIGP:
 264                 kvm->arch.user_sigp = 1;
 265                 r = 0;
 266                 break;
 267         default:
 268                 r = -EINVAL;
 269                 break;
 270         }
 271         return r;
 272 }
 273
 274 static int kvm_s390_get_mem_control(struct kvm *kvm, struct kvm_device_attr *attr)
 275 {
 276         int ret;
 277
 278         switch (attr->attr) {
 279         case KVM_S390_VM_MEM_LIMIT_SIZE:
 280                 ret = 0;
 281                 if (put_user(kvm->arch.gmap->asce_end, (u64 __user *)attr->addr))
 282                         ret = -EFAULT;
 283                 break;
 284         default:
 285                 ret = -ENXIO;
 286                 break;
 287         }
 288         return ret;
 289 }
 290
 291 static int kvm_s390_set_mem_control(struct kvm *kvm, struct kvm_device_attr *attr)
 292 {
 293         int ret;
 294         unsigned int idx;
 295         switch (attr->attr) {
 296         case KVM_S390_VM_MEM_ENABLE_CMMA:
 297                 ret = -EBUSY;
 298                 mutex_lock(&kvm->lock);
 299                 if (atomic_read(&kvm->online_vcpus) == 0) {
 300                         kvm->arch.use_cmma = 1;
 301                         ret = 0;
 302                 }
 303                 mutex_unlock(&kvm->lock);
 304                 break;
 305         case KVM_S390_VM_MEM_CLR_CMMA:
 306                 mutex_lock(&kvm->lock);
 307                 idx = srcu_read_lock(&kvm->srcu);
 308                 s390_reset_cmma(kvm->arch.gmap->mm);
 309                 srcu_read_unlock(&kvm->srcu, idx);
 310                 mutex_unlock(&kvm->lock);
 311                 ret = 0;
 312                 break;
 313         case KVM_S390_VM_MEM_LIMIT_SIZE: {
 314                 unsigned long new_limit;
 315
 316                 if (kvm_is_ucontrol(kvm))
 317                         return -EINVAL;
 318
 319                 if (get_user(new_limit, (u64 __user *)attr->addr))
 320                         return -EFAULT;
 321
 322                 if (new_limit > kvm->arch.gmap->asce_end)
 323                         return -E2BIG;
 324
 325                 ret = -EBUSY;
 326                 mutex_lock(&kvm->lock);
 327                 if (atomic_read(&kvm->online_vcpus) == 0) {
 328                         /* gmap_alloc will round the limit up */
 329                         struct gmap *new = gmap_alloc(current->mm, new_limit);
 330
 331                         if (!new) {
 332                                 ret = -ENOMEM;
 333                         } else {
 334                                 gmap_free(kvm->arch.gmap);
 335                                 new->private = kvm;
 336                                 kvm->arch.gmap = new;
 337                                 ret = 0;
 338                         }
 339                 }
 340                 mutex_unlock(&kvm->lock);
 341                 break;
 342         }
 343         default:
 344                 ret = -ENXIO;
 345                 break;
 346         }
 347         return ret;
 348 }
 349
 350 static void kvm_s390_vcpu_crypto_setup(struct kvm_vcpu *vcpu);
 351
 352 static int kvm_s390_vm_set_crypto(struct kvm *kvm, struct kvm_device_attr *attr)
 353 {
 354         struct kvm_vcpu *vcpu;
 355         int i;
 356
 357         if (!test_kvm_facility(kvm, 76))
 358                 return -EINVAL;
 359
 360         mutex_lock(&kvm->lock);
 361         switch (attr->attr) {
 362         case KVM_S390_VM_CRYPTO_ENABLE_AES_KW:
 363                 get_random_bytes(
 364                         kvm->arch.crypto.crycb->aes_wrapping_key_mask,
 365                         sizeof(kvm->arch.crypto.crycb->aes_wrapping_key_mask));
 366                 kvm->arch.crypto.aes_kw = 1;
 367                 break;
 368         case KVM_S390_VM_CRYPTO_ENABLE_DEA_KW:
 369                 get_random_bytes(
 370                         kvm->arch.crypto.crycb->dea_wrapping_key_mask,
 371                         sizeof(kvm->arch.crypto.crycb->dea_wrapping_key_mask));
 372                 kvm->arch.crypto.dea_kw = 1;
 373                 break;
 374         case KVM_S390_VM_CRYPTO_DISABLE_AES_KW:
 375                 kvm->arch.crypto.aes_kw = 0;
 376                 memset(kvm->arch.crypto.crycb->aes_wrapping_key_mask, 0,
 377                         sizeof(kvm->arch.crypto.crycb->aes_wrapping_key_mask));
 378                 break;
 379         case KVM_S390_VM_CRYPTO_DISABLE_DEA_KW:
 380                 kvm->arch.crypto.dea_kw = 0;
 381                 memset(kvm->arch.crypto.crycb->dea_wrapping_key_mask, 0,
 382                         sizeof(kvm->arch.crypto.crycb->dea_wrapping_key_mask));
 383                 break;
 384         default:
 385                 mutex_unlock(&kvm->lock);
 386                 return -ENXIO;
 387         }
 388
 389         kvm_for_each_vcpu(i, vcpu, kvm) {
 390                 kvm_s390_vcpu_crypto_setup(vcpu);
 391                 exit_sie(vcpu);
 392         }
 393         mutex_unlock(&kvm->lock);
 394         return 0;
 395 }
 396
 397 static int kvm_s390_set_tod_high(struct kvm *kvm, struct kvm_device_attr *attr)
 398 {
 399         u8 gtod_high;
 400
 401         if (copy_from_user(&gtod_high, (void __user *)attr->addr,
 402                                            sizeof(gtod_high)))
 403                 return -EFAULT;
 404
 405         if (gtod_high != 0)
 406                 return -EINVAL;
 407
 408         return 0;
 409 }
 410
 411 static int kvm_s390_set_tod_low(struct kvm *kvm, struct kvm_device_attr *attr)
 412 {
 413         struct kvm_vcpu *cur_vcpu;
 414         unsigned int vcpu_idx;
 415         u64 host_tod, gtod;
 416         int r;
 417
 418         if (copy_from_user(&gtod, (void __user *)attr->addr, sizeof(gtod)))
 419                 return -EFAULT;
 420
 421         r = store_tod_clock(&host_tod);
 422         if (r)
 423                 return r;
 424
 425         mutex_lock(&kvm->lock);
 426         kvm->arch.epoch = gtod - host_tod;
 427         kvm_for_each_vcpu(vcpu_idx, cur_vcpu, kvm) {
 428                 cur_vcpu->arch.sie_block->epoch = kvm->arch.epoch;
 429                 exit_sie(cur_vcpu);
 430         }
 431         mutex_unlock(&kvm->lock);
 432         return 0;
 433 }
 434
 435 static int kvm_s390_set_tod(struct kvm *kvm, struct kvm_device_attr *attr)
 436 {
 437         int ret;
 438
 439         if (attr->flags)
 440                 return -EINVAL;
 441
 442         switch (attr->attr) {
 443         case KVM_S390_VM_TOD_HIGH:
 444                 ret = kvm_s390_set_tod_high(kvm, attr);
 445                 break;
 446         case KVM_S390_VM_TOD_LOW:
 447                 ret = kvm_s390_set_tod_low(kvm, attr);
 448                 break;
 449         default:
 450                 ret = -ENXIO;
 451                 break;
 452         }
 453         return ret;
 454 }
 455
 456 static int kvm_s390_get_tod_high(struct kvm *kvm, struct kvm_device_attr *attr)
 457 {
 458         u8 gtod_high = 0;
 459
 460         if (copy_to_user((void __user *)attr->addr, &gtod_high,
 461                                          sizeof(gtod_high)))
 462                 return -EFAULT;
 463
 464         return 0;
 465 }
 466
 467 static int kvm_s390_get_tod_low(struct kvm *kvm, struct kvm_device_attr *attr)
 468 {
 469         u64 host_tod, gtod;
 470         int r;
 471
 472         r = store_tod_clock(&host_tod);
 473         if (r)
 474                 return r;
 475
 476         gtod = host_tod + kvm->arch.epoch;
 477         if (copy_to_user((void __user *)attr->addr, &gtod, sizeof(gtod)))
 478                 return -EFAULT;
 479
 480         return 0;
 481 }
 482
 483 static int kvm_s390_get_tod(struct kvm *kvm, struct kvm_device_attr *attr)
 484 {
 485         int ret;
 486
 487         if (attr->flags)
 488                 return -EINVAL;
 489
 490         switch (attr->attr) {
 491         case KVM_S390_VM_TOD_HIGH:
 492                 ret = kvm_s390_get_tod_high(kvm, attr);
 493                 break;
 494         case KVM_S390_VM_TOD_LOW:
 495                 ret = kvm_s390_get_tod_low(kvm, attr);
 496                 break;
 497         default:
 498                 ret = -ENXIO;
 499                 break;
 500         }
 501         return ret;
 502 }
 503
 504 static int kvm_s390_set_processor(struct kvm *kvm, struct kvm_device_attr *attr)
 505 {
 506         struct kvm_s390_vm_cpu_processor *proc;
 507         int ret = 0;
 508
 509         mutex_lock(&kvm->lock);
 510         if (atomic_read(&kvm->online_vcpus)) {
 511                 ret = -EBUSY;
 512                 goto out;
 513         }
 514         proc = kzalloc(sizeof(*proc), GFP_KERNEL);
 515         if (!proc) {
 516                 ret = -ENOMEM;
 517                 goto out;
 518         }
 519         if (!copy_from_user(proc, (void __user *)attr->addr,
 520                             sizeof(*proc))) {
 521                 memcpy(&kvm->arch.model.cpu_id, &proc->cpuid,
 522                        sizeof(struct cpuid));
 523                 kvm->arch.model.ibc = proc->ibc;
 524                 memcpy(kvm->arch.model.fac->list, proc->fac_list,
 525                        S390_ARCH_FAC_LIST_SIZE_BYTE);
 526         } else
 527                 ret = -EFAULT;
 528         kfree(proc);
 529 out:
 530         mutex_unlock(&kvm->lock);
 531         return ret;
 532 }
 533
 534 static int kvm_s390_set_cpu_model(struct kvm *kvm, struct kvm_device_attr *attr)
 535 {
 536         int ret = -ENXIO;
 537
 538         switch (attr->attr) {
 539         case KVM_S390_VM_CPU_PROCESSOR:
 540                 ret = kvm_s390_set_processor(kvm, attr);
 541                 break;
 542         }
 543         return ret;
 544 }
 545
 546 static int kvm_s390_get_processor(struct kvm *kvm, struct kvm_device_attr *attr)
 547 {
 548         struct kvm_s390_vm_cpu_processor *proc;
 549         int ret = 0;
 550
 551         proc = kzalloc(sizeof(*proc), GFP_KERNEL);
 552         if (!proc) {
 553                 ret = -ENOMEM;
 554                 goto out;
 555         }
 556         memcpy(&proc->cpuid, &kvm->arch.model.cpu_id, sizeof(struct cpuid));
 557         proc->ibc = kvm->arch.model.ibc;
 558         memcpy(&proc->fac_list, kvm->arch.model.fac->list, S390_ARCH_FAC_LIST_SIZE_BYTE);
 559         if (copy_to_user((void __user *)attr->addr, proc, sizeof(*proc)))
 560                 ret = -EFAULT;
 561         kfree(proc);
 562 out:
 563         return ret;
 564 }
 565
 566 static int kvm_s390_get_machine(struct kvm *kvm, struct kvm_device_attr *attr)
 567 {
 568         struct kvm_s390_vm_cpu_machine *mach;
 569         int ret = 0;
 570
 571         mach = kzalloc(sizeof(*mach), GFP_KERNEL);
 572         if (!mach) {
 573                 ret = -ENOMEM;
 574                 goto out;
 575         }
 576         get_cpu_id((struct cpuid *) &mach->cpuid);
 577         mach->ibc = sclp_get_ibc();
 578         memcpy(&mach->fac_mask, kvm->arch.model.fac->mask,
 579                S390_ARCH_FAC_LIST_SIZE_BYTE);
 580         memcpy((unsigned long *)&mach->fac_list, S390_lowcore.stfle_fac_list,
 581                S390_ARCH_FAC_LIST_SIZE_BYTE);
 582         if (copy_to_user((void __user *)attr->addr, mach, sizeof(*mach)))
 583                 ret = -EFAULT;
 584         kfree(mach);
 585 out:
 586         return ret;
 587 }
 588
 589 static int kvm_s390_get_cpu_model(struct kvm *kvm, struct kvm_device_attr *attr)
 590 {
 591         int ret = -ENXIO;
 592
 593         switch (attr->attr) {
 594         case KVM_S390_VM_CPU_PROCESSOR:
 595                 ret = kvm_s390_get_processor(kvm, attr);
 596                 break;
 597         case KVM_S390_VM_CPU_MACHINE:
 598                 ret = kvm_s390_get_machine(kvm, attr);
 599                 break;
 600         }
 601         return ret;
 602 }
 603
 604 static int kvm_s390_vm_set_attr(struct kvm *kvm, struct kvm_device_attr *attr)
 605 {
 606         int ret;
 607
 608         switch (attr->group) {
 609         case KVM_S390_VM_MEM_CTRL:
 610                 ret = kvm_s390_set_mem_control(kvm, attr);
 611                 break;
 612         case KVM_S390_VM_TOD:
 613                 ret = kvm_s390_set_tod(kvm, attr);
 614                 break;
 615         case KVM_S390_VM_CPU_MODEL:
 616                 ret = kvm_s390_set_cpu_model(kvm, attr);
 617                 break;
 618         case KVM_S390_VM_CRYPTO:
 619                 ret = kvm_s390_vm_set_crypto(kvm, attr);
 620                 break;
 621         default:
 622                 ret = -ENXIO;
 623                 break;
 624         }
 625
 626         return ret;
 627 }
 628
 629 static int kvm_s390_vm_get_attr(struct kvm *kvm, struct kvm_device_attr *attr)
 630 {
 631         int ret;
 632
 633         switch (attr->group) {
 634         case KVM_S390_VM_MEM_CTRL:
 635                 ret = kvm_s390_get_mem_control(kvm, attr);
 636                 break;
 637         case KVM_S390_VM_TOD:
 638                 ret = kvm_s390_get_tod(kvm, attr);
 639                 break;
 640         case KVM_S390_VM_CPU_MODEL:
 641                 ret = kvm_s390_get_cpu_model(kvm, attr);
 642                 break;
 643         default:
 644                 ret = -ENXIO;
 645                 break;
 646         }
 647
 648         return ret;
 649 }
 650
 651 static int kvm_s390_vm_has_attr(struct kvm *kvm, struct kvm_device_attr *attr)
 652 {
 653         int ret;
 654
 655         switch (attr->group) {
 656         case KVM_S390_VM_MEM_CTRL:
 657                 switch (attr->attr) {
 658                 case KVM_S390_VM_MEM_ENABLE_CMMA:
 659                 case KVM_S390_VM_MEM_CLR_CMMA:
 660                 case KVM_S390_VM_MEM_LIMIT_SIZE:
 661                         ret = 0;
 662                         break;
 663                 default:
 664                         ret = -ENXIO;
 665                         break;
 666                 }
 667                 break;
 668         case KVM_S390_VM_TOD:
 669                 switch (attr->attr) {
 670                 case KVM_S390_VM_TOD_LOW:
 671                 case KVM_S390_VM_TOD_HIGH:
 672                         ret = 0;
 673                         break;
 674                 default:
 675                         ret = -ENXIO;
 676                         break;
 677                 }
 678                 break;
 679         case KVM_S390_VM_CPU_MODEL:
 680                 switch (attr->attr) {
 681                 case KVM_S390_VM_CPU_PROCESSOR:
 682                 case KVM_S390_VM_CPU_MACHINE:
 683                         ret = 0;
 684                         break;
 685                 default:
 686                         ret = -ENXIO;
 687                         break;
 688                 }
 689                 break;
 690         case KVM_S390_VM_CRYPTO:
 691                 switch (attr->attr) {
 692                 case KVM_S390_VM_CRYPTO_ENABLE_AES_KW:
 693                 case KVM_S390_VM_CRYPTO_ENABLE_DEA_KW:
 694                 case KVM_S390_VM_CRYPTO_DISABLE_AES_KW:
 695                 case KVM_S390_VM_CRYPTO_DISABLE_DEA_KW:
 696                         ret = 0;
 697                         break;
 698                 default:
 699                         ret = -ENXIO;
 700                         break;
 701                 }
 702                 break;
 703         default:
 704                 ret = -ENXIO;
 705                 break;
 706         }
 707
 708         return ret;
 709 }
 710
 711 long kvm_arch_vm_ioctl(struct file *filp,
 712                        unsigned int ioctl, unsigned long arg)
 713 {
 714         struct kvm *kvm = filp->private_data;
 715         void __user *argp = (void __user *)arg;
 716         struct kvm_device_attr attr;
 717         int r;
 718
 719         switch (ioctl) {
 720         case KVM_S390_INTERRUPT: {
 721                 struct kvm_s390_interrupt s390int;
 722
 723                 r = -EFAULT;
 724                 if (copy_from_user(&s390int, argp, sizeof(s390int)))
 725                         break;
 726                 r = kvm_s390_inject_vm(kvm, &s390int);
 727                 break;
 728         }
 729         case KVM_ENABLE_CAP: {
 730                 struct kvm_enable_cap cap;
 731                 r = -EFAULT;
 732                 if (copy_from_user(&cap, argp, sizeof(cap)))
 733                         break;
 734                 r = kvm_vm_ioctl_enable_cap(kvm, &cap);
 735                 break;
 736         }
 737         case KVM_CREATE_IRQCHIP: {
 738                 struct kvm_irq_routing_entry routing;
 739
 740                 r = -EINVAL;
 741                 if (kvm->arch.use_irqchip) {
 742                         /* Set up dummy routing. */
 743                         memset(&routing, 0, sizeof(routing));
 744                         kvm_set_irq_routing(kvm, &routing, 0, 0);
 745                         r = 0;
 746                 }
 747                 break;
 748         }
 749         case KVM_SET_DEVICE_ATTR: {
 750                 r = -EFAULT;
 751                 if (copy_from_user(&attr, (void __user *)arg, sizeof(attr)))
 752                         break;
 753                 r = kvm_s390_vm_set_attr(kvm, &attr);
 754                 break;
 755         }
 756         case KVM_GET_DEVICE_ATTR: {
 757                 r = -EFAULT;
 758                 if (copy_from_user(&attr, (void __user *)arg, sizeof(attr)))
 759                         break;
 760                 r = kvm_s390_vm_get_attr(kvm, &attr);
 761                 break;
 762         }
 763         case KVM_HAS_DEVICE_ATTR: {
 764                 r = -EFAULT;
 765                 if (copy_from_user(&attr, (void __user *)arg, sizeof(attr)))
 766                         break;
 767                 r = kvm_s390_vm_has_attr(kvm, &attr);
 768                 break;
 769         }
 770         default:
 771                 r = -ENOTTY;
 772         }
 773
 774         return r;
 775 }
 776
 777 static int kvm_s390_query_ap_config(u8 *config)
 778 {
 779         u32 fcn_code = 0x04000000UL;
 780         u32 cc = 0;
 781
 782         memset(config, 0, 128);
 783         asm volatile(
 784                 "lgr 0,%1\n"
 785                 "lgr 2,%2\n"
 786                 ".long 0xb2af0000\n"            /* PQAP(QCI) */
 787                 "0: ipm %0\n"
 788                 "srl %0,28\n"
 789                 "1:\n"
 790                 EX_TABLE(0b, 1b)
 791                 : "+r" (cc)
 792                 : "r" (fcn_code), "r" (config)
 793                 : "cc", "0", "2", "memory"
 794         );
 795
 796         return cc;
 797 }
 798
 799 static int kvm_s390_apxa_installed(void)
 800 {
 801         u8 config[128];
 802         int cc;
 803
 804         if (test_facility(2) && test_facility(12)) {
 805                 cc = kvm_s390_query_ap_config(config);
 806
 807                 if (cc)
 808                         pr_err("PQAP(QCI) failed with cc=%d", cc);
 809                 else
 810                         return config[0] & 0x40;
 811         }
 812
 813         return 0;
 814 }
 815
 816 static void kvm_s390_set_crycb_format(struct kvm *kvm)
 817 {
 818         kvm->arch.crypto.crycbd = (__u32)(unsigned long) kvm->arch.crypto.crycb;
 819
 820         if (kvm_s390_apxa_installed())
 821                 kvm->arch.crypto.crycbd |= CRYCB_FORMAT2;
 822         else
 823                 kvm->arch.crypto.crycbd |= CRYCB_FORMAT1;
 824 }
 825
 826 static void kvm_s390_get_cpu_id(struct cpuid *cpu_id)
 827 {
 828         get_cpu_id(cpu_id);
 829         cpu_id->version = 0xff;
 830 }
 831
 832 static int kvm_s390_crypto_init(struct kvm *kvm)
 833 {
 834         if (!test_kvm_facility(kvm, 76))
 835                 return 0;
 836
 837         kvm->arch.crypto.crycb = kzalloc(sizeof(*kvm->arch.crypto.crycb),
 838                                          GFP_KERNEL | GFP_DMA);
 839         if (!kvm->arch.crypto.crycb)
 840                 return -ENOMEM;
 841
 842         kvm_s390_set_crycb_format(kvm);
 843
 844         /* Enable AES/DEA protected key functions by default */
 845         kvm->arch.crypto.aes_kw = 1;
 846         kvm->arch.crypto.dea_kw = 1;
 847         get_random_bytes(kvm->arch.crypto.crycb->aes_wrapping_key_mask,
 848                          sizeof(kvm->arch.crypto.crycb->aes_wrapping_key_mask));
 849         get_random_bytes(kvm->arch.crypto.crycb->dea_wrapping_key_mask,
 850                          sizeof(kvm->arch.crypto.crycb->dea_wrapping_key_mask));
 851
 852         return 0;
 853 }
 854
 855 int kvm_arch_init_vm(struct kvm *kvm, unsigned long type)
 856 {
 857         int i, rc;
 858         char debug_name[16];
 859         static unsigned long sca_offset;
 860
 861         rc = -EINVAL;
 862 #ifdef CONFIG_KVM_S390_UCONTROL
 863         if (type & ~KVM_VM_S390_UCONTROL)
 864                 goto out_err;
 865         if ((type & KVM_VM_S390_UCONTROL) && (!capable(CAP_SYS_ADMIN)))
 866                 goto out_err;
 867 #else
 868         if (type)
 869                 goto out_err;
 870 #endif
 871
 872         rc = s390_enable_sie();
 873         if (rc)
 874                 goto out_err;
 875
 876         rc = -ENOMEM;
 877
 878         kvm->arch.sca = (struct sca_block *) get_zeroed_page(GFP_KERNEL);
 879         if (!kvm->arch.sca)
 880                 goto out_err;
 881         spin_lock(&kvm_lock);
 882         sca_offset = (sca_offset + 16) & 0x7f0;
 883         kvm->arch.sca = (struct sca_block *) ((char *) kvm->arch.sca + sca_offset);
 884         spin_unlock(&kvm_lock);
 885
 886         sprintf(debug_name, "kvm-%u", current->pid);
 887
 888         kvm->arch.dbf = debug_register(debug_name, 8, 2, 8 * sizeof(long));
 889         if (!kvm->arch.dbf)
 890                 goto out_nodbf;
 891
 892         /*
 893          * The architectural maximum amount of facilities is 16 kbit. To store
 894          * this amount, 2 kbyte of memory is required. Thus we need a full
 895          * page to hold the guest facility list (arch.model.fac->list) and the
 896          * facility mask (arch.model.fac->mask). Its address size has to be
 897          * 31 bits and word aligned.
 898          */
 899         kvm->arch.model.fac =
 900                 (struct kvm_s390_fac *) get_zeroed_page(GFP_KERNEL | GFP_DMA);
 901         if (!kvm->arch.model.fac)
 902                 goto out_nofac;
 903
 904         /* Populate the facility mask initially. */
 905         memcpy(kvm->arch.model.fac->mask, S390_lowcore.stfle_fac_list,
 906                S390_ARCH_FAC_LIST_SIZE_BYTE);
 907         for (i = 0; i < S390_ARCH_FAC_LIST_SIZE_U64; i++) {
 908                 if (i < kvm_s390_fac_list_mask_size())
 909                         kvm->arch.model.fac->mask[i] &= kvm_s390_fac_list_mask[i];
 910                 else
 911                         kvm->arch.model.fac->mask[i] = 0UL;
 912         }
 913
 914         /* Populate the facility list initially. */
 915         memcpy(kvm->arch.model.fac->list, kvm->arch.model.fac->mask,
 916                S390_ARCH_FAC_LIST_SIZE_BYTE);
 917
 918         kvm_s390_get_cpu_id(&kvm->arch.model.cpu_id);
 919         kvm->arch.model.ibc = sclp_get_ibc() & 0x0fff;
 920
 921         if (kvm_s390_crypto_init(kvm) < 0)
 922                 goto out_crypto;
 923
 924         spin_lock_init(&kvm->arch.float_int.lock);
 925         INIT_LIST_HEAD(&kvm->arch.float_int.list);
 926         init_waitqueue_head(&kvm->arch.ipte_wq);
 927         mutex_init(&kvm->arch.ipte_mutex);
 928
 929         debug_register_view(kvm->arch.dbf, &debug_sprintf_view);
 930         VM_EVENT(kvm, 3, "%s", "vm created");
 931
 932         if (type & KVM_VM_S390_UCONTROL) {
 933                 kvm->arch.gmap = NULL;
 934         } else {
 935                 kvm->arch.gmap = gmap_alloc(current->mm, (1UL << 44) - 1);
 936                 if (!kvm->arch.gmap)
 937                         goto out_nogmap;
 938                 kvm->arch.gmap->private = kvm;
 939                 kvm->arch.gmap->pfault_enabled = 0;
 940         }
 941
 942         kvm->arch.css_support = 0;
 943         kvm->arch.use_irqchip = 0;
 944         kvm->arch.epoch = 0;
 945
 946         spin_lock_init(&kvm->arch.start_stop_lock);
 947
 948         return 0;
 949 out_nogmap:
 950         kfree(kvm->arch.crypto.crycb);
 951 out_crypto:
 952         free_page((unsigned long)kvm->arch.model.fac);
 953 out_nofac:
 954         debug_unregister(kvm->arch.dbf);
 955 out_nodbf:
 956         free_page((unsigned long)(kvm->arch.sca));
 957 out_err:
 958         return rc;
 959 }
 960
 961 void kvm_arch_vcpu_destroy(struct kvm_vcpu *vcpu)
 962 {
 963         VCPU_EVENT(vcpu, 3, "%s", "free cpu");
 964         trace_kvm_s390_destroy_vcpu(vcpu->vcpu_id);
 965         kvm_s390_clear_local_irqs(vcpu);
 966         kvm_clear_async_pf_completion_queue(vcpu);
 967         if (!kvm_is_ucontrol(vcpu->kvm)) {
 968                 clear_bit(63 - vcpu->vcpu_id,
 969                           (unsigned long *) &vcpu->kvm->arch.sca->mcn);
 970                 if (vcpu->kvm->arch.sca->cpu[vcpu->vcpu_id].sda ==
 971                     (__u64) vcpu->arch.sie_block)
 972                         vcpu->kvm->arch.sca->cpu[vcpu->vcpu_id].sda = 0;
 973         }
 974         smp_mb();
 975
 976         if (kvm_is_ucontrol(vcpu->kvm))
 977                 gmap_free(vcpu->arch.gmap);
 978
 979         if (kvm_s390_cmma_enabled(vcpu->kvm))
 980                 kvm_s390_vcpu_unsetup_cmma(vcpu);
 981         free_page((unsigned long)(vcpu->arch.sie_block));
 982
 983         kvm_vcpu_uninit(vcpu);
 984         kmem_cache_free(kvm_vcpu_cache, vcpu);
 985 }
 986
 987 static void kvm_free_vcpus(struct kvm *kvm)
 988 {
 989         unsigned int i;
 990         struct kvm_vcpu *vcpu;
 991
 992         kvm_for_each_vcpu(i, vcpu, kvm)
 993                 kvm_arch_vcpu_destroy(vcpu);
 994
 995         mutex_lock(&kvm->lock);
 996         for (i = 0; i < atomic_read(&kvm->online_vcpus); i++)
 997                 kvm->vcpus[i] = NULL;
 998
 999         atomic_set(&kvm->online_vcpus, 0);
1000         mutex_unlock(&kvm->lock);
1001 }
1002
1003 void kvm_arch_destroy_vm(struct kvm *kvm)
1004 {
1005         kvm_free_vcpus(kvm);
1006         free_page((unsigned long)kvm->arch.model.fac);
1007         free_page((unsigned long)(kvm->arch.sca));
1008         debug_unregister(kvm->arch.dbf);
1009         kfree(kvm->arch.crypto.crycb);
1010         if (!kvm_is_ucontrol(kvm))
1011                 gmap_free(kvm->arch.gmap);
1012         kvm_s390_destroy_adapters(kvm);
1013         kvm_s390_clear_float_irqs(kvm);
1014 }
1015
1016 /* Section: vcpu related */
1017 static int __kvm_ucontrol_vcpu_init(struct kvm_vcpu *vcpu)
1018 {
1019         vcpu->arch.gmap = gmap_alloc(current->mm, -1UL);
1020         if (!vcpu->arch.gmap)
1021                 return -ENOMEM;
1022         vcpu->arch.gmap->private = vcpu->kvm;
1023
1024         return 0;
1025 }
1026
1027 int kvm_arch_vcpu_init(struct kvm_vcpu *vcpu)
1028 {
1029         vcpu->arch.pfault_token = KVM_S390_PFAULT_TOKEN_INVALID;
1030         kvm_clear_async_pf_completion_queue(vcpu);
1031         vcpu->run->kvm_valid_regs = KVM_SYNC_PREFIX |
1032                                     KVM_SYNC_GPRS |
1033                                     KVM_SYNC_ACRS |
1034                                     KVM_SYNC_CRS |
1035                                     KVM_SYNC_ARCH0 |
1036                                     KVM_SYNC_PFAULT;
1037
1038         if (kvm_is_ucontrol(vcpu->kvm))
1039                 return __kvm_ucontrol_vcpu_init(vcpu);
1040
1041         return 0;
1042 }
1043
1044 void kvm_arch_vcpu_load(struct kvm_vcpu *vcpu, int cpu)
1045 {
1046         save_fp_ctl(&vcpu->arch.host_fpregs.fpc);
1047         save_fp_regs(vcpu->arch.host_fpregs.fprs);
1048         save_access_regs(vcpu->arch.host_acrs);
1049         restore_fp_ctl(&vcpu->arch.guest_fpregs.fpc);
1050         restore_fp_regs(vcpu->arch.guest_fpregs.fprs);
1051         restore_access_regs(vcpu->run->s.regs.acrs);
1052         gmap_enable(vcpu->arch.gmap);
1053         atomic_set_mask(CPUSTAT_RUNNING, &vcpu->arch.sie_block->cpuflags);
1054 }
1055
1056 void kvm_arch_vcpu_put(struct kvm_vcpu *vcpu)
1057 {
1058         atomic_clear_mask(CPUSTAT_RUNNING, &vcpu->arch.sie_block->cpuflags);
1059         gmap_disable(vcpu->arch.gmap);
1060         save_fp_ctl(&vcpu->arch.guest_fpregs.fpc);
1061         save_fp_regs(vcpu->arch.guest_fpregs.fprs);
1062         save_access_regs(vcpu->run->s.regs.acrs);
1063         restore_fp_ctl(&vcpu->arch.host_fpregs.fpc);
1064         restore_fp_regs(vcpu->arch.host_fpregs.fprs);
1065         restore_access_regs(vcpu->arch.host_acrs);
1066 }
1067
1068 static void kvm_s390_vcpu_initial_reset(struct kvm_vcpu *vcpu)
1069 {
1070         /* this equals initial cpu reset in pop, but we don't switch to ESA */
1071         vcpu->arch.sie_block->gpsw.mask = 0UL;
1072         vcpu->arch.sie_block->gpsw.addr = 0UL;
1073         kvm_s390_set_prefix(vcpu, 0);
1074         vcpu->arch.sie_block->cputm     = 0UL;
1075         vcpu->arch.sie_block->ckc       = 0UL;
1076         vcpu->arch.sie_block->todpr     = 0;
1077         memset(vcpu->arch.sie_block->gcr, 0, 16 * sizeof(__u64));
1078         vcpu->arch.sie_block->gcr[0]  = 0xE0UL;
1079         vcpu->arch.sie_block->gcr[14] = 0xC2000000UL;
1080         vcpu->arch.guest_fpregs.fpc = 0;
1081         asm volatile("lfpc %0" : : "Q" (vcpu->arch.guest_fpregs.fpc));
1082         vcpu->arch.sie_block->gbea = 1;
1083         vcpu->arch.sie_block->pp = 0;
1084         vcpu->arch.pfault_token = KVM_S390_PFAULT_TOKEN_INVALID;
1085         kvm_clear_async_pf_completion_queue(vcpu);
1086         if (!kvm_s390_user_cpu_state_ctrl(vcpu->kvm))
1087                 kvm_s390_vcpu_stop(vcpu);
1088         kvm_s390_clear_local_irqs(vcpu);
1089 }
1090
1091 void kvm_arch_vcpu_postcreate(struct kvm_vcpu *vcpu)
1092 {
1093         mutex_lock(&vcpu->kvm->lock);
1094         vcpu->arch.sie_block->epoch = vcpu->kvm->arch.epoch;
1095         mutex_unlock(&vcpu->kvm->lock);
1096         if (!kvm_is_ucontrol(vcpu->kvm))
1097                 vcpu->arch.gmap = vcpu->kvm->arch.gmap;
1098 }
1099
1100 static void kvm_s390_vcpu_crypto_setup(struct kvm_vcpu *vcpu)
1101 {
1102         if (!test_kvm_facility(vcpu->kvm, 76))
1103                 return;
1104
1105         vcpu->arch.sie_block->ecb3 &= ~(ECB3_AES | ECB3_DEA);
1106
1107         if (vcpu->kvm->arch.crypto.aes_kw)
1108                 vcpu->arch.sie_block->ecb3 |= ECB3_AES;
1109         if (vcpu->kvm->arch.crypto.dea_kw)
1110                 vcpu->arch.sie_block->ecb3 |= ECB3_DEA;
1111
1112         vcpu->arch.sie_block->crycbd = vcpu->kvm->arch.crypto.crycbd;
1113 }
1114
1115 void kvm_s390_vcpu_unsetup_cmma(struct kvm_vcpu *vcpu)
1116 {
1117         free_page(vcpu->arch.sie_block->cbrlo);
1118         vcpu->arch.sie_block->cbrlo = 0;
1119 }
1120
1121 int kvm_s390_vcpu_setup_cmma(struct kvm_vcpu *vcpu)
1122 {
1123         vcpu->arch.sie_block->cbrlo = get_zeroed_page(GFP_KERNEL);
1124         if (!vcpu->arch.sie_block->cbrlo)
1125                 return -ENOMEM;
1126
1127         vcpu->arch.sie_block->ecb2 |= 0x80;
1128         vcpu->arch.sie_block->ecb2 &= ~0x08;
1129         return 0;
1130 }
1131
1132 int kvm_arch_vcpu_setup(struct kvm_vcpu *vcpu)
1133 {
1134         int rc = 0;
1135
1136         atomic_set(&vcpu->arch.sie_block->cpuflags, CPUSTAT_ZARCH |
1137                                                     CPUSTAT_SM |
1138                                                     CPUSTAT_STOPPED |
1139                                                     CPUSTAT_GED);
1140         vcpu->arch.sie_block->ecb   = 6;
1141         if (test_kvm_facility(vcpu->kvm, 50) && test_kvm_facility(vcpu->kvm, 73))
1142                 vcpu->arch.sie_block->ecb |= 0x10;
1143
1144         vcpu->arch.sie_block->ecb2  = 8;
1145         vcpu->arch.sie_block->eca   = 0xC1002000U;
1146         if (sclp_has_siif())
1147                 vcpu->arch.sie_block->eca |= 1;
1148         if (sclp_has_sigpif())
1149                 vcpu->arch.sie_block->eca |= 0x10000000U;
1150         vcpu->arch.sie_block->ictl |= ICTL_ISKE | ICTL_SSKE | ICTL_RRBE |
1151                                       ICTL_TPROT;
1152
1153         if (kvm_s390_cmma_enabled(vcpu->kvm)) {
1154                 rc = kvm_s390_vcpu_setup_cmma(vcpu);
1155                 if (rc)
1156                         return rc;
1157         }
1158         hrtimer_init(&vcpu->arch.ckc_timer, CLOCK_MONOTONIC, HRTIMER_MODE_REL);
1159         vcpu->arch.ckc_timer.function = kvm_s390_idle_wakeup;
1160
1161         mutex_lock(&vcpu->kvm->lock);
1162         vcpu->arch.cpu_id = vcpu->kvm->arch.model.cpu_id;
1163         vcpu->arch.sie_block->ibc = vcpu->kvm->arch.model.ibc;
1164         mutex_unlock(&vcpu->kvm->lock);
1165
1166         kvm_s390_vcpu_crypto_setup(vcpu);
1167
1168         return rc;
1169 }
1170
1171 struct kvm_vcpu *kvm_arch_vcpu_create(struct kvm *kvm,
1172                                       unsigned int id)
1173 {
1174         struct kvm_vcpu *vcpu;
1175         struct sie_page *sie_page;
1176         int rc = -EINVAL;
1177
1178         if (id >= KVM_MAX_VCPUS)
1179                 goto out;
1180
1181         rc = -ENOMEM;
1182
1183         vcpu = kmem_cache_zalloc(kvm_vcpu_cache, GFP_KERNEL);
1184         if (!vcpu)
1185                 goto out;
1186
1187         sie_page = (struct sie_page *) get_zeroed_page(GFP_KERNEL);
1188         if (!sie_page)
1189                 goto out_free_cpu;
1190
1191         vcpu->arch.sie_block = &sie_page->sie_block;
1192         vcpu->arch.sie_block->itdba = (unsigned long) &sie_page->itdb;
1193
1194         vcpu->arch.sie_block->icpua = id;
1195         if (!kvm_is_ucontrol(kvm)) {
1196                 if (!kvm->arch.sca) {
1197                         WARN_ON_ONCE(1);
1198                         goto out_free_cpu;
1199                 }
1200                 if (!kvm->arch.sca->cpu[id].sda)
1201                         kvm->arch.sca->cpu[id].sda =
1202                                 (__u64) vcpu->arch.sie_block;
1203                 vcpu->arch.sie_block->scaoh =
1204                         (__u32)(((__u64)kvm->arch.sca) >> 32);
1205                 vcpu->arch.sie_block->scaol = (__u32)(__u64)kvm->arch.sca;
1206                 set_bit(63 - id, (unsigned long *) &kvm->arch.sca->mcn);
1207         }
1208         vcpu->arch.sie_block->fac = (int) (long) kvm->arch.model.fac->list;
1209
1210         spin_lock_init(&vcpu->arch.local_int.lock);
1211         vcpu->arch.local_int.float_int = &kvm->arch.float_int;
1212         vcpu->arch.local_int.wq = &vcpu->wq;
1213         vcpu->arch.local_int.cpuflags = &vcpu->arch.sie_block->cpuflags;
1214
1215         rc = kvm_vcpu_init(vcpu, kvm, id);
1216         if (rc)
1217                 goto out_free_sie_block;
1218         VM_EVENT(kvm, 3, "create cpu %d at %p, sie block at %p", id, vcpu,
1219                  vcpu->arch.sie_block);
1220         trace_kvm_s390_create_vcpu(id, vcpu, vcpu->arch.sie_block);
1221
1222         return vcpu;
1223 out_free_sie_block:
1224         free_page((unsigned long)(vcpu->arch.sie_block));
1225 out_free_cpu:
1226         kmem_cache_free(kvm_vcpu_cache, vcpu);
1227 out:
1228         return ERR_PTR(rc);
1229 }
1230
1231 int kvm_arch_vcpu_runnable(struct kvm_vcpu *vcpu)
1232 {
1233         return kvm_s390_vcpu_has_irq(vcpu, 0);
1234 }
1235
1236 void s390_vcpu_block(struct kvm_vcpu *vcpu)
1237 {
1238         atomic_set_mask(PROG_BLOCK_SIE, &vcpu->arch.sie_block->prog20);
1239 }
1240
1241 void s390_vcpu_unblock(struct kvm_vcpu *vcpu)
1242 {
1243         atomic_clear_mask(PROG_BLOCK_SIE, &vcpu->arch.sie_block->prog20);
1244 }
1245
1246 /*
1247  * Kick a guest cpu out of SIE and wait until SIE is not running.
1248  * If the CPU is not running (e.g. waiting as idle) the function will
1249  * return immediately. */
1250 void exit_sie(struct kvm_vcpu *vcpu)
1251 {
1252         atomic_set_mask(CPUSTAT_STOP_INT, &vcpu->arch.sie_block->cpuflags);
1253         while (vcpu->arch.sie_block->prog0c & PROG_IN_SIE)
1254                 cpu_relax();
1255 }
1256
1257 /* Kick a guest cpu out of SIE and prevent SIE-reentry */
1258 void exit_sie_sync(struct kvm_vcpu *vcpu)
1259 {
1260         s390_vcpu_block(vcpu);
1261         exit_sie(vcpu);
1262 }
1263
1264 static void kvm_gmap_notifier(struct gmap *gmap, unsigned long address)
1265 {
1266         int i;
1267         struct kvm *kvm = gmap->private;
1268         struct kvm_vcpu *vcpu;
1269
1270         kvm_for_each_vcpu(i, vcpu, kvm) {
1271                 /* match against both prefix pages */
1272                 if (kvm_s390_get_prefix(vcpu) == (address & ~0x1000UL)) {
1273                         VCPU_EVENT(vcpu, 2, "gmap notifier for %lx", address);
1274                         kvm_make_request(KVM_REQ_MMU_RELOAD, vcpu);
1275                         exit_sie_sync(vcpu);
1276                 }
1277         }
1278 }
1279
1280 int kvm_arch_vcpu_should_kick(struct kvm_vcpu *vcpu)
1281 {
1282         /* kvm common code refers to this, but never calls it */
1283         BUG();
1284         return 0;
1285 }
1286
1287 static int kvm_arch_vcpu_ioctl_get_one_reg(struct kvm_vcpu *vcpu,
1288                                            struct kvm_one_reg *reg)
1289 {
1290         int r = -EINVAL;
1291
1292         switch (reg->id) {
1293         case KVM_REG_S390_TODPR:
1294                 r = put_user(vcpu->arch.sie_block->todpr,
1295                              (u32 __user *)reg->addr);
1296                 break;
1297         case KVM_REG_S390_EPOCHDIFF:
1298                 r = put_user(vcpu->arch.sie_block->epoch,
1299                              (u64 __user *)reg->addr);
1300                 break;
1301         case KVM_REG_S390_CPU_TIMER:
1302                 r = put_user(vcpu->arch.sie_block->cputm,
1303                              (u64 __user *)reg->addr);
1304                 break;
1305         case KVM_REG_S390_CLOCK_COMP:
1306                 r = put_user(vcpu->arch.sie_block->ckc,
1307                              (u64 __user *)reg->addr);
1308                 break;
1309         case KVM_REG_S390_PFTOKEN:
1310                 r = put_user(vcpu->arch.pfault_token,
1311                              (u64 __user *)reg->addr);
1312                 break;
1313         case KVM_REG_S390_PFCOMPARE:
1314                 r = put_user(vcpu->arch.pfault_compare,
1315                              (u64 __user *)reg->addr);
1316                 break;
1317         case KVM_REG_S390_PFSELECT:
1318                 r = put_user(vcpu->arch.pfault_select,
1319                              (u64 __user *)reg->addr);
1320                 break;
1321         case KVM_REG_S390_PP:
1322                 r = put_user(vcpu->arch.sie_block->pp,
1323                              (u64 __user *)reg->addr);
1324                 break;
1325         case KVM_REG_S390_GBEA:
1326                 r = put_user(vcpu->arch.sie_block->gbea,
1327                              (u64 __user *)reg->addr);
1328                 break;
1329         default:
1330                 break;
1331         }
1332
1333         return r;
1334 }
1335
1336 static int kvm_arch_vcpu_ioctl_set_one_reg(struct kvm_vcpu *vcpu,
1337                                            struct kvm_one_reg *reg)
1338 {
1339         int r = -EINVAL;
1340
1341         switch (reg->id) {
1342         case KVM_REG_S390_TODPR:
1343                 r = get_user(vcpu->arch.sie_block->todpr,
1344                              (u32 __user *)reg->addr);
1345                 break;
1346         case KVM_REG_S390_EPOCHDIFF:
1347                 r = get_user(vcpu->arch.sie_block->epoch,
1348                              (u64 __user *)reg->addr);
1349                 break;
1350         case KVM_REG_S390_CPU_TIMER:
1351                 r = get_user(vcpu->arch.sie_block->cputm,
1352                              (u64 __user *)reg->addr);
1353                 break;
1354         case KVM_REG_S390_CLOCK_COMP:
1355                 r = get_user(vcpu->arch.sie_block->ckc,
1356                              (u64 __user *)reg->addr);
1357                 break;
1358         case KVM_REG_S390_PFTOKEN:
1359                 r = get_user(vcpu->arch.pfault_token,
1360                              (u64 __user *)reg->addr);
1361                 if (vcpu->arch.pfault_token == KVM_S390_PFAULT_TOKEN_INVALID)
1362                         kvm_clear_async_pf_completion_queue(vcpu);
1363                 break;
1364         case KVM_REG_S390_PFCOMPARE:
1365                 r = get_user(vcpu->arch.pfault_compare,
1366                              (u64 __user *)reg->addr);
1367                 break;
1368         case KVM_REG_S390_PFSELECT:
1369                 r = get_user(vcpu->arch.pfault_select,
1370                              (u64 __user *)reg->addr);
1371                 break;
1372         case KVM_REG_S390_PP:
1373                 r = get_user(vcpu->arch.sie_block->pp,
1374                              (u64 __user *)reg->addr);
1375                 break;
1376         case KVM_REG_S390_GBEA:
1377                 r = get_user(vcpu->arch.sie_block->gbea,
1378                              (u64 __user *)reg->addr);
1379                 break;
1380         default:
1381                 break;
1382         }
1383
1384         return r;
1385 }
1386
1387 static int kvm_arch_vcpu_ioctl_initial_reset(struct kvm_vcpu *vcpu)
1388 {
1389         kvm_s390_vcpu_initial_reset(vcpu);
1390         return 0;
1391 }
1392
1393 int kvm_arch_vcpu_ioctl_set_regs(struct kvm_vcpu *vcpu, struct kvm_regs *regs)
1394 {
1395         memcpy(&vcpu->run->s.regs.gprs, &regs->gprs, sizeof(regs->gprs));
1396         return 0;
1397 }
1398
1399 int kvm_arch_vcpu_ioctl_get_regs(struct kvm_vcpu *vcpu, struct kvm_regs *regs)
1400 {
1401         memcpy(&regs->gprs, &vcpu->run->s.regs.gprs, sizeof(regs->gprs));
1402         return 0;
1403 }
1404
1405 int kvm_arch_vcpu_ioctl_set_sregs(struct kvm_vcpu *vcpu,
1406                                   struct kvm_sregs *sregs)
1407 {
1408         memcpy(&vcpu->run->s.regs.acrs, &sregs->acrs, sizeof(sregs->acrs));
1409         memcpy(&vcpu->arch.sie_block->gcr, &sregs->crs, sizeof(sregs->crs));
1410         restore_access_regs(vcpu->run->s.regs.acrs);
1411         return 0;
1412 }
1413
1414 int kvm_arch_vcpu_ioctl_get_sregs(struct kvm_vcpu *vcpu,
1415                                   struct kvm_sregs *sregs)
1416 {
1417         memcpy(&sregs->acrs, &vcpu->run->s.regs.acrs, sizeof(sregs->acrs));
1418         memcpy(&sregs->crs, &vcpu->arch.sie_block->gcr, sizeof(sregs->crs));
1419         return 0;
1420 }
1421
1422 int kvm_arch_vcpu_ioctl_set_fpu(struct kvm_vcpu *vcpu, struct kvm_fpu *fpu)
1423 {
1424         if (test_fp_ctl(fpu->fpc))
1425                 return -EINVAL;
1426         memcpy(&vcpu->arch.guest_fpregs.fprs, &fpu->fprs, sizeof(fpu->fprs));
1427         vcpu->arch.guest_fpregs.fpc = fpu->fpc;
1428         restore_fp_ctl(&vcpu->arch.guest_fpregs.fpc);
1429         restore_fp_regs(vcpu->arch.guest_fpregs.fprs);
1430         return 0;
1431 }
1432
1433 int kvm_arch_vcpu_ioctl_get_fpu(struct kvm_vcpu *vcpu, struct kvm_fpu *fpu)
1434 {
1435         memcpy(&fpu->fprs, &vcpu->arch.guest_fpregs.fprs, sizeof(fpu->fprs));
1436         fpu->fpc = vcpu->arch.guest_fpregs.fpc;
1437         return 0;
1438 }
1439
1440 static int kvm_arch_vcpu_ioctl_set_initial_psw(struct kvm_vcpu *vcpu, psw_t psw)
1441 {
1442         int rc = 0;
1443
1444         if (!is_vcpu_stopped(vcpu))
1445                 rc = -EBUSY;
1446         else {
1447                 vcpu->run->psw_mask = psw.mask;
1448                 vcpu->run->psw_addr = psw.addr;
1449         }
1450         return rc;
1451 }
1452
1453 int kvm_arch_vcpu_ioctl_translate(struct kvm_vcpu *vcpu,
1454                                   struct kvm_translation *tr)
1455 {
1456         return -EINVAL; /* not implemented yet */
1457 }
1458
1459 #define VALID_GUESTDBG_FLAGS (KVM_GUESTDBG_SINGLESTEP | \
1460                               KVM_GUESTDBG_USE_HW_BP | \
1461                               KVM_GUESTDBG_ENABLE)
1462
1463 int kvm_arch_vcpu_ioctl_set_guest_debug(struct kvm_vcpu *vcpu,
1464                                         struct kvm_guest_debug *dbg)
1465 {
1466         int rc = 0;
1467
1468         vcpu->guest_debug = 0;
1469         kvm_s390_clear_bp_data(vcpu);
1470
1471         if (dbg->control & ~VALID_GUESTDBG_FLAGS)
1472                 return -EINVAL;
1473
1474         if (dbg->control & KVM_GUESTDBG_ENABLE) {
1475                 vcpu->guest_debug = dbg->control;
1476                 /* enforce guest PER */
1477                 atomic_set_mask(CPUSTAT_P, &vcpu->arch.sie_block->cpuflags);
1478
1479                 if (dbg->control & KVM_GUESTDBG_USE_HW_BP)
1480                         rc = kvm_s390_import_bp_data(vcpu, dbg);
1481         } else {
1482                 atomic_clear_mask(CPUSTAT_P, &vcpu->arch.sie_block->cpuflags);
1483                 vcpu->arch.guestdbg.last_bp = 0;
1484         }
1485
1486         if (rc) {
1487                 vcpu->guest_debug = 0;
1488                 kvm_s390_clear_bp_data(vcpu);
1489                 atomic_clear_mask(CPUSTAT_P, &vcpu->arch.sie_block->cpuflags);
1490         }
1491
1492         return rc;
1493 }
1494
1495 int kvm_arch_vcpu_ioctl_get_mpstate(struct kvm_vcpu *vcpu,
1496                                     struct kvm_mp_state *mp_state)
1497 {
1498         /* CHECK_STOP and LOAD are not supported yet */
1499         return is_vcpu_stopped(vcpu) ? KVM_MP_STATE_STOPPED :
1500                                        KVM_MP_STATE_OPERATING;
1501 }
1502
1503 int kvm_arch_vcpu_ioctl_set_mpstate(struct kvm_vcpu *vcpu,
1504                                     struct kvm_mp_state *mp_state)
1505 {
1506         int rc = 0;
1507
1508         /* user space knows about this interface - let it control the state */
1509         vcpu->kvm->arch.user_cpu_state_ctrl = 1;
1510
1511         switch (mp_state->mp_state) {
1512         case KVM_MP_STATE_STOPPED:
1513                 kvm_s390_vcpu_stop(vcpu);
1514                 break;
1515         case KVM_MP_STATE_OPERATING:
1516                 kvm_s390_vcpu_start(vcpu);
1517                 break;
1518         case KVM_MP_STATE_LOAD:
1519         case KVM_MP_STATE_CHECK_STOP:
1520                 /* fall through - CHECK_STOP and LOAD are not supported yet */
1521         default:
1522                 rc = -ENXIO;
1523         }
1524
1525         return rc;
1526 }
1527
1528 bool kvm_s390_cmma_enabled(struct kvm *kvm)
1529 {
1530         if (!MACHINE_IS_LPAR)
1531                 return false;
1532         /* only enable for z10 and later */
1533         if (!MACHINE_HAS_EDAT1)
1534                 return false;
1535         if (!kvm->arch.use_cmma)
1536                 return false;
1537         return true;
1538 }
1539
1540 static bool ibs_enabled(struct kvm_vcpu *vcpu)
1541 {
1542         return atomic_read(&vcpu->arch.sie_block->cpuflags) & CPUSTAT_IBS;
1543 }
1544
1545 static int kvm_s390_handle_requests(struct kvm_vcpu *vcpu)
1546 {
1547 retry:
1548         s390_vcpu_unblock(vcpu);
1549         /*
1550          * We use MMU_RELOAD just to re-arm the ipte notifier for the
1551          * guest prefix page. gmap_ipte_notify will wait on the ptl lock.
1552          * This ensures that the ipte instruction for this request has
1553          * already finished. We might race against a second unmapper that
1554          * wants to set the blocking bit. Lets just retry the request loop.
1555          */
1556         if (kvm_check_request(KVM_REQ_MMU_RELOAD, vcpu)) {
1557                 int rc;
1558                 rc = gmap_ipte_notify(vcpu->arch.gmap,
1559                                       kvm_s390_get_prefix(vcpu),
1560                                       PAGE_SIZE * 2);
1561                 if (rc)
1562                         return rc;
1563                 goto retry;
1564         }
1565
1566         if (kvm_check_request(KVM_REQ_TLB_FLUSH, vcpu)) {
1567                 vcpu->arch.sie_block->ihcpu = 0xffff;
1568                 goto retry;
1569         }
1570
1571         if (kvm_check_request(KVM_REQ_ENABLE_IBS, vcpu)) {
1572                 if (!ibs_enabled(vcpu)) {
1573                         trace_kvm_s390_enable_disable_ibs(vcpu->vcpu_id, 1);
1574                         atomic_set_mask(CPUSTAT_IBS,
1575                                         &vcpu->arch.sie_block->cpuflags);
1576                 }
1577                 goto retry;
1578         }
1579
1580         if (kvm_check_request(KVM_REQ_DISABLE_IBS, vcpu)) {
1581                 if (ibs_enabled(vcpu)) {
1582                         trace_kvm_s390_enable_disable_ibs(vcpu->vcpu_id, 0);
1583                         atomic_clear_mask(CPUSTAT_IBS,
1584                                           &vcpu->arch.sie_block->cpuflags);
1585                 }
1586                 goto retry;
1587         }
1588
1589         /* nothing to do, just clear the request */
1590         clear_bit(KVM_REQ_UNHALT, &vcpu->requests);
1591
1592         return 0;
1593 }
1594
1595 /**
1596  * kvm_arch_fault_in_page - fault-in guest page if necessary
1597  * @vcpu: The corresponding virtual cpu
1598  * @gpa: Guest physical address
1599  * @writable: Whether the page should be writable or not
1600  *
1601  * Make sure that a guest page has been faulted-in on the host.
1602  *
1603  * Return: Zero on success, negative error code otherwise.
1604  */
1605 long kvm_arch_fault_in_page(struct kvm_vcpu *vcpu, gpa_t gpa, int writable)
1606 {
1607         return gmap_fault(vcpu->arch.gmap, gpa,
1608                           writable ? FAULT_FLAG_WRITE : 0);
1609 }
1610
1611 static void __kvm_inject_pfault_token(struct kvm_vcpu *vcpu, bool start_token,
1612                                       unsigned long token)
1613 {
1614         struct kvm_s390_interrupt inti;
1615         struct kvm_s390_irq irq;
1616
1617         if (start_token) {
1618                 irq.u.ext.ext_params2 = token;
1619                 irq.type = KVM_S390_INT_PFAULT_INIT;
1620                 WARN_ON_ONCE(kvm_s390_inject_vcpu(vcpu, &irq));
1621         } else {
1622                 inti.type = KVM_S390_INT_PFAULT_DONE;
1623                 inti.parm64 = token;
1624                 WARN_ON_ONCE(kvm_s390_inject_vm(vcpu->kvm, &inti));
1625         }
1626 }
1627
1628 void kvm_arch_async_page_not_present(struct kvm_vcpu *vcpu,
1629                                      struct kvm_async_pf *work)
1630 {
1631         trace_kvm_s390_pfault_init(vcpu, work->arch.pfault_token);
1632         __kvm_inject_pfault_token(vcpu, true, work->arch.pfault_token);
1633 }
1634
1635 void kvm_arch_async_page_present(struct kvm_vcpu *vcpu,
1636                                  struct kvm_async_pf *work)
1637 {
1638         trace_kvm_s390_pfault_done(vcpu, work->arch.pfault_token);
1639         __kvm_inject_pfault_token(vcpu, false, work->arch.pfault_token);
1640 }
1641
1642 void kvm_arch_async_page_ready(struct kvm_vcpu *vcpu,
1643                                struct kvm_async_pf *work)
1644 {
1645         /* s390 will always inject the page directly */
1646 }
1647
1648 bool kvm_arch_can_inject_async_page_present(struct kvm_vcpu *vcpu)
1649 {
1650         /*
1651          * s390 will always inject the page directly,
1652          * but we still want check_async_completion to cleanup
1653          */
1654         return true;
1655 }
1656
1657 static int kvm_arch_setup_async_pf(struct kvm_vcpu *vcpu)
1658 {
1659         hva_t hva;
1660         struct kvm_arch_async_pf arch;
1661         int rc;
1662
1663         if (vcpu->arch.pfault_token == KVM_S390_PFAULT_TOKEN_INVALID)
1664                 return 0;
1665         if ((vcpu->arch.sie_block->gpsw.mask & vcpu->arch.pfault_select) !=
1666             vcpu->arch.pfault_compare)
1667                 return 0;
1668         if (psw_extint_disabled(vcpu))
1669                 return 0;
1670         if (kvm_s390_vcpu_has_irq(vcpu, 0))
1671                 return 0;
1672         if (!(vcpu->arch.sie_block->gcr[0] & 0x200ul))
1673                 return 0;
1674         if (!vcpu->arch.gmap->pfault_enabled)
1675                 return 0;
1676
1677         hva = gfn_to_hva(vcpu->kvm, gpa_to_gfn(current->thread.gmap_addr));
1678         hva += current->thread.gmap_addr & ~PAGE_MASK;
1679         if (read_guest_real(vcpu, vcpu->arch.pfault_token, &arch.pfault_token, 8))
1680                 return 0;
1681
1682         rc = kvm_setup_async_pf(vcpu, current->thread.gmap_addr, hva, &arch);
1683         return rc;
1684 }
1685
1686 static int vcpu_pre_run(struct kvm_vcpu *vcpu)
1687 {
1688         int rc, cpuflags;
1689
1690         /*
1691          * On s390 notifications for arriving pages will be delivered directly
1692          * to the guest but the house keeping for completed pfaults is
1693          * handled outside the worker.
1694          */
1695         kvm_check_async_pf_completion(vcpu);
1696
1697         memcpy(&vcpu->arch.sie_block->gg14, &vcpu->run->s.regs.gprs[14], 16);
1698
1699         if (need_resched())
1700                 schedule();
1701
1702         if (test_cpu_flag(CIF_MCCK_PENDING))
1703                 s390_handle_mcck();
1704
1705         if (!kvm_is_ucontrol(vcpu->kvm)) {
1706                 rc = kvm_s390_deliver_pending_interrupts(vcpu);
1707                 if (rc)
1708                         return rc;
1709         }
1710
1711         rc = kvm_s390_handle_requests(vcpu);
1712         if (rc)
1713                 return rc;
1714
1715         if (guestdbg_enabled(vcpu)) {
1716                 kvm_s390_backup_guest_per_regs(vcpu);
1717                 kvm_s390_patch_guest_per_regs(vcpu);
1718         }
1719
1720         vcpu->arch.sie_block->icptcode = 0;
1721         cpuflags = atomic_read(&vcpu->arch.sie_block->cpuflags);
1722         VCPU_EVENT(vcpu, 6, "entering sie flags %x", cpuflags);
1723         trace_kvm_s390_sie_enter(vcpu, cpuflags);
1724
1725         return 0;
1726 }
1727
1728 static int vcpu_post_run(struct kvm_vcpu *vcpu, int exit_reason)
1729 {
1730         int rc = -1;
1731
1732         VCPU_EVENT(vcpu, 6, "exit sie icptcode %d",
1733                    vcpu->arch.sie_block->icptcode);
1734         trace_kvm_s390_sie_exit(vcpu, vcpu->arch.sie_block->icptcode);
1735
1736         if (guestdbg_enabled(vcpu))
1737                 kvm_s390_restore_guest_per_regs(vcpu);
1738
1739         if (exit_reason >= 0) {
1740                 rc = 0;
1741         } else if (kvm_is_ucontrol(vcpu->kvm)) {
1742                 vcpu->run->exit_reason = KVM_EXIT_S390_UCONTROL;
1743                 vcpu->run->s390_ucontrol.trans_exc_code =
1744                                                 current->thread.gmap_addr;
1745                 vcpu->run->s390_ucontrol.pgm_code = 0x10;
1746                 rc = -EREMOTE;
1747
1748         } else if (current->thread.gmap_pfault) {
1749                 trace_kvm_s390_major_guest_pfault(vcpu);
1750                 current->thread.gmap_pfault = 0;
1751                 if (kvm_arch_setup_async_pf(vcpu)) {
1752                         rc = 0;
1753                 } else {
1754                         gpa_t gpa = current->thread.gmap_addr;
1755                         rc = kvm_arch_fault_in_page(vcpu, gpa, 1);
1756                 }
1757         }
1758
1759         if (rc == -1) {
1760                 VCPU_EVENT(vcpu, 3, "%s", "fault in sie instruction");
1761                 trace_kvm_s390_sie_fault(vcpu);
1762                 rc = kvm_s390_inject_program_int(vcpu, PGM_ADDRESSING);
1763         }
1764
1765         memcpy(&vcpu->run->s.regs.gprs[14], &vcpu->arch.sie_block->gg14, 16);
1766
1767         if (rc == 0) {
1768                 if (kvm_is_ucontrol(vcpu->kvm))
1769                         /* Don't exit for host interrupts. */
1770                         rc = vcpu->arch.sie_block->icptcode ? -EOPNOTSUPP : 0;
1771                 else
1772                         rc = kvm_handle_sie_intercept(vcpu);
1773         }
1774
1775         return rc;
1776 }
1777
1778 static int __vcpu_run(struct kvm_vcpu *vcpu)
1779 {
1780         int rc, exit_reason;
1781
1782         /*
1783          * We try to hold kvm->srcu during most of vcpu_run (except when run-
1784          * ning the guest), so that memslots (and other stuff) are protected
1785          */
1786         vcpu->srcu_idx = srcu_read_lock(&vcpu->kvm->srcu);
1787
1788         do {
1789                 rc = vcpu_pre_run(vcpu);
1790                 if (rc)
1791                         break;
1792
1793                 srcu_read_unlock(&vcpu->kvm->srcu, vcpu->srcu_idx);
1794                 /*
1795                  * As PF_VCPU will be used in fault handler, between
1796                  * guest_enter and guest_exit should be no uaccess.
1797                  */
1798                 preempt_disable();
1799                 kvm_guest_enter();
1800                 preempt_enable();
1801                 exit_reason = sie64a(vcpu->arch.sie_block,
1802                                      vcpu->run->s.regs.gprs);
1803                 kvm_guest_exit();
1804                 vcpu->srcu_idx = srcu_read_lock(&vcpu->kvm->srcu);
1805
1806                 rc = vcpu_post_run(vcpu, exit_reason);
1807         } while (!signal_pending(current) && !guestdbg_exit_pending(vcpu) && !rc);
1808
1809         srcu_read_unlock(&vcpu->kvm->srcu, vcpu->srcu_idx);
1810         return rc;
1811 }
1812
1813 static void sync_regs(struct kvm_vcpu *vcpu, struct kvm_run *kvm_run)
1814 {
1815         vcpu->arch.sie_block->gpsw.mask = kvm_run->psw_mask;
1816         vcpu->arch.sie_block->gpsw.addr = kvm_run->psw_addr;
1817         if (kvm_run->kvm_dirty_regs & KVM_SYNC_PREFIX)
1818                 kvm_s390_set_prefix(vcpu, kvm_run->s.regs.prefix);
1819         if (kvm_run->kvm_dirty_regs & KVM_SYNC_CRS) {
1820                 memcpy(&vcpu->arch.sie_block->gcr, &kvm_run->s.regs.crs, 128);
1821                 /* some control register changes require a tlb flush */
1822                 kvm_make_request(KVM_REQ_TLB_FLUSH, vcpu);
1823         }
1824         if (kvm_run->kvm_dirty_regs & KVM_SYNC_ARCH0) {
1825                 vcpu->arch.sie_block->cputm = kvm_run->s.regs.cputm;
1826                 vcpu->arch.sie_block->ckc = kvm_run->s.regs.ckc;
1827                 vcpu->arch.sie_block->todpr = kvm_run->s.regs.todpr;
1828                 vcpu->arch.sie_block->pp = kvm_run->s.regs.pp;
1829                 vcpu->arch.sie_block->gbea = kvm_run->s.regs.gbea;
1830         }
1831         if (kvm_run->kvm_dirty_regs & KVM_SYNC_PFAULT) {
1832                 vcpu->arch.pfault_token = kvm_run->s.regs.pft;
1833                 vcpu->arch.pfault_select = kvm_run->s.regs.pfs;
1834                 vcpu->arch.pfault_compare = kvm_run->s.regs.pfc;
1835                 if (vcpu->arch.pfault_token == KVM_S390_PFAULT_TOKEN_INVALID)
1836                         kvm_clear_async_pf_completion_queue(vcpu);
1837         }
1838         kvm_run->kvm_dirty_regs = 0;
1839 }
1840
1841 static void store_regs(struct kvm_vcpu *vcpu, struct kvm_run *kvm_run)
1842 {
1843         kvm_run->psw_mask = vcpu->arch.sie_block->gpsw.mask;
1844         kvm_run->psw_addr = vcpu->arch.sie_block->gpsw.addr;
1845         kvm_run->s.regs.prefix = kvm_s390_get_prefix(vcpu);
1846         memcpy(&kvm_run->s.regs.crs, &vcpu->arch.sie_block->gcr, 128);
1847         kvm_run->s.regs.cputm = vcpu->arch.sie_block->cputm;
1848         kvm_run->s.regs.ckc = vcpu->arch.sie_block->ckc;
1849         kvm_run->s.regs.todpr = vcpu->arch.sie_block->todpr;
1850         kvm_run->s.regs.pp = vcpu->arch.sie_block->pp;
1851         kvm_run->s.regs.gbea = vcpu->arch.sie_block->gbea;
1852         kvm_run->s.regs.pft = vcpu->arch.pfault_token;
1853         kvm_run->s.regs.pfs = vcpu->arch.pfault_select;
1854         kvm_run->s.regs.pfc = vcpu->arch.pfault_compare;
1855 }
1856
1857 int kvm_arch_vcpu_ioctl_run(struct kvm_vcpu *vcpu, struct kvm_run *kvm_run)
1858 {
1859         int rc;
1860         sigset_t sigsaved;
1861
1862         if (guestdbg_exit_pending(vcpu)) {
1863                 kvm_s390_prepare_debug_exit(vcpu);
1864                 return 0;
1865         }
1866
1867         if (vcpu->sigset_active)
1868                 sigprocmask(SIG_SETMASK, &vcpu->sigset, &sigsaved);
1869
1870         if (!kvm_s390_user_cpu_state_ctrl(vcpu->kvm)) {
1871                 kvm_s390_vcpu_start(vcpu);
1872         } else if (is_vcpu_stopped(vcpu)) {
1873                 pr_err_ratelimited("kvm-s390: can't run stopped vcpu %d\n",
1874                                    vcpu->vcpu_id);
1875                 return -EINVAL;
1876         }
1877
1878         sync_regs(vcpu, kvm_run);
1879
1880         might_fault();
1881         rc = __vcpu_run(vcpu);
1882
1883         if (signal_pending(current) && !rc) {
1884                 kvm_run->exit_reason = KVM_EXIT_INTR;
1885                 rc = -EINTR;
1886         }
1887
1888         if (guestdbg_exit_pending(vcpu) && !rc)  {
1889                 kvm_s390_prepare_debug_exit(vcpu);
1890                 rc = 0;
1891         }
1892
1893         if (rc == -EOPNOTSUPP) {
1894                 /* intercept cannot be handled in-kernel, prepare kvm-run */
1895                 kvm_run->exit_reason         = KVM_EXIT_S390_SIEIC;
1896                 kvm_run->s390_sieic.icptcode = vcpu->arch.sie_block->icptcode;
1897                 kvm_run->s390_sieic.ipa      = vcpu->arch.sie_block->ipa;
1898                 kvm_run->s390_sieic.ipb      = vcpu->arch.sie_block->ipb;
1899                 rc = 0;
1900         }
1901
1902         if (rc == -EREMOTE) {
1903                 /* intercept was handled, but userspace support is needed
1904                  * kvm_run has been prepared by the handler */
1905                 rc = 0;
1906         }
1907
1908         store_regs(vcpu, kvm_run);
1909
1910         if (vcpu->sigset_active)
1911                 sigprocmask(SIG_SETMASK, &sigsaved, NULL);
1912
1913         vcpu->stat.exit_userspace++;
1914         return rc;
1915 }
1916
1917 /*
1918  * store status at address
1919  * we use have two special cases:
1920  * KVM_S390_STORE_STATUS_NOADDR: -> 0x1200 on 64 bit
1921  * KVM_S390_STORE_STATUS_PREFIXED: -> prefix
1922  */
1923 int kvm_s390_store_status_unloaded(struct kvm_vcpu *vcpu, unsigned long gpa)
1924 {
1925         unsigned char archmode = 1;
1926         unsigned int px;
1927         u64 clkcomp;
1928         int rc;
1929
1930         if (gpa == KVM_S390_STORE_STATUS_NOADDR) {
1931                 if (write_guest_abs(vcpu, 163, &archmode, 1))
1932                         return -EFAULT;
1933                 gpa = SAVE_AREA_BASE;
1934         } else if (gpa == KVM_S390_STORE_STATUS_PREFIXED) {
1935                 if (write_guest_real(vcpu, 163, &archmode, 1))
1936                         return -EFAULT;
1937                 gpa = kvm_s390_real_to_abs(vcpu, SAVE_AREA_BASE);
1938         }
1939         rc = write_guest_abs(vcpu, gpa + offsetof(struct save_area, fp_regs),
1940                              vcpu->arch.guest_fpregs.fprs, 128);
1941         rc |= write_guest_abs(vcpu, gpa + offsetof(struct save_area, gp_regs),
1942                               vcpu->run->s.regs.gprs, 128);
1943         rc |= write_guest_abs(vcpu, gpa + offsetof(struct save_area, psw),
1944                               &vcpu->arch.sie_block->gpsw, 16);
1945         px = kvm_s390_get_prefix(vcpu);
1946         rc |= write_guest_abs(vcpu, gpa + offsetof(struct save_area, pref_reg),
1947                               &px, 4);
1948         rc |= write_guest_abs(vcpu,
1949                               gpa + offsetof(struct save_area, fp_ctrl_reg),
1950                               &vcpu->arch.guest_fpregs.fpc, 4);
1951         rc |= write_guest_abs(vcpu, gpa + offsetof(struct save_area, tod_reg),
1952                               &vcpu->arch.sie_block->todpr, 4);
1953         rc |= write_guest_abs(vcpu, gpa + offsetof(struct save_area, timer),
1954                               &vcpu->arch.sie_block->cputm, 8);
1955         clkcomp = vcpu->arch.sie_block->ckc >> 8;
1956         rc |= write_guest_abs(vcpu, gpa + offsetof(struct save_area, clk_cmp),
1957                               &clkcomp, 8);
1958         rc |= write_guest_abs(vcpu, gpa + offsetof(struct save_area, acc_regs),
1959                               &vcpu->run->s.regs.acrs, 64);
1960         rc |= write_guest_abs(vcpu, gpa + offsetof(struct save_area, ctrl_regs),
1961                               &vcpu->arch.sie_block->gcr, 128);
1962         return rc ? -EFAULT : 0;
1963 }
1964
1965 int kvm_s390_vcpu_store_status(struct kvm_vcpu *vcpu, unsigned long addr)
1966 {
1967         /*
1968          * The guest FPRS and ACRS are in the host FPRS/ACRS due to the lazy
1969          * copying in vcpu load/put. Lets update our copies before we save
1970          * it into the save area
1971          */
1972         save_fp_ctl(&vcpu->arch.guest_fpregs.fpc);
1973         save_fp_regs(vcpu->arch.guest_fpregs.fprs);
1974         save_access_regs(vcpu->run->s.regs.acrs);
1975
1976         return kvm_s390_store_status_unloaded(vcpu, addr);
1977 }
1978
1979 static void __disable_ibs_on_vcpu(struct kvm_vcpu *vcpu)
1980 {
1981         kvm_check_request(KVM_REQ_ENABLE_IBS, vcpu);
1982         kvm_make_request(KVM_REQ_DISABLE_IBS, vcpu);
1983         exit_sie_sync(vcpu);
1984 }
1985
1986 static void __disable_ibs_on_all_vcpus(struct kvm *kvm)
1987 {
1988         unsigned int i;
1989         struct kvm_vcpu *vcpu;
1990
1991         kvm_for_each_vcpu(i, vcpu, kvm) {
1992                 __disable_ibs_on_vcpu(vcpu);
1993         }
1994 }
1995
1996 static void __enable_ibs_on_vcpu(struct kvm_vcpu *vcpu)
1997 {
1998         kvm_check_request(KVM_REQ_DISABLE_IBS, vcpu);
1999         kvm_make_request(KVM_REQ_ENABLE_IBS, vcpu);
2000         exit_sie_sync(vcpu);
2001 }
2002
2003 void kvm_s390_vcpu_start(struct kvm_vcpu *vcpu)
2004 {
2005         int i, online_vcpus, started_vcpus = 0;
2006
2007         if (!is_vcpu_stopped(vcpu))
2008                 return;
2009
2010         trace_kvm_s390_vcpu_start_stop(vcpu->vcpu_id, 1);
2011         /* Only one cpu at a time may enter/leave the STOPPED state. */
2012         spin_lock(&vcpu->kvm->arch.start_stop_lock);
2013         online_vcpus = atomic_read(&vcpu->kvm->online_vcpus);
2014
2015         for (i = 0; i < online_vcpus; i++) {
2016                 if (!is_vcpu_stopped(vcpu->kvm->vcpus[i]))
2017                         started_vcpus++;
2018         }
2019
2020         if (started_vcpus == 0) {
2021                 /* we're the only active VCPU -> speed it up */
2022                 __enable_ibs_on_vcpu(vcpu);
2023         } else if (started_vcpus == 1) {
2024                 /*
2025                  * As we are starting a second VCPU, we have to disable
2026                  * the IBS facility on all VCPUs to remove potentially
2027                  * oustanding ENABLE requests.
2028                  */
2029                 __disable_ibs_on_all_vcpus(vcpu->kvm);
2030         }
2031
2032         atomic_clear_mask(CPUSTAT_STOPPED, &vcpu->arch.sie_block->cpuflags);
2033         /*
2034          * Another VCPU might have used IBS while we were offline.
2035          * Let's play safe and flush the VCPU at startup.
2036          */
2037         kvm_make_request(KVM_REQ_TLB_FLUSH, vcpu);
2038         spin_unlock(&vcpu->kvm->arch.start_stop_lock);
2039         return;
2040 }
2041
2042 void kvm_s390_vcpu_stop(struct kvm_vcpu *vcpu)
2043 {
2044         int i, online_vcpus, started_vcpus = 0;
2045         struct kvm_vcpu *started_vcpu = NULL;
2046
2047         if (is_vcpu_stopped(vcpu))
2048                 return;
2049
2050         trace_kvm_s390_vcpu_start_stop(vcpu->vcpu_id, 0);
2051         /* Only one cpu at a time may enter/leave the STOPPED state. */
2052         spin_lock(&vcpu->kvm->arch.start_stop_lock);
2053         online_vcpus = atomic_read(&vcpu->kvm->online_vcpus);
2054
2055         /* SIGP STOP and SIGP STOP AND STORE STATUS has been fully processed */
2056         kvm_s390_clear_stop_irq(vcpu);
2057
2058         atomic_set_mask(CPUSTAT_STOPPED, &vcpu->arch.sie_block->cpuflags);
2059         __disable_ibs_on_vcpu(vcpu);
2060
2061         for (i = 0; i < online_vcpus; i++) {
2062                 if (!is_vcpu_stopped(vcpu->kvm->vcpus[i])) {
2063                         started_vcpus++;
2064                         started_vcpu = vcpu->kvm->vcpus[i];
2065                 }
2066         }
2067
2068         if (started_vcpus == 1) {
2069                 /*
2070                  * As we only have one VCPU left, we want to enable the
2071                  * IBS facility for that VCPU to speed it up.
2072                  */
2073                 __enable_ibs_on_vcpu(started_vcpu);
2074         }
2075
2076         spin_unlock(&vcpu->kvm->arch.start_stop_lock);
2077         return;
2078 }
2079
2080 static int kvm_vcpu_ioctl_enable_cap(struct kvm_vcpu *vcpu,
2081                                      struct kvm_enable_cap *cap)
2082 {
2083         int r;
2084
2085         if (cap->flags)
2086                 return -EINVAL;
2087
2088         switch (cap->cap) {
2089         case KVM_CAP_S390_CSS_SUPPORT:
2090                 if (!vcpu->kvm->arch.css_support) {
2091                         vcpu->kvm->arch.css_support = 1;
2092                         trace_kvm_s390_enable_css(vcpu->kvm);
2093                 }
2094                 r = 0;
2095                 break;
2096         default:
2097                 r = -EINVAL;
2098                 break;
2099         }
2100         return r;
2101 }
2102
2103 long kvm_arch_vcpu_ioctl(struct file *filp,
2104                          unsigned int ioctl, unsigned long arg)
2105 {
2106         struct kvm_vcpu *vcpu = filp->private_data;
2107         void __user *argp = (void __user *)arg;
2108         int idx;
2109         long r;
2110
2111         switch (ioctl) {
2112         case KVM_S390_INTERRUPT: {
2113                 struct kvm_s390_interrupt s390int;
2114                 struct kvm_s390_irq s390irq;
2115
2116                 r = -EFAULT;
2117                 if (copy_from_user(&s390int, argp, sizeof(s390int)))
2118                         break;
2119                 if (s390int_to_s390irq(&s390int, &s390irq))
2120                         return -EINVAL;
2121                 r = kvm_s390_inject_vcpu(vcpu, &s390irq);
2122                 break;
2123         }
2124         case KVM_S390_STORE_STATUS:
2125                 idx = srcu_read_lock(&vcpu->kvm->srcu);
2126                 r = kvm_s390_vcpu_store_status(vcpu, arg);
2127                 srcu_read_unlock(&vcpu->kvm->srcu, idx);
2128                 break;
2129         case KVM_S390_SET_INITIAL_PSW: {
2130                 psw_t psw;
2131
2132                 r = -EFAULT;
2133                 if (copy_from_user(&psw, argp, sizeof(psw)))
2134                         break;
2135                 r = kvm_arch_vcpu_ioctl_set_initial_psw(vcpu, psw);
2136                 break;
2137         }
2138         case KVM_S390_INITIAL_RESET:
2139                 r = kvm_arch_vcpu_ioctl_initial_reset(vcpu);
2140                 break;
2141         case KVM_SET_ONE_REG:
2142         case KVM_GET_ONE_REG: {
2143                 struct kvm_one_reg reg;
2144                 r = -EFAULT;
2145                 if (copy_from_user(&reg, argp, sizeof(reg)))
2146                         break;
2147                 if (ioctl == KVM_SET_ONE_REG)
2148                         r = kvm_arch_vcpu_ioctl_set_one_reg(vcpu, &reg);
2149                 else
2150                         r = kvm_arch_vcpu_ioctl_get_one_reg(vcpu, &reg);
2151                 break;
2152         }
2153 #ifdef CONFIG_KVM_S390_UCONTROL
2154         case KVM_S390_UCAS_MAP: {
2155                 struct kvm_s390_ucas_mapping ucasmap;
2156
2157                 if (copy_from_user(&ucasmap, argp, sizeof(ucasmap))) {
2158                         r = -EFAULT;
2159                         break;
2160                 }
2161
2162                 if (!kvm_is_ucontrol(vcpu->kvm)) {
2163                         r = -EINVAL;
2164                         break;
2165                 }
2166
2167                 r = gmap_map_segment(vcpu->arch.gmap, ucasmap.user_addr,
2168                                      ucasmap.vcpu_addr, ucasmap.length);
2169                 break;
2170         }
2171         case KVM_S390_UCAS_UNMAP: {
2172                 struct kvm_s390_ucas_mapping ucasmap;
2173
2174                 if (copy_from_user(&ucasmap, argp, sizeof(ucasmap))) {
2175                         r = -EFAULT;
2176                         break;
2177                 }
2178
2179                 if (!kvm_is_ucontrol(vcpu->kvm)) {
2180                         r = -EINVAL;
2181                         break;
2182                 }
2183
2184                 r = gmap_unmap_segment(vcpu->arch.gmap, ucasmap.vcpu_addr,
2185                         ucasmap.length);
2186                 break;
2187         }
2188 #endif
2189         case KVM_S390_VCPU_FAULT: {
2190                 r = gmap_fault(vcpu->arch.gmap, arg, 0);
2191                 break;
2192         }
2193         case KVM_ENABLE_CAP:
2194         {
2195                 struct kvm_enable_cap cap;
2196                 r = -EFAULT;
2197                 if (copy_from_user(&cap, argp, sizeof(cap)))
2198                         break;
2199                 r = kvm_vcpu_ioctl_enable_cap(vcpu, &cap);
2200                 break;
2201         }
2202         default:
2203                 r = -ENOTTY;
2204         }
2205         return r;
2206 }
2207
2208 int kvm_arch_vcpu_fault(struct kvm_vcpu *vcpu, struct vm_fault *vmf)
2209 {
2210 #ifdef CONFIG_KVM_S390_UCONTROL
2211         if ((vmf->pgoff == KVM_S390_SIE_PAGE_OFFSET)
2212                  && (kvm_is_ucontrol(vcpu->kvm))) {
2213                 vmf->page = virt_to_page(vcpu->arch.sie_block);
2214                 get_page(vmf->page);
2215                 return 0;
2216         }
2217 #endif
2218         return VM_FAULT_SIGBUS;
2219 }
2220
2221 int kvm_arch_create_memslot(struct kvm *kvm, struct kvm_memory_slot *slot,
2222                             unsigned long npages)
2223 {
2224         return 0;
2225 }
2226
2227 /* Section: memory related */
2228 int kvm_arch_prepare_memory_region(struct kvm *kvm,
2229                                    struct kvm_memory_slot *memslot,
2230                                    struct kvm_userspace_memory_region *mem,
2231                                    enum kvm_mr_change change)
2232 {
2233         /* A few sanity checks. We can have memory slots which have to be
2234            located/ended at a segment boundary (1MB). The memory in userland is
2235            ok to be fragmented into various different vmas. It is okay to mmap()
2236            and munmap() stuff in this slot after doing this call at any time */
2237
2238         if (mem->userspace_addr & 0xffffful)
2239                 return -EINVAL;
2240
2241         if (mem->memory_size & 0xffffful)
2242                 return -EINVAL;
2243
2244         return 0;
2245 }
2246
2247 void kvm_arch_commit_memory_region(struct kvm *kvm,
2248                                 struct kvm_userspace_memory_region *mem,
2249                                 const struct kvm_memory_slot *old,
2250                                 enum kvm_mr_change change)
2251 {
2252         int rc;
2253
2254         /* If the basics of the memslot do not change, we do not want
2255          * to update the gmap. Every update causes several unnecessary
2256          * segment translation exceptions. This is usually handled just
2257          * fine by the normal fault handler + gmap, but it will also
2258          * cause faults on the prefix page of running guest CPUs.
2259          */
2260         if (old->userspace_addr == mem->userspace_addr &&
2261             old->base_gfn * PAGE_SIZE == mem->guest_phys_addr &&
2262             old->npages * PAGE_SIZE == mem->memory_size)
2263                 return;
2264
2265         rc = gmap_map_segment(kvm->arch.gmap, mem->userspace_addr,
2266                 mem->guest_phys_addr, mem->memory_size);
2267         if (rc)
2268                 printk(KERN_WARNING "kvm-s390: failed to commit memory region\n");
2269         return;
2270 }
2271
2272 static int __init kvm_s390_init(void)
2273 {
2274         return kvm_init(NULL, sizeof(struct kvm_vcpu), 0, THIS_MODULE);
2275 }
2276
2277 static void __exit kvm_s390_exit(void)
2278 {
2279         kvm_exit();
2280 }
2281
2282 module_init(kvm_s390_init);
2283 module_exit(kvm_s390_exit);
2284
2285 /*
2286  * Enable autoloading of the kvm module.
2287  * Note that we add the module alias here instead of virt/kvm/kvm_main.c
2288  * since x86 takes a different approach.
2289  */
2290 #include <linux/miscdevice.h>
2291 MODULE_ALIAS_MISCDEV(KVM_MINOR);
2292 MODULE_ALIAS("devname:kvm");