arch/s390/kvm/kvm-s390.c

   1 /*
   2  * hosting zSeries kernel virtual machines
   3  *
   4  * Copyright IBM Corp. 2008, 2009
   5  *
   6  * This program is free software; you can redistribute it and/or modify
   7  * it under the terms of the GNU General Public License (version 2 only)
   8  * as published by the Free Software Foundation.
   9  *
  10  *    Author(s): Carsten Otte <cotte@de.ibm.com>
  11  *               Christian Borntraeger <borntraeger@de.ibm.com>
  12  *               Heiko Carstens <heiko.carstens@de.ibm.com>
  13  *               Christian Ehrhardt <ehrhardt@de.ibm.com>
  14  *               Jason J. Herne <jjherne@us.ibm.com>
  15  */
  16
  17 #include <linux/compiler.h>
  18 #include <linux/err.h>
  19 #include <linux/fs.h>
  20 #include <linux/hrtimer.h>
  21 #include <linux/init.h>
  22 #include <linux/kvm.h>
  23 #include <linux/kvm_host.h>
  24 #include <linux/mman.h>
  25 #include <linux/module.h>
  26 #include <linux/random.h>
  27 #include <linux/slab.h>
  28 #include <linux/timer.h>
  29 #include <linux/vmalloc.h>
  30 #include <linux/bitmap.h>
  31 #include <asm/asm-offsets.h>
  32 #include <asm/lowcore.h>
  33 #include <asm/stp.h>
  34 #include <asm/pgtable.h>
  35 #include <asm/gmap.h>
  36 #include <asm/nmi.h>
  37 #include <asm/switch_to.h>
  38 #include <asm/isc.h>
  39 #include <asm/sclp.h>
  40 #include <asm/cpacf.h>
  41 #include <asm/timex.h>
  42 #include "kvm-s390.h"
  43 #include "gaccess.h"
  44
  45 #define KMSG_COMPONENT "kvm-s390"
  46 #undef pr_fmt
  47 #define pr_fmt(fmt) KMSG_COMPONENT ": " fmt
  48
  49 #define CREATE_TRACE_POINTS
  50 #include "trace.h"
  51 #include "trace-s390.h"
  52
  53 #define MEM_OP_MAX_SIZE 65536   /* Maximum transfer size for KVM_S390_MEM_OP */
  54 #define LOCAL_IRQS 32
  55 #define VCPU_IRQS_MAX_BUF (sizeof(struct kvm_s390_irq) * \
  56                            (KVM_MAX_VCPUS + LOCAL_IRQS))
  57
  58 #define VCPU_STAT(x) offsetof(struct kvm_vcpu, stat.x), KVM_STAT_VCPU
  59
  60 struct kvm_stats_debugfs_item debugfs_entries[] = {
  61         { "userspace_handled", VCPU_STAT(exit_userspace) },
  62         { "exit_null", VCPU_STAT(exit_null) },
  63         { "exit_validity", VCPU_STAT(exit_validity) },
  64         { "exit_stop_request", VCPU_STAT(exit_stop_request) },
  65         { "exit_external_request", VCPU_STAT(exit_external_request) },
  66         { "exit_external_interrupt", VCPU_STAT(exit_external_interrupt) },
  67         { "exit_instruction", VCPU_STAT(exit_instruction) },
  68         { "exit_pei", VCPU_STAT(exit_pei) },
  69         { "exit_program_interruption", VCPU_STAT(exit_program_interruption) },
  70         { "exit_instr_and_program_int", VCPU_STAT(exit_instr_and_program) },
  71         { "exit_operation_exception", VCPU_STAT(exit_operation_exception) },
  72         { "halt_successful_poll", VCPU_STAT(halt_successful_poll) },
  73         { "halt_attempted_poll", VCPU_STAT(halt_attempted_poll) },
  74         { "halt_poll_invalid", VCPU_STAT(halt_poll_invalid) },
  75         { "halt_wakeup", VCPU_STAT(halt_wakeup) },
  76         { "instruction_lctlg", VCPU_STAT(instruction_lctlg) },
  77         { "instruction_lctl", VCPU_STAT(instruction_lctl) },
  78         { "instruction_stctl", VCPU_STAT(instruction_stctl) },
  79         { "instruction_stctg", VCPU_STAT(instruction_stctg) },
  80         { "deliver_emergency_signal", VCPU_STAT(deliver_emergency_signal) },
  81         { "deliver_external_call", VCPU_STAT(deliver_external_call) },
  82         { "deliver_service_signal", VCPU_STAT(deliver_service_signal) },
  83         { "deliver_virtio_interrupt", VCPU_STAT(deliver_virtio_interrupt) },
  84         { "deliver_stop_signal", VCPU_STAT(deliver_stop_signal) },
  85         { "deliver_prefix_signal", VCPU_STAT(deliver_prefix_signal) },
  86         { "deliver_restart_signal", VCPU_STAT(deliver_restart_signal) },
  87         { "deliver_program_interruption", VCPU_STAT(deliver_program_int) },
  88         { "exit_wait_state", VCPU_STAT(exit_wait_state) },
  89         { "instruction_pfmf", VCPU_STAT(instruction_pfmf) },
  90         { "instruction_stidp", VCPU_STAT(instruction_stidp) },
  91         { "instruction_spx", VCPU_STAT(instruction_spx) },
  92         { "instruction_stpx", VCPU_STAT(instruction_stpx) },
  93         { "instruction_stap", VCPU_STAT(instruction_stap) },
  94         { "instruction_storage_key", VCPU_STAT(instruction_storage_key) },
  95         { "instruction_ipte_interlock", VCPU_STAT(instruction_ipte_interlock) },
  96         { "instruction_stsch", VCPU_STAT(instruction_stsch) },
  97         { "instruction_chsc", VCPU_STAT(instruction_chsc) },
  98         { "instruction_essa", VCPU_STAT(instruction_essa) },
  99         { "instruction_stsi", VCPU_STAT(instruction_stsi) },
 100         { "instruction_stfl", VCPU_STAT(instruction_stfl) },
 101         { "instruction_tprot", VCPU_STAT(instruction_tprot) },
 102         { "instruction_sthyi", VCPU_STAT(instruction_sthyi) },
 103         { "instruction_sie", VCPU_STAT(instruction_sie) },
 104         { "instruction_sigp_sense", VCPU_STAT(instruction_sigp_sense) },
 105         { "instruction_sigp_sense_running", VCPU_STAT(instruction_sigp_sense_running) },
 106         { "instruction_sigp_external_call", VCPU_STAT(instruction_sigp_external_call) },
 107         { "instruction_sigp_emergency", VCPU_STAT(instruction_sigp_emergency) },
 108         { "instruction_sigp_cond_emergency", VCPU_STAT(instruction_sigp_cond_emergency) },
 109         { "instruction_sigp_start", VCPU_STAT(instruction_sigp_start) },
 110         { "instruction_sigp_stop", VCPU_STAT(instruction_sigp_stop) },
 111         { "instruction_sigp_stop_store_status", VCPU_STAT(instruction_sigp_stop_store_status) },
 112         { "instruction_sigp_store_status", VCPU_STAT(instruction_sigp_store_status) },
 113         { "instruction_sigp_store_adtl_status", VCPU_STAT(instruction_sigp_store_adtl_status) },
 114         { "instruction_sigp_set_arch", VCPU_STAT(instruction_sigp_arch) },
 115         { "instruction_sigp_set_prefix", VCPU_STAT(instruction_sigp_prefix) },
 116         { "instruction_sigp_restart", VCPU_STAT(instruction_sigp_restart) },
 117         { "instruction_sigp_cpu_reset", VCPU_STAT(instruction_sigp_cpu_reset) },
 118         { "instruction_sigp_init_cpu_reset", VCPU_STAT(instruction_sigp_init_cpu_reset) },
 119         { "instruction_sigp_unknown", VCPU_STAT(instruction_sigp_unknown) },
 120         { "diagnose_10", VCPU_STAT(diagnose_10) },
 121         { "diagnose_44", VCPU_STAT(diagnose_44) },
 122         { "diagnose_9c", VCPU_STAT(diagnose_9c) },
 123         { "diagnose_258", VCPU_STAT(diagnose_258) },
 124         { "diagnose_308", VCPU_STAT(diagnose_308) },
 125         { "diagnose_500", VCPU_STAT(diagnose_500) },
 126         { NULL }
 127 };
 128
 129 /* allow nested virtualization in KVM (if enabled by user space) */
 130 static int nested;
 131 module_param(nested, int, S_IRUGO);
 132 MODULE_PARM_DESC(nested, "Nested virtualization support");
 133
 134 /* upper facilities limit for kvm */
 135 unsigned long kvm_s390_fac_list_mask[16] = {
 136         0xffe6000000000000UL,
 137         0x005e000000000000UL,
 138 };
 139
 140 unsigned long kvm_s390_fac_list_mask_size(void)
 141 {
 142         BUILD_BUG_ON(ARRAY_SIZE(kvm_s390_fac_list_mask) > S390_ARCH_FAC_MASK_SIZE_U64);
 143         return ARRAY_SIZE(kvm_s390_fac_list_mask);
 144 }
 145
 146 /* available cpu features supported by kvm */
 147 static DECLARE_BITMAP(kvm_s390_available_cpu_feat, KVM_S390_VM_CPU_FEAT_NR_BITS);
 148 /* available subfunctions indicated via query / "test bit" */
 149 static struct kvm_s390_vm_cpu_subfunc kvm_s390_available_subfunc;
 150
 151 static struct gmap_notifier gmap_notifier;
 152 static struct gmap_notifier vsie_gmap_notifier;
 153 debug_info_t *kvm_s390_dbf;
 154
 155 /* Section: not file related */
 156 int kvm_arch_hardware_enable(void)
 157 {
 158         /* every s390 is virtualization enabled ;-) */
 159         return 0;
 160 }
 161
 162 static void kvm_gmap_notifier(struct gmap *gmap, unsigned long start,
 163                               unsigned long end);
 164
 165 /*
 166  * This callback is executed during stop_machine(). All CPUs are therefore
 167  * temporarily stopped. In order not to change guest behavior, we have to
 168  * disable preemption whenever we touch the epoch of kvm and the VCPUs,
 169  * so a CPU won't be stopped while calculating with the epoch.
 170  */
 171 static int kvm_clock_sync(struct notifier_block *notifier, unsigned long val,
 172                           void *v)
 173 {
 174         struct kvm *kvm;
 175         struct kvm_vcpu *vcpu;
 176         int i;
 177         unsigned long long *delta = v;
 178
 179         list_for_each_entry(kvm, &vm_list, vm_list) {
 180                 kvm->arch.epoch -= *delta;
 181                 kvm_for_each_vcpu(i, vcpu, kvm) {
 182                         vcpu->arch.sie_block->epoch -= *delta;
 183                         if (vcpu->arch.cputm_enabled)
 184                                 vcpu->arch.cputm_start += *delta;
 185                         if (vcpu->arch.vsie_block)
 186                                 vcpu->arch.vsie_block->epoch -= *delta;
 187                 }
 188         }
 189         return NOTIFY_OK;
 190 }
 191
 192 static struct notifier_block kvm_clock_notifier = {
 193         .notifier_call = kvm_clock_sync,
 194 };
 195
 196 int kvm_arch_hardware_setup(void)
 197 {
 198         gmap_notifier.notifier_call = kvm_gmap_notifier;
 199         gmap_register_pte_notifier(&gmap_notifier);
 200         vsie_gmap_notifier.notifier_call = kvm_s390_vsie_gmap_notifier;
 201         gmap_register_pte_notifier(&vsie_gmap_notifier);
 202         atomic_notifier_chain_register(&s390_epoch_delta_notifier,
 203                                        &kvm_clock_notifier);
 204         return 0;
 205 }
 206
 207 void kvm_arch_hardware_unsetup(void)
 208 {
 209         gmap_unregister_pte_notifier(&gmap_notifier);
 210         gmap_unregister_pte_notifier(&vsie_gmap_notifier);
 211         atomic_notifier_chain_unregister(&s390_epoch_delta_notifier,
 212                                          &kvm_clock_notifier);
 213 }
 214
 215 static void allow_cpu_feat(unsigned long nr)
 216 {
 217         set_bit_inv(nr, kvm_s390_available_cpu_feat);
 218 }
 219
 220 static inline int plo_test_bit(unsigned char nr)
 221 {
 222         register unsigned long r0 asm("0") = (unsigned long) nr | 0x100;
 223         int cc = 3; /* subfunction not available */
 224
 225         asm volatile(
 226                 /* Parameter registers are ignored for "test bit" */
 227                 "       plo     0,0,0,0(0)\n"
 228                 "       ipm     %0\n"
 229                 "       srl     %0,28\n"
 230                 : "=d" (cc)
 231                 : "d" (r0)
 232                 : "cc");
 233         return cc == 0;
 234 }
 235
 236 static void kvm_s390_cpu_feat_init(void)
 237 {
 238         int i;
 239
 240         for (i = 0; i < 256; ++i) {
 241                 if (plo_test_bit(i))
 242                         kvm_s390_available_subfunc.plo[i >> 3] |= 0x80 >> (i & 7);
 243         }
 244
 245         if (test_facility(28)) /* TOD-clock steering */
 246                 ptff(kvm_s390_available_subfunc.ptff,
 247                      sizeof(kvm_s390_available_subfunc.ptff),
 248                      PTFF_QAF);
 249
 250         if (test_facility(17)) { /* MSA */
 251                 __cpacf_query(CPACF_KMAC, kvm_s390_available_subfunc.kmac);
 252                 __cpacf_query(CPACF_KMC, kvm_s390_available_subfunc.kmc);
 253                 __cpacf_query(CPACF_KM, kvm_s390_available_subfunc.km);
 254                 __cpacf_query(CPACF_KIMD, kvm_s390_available_subfunc.kimd);
 255                 __cpacf_query(CPACF_KLMD, kvm_s390_available_subfunc.klmd);
 256         }
 257         if (test_facility(76)) /* MSA3 */
 258                 __cpacf_query(CPACF_PCKMO, kvm_s390_available_subfunc.pckmo);
 259         if (test_facility(77)) { /* MSA4 */
 260                 __cpacf_query(CPACF_KMCTR, kvm_s390_available_subfunc.kmctr);
 261                 __cpacf_query(CPACF_KMF, kvm_s390_available_subfunc.kmf);
 262                 __cpacf_query(CPACF_KMO, kvm_s390_available_subfunc.kmo);
 263                 __cpacf_query(CPACF_PCC, kvm_s390_available_subfunc.pcc);
 264         }
 265         if (test_facility(57)) /* MSA5 */
 266                 __cpacf_query(CPACF_PPNO, kvm_s390_available_subfunc.ppno);
 267
 268         if (MACHINE_HAS_ESOP)
 269                 allow_cpu_feat(KVM_S390_VM_CPU_FEAT_ESOP);
 270         /*
 271          * We need SIE support, ESOP (PROT_READ protection for gmap_shadow),
 272          * 64bit SCAO (SCA passthrough) and IDTE (for gmap_shadow unshadowing).
 273          */
 274         if (!sclp.has_sief2 || !MACHINE_HAS_ESOP || !sclp.has_64bscao ||
 275             !test_facility(3) || !nested)
 276                 return;
 277         allow_cpu_feat(KVM_S390_VM_CPU_FEAT_SIEF2);
 278         if (sclp.has_64bscao)
 279                 allow_cpu_feat(KVM_S390_VM_CPU_FEAT_64BSCAO);
 280         if (sclp.has_siif)
 281                 allow_cpu_feat(KVM_S390_VM_CPU_FEAT_SIIF);
 282         if (sclp.has_gpere)
 283                 allow_cpu_feat(KVM_S390_VM_CPU_FEAT_GPERE);
 284         if (sclp.has_gsls)
 285                 allow_cpu_feat(KVM_S390_VM_CPU_FEAT_GSLS);
 286         if (sclp.has_ib)
 287                 allow_cpu_feat(KVM_S390_VM_CPU_FEAT_IB);
 288         if (sclp.has_cei)
 289                 allow_cpu_feat(KVM_S390_VM_CPU_FEAT_CEI);
 290         if (sclp.has_ibs)
 291                 allow_cpu_feat(KVM_S390_VM_CPU_FEAT_IBS);
 292         /*
 293          * KVM_S390_VM_CPU_FEAT_SKEY: Wrong shadow of PTE.I bits will make
 294          * all skey handling functions read/set the skey from the PGSTE
 295          * instead of the real storage key.
 296          *
 297          * KVM_S390_VM_CPU_FEAT_CMMA: Wrong shadow of PTE.I bits will make
 298          * pages being detected as preserved although they are resident.
 299          *
 300          * KVM_S390_VM_CPU_FEAT_PFMFI: Wrong shadow of PTE.I bits will
 301          * have the same effect as for KVM_S390_VM_CPU_FEAT_SKEY.
 302          *
 303          * For KVM_S390_VM_CPU_FEAT_SKEY, KVM_S390_VM_CPU_FEAT_CMMA and
 304          * KVM_S390_VM_CPU_FEAT_PFMFI, all PTE.I and PGSTE bits have to be
 305          * correctly shadowed. We can do that for the PGSTE but not for PTE.I.
 306          *
 307          * KVM_S390_VM_CPU_FEAT_SIGPIF: Wrong SCB addresses in the SCA. We
 308          * cannot easily shadow the SCA because of the ipte lock.
 309          */
 310 }
 311
 312 int kvm_arch_init(void *opaque)
 313 {
 314         kvm_s390_dbf = debug_register("kvm-trace", 32, 1, 7 * sizeof(long));
 315         if (!kvm_s390_dbf)
 316                 return -ENOMEM;
 317
 318         if (debug_register_view(kvm_s390_dbf, &debug_sprintf_view)) {
 319                 debug_unregister(kvm_s390_dbf);
 320                 return -ENOMEM;
 321         }
 322
 323         kvm_s390_cpu_feat_init();
 324
 325         /* Register floating interrupt controller interface. */
 326         return kvm_register_device_ops(&kvm_flic_ops, KVM_DEV_TYPE_FLIC);
 327 }
 328
 329 void kvm_arch_exit(void)
 330 {
 331         debug_unregister(kvm_s390_dbf);
 332 }
 333
 334 /* Section: device related */
 335 long kvm_arch_dev_ioctl(struct file *filp,
 336                         unsigned int ioctl, unsigned long arg)
 337 {
 338         if (ioctl == KVM_S390_ENABLE_SIE)
 339                 return s390_enable_sie();
 340         return -EINVAL;
 341 }
 342
 343 int kvm_vm_ioctl_check_extension(struct kvm *kvm, long ext)
 344 {
 345         int r;
 346
 347         switch (ext) {
 348         case KVM_CAP_S390_PSW:
 349         case KVM_CAP_S390_GMAP:
 350         case KVM_CAP_SYNC_MMU:
 351 #ifdef CONFIG_KVM_S390_UCONTROL
 352         case KVM_CAP_S390_UCONTROL:
 353 #endif
 354         case KVM_CAP_ASYNC_PF:
 355         case KVM_CAP_SYNC_REGS:
 356         case KVM_CAP_ONE_REG:
 357         case KVM_CAP_ENABLE_CAP:
 358         case KVM_CAP_S390_CSS_SUPPORT:
 359         case KVM_CAP_IOEVENTFD:
 360         case KVM_CAP_DEVICE_CTRL:
 361         case KVM_CAP_ENABLE_CAP_VM:
 362         case KVM_CAP_S390_IRQCHIP:
 363         case KVM_CAP_VM_ATTRIBUTES:
 364         case KVM_CAP_MP_STATE:
 365         case KVM_CAP_S390_INJECT_IRQ:
 366         case KVM_CAP_S390_USER_SIGP:
 367         case KVM_CAP_S390_USER_STSI:
 368         case KVM_CAP_S390_SKEYS:
 369         case KVM_CAP_S390_IRQ_STATE:
 370         case KVM_CAP_S390_USER_INSTR0:
 371                 r = 1;
 372                 break;
 373         case KVM_CAP_S390_MEM_OP:
 374                 r = MEM_OP_MAX_SIZE;
 375                 break;
 376         case KVM_CAP_NR_VCPUS:
 377         case KVM_CAP_MAX_VCPUS:
 378                 r = KVM_S390_BSCA_CPU_SLOTS;
 379                 if (sclp.has_esca && sclp.has_64bscao)
 380                         r = KVM_S390_ESCA_CPU_SLOTS;
 381                 break;
 382         case KVM_CAP_NR_MEMSLOTS:
 383                 r = KVM_USER_MEM_SLOTS;
 384                 break;
 385         case KVM_CAP_S390_COW:
 386                 r = MACHINE_HAS_ESOP;
 387                 break;
 388         case KVM_CAP_S390_VECTOR_REGISTERS:
 389                 r = MACHINE_HAS_VX;
 390                 break;
 391         case KVM_CAP_S390_RI:
 392                 r = test_facility(64);
 393                 break;
 394         default:
 395                 r = 0;
 396         }
 397         return r;
 398 }
 399
 400 static void kvm_s390_sync_dirty_log(struct kvm *kvm,
 401                                         struct kvm_memory_slot *memslot)
 402 {
 403         gfn_t cur_gfn, last_gfn;
 404         unsigned long address;
 405         struct gmap *gmap = kvm->arch.gmap;
 406
 407         /* Loop over all guest pages */
 408         last_gfn = memslot->base_gfn + memslot->npages;
 409         for (cur_gfn = memslot->base_gfn; cur_gfn <= last_gfn; cur_gfn++) {
 410                 address = gfn_to_hva_memslot(memslot, cur_gfn);
 411
 412                 if (test_and_clear_guest_dirty(gmap->mm, address))
 413                         mark_page_dirty(kvm, cur_gfn);
 414                 if (fatal_signal_pending(current))
 415                         return;
 416                 cond_resched();
 417         }
 418 }
 419
 420 /* Section: vm related */
 421 static void sca_del_vcpu(struct kvm_vcpu *vcpu);
 422
 423 /*
 424  * Get (and clear) the dirty memory log for a memory slot.
 425  */
 426 int kvm_vm_ioctl_get_dirty_log(struct kvm *kvm,
 427                                struct kvm_dirty_log *log)
 428 {
 429         int r;
 430         unsigned long n;
 431         struct kvm_memslots *slots;
 432         struct kvm_memory_slot *memslot;
 433         int is_dirty = 0;
 434
 435         mutex_lock(&kvm->slots_lock);
 436
 437         r = -EINVAL;
 438         if (log->slot >= KVM_USER_MEM_SLOTS)
 439                 goto out;
 440
 441         slots = kvm_memslots(kvm);
 442         memslot = id_to_memslot(slots, log->slot);
 443         r = -ENOENT;
 444         if (!memslot->dirty_bitmap)
 445                 goto out;
 446
 447         kvm_s390_sync_dirty_log(kvm, memslot);
 448         r = kvm_get_dirty_log(kvm, log, &is_dirty);
 449         if (r)
 450                 goto out;
 451
 452         /* Clear the dirty log */
 453         if (is_dirty) {
 454                 n = kvm_dirty_bitmap_bytes(memslot);
 455                 memset(memslot->dirty_bitmap, 0, n);
 456         }
 457         r = 0;
 458 out:
 459         mutex_unlock(&kvm->slots_lock);
 460         return r;
 461 }
 462
 463 static void icpt_operexc_on_all_vcpus(struct kvm *kvm)
 464 {
 465         unsigned int i;
 466         struct kvm_vcpu *vcpu;
 467
 468         kvm_for_each_vcpu(i, vcpu, kvm) {
 469                 kvm_s390_sync_request(KVM_REQ_ICPT_OPEREXC, vcpu);
 470         }
 471 }
 472
 473 static int kvm_vm_ioctl_enable_cap(struct kvm *kvm, struct kvm_enable_cap *cap)
 474 {
 475         int r;
 476
 477         if (cap->flags)
 478                 return -EINVAL;
 479
 480         switch (cap->cap) {
 481         case KVM_CAP_S390_IRQCHIP:
 482                 VM_EVENT(kvm, 3, "%s", "ENABLE: CAP_S390_IRQCHIP");
 483                 kvm->arch.use_irqchip = 1;
 484                 r = 0;
 485                 break;
 486         case KVM_CAP_S390_USER_SIGP:
 487                 VM_EVENT(kvm, 3, "%s", "ENABLE: CAP_S390_USER_SIGP");
 488                 kvm->arch.user_sigp = 1;
 489                 r = 0;
 490                 break;
 491         case KVM_CAP_S390_VECTOR_REGISTERS:
 492                 mutex_lock(&kvm->lock);
 493                 if (kvm->created_vcpus) {
 494                         r = -EBUSY;
 495                 } else if (MACHINE_HAS_VX) {
 496                         set_kvm_facility(kvm->arch.model.fac_mask, 129);
 497                         set_kvm_facility(kvm->arch.model.fac_list, 129);
 498                         r = 0;
 499                 } else
 500                         r = -EINVAL;
 501                 mutex_unlock(&kvm->lock);
 502                 VM_EVENT(kvm, 3, "ENABLE: CAP_S390_VECTOR_REGISTERS %s",
 503                          r ? "(not available)" : "(success)");
 504                 break;
 505         case KVM_CAP_S390_RI:
 506                 r = -EINVAL;
 507                 mutex_lock(&kvm->lock);
 508                 if (kvm->created_vcpus) {
 509                         r = -EBUSY;
 510                 } else if (test_facility(64)) {
 511                         set_kvm_facility(kvm->arch.model.fac_mask, 64);
 512                         set_kvm_facility(kvm->arch.model.fac_list, 64);
 513                         r = 0;
 514                 }
 515                 mutex_unlock(&kvm->lock);
 516                 VM_EVENT(kvm, 3, "ENABLE: CAP_S390_RI %s",
 517                          r ? "(not available)" : "(success)");
 518                 break;
 519         case KVM_CAP_S390_USER_STSI:
 520                 VM_EVENT(kvm, 3, "%s", "ENABLE: CAP_S390_USER_STSI");
 521                 kvm->arch.user_stsi = 1;
 522                 r = 0;
 523                 break;
 524         case KVM_CAP_S390_USER_INSTR0:
 525                 VM_EVENT(kvm, 3, "%s", "ENABLE: CAP_S390_USER_INSTR0");
 526                 kvm->arch.user_instr0 = 1;
 527                 icpt_operexc_on_all_vcpus(kvm);
 528                 r = 0;
 529                 break;
 530         default:
 531                 r = -EINVAL;
 532                 break;
 533         }
 534         return r;
 535 }
 536
 537 static int kvm_s390_get_mem_control(struct kvm *kvm, struct kvm_device_attr *attr)
 538 {
 539         int ret;
 540
 541         switch (attr->attr) {
 542         case KVM_S390_VM_MEM_LIMIT_SIZE:
 543                 ret = 0;
 544                 VM_EVENT(kvm, 3, "QUERY: max guest memory: %lu bytes",
 545                          kvm->arch.mem_limit);
 546                 if (put_user(kvm->arch.mem_limit, (u64 __user *)attr->addr))
 547                         ret = -EFAULT;
 548                 break;
 549         default:
 550                 ret = -ENXIO;
 551                 break;
 552         }
 553         return ret;
 554 }
 555
 556 static int kvm_s390_set_mem_control(struct kvm *kvm, struct kvm_device_attr *attr)
 557 {
 558         int ret;
 559         unsigned int idx;
 560         switch (attr->attr) {
 561         case KVM_S390_VM_MEM_ENABLE_CMMA:
 562                 ret = -ENXIO;
 563                 if (!sclp.has_cmma)
 564                         break;
 565
 566                 ret = -EBUSY;
 567                 VM_EVENT(kvm, 3, "%s", "ENABLE: CMMA support");
 568                 mutex_lock(&kvm->lock);
 569                 if (!kvm->created_vcpus) {
 570                         kvm->arch.use_cmma = 1;
 571                         ret = 0;
 572                 }
 573                 mutex_unlock(&kvm->lock);
 574                 break;
 575         case KVM_S390_VM_MEM_CLR_CMMA:
 576                 ret = -ENXIO;
 577                 if (!sclp.has_cmma)
 578                         break;
 579                 ret = -EINVAL;
 580                 if (!kvm->arch.use_cmma)
 581                         break;
 582
 583                 VM_EVENT(kvm, 3, "%s", "RESET: CMMA states");
 584                 mutex_lock(&kvm->lock);
 585                 idx = srcu_read_lock(&kvm->srcu);
 586                 s390_reset_cmma(kvm->arch.gmap->mm);
 587                 srcu_read_unlock(&kvm->srcu, idx);
 588                 mutex_unlock(&kvm->lock);
 589                 ret = 0;
 590                 break;
 591         case KVM_S390_VM_MEM_LIMIT_SIZE: {
 592                 unsigned long new_limit;
 593
 594                 if (kvm_is_ucontrol(kvm))
 595                         return -EINVAL;
 596
 597                 if (get_user(new_limit, (u64 __user *)attr->addr))
 598                         return -EFAULT;
 599
 600                 if (kvm->arch.mem_limit != KVM_S390_NO_MEM_LIMIT &&
 601                     new_limit > kvm->arch.mem_limit)
 602                         return -E2BIG;
 603
 604                 if (!new_limit)
 605                         return -EINVAL;
 606
 607                 /* gmap_create takes last usable address */
 608                 if (new_limit != KVM_S390_NO_MEM_LIMIT)
 609                         new_limit -= 1;
 610
 611                 ret = -EBUSY;
 612                 mutex_lock(&kvm->lock);
 613                 if (!kvm->created_vcpus) {
 614                         /* gmap_create will round the limit up */
 615                         struct gmap *new = gmap_create(current->mm, new_limit);
 616
 617                         if (!new) {
 618                                 ret = -ENOMEM;
 619                         } else {
 620                                 gmap_remove(kvm->arch.gmap);
 621                                 new->private = kvm;
 622                                 kvm->arch.gmap = new;
 623                                 ret = 0;
 624                         }
 625                 }
 626                 mutex_unlock(&kvm->lock);
 627                 VM_EVENT(kvm, 3, "SET: max guest address: %lu", new_limit);
 628                 VM_EVENT(kvm, 3, "New guest asce: 0x%pK",
 629                          (void *) kvm->arch.gmap->asce);
 630                 break;
 631         }
 632         default:
 633                 ret = -ENXIO;
 634                 break;
 635         }
 636         return ret;
 637 }
 638
 639 static void kvm_s390_vcpu_crypto_setup(struct kvm_vcpu *vcpu);
 640
 641 static int kvm_s390_vm_set_crypto(struct kvm *kvm, struct kvm_device_attr *attr)
 642 {
 643         struct kvm_vcpu *vcpu;
 644         int i;
 645
 646         if (!test_kvm_facility(kvm, 76))
 647                 return -EINVAL;
 648
 649         mutex_lock(&kvm->lock);
 650         switch (attr->attr) {
 651         case KVM_S390_VM_CRYPTO_ENABLE_AES_KW:
 652                 get_random_bytes(
 653                         kvm->arch.crypto.crycb->aes_wrapping_key_mask,
 654                         sizeof(kvm->arch.crypto.crycb->aes_wrapping_key_mask));
 655                 kvm->arch.crypto.aes_kw = 1;
 656                 VM_EVENT(kvm, 3, "%s", "ENABLE: AES keywrapping support");
 657                 break;
 658         case KVM_S390_VM_CRYPTO_ENABLE_DEA_KW:
 659                 get_random_bytes(
 660                         kvm->arch.crypto.crycb->dea_wrapping_key_mask,
 661                         sizeof(kvm->arch.crypto.crycb->dea_wrapping_key_mask));
 662                 kvm->arch.crypto.dea_kw = 1;
 663                 VM_EVENT(kvm, 3, "%s", "ENABLE: DEA keywrapping support");
 664                 break;
 665         case KVM_S390_VM_CRYPTO_DISABLE_AES_KW:
 666                 kvm->arch.crypto.aes_kw = 0;
 667                 memset(kvm->arch.crypto.crycb->aes_wrapping_key_mask, 0,
 668                         sizeof(kvm->arch.crypto.crycb->aes_wrapping_key_mask));
 669                 VM_EVENT(kvm, 3, "%s", "DISABLE: AES keywrapping support");
 670                 break;
 671         case KVM_S390_VM_CRYPTO_DISABLE_DEA_KW:
 672                 kvm->arch.crypto.dea_kw = 0;
 673                 memset(kvm->arch.crypto.crycb->dea_wrapping_key_mask, 0,
 674                         sizeof(kvm->arch.crypto.crycb->dea_wrapping_key_mask));
 675                 VM_EVENT(kvm, 3, "%s", "DISABLE: DEA keywrapping support");
 676                 break;
 677         default:
 678                 mutex_unlock(&kvm->lock);
 679                 return -ENXIO;
 680         }
 681
 682         kvm_for_each_vcpu(i, vcpu, kvm) {
 683                 kvm_s390_vcpu_crypto_setup(vcpu);
 684                 exit_sie(vcpu);
 685         }
 686         mutex_unlock(&kvm->lock);
 687         return 0;
 688 }
 689
 690 static int kvm_s390_set_tod_high(struct kvm *kvm, struct kvm_device_attr *attr)
 691 {
 692         u8 gtod_high;
 693
 694         if (copy_from_user(&gtod_high, (void __user *)attr->addr,
 695                                            sizeof(gtod_high)))
 696                 return -EFAULT;
 697
 698         if (gtod_high != 0)
 699                 return -EINVAL;
 700         VM_EVENT(kvm, 3, "SET: TOD extension: 0x%x", gtod_high);
 701
 702         return 0;
 703 }
 704
 705 static int kvm_s390_set_tod_low(struct kvm *kvm, struct kvm_device_attr *attr)
 706 {
 707         u64 gtod;
 708
 709         if (copy_from_user(&gtod, (void __user *)attr->addr, sizeof(gtod)))
 710                 return -EFAULT;
 711
 712         kvm_s390_set_tod_clock(kvm, gtod);
 713         VM_EVENT(kvm, 3, "SET: TOD base: 0x%llx", gtod);
 714         return 0;
 715 }
 716
 717 static int kvm_s390_set_tod(struct kvm *kvm, struct kvm_device_attr *attr)
 718 {
 719         int ret;
 720
 721         if (attr->flags)
 722                 return -EINVAL;
 723
 724         switch (attr->attr) {
 725         case KVM_S390_VM_TOD_HIGH:
 726                 ret = kvm_s390_set_tod_high(kvm, attr);
 727                 break;
 728         case KVM_S390_VM_TOD_LOW:
 729                 ret = kvm_s390_set_tod_low(kvm, attr);
 730                 break;
 731         default:
 732                 ret = -ENXIO;
 733                 break;
 734         }
 735         return ret;
 736 }
 737
 738 static int kvm_s390_get_tod_high(struct kvm *kvm, struct kvm_device_attr *attr)
 739 {
 740         u8 gtod_high = 0;
 741
 742         if (copy_to_user((void __user *)attr->addr, &gtod_high,
 743                                          sizeof(gtod_high)))
 744                 return -EFAULT;
 745         VM_EVENT(kvm, 3, "QUERY: TOD extension: 0x%x", gtod_high);
 746
 747         return 0;
 748 }
 749
 750 static int kvm_s390_get_tod_low(struct kvm *kvm, struct kvm_device_attr *attr)
 751 {
 752         u64 gtod;
 753
 754         gtod = kvm_s390_get_tod_clock_fast(kvm);
 755         if (copy_to_user((void __user *)attr->addr, &gtod, sizeof(gtod)))
 756                 return -EFAULT;
 757         VM_EVENT(kvm, 3, "QUERY: TOD base: 0x%llx", gtod);
 758
 759         return 0;
 760 }
 761
 762 static int kvm_s390_get_tod(struct kvm *kvm, struct kvm_device_attr *attr)
 763 {
 764         int ret;
 765
 766         if (attr->flags)
 767                 return -EINVAL;
 768
 769         switch (attr->attr) {
 770         case KVM_S390_VM_TOD_HIGH:
 771                 ret = kvm_s390_get_tod_high(kvm, attr);
 772                 break;
 773         case KVM_S390_VM_TOD_LOW:
 774                 ret = kvm_s390_get_tod_low(kvm, attr);
 775                 break;
 776         default:
 777                 ret = -ENXIO;
 778                 break;
 779         }
 780         return ret;
 781 }
 782
 783 static int kvm_s390_set_processor(struct kvm *kvm, struct kvm_device_attr *attr)
 784 {
 785         struct kvm_s390_vm_cpu_processor *proc;
 786         u16 lowest_ibc, unblocked_ibc;
 787         int ret = 0;
 788
 789         mutex_lock(&kvm->lock);
 790         if (kvm->created_vcpus) {
 791                 ret = -EBUSY;
 792                 goto out;
 793         }
 794         proc = kzalloc(sizeof(*proc), GFP_KERNEL);
 795         if (!proc) {
 796                 ret = -ENOMEM;
 797                 goto out;
 798         }
 799         if (!copy_from_user(proc, (void __user *)attr->addr,
 800                             sizeof(*proc))) {
 801                 kvm->arch.model.cpuid = proc->cpuid;
 802                 lowest_ibc = sclp.ibc >> 16 & 0xfff;
 803                 unblocked_ibc = sclp.ibc & 0xfff;
 804                 if (lowest_ibc && proc->ibc) {
 805                         if (proc->ibc > unblocked_ibc)
 806                                 kvm->arch.model.ibc = unblocked_ibc;
 807                         else if (proc->ibc < lowest_ibc)
 808                                 kvm->arch.model.ibc = lowest_ibc;
 809                         else
 810                                 kvm->arch.model.ibc = proc->ibc;
 811                 }
 812                 memcpy(kvm->arch.model.fac_list, proc->fac_list,
 813                        S390_ARCH_FAC_LIST_SIZE_BYTE);
 814         } else
 815                 ret = -EFAULT;
 816         kfree(proc);
 817 out:
 818         mutex_unlock(&kvm->lock);
 819         return ret;
 820 }
 821
 822 static int kvm_s390_set_processor_feat(struct kvm *kvm,
 823                                        struct kvm_device_attr *attr)
 824 {
 825         struct kvm_s390_vm_cpu_feat data;
 826         int ret = -EBUSY;
 827
 828         if (copy_from_user(&data, (void __user *)attr->addr, sizeof(data)))
 829                 return -EFAULT;
 830         if (!bitmap_subset((unsigned long *) data.feat,
 831                            kvm_s390_available_cpu_feat,
 832                            KVM_S390_VM_CPU_FEAT_NR_BITS))
 833                 return -EINVAL;
 834
 835         mutex_lock(&kvm->lock);
 836         if (!atomic_read(&kvm->online_vcpus)) {
 837                 bitmap_copy(kvm->arch.cpu_feat, (unsigned long *) data.feat,
 838                             KVM_S390_VM_CPU_FEAT_NR_BITS);
 839                 ret = 0;
 840         }
 841         mutex_unlock(&kvm->lock);
 842         return ret;
 843 }
 844
 845 static int kvm_s390_set_processor_subfunc(struct kvm *kvm,
 846                                           struct kvm_device_attr *attr)
 847 {
 848         /*
 849          * Once supported by kernel + hw, we have to store the subfunctions
 850          * in kvm->arch and remember that user space configured them.
 851          */
 852         return -ENXIO;
 853 }
 854
 855 static int kvm_s390_set_cpu_model(struct kvm *kvm, struct kvm_device_attr *attr)
 856 {
 857         int ret = -ENXIO;
 858
 859         switch (attr->attr) {
 860         case KVM_S390_VM_CPU_PROCESSOR:
 861                 ret = kvm_s390_set_processor(kvm, attr);
 862                 break;
 863         case KVM_S390_VM_CPU_PROCESSOR_FEAT:
 864                 ret = kvm_s390_set_processor_feat(kvm, attr);
 865                 break;
 866         case KVM_S390_VM_CPU_PROCESSOR_SUBFUNC:
 867                 ret = kvm_s390_set_processor_subfunc(kvm, attr);
 868                 break;
 869         }
 870         return ret;
 871 }
 872
 873 static int kvm_s390_get_processor(struct kvm *kvm, struct kvm_device_attr *attr)
 874 {
 875         struct kvm_s390_vm_cpu_processor *proc;
 876         int ret = 0;
 877
 878         proc = kzalloc(sizeof(*proc), GFP_KERNEL);
 879         if (!proc) {
 880                 ret = -ENOMEM;
 881                 goto out;
 882         }
 883         proc->cpuid = kvm->arch.model.cpuid;
 884         proc->ibc = kvm->arch.model.ibc;
 885         memcpy(&proc->fac_list, kvm->arch.model.fac_list,
 886                S390_ARCH_FAC_LIST_SIZE_BYTE);
 887         if (copy_to_user((void __user *)attr->addr, proc, sizeof(*proc)))
 888                 ret = -EFAULT;
 889         kfree(proc);
 890 out:
 891         return ret;
 892 }
 893
 894 static int kvm_s390_get_machine(struct kvm *kvm, struct kvm_device_attr *attr)
 895 {
 896         struct kvm_s390_vm_cpu_machine *mach;
 897         int ret = 0;
 898
 899         mach = kzalloc(sizeof(*mach), GFP_KERNEL);
 900         if (!mach) {
 901                 ret = -ENOMEM;
 902                 goto out;
 903         }
 904         get_cpu_id((struct cpuid *) &mach->cpuid);
 905         mach->ibc = sclp.ibc;
 906         memcpy(&mach->fac_mask, kvm->arch.model.fac_mask,
 907                S390_ARCH_FAC_LIST_SIZE_BYTE);
 908         memcpy((unsigned long *)&mach->fac_list, S390_lowcore.stfle_fac_list,
 909                S390_ARCH_FAC_LIST_SIZE_BYTE);
 910         if (copy_to_user((void __user *)attr->addr, mach, sizeof(*mach)))
 911                 ret = -EFAULT;
 912         kfree(mach);
 913 out:
 914         return ret;
 915 }
 916
 917 static int kvm_s390_get_processor_feat(struct kvm *kvm,
 918                                        struct kvm_device_attr *attr)
 919 {
 920         struct kvm_s390_vm_cpu_feat data;
 921
 922         bitmap_copy((unsigned long *) data.feat, kvm->arch.cpu_feat,
 923                     KVM_S390_VM_CPU_FEAT_NR_BITS);
 924         if (copy_to_user((void __user *)attr->addr, &data, sizeof(data)))
 925                 return -EFAULT;
 926         return 0;
 927 }
 928
 929 static int kvm_s390_get_machine_feat(struct kvm *kvm,
 930                                      struct kvm_device_attr *attr)
 931 {
 932         struct kvm_s390_vm_cpu_feat data;
 933
 934         bitmap_copy((unsigned long *) data.feat,
 935                     kvm_s390_available_cpu_feat,
 936                     KVM_S390_VM_CPU_FEAT_NR_BITS);
 937         if (copy_to_user((void __user *)attr->addr, &data, sizeof(data)))
 938                 return -EFAULT;
 939         return 0;
 940 }
 941
 942 static int kvm_s390_get_processor_subfunc(struct kvm *kvm,
 943                                           struct kvm_device_attr *attr)
 944 {
 945         /*
 946          * Once we can actually configure subfunctions (kernel + hw support),
 947          * we have to check if they were already set by user space, if so copy
 948          * them from kvm->arch.
 949          */
 950         return -ENXIO;
 951 }
 952
 953 static int kvm_s390_get_machine_subfunc(struct kvm *kvm,
 954                                         struct kvm_device_attr *attr)
 955 {
 956         if (copy_to_user((void __user *)attr->addr, &kvm_s390_available_subfunc,
 957             sizeof(struct kvm_s390_vm_cpu_subfunc)))
 958                 return -EFAULT;
 959         return 0;
 960 }
 961 static int kvm_s390_get_cpu_model(struct kvm *kvm, struct kvm_device_attr *attr)
 962 {
 963         int ret = -ENXIO;
 964
 965         switch (attr->attr) {
 966         case KVM_S390_VM_CPU_PROCESSOR:
 967                 ret = kvm_s390_get_processor(kvm, attr);
 968                 break;
 969         case KVM_S390_VM_CPU_MACHINE:
 970                 ret = kvm_s390_get_machine(kvm, attr);
 971                 break;
 972         case KVM_S390_VM_CPU_PROCESSOR_FEAT:
 973                 ret = kvm_s390_get_processor_feat(kvm, attr);
 974                 break;
 975         case KVM_S390_VM_CPU_MACHINE_FEAT:
 976                 ret = kvm_s390_get_machine_feat(kvm, attr);
 977                 break;
 978         case KVM_S390_VM_CPU_PROCESSOR_SUBFUNC:
 979                 ret = kvm_s390_get_processor_subfunc(kvm, attr);
 980                 break;
 981         case KVM_S390_VM_CPU_MACHINE_SUBFUNC:
 982                 ret = kvm_s390_get_machine_subfunc(kvm, attr);
 983                 break;
 984         }
 985         return ret;
 986 }
 987
 988 static int kvm_s390_vm_set_attr(struct kvm *kvm, struct kvm_device_attr *attr)
 989 {
 990         int ret;
 991
 992         switch (attr->group) {
 993         case KVM_S390_VM_MEM_CTRL:
 994                 ret = kvm_s390_set_mem_control(kvm, attr);
 995                 break;
 996         case KVM_S390_VM_TOD:
 997                 ret = kvm_s390_set_tod(kvm, attr);
 998                 break;
 999         case KVM_S390_VM_CPU_MODEL:
1000                 ret = kvm_s390_set_cpu_model(kvm, attr);
1001                 break;
1002         case KVM_S390_VM_CRYPTO:
1003                 ret = kvm_s390_vm_set_crypto(kvm, attr);
1004                 break;
1005         default:
1006                 ret = -ENXIO;
1007                 break;
1008         }
1009
1010         return ret;
1011 }
1012
1013 static int kvm_s390_vm_get_attr(struct kvm *kvm, struct kvm_device_attr *attr)
1014 {
1015         int ret;
1016
1017         switch (attr->group) {
1018         case KVM_S390_VM_MEM_CTRL:
1019                 ret = kvm_s390_get_mem_control(kvm, attr);
1020                 break;
1021         case KVM_S390_VM_TOD:
1022                 ret = kvm_s390_get_tod(kvm, attr);
1023                 break;
1024         case KVM_S390_VM_CPU_MODEL:
1025                 ret = kvm_s390_get_cpu_model(kvm, attr);
1026                 break;
1027         default:
1028                 ret = -ENXIO;
1029                 break;
1030         }
1031
1032         return ret;
1033 }
1034
1035 static int kvm_s390_vm_has_attr(struct kvm *kvm, struct kvm_device_attr *attr)
1036 {
1037         int ret;
1038
1039         switch (attr->group) {
1040         case KVM_S390_VM_MEM_CTRL:
1041                 switch (attr->attr) {
1042                 case KVM_S390_VM_MEM_ENABLE_CMMA:
1043                 case KVM_S390_VM_MEM_CLR_CMMA:
1044                         ret = sclp.has_cmma ? 0 : -ENXIO;
1045                         break;
1046                 case KVM_S390_VM_MEM_LIMIT_SIZE:
1047                         ret = 0;
1048                         break;
1049                 default:
1050                         ret = -ENXIO;
1051                         break;
1052                 }
1053                 break;
1054         case KVM_S390_VM_TOD:
1055                 switch (attr->attr) {
1056                 case KVM_S390_VM_TOD_LOW:
1057                 case KVM_S390_VM_TOD_HIGH:
1058                         ret = 0;
1059                         break;
1060                 default:
1061                         ret = -ENXIO;
1062                         break;
1063                 }
1064                 break;
1065         case KVM_S390_VM_CPU_MODEL:
1066                 switch (attr->attr) {
1067                 case KVM_S390_VM_CPU_PROCESSOR:
1068                 case KVM_S390_VM_CPU_MACHINE:
1069                 case KVM_S390_VM_CPU_PROCESSOR_FEAT:
1070                 case KVM_S390_VM_CPU_MACHINE_FEAT:
1071                 case KVM_S390_VM_CPU_MACHINE_SUBFUNC:
1072                         ret = 0;
1073                         break;
1074                 /* configuring subfunctions is not supported yet */
1075                 case KVM_S390_VM_CPU_PROCESSOR_SUBFUNC:
1076                 default:
1077                         ret = -ENXIO;
1078                         break;
1079                 }
1080                 break;
1081         case KVM_S390_VM_CRYPTO:
1082                 switch (attr->attr) {
1083                 case KVM_S390_VM_CRYPTO_ENABLE_AES_KW:
1084                 case KVM_S390_VM_CRYPTO_ENABLE_DEA_KW:
1085                 case KVM_S390_VM_CRYPTO_DISABLE_AES_KW:
1086                 case KVM_S390_VM_CRYPTO_DISABLE_DEA_KW:
1087                         ret = 0;
1088                         break;
1089                 default:
1090                         ret = -ENXIO;
1091                         break;
1092                 }
1093                 break;
1094         default:
1095                 ret = -ENXIO;
1096                 break;
1097         }
1098
1099         return ret;
1100 }
1101
1102 static long kvm_s390_get_skeys(struct kvm *kvm, struct kvm_s390_skeys *args)
1103 {
1104         uint8_t *keys;
1105         uint64_t hva;
1106         int i, r = 0;
1107
1108         if (args->flags != 0)
1109                 return -EINVAL;
1110
1111         /* Is this guest using storage keys? */
1112         if (!mm_use_skey(current->mm))
1113                 return KVM_S390_GET_SKEYS_NONE;
1114
1115         /* Enforce sane limit on memory allocation */
1116         if (args->count < 1 || args->count > KVM_S390_SKEYS_MAX)
1117                 return -EINVAL;
1118
1119         keys = kmalloc_array(args->count, sizeof(uint8_t),
1120                              GFP_KERNEL | __GFP_NOWARN);
1121         if (!keys)
1122                 keys = vmalloc(sizeof(uint8_t) * args->count);
1123         if (!keys)
1124                 return -ENOMEM;
1125
1126         down_read(&current->mm->mmap_sem);
1127         for (i = 0; i < args->count; i++) {
1128                 hva = gfn_to_hva(kvm, args->start_gfn + i);
1129                 if (kvm_is_error_hva(hva)) {
1130                         r = -EFAULT;
1131                         break;
1132                 }
1133
1134                 r = get_guest_storage_key(current->mm, hva, &keys[i]);
1135                 if (r)
1136                         break;
1137         }
1138         up_read(&current->mm->mmap_sem);
1139
1140         if (!r) {
1141                 r = copy_to_user((uint8_t __user *)args->skeydata_addr, keys,
1142                                  sizeof(uint8_t) * args->count);
1143                 if (r)
1144                         r = -EFAULT;
1145         }
1146
1147         kvfree(keys);
1148         return r;
1149 }
1150
1151 static long kvm_s390_set_skeys(struct kvm *kvm, struct kvm_s390_skeys *args)
1152 {
1153         uint8_t *keys;
1154         uint64_t hva;
1155         int i, r = 0;
1156
1157         if (args->flags != 0)
1158                 return -EINVAL;
1159
1160         /* Enforce sane limit on memory allocation */
1161         if (args->count < 1 || args->count > KVM_S390_SKEYS_MAX)
1162                 return -EINVAL;
1163
1164         keys = kmalloc_array(args->count, sizeof(uint8_t),
1165                              GFP_KERNEL | __GFP_NOWARN);
1166         if (!keys)
1167                 keys = vmalloc(sizeof(uint8_t) * args->count);
1168         if (!keys)
1169                 return -ENOMEM;
1170
1171         r = copy_from_user(keys, (uint8_t __user *)args->skeydata_addr,
1172                            sizeof(uint8_t) * args->count);
1173         if (r) {
1174                 r = -EFAULT;
1175                 goto out;
1176         }
1177
1178         /* Enable storage key handling for the guest */
1179         r = s390_enable_skey();
1180         if (r)
1181                 goto out;
1182
1183         down_read(&current->mm->mmap_sem);
1184         for (i = 0; i < args->count; i++) {
1185                 hva = gfn_to_hva(kvm, args->start_gfn + i);
1186                 if (kvm_is_error_hva(hva)) {
1187                         r = -EFAULT;
1188                         break;
1189                 }
1190
1191                 /* Lowest order bit is reserved */
1192                 if (keys[i] & 0x01) {
1193                         r = -EINVAL;
1194                         break;
1195                 }
1196
1197                 r = set_guest_storage_key(current->mm, hva, keys[i], 0);
1198                 if (r)
1199                         break;
1200         }
1201         up_read(&current->mm->mmap_sem);
1202 out:
1203         kvfree(keys);
1204         return r;
1205 }
1206
1207 long kvm_arch_vm_ioctl(struct file *filp,
1208                        unsigned int ioctl, unsigned long arg)
1209 {
1210         struct kvm *kvm = filp->private_data;
1211         void __user *argp = (void __user *)arg;
1212         struct kvm_device_attr attr;
1213         int r;
1214
1215         switch (ioctl) {
1216         case KVM_S390_INTERRUPT: {
1217                 struct kvm_s390_interrupt s390int;
1218
1219                 r = -EFAULT;
1220                 if (copy_from_user(&s390int, argp, sizeof(s390int)))
1221                         break;
1222                 r = kvm_s390_inject_vm(kvm, &s390int);
1223                 break;
1224         }
1225         case KVM_ENABLE_CAP: {
1226                 struct kvm_enable_cap cap;
1227                 r = -EFAULT;
1228                 if (copy_from_user(&cap, argp, sizeof(cap)))
1229                         break;
1230                 r = kvm_vm_ioctl_enable_cap(kvm, &cap);
1231                 break;
1232         }
1233         case KVM_CREATE_IRQCHIP: {
1234                 struct kvm_irq_routing_entry routing;
1235
1236                 r = -EINVAL;
1237                 if (kvm->arch.use_irqchip) {
1238                         /* Set up dummy routing. */
1239                         memset(&routing, 0, sizeof(routing));
1240                         r = kvm_set_irq_routing(kvm, &routing, 0, 0);
1241                 }
1242                 break;
1243         }
1244         case KVM_SET_DEVICE_ATTR: {
1245                 r = -EFAULT;
1246                 if (copy_from_user(&attr, (void __user *)arg, sizeof(attr)))
1247                         break;
1248                 r = kvm_s390_vm_set_attr(kvm, &attr);
1249                 break;
1250         }
1251         case KVM_GET_DEVICE_ATTR: {
1252                 r = -EFAULT;
1253                 if (copy_from_user(&attr, (void __user *)arg, sizeof(attr)))
1254                         break;
1255                 r = kvm_s390_vm_get_attr(kvm, &attr);
1256                 break;
1257         }
1258         case KVM_HAS_DEVICE_ATTR: {
1259                 r = -EFAULT;
1260                 if (copy_from_user(&attr, (void __user *)arg, sizeof(attr)))
1261                         break;
1262                 r = kvm_s390_vm_has_attr(kvm, &attr);
1263                 break;
1264         }
1265         case KVM_S390_GET_SKEYS: {
1266                 struct kvm_s390_skeys args;
1267
1268                 r = -EFAULT;
1269                 if (copy_from_user(&args, argp,
1270                                    sizeof(struct kvm_s390_skeys)))
1271                         break;
1272                 r = kvm_s390_get_skeys(kvm, &args);
1273                 break;
1274         }
1275         case KVM_S390_SET_SKEYS: {
1276                 struct kvm_s390_skeys args;
1277
1278                 r = -EFAULT;
1279                 if (copy_from_user(&args, argp,
1280                                    sizeof(struct kvm_s390_skeys)))
1281                         break;
1282                 r = kvm_s390_set_skeys(kvm, &args);
1283                 break;
1284         }
1285         default:
1286                 r = -ENOTTY;
1287         }
1288
1289         return r;
1290 }
1291
1292 static int kvm_s390_query_ap_config(u8 *config)
1293 {
1294         u32 fcn_code = 0x04000000UL;
1295         u32 cc = 0;
1296
1297         memset(config, 0, 128);
1298         asm volatile(
1299                 "lgr 0,%1\n"
1300                 "lgr 2,%2\n"
1301                 ".long 0xb2af0000\n"            /* PQAP(QCI) */
1302                 "0: ipm %0\n"
1303                 "srl %0,28\n"
1304                 "1:\n"
1305                 EX_TABLE(0b, 1b)
1306                 : "+r" (cc)
1307                 : "r" (fcn_code), "r" (config)
1308                 : "cc", "0", "2", "memory"
1309         );
1310
1311         return cc;
1312 }
1313
1314 static int kvm_s390_apxa_installed(void)
1315 {
1316         u8 config[128];
1317         int cc;
1318
1319         if (test_facility(12)) {
1320                 cc = kvm_s390_query_ap_config(config);
1321
1322                 if (cc)
1323                         pr_err("PQAP(QCI) failed with cc=%d", cc);
1324                 else
1325                         return config[0] & 0x40;
1326         }
1327
1328         return 0;
1329 }
1330
1331 static void kvm_s390_set_crycb_format(struct kvm *kvm)
1332 {
1333         kvm->arch.crypto.crycbd = (__u32)(unsigned long) kvm->arch.crypto.crycb;
1334
1335         if (kvm_s390_apxa_installed())
1336                 kvm->arch.crypto.crycbd |= CRYCB_FORMAT2;
1337         else
1338                 kvm->arch.crypto.crycbd |= CRYCB_FORMAT1;
1339 }
1340
1341 static u64 kvm_s390_get_initial_cpuid(void)
1342 {
1343         struct cpuid cpuid;
1344
1345         get_cpu_id(&cpuid);
1346         cpuid.version = 0xff;
1347         return *((u64 *) &cpuid);
1348 }
1349
1350 static void kvm_s390_crypto_init(struct kvm *kvm)
1351 {
1352         if (!test_kvm_facility(kvm, 76))
1353                 return;
1354
1355         kvm->arch.crypto.crycb = &kvm->arch.sie_page2->crycb;
1356         kvm_s390_set_crycb_format(kvm);
1357
1358         /* Enable AES/DEA protected key functions by default */
1359         kvm->arch.crypto.aes_kw = 1;
1360         kvm->arch.crypto.dea_kw = 1;
1361         get_random_bytes(kvm->arch.crypto.crycb->aes_wrapping_key_mask,
1362                          sizeof(kvm->arch.crypto.crycb->aes_wrapping_key_mask));
1363         get_random_bytes(kvm->arch.crypto.crycb->dea_wrapping_key_mask,
1364                          sizeof(kvm->arch.crypto.crycb->dea_wrapping_key_mask));
1365 }
1366
1367 static void sca_dispose(struct kvm *kvm)
1368 {
1369         if (kvm->arch.use_esca)
1370                 free_pages_exact(kvm->arch.sca, sizeof(struct esca_block));
1371         else
1372                 free_page((unsigned long)(kvm->arch.sca));
1373         kvm->arch.sca = NULL;
1374 }
1375
1376 int kvm_arch_init_vm(struct kvm *kvm, unsigned long type)
1377 {
1378         gfp_t alloc_flags = GFP_KERNEL;
1379         int i, rc;
1380         char debug_name[16];
1381         static unsigned long sca_offset;
1382
1383         rc = -EINVAL;
1384 #ifdef CONFIG_KVM_S390_UCONTROL
1385         if (type & ~KVM_VM_S390_UCONTROL)
1386                 goto out_err;
1387         if ((type & KVM_VM_S390_UCONTROL) && (!capable(CAP_SYS_ADMIN)))
1388                 goto out_err;
1389 #else
1390         if (type)
1391                 goto out_err;
1392 #endif
1393
1394         rc = s390_enable_sie();
1395         if (rc)
1396                 goto out_err;
1397
1398         rc = -ENOMEM;
1399
1400         ratelimit_state_init(&kvm->arch.sthyi_limit, 5 * HZ, 500);
1401
1402         kvm->arch.use_esca = 0; /* start with basic SCA */
1403         if (!sclp.has_64bscao)
1404                 alloc_flags |= GFP_DMA;
1405         rwlock_init(&kvm->arch.sca_lock);
1406         kvm->arch.sca = (struct bsca_block *) get_zeroed_page(alloc_flags);
1407         if (!kvm->arch.sca)
1408                 goto out_err;
1409         spin_lock(&kvm_lock);
1410         sca_offset += 16;
1411         if (sca_offset + sizeof(struct bsca_block) > PAGE_SIZE)
1412                 sca_offset = 0;
1413         kvm->arch.sca = (struct bsca_block *)
1414                         ((char *) kvm->arch.sca + sca_offset);
1415         spin_unlock(&kvm_lock);
1416
1417         sprintf(debug_name, "kvm-%u", current->pid);
1418
1419         kvm->arch.dbf = debug_register(debug_name, 32, 1, 7 * sizeof(long));
1420         if (!kvm->arch.dbf)
1421                 goto out_err;
1422
1423         kvm->arch.sie_page2 =
1424              (struct sie_page2 *) get_zeroed_page(GFP_KERNEL | GFP_DMA);
1425         if (!kvm->arch.sie_page2)
1426                 goto out_err;
1427
1428         /* Populate the facility mask initially. */
1429         memcpy(kvm->arch.model.fac_mask, S390_lowcore.stfle_fac_list,
1430                S390_ARCH_FAC_LIST_SIZE_BYTE);
1431         for (i = 0; i < S390_ARCH_FAC_LIST_SIZE_U64; i++) {
1432                 if (i < kvm_s390_fac_list_mask_size())
1433                         kvm->arch.model.fac_mask[i] &= kvm_s390_fac_list_mask[i];
1434                 else
1435                         kvm->arch.model.fac_mask[i] = 0UL;
1436         }
1437
1438         /* Populate the facility list initially. */
1439         kvm->arch.model.fac_list = kvm->arch.sie_page2->fac_list;
1440         memcpy(kvm->arch.model.fac_list, kvm->arch.model.fac_mask,
1441                S390_ARCH_FAC_LIST_SIZE_BYTE);
1442
1443         set_kvm_facility(kvm->arch.model.fac_mask, 74);
1444         set_kvm_facility(kvm->arch.model.fac_list, 74);
1445
1446         kvm->arch.model.cpuid = kvm_s390_get_initial_cpuid();
1447         kvm->arch.model.ibc = sclp.ibc & 0x0fff;
1448
1449         kvm_s390_crypto_init(kvm);
1450
1451         spin_lock_init(&kvm->arch.float_int.lock);
1452         for (i = 0; i < FIRQ_LIST_COUNT; i++)
1453                 INIT_LIST_HEAD(&kvm->arch.float_int.lists[i]);
1454         init_waitqueue_head(&kvm->arch.ipte_wq);
1455         mutex_init(&kvm->arch.ipte_mutex);
1456
1457         debug_register_view(kvm->arch.dbf, &debug_sprintf_view);
1458         VM_EVENT(kvm, 3, "vm created with type %lu", type);
1459
1460         if (type & KVM_VM_S390_UCONTROL) {
1461                 kvm->arch.gmap = NULL;
1462                 kvm->arch.mem_limit = KVM_S390_NO_MEM_LIMIT;
1463         } else {
1464                 if (sclp.hamax == U64_MAX)
1465                         kvm->arch.mem_limit = TASK_MAX_SIZE;
1466                 else
1467                         kvm->arch.mem_limit = min_t(unsigned long, TASK_MAX_SIZE,
1468                                                     sclp.hamax + 1);
1469                 kvm->arch.gmap = gmap_create(current->mm, kvm->arch.mem_limit - 1);
1470                 if (!kvm->arch.gmap)
1471                         goto out_err;
1472                 kvm->arch.gmap->private = kvm;
1473                 kvm->arch.gmap->pfault_enabled = 0;
1474         }
1475
1476         kvm->arch.css_support = 0;
1477         kvm->arch.use_irqchip = 0;
1478         kvm->arch.epoch = 0;
1479
1480         spin_lock_init(&kvm->arch.start_stop_lock);
1481         kvm_s390_vsie_init(kvm);
1482         KVM_EVENT(3, "vm 0x%pK created by pid %u", kvm, current->pid);
1483
1484         return 0;
1485 out_err:
1486         free_page((unsigned long)kvm->arch.sie_page2);
1487         debug_unregister(kvm->arch.dbf);
1488         sca_dispose(kvm);
1489         KVM_EVENT(3, "creation of vm failed: %d", rc);
1490         return rc;
1491 }
1492
1493 void kvm_arch_vcpu_destroy(struct kvm_vcpu *vcpu)
1494 {
1495         VCPU_EVENT(vcpu, 3, "%s", "free cpu");
1496         trace_kvm_s390_destroy_vcpu(vcpu->vcpu_id);
1497         kvm_s390_clear_local_irqs(vcpu);
1498         kvm_clear_async_pf_completion_queue(vcpu);
1499         if (!kvm_is_ucontrol(vcpu->kvm))
1500                 sca_del_vcpu(vcpu);
1501
1502         if (kvm_is_ucontrol(vcpu->kvm))
1503                 gmap_remove(vcpu->arch.gmap);
1504
1505         if (vcpu->kvm->arch.use_cmma)
1506                 kvm_s390_vcpu_unsetup_cmma(vcpu);
1507         free_page((unsigned long)(vcpu->arch.sie_block));
1508
1509         kvm_vcpu_uninit(vcpu);
1510         kmem_cache_free(kvm_vcpu_cache, vcpu);
1511 }
1512
1513 static void kvm_free_vcpus(struct kvm *kvm)
1514 {
1515         unsigned int i;
1516         struct kvm_vcpu *vcpu;
1517
1518         kvm_for_each_vcpu(i, vcpu, kvm)
1519                 kvm_arch_vcpu_destroy(vcpu);
1520
1521         mutex_lock(&kvm->lock);
1522         for (i = 0; i < atomic_read(&kvm->online_vcpus); i++)
1523                 kvm->vcpus[i] = NULL;
1524
1525         atomic_set(&kvm->online_vcpus, 0);
1526         mutex_unlock(&kvm->lock);
1527 }
1528
1529 void kvm_arch_destroy_vm(struct kvm *kvm)
1530 {
1531         kvm_free_vcpus(kvm);
1532         sca_dispose(kvm);
1533         debug_unregister(kvm->arch.dbf);
1534         free_page((unsigned long)kvm->arch.sie_page2);
1535         if (!kvm_is_ucontrol(kvm))
1536                 gmap_remove(kvm->arch.gmap);
1537         kvm_s390_destroy_adapters(kvm);
1538         kvm_s390_clear_float_irqs(kvm);
1539         kvm_s390_vsie_destroy(kvm);
1540         KVM_EVENT(3, "vm 0x%pK destroyed", kvm);
1541 }
1542
1543 /* Section: vcpu related */
1544 static int __kvm_ucontrol_vcpu_init(struct kvm_vcpu *vcpu)
1545 {
1546         vcpu->arch.gmap = gmap_create(current->mm, -1UL);
1547         if (!vcpu->arch.gmap)
1548                 return -ENOMEM;
1549         vcpu->arch.gmap->private = vcpu->kvm;
1550
1551         return 0;
1552 }
1553
1554 static void sca_del_vcpu(struct kvm_vcpu *vcpu)
1555 {
1556         read_lock(&vcpu->kvm->arch.sca_lock);
1557         if (vcpu->kvm->arch.use_esca) {
1558                 struct esca_block *sca = vcpu->kvm->arch.sca;
1559
1560                 clear_bit_inv(vcpu->vcpu_id, (unsigned long *) sca->mcn);
1561                 sca->cpu[vcpu->vcpu_id].sda = 0;
1562         } else {
1563                 struct bsca_block *sca = vcpu->kvm->arch.sca;
1564
1565                 clear_bit_inv(vcpu->vcpu_id, (unsigned long *) &sca->mcn);
1566                 sca->cpu[vcpu->vcpu_id].sda = 0;
1567         }
1568         read_unlock(&vcpu->kvm->arch.sca_lock);
1569 }
1570
1571 static void sca_add_vcpu(struct kvm_vcpu *vcpu)
1572 {
1573         read_lock(&vcpu->kvm->arch.sca_lock);
1574         if (vcpu->kvm->arch.use_esca) {
1575                 struct esca_block *sca = vcpu->kvm->arch.sca;
1576
1577                 sca->cpu[vcpu->vcpu_id].sda = (__u64) vcpu->arch.sie_block;
1578                 vcpu->arch.sie_block->scaoh = (__u32)(((__u64)sca) >> 32);
1579                 vcpu->arch.sie_block->scaol = (__u32)(__u64)sca & ~0x3fU;
1580                 vcpu->arch.sie_block->ecb2 |= 0x04U;
1581                 set_bit_inv(vcpu->vcpu_id, (unsigned long *) sca->mcn);
1582         } else {
1583                 struct bsca_block *sca = vcpu->kvm->arch.sca;
1584
1585                 sca->cpu[vcpu->vcpu_id].sda = (__u64) vcpu->arch.sie_block;
1586                 vcpu->arch.sie_block->scaoh = (__u32)(((__u64)sca) >> 32);
1587                 vcpu->arch.sie_block->scaol = (__u32)(__u64)sca;
1588                 set_bit_inv(vcpu->vcpu_id, (unsigned long *) &sca->mcn);
1589         }
1590         read_unlock(&vcpu->kvm->arch.sca_lock);
1591 }
1592
1593 /* Basic SCA to Extended SCA data copy routines */
1594 static inline void sca_copy_entry(struct esca_entry *d, struct bsca_entry *s)
1595 {
1596         d->sda = s->sda;
1597         d->sigp_ctrl.c = s->sigp_ctrl.c;
1598         d->sigp_ctrl.scn = s->sigp_ctrl.scn;
1599 }
1600
1601 static void sca_copy_b_to_e(struct esca_block *d, struct bsca_block *s)
1602 {
1603         int i;
1604
1605         d->ipte_control = s->ipte_control;
1606         d->mcn[0] = s->mcn;
1607         for (i = 0; i < KVM_S390_BSCA_CPU_SLOTS; i++)
1608                 sca_copy_entry(&d->cpu[i], &s->cpu[i]);
1609 }
1610
1611 static int sca_switch_to_extended(struct kvm *kvm)
1612 {
1613         struct bsca_block *old_sca = kvm->arch.sca;
1614         struct esca_block *new_sca;
1615         struct kvm_vcpu *vcpu;
1616         unsigned int vcpu_idx;
1617         u32 scaol, scaoh;
1618
1619         new_sca = alloc_pages_exact(sizeof(*new_sca), GFP_KERNEL|__GFP_ZERO);
1620         if (!new_sca)
1621                 return -ENOMEM;
1622
1623         scaoh = (u32)((u64)(new_sca) >> 32);
1624         scaol = (u32)(u64)(new_sca) & ~0x3fU;
1625
1626         kvm_s390_vcpu_block_all(kvm);
1627         write_lock(&kvm->arch.sca_lock);
1628
1629         sca_copy_b_to_e(new_sca, old_sca);
1630
1631         kvm_for_each_vcpu(vcpu_idx, vcpu, kvm) {
1632                 vcpu->arch.sie_block->scaoh = scaoh;
1633                 vcpu->arch.sie_block->scaol = scaol;
1634                 vcpu->arch.sie_block->ecb2 |= 0x04U;
1635         }
1636         kvm->arch.sca = new_sca;
1637         kvm->arch.use_esca = 1;
1638
1639         write_unlock(&kvm->arch.sca_lock);
1640         kvm_s390_vcpu_unblock_all(kvm);
1641
1642         free_page((unsigned long)old_sca);
1643
1644         VM_EVENT(kvm, 2, "Switched to ESCA (0x%pK -> 0x%pK)",
1645                  old_sca, kvm->arch.sca);
1646         return 0;
1647 }
1648
1649 static int sca_can_add_vcpu(struct kvm *kvm, unsigned int id)
1650 {
1651         int rc;
1652
1653         if (id < KVM_S390_BSCA_CPU_SLOTS)
1654                 return true;
1655         if (!sclp.has_esca || !sclp.has_64bscao)
1656                 return false;
1657
1658         mutex_lock(&kvm->lock);
1659         rc = kvm->arch.use_esca ? 0 : sca_switch_to_extended(kvm);
1660         mutex_unlock(&kvm->lock);
1661
1662         return rc == 0 && id < KVM_S390_ESCA_CPU_SLOTS;
1663 }
1664
1665 int kvm_arch_vcpu_init(struct kvm_vcpu *vcpu)
1666 {
1667         vcpu->arch.pfault_token = KVM_S390_PFAULT_TOKEN_INVALID;
1668         kvm_clear_async_pf_completion_queue(vcpu);
1669         vcpu->run->kvm_valid_regs = KVM_SYNC_PREFIX |
1670                                     KVM_SYNC_GPRS |
1671                                     KVM_SYNC_ACRS |
1672                                     KVM_SYNC_CRS |
1673                                     KVM_SYNC_ARCH0 |
1674                                     KVM_SYNC_PFAULT;
1675         kvm_s390_set_prefix(vcpu, 0);
1676         if (test_kvm_facility(vcpu->kvm, 64))
1677                 vcpu->run->kvm_valid_regs |= KVM_SYNC_RICCB;
1678         /* fprs can be synchronized via vrs, even if the guest has no vx. With
1679          * MACHINE_HAS_VX, (load|store)_fpu_regs() will work with vrs format.
1680          */
1681         if (MACHINE_HAS_VX)
1682                 vcpu->run->kvm_valid_regs |= KVM_SYNC_VRS;
1683         else
1684                 vcpu->run->kvm_valid_regs |= KVM_SYNC_FPRS;
1685
1686         if (kvm_is_ucontrol(vcpu->kvm))
1687                 return __kvm_ucontrol_vcpu_init(vcpu);
1688
1689         return 0;
1690 }
1691
1692 /* needs disabled preemption to protect from TOD sync and vcpu_load/put */
1693 static void __start_cpu_timer_accounting(struct kvm_vcpu *vcpu)
1694 {
1695         WARN_ON_ONCE(vcpu->arch.cputm_start != 0);
1696         raw_write_seqcount_begin(&vcpu->arch.cputm_seqcount);
1697         vcpu->arch.cputm_start = get_tod_clock_fast();
1698         raw_write_seqcount_end(&vcpu->arch.cputm_seqcount);
1699 }
1700
1701 /* needs disabled preemption to protect from TOD sync and vcpu_load/put */
1702 static void __stop_cpu_timer_accounting(struct kvm_vcpu *vcpu)
1703 {
1704         WARN_ON_ONCE(vcpu->arch.cputm_start == 0);
1705         raw_write_seqcount_begin(&vcpu->arch.cputm_seqcount);
1706         vcpu->arch.sie_block->cputm -= get_tod_clock_fast() - vcpu->arch.cputm_start;
1707         vcpu->arch.cputm_start = 0;
1708         raw_write_seqcount_end(&vcpu->arch.cputm_seqcount);
1709 }
1710
1711 /* needs disabled preemption to protect from TOD sync and vcpu_load/put */
1712 static void __enable_cpu_timer_accounting(struct kvm_vcpu *vcpu)
1713 {
1714         WARN_ON_ONCE(vcpu->arch.cputm_enabled);
1715         vcpu->arch.cputm_enabled = true;
1716         __start_cpu_timer_accounting(vcpu);
1717 }
1718
1719 /* needs disabled preemption to protect from TOD sync and vcpu_load/put */
1720 static void __disable_cpu_timer_accounting(struct kvm_vcpu *vcpu)
1721 {
1722         WARN_ON_ONCE(!vcpu->arch.cputm_enabled);
1723         __stop_cpu_timer_accounting(vcpu);
1724         vcpu->arch.cputm_enabled = false;
1725 }
1726
1727 static void enable_cpu_timer_accounting(struct kvm_vcpu *vcpu)
1728 {
1729         preempt_disable(); /* protect from TOD sync and vcpu_load/put */
1730         __enable_cpu_timer_accounting(vcpu);
1731         preempt_enable();
1732 }
1733
1734 static void disable_cpu_timer_accounting(struct kvm_vcpu *vcpu)
1735 {
1736         preempt_disable(); /* protect from TOD sync and vcpu_load/put */
1737         __disable_cpu_timer_accounting(vcpu);
1738         preempt_enable();
1739 }
1740
1741 /* set the cpu timer - may only be called from the VCPU thread itself */
1742 void kvm_s390_set_cpu_timer(struct kvm_vcpu *vcpu, __u64 cputm)
1743 {
1744         preempt_disable(); /* protect from TOD sync and vcpu_load/put */
1745         raw_write_seqcount_begin(&vcpu->arch.cputm_seqcount);
1746         if (vcpu->arch.cputm_enabled)
1747                 vcpu->arch.cputm_start = get_tod_clock_fast();
1748         vcpu->arch.sie_block->cputm = cputm;
1749         raw_write_seqcount_end(&vcpu->arch.cputm_seqcount);
1750         preempt_enable();
1751 }
1752
1753 /* update and get the cpu timer - can also be called from other VCPU threads */
1754 __u64 kvm_s390_get_cpu_timer(struct kvm_vcpu *vcpu)
1755 {
1756         unsigned int seq;
1757         __u64 value;
1758
1759         if (unlikely(!vcpu->arch.cputm_enabled))
1760                 return vcpu->arch.sie_block->cputm;
1761
1762         preempt_disable(); /* protect from TOD sync and vcpu_load/put */
1763         do {
1764                 seq = raw_read_seqcount(&vcpu->arch.cputm_seqcount);
1765                 /*
1766                  * If the writer would ever execute a read in the critical
1767                  * section, e.g. in irq context, we have a deadlock.
1768                  */
1769                 WARN_ON_ONCE((seq & 1) && smp_processor_id() == vcpu->cpu);
1770                 value = vcpu->arch.sie_block->cputm;
1771                 /* if cputm_start is 0, accounting is being started/stopped */
1772                 if (likely(vcpu->arch.cputm_start))
1773                         value -= get_tod_clock_fast() - vcpu->arch.cputm_start;
1774         } while (read_seqcount_retry(&vcpu->arch.cputm_seqcount, seq & ~1));
1775         preempt_enable();
1776         return value;
1777 }
1778
1779 void kvm_arch_vcpu_load(struct kvm_vcpu *vcpu, int cpu)
1780 {
1781         /* Save host register state */
1782         save_fpu_regs();
1783         vcpu->arch.host_fpregs.fpc = current->thread.fpu.fpc;
1784         vcpu->arch.host_fpregs.regs = current->thread.fpu.regs;
1785
1786         if (MACHINE_HAS_VX)
1787                 current->thread.fpu.regs = vcpu->run->s.regs.vrs;
1788         else
1789                 current->thread.fpu.regs = vcpu->run->s.regs.fprs;
1790         current->thread.fpu.fpc = vcpu->run->s.regs.fpc;
1791         if (test_fp_ctl(current->thread.fpu.fpc))
1792                 /* User space provided an invalid FPC, let's clear it */
1793                 current->thread.fpu.fpc = 0;
1794
1795         save_access_regs(vcpu->arch.host_acrs);
1796         restore_access_regs(vcpu->run->s.regs.acrs);
1797         gmap_enable(vcpu->arch.enabled_gmap);
1798         atomic_or(CPUSTAT_RUNNING, &vcpu->arch.sie_block->cpuflags);
1799         if (vcpu->arch.cputm_enabled && !is_vcpu_idle(vcpu))
1800                 __start_cpu_timer_accounting(vcpu);
1801         vcpu->cpu = cpu;
1802 }
1803
1804 void kvm_arch_vcpu_put(struct kvm_vcpu *vcpu)
1805 {
1806         vcpu->cpu = -1;
1807         if (vcpu->arch.cputm_enabled && !is_vcpu_idle(vcpu))
1808                 __stop_cpu_timer_accounting(vcpu);
1809         atomic_andnot(CPUSTAT_RUNNING, &vcpu->arch.sie_block->cpuflags);
1810         vcpu->arch.enabled_gmap = gmap_get_enabled();
1811         gmap_disable(vcpu->arch.enabled_gmap);
1812
1813         /* Save guest register state */
1814         save_fpu_regs();
1815         vcpu->run->s.regs.fpc = current->thread.fpu.fpc;
1816
1817         /* Restore host register state */
1818         current->thread.fpu.fpc = vcpu->arch.host_fpregs.fpc;
1819         current->thread.fpu.regs = vcpu->arch.host_fpregs.regs;
1820
1821         save_access_regs(vcpu->run->s.regs.acrs);
1822         restore_access_regs(vcpu->arch.host_acrs);
1823 }
1824
1825 static void kvm_s390_vcpu_initial_reset(struct kvm_vcpu *vcpu)
1826 {
1827         /* this equals initial cpu reset in pop, but we don't switch to ESA */
1828         vcpu->arch.sie_block->gpsw.mask = 0UL;
1829         vcpu->arch.sie_block->gpsw.addr = 0UL;
1830         kvm_s390_set_prefix(vcpu, 0);
1831         kvm_s390_set_cpu_timer(vcpu, 0);
1832         vcpu->arch.sie_block->ckc       = 0UL;
1833         vcpu->arch.sie_block->todpr     = 0;
1834         memset(vcpu->arch.sie_block->gcr, 0, 16 * sizeof(__u64));
1835         vcpu->arch.sie_block->gcr[0]  = 0xE0UL;
1836         vcpu->arch.sie_block->gcr[14] = 0xC2000000UL;
1837         /* make sure the new fpc will be lazily loaded */
1838         save_fpu_regs();
1839         current->thread.fpu.fpc = 0;
1840         vcpu->arch.sie_block->gbea = 1;
1841         vcpu->arch.sie_block->pp = 0;
1842         vcpu->arch.pfault_token = KVM_S390_PFAULT_TOKEN_INVALID;
1843         kvm_clear_async_pf_completion_queue(vcpu);
1844         if (!kvm_s390_user_cpu_state_ctrl(vcpu->kvm))
1845                 kvm_s390_vcpu_stop(vcpu);
1846         kvm_s390_clear_local_irqs(vcpu);
1847 }
1848
1849 void kvm_arch_vcpu_postcreate(struct kvm_vcpu *vcpu)
1850 {
1851         mutex_lock(&vcpu->kvm->lock);
1852         preempt_disable();
1853         vcpu->arch.sie_block->epoch = vcpu->kvm->arch.epoch;
1854         preempt_enable();
1855         mutex_unlock(&vcpu->kvm->lock);
1856         if (!kvm_is_ucontrol(vcpu->kvm)) {
1857                 vcpu->arch.gmap = vcpu->kvm->arch.gmap;
1858                 sca_add_vcpu(vcpu);
1859         }
1860         if (test_kvm_facility(vcpu->kvm, 74) || vcpu->kvm->arch.user_instr0)
1861                 vcpu->arch.sie_block->ictl |= ICTL_OPEREXC;
1862         /* make vcpu_load load the right gmap on the first trigger */
1863         vcpu->arch.enabled_gmap = vcpu->arch.gmap;
1864 }
1865
1866 static void kvm_s390_vcpu_crypto_setup(struct kvm_vcpu *vcpu)
1867 {
1868         if (!test_kvm_facility(vcpu->kvm, 76))
1869                 return;
1870
1871         vcpu->arch.sie_block->ecb3 &= ~(ECB3_AES | ECB3_DEA);
1872
1873         if (vcpu->kvm->arch.crypto.aes_kw)
1874                 vcpu->arch.sie_block->ecb3 |= ECB3_AES;
1875         if (vcpu->kvm->arch.crypto.dea_kw)
1876                 vcpu->arch.sie_block->ecb3 |= ECB3_DEA;
1877
1878         vcpu->arch.sie_block->crycbd = vcpu->kvm->arch.crypto.crycbd;
1879 }
1880
1881 void kvm_s390_vcpu_unsetup_cmma(struct kvm_vcpu *vcpu)
1882 {
1883         free_page(vcpu->arch.sie_block->cbrlo);
1884         vcpu->arch.sie_block->cbrlo = 0;
1885 }
1886
1887 int kvm_s390_vcpu_setup_cmma(struct kvm_vcpu *vcpu)
1888 {
1889         vcpu->arch.sie_block->cbrlo = get_zeroed_page(GFP_KERNEL);
1890         if (!vcpu->arch.sie_block->cbrlo)
1891                 return -ENOMEM;
1892
1893         vcpu->arch.sie_block->ecb2 |= 0x80;
1894         vcpu->arch.sie_block->ecb2 &= ~0x08;
1895         return 0;
1896 }
1897
1898 static void kvm_s390_vcpu_setup_model(struct kvm_vcpu *vcpu)
1899 {
1900         struct kvm_s390_cpu_model *model = &vcpu->kvm->arch.model;
1901
1902         vcpu->arch.sie_block->ibc = model->ibc;
1903         if (test_kvm_facility(vcpu->kvm, 7))
1904                 vcpu->arch.sie_block->fac = (u32)(u64) model->fac_list;
1905 }
1906
1907 int kvm_arch_vcpu_setup(struct kvm_vcpu *vcpu)
1908 {
1909         int rc = 0;
1910
1911         atomic_set(&vcpu->arch.sie_block->cpuflags, CPUSTAT_ZARCH |
1912                                                     CPUSTAT_SM |
1913                                                     CPUSTAT_STOPPED);
1914
1915         if (test_kvm_facility(vcpu->kvm, 78))
1916                 atomic_or(CPUSTAT_GED2, &vcpu->arch.sie_block->cpuflags);
1917         else if (test_kvm_facility(vcpu->kvm, 8))
1918                 atomic_or(CPUSTAT_GED, &vcpu->arch.sie_block->cpuflags);
1919
1920         kvm_s390_vcpu_setup_model(vcpu);
1921
1922         /* pgste_set_pte has special handling for !MACHINE_HAS_ESOP */
1923         if (MACHINE_HAS_ESOP)
1924                 vcpu->arch.sie_block->ecb |= 0x02;
1925         if (test_kvm_facility(vcpu->kvm, 9))
1926                 vcpu->arch.sie_block->ecb |= 0x04;
1927         if (test_kvm_facility(vcpu->kvm, 73))
1928                 vcpu->arch.sie_block->ecb |= 0x10;
1929
1930         if (test_kvm_facility(vcpu->kvm, 8) && sclp.has_pfmfi)
1931                 vcpu->arch.sie_block->ecb2 |= 0x08;
1932         vcpu->arch.sie_block->eca = 0x1002000U;
1933         if (sclp.has_cei)
1934                 vcpu->arch.sie_block->eca |= 0x80000000U;
1935         if (sclp.has_ib)
1936                 vcpu->arch.sie_block->eca |= 0x40000000U;
1937         if (sclp.has_siif)
1938                 vcpu->arch.sie_block->eca |= 1;
1939         if (sclp.has_sigpif)
1940                 vcpu->arch.sie_block->eca |= 0x10000000U;
1941         if (test_kvm_facility(vcpu->kvm, 129)) {
1942                 vcpu->arch.sie_block->eca |= 0x00020000;
1943                 vcpu->arch.sie_block->ecd |= 0x20000000;
1944         }
1945         vcpu->arch.sie_block->riccbd = (unsigned long) &vcpu->run->s.regs.riccb;
1946         vcpu->arch.sie_block->ictl |= ICTL_ISKE | ICTL_SSKE | ICTL_RRBE;
1947
1948         if (vcpu->kvm->arch.use_cmma) {
1949                 rc = kvm_s390_vcpu_setup_cmma(vcpu);
1950                 if (rc)
1951                         return rc;
1952         }
1953         hrtimer_init(&vcpu->arch.ckc_timer, CLOCK_MONOTONIC, HRTIMER_MODE_REL);
1954         vcpu->arch.ckc_timer.function = kvm_s390_idle_wakeup;
1955
1956         kvm_s390_vcpu_crypto_setup(vcpu);
1957
1958         return rc;
1959 }
1960
1961 struct kvm_vcpu *kvm_arch_vcpu_create(struct kvm *kvm,
1962                                       unsigned int id)
1963 {
1964         struct kvm_vcpu *vcpu;
1965         struct sie_page *sie_page;
1966         int rc = -EINVAL;
1967
1968         if (!kvm_is_ucontrol(kvm) && !sca_can_add_vcpu(kvm, id))
1969                 goto out;
1970
1971         rc = -ENOMEM;
1972
1973         vcpu = kmem_cache_zalloc(kvm_vcpu_cache, GFP_KERNEL);
1974         if (!vcpu)
1975                 goto out;
1976
1977         sie_page = (struct sie_page *) get_zeroed_page(GFP_KERNEL);
1978         if (!sie_page)
1979                 goto out_free_cpu;
1980
1981         vcpu->arch.sie_block = &sie_page->sie_block;
1982         vcpu->arch.sie_block->itdba = (unsigned long) &sie_page->itdb;
1983
1984         /* the real guest size will always be smaller than msl */
1985         vcpu->arch.sie_block->mso = 0;
1986         vcpu->arch.sie_block->msl = sclp.hamax;
1987
1988         vcpu->arch.sie_block->icpua = id;
1989         spin_lock_init(&vcpu->arch.local_int.lock);
1990         vcpu->arch.local_int.float_int = &kvm->arch.float_int;
1991         vcpu->arch.local_int.wq = &vcpu->wq;
1992         vcpu->arch.local_int.cpuflags = &vcpu->arch.sie_block->cpuflags;
1993         seqcount_init(&vcpu->arch.cputm_seqcount);
1994
1995         rc = kvm_vcpu_init(vcpu, kvm, id);
1996         if (rc)
1997                 goto out_free_sie_block;
1998         VM_EVENT(kvm, 3, "create cpu %d at 0x%pK, sie block at 0x%pK", id, vcpu,
1999                  vcpu->arch.sie_block);
2000         trace_kvm_s390_create_vcpu(id, vcpu, vcpu->arch.sie_block);
2001
2002         return vcpu;
2003 out_free_sie_block:
2004         free_page((unsigned long)(vcpu->arch.sie_block));
2005 out_free_cpu:
2006         kmem_cache_free(kvm_vcpu_cache, vcpu);
2007 out:
2008         return ERR_PTR(rc);
2009 }
2010
2011 int kvm_arch_vcpu_runnable(struct kvm_vcpu *vcpu)
2012 {
2013         return kvm_s390_vcpu_has_irq(vcpu, 0);
2014 }
2015
2016 void kvm_s390_vcpu_block(struct kvm_vcpu *vcpu)
2017 {
2018         atomic_or(PROG_BLOCK_SIE, &vcpu->arch.sie_block->prog20);
2019         exit_sie(vcpu);
2020 }
2021
2022 void kvm_s390_vcpu_unblock(struct kvm_vcpu *vcpu)
2023 {
2024         atomic_andnot(PROG_BLOCK_SIE, &vcpu->arch.sie_block->prog20);
2025 }
2026
2027 static void kvm_s390_vcpu_request(struct kvm_vcpu *vcpu)
2028 {
2029         atomic_or(PROG_REQUEST, &vcpu->arch.sie_block->prog20);
2030         exit_sie(vcpu);
2031 }
2032
2033 static void kvm_s390_vcpu_request_handled(struct kvm_vcpu *vcpu)
2034 {
2035         atomic_andnot(PROG_REQUEST, &vcpu->arch.sie_block->prog20);
2036 }
2037
2038 /*
2039  * Kick a guest cpu out of SIE and wait until SIE is not running.
2040  * If the CPU is not running (e.g. waiting as idle) the function will
2041  * return immediately. */
2042 void exit_sie(struct kvm_vcpu *vcpu)
2043 {
2044         atomic_or(CPUSTAT_STOP_INT, &vcpu->arch.sie_block->cpuflags);
2045         while (vcpu->arch.sie_block->prog0c & PROG_IN_SIE)
2046                 cpu_relax();
2047 }
2048
2049 /* Kick a guest cpu out of SIE to process a request synchronously */
2050 void kvm_s390_sync_request(int req, struct kvm_vcpu *vcpu)
2051 {
2052         kvm_make_request(req, vcpu);
2053         kvm_s390_vcpu_request(vcpu);
2054 }
2055
2056 static void kvm_gmap_notifier(struct gmap *gmap, unsigned long start,
2057                               unsigned long end)
2058 {
2059         struct kvm *kvm = gmap->private;
2060         struct kvm_vcpu *vcpu;
2061         unsigned long prefix;
2062         int i;
2063
2064         if (gmap_is_shadow(gmap))
2065                 return;
2066         if (start >= 1UL << 31)
2067                 /* We are only interested in prefix pages */
2068                 return;
2069         kvm_for_each_vcpu(i, vcpu, kvm) {
2070                 /* match against both prefix pages */
2071                 prefix = kvm_s390_get_prefix(vcpu);
2072                 if (prefix <= end && start <= prefix + 2*PAGE_SIZE - 1) {
2073                         VCPU_EVENT(vcpu, 2, "gmap notifier for %lx-%lx",
2074                                    start, end);
2075                         kvm_s390_sync_request(KVM_REQ_MMU_RELOAD, vcpu);
2076                 }
2077         }
2078 }
2079
2080 int kvm_arch_vcpu_should_kick(struct kvm_vcpu *vcpu)
2081 {
2082         /* kvm common code refers to this, but never calls it */
2083         BUG();
2084         return 0;
2085 }
2086
2087 static int kvm_arch_vcpu_ioctl_get_one_reg(struct kvm_vcpu *vcpu,
2088                                            struct kvm_one_reg *reg)
2089 {
2090         int r = -EINVAL;
2091
2092         switch (reg->id) {
2093         case KVM_REG_S390_TODPR:
2094                 r = put_user(vcpu->arch.sie_block->todpr,
2095                              (u32 __user *)reg->addr);
2096                 break;
2097         case KVM_REG_S390_EPOCHDIFF:
2098                 r = put_user(vcpu->arch.sie_block->epoch,
2099                              (u64 __user *)reg->addr);
2100                 break;
2101         case KVM_REG_S390_CPU_TIMER:
2102                 r = put_user(kvm_s390_get_cpu_timer(vcpu),
2103                              (u64 __user *)reg->addr);
2104                 break;
2105         case KVM_REG_S390_CLOCK_COMP:
2106                 r = put_user(vcpu->arch.sie_block->ckc,
2107                              (u64 __user *)reg->addr);
2108                 break;
2109         case KVM_REG_S390_PFTOKEN:
2110                 r = put_user(vcpu->arch.pfault_token,
2111                              (u64 __user *)reg->addr);
2112                 break;
2113         case KVM_REG_S390_PFCOMPARE:
2114                 r = put_user(vcpu->arch.pfault_compare,
2115                              (u64 __user *)reg->addr);
2116                 break;
2117         case KVM_REG_S390_PFSELECT:
2118                 r = put_user(vcpu->arch.pfault_select,
2119                              (u64 __user *)reg->addr);
2120                 break;
2121         case KVM_REG_S390_PP:
2122                 r = put_user(vcpu->arch.sie_block->pp,
2123                              (u64 __user *)reg->addr);
2124                 break;
2125         case KVM_REG_S390_GBEA:
2126                 r = put_user(vcpu->arch.sie_block->gbea,
2127                              (u64 __user *)reg->addr);
2128                 break;
2129         default:
2130                 break;
2131         }
2132
2133         return r;
2134 }
2135
2136 static int kvm_arch_vcpu_ioctl_set_one_reg(struct kvm_vcpu *vcpu,
2137                                            struct kvm_one_reg *reg)
2138 {
2139         int r = -EINVAL;
2140         __u64 val;
2141
2142         switch (reg->id) {
2143         case KVM_REG_S390_TODPR:
2144                 r = get_user(vcpu->arch.sie_block->todpr,
2145                              (u32 __user *)reg->addr);
2146                 break;
2147         case KVM_REG_S390_EPOCHDIFF:
2148                 r = get_user(vcpu->arch.sie_block->epoch,
2149                              (u64 __user *)reg->addr);
2150                 break;
2151         case KVM_REG_S390_CPU_TIMER:
2152                 r = get_user(val, (u64 __user *)reg->addr);
2153                 if (!r)
2154                         kvm_s390_set_cpu_timer(vcpu, val);
2155                 break;
2156         case KVM_REG_S390_CLOCK_COMP:
2157                 r = get_user(vcpu->arch.sie_block->ckc,
2158                              (u64 __user *)reg->addr);
2159                 break;
2160         case KVM_REG_S390_PFTOKEN:
2161                 r = get_user(vcpu->arch.pfault_token,
2162                              (u64 __user *)reg->addr);
2163                 if (vcpu->arch.pfault_token == KVM_S390_PFAULT_TOKEN_INVALID)
2164                         kvm_clear_async_pf_completion_queue(vcpu);
2165                 break;
2166         case KVM_REG_S390_PFCOMPARE:
2167                 r = get_user(vcpu->arch.pfault_compare,
2168                              (u64 __user *)reg->addr);
2169                 break;
2170         case KVM_REG_S390_PFSELECT:
2171                 r = get_user(vcpu->arch.pfault_select,
2172                              (u64 __user *)reg->addr);
2173                 break;
2174         case KVM_REG_S390_PP:
2175                 r = get_user(vcpu->arch.sie_block->pp,
2176                              (u64 __user *)reg->addr);
2177                 break;
2178         case KVM_REG_S390_GBEA:
2179                 r = get_user(vcpu->arch.sie_block->gbea,
2180                              (u64 __user *)reg->addr);
2181                 break;
2182         default:
2183                 break;
2184         }
2185
2186         return r;
2187 }
2188
2189 static int kvm_arch_vcpu_ioctl_initial_reset(struct kvm_vcpu *vcpu)
2190 {
2191         kvm_s390_vcpu_initial_reset(vcpu);
2192         return 0;
2193 }
2194
2195 int kvm_arch_vcpu_ioctl_set_regs(struct kvm_vcpu *vcpu, struct kvm_regs *regs)
2196 {
2197         memcpy(&vcpu->run->s.regs.gprs, &regs->gprs, sizeof(regs->gprs));
2198         return 0;
2199 }
2200
2201 int kvm_arch_vcpu_ioctl_get_regs(struct kvm_vcpu *vcpu, struct kvm_regs *regs)
2202 {
2203         memcpy(&regs->gprs, &vcpu->run->s.regs.gprs, sizeof(regs->gprs));
2204         return 0;
2205 }
2206
2207 int kvm_arch_vcpu_ioctl_set_sregs(struct kvm_vcpu *vcpu,
2208                                   struct kvm_sregs *sregs)
2209 {
2210         memcpy(&vcpu->run->s.regs.acrs, &sregs->acrs, sizeof(sregs->acrs));
2211         memcpy(&vcpu->arch.sie_block->gcr, &sregs->crs, sizeof(sregs->crs));
2212         restore_access_regs(vcpu->run->s.regs.acrs);
2213         return 0;
2214 }
2215
2216 int kvm_arch_vcpu_ioctl_get_sregs(struct kvm_vcpu *vcpu,
2217                                   struct kvm_sregs *sregs)
2218 {
2219         memcpy(&sregs->acrs, &vcpu->run->s.regs.acrs, sizeof(sregs->acrs));
2220         memcpy(&sregs->crs, &vcpu->arch.sie_block->gcr, sizeof(sregs->crs));
2221         return 0;
2222 }
2223
2224 int kvm_arch_vcpu_ioctl_set_fpu(struct kvm_vcpu *vcpu, struct kvm_fpu *fpu)
2225 {
2226         /* make sure the new values will be lazily loaded */
2227         save_fpu_regs();
2228         if (test_fp_ctl(fpu->fpc))
2229                 return -EINVAL;
2230         current->thread.fpu.fpc = fpu->fpc;
2231         if (MACHINE_HAS_VX)
2232                 convert_fp_to_vx(current->thread.fpu.vxrs, (freg_t *)fpu->fprs);
2233         else
2234                 memcpy(current->thread.fpu.fprs, &fpu->fprs, sizeof(fpu->fprs));
2235         return 0;
2236 }
2237
2238 int kvm_arch_vcpu_ioctl_get_fpu(struct kvm_vcpu *vcpu, struct kvm_fpu *fpu)
2239 {
2240         /* make sure we have the latest values */
2241         save_fpu_regs();
2242         if (MACHINE_HAS_VX)
2243                 convert_vx_to_fp((freg_t *)fpu->fprs, current->thread.fpu.vxrs);
2244         else
2245                 memcpy(fpu->fprs, current->thread.fpu.fprs, sizeof(fpu->fprs));
2246         fpu->fpc = current->thread.fpu.fpc;
2247         return 0;
2248 }
2249
2250 static int kvm_arch_vcpu_ioctl_set_initial_psw(struct kvm_vcpu *vcpu, psw_t psw)
2251 {
2252         int rc = 0;
2253
2254         if (!is_vcpu_stopped(vcpu))
2255                 rc = -EBUSY;
2256         else {
2257                 vcpu->run->psw_mask = psw.mask;
2258                 vcpu->run->psw_addr = psw.addr;
2259         }
2260         return rc;
2261 }
2262
2263 int kvm_arch_vcpu_ioctl_translate(struct kvm_vcpu *vcpu,
2264                                   struct kvm_translation *tr)
2265 {
2266         return -EINVAL; /* not implemented yet */
2267 }
2268
2269 #define VALID_GUESTDBG_FLAGS (KVM_GUESTDBG_SINGLESTEP | \
2270                               KVM_GUESTDBG_USE_HW_BP | \
2271                               KVM_GUESTDBG_ENABLE)
2272
2273 int kvm_arch_vcpu_ioctl_set_guest_debug(struct kvm_vcpu *vcpu,
2274                                         struct kvm_guest_debug *dbg)
2275 {
2276         int rc = 0;
2277
2278         vcpu->guest_debug = 0;
2279         kvm_s390_clear_bp_data(vcpu);
2280
2281         if (dbg->control & ~VALID_GUESTDBG_FLAGS)
2282                 return -EINVAL;
2283         if (!sclp.has_gpere)
2284                 return -EINVAL;
2285
2286         if (dbg->control & KVM_GUESTDBG_ENABLE) {
2287                 vcpu->guest_debug = dbg->control;
2288                 /* enforce guest PER */
2289                 atomic_or(CPUSTAT_P, &vcpu->arch.sie_block->cpuflags);
2290
2291                 if (dbg->control & KVM_GUESTDBG_USE_HW_BP)
2292                         rc = kvm_s390_import_bp_data(vcpu, dbg);
2293         } else {
2294                 atomic_andnot(CPUSTAT_P, &vcpu->arch.sie_block->cpuflags);
2295                 vcpu->arch.guestdbg.last_bp = 0;
2296         }
2297
2298         if (rc) {
2299                 vcpu->guest_debug = 0;
2300                 kvm_s390_clear_bp_data(vcpu);
2301                 atomic_andnot(CPUSTAT_P, &vcpu->arch.sie_block->cpuflags);
2302         }
2303
2304         return rc;
2305 }
2306
2307 int kvm_arch_vcpu_ioctl_get_mpstate(struct kvm_vcpu *vcpu,
2308                                     struct kvm_mp_state *mp_state)
2309 {
2310         /* CHECK_STOP and LOAD are not supported yet */
2311         return is_vcpu_stopped(vcpu) ? KVM_MP_STATE_STOPPED :
2312                                        KVM_MP_STATE_OPERATING;
2313 }
2314
2315 int kvm_arch_vcpu_ioctl_set_mpstate(struct kvm_vcpu *vcpu,
2316                                     struct kvm_mp_state *mp_state)
2317 {
2318         int rc = 0;
2319
2320         /* user space knows about this interface - let it control the state */
2321         vcpu->kvm->arch.user_cpu_state_ctrl = 1;
2322
2323         switch (mp_state->mp_state) {
2324         case KVM_MP_STATE_STOPPED:
2325                 kvm_s390_vcpu_stop(vcpu);
2326                 break;
2327         case KVM_MP_STATE_OPERATING:
2328                 kvm_s390_vcpu_start(vcpu);
2329                 break;
2330         case KVM_MP_STATE_LOAD:
2331         case KVM_MP_STATE_CHECK_STOP:
2332                 /* fall through - CHECK_STOP and LOAD are not supported yet */
2333         default:
2334                 rc = -ENXIO;
2335         }
2336
2337         return rc;
2338 }
2339
2340 static bool ibs_enabled(struct kvm_vcpu *vcpu)
2341 {
2342         return atomic_read(&vcpu->arch.sie_block->cpuflags) & CPUSTAT_IBS;
2343 }
2344
2345 static int kvm_s390_handle_requests(struct kvm_vcpu *vcpu)
2346 {
2347 retry:
2348         kvm_s390_vcpu_request_handled(vcpu);
2349         if (!vcpu->requests)
2350                 return 0;
2351         /*
2352          * We use MMU_RELOAD just to re-arm the ipte notifier for the
2353          * guest prefix page. gmap_mprotect_notify will wait on the ptl lock.
2354          * This ensures that the ipte instruction for this request has
2355          * already finished. We might race against a second unmapper that
2356          * wants to set the blocking bit. Lets just retry the request loop.
2357          */
2358         if (kvm_check_request(KVM_REQ_MMU_RELOAD, vcpu)) {
2359                 int rc;
2360                 rc = gmap_mprotect_notify(vcpu->arch.gmap,
2361                                           kvm_s390_get_prefix(vcpu),
2362                                           PAGE_SIZE * 2, PROT_WRITE);
2363                 if (rc) {
2364                         kvm_make_request(KVM_REQ_MMU_RELOAD, vcpu);
2365                         return rc;
2366                 }
2367                 goto retry;
2368         }
2369
2370         if (kvm_check_request(KVM_REQ_TLB_FLUSH, vcpu)) {
2371                 vcpu->arch.sie_block->ihcpu = 0xffff;
2372                 goto retry;
2373         }
2374
2375         if (kvm_check_request(KVM_REQ_ENABLE_IBS, vcpu)) {
2376                 if (!ibs_enabled(vcpu)) {
2377                         trace_kvm_s390_enable_disable_ibs(vcpu->vcpu_id, 1);
2378                         atomic_or(CPUSTAT_IBS,
2379                                         &vcpu->arch.sie_block->cpuflags);
2380                 }
2381                 goto retry;
2382         }
2383
2384         if (kvm_check_request(KVM_REQ_DISABLE_IBS, vcpu)) {
2385                 if (ibs_enabled(vcpu)) {
2386                         trace_kvm_s390_enable_disable_ibs(vcpu->vcpu_id, 0);
2387                         atomic_andnot(CPUSTAT_IBS,
2388                                           &vcpu->arch.sie_block->cpuflags);
2389                 }
2390                 goto retry;
2391         }
2392
2393         if (kvm_check_request(KVM_REQ_ICPT_OPEREXC, vcpu)) {
2394                 vcpu->arch.sie_block->ictl |= ICTL_OPEREXC;
2395                 goto retry;
2396         }
2397
2398         /* nothing to do, just clear the request */
2399         clear_bit(KVM_REQ_UNHALT, &vcpu->requests);
2400
2401         return 0;
2402 }
2403
2404 void kvm_s390_set_tod_clock(struct kvm *kvm, u64 tod)
2405 {
2406         struct kvm_vcpu *vcpu;
2407         int i;
2408
2409         mutex_lock(&kvm->lock);
2410         preempt_disable();
2411         kvm->arch.epoch = tod - get_tod_clock();
2412         kvm_s390_vcpu_block_all(kvm);
2413         kvm_for_each_vcpu(i, vcpu, kvm)
2414                 vcpu->arch.sie_block->epoch = kvm->arch.epoch;
2415         kvm_s390_vcpu_unblock_all(kvm);
2416         preempt_enable();
2417         mutex_unlock(&kvm->lock);
2418 }
2419
2420 /**
2421  * kvm_arch_fault_in_page - fault-in guest page if necessary
2422  * @vcpu: The corresponding virtual cpu
2423  * @gpa: Guest physical address
2424  * @writable: Whether the page should be writable or not
2425  *
2426  * Make sure that a guest page has been faulted-in on the host.
2427  *
2428  * Return: Zero on success, negative error code otherwise.
2429  */
2430 long kvm_arch_fault_in_page(struct kvm_vcpu *vcpu, gpa_t gpa, int writable)
2431 {
2432         return gmap_fault(vcpu->arch.gmap, gpa,
2433                           writable ? FAULT_FLAG_WRITE : 0);
2434 }
2435
2436 static void __kvm_inject_pfault_token(struct kvm_vcpu *vcpu, bool start_token,
2437                                       unsigned long token)
2438 {
2439         struct kvm_s390_interrupt inti;
2440         struct kvm_s390_irq irq;
2441
2442         if (start_token) {
2443                 irq.u.ext.ext_params2 = token;
2444                 irq.type = KVM_S390_INT_PFAULT_INIT;
2445                 WARN_ON_ONCE(kvm_s390_inject_vcpu(vcpu, &irq));
2446         } else {
2447                 inti.type = KVM_S390_INT_PFAULT_DONE;
2448                 inti.parm64 = token;
2449                 WARN_ON_ONCE(kvm_s390_inject_vm(vcpu->kvm, &inti));
2450         }
2451 }
2452
2453 void kvm_arch_async_page_not_present(struct kvm_vcpu *vcpu,
2454                                      struct kvm_async_pf *work)
2455 {
2456         trace_kvm_s390_pfault_init(vcpu, work->arch.pfault_token);
2457         __kvm_inject_pfault_token(vcpu, true, work->arch.pfault_token);
2458 }
2459
2460 void kvm_arch_async_page_present(struct kvm_vcpu *vcpu,
2461                                  struct kvm_async_pf *work)
2462 {
2463         trace_kvm_s390_pfault_done(vcpu, work->arch.pfault_token);
2464         __kvm_inject_pfault_token(vcpu, false, work->arch.pfault_token);
2465 }
2466
2467 void kvm_arch_async_page_ready(struct kvm_vcpu *vcpu,
2468                                struct kvm_async_pf *work)
2469 {
2470         /* s390 will always inject the page directly */
2471 }
2472
2473 bool kvm_arch_can_inject_async_page_present(struct kvm_vcpu *vcpu)
2474 {
2475         /*
2476          * s390 will always inject the page directly,
2477          * but we still want check_async_completion to cleanup
2478          */
2479         return true;
2480 }
2481
2482 static int kvm_arch_setup_async_pf(struct kvm_vcpu *vcpu)
2483 {
2484         hva_t hva;
2485         struct kvm_arch_async_pf arch;
2486         int rc;
2487
2488         if (vcpu->arch.pfault_token == KVM_S390_PFAULT_TOKEN_INVALID)
2489                 return 0;
2490         if ((vcpu->arch.sie_block->gpsw.mask & vcpu->arch.pfault_select) !=
2491             vcpu->arch.pfault_compare)
2492                 return 0;
2493         if (psw_extint_disabled(vcpu))
2494                 return 0;
2495         if (kvm_s390_vcpu_has_irq(vcpu, 0))
2496                 return 0;
2497         if (!(vcpu->arch.sie_block->gcr[0] & 0x200ul))
2498                 return 0;
2499         if (!vcpu->arch.gmap->pfault_enabled)
2500                 return 0;
2501
2502         hva = gfn_to_hva(vcpu->kvm, gpa_to_gfn(current->thread.gmap_addr));
2503         hva += current->thread.gmap_addr & ~PAGE_MASK;
2504         if (read_guest_real(vcpu, vcpu->arch.pfault_token, &arch.pfault_token, 8))
2505                 return 0;
2506
2507         rc = kvm_setup_async_pf(vcpu, current->thread.gmap_addr, hva, &arch);
2508         return rc;
2509 }
2510
2511 static int vcpu_pre_run(struct kvm_vcpu *vcpu)
2512 {
2513         int rc, cpuflags;
2514
2515         /*
2516          * On s390 notifications for arriving pages will be delivered directly
2517          * to the guest but the house keeping for completed pfaults is
2518          * handled outside the worker.
2519          */
2520         kvm_check_async_pf_completion(vcpu);
2521
2522         vcpu->arch.sie_block->gg14 = vcpu->run->s.regs.gprs[14];
2523         vcpu->arch.sie_block->gg15 = vcpu->run->s.regs.gprs[15];
2524
2525         if (need_resched())
2526                 schedule();
2527
2528         if (test_cpu_flag(CIF_MCCK_PENDING))
2529                 s390_handle_mcck();
2530
2531         if (!kvm_is_ucontrol(vcpu->kvm)) {
2532                 rc = kvm_s390_deliver_pending_interrupts(vcpu);
2533                 if (rc)
2534                         return rc;
2535         }
2536
2537         rc = kvm_s390_handle_requests(vcpu);
2538         if (rc)
2539                 return rc;
2540
2541         if (guestdbg_enabled(vcpu)) {
2542                 kvm_s390_backup_guest_per_regs(vcpu);
2543                 kvm_s390_patch_guest_per_regs(vcpu);
2544         }
2545
2546         vcpu->arch.sie_block->icptcode = 0;
2547         cpuflags = atomic_read(&vcpu->arch.sie_block->cpuflags);
2548         VCPU_EVENT(vcpu, 6, "entering sie flags %x", cpuflags);
2549         trace_kvm_s390_sie_enter(vcpu, cpuflags);
2550
2551         return 0;
2552 }
2553
2554 static int vcpu_post_run_fault_in_sie(struct kvm_vcpu *vcpu)
2555 {
2556         struct kvm_s390_pgm_info pgm_info = {
2557                 .code = PGM_ADDRESSING,
2558         };
2559         u8 opcode, ilen;
2560         int rc;
2561
2562         VCPU_EVENT(vcpu, 3, "%s", "fault in sie instruction");
2563         trace_kvm_s390_sie_fault(vcpu);
2564
2565         /*
2566          * We want to inject an addressing exception, which is defined as a
2567          * suppressing or terminating exception. However, since we came here
2568          * by a DAT access exception, the PSW still points to the faulting
2569          * instruction since DAT exceptions are nullifying. So we've got
2570          * to look up the current opcode to get the length of the instruction
2571          * to be able to forward the PSW.
2572          */
2573         rc = read_guest_instr(vcpu, &opcode, 1);
2574         ilen = insn_length(opcode);
2575         if (rc < 0) {
2576                 return rc;
2577         } else if (rc) {
2578                 /* Instruction-Fetching Exceptions - we can't detect the ilen.
2579                  * Forward by arbitrary ilc, injection will take care of
2580                  * nullification if necessary.
2581                  */
2582                 pgm_info = vcpu->arch.pgm;
2583                 ilen = 4;
2584         }
2585         pgm_info.flags = ilen | KVM_S390_PGM_FLAGS_ILC_VALID;
2586         kvm_s390_forward_psw(vcpu, ilen);
2587         return kvm_s390_inject_prog_irq(vcpu, &pgm_info);
2588 }
2589
2590 static int vcpu_post_run(struct kvm_vcpu *vcpu, int exit_reason)
2591 {
2592         VCPU_EVENT(vcpu, 6, "exit sie icptcode %d",
2593                    vcpu->arch.sie_block->icptcode);
2594         trace_kvm_s390_sie_exit(vcpu, vcpu->arch.sie_block->icptcode);
2595
2596         if (guestdbg_enabled(vcpu))
2597                 kvm_s390_restore_guest_per_regs(vcpu);
2598
2599         vcpu->run->s.regs.gprs[14] = vcpu->arch.sie_block->gg14;
2600         vcpu->run->s.regs.gprs[15] = vcpu->arch.sie_block->gg15;
2601
2602         if (vcpu->arch.sie_block->icptcode > 0) {
2603                 int rc = kvm_handle_sie_intercept(vcpu);
2604
2605                 if (rc != -EOPNOTSUPP)
2606                         return rc;
2607                 vcpu->run->exit_reason = KVM_EXIT_S390_SIEIC;
2608                 vcpu->run->s390_sieic.icptcode = vcpu->arch.sie_block->icptcode;
2609                 vcpu->run->s390_sieic.ipa = vcpu->arch.sie_block->ipa;
2610                 vcpu->run->s390_sieic.ipb = vcpu->arch.sie_block->ipb;
2611                 return -EREMOTE;
2612         } else if (exit_reason != -EFAULT) {
2613                 vcpu->stat.exit_null++;
2614                 return 0;
2615         } else if (kvm_is_ucontrol(vcpu->kvm)) {
2616                 vcpu->run->exit_reason = KVM_EXIT_S390_UCONTROL;
2617                 vcpu->run->s390_ucontrol.trans_exc_code =
2618                                                 current->thread.gmap_addr;
2619                 vcpu->run->s390_ucontrol.pgm_code = 0x10;
2620                 return -EREMOTE;
2621         } else if (current->thread.gmap_pfault) {
2622                 trace_kvm_s390_major_guest_pfault(vcpu);
2623                 current->thread.gmap_pfault = 0;
2624                 if (kvm_arch_setup_async_pf(vcpu))
2625                         return 0;
2626                 return kvm_arch_fault_in_page(vcpu, current->thread.gmap_addr, 1);
2627         }
2628         return vcpu_post_run_fault_in_sie(vcpu);
2629 }
2630
2631 static int __vcpu_run(struct kvm_vcpu *vcpu)
2632 {
2633         int rc, exit_reason;
2634
2635         /*
2636          * We try to hold kvm->srcu during most of vcpu_run (except when run-
2637          * ning the guest), so that memslots (and other stuff) are protected
2638          */
2639         vcpu->srcu_idx = srcu_read_lock(&vcpu->kvm->srcu);
2640
2641         do {
2642                 rc = vcpu_pre_run(vcpu);
2643                 if (rc)
2644                         break;
2645
2646                 srcu_read_unlock(&vcpu->kvm->srcu, vcpu->srcu_idx);
2647                 /*
2648                  * As PF_VCPU will be used in fault handler, between
2649                  * guest_enter and guest_exit should be no uaccess.
2650                  */
2651                 local_irq_disable();
2652                 guest_enter_irqoff();
2653                 __disable_cpu_timer_accounting(vcpu);
2654                 local_irq_enable();
2655                 exit_reason = sie64a(vcpu->arch.sie_block,
2656                                      vcpu->run->s.regs.gprs);
2657                 local_irq_disable();
2658                 __enable_cpu_timer_accounting(vcpu);
2659                 guest_exit_irqoff();
2660                 local_irq_enable();
2661                 vcpu->srcu_idx = srcu_read_lock(&vcpu->kvm->srcu);
2662
2663                 rc = vcpu_post_run(vcpu, exit_reason);
2664         } while (!signal_pending(current) && !guestdbg_exit_pending(vcpu) && !rc);
2665
2666         srcu_read_unlock(&vcpu->kvm->srcu, vcpu->srcu_idx);
2667         return rc;
2668 }
2669
2670 static void sync_regs(struct kvm_vcpu *vcpu, struct kvm_run *kvm_run)
2671 {
2672         vcpu->arch.sie_block->gpsw.mask = kvm_run->psw_mask;
2673         vcpu->arch.sie_block->gpsw.addr = kvm_run->psw_addr;
2674         if (kvm_run->kvm_dirty_regs & KVM_SYNC_PREFIX)
2675                 kvm_s390_set_prefix(vcpu, kvm_run->s.regs.prefix);
2676         if (kvm_run->kvm_dirty_regs & KVM_SYNC_CRS) {
2677                 memcpy(&vcpu->arch.sie_block->gcr, &kvm_run->s.regs.crs, 128);
2678                 /* some control register changes require a tlb flush */
2679                 kvm_make_request(KVM_REQ_TLB_FLUSH, vcpu);
2680         }
2681         if (kvm_run->kvm_dirty_regs & KVM_SYNC_ARCH0) {
2682                 kvm_s390_set_cpu_timer(vcpu, kvm_run->s.regs.cputm);
2683                 vcpu->arch.sie_block->ckc = kvm_run->s.regs.ckc;
2684                 vcpu->arch.sie_block->todpr = kvm_run->s.regs.todpr;
2685                 vcpu->arch.sie_block->pp = kvm_run->s.regs.pp;
2686                 vcpu->arch.sie_block->gbea = kvm_run->s.regs.gbea;
2687         }
2688         if (kvm_run->kvm_dirty_regs & KVM_SYNC_PFAULT) {
2689                 vcpu->arch.pfault_token = kvm_run->s.regs.pft;
2690                 vcpu->arch.pfault_select = kvm_run->s.regs.pfs;
2691                 vcpu->arch.pfault_compare = kvm_run->s.regs.pfc;
2692                 if (vcpu->arch.pfault_token == KVM_S390_PFAULT_TOKEN_INVALID)
2693                         kvm_clear_async_pf_completion_queue(vcpu);
2694         }
2695         /*
2696          * If userspace sets the riccb (e.g. after migration) to a valid state,
2697          * we should enable RI here instead of doing the lazy enablement.
2698          */
2699         if ((kvm_run->kvm_dirty_regs & KVM_SYNC_RICCB) &&
2700             test_kvm_facility(vcpu->kvm, 64)) {
2701                 struct runtime_instr_cb *riccb =
2702                         (struct runtime_instr_cb *) &kvm_run->s.regs.riccb;
2703
2704                 if (riccb->valid)
2705                         vcpu->arch.sie_block->ecb3 |= 0x01;
2706         }
2707
2708         kvm_run->kvm_dirty_regs = 0;
2709 }
2710
2711 static void store_regs(struct kvm_vcpu *vcpu, struct kvm_run *kvm_run)
2712 {
2713         kvm_run->psw_mask = vcpu->arch.sie_block->gpsw.mask;
2714         kvm_run->psw_addr = vcpu->arch.sie_block->gpsw.addr;
2715         kvm_run->s.regs.prefix = kvm_s390_get_prefix(vcpu);
2716         memcpy(&kvm_run->s.regs.crs, &vcpu->arch.sie_block->gcr, 128);
2717         kvm_run->s.regs.cputm = kvm_s390_get_cpu_timer(vcpu);
2718         kvm_run->s.regs.ckc = vcpu->arch.sie_block->ckc;
2719         kvm_run->s.regs.todpr = vcpu->arch.sie_block->todpr;
2720         kvm_run->s.regs.pp = vcpu->arch.sie_block->pp;
2721         kvm_run->s.regs.gbea = vcpu->arch.sie_block->gbea;
2722         kvm_run->s.regs.pft = vcpu->arch.pfault_token;
2723         kvm_run->s.regs.pfs = vcpu->arch.pfault_select;
2724         kvm_run->s.regs.pfc = vcpu->arch.pfault_compare;
2725 }
2726
2727 int kvm_arch_vcpu_ioctl_run(struct kvm_vcpu *vcpu, struct kvm_run *kvm_run)
2728 {
2729         int rc;
2730         sigset_t sigsaved;
2731
2732         if (guestdbg_exit_pending(vcpu)) {
2733                 kvm_s390_prepare_debug_exit(vcpu);
2734                 return 0;
2735         }
2736
2737         if (vcpu->sigset_active)
2738                 sigprocmask(SIG_SETMASK, &vcpu->sigset, &sigsaved);
2739
2740         if (!kvm_s390_user_cpu_state_ctrl(vcpu->kvm)) {
2741                 kvm_s390_vcpu_start(vcpu);
2742         } else if (is_vcpu_stopped(vcpu)) {
2743                 pr_err_ratelimited("can't run stopped vcpu %d\n",
2744                                    vcpu->vcpu_id);
2745                 return -EINVAL;
2746         }
2747
2748         sync_regs(vcpu, kvm_run);
2749         enable_cpu_timer_accounting(vcpu);
2750
2751         might_fault();
2752         rc = __vcpu_run(vcpu);
2753
2754         if (signal_pending(current) && !rc) {
2755                 kvm_run->exit_reason = KVM_EXIT_INTR;
2756                 rc = -EINTR;
2757         }
2758
2759         if (guestdbg_exit_pending(vcpu) && !rc)  {
2760                 kvm_s390_prepare_debug_exit(vcpu);
2761                 rc = 0;
2762         }
2763
2764         if (rc == -EREMOTE) {
2765                 /* userspace support is needed, kvm_run has been prepared */
2766                 rc = 0;
2767         }
2768
2769         disable_cpu_timer_accounting(vcpu);
2770         store_regs(vcpu, kvm_run);
2771
2772         if (vcpu->sigset_active)
2773                 sigprocmask(SIG_SETMASK, &sigsaved, NULL);
2774
2775         vcpu->stat.exit_userspace++;
2776         return rc;
2777 }
2778
2779 /*
2780  * store status at address
2781  * we use have two special cases:
2782  * KVM_S390_STORE_STATUS_NOADDR: -> 0x1200 on 64 bit
2783  * KVM_S390_STORE_STATUS_PREFIXED: -> prefix
2784  */
2785 int kvm_s390_store_status_unloaded(struct kvm_vcpu *vcpu, unsigned long gpa)
2786 {
2787         unsigned char archmode = 1;
2788         freg_t fprs[NUM_FPRS];
2789         unsigned int px;
2790         u64 clkcomp, cputm;
2791         int rc;
2792
2793         px = kvm_s390_get_prefix(vcpu);
2794         if (gpa == KVM_S390_STORE_STATUS_NOADDR) {
2795                 if (write_guest_abs(vcpu, 163, &archmode, 1))
2796                         return -EFAULT;
2797                 gpa = 0;
2798         } else if (gpa == KVM_S390_STORE_STATUS_PREFIXED) {
2799                 if (write_guest_real(vcpu, 163, &archmode, 1))
2800                         return -EFAULT;
2801                 gpa = px;
2802         } else
2803                 gpa -= __LC_FPREGS_SAVE_AREA;
2804
2805         /* manually convert vector registers if necessary */
2806         if (MACHINE_HAS_VX) {
2807                 convert_vx_to_fp(fprs, (__vector128 *) vcpu->run->s.regs.vrs);
2808                 rc = write_guest_abs(vcpu, gpa + __LC_FPREGS_SAVE_AREA,
2809                                      fprs, 128);
2810         } else {
2811                 rc = write_guest_abs(vcpu, gpa + __LC_FPREGS_SAVE_AREA,
2812                                      vcpu->run->s.regs.fprs, 128);
2813         }
2814         rc |= write_guest_abs(vcpu, gpa + __LC_GPREGS_SAVE_AREA,
2815                               vcpu->run->s.regs.gprs, 128);
2816         rc |= write_guest_abs(vcpu, gpa + __LC_PSW_SAVE_AREA,
2817                               &vcpu->arch.sie_block->gpsw, 16);
2818         rc |= write_guest_abs(vcpu, gpa + __LC_PREFIX_SAVE_AREA,
2819                               &px, 4);
2820         rc |= write_guest_abs(vcpu, gpa + __LC_FP_CREG_SAVE_AREA,
2821                               &vcpu->run->s.regs.fpc, 4);
2822         rc |= write_guest_abs(vcpu, gpa + __LC_TOD_PROGREG_SAVE_AREA,
2823                               &vcpu->arch.sie_block->todpr, 4);
2824         cputm = kvm_s390_get_cpu_timer(vcpu);
2825         rc |= write_guest_abs(vcpu, gpa + __LC_CPU_TIMER_SAVE_AREA,
2826                               &cputm, 8);
2827         clkcomp = vcpu->arch.sie_block->ckc >> 8;
2828         rc |= write_guest_abs(vcpu, gpa + __LC_CLOCK_COMP_SAVE_AREA,
2829                               &clkcomp, 8);
2830         rc |= write_guest_abs(vcpu, gpa + __LC_AREGS_SAVE_AREA,
2831                               &vcpu->run->s.regs.acrs, 64);
2832         rc |= write_guest_abs(vcpu, gpa + __LC_CREGS_SAVE_AREA,
2833                               &vcpu->arch.sie_block->gcr, 128);
2834         return rc ? -EFAULT : 0;
2835 }
2836
2837 int kvm_s390_vcpu_store_status(struct kvm_vcpu *vcpu, unsigned long addr)
2838 {
2839         /*
2840          * The guest FPRS and ACRS are in the host FPRS/ACRS due to the lazy
2841          * copying in vcpu load/put. Lets update our copies before we save
2842          * it into the save area
2843          */
2844         save_fpu_regs();
2845         vcpu->run->s.regs.fpc = current->thread.fpu.fpc;
2846         save_access_regs(vcpu->run->s.regs.acrs);
2847
2848         return kvm_s390_store_status_unloaded(vcpu, addr);
2849 }
2850
2851 static void __disable_ibs_on_vcpu(struct kvm_vcpu *vcpu)
2852 {
2853         kvm_check_request(KVM_REQ_ENABLE_IBS, vcpu);
2854         kvm_s390_sync_request(KVM_REQ_DISABLE_IBS, vcpu);
2855 }
2856
2857 static void __disable_ibs_on_all_vcpus(struct kvm *kvm)
2858 {
2859         unsigned int i;
2860         struct kvm_vcpu *vcpu;
2861
2862         kvm_for_each_vcpu(i, vcpu, kvm) {
2863                 __disable_ibs_on_vcpu(vcpu);
2864         }
2865 }
2866
2867 static void __enable_ibs_on_vcpu(struct kvm_vcpu *vcpu)
2868 {
2869         if (!sclp.has_ibs)
2870                 return;
2871         kvm_check_request(KVM_REQ_DISABLE_IBS, vcpu);
2872         kvm_s390_sync_request(KVM_REQ_ENABLE_IBS, vcpu);
2873 }
2874
2875 void kvm_s390_vcpu_start(struct kvm_vcpu *vcpu)
2876 {
2877         int i, online_vcpus, started_vcpus = 0;
2878
2879         if (!is_vcpu_stopped(vcpu))
2880                 return;
2881
2882         trace_kvm_s390_vcpu_start_stop(vcpu->vcpu_id, 1);
2883         /* Only one cpu at a time may enter/leave the STOPPED state. */
2884         spin_lock(&vcpu->kvm->arch.start_stop_lock);
2885         online_vcpus = atomic_read(&vcpu->kvm->online_vcpus);
2886
2887         for (i = 0; i < online_vcpus; i++) {
2888                 if (!is_vcpu_stopped(vcpu->kvm->vcpus[i]))
2889                         started_vcpus++;
2890         }
2891
2892         if (started_vcpus == 0) {
2893                 /* we're the only active VCPU -> speed it up */
2894                 __enable_ibs_on_vcpu(vcpu);
2895         } else if (started_vcpus == 1) {
2896                 /*
2897                  * As we are starting a second VCPU, we have to disable
2898                  * the IBS facility on all VCPUs to remove potentially
2899                  * oustanding ENABLE requests.
2900                  */
2901                 __disable_ibs_on_all_vcpus(vcpu->kvm);
2902         }
2903
2904         atomic_andnot(CPUSTAT_STOPPED, &vcpu->arch.sie_block->cpuflags);
2905         /*
2906          * Another VCPU might have used IBS while we were offline.
2907          * Let's play safe and flush the VCPU at startup.
2908          */
2909         kvm_make_request(KVM_REQ_TLB_FLUSH, vcpu);
2910         spin_unlock(&vcpu->kvm->arch.start_stop_lock);
2911         return;
2912 }
2913
2914 void kvm_s390_vcpu_stop(struct kvm_vcpu *vcpu)
2915 {
2916         int i, online_vcpus, started_vcpus = 0;
2917         struct kvm_vcpu *started_vcpu = NULL;
2918
2919         if (is_vcpu_stopped(vcpu))
2920                 return;
2921
2922         trace_kvm_s390_vcpu_start_stop(vcpu->vcpu_id, 0);
2923         /* Only one cpu at a time may enter/leave the STOPPED state. */
2924         spin_lock(&vcpu->kvm->arch.start_stop_lock);
2925         online_vcpus = atomic_read(&vcpu->kvm->online_vcpus);
2926
2927         /* SIGP STOP and SIGP STOP AND STORE STATUS has been fully processed */
2928         kvm_s390_clear_stop_irq(vcpu);
2929
2930         atomic_or(CPUSTAT_STOPPED, &vcpu->arch.sie_block->cpuflags);
2931         __disable_ibs_on_vcpu(vcpu);
2932
2933         for (i = 0; i < online_vcpus; i++) {
2934                 if (!is_vcpu_stopped(vcpu->kvm->vcpus[i])) {
2935                         started_vcpus++;
2936                         started_vcpu = vcpu->kvm->vcpus[i];
2937                 }
2938         }
2939
2940         if (started_vcpus == 1) {
2941                 /*
2942                  * As we only have one VCPU left, we want to enable the
2943                  * IBS facility for that VCPU to speed it up.
2944                  */
2945                 __enable_ibs_on_vcpu(started_vcpu);
2946         }
2947
2948         spin_unlock(&vcpu->kvm->arch.start_stop_lock);
2949         return;
2950 }
2951
2952 static int kvm_vcpu_ioctl_enable_cap(struct kvm_vcpu *vcpu,
2953                                      struct kvm_enable_cap *cap)
2954 {
2955         int r;
2956
2957         if (cap->flags)
2958                 return -EINVAL;
2959
2960         switch (cap->cap) {
2961         case KVM_CAP_S390_CSS_SUPPORT:
2962                 if (!vcpu->kvm->arch.css_support) {
2963                         vcpu->kvm->arch.css_support = 1;
2964                         VM_EVENT(vcpu->kvm, 3, "%s", "ENABLE: CSS support");
2965                         trace_kvm_s390_enable_css(vcpu->kvm);
2966                 }
2967                 r = 0;
2968                 break;
2969         default:
2970                 r = -EINVAL;
2971                 break;
2972         }
2973         return r;
2974 }
2975
2976 static long kvm_s390_guest_mem_op(struct kvm_vcpu *vcpu,
2977                                   struct kvm_s390_mem_op *mop)
2978 {
2979         void __user *uaddr = (void __user *)mop->buf;
2980         void *tmpbuf = NULL;
2981         int r, srcu_idx;
2982         const u64 supported_flags = KVM_S390_MEMOP_F_INJECT_EXCEPTION
2983                                     | KVM_S390_MEMOP_F_CHECK_ONLY;
2984
2985         if (mop->flags & ~supported_flags)
2986                 return -EINVAL;
2987
2988         if (mop->size > MEM_OP_MAX_SIZE)
2989                 return -E2BIG;
2990
2991         if (!(mop->flags & KVM_S390_MEMOP_F_CHECK_ONLY)) {
2992                 tmpbuf = vmalloc(mop->size);
2993                 if (!tmpbuf)
2994                         return -ENOMEM;
2995         }
2996
2997         srcu_idx = srcu_read_lock(&vcpu->kvm->srcu);
2998
2999         switch (mop->op) {
3000         case KVM_S390_MEMOP_LOGICAL_READ:
3001                 if (mop->flags & KVM_S390_MEMOP_F_CHECK_ONLY) {
3002                         r = check_gva_range(vcpu, mop->gaddr, mop->ar,
3003                                             mop->size, GACC_FETCH);
3004                         break;
3005                 }
3006                 r = read_guest(vcpu, mop->gaddr, mop->ar, tmpbuf, mop->size);
3007                 if (r == 0) {
3008                         if (copy_to_user(uaddr, tmpbuf, mop->size))
3009                                 r = -EFAULT;
3010                 }
3011                 break;
3012         case KVM_S390_MEMOP_LOGICAL_WRITE:
3013                 if (mop->flags & KVM_S390_MEMOP_F_CHECK_ONLY) {
3014                         r = check_gva_range(vcpu, mop->gaddr, mop->ar,
3015                                             mop->size, GACC_STORE);
3016                         break;
3017                 }
3018                 if (copy_from_user(tmpbuf, uaddr, mop->size)) {
3019                         r = -EFAULT;
3020                         break;
3021                 }
3022                 r = write_guest(vcpu, mop->gaddr, mop->ar, tmpbuf, mop->size);
3023                 break;
3024         default:
3025                 r = -EINVAL;
3026         }
3027
3028         srcu_read_unlock(&vcpu->kvm->srcu, srcu_idx);
3029
3030         if (r > 0 && (mop->flags & KVM_S390_MEMOP_F_INJECT_EXCEPTION) != 0)
3031                 kvm_s390_inject_prog_irq(vcpu, &vcpu->arch.pgm);
3032
3033         vfree(tmpbuf);
3034         return r;
3035 }
3036
3037 long kvm_arch_vcpu_ioctl(struct file *filp,
3038                          unsigned int ioctl, unsigned long arg)
3039 {
3040         struct kvm_vcpu *vcpu = filp->private_data;
3041         void __user *argp = (void __user *)arg;
3042         int idx;
3043         long r;
3044
3045         switch (ioctl) {
3046         case KVM_S390_IRQ: {
3047                 struct kvm_s390_irq s390irq;
3048
3049                 r = -EFAULT;
3050                 if (copy_from_user(&s390irq, argp, sizeof(s390irq)))
3051                         break;
3052                 r = kvm_s390_inject_vcpu(vcpu, &s390irq);
3053                 break;
3054         }
3055         case KVM_S390_INTERRUPT: {
3056                 struct kvm_s390_interrupt s390int;
3057                 struct kvm_s390_irq s390irq;
3058
3059                 r = -EFAULT;
3060                 if (copy_from_user(&s390int, argp, sizeof(s390int)))
3061                         break;
3062                 if (s390int_to_s390irq(&s390int, &s390irq))
3063                         return -EINVAL;
3064                 r = kvm_s390_inject_vcpu(vcpu, &s390irq);
3065                 break;
3066         }
3067         case KVM_S390_STORE_STATUS:
3068                 idx = srcu_read_lock(&vcpu->kvm->srcu);
3069                 r = kvm_s390_vcpu_store_status(vcpu, arg);
3070                 srcu_read_unlock(&vcpu->kvm->srcu, idx);
3071                 break;
3072         case KVM_S390_SET_INITIAL_PSW: {
3073                 psw_t psw;
3074
3075                 r = -EFAULT;
3076                 if (copy_from_user(&psw, argp, sizeof(psw)))
3077                         break;
3078                 r = kvm_arch_vcpu_ioctl_set_initial_psw(vcpu, psw);
3079                 break;
3080         }
3081         case KVM_S390_INITIAL_RESET:
3082                 r = kvm_arch_vcpu_ioctl_initial_reset(vcpu);
3083                 break;
3084         case KVM_SET_ONE_REG:
3085         case KVM_GET_ONE_REG: {
3086                 struct kvm_one_reg reg;
3087                 r = -EFAULT;
3088                 if (copy_from_user(&reg, argp, sizeof(reg)))
3089                         break;
3090                 if (ioctl == KVM_SET_ONE_REG)
3091                         r = kvm_arch_vcpu_ioctl_set_one_reg(vcpu, &reg);
3092                 else
3093                         r = kvm_arch_vcpu_ioctl_get_one_reg(vcpu, &reg);
3094                 break;
3095         }
3096 #ifdef CONFIG_KVM_S390_UCONTROL
3097         case KVM_S390_UCAS_MAP: {
3098                 struct kvm_s390_ucas_mapping ucasmap;
3099
3100                 if (copy_from_user(&ucasmap, argp, sizeof(ucasmap))) {
3101                         r = -EFAULT;
3102                         break;
3103                 }
3104
3105                 if (!kvm_is_ucontrol(vcpu->kvm)) {
3106                         r = -EINVAL;
3107                         break;
3108                 }
3109
3110                 r = gmap_map_segment(vcpu->arch.gmap, ucasmap.user_addr,
3111                                      ucasmap.vcpu_addr, ucasmap.length);
3112                 break;
3113         }
3114         case KVM_S390_UCAS_UNMAP: {
3115                 struct kvm_s390_ucas_mapping ucasmap;
3116
3117                 if (copy_from_user(&ucasmap, argp, sizeof(ucasmap))) {
3118                         r = -EFAULT;
3119                         break;
3120                 }
3121
3122                 if (!kvm_is_ucontrol(vcpu->kvm)) {
3123                         r = -EINVAL;
3124                         break;
3125                 }
3126
3127                 r = gmap_unmap_segment(vcpu->arch.gmap, ucasmap.vcpu_addr,
3128                         ucasmap.length);
3129                 break;
3130         }
3131 #endif
3132         case KVM_S390_VCPU_FAULT: {
3133                 r = gmap_fault(vcpu->arch.gmap, arg, 0);
3134                 break;
3135         }
3136         case KVM_ENABLE_CAP:
3137         {
3138                 struct kvm_enable_cap cap;
3139                 r = -EFAULT;
3140                 if (copy_from_user(&cap, argp, sizeof(cap)))
3141                         break;
3142                 r = kvm_vcpu_ioctl_enable_cap(vcpu, &cap);
3143                 break;
3144         }
3145         case KVM_S390_MEM_OP: {
3146                 struct kvm_s390_mem_op mem_op;
3147
3148                 if (copy_from_user(&mem_op, argp, sizeof(mem_op)) == 0)
3149                         r = kvm_s390_guest_mem_op(vcpu, &mem_op);
3150                 else
3151                         r = -EFAULT;
3152                 break;
3153         }
3154         case KVM_S390_SET_IRQ_STATE: {
3155                 struct kvm_s390_irq_state irq_state;
3156
3157                 r = -EFAULT;
3158                 if (copy_from_user(&irq_state, argp, sizeof(irq_state)))
3159                         break;
3160                 if (irq_state.len > VCPU_IRQS_MAX_BUF ||
3161                     irq_state.len == 0 ||
3162                     irq_state.len % sizeof(struct kvm_s390_irq) > 0) {
3163                         r = -EINVAL;
3164                         break;
3165                 }
3166                 r = kvm_s390_set_irq_state(vcpu,
3167                                            (void __user *) irq_state.buf,
3168                                            irq_state.len);
3169                 break;
3170         }
3171         case KVM_S390_GET_IRQ_STATE: {
3172                 struct kvm_s390_irq_state irq_state;
3173
3174                 r = -EFAULT;
3175                 if (copy_from_user(&irq_state, argp, sizeof(irq_state)))
3176                         break;
3177                 if (irq_state.len == 0) {
3178                         r = -EINVAL;
3179                         break;
3180                 }
3181                 r = kvm_s390_get_irq_state(vcpu,
3182                                            (__u8 __user *)  irq_state.buf,
3183                                            irq_state.len);
3184                 break;
3185         }
3186         default:
3187                 r = -ENOTTY;
3188         }
3189         return r;
3190 }
3191
3192 int kvm_arch_vcpu_fault(struct kvm_vcpu *vcpu, struct vm_fault *vmf)
3193 {
3194 #ifdef CONFIG_KVM_S390_UCONTROL
3195         if ((vmf->pgoff == KVM_S390_SIE_PAGE_OFFSET)
3196                  && (kvm_is_ucontrol(vcpu->kvm))) {
3197                 vmf->page = virt_to_page(vcpu->arch.sie_block);
3198                 get_page(vmf->page);
3199                 return 0;
3200         }
3201 #endif
3202         return VM_FAULT_SIGBUS;
3203 }
3204
3205 int kvm_arch_create_memslot(struct kvm *kvm, struct kvm_memory_slot *slot,
3206                             unsigned long npages)
3207 {
3208         return 0;
3209 }
3210
3211 /* Section: memory related */
3212 int kvm_arch_prepare_memory_region(struct kvm *kvm,
3213                                    struct kvm_memory_slot *memslot,
3214                                    const struct kvm_userspace_memory_region *mem,
3215                                    enum kvm_mr_change change)
3216 {
3217         /* A few sanity checks. We can have memory slots which have to be
3218            located/ended at a segment boundary (1MB). The memory in userland is
3219            ok to be fragmented into various different vmas. It is okay to mmap()
3220            and munmap() stuff in this slot after doing this call at any time */
3221
3222         if (mem->userspace_addr & 0xffffful)
3223                 return -EINVAL;
3224
3225         if (mem->memory_size & 0xffffful)
3226                 return -EINVAL;
3227
3228         if (mem->guest_phys_addr + mem->memory_size > kvm->arch.mem_limit)
3229                 return -EINVAL;
3230
3231         return 0;
3232 }
3233
3234 void kvm_arch_commit_memory_region(struct kvm *kvm,
3235                                 const struct kvm_userspace_memory_region *mem,
3236                                 const struct kvm_memory_slot *old,
3237                                 const struct kvm_memory_slot *new,
3238                                 enum kvm_mr_change change)
3239 {
3240         int rc;
3241
3242         /* If the basics of the memslot do not change, we do not want
3243          * to update the gmap. Every update causes several unnecessary
3244          * segment translation exceptions. This is usually handled just
3245          * fine by the normal fault handler + gmap, but it will also
3246          * cause faults on the prefix page of running guest CPUs.
3247          */
3248         if (old->userspace_addr == mem->userspace_addr &&
3249             old->base_gfn * PAGE_SIZE == mem->guest_phys_addr &&
3250             old->npages * PAGE_SIZE == mem->memory_size)
3251                 return;
3252
3253         rc = gmap_map_segment(kvm->arch.gmap, mem->userspace_addr,
3254                 mem->guest_phys_addr, mem->memory_size);
3255         if (rc)
3256                 pr_warn("failed to commit memory region\n");
3257         return;
3258 }
3259
3260 static inline unsigned long nonhyp_mask(int i)
3261 {
3262         unsigned int nonhyp_fai = (sclp.hmfai << i * 2) >> 30;
3263
3264         return 0x0000ffffffffffffUL >> (nonhyp_fai << 4);
3265 }
3266
3267 void kvm_arch_vcpu_block_finish(struct kvm_vcpu *vcpu)
3268 {
3269         vcpu->valid_wakeup = false;
3270 }
3271
3272 static int __init kvm_s390_init(void)
3273 {
3274         int i;
3275
3276         if (!sclp.has_sief2) {
3277                 pr_info("SIE not available\n");
3278                 return -ENODEV;
3279         }
3280
3281         for (i = 0; i < 16; i++)
3282                 kvm_s390_fac_list_mask[i] |=
3283                         S390_lowcore.stfle_fac_list[i] & nonhyp_mask(i);
3284
3285         return kvm_init(NULL, sizeof(struct kvm_vcpu), 0, THIS_MODULE);
3286 }
3287
3288 static void __exit kvm_s390_exit(void)
3289 {
3290         kvm_exit();
3291 }
3292
3293 module_init(kvm_s390_init);
3294 module_exit(kvm_s390_exit);
3295
3296 /*
3297  * Enable autoloading of the kvm module.
3298  * Note that we add the module alias here instead of virt/kvm/kvm_main.c
3299  * since x86 takes a different approach.
3300  */
3301 #include <linux/miscdevice.h>
3302 MODULE_ALIAS_MISCDEV(KVM_MINOR);
3303 MODULE_ALIAS("devname:kvm");