1 // SPDX-License-Identifier: GPL-2.0
3 * hosting IBM Z kernel virtual machines (s390x)
5 * Copyright IBM Corp. 2008, 2018
7 * Author(s): Carsten Otte <cotte@de.ibm.com>
8 * Christian Borntraeger <borntraeger@de.ibm.com>
9 * Heiko Carstens <heiko.carstens@de.ibm.com>
10 * Christian Ehrhardt <ehrhardt@de.ibm.com>
11 * Jason J. Herne <jjherne@us.ibm.com>
14 #define KMSG_COMPONENT "kvm-s390"
15 #define pr_fmt(fmt) KMSG_COMPONENT ": " fmt
17 #include <linux/compiler.h>
18 #include <linux/err.h>
20 #include <linux/hrtimer.h>
21 #include <linux/init.h>
22 #include <linux/kvm.h>
23 #include <linux/kvm_host.h>
24 #include <linux/mman.h>
25 #include <linux/module.h>
26 #include <linux/moduleparam.h>
27 #include <linux/random.h>
28 #include <linux/slab.h>
29 #include <linux/timer.h>
30 #include <linux/vmalloc.h>
31 #include <linux/bitmap.h>
32 #include <linux/sched/signal.h>
33 #include <linux/string.h>
35 #include <asm/asm-offsets.h>
36 #include <asm/lowcore.h>
38 #include <asm/pgtable.h>
41 #include <asm/switch_to.h>
44 #include <asm/cpacf.h>
45 #include <asm/timex.h>
50 #define CREATE_TRACE_POINTS
52 #include "trace-s390.h"
54 #define MEM_OP_MAX_SIZE 65536 /* Maximum transfer size for KVM_S390_MEM_OP */
56 #define VCPU_IRQS_MAX_BUF (sizeof(struct kvm_s390_irq) * \
57 (KVM_MAX_VCPUS + LOCAL_IRQS))
59 #define VCPU_STAT(x) offsetof(struct kvm_vcpu, stat.x), KVM_STAT_VCPU
60 #define VM_STAT(x) offsetof(struct kvm, stat.x), KVM_STAT_VM
62 struct kvm_stats_debugfs_item debugfs_entries[] = {
63 { "userspace_handled", VCPU_STAT(exit_userspace) },
64 { "exit_null", VCPU_STAT(exit_null) },
65 { "exit_validity", VCPU_STAT(exit_validity) },
66 { "exit_stop_request", VCPU_STAT(exit_stop_request) },
67 { "exit_external_request", VCPU_STAT(exit_external_request) },
68 { "exit_io_request", VCPU_STAT(exit_io_request) },
69 { "exit_external_interrupt", VCPU_STAT(exit_external_interrupt) },
70 { "exit_instruction", VCPU_STAT(exit_instruction) },
71 { "exit_pei", VCPU_STAT(exit_pei) },
72 { "exit_program_interruption", VCPU_STAT(exit_program_interruption) },
73 { "exit_instr_and_program_int", VCPU_STAT(exit_instr_and_program) },
74 { "exit_operation_exception", VCPU_STAT(exit_operation_exception) },
75 { "halt_successful_poll", VCPU_STAT(halt_successful_poll) },
76 { "halt_attempted_poll", VCPU_STAT(halt_attempted_poll) },
77 { "halt_poll_invalid", VCPU_STAT(halt_poll_invalid) },
78 { "halt_no_poll_steal", VCPU_STAT(halt_no_poll_steal) },
79 { "halt_wakeup", VCPU_STAT(halt_wakeup) },
80 { "instruction_lctlg", VCPU_STAT(instruction_lctlg) },
81 { "instruction_lctl", VCPU_STAT(instruction_lctl) },
82 { "instruction_stctl", VCPU_STAT(instruction_stctl) },
83 { "instruction_stctg", VCPU_STAT(instruction_stctg) },
84 { "deliver_ckc", VCPU_STAT(deliver_ckc) },
85 { "deliver_cputm", VCPU_STAT(deliver_cputm) },
86 { "deliver_emergency_signal", VCPU_STAT(deliver_emergency_signal) },
87 { "deliver_external_call", VCPU_STAT(deliver_external_call) },
88 { "deliver_service_signal", VCPU_STAT(deliver_service_signal) },
89 { "deliver_virtio", VCPU_STAT(deliver_virtio) },
90 { "deliver_stop_signal", VCPU_STAT(deliver_stop_signal) },
91 { "deliver_prefix_signal", VCPU_STAT(deliver_prefix_signal) },
92 { "deliver_restart_signal", VCPU_STAT(deliver_restart_signal) },
93 { "deliver_program", VCPU_STAT(deliver_program) },
94 { "deliver_io", VCPU_STAT(deliver_io) },
95 { "deliver_machine_check", VCPU_STAT(deliver_machine_check) },
96 { "exit_wait_state", VCPU_STAT(exit_wait_state) },
97 { "inject_ckc", VCPU_STAT(inject_ckc) },
98 { "inject_cputm", VCPU_STAT(inject_cputm) },
99 { "inject_external_call", VCPU_STAT(inject_external_call) },
100 { "inject_float_mchk", VM_STAT(inject_float_mchk) },
101 { "inject_emergency_signal", VCPU_STAT(inject_emergency_signal) },
102 { "inject_io", VM_STAT(inject_io) },
103 { "inject_mchk", VCPU_STAT(inject_mchk) },
104 { "inject_pfault_done", VM_STAT(inject_pfault_done) },
105 { "inject_program", VCPU_STAT(inject_program) },
106 { "inject_restart", VCPU_STAT(inject_restart) },
107 { "inject_service_signal", VM_STAT(inject_service_signal) },
108 { "inject_set_prefix", VCPU_STAT(inject_set_prefix) },
109 { "inject_stop_signal", VCPU_STAT(inject_stop_signal) },
110 { "inject_pfault_init", VCPU_STAT(inject_pfault_init) },
111 { "inject_virtio", VM_STAT(inject_virtio) },
112 { "instruction_epsw", VCPU_STAT(instruction_epsw) },
113 { "instruction_gs", VCPU_STAT(instruction_gs) },
114 { "instruction_io_other", VCPU_STAT(instruction_io_other) },
115 { "instruction_lpsw", VCPU_STAT(instruction_lpsw) },
116 { "instruction_lpswe", VCPU_STAT(instruction_lpswe) },
117 { "instruction_pfmf", VCPU_STAT(instruction_pfmf) },
118 { "instruction_ptff", VCPU_STAT(instruction_ptff) },
119 { "instruction_stidp", VCPU_STAT(instruction_stidp) },
120 { "instruction_sck", VCPU_STAT(instruction_sck) },
121 { "instruction_sckpf", VCPU_STAT(instruction_sckpf) },
122 { "instruction_spx", VCPU_STAT(instruction_spx) },
123 { "instruction_stpx", VCPU_STAT(instruction_stpx) },
124 { "instruction_stap", VCPU_STAT(instruction_stap) },
125 { "instruction_iske", VCPU_STAT(instruction_iske) },
126 { "instruction_ri", VCPU_STAT(instruction_ri) },
127 { "instruction_rrbe", VCPU_STAT(instruction_rrbe) },
128 { "instruction_sske", VCPU_STAT(instruction_sske) },
129 { "instruction_ipte_interlock", VCPU_STAT(instruction_ipte_interlock) },
130 { "instruction_essa", VCPU_STAT(instruction_essa) },
131 { "instruction_stsi", VCPU_STAT(instruction_stsi) },
132 { "instruction_stfl", VCPU_STAT(instruction_stfl) },
133 { "instruction_tb", VCPU_STAT(instruction_tb) },
134 { "instruction_tpi", VCPU_STAT(instruction_tpi) },
135 { "instruction_tprot", VCPU_STAT(instruction_tprot) },
136 { "instruction_tsch", VCPU_STAT(instruction_tsch) },
137 { "instruction_sthyi", VCPU_STAT(instruction_sthyi) },
138 { "instruction_sie", VCPU_STAT(instruction_sie) },
139 { "instruction_sigp_sense", VCPU_STAT(instruction_sigp_sense) },
140 { "instruction_sigp_sense_running", VCPU_STAT(instruction_sigp_sense_running) },
141 { "instruction_sigp_external_call", VCPU_STAT(instruction_sigp_external_call) },
142 { "instruction_sigp_emergency", VCPU_STAT(instruction_sigp_emergency) },
143 { "instruction_sigp_cond_emergency", VCPU_STAT(instruction_sigp_cond_emergency) },
144 { "instruction_sigp_start", VCPU_STAT(instruction_sigp_start) },
145 { "instruction_sigp_stop", VCPU_STAT(instruction_sigp_stop) },
146 { "instruction_sigp_stop_store_status", VCPU_STAT(instruction_sigp_stop_store_status) },
147 { "instruction_sigp_store_status", VCPU_STAT(instruction_sigp_store_status) },
148 { "instruction_sigp_store_adtl_status", VCPU_STAT(instruction_sigp_store_adtl_status) },
149 { "instruction_sigp_set_arch", VCPU_STAT(instruction_sigp_arch) },
150 { "instruction_sigp_set_prefix", VCPU_STAT(instruction_sigp_prefix) },
151 { "instruction_sigp_restart", VCPU_STAT(instruction_sigp_restart) },
152 { "instruction_sigp_cpu_reset", VCPU_STAT(instruction_sigp_cpu_reset) },
153 { "instruction_sigp_init_cpu_reset", VCPU_STAT(instruction_sigp_init_cpu_reset) },
154 { "instruction_sigp_unknown", VCPU_STAT(instruction_sigp_unknown) },
155 { "instruction_diag_10", VCPU_STAT(diagnose_10) },
156 { "instruction_diag_44", VCPU_STAT(diagnose_44) },
157 { "instruction_diag_9c", VCPU_STAT(diagnose_9c) },
158 { "diag_9c_ignored", VCPU_STAT(diagnose_9c_ignored) },
159 { "instruction_diag_258", VCPU_STAT(diagnose_258) },
160 { "instruction_diag_308", VCPU_STAT(diagnose_308) },
161 { "instruction_diag_500", VCPU_STAT(diagnose_500) },
162 { "instruction_diag_other", VCPU_STAT(diagnose_other) },
166 struct kvm_s390_tod_clock_ext {
172 /* allow nested virtualization in KVM (if enabled by user space) */
174 module_param(nested, int, S_IRUGO);
175 MODULE_PARM_DESC(nested, "Nested virtualization support");
177 /* allow 1m huge page guest backing, if !nested */
179 module_param(hpage, int, 0444);
180 MODULE_PARM_DESC(hpage, "1m huge page backing support");
182 /* maximum percentage of steal time for polling. >100 is treated like 100 */
183 static u8 halt_poll_max_steal = 10;
184 module_param(halt_poll_max_steal, byte, 0644);
185 MODULE_PARM_DESC(halt_poll_max_steal, "Maximum percentage of steal time to allow polling");
188 * For now we handle at most 16 double words as this is what the s390 base
189 * kernel handles and stores in the prefix page. If we ever need to go beyond
190 * this, this requires changes to code, but the external uapi can stay.
192 #define SIZE_INTERNAL 16
195 * Base feature mask that defines default mask for facilities. Consists of the
196 * defines in FACILITIES_KVM and the non-hypervisor managed bits.
198 static unsigned long kvm_s390_fac_base[SIZE_INTERNAL] = { FACILITIES_KVM };
200 * Extended feature mask. Consists of the defines in FACILITIES_KVM_CPUMODEL
201 * and defines the facilities that can be enabled via a cpu model.
203 static unsigned long kvm_s390_fac_ext[SIZE_INTERNAL] = { FACILITIES_KVM_CPUMODEL };
205 static unsigned long kvm_s390_fac_size(void)
207 BUILD_BUG_ON(SIZE_INTERNAL > S390_ARCH_FAC_MASK_SIZE_U64);
208 BUILD_BUG_ON(SIZE_INTERNAL > S390_ARCH_FAC_LIST_SIZE_U64);
209 BUILD_BUG_ON(SIZE_INTERNAL * sizeof(unsigned long) >
210 sizeof(S390_lowcore.stfle_fac_list));
212 return SIZE_INTERNAL;
215 /* available cpu features supported by kvm */
216 static DECLARE_BITMAP(kvm_s390_available_cpu_feat, KVM_S390_VM_CPU_FEAT_NR_BITS);
217 /* available subfunctions indicated via query / "test bit" */
218 static struct kvm_s390_vm_cpu_subfunc kvm_s390_available_subfunc;
220 static struct gmap_notifier gmap_notifier;
221 static struct gmap_notifier vsie_gmap_notifier;
222 debug_info_t *kvm_s390_dbf;
224 /* Section: not file related */
225 int kvm_arch_hardware_enable(void)
227 /* every s390 is virtualization enabled ;-) */
231 int kvm_arch_check_processor_compat(void)
236 static void kvm_gmap_notifier(struct gmap *gmap, unsigned long start,
239 static void kvm_clock_sync_scb(struct kvm_s390_sie_block *scb, u64 delta)
244 * The TOD jumps by delta, we have to compensate this by adding
245 * -delta to the epoch.
249 /* sign-extension - we're adding to signed values below */
254 if (scb->ecd & ECD_MEF) {
255 scb->epdx += delta_idx;
256 if (scb->epoch < delta)
262 * This callback is executed during stop_machine(). All CPUs are therefore
263 * temporarily stopped. In order not to change guest behavior, we have to
264 * disable preemption whenever we touch the epoch of kvm and the VCPUs,
265 * so a CPU won't be stopped while calculating with the epoch.
267 static int kvm_clock_sync(struct notifier_block *notifier, unsigned long val,
271 struct kvm_vcpu *vcpu;
273 unsigned long long *delta = v;
275 list_for_each_entry(kvm, &vm_list, vm_list) {
276 kvm_for_each_vcpu(i, vcpu, kvm) {
277 kvm_clock_sync_scb(vcpu->arch.sie_block, *delta);
279 kvm->arch.epoch = vcpu->arch.sie_block->epoch;
280 kvm->arch.epdx = vcpu->arch.sie_block->epdx;
282 if (vcpu->arch.cputm_enabled)
283 vcpu->arch.cputm_start += *delta;
284 if (vcpu->arch.vsie_block)
285 kvm_clock_sync_scb(vcpu->arch.vsie_block,
292 static struct notifier_block kvm_clock_notifier = {
293 .notifier_call = kvm_clock_sync,
296 int kvm_arch_hardware_setup(void)
298 gmap_notifier.notifier_call = kvm_gmap_notifier;
299 gmap_register_pte_notifier(&gmap_notifier);
300 vsie_gmap_notifier.notifier_call = kvm_s390_vsie_gmap_notifier;
301 gmap_register_pte_notifier(&vsie_gmap_notifier);
302 atomic_notifier_chain_register(&s390_epoch_delta_notifier,
303 &kvm_clock_notifier);
307 void kvm_arch_hardware_unsetup(void)
309 gmap_unregister_pte_notifier(&gmap_notifier);
310 gmap_unregister_pte_notifier(&vsie_gmap_notifier);
311 atomic_notifier_chain_unregister(&s390_epoch_delta_notifier,
312 &kvm_clock_notifier);
315 static void allow_cpu_feat(unsigned long nr)
317 set_bit_inv(nr, kvm_s390_available_cpu_feat);
320 static inline int plo_test_bit(unsigned char nr)
322 register unsigned long r0 asm("0") = (unsigned long) nr | 0x100;
326 /* Parameter registers are ignored for "test bit" */
336 static __always_inline void __insn32_query(unsigned int opcode, u8 *query)
338 register unsigned long r0 asm("0") = 0; /* query function */
339 register unsigned long r1 asm("1") = (unsigned long) query;
342 /* Parameter regs are ignored */
343 " .insn rrf,%[opc] << 16,2,4,6,0\n"
345 : "d" (r0), "a" (r1), [opc] "i" (opcode)
349 #define INSN_SORTL 0xb938
350 #define INSN_DFLTCC 0xb939
352 static void kvm_s390_cpu_feat_init(void)
356 for (i = 0; i < 256; ++i) {
358 kvm_s390_available_subfunc.plo[i >> 3] |= 0x80 >> (i & 7);
361 if (test_facility(28)) /* TOD-clock steering */
362 ptff(kvm_s390_available_subfunc.ptff,
363 sizeof(kvm_s390_available_subfunc.ptff),
366 if (test_facility(17)) { /* MSA */
367 __cpacf_query(CPACF_KMAC, (cpacf_mask_t *)
368 kvm_s390_available_subfunc.kmac);
369 __cpacf_query(CPACF_KMC, (cpacf_mask_t *)
370 kvm_s390_available_subfunc.kmc);
371 __cpacf_query(CPACF_KM, (cpacf_mask_t *)
372 kvm_s390_available_subfunc.km);
373 __cpacf_query(CPACF_KIMD, (cpacf_mask_t *)
374 kvm_s390_available_subfunc.kimd);
375 __cpacf_query(CPACF_KLMD, (cpacf_mask_t *)
376 kvm_s390_available_subfunc.klmd);
378 if (test_facility(76)) /* MSA3 */
379 __cpacf_query(CPACF_PCKMO, (cpacf_mask_t *)
380 kvm_s390_available_subfunc.pckmo);
381 if (test_facility(77)) { /* MSA4 */
382 __cpacf_query(CPACF_KMCTR, (cpacf_mask_t *)
383 kvm_s390_available_subfunc.kmctr);
384 __cpacf_query(CPACF_KMF, (cpacf_mask_t *)
385 kvm_s390_available_subfunc.kmf);
386 __cpacf_query(CPACF_KMO, (cpacf_mask_t *)
387 kvm_s390_available_subfunc.kmo);
388 __cpacf_query(CPACF_PCC, (cpacf_mask_t *)
389 kvm_s390_available_subfunc.pcc);
391 if (test_facility(57)) /* MSA5 */
392 __cpacf_query(CPACF_PRNO, (cpacf_mask_t *)
393 kvm_s390_available_subfunc.ppno);
395 if (test_facility(146)) /* MSA8 */
396 __cpacf_query(CPACF_KMA, (cpacf_mask_t *)
397 kvm_s390_available_subfunc.kma);
399 if (test_facility(155)) /* MSA9 */
400 __cpacf_query(CPACF_KDSA, (cpacf_mask_t *)
401 kvm_s390_available_subfunc.kdsa);
403 if (test_facility(150)) /* SORTL */
404 __insn32_query(INSN_SORTL, kvm_s390_available_subfunc.sortl);
406 if (test_facility(151)) /* DFLTCC */
407 __insn32_query(INSN_DFLTCC, kvm_s390_available_subfunc.dfltcc);
409 if (MACHINE_HAS_ESOP)
410 allow_cpu_feat(KVM_S390_VM_CPU_FEAT_ESOP);
412 * We need SIE support, ESOP (PROT_READ protection for gmap_shadow),
413 * 64bit SCAO (SCA passthrough) and IDTE (for gmap_shadow unshadowing).
415 if (!sclp.has_sief2 || !MACHINE_HAS_ESOP || !sclp.has_64bscao ||
416 !test_facility(3) || !nested)
418 allow_cpu_feat(KVM_S390_VM_CPU_FEAT_SIEF2);
419 if (sclp.has_64bscao)
420 allow_cpu_feat(KVM_S390_VM_CPU_FEAT_64BSCAO);
422 allow_cpu_feat(KVM_S390_VM_CPU_FEAT_SIIF);
424 allow_cpu_feat(KVM_S390_VM_CPU_FEAT_GPERE);
426 allow_cpu_feat(KVM_S390_VM_CPU_FEAT_GSLS);
428 allow_cpu_feat(KVM_S390_VM_CPU_FEAT_IB);
430 allow_cpu_feat(KVM_S390_VM_CPU_FEAT_CEI);
432 allow_cpu_feat(KVM_S390_VM_CPU_FEAT_IBS);
434 allow_cpu_feat(KVM_S390_VM_CPU_FEAT_KSS);
436 * KVM_S390_VM_CPU_FEAT_SKEY: Wrong shadow of PTE.I bits will make
437 * all skey handling functions read/set the skey from the PGSTE
438 * instead of the real storage key.
440 * KVM_S390_VM_CPU_FEAT_CMMA: Wrong shadow of PTE.I bits will make
441 * pages being detected as preserved although they are resident.
443 * KVM_S390_VM_CPU_FEAT_PFMFI: Wrong shadow of PTE.I bits will
444 * have the same effect as for KVM_S390_VM_CPU_FEAT_SKEY.
446 * For KVM_S390_VM_CPU_FEAT_SKEY, KVM_S390_VM_CPU_FEAT_CMMA and
447 * KVM_S390_VM_CPU_FEAT_PFMFI, all PTE.I and PGSTE bits have to be
448 * correctly shadowed. We can do that for the PGSTE but not for PTE.I.
450 * KVM_S390_VM_CPU_FEAT_SIGPIF: Wrong SCB addresses in the SCA. We
451 * cannot easily shadow the SCA because of the ipte lock.
455 int kvm_arch_init(void *opaque)
459 kvm_s390_dbf = debug_register("kvm-trace", 32, 1, 7 * sizeof(long));
463 if (debug_register_view(kvm_s390_dbf, &debug_sprintf_view))
466 kvm_s390_cpu_feat_init();
468 /* Register floating interrupt controller interface. */
469 rc = kvm_register_device_ops(&kvm_flic_ops, KVM_DEV_TYPE_FLIC);
471 pr_err("A FLIC registration call failed with rc=%d\n", rc);
475 rc = kvm_s390_gib_init(GAL_ISC);
486 void kvm_arch_exit(void)
488 kvm_s390_gib_destroy();
489 debug_unregister(kvm_s390_dbf);
492 /* Section: device related */
493 long kvm_arch_dev_ioctl(struct file *filp,
494 unsigned int ioctl, unsigned long arg)
496 if (ioctl == KVM_S390_ENABLE_SIE)
497 return s390_enable_sie();
501 int kvm_vm_ioctl_check_extension(struct kvm *kvm, long ext)
506 case KVM_CAP_S390_PSW:
507 case KVM_CAP_S390_GMAP:
508 case KVM_CAP_SYNC_MMU:
509 #ifdef CONFIG_KVM_S390_UCONTROL
510 case KVM_CAP_S390_UCONTROL:
512 case KVM_CAP_ASYNC_PF:
513 case KVM_CAP_SYNC_REGS:
514 case KVM_CAP_ONE_REG:
515 case KVM_CAP_ENABLE_CAP:
516 case KVM_CAP_S390_CSS_SUPPORT:
517 case KVM_CAP_IOEVENTFD:
518 case KVM_CAP_DEVICE_CTRL:
519 case KVM_CAP_S390_IRQCHIP:
520 case KVM_CAP_VM_ATTRIBUTES:
521 case KVM_CAP_MP_STATE:
522 case KVM_CAP_IMMEDIATE_EXIT:
523 case KVM_CAP_S390_INJECT_IRQ:
524 case KVM_CAP_S390_USER_SIGP:
525 case KVM_CAP_S390_USER_STSI:
526 case KVM_CAP_S390_SKEYS:
527 case KVM_CAP_S390_IRQ_STATE:
528 case KVM_CAP_S390_USER_INSTR0:
529 case KVM_CAP_S390_CMMA_MIGRATION:
530 case KVM_CAP_S390_AIS:
531 case KVM_CAP_S390_AIS_MIGRATION:
532 case KVM_CAP_S390_VCPU_RESETS:
535 case KVM_CAP_S390_HPAGE_1M:
537 if (hpage && !kvm_is_ucontrol(kvm))
540 case KVM_CAP_S390_MEM_OP:
543 case KVM_CAP_NR_VCPUS:
544 case KVM_CAP_MAX_VCPUS:
545 case KVM_CAP_MAX_VCPU_ID:
546 r = KVM_S390_BSCA_CPU_SLOTS;
547 if (!kvm_s390_use_sca_entries())
549 else if (sclp.has_esca && sclp.has_64bscao)
550 r = KVM_S390_ESCA_CPU_SLOTS;
552 case KVM_CAP_S390_COW:
553 r = MACHINE_HAS_ESOP;
555 case KVM_CAP_S390_VECTOR_REGISTERS:
558 case KVM_CAP_S390_RI:
559 r = test_facility(64);
561 case KVM_CAP_S390_GS:
562 r = test_facility(133);
564 case KVM_CAP_S390_BPB:
565 r = test_facility(82);
573 static void kvm_s390_sync_dirty_log(struct kvm *kvm,
574 struct kvm_memory_slot *memslot)
577 gfn_t cur_gfn, last_gfn;
578 unsigned long gaddr, vmaddr;
579 struct gmap *gmap = kvm->arch.gmap;
580 DECLARE_BITMAP(bitmap, _PAGE_ENTRIES);
582 /* Loop over all guest segments */
583 cur_gfn = memslot->base_gfn;
584 last_gfn = memslot->base_gfn + memslot->npages;
585 for (; cur_gfn <= last_gfn; cur_gfn += _PAGE_ENTRIES) {
586 gaddr = gfn_to_gpa(cur_gfn);
587 vmaddr = gfn_to_hva_memslot(memslot, cur_gfn);
588 if (kvm_is_error_hva(vmaddr))
591 bitmap_zero(bitmap, _PAGE_ENTRIES);
592 gmap_sync_dirty_log_pmd(gmap, bitmap, gaddr, vmaddr);
593 for (i = 0; i < _PAGE_ENTRIES; i++) {
594 if (test_bit(i, bitmap))
595 mark_page_dirty(kvm, cur_gfn + i);
598 if (fatal_signal_pending(current))
604 /* Section: vm related */
605 static void sca_del_vcpu(struct kvm_vcpu *vcpu);
608 * Get (and clear) the dirty memory log for a memory slot.
610 int kvm_vm_ioctl_get_dirty_log(struct kvm *kvm,
611 struct kvm_dirty_log *log)
615 struct kvm_memslots *slots;
616 struct kvm_memory_slot *memslot;
619 if (kvm_is_ucontrol(kvm))
622 mutex_lock(&kvm->slots_lock);
625 if (log->slot >= KVM_USER_MEM_SLOTS)
628 slots = kvm_memslots(kvm);
629 memslot = id_to_memslot(slots, log->slot);
631 if (!memslot->dirty_bitmap)
634 kvm_s390_sync_dirty_log(kvm, memslot);
635 r = kvm_get_dirty_log(kvm, log, &is_dirty);
639 /* Clear the dirty log */
641 n = kvm_dirty_bitmap_bytes(memslot);
642 memset(memslot->dirty_bitmap, 0, n);
646 mutex_unlock(&kvm->slots_lock);
650 static void icpt_operexc_on_all_vcpus(struct kvm *kvm)
653 struct kvm_vcpu *vcpu;
655 kvm_for_each_vcpu(i, vcpu, kvm) {
656 kvm_s390_sync_request(KVM_REQ_ICPT_OPEREXC, vcpu);
660 int kvm_vm_ioctl_enable_cap(struct kvm *kvm, struct kvm_enable_cap *cap)
668 case KVM_CAP_S390_IRQCHIP:
669 VM_EVENT(kvm, 3, "%s", "ENABLE: CAP_S390_IRQCHIP");
670 kvm->arch.use_irqchip = 1;
673 case KVM_CAP_S390_USER_SIGP:
674 VM_EVENT(kvm, 3, "%s", "ENABLE: CAP_S390_USER_SIGP");
675 kvm->arch.user_sigp = 1;
678 case KVM_CAP_S390_VECTOR_REGISTERS:
679 mutex_lock(&kvm->lock);
680 if (kvm->created_vcpus) {
682 } else if (MACHINE_HAS_VX) {
683 set_kvm_facility(kvm->arch.model.fac_mask, 129);
684 set_kvm_facility(kvm->arch.model.fac_list, 129);
685 if (test_facility(134)) {
686 set_kvm_facility(kvm->arch.model.fac_mask, 134);
687 set_kvm_facility(kvm->arch.model.fac_list, 134);
689 if (test_facility(135)) {
690 set_kvm_facility(kvm->arch.model.fac_mask, 135);
691 set_kvm_facility(kvm->arch.model.fac_list, 135);
693 if (test_facility(148)) {
694 set_kvm_facility(kvm->arch.model.fac_mask, 148);
695 set_kvm_facility(kvm->arch.model.fac_list, 148);
697 if (test_facility(152)) {
698 set_kvm_facility(kvm->arch.model.fac_mask, 152);
699 set_kvm_facility(kvm->arch.model.fac_list, 152);
704 mutex_unlock(&kvm->lock);
705 VM_EVENT(kvm, 3, "ENABLE: CAP_S390_VECTOR_REGISTERS %s",
706 r ? "(not available)" : "(success)");
708 case KVM_CAP_S390_RI:
710 mutex_lock(&kvm->lock);
711 if (kvm->created_vcpus) {
713 } else if (test_facility(64)) {
714 set_kvm_facility(kvm->arch.model.fac_mask, 64);
715 set_kvm_facility(kvm->arch.model.fac_list, 64);
718 mutex_unlock(&kvm->lock);
719 VM_EVENT(kvm, 3, "ENABLE: CAP_S390_RI %s",
720 r ? "(not available)" : "(success)");
722 case KVM_CAP_S390_AIS:
723 mutex_lock(&kvm->lock);
724 if (kvm->created_vcpus) {
727 set_kvm_facility(kvm->arch.model.fac_mask, 72);
728 set_kvm_facility(kvm->arch.model.fac_list, 72);
731 mutex_unlock(&kvm->lock);
732 VM_EVENT(kvm, 3, "ENABLE: AIS %s",
733 r ? "(not available)" : "(success)");
735 case KVM_CAP_S390_GS:
737 mutex_lock(&kvm->lock);
738 if (kvm->created_vcpus) {
740 } else if (test_facility(133)) {
741 set_kvm_facility(kvm->arch.model.fac_mask, 133);
742 set_kvm_facility(kvm->arch.model.fac_list, 133);
745 mutex_unlock(&kvm->lock);
746 VM_EVENT(kvm, 3, "ENABLE: CAP_S390_GS %s",
747 r ? "(not available)" : "(success)");
749 case KVM_CAP_S390_HPAGE_1M:
750 mutex_lock(&kvm->lock);
751 if (kvm->created_vcpus)
753 else if (!hpage || kvm->arch.use_cmma || kvm_is_ucontrol(kvm))
757 down_write(&kvm->mm->mmap_sem);
758 kvm->mm->context.allow_gmap_hpage_1m = 1;
759 up_write(&kvm->mm->mmap_sem);
761 * We might have to create fake 4k page
762 * tables. To avoid that the hardware works on
763 * stale PGSTEs, we emulate these instructions.
765 kvm->arch.use_skf = 0;
766 kvm->arch.use_pfmfi = 0;
768 mutex_unlock(&kvm->lock);
769 VM_EVENT(kvm, 3, "ENABLE: CAP_S390_HPAGE %s",
770 r ? "(not available)" : "(success)");
772 case KVM_CAP_S390_USER_STSI:
773 VM_EVENT(kvm, 3, "%s", "ENABLE: CAP_S390_USER_STSI");
774 kvm->arch.user_stsi = 1;
777 case KVM_CAP_S390_USER_INSTR0:
778 VM_EVENT(kvm, 3, "%s", "ENABLE: CAP_S390_USER_INSTR0");
779 kvm->arch.user_instr0 = 1;
780 icpt_operexc_on_all_vcpus(kvm);
790 static int kvm_s390_get_mem_control(struct kvm *kvm, struct kvm_device_attr *attr)
794 switch (attr->attr) {
795 case KVM_S390_VM_MEM_LIMIT_SIZE:
797 VM_EVENT(kvm, 3, "QUERY: max guest memory: %lu bytes",
798 kvm->arch.mem_limit);
799 if (put_user(kvm->arch.mem_limit, (u64 __user *)attr->addr))
809 static int kvm_s390_set_mem_control(struct kvm *kvm, struct kvm_device_attr *attr)
813 switch (attr->attr) {
814 case KVM_S390_VM_MEM_ENABLE_CMMA:
819 VM_EVENT(kvm, 3, "%s", "ENABLE: CMMA support");
820 mutex_lock(&kvm->lock);
821 if (kvm->created_vcpus)
823 else if (kvm->mm->context.allow_gmap_hpage_1m)
826 kvm->arch.use_cmma = 1;
827 /* Not compatible with cmma. */
828 kvm->arch.use_pfmfi = 0;
831 mutex_unlock(&kvm->lock);
833 case KVM_S390_VM_MEM_CLR_CMMA:
838 if (!kvm->arch.use_cmma)
841 VM_EVENT(kvm, 3, "%s", "RESET: CMMA states");
842 mutex_lock(&kvm->lock);
843 idx = srcu_read_lock(&kvm->srcu);
844 s390_reset_cmma(kvm->arch.gmap->mm);
845 srcu_read_unlock(&kvm->srcu, idx);
846 mutex_unlock(&kvm->lock);
849 case KVM_S390_VM_MEM_LIMIT_SIZE: {
850 unsigned long new_limit;
852 if (kvm_is_ucontrol(kvm))
855 if (get_user(new_limit, (u64 __user *)attr->addr))
858 if (kvm->arch.mem_limit != KVM_S390_NO_MEM_LIMIT &&
859 new_limit > kvm->arch.mem_limit)
865 /* gmap_create takes last usable address */
866 if (new_limit != KVM_S390_NO_MEM_LIMIT)
870 mutex_lock(&kvm->lock);
871 if (!kvm->created_vcpus) {
872 /* gmap_create will round the limit up */
873 struct gmap *new = gmap_create(current->mm, new_limit);
878 gmap_remove(kvm->arch.gmap);
880 kvm->arch.gmap = new;
884 mutex_unlock(&kvm->lock);
885 VM_EVENT(kvm, 3, "SET: max guest address: %lu", new_limit);
886 VM_EVENT(kvm, 3, "New guest asce: 0x%pK",
887 (void *) kvm->arch.gmap->asce);
897 static void kvm_s390_vcpu_crypto_setup(struct kvm_vcpu *vcpu);
899 void kvm_s390_vcpu_crypto_reset_all(struct kvm *kvm)
901 struct kvm_vcpu *vcpu;
904 kvm_s390_vcpu_block_all(kvm);
906 kvm_for_each_vcpu(i, vcpu, kvm) {
907 kvm_s390_vcpu_crypto_setup(vcpu);
908 /* recreate the shadow crycb by leaving the VSIE handler */
909 kvm_s390_sync_request(KVM_REQ_VSIE_RESTART, vcpu);
912 kvm_s390_vcpu_unblock_all(kvm);
915 static int kvm_s390_vm_set_crypto(struct kvm *kvm, struct kvm_device_attr *attr)
917 mutex_lock(&kvm->lock);
918 switch (attr->attr) {
919 case KVM_S390_VM_CRYPTO_ENABLE_AES_KW:
920 if (!test_kvm_facility(kvm, 76)) {
921 mutex_unlock(&kvm->lock);
925 kvm->arch.crypto.crycb->aes_wrapping_key_mask,
926 sizeof(kvm->arch.crypto.crycb->aes_wrapping_key_mask));
927 kvm->arch.crypto.aes_kw = 1;
928 VM_EVENT(kvm, 3, "%s", "ENABLE: AES keywrapping support");
930 case KVM_S390_VM_CRYPTO_ENABLE_DEA_KW:
931 if (!test_kvm_facility(kvm, 76)) {
932 mutex_unlock(&kvm->lock);
936 kvm->arch.crypto.crycb->dea_wrapping_key_mask,
937 sizeof(kvm->arch.crypto.crycb->dea_wrapping_key_mask));
938 kvm->arch.crypto.dea_kw = 1;
939 VM_EVENT(kvm, 3, "%s", "ENABLE: DEA keywrapping support");
941 case KVM_S390_VM_CRYPTO_DISABLE_AES_KW:
942 if (!test_kvm_facility(kvm, 76)) {
943 mutex_unlock(&kvm->lock);
946 kvm->arch.crypto.aes_kw = 0;
947 memset(kvm->arch.crypto.crycb->aes_wrapping_key_mask, 0,
948 sizeof(kvm->arch.crypto.crycb->aes_wrapping_key_mask));
949 VM_EVENT(kvm, 3, "%s", "DISABLE: AES keywrapping support");
951 case KVM_S390_VM_CRYPTO_DISABLE_DEA_KW:
952 if (!test_kvm_facility(kvm, 76)) {
953 mutex_unlock(&kvm->lock);
956 kvm->arch.crypto.dea_kw = 0;
957 memset(kvm->arch.crypto.crycb->dea_wrapping_key_mask, 0,
958 sizeof(kvm->arch.crypto.crycb->dea_wrapping_key_mask));
959 VM_EVENT(kvm, 3, "%s", "DISABLE: DEA keywrapping support");
961 case KVM_S390_VM_CRYPTO_ENABLE_APIE:
962 if (!ap_instructions_available()) {
963 mutex_unlock(&kvm->lock);
966 kvm->arch.crypto.apie = 1;
968 case KVM_S390_VM_CRYPTO_DISABLE_APIE:
969 if (!ap_instructions_available()) {
970 mutex_unlock(&kvm->lock);
973 kvm->arch.crypto.apie = 0;
976 mutex_unlock(&kvm->lock);
980 kvm_s390_vcpu_crypto_reset_all(kvm);
981 mutex_unlock(&kvm->lock);
985 static void kvm_s390_sync_request_broadcast(struct kvm *kvm, int req)
988 struct kvm_vcpu *vcpu;
990 kvm_for_each_vcpu(cx, vcpu, kvm)
991 kvm_s390_sync_request(req, vcpu);
995 * Must be called with kvm->srcu held to avoid races on memslots, and with
996 * kvm->slots_lock to avoid races with ourselves and kvm_s390_vm_stop_migration.
998 static int kvm_s390_vm_start_migration(struct kvm *kvm)
1000 struct kvm_memory_slot *ms;
1001 struct kvm_memslots *slots;
1002 unsigned long ram_pages = 0;
1005 /* migration mode already enabled */
1006 if (kvm->arch.migration_mode)
1008 slots = kvm_memslots(kvm);
1009 if (!slots || !slots->used_slots)
1012 if (!kvm->arch.use_cmma) {
1013 kvm->arch.migration_mode = 1;
1016 /* mark all the pages in active slots as dirty */
1017 for (slotnr = 0; slotnr < slots->used_slots; slotnr++) {
1018 ms = slots->memslots + slotnr;
1019 if (!ms->dirty_bitmap)
1022 * The second half of the bitmap is only used on x86,
1023 * and would be wasted otherwise, so we put it to good
1024 * use here to keep track of the state of the storage
1027 memset(kvm_second_dirty_bitmap(ms), 0xff, kvm_dirty_bitmap_bytes(ms));
1028 ram_pages += ms->npages;
1030 atomic64_set(&kvm->arch.cmma_dirty_pages, ram_pages);
1031 kvm->arch.migration_mode = 1;
1032 kvm_s390_sync_request_broadcast(kvm, KVM_REQ_START_MIGRATION);
1037 * Must be called with kvm->slots_lock to avoid races with ourselves and
1038 * kvm_s390_vm_start_migration.
1040 static int kvm_s390_vm_stop_migration(struct kvm *kvm)
1042 /* migration mode already disabled */
1043 if (!kvm->arch.migration_mode)
1045 kvm->arch.migration_mode = 0;
1046 if (kvm->arch.use_cmma)
1047 kvm_s390_sync_request_broadcast(kvm, KVM_REQ_STOP_MIGRATION);
1051 static int kvm_s390_vm_set_migration(struct kvm *kvm,
1052 struct kvm_device_attr *attr)
1056 mutex_lock(&kvm->slots_lock);
1057 switch (attr->attr) {
1058 case KVM_S390_VM_MIGRATION_START:
1059 res = kvm_s390_vm_start_migration(kvm);
1061 case KVM_S390_VM_MIGRATION_STOP:
1062 res = kvm_s390_vm_stop_migration(kvm);
1067 mutex_unlock(&kvm->slots_lock);
1072 static int kvm_s390_vm_get_migration(struct kvm *kvm,
1073 struct kvm_device_attr *attr)
1075 u64 mig = kvm->arch.migration_mode;
1077 if (attr->attr != KVM_S390_VM_MIGRATION_STATUS)
1080 if (copy_to_user((void __user *)attr->addr, &mig, sizeof(mig)))
1085 static int kvm_s390_set_tod_ext(struct kvm *kvm, struct kvm_device_attr *attr)
1087 struct kvm_s390_vm_tod_clock gtod;
1089 if (copy_from_user(>od, (void __user *)attr->addr, sizeof(gtod)))
1092 if (!test_kvm_facility(kvm, 139) && gtod.epoch_idx)
1094 kvm_s390_set_tod_clock(kvm, >od);
1096 VM_EVENT(kvm, 3, "SET: TOD extension: 0x%x, TOD base: 0x%llx",
1097 gtod.epoch_idx, gtod.tod);
1102 static int kvm_s390_set_tod_high(struct kvm *kvm, struct kvm_device_attr *attr)
1106 if (copy_from_user(>od_high, (void __user *)attr->addr,
1112 VM_EVENT(kvm, 3, "SET: TOD extension: 0x%x", gtod_high);
1117 static int kvm_s390_set_tod_low(struct kvm *kvm, struct kvm_device_attr *attr)
1119 struct kvm_s390_vm_tod_clock gtod = { 0 };
1121 if (copy_from_user(>od.tod, (void __user *)attr->addr,
1125 kvm_s390_set_tod_clock(kvm, >od);
1126 VM_EVENT(kvm, 3, "SET: TOD base: 0x%llx", gtod.tod);
1130 static int kvm_s390_set_tod(struct kvm *kvm, struct kvm_device_attr *attr)
1137 switch (attr->attr) {
1138 case KVM_S390_VM_TOD_EXT:
1139 ret = kvm_s390_set_tod_ext(kvm, attr);
1141 case KVM_S390_VM_TOD_HIGH:
1142 ret = kvm_s390_set_tod_high(kvm, attr);
1144 case KVM_S390_VM_TOD_LOW:
1145 ret = kvm_s390_set_tod_low(kvm, attr);
1154 static void kvm_s390_get_tod_clock(struct kvm *kvm,
1155 struct kvm_s390_vm_tod_clock *gtod)
1157 struct kvm_s390_tod_clock_ext htod;
1161 get_tod_clock_ext((char *)&htod);
1163 gtod->tod = htod.tod + kvm->arch.epoch;
1164 gtod->epoch_idx = 0;
1165 if (test_kvm_facility(kvm, 139)) {
1166 gtod->epoch_idx = htod.epoch_idx + kvm->arch.epdx;
1167 if (gtod->tod < htod.tod)
1168 gtod->epoch_idx += 1;
1174 static int kvm_s390_get_tod_ext(struct kvm *kvm, struct kvm_device_attr *attr)
1176 struct kvm_s390_vm_tod_clock gtod;
1178 memset(>od, 0, sizeof(gtod));
1179 kvm_s390_get_tod_clock(kvm, >od);
1180 if (copy_to_user((void __user *)attr->addr, >od, sizeof(gtod)))
1183 VM_EVENT(kvm, 3, "QUERY: TOD extension: 0x%x, TOD base: 0x%llx",
1184 gtod.epoch_idx, gtod.tod);
1188 static int kvm_s390_get_tod_high(struct kvm *kvm, struct kvm_device_attr *attr)
1192 if (copy_to_user((void __user *)attr->addr, >od_high,
1195 VM_EVENT(kvm, 3, "QUERY: TOD extension: 0x%x", gtod_high);
1200 static int kvm_s390_get_tod_low(struct kvm *kvm, struct kvm_device_attr *attr)
1204 gtod = kvm_s390_get_tod_clock_fast(kvm);
1205 if (copy_to_user((void __user *)attr->addr, >od, sizeof(gtod)))
1207 VM_EVENT(kvm, 3, "QUERY: TOD base: 0x%llx", gtod);
1212 static int kvm_s390_get_tod(struct kvm *kvm, struct kvm_device_attr *attr)
1219 switch (attr->attr) {
1220 case KVM_S390_VM_TOD_EXT:
1221 ret = kvm_s390_get_tod_ext(kvm, attr);
1223 case KVM_S390_VM_TOD_HIGH:
1224 ret = kvm_s390_get_tod_high(kvm, attr);
1226 case KVM_S390_VM_TOD_LOW:
1227 ret = kvm_s390_get_tod_low(kvm, attr);
1236 static int kvm_s390_set_processor(struct kvm *kvm, struct kvm_device_attr *attr)
1238 struct kvm_s390_vm_cpu_processor *proc;
1239 u16 lowest_ibc, unblocked_ibc;
1242 mutex_lock(&kvm->lock);
1243 if (kvm->created_vcpus) {
1247 proc = kzalloc(sizeof(*proc), GFP_KERNEL);
1252 if (!copy_from_user(proc, (void __user *)attr->addr,
1254 kvm->arch.model.cpuid = proc->cpuid;
1255 lowest_ibc = sclp.ibc >> 16 & 0xfff;
1256 unblocked_ibc = sclp.ibc & 0xfff;
1257 if (lowest_ibc && proc->ibc) {
1258 if (proc->ibc > unblocked_ibc)
1259 kvm->arch.model.ibc = unblocked_ibc;
1260 else if (proc->ibc < lowest_ibc)
1261 kvm->arch.model.ibc = lowest_ibc;
1263 kvm->arch.model.ibc = proc->ibc;
1265 memcpy(kvm->arch.model.fac_list, proc->fac_list,
1266 S390_ARCH_FAC_LIST_SIZE_BYTE);
1267 VM_EVENT(kvm, 3, "SET: guest ibc: 0x%4.4x, guest cpuid: 0x%16.16llx",
1268 kvm->arch.model.ibc,
1269 kvm->arch.model.cpuid);
1270 VM_EVENT(kvm, 3, "SET: guest faclist: 0x%16.16llx.%16.16llx.%16.16llx",
1271 kvm->arch.model.fac_list[0],
1272 kvm->arch.model.fac_list[1],
1273 kvm->arch.model.fac_list[2]);
1278 mutex_unlock(&kvm->lock);
1282 static int kvm_s390_set_processor_feat(struct kvm *kvm,
1283 struct kvm_device_attr *attr)
1285 struct kvm_s390_vm_cpu_feat data;
1287 if (copy_from_user(&data, (void __user *)attr->addr, sizeof(data)))
1289 if (!bitmap_subset((unsigned long *) data.feat,
1290 kvm_s390_available_cpu_feat,
1291 KVM_S390_VM_CPU_FEAT_NR_BITS))
1294 mutex_lock(&kvm->lock);
1295 if (kvm->created_vcpus) {
1296 mutex_unlock(&kvm->lock);
1299 bitmap_copy(kvm->arch.cpu_feat, (unsigned long *) data.feat,
1300 KVM_S390_VM_CPU_FEAT_NR_BITS);
1301 mutex_unlock(&kvm->lock);
1302 VM_EVENT(kvm, 3, "SET: guest feat: 0x%16.16llx.0x%16.16llx.0x%16.16llx",
1309 static int kvm_s390_set_processor_subfunc(struct kvm *kvm,
1310 struct kvm_device_attr *attr)
1312 mutex_lock(&kvm->lock);
1313 if (kvm->created_vcpus) {
1314 mutex_unlock(&kvm->lock);
1318 if (copy_from_user(&kvm->arch.model.subfuncs, (void __user *)attr->addr,
1319 sizeof(struct kvm_s390_vm_cpu_subfunc))) {
1320 mutex_unlock(&kvm->lock);
1323 mutex_unlock(&kvm->lock);
1325 VM_EVENT(kvm, 3, "SET: guest PLO subfunc 0x%16.16lx.%16.16lx.%16.16lx.%16.16lx",
1326 ((unsigned long *) &kvm->arch.model.subfuncs.plo)[0],
1327 ((unsigned long *) &kvm->arch.model.subfuncs.plo)[1],
1328 ((unsigned long *) &kvm->arch.model.subfuncs.plo)[2],
1329 ((unsigned long *) &kvm->arch.model.subfuncs.plo)[3]);
1330 VM_EVENT(kvm, 3, "SET: guest PTFF subfunc 0x%16.16lx.%16.16lx",
1331 ((unsigned long *) &kvm->arch.model.subfuncs.ptff)[0],
1332 ((unsigned long *) &kvm->arch.model.subfuncs.ptff)[1]);
1333 VM_EVENT(kvm, 3, "SET: guest KMAC subfunc 0x%16.16lx.%16.16lx",
1334 ((unsigned long *) &kvm->arch.model.subfuncs.kmac)[0],
1335 ((unsigned long *) &kvm->arch.model.subfuncs.kmac)[1]);
1336 VM_EVENT(kvm, 3, "SET: guest KMC subfunc 0x%16.16lx.%16.16lx",
1337 ((unsigned long *) &kvm->arch.model.subfuncs.kmc)[0],
1338 ((unsigned long *) &kvm->arch.model.subfuncs.kmc)[1]);
1339 VM_EVENT(kvm, 3, "SET: guest KM subfunc 0x%16.16lx.%16.16lx",
1340 ((unsigned long *) &kvm->arch.model.subfuncs.km)[0],
1341 ((unsigned long *) &kvm->arch.model.subfuncs.km)[1]);
1342 VM_EVENT(kvm, 3, "SET: guest KIMD subfunc 0x%16.16lx.%16.16lx",
1343 ((unsigned long *) &kvm->arch.model.subfuncs.kimd)[0],
1344 ((unsigned long *) &kvm->arch.model.subfuncs.kimd)[1]);
1345 VM_EVENT(kvm, 3, "SET: guest KLMD subfunc 0x%16.16lx.%16.16lx",
1346 ((unsigned long *) &kvm->arch.model.subfuncs.klmd)[0],
1347 ((unsigned long *) &kvm->arch.model.subfuncs.klmd)[1]);
1348 VM_EVENT(kvm, 3, "SET: guest PCKMO subfunc 0x%16.16lx.%16.16lx",
1349 ((unsigned long *) &kvm->arch.model.subfuncs.pckmo)[0],
1350 ((unsigned long *) &kvm->arch.model.subfuncs.pckmo)[1]);
1351 VM_EVENT(kvm, 3, "SET: guest KMCTR subfunc 0x%16.16lx.%16.16lx",
1352 ((unsigned long *) &kvm->arch.model.subfuncs.kmctr)[0],
1353 ((unsigned long *) &kvm->arch.model.subfuncs.kmctr)[1]);
1354 VM_EVENT(kvm, 3, "SET: guest KMF subfunc 0x%16.16lx.%16.16lx",
1355 ((unsigned long *) &kvm->arch.model.subfuncs.kmf)[0],
1356 ((unsigned long *) &kvm->arch.model.subfuncs.kmf)[1]);
1357 VM_EVENT(kvm, 3, "SET: guest KMO subfunc 0x%16.16lx.%16.16lx",
1358 ((unsigned long *) &kvm->arch.model.subfuncs.kmo)[0],
1359 ((unsigned long *) &kvm->arch.model.subfuncs.kmo)[1]);
1360 VM_EVENT(kvm, 3, "SET: guest PCC subfunc 0x%16.16lx.%16.16lx",
1361 ((unsigned long *) &kvm->arch.model.subfuncs.pcc)[0],
1362 ((unsigned long *) &kvm->arch.model.subfuncs.pcc)[1]);
1363 VM_EVENT(kvm, 3, "SET: guest PPNO subfunc 0x%16.16lx.%16.16lx",
1364 ((unsigned long *) &kvm->arch.model.subfuncs.ppno)[0],
1365 ((unsigned long *) &kvm->arch.model.subfuncs.ppno)[1]);
1366 VM_EVENT(kvm, 3, "SET: guest KMA subfunc 0x%16.16lx.%16.16lx",
1367 ((unsigned long *) &kvm->arch.model.subfuncs.kma)[0],
1368 ((unsigned long *) &kvm->arch.model.subfuncs.kma)[1]);
1369 VM_EVENT(kvm, 3, "SET: guest KDSA subfunc 0x%16.16lx.%16.16lx",
1370 ((unsigned long *) &kvm->arch.model.subfuncs.kdsa)[0],
1371 ((unsigned long *) &kvm->arch.model.subfuncs.kdsa)[1]);
1372 VM_EVENT(kvm, 3, "SET: guest SORTL subfunc 0x%16.16lx.%16.16lx.%16.16lx.%16.16lx",
1373 ((unsigned long *) &kvm->arch.model.subfuncs.sortl)[0],
1374 ((unsigned long *) &kvm->arch.model.subfuncs.sortl)[1],
1375 ((unsigned long *) &kvm->arch.model.subfuncs.sortl)[2],
1376 ((unsigned long *) &kvm->arch.model.subfuncs.sortl)[3]);
1377 VM_EVENT(kvm, 3, "SET: guest DFLTCC subfunc 0x%16.16lx.%16.16lx.%16.16lx.%16.16lx",
1378 ((unsigned long *) &kvm->arch.model.subfuncs.dfltcc)[0],
1379 ((unsigned long *) &kvm->arch.model.subfuncs.dfltcc)[1],
1380 ((unsigned long *) &kvm->arch.model.subfuncs.dfltcc)[2],
1381 ((unsigned long *) &kvm->arch.model.subfuncs.dfltcc)[3]);
1386 static int kvm_s390_set_cpu_model(struct kvm *kvm, struct kvm_device_attr *attr)
1390 switch (attr->attr) {
1391 case KVM_S390_VM_CPU_PROCESSOR:
1392 ret = kvm_s390_set_processor(kvm, attr);
1394 case KVM_S390_VM_CPU_PROCESSOR_FEAT:
1395 ret = kvm_s390_set_processor_feat(kvm, attr);
1397 case KVM_S390_VM_CPU_PROCESSOR_SUBFUNC:
1398 ret = kvm_s390_set_processor_subfunc(kvm, attr);
1404 static int kvm_s390_get_processor(struct kvm *kvm, struct kvm_device_attr *attr)
1406 struct kvm_s390_vm_cpu_processor *proc;
1409 proc = kzalloc(sizeof(*proc), GFP_KERNEL);
1414 proc->cpuid = kvm->arch.model.cpuid;
1415 proc->ibc = kvm->arch.model.ibc;
1416 memcpy(&proc->fac_list, kvm->arch.model.fac_list,
1417 S390_ARCH_FAC_LIST_SIZE_BYTE);
1418 VM_EVENT(kvm, 3, "GET: guest ibc: 0x%4.4x, guest cpuid: 0x%16.16llx",
1419 kvm->arch.model.ibc,
1420 kvm->arch.model.cpuid);
1421 VM_EVENT(kvm, 3, "GET: guest faclist: 0x%16.16llx.%16.16llx.%16.16llx",
1422 kvm->arch.model.fac_list[0],
1423 kvm->arch.model.fac_list[1],
1424 kvm->arch.model.fac_list[2]);
1425 if (copy_to_user((void __user *)attr->addr, proc, sizeof(*proc)))
1432 static int kvm_s390_get_machine(struct kvm *kvm, struct kvm_device_attr *attr)
1434 struct kvm_s390_vm_cpu_machine *mach;
1437 mach = kzalloc(sizeof(*mach), GFP_KERNEL);
1442 get_cpu_id((struct cpuid *) &mach->cpuid);
1443 mach->ibc = sclp.ibc;
1444 memcpy(&mach->fac_mask, kvm->arch.model.fac_mask,
1445 S390_ARCH_FAC_LIST_SIZE_BYTE);
1446 memcpy((unsigned long *)&mach->fac_list, S390_lowcore.stfle_fac_list,
1447 sizeof(S390_lowcore.stfle_fac_list));
1448 VM_EVENT(kvm, 3, "GET: host ibc: 0x%4.4x, host cpuid: 0x%16.16llx",
1449 kvm->arch.model.ibc,
1450 kvm->arch.model.cpuid);
1451 VM_EVENT(kvm, 3, "GET: host facmask: 0x%16.16llx.%16.16llx.%16.16llx",
1455 VM_EVENT(kvm, 3, "GET: host faclist: 0x%16.16llx.%16.16llx.%16.16llx",
1459 if (copy_to_user((void __user *)attr->addr, mach, sizeof(*mach)))
1466 static int kvm_s390_get_processor_feat(struct kvm *kvm,
1467 struct kvm_device_attr *attr)
1469 struct kvm_s390_vm_cpu_feat data;
1471 bitmap_copy((unsigned long *) data.feat, kvm->arch.cpu_feat,
1472 KVM_S390_VM_CPU_FEAT_NR_BITS);
1473 if (copy_to_user((void __user *)attr->addr, &data, sizeof(data)))
1475 VM_EVENT(kvm, 3, "GET: guest feat: 0x%16.16llx.0x%16.16llx.0x%16.16llx",
1482 static int kvm_s390_get_machine_feat(struct kvm *kvm,
1483 struct kvm_device_attr *attr)
1485 struct kvm_s390_vm_cpu_feat data;
1487 bitmap_copy((unsigned long *) data.feat,
1488 kvm_s390_available_cpu_feat,
1489 KVM_S390_VM_CPU_FEAT_NR_BITS);
1490 if (copy_to_user((void __user *)attr->addr, &data, sizeof(data)))
1492 VM_EVENT(kvm, 3, "GET: host feat: 0x%16.16llx.0x%16.16llx.0x%16.16llx",
1499 static int kvm_s390_get_processor_subfunc(struct kvm *kvm,
1500 struct kvm_device_attr *attr)
1502 if (copy_to_user((void __user *)attr->addr, &kvm->arch.model.subfuncs,
1503 sizeof(struct kvm_s390_vm_cpu_subfunc)))
1506 VM_EVENT(kvm, 3, "GET: guest PLO subfunc 0x%16.16lx.%16.16lx.%16.16lx.%16.16lx",
1507 ((unsigned long *) &kvm->arch.model.subfuncs.plo)[0],
1508 ((unsigned long *) &kvm->arch.model.subfuncs.plo)[1],
1509 ((unsigned long *) &kvm->arch.model.subfuncs.plo)[2],
1510 ((unsigned long *) &kvm->arch.model.subfuncs.plo)[3]);
1511 VM_EVENT(kvm, 3, "GET: guest PTFF subfunc 0x%16.16lx.%16.16lx",
1512 ((unsigned long *) &kvm->arch.model.subfuncs.ptff)[0],
1513 ((unsigned long *) &kvm->arch.model.subfuncs.ptff)[1]);
1514 VM_EVENT(kvm, 3, "GET: guest KMAC subfunc 0x%16.16lx.%16.16lx",
1515 ((unsigned long *) &kvm->arch.model.subfuncs.kmac)[0],
1516 ((unsigned long *) &kvm->arch.model.subfuncs.kmac)[1]);
1517 VM_EVENT(kvm, 3, "GET: guest KMC subfunc 0x%16.16lx.%16.16lx",
1518 ((unsigned long *) &kvm->arch.model.subfuncs.kmc)[0],
1519 ((unsigned long *) &kvm->arch.model.subfuncs.kmc)[1]);
1520 VM_EVENT(kvm, 3, "GET: guest KM subfunc 0x%16.16lx.%16.16lx",
1521 ((unsigned long *) &kvm->arch.model.subfuncs.km)[0],
1522 ((unsigned long *) &kvm->arch.model.subfuncs.km)[1]);
1523 VM_EVENT(kvm, 3, "GET: guest KIMD subfunc 0x%16.16lx.%16.16lx",
1524 ((unsigned long *) &kvm->arch.model.subfuncs.kimd)[0],
1525 ((unsigned long *) &kvm->arch.model.subfuncs.kimd)[1]);
1526 VM_EVENT(kvm, 3, "GET: guest KLMD subfunc 0x%16.16lx.%16.16lx",
1527 ((unsigned long *) &kvm->arch.model.subfuncs.klmd)[0],
1528 ((unsigned long *) &kvm->arch.model.subfuncs.klmd)[1]);
1529 VM_EVENT(kvm, 3, "GET: guest PCKMO subfunc 0x%16.16lx.%16.16lx",
1530 ((unsigned long *) &kvm->arch.model.subfuncs.pckmo)[0],
1531 ((unsigned long *) &kvm->arch.model.subfuncs.pckmo)[1]);
1532 VM_EVENT(kvm, 3, "GET: guest KMCTR subfunc 0x%16.16lx.%16.16lx",
1533 ((unsigned long *) &kvm->arch.model.subfuncs.kmctr)[0],
1534 ((unsigned long *) &kvm->arch.model.subfuncs.kmctr)[1]);
1535 VM_EVENT(kvm, 3, "GET: guest KMF subfunc 0x%16.16lx.%16.16lx",
1536 ((unsigned long *) &kvm->arch.model.subfuncs.kmf)[0],
1537 ((unsigned long *) &kvm->arch.model.subfuncs.kmf)[1]);
1538 VM_EVENT(kvm, 3, "GET: guest KMO subfunc 0x%16.16lx.%16.16lx",
1539 ((unsigned long *) &kvm->arch.model.subfuncs.kmo)[0],
1540 ((unsigned long *) &kvm->arch.model.subfuncs.kmo)[1]);
1541 VM_EVENT(kvm, 3, "GET: guest PCC subfunc 0x%16.16lx.%16.16lx",
1542 ((unsigned long *) &kvm->arch.model.subfuncs.pcc)[0],
1543 ((unsigned long *) &kvm->arch.model.subfuncs.pcc)[1]);
1544 VM_EVENT(kvm, 3, "GET: guest PPNO subfunc 0x%16.16lx.%16.16lx",
1545 ((unsigned long *) &kvm->arch.model.subfuncs.ppno)[0],
1546 ((unsigned long *) &kvm->arch.model.subfuncs.ppno)[1]);
1547 VM_EVENT(kvm, 3, "GET: guest KMA subfunc 0x%16.16lx.%16.16lx",
1548 ((unsigned long *) &kvm->arch.model.subfuncs.kma)[0],
1549 ((unsigned long *) &kvm->arch.model.subfuncs.kma)[1]);
1550 VM_EVENT(kvm, 3, "GET: guest KDSA subfunc 0x%16.16lx.%16.16lx",
1551 ((unsigned long *) &kvm->arch.model.subfuncs.kdsa)[0],
1552 ((unsigned long *) &kvm->arch.model.subfuncs.kdsa)[1]);
1553 VM_EVENT(kvm, 3, "GET: guest SORTL subfunc 0x%16.16lx.%16.16lx.%16.16lx.%16.16lx",
1554 ((unsigned long *) &kvm->arch.model.subfuncs.sortl)[0],
1555 ((unsigned long *) &kvm->arch.model.subfuncs.sortl)[1],
1556 ((unsigned long *) &kvm->arch.model.subfuncs.sortl)[2],
1557 ((unsigned long *) &kvm->arch.model.subfuncs.sortl)[3]);
1558 VM_EVENT(kvm, 3, "GET: guest DFLTCC subfunc 0x%16.16lx.%16.16lx.%16.16lx.%16.16lx",
1559 ((unsigned long *) &kvm->arch.model.subfuncs.dfltcc)[0],
1560 ((unsigned long *) &kvm->arch.model.subfuncs.dfltcc)[1],
1561 ((unsigned long *) &kvm->arch.model.subfuncs.dfltcc)[2],
1562 ((unsigned long *) &kvm->arch.model.subfuncs.dfltcc)[3]);
1567 static int kvm_s390_get_machine_subfunc(struct kvm *kvm,
1568 struct kvm_device_attr *attr)
1570 if (copy_to_user((void __user *)attr->addr, &kvm_s390_available_subfunc,
1571 sizeof(struct kvm_s390_vm_cpu_subfunc)))
1574 VM_EVENT(kvm, 3, "GET: host PLO subfunc 0x%16.16lx.%16.16lx.%16.16lx.%16.16lx",
1575 ((unsigned long *) &kvm_s390_available_subfunc.plo)[0],
1576 ((unsigned long *) &kvm_s390_available_subfunc.plo)[1],
1577 ((unsigned long *) &kvm_s390_available_subfunc.plo)[2],
1578 ((unsigned long *) &kvm_s390_available_subfunc.plo)[3]);
1579 VM_EVENT(kvm, 3, "GET: host PTFF subfunc 0x%16.16lx.%16.16lx",
1580 ((unsigned long *) &kvm_s390_available_subfunc.ptff)[0],
1581 ((unsigned long *) &kvm_s390_available_subfunc.ptff)[1]);
1582 VM_EVENT(kvm, 3, "GET: host KMAC subfunc 0x%16.16lx.%16.16lx",
1583 ((unsigned long *) &kvm_s390_available_subfunc.kmac)[0],
1584 ((unsigned long *) &kvm_s390_available_subfunc.kmac)[1]);
1585 VM_EVENT(kvm, 3, "GET: host KMC subfunc 0x%16.16lx.%16.16lx",
1586 ((unsigned long *) &kvm_s390_available_subfunc.kmc)[0],
1587 ((unsigned long *) &kvm_s390_available_subfunc.kmc)[1]);
1588 VM_EVENT(kvm, 3, "GET: host KM subfunc 0x%16.16lx.%16.16lx",
1589 ((unsigned long *) &kvm_s390_available_subfunc.km)[0],
1590 ((unsigned long *) &kvm_s390_available_subfunc.km)[1]);
1591 VM_EVENT(kvm, 3, "GET: host KIMD subfunc 0x%16.16lx.%16.16lx",
1592 ((unsigned long *) &kvm_s390_available_subfunc.kimd)[0],
1593 ((unsigned long *) &kvm_s390_available_subfunc.kimd)[1]);
1594 VM_EVENT(kvm, 3, "GET: host KLMD subfunc 0x%16.16lx.%16.16lx",
1595 ((unsigned long *) &kvm_s390_available_subfunc.klmd)[0],
1596 ((unsigned long *) &kvm_s390_available_subfunc.klmd)[1]);
1597 VM_EVENT(kvm, 3, "GET: host PCKMO subfunc 0x%16.16lx.%16.16lx",
1598 ((unsigned long *) &kvm_s390_available_subfunc.pckmo)[0],
1599 ((unsigned long *) &kvm_s390_available_subfunc.pckmo)[1]);
1600 VM_EVENT(kvm, 3, "GET: host KMCTR subfunc 0x%16.16lx.%16.16lx",
1601 ((unsigned long *) &kvm_s390_available_subfunc.kmctr)[0],
1602 ((unsigned long *) &kvm_s390_available_subfunc.kmctr)[1]);
1603 VM_EVENT(kvm, 3, "GET: host KMF subfunc 0x%16.16lx.%16.16lx",
1604 ((unsigned long *) &kvm_s390_available_subfunc.kmf)[0],
1605 ((unsigned long *) &kvm_s390_available_subfunc.kmf)[1]);
1606 VM_EVENT(kvm, 3, "GET: host KMO subfunc 0x%16.16lx.%16.16lx",
1607 ((unsigned long *) &kvm_s390_available_subfunc.kmo)[0],
1608 ((unsigned long *) &kvm_s390_available_subfunc.kmo)[1]);
1609 VM_EVENT(kvm, 3, "GET: host PCC subfunc 0x%16.16lx.%16.16lx",
1610 ((unsigned long *) &kvm_s390_available_subfunc.pcc)[0],
1611 ((unsigned long *) &kvm_s390_available_subfunc.pcc)[1]);
1612 VM_EVENT(kvm, 3, "GET: host PPNO subfunc 0x%16.16lx.%16.16lx",
1613 ((unsigned long *) &kvm_s390_available_subfunc.ppno)[0],
1614 ((unsigned long *) &kvm_s390_available_subfunc.ppno)[1]);
1615 VM_EVENT(kvm, 3, "GET: host KMA subfunc 0x%16.16lx.%16.16lx",
1616 ((unsigned long *) &kvm_s390_available_subfunc.kma)[0],
1617 ((unsigned long *) &kvm_s390_available_subfunc.kma)[1]);
1618 VM_EVENT(kvm, 3, "GET: host KDSA subfunc 0x%16.16lx.%16.16lx",
1619 ((unsigned long *) &kvm_s390_available_subfunc.kdsa)[0],
1620 ((unsigned long *) &kvm_s390_available_subfunc.kdsa)[1]);
1621 VM_EVENT(kvm, 3, "GET: host SORTL subfunc 0x%16.16lx.%16.16lx.%16.16lx.%16.16lx",
1622 ((unsigned long *) &kvm_s390_available_subfunc.sortl)[0],
1623 ((unsigned long *) &kvm_s390_available_subfunc.sortl)[1],
1624 ((unsigned long *) &kvm_s390_available_subfunc.sortl)[2],
1625 ((unsigned long *) &kvm_s390_available_subfunc.sortl)[3]);
1626 VM_EVENT(kvm, 3, "GET: host DFLTCC subfunc 0x%16.16lx.%16.16lx.%16.16lx.%16.16lx",
1627 ((unsigned long *) &kvm_s390_available_subfunc.dfltcc)[0],
1628 ((unsigned long *) &kvm_s390_available_subfunc.dfltcc)[1],
1629 ((unsigned long *) &kvm_s390_available_subfunc.dfltcc)[2],
1630 ((unsigned long *) &kvm_s390_available_subfunc.dfltcc)[3]);
1635 static int kvm_s390_get_cpu_model(struct kvm *kvm, struct kvm_device_attr *attr)
1639 switch (attr->attr) {
1640 case KVM_S390_VM_CPU_PROCESSOR:
1641 ret = kvm_s390_get_processor(kvm, attr);
1643 case KVM_S390_VM_CPU_MACHINE:
1644 ret = kvm_s390_get_machine(kvm, attr);
1646 case KVM_S390_VM_CPU_PROCESSOR_FEAT:
1647 ret = kvm_s390_get_processor_feat(kvm, attr);
1649 case KVM_S390_VM_CPU_MACHINE_FEAT:
1650 ret = kvm_s390_get_machine_feat(kvm, attr);
1652 case KVM_S390_VM_CPU_PROCESSOR_SUBFUNC:
1653 ret = kvm_s390_get_processor_subfunc(kvm, attr);
1655 case KVM_S390_VM_CPU_MACHINE_SUBFUNC:
1656 ret = kvm_s390_get_machine_subfunc(kvm, attr);
1662 static int kvm_s390_vm_set_attr(struct kvm *kvm, struct kvm_device_attr *attr)
1666 switch (attr->group) {
1667 case KVM_S390_VM_MEM_CTRL:
1668 ret = kvm_s390_set_mem_control(kvm, attr);
1670 case KVM_S390_VM_TOD:
1671 ret = kvm_s390_set_tod(kvm, attr);
1673 case KVM_S390_VM_CPU_MODEL:
1674 ret = kvm_s390_set_cpu_model(kvm, attr);
1676 case KVM_S390_VM_CRYPTO:
1677 ret = kvm_s390_vm_set_crypto(kvm, attr);
1679 case KVM_S390_VM_MIGRATION:
1680 ret = kvm_s390_vm_set_migration(kvm, attr);
1690 static int kvm_s390_vm_get_attr(struct kvm *kvm, struct kvm_device_attr *attr)
1694 switch (attr->group) {
1695 case KVM_S390_VM_MEM_CTRL:
1696 ret = kvm_s390_get_mem_control(kvm, attr);
1698 case KVM_S390_VM_TOD:
1699 ret = kvm_s390_get_tod(kvm, attr);
1701 case KVM_S390_VM_CPU_MODEL:
1702 ret = kvm_s390_get_cpu_model(kvm, attr);
1704 case KVM_S390_VM_MIGRATION:
1705 ret = kvm_s390_vm_get_migration(kvm, attr);
1715 static int kvm_s390_vm_has_attr(struct kvm *kvm, struct kvm_device_attr *attr)
1719 switch (attr->group) {
1720 case KVM_S390_VM_MEM_CTRL:
1721 switch (attr->attr) {
1722 case KVM_S390_VM_MEM_ENABLE_CMMA:
1723 case KVM_S390_VM_MEM_CLR_CMMA:
1724 ret = sclp.has_cmma ? 0 : -ENXIO;
1726 case KVM_S390_VM_MEM_LIMIT_SIZE:
1734 case KVM_S390_VM_TOD:
1735 switch (attr->attr) {
1736 case KVM_S390_VM_TOD_LOW:
1737 case KVM_S390_VM_TOD_HIGH:
1745 case KVM_S390_VM_CPU_MODEL:
1746 switch (attr->attr) {
1747 case KVM_S390_VM_CPU_PROCESSOR:
1748 case KVM_S390_VM_CPU_MACHINE:
1749 case KVM_S390_VM_CPU_PROCESSOR_FEAT:
1750 case KVM_S390_VM_CPU_MACHINE_FEAT:
1751 case KVM_S390_VM_CPU_MACHINE_SUBFUNC:
1752 case KVM_S390_VM_CPU_PROCESSOR_SUBFUNC:
1760 case KVM_S390_VM_CRYPTO:
1761 switch (attr->attr) {
1762 case KVM_S390_VM_CRYPTO_ENABLE_AES_KW:
1763 case KVM_S390_VM_CRYPTO_ENABLE_DEA_KW:
1764 case KVM_S390_VM_CRYPTO_DISABLE_AES_KW:
1765 case KVM_S390_VM_CRYPTO_DISABLE_DEA_KW:
1768 case KVM_S390_VM_CRYPTO_ENABLE_APIE:
1769 case KVM_S390_VM_CRYPTO_DISABLE_APIE:
1770 ret = ap_instructions_available() ? 0 : -ENXIO;
1777 case KVM_S390_VM_MIGRATION:
1788 static long kvm_s390_get_skeys(struct kvm *kvm, struct kvm_s390_skeys *args)
1792 int srcu_idx, i, r = 0;
1794 if (args->flags != 0)
1797 /* Is this guest using storage keys? */
1798 if (!mm_uses_skeys(current->mm))
1799 return KVM_S390_GET_SKEYS_NONE;
1801 /* Enforce sane limit on memory allocation */
1802 if (args->count < 1 || args->count > KVM_S390_SKEYS_MAX)
1805 keys = kvmalloc_array(args->count, sizeof(uint8_t), GFP_KERNEL);
1809 down_read(¤t->mm->mmap_sem);
1810 srcu_idx = srcu_read_lock(&kvm->srcu);
1811 for (i = 0; i < args->count; i++) {
1812 hva = gfn_to_hva(kvm, args->start_gfn + i);
1813 if (kvm_is_error_hva(hva)) {
1818 r = get_guest_storage_key(current->mm, hva, &keys[i]);
1822 srcu_read_unlock(&kvm->srcu, srcu_idx);
1823 up_read(¤t->mm->mmap_sem);
1826 r = copy_to_user((uint8_t __user *)args->skeydata_addr, keys,
1827 sizeof(uint8_t) * args->count);
1836 static long kvm_s390_set_skeys(struct kvm *kvm, struct kvm_s390_skeys *args)
1840 int srcu_idx, i, r = 0;
1843 if (args->flags != 0)
1846 /* Enforce sane limit on memory allocation */
1847 if (args->count < 1 || args->count > KVM_S390_SKEYS_MAX)
1850 keys = kvmalloc_array(args->count, sizeof(uint8_t), GFP_KERNEL);
1854 r = copy_from_user(keys, (uint8_t __user *)args->skeydata_addr,
1855 sizeof(uint8_t) * args->count);
1861 /* Enable storage key handling for the guest */
1862 r = s390_enable_skey();
1867 down_read(¤t->mm->mmap_sem);
1868 srcu_idx = srcu_read_lock(&kvm->srcu);
1869 while (i < args->count) {
1871 hva = gfn_to_hva(kvm, args->start_gfn + i);
1872 if (kvm_is_error_hva(hva)) {
1877 /* Lowest order bit is reserved */
1878 if (keys[i] & 0x01) {
1883 r = set_guest_storage_key(current->mm, hva, keys[i], 0);
1885 r = fixup_user_fault(current, current->mm, hva,
1886 FAULT_FLAG_WRITE, &unlocked);
1893 srcu_read_unlock(&kvm->srcu, srcu_idx);
1894 up_read(¤t->mm->mmap_sem);
1901 * Base address and length must be sent at the start of each block, therefore
1902 * it's cheaper to send some clean data, as long as it's less than the size of
1905 #define KVM_S390_MAX_BIT_DISTANCE (2 * sizeof(void *))
1906 /* for consistency */
1907 #define KVM_S390_CMMA_SIZE_MAX ((u32)KVM_S390_SKEYS_MAX)
1910 * Similar to gfn_to_memslot, but returns the index of a memslot also when the
1911 * address falls in a hole. In that case the index of one of the memslots
1912 * bordering the hole is returned.
1914 static int gfn_to_memslot_approx(struct kvm_memslots *slots, gfn_t gfn)
1916 int start = 0, end = slots->used_slots;
1917 int slot = atomic_read(&slots->lru_slot);
1918 struct kvm_memory_slot *memslots = slots->memslots;
1920 if (gfn >= memslots[slot].base_gfn &&
1921 gfn < memslots[slot].base_gfn + memslots[slot].npages)
1924 while (start < end) {
1925 slot = start + (end - start) / 2;
1927 if (gfn >= memslots[slot].base_gfn)
1933 if (gfn >= memslots[start].base_gfn &&
1934 gfn < memslots[start].base_gfn + memslots[start].npages) {
1935 atomic_set(&slots->lru_slot, start);
1941 static int kvm_s390_peek_cmma(struct kvm *kvm, struct kvm_s390_cmma_log *args,
1942 u8 *res, unsigned long bufsize)
1944 unsigned long pgstev, hva, cur_gfn = args->start_gfn;
1947 while (args->count < bufsize) {
1948 hva = gfn_to_hva(kvm, cur_gfn);
1950 * We return an error if the first value was invalid, but we
1951 * return successfully if at least one value was copied.
1953 if (kvm_is_error_hva(hva))
1954 return args->count ? 0 : -EFAULT;
1955 if (get_pgste(kvm->mm, hva, &pgstev) < 0)
1957 res[args->count++] = (pgstev >> 24) & 0x43;
1964 static unsigned long kvm_s390_next_dirty_cmma(struct kvm_memslots *slots,
1965 unsigned long cur_gfn)
1967 int slotidx = gfn_to_memslot_approx(slots, cur_gfn);
1968 struct kvm_memory_slot *ms = slots->memslots + slotidx;
1969 unsigned long ofs = cur_gfn - ms->base_gfn;
1971 if (ms->base_gfn + ms->npages <= cur_gfn) {
1973 /* If we are above the highest slot, wrap around */
1975 slotidx = slots->used_slots - 1;
1977 ms = slots->memslots + slotidx;
1980 ofs = find_next_bit(kvm_second_dirty_bitmap(ms), ms->npages, ofs);
1981 while ((slotidx > 0) && (ofs >= ms->npages)) {
1983 ms = slots->memslots + slotidx;
1984 ofs = find_next_bit(kvm_second_dirty_bitmap(ms), ms->npages, 0);
1986 return ms->base_gfn + ofs;
1989 static int kvm_s390_get_cmma(struct kvm *kvm, struct kvm_s390_cmma_log *args,
1990 u8 *res, unsigned long bufsize)
1992 unsigned long mem_end, cur_gfn, next_gfn, hva, pgstev;
1993 struct kvm_memslots *slots = kvm_memslots(kvm);
1994 struct kvm_memory_slot *ms;
1996 cur_gfn = kvm_s390_next_dirty_cmma(slots, args->start_gfn);
1997 ms = gfn_to_memslot(kvm, cur_gfn);
1999 args->start_gfn = cur_gfn;
2002 next_gfn = kvm_s390_next_dirty_cmma(slots, cur_gfn + 1);
2003 mem_end = slots->memslots[0].base_gfn + slots->memslots[0].npages;
2005 while (args->count < bufsize) {
2006 hva = gfn_to_hva(kvm, cur_gfn);
2007 if (kvm_is_error_hva(hva))
2009 /* Decrement only if we actually flipped the bit to 0 */
2010 if (test_and_clear_bit(cur_gfn - ms->base_gfn, kvm_second_dirty_bitmap(ms)))
2011 atomic64_dec(&kvm->arch.cmma_dirty_pages);
2012 if (get_pgste(kvm->mm, hva, &pgstev) < 0)
2014 /* Save the value */
2015 res[args->count++] = (pgstev >> 24) & 0x43;
2016 /* If the next bit is too far away, stop. */
2017 if (next_gfn > cur_gfn + KVM_S390_MAX_BIT_DISTANCE)
2019 /* If we reached the previous "next", find the next one */
2020 if (cur_gfn == next_gfn)
2021 next_gfn = kvm_s390_next_dirty_cmma(slots, cur_gfn + 1);
2022 /* Reached the end of memory or of the buffer, stop */
2023 if ((next_gfn >= mem_end) ||
2024 (next_gfn - args->start_gfn >= bufsize))
2027 /* Reached the end of the current memslot, take the next one. */
2028 if (cur_gfn - ms->base_gfn >= ms->npages) {
2029 ms = gfn_to_memslot(kvm, cur_gfn);
2038 * This function searches for the next page with dirty CMMA attributes, and
2039 * saves the attributes in the buffer up to either the end of the buffer or
2040 * until a block of at least KVM_S390_MAX_BIT_DISTANCE clean bits is found;
2041 * no trailing clean bytes are saved.
2042 * In case no dirty bits were found, or if CMMA was not enabled or used, the
2043 * output buffer will indicate 0 as length.
2045 static int kvm_s390_get_cmma_bits(struct kvm *kvm,
2046 struct kvm_s390_cmma_log *args)
2048 unsigned long bufsize;
2049 int srcu_idx, peek, ret;
2052 if (!kvm->arch.use_cmma)
2054 /* Invalid/unsupported flags were specified */
2055 if (args->flags & ~KVM_S390_CMMA_PEEK)
2057 /* Migration mode query, and we are not doing a migration */
2058 peek = !!(args->flags & KVM_S390_CMMA_PEEK);
2059 if (!peek && !kvm->arch.migration_mode)
2061 /* CMMA is disabled or was not used, or the buffer has length zero */
2062 bufsize = min(args->count, KVM_S390_CMMA_SIZE_MAX);
2063 if (!bufsize || !kvm->mm->context.uses_cmm) {
2064 memset(args, 0, sizeof(*args));
2067 /* We are not peeking, and there are no dirty pages */
2068 if (!peek && !atomic64_read(&kvm->arch.cmma_dirty_pages)) {
2069 memset(args, 0, sizeof(*args));
2073 values = vmalloc(bufsize);
2077 down_read(&kvm->mm->mmap_sem);
2078 srcu_idx = srcu_read_lock(&kvm->srcu);
2080 ret = kvm_s390_peek_cmma(kvm, args, values, bufsize);
2082 ret = kvm_s390_get_cmma(kvm, args, values, bufsize);
2083 srcu_read_unlock(&kvm->srcu, srcu_idx);
2084 up_read(&kvm->mm->mmap_sem);
2086 if (kvm->arch.migration_mode)
2087 args->remaining = atomic64_read(&kvm->arch.cmma_dirty_pages);
2089 args->remaining = 0;
2091 if (copy_to_user((void __user *)args->values, values, args->count))
2099 * This function sets the CMMA attributes for the given pages. If the input
2100 * buffer has zero length, no action is taken, otherwise the attributes are
2101 * set and the mm->context.uses_cmm flag is set.
2103 static int kvm_s390_set_cmma_bits(struct kvm *kvm,
2104 const struct kvm_s390_cmma_log *args)
2106 unsigned long hva, mask, pgstev, i;
2108 int srcu_idx, r = 0;
2112 if (!kvm->arch.use_cmma)
2114 /* invalid/unsupported flags */
2115 if (args->flags != 0)
2117 /* Enforce sane limit on memory allocation */
2118 if (args->count > KVM_S390_CMMA_SIZE_MAX)
2121 if (args->count == 0)
2124 bits = vmalloc(array_size(sizeof(*bits), args->count));
2128 r = copy_from_user(bits, (void __user *)args->values, args->count);
2134 down_read(&kvm->mm->mmap_sem);
2135 srcu_idx = srcu_read_lock(&kvm->srcu);
2136 for (i = 0; i < args->count; i++) {
2137 hva = gfn_to_hva(kvm, args->start_gfn + i);
2138 if (kvm_is_error_hva(hva)) {
2144 pgstev = pgstev << 24;
2145 mask &= _PGSTE_GPS_USAGE_MASK | _PGSTE_GPS_NODAT;
2146 set_pgste_bits(kvm->mm, hva, mask, pgstev);
2148 srcu_read_unlock(&kvm->srcu, srcu_idx);
2149 up_read(&kvm->mm->mmap_sem);
2151 if (!kvm->mm->context.uses_cmm) {
2152 down_write(&kvm->mm->mmap_sem);
2153 kvm->mm->context.uses_cmm = 1;
2154 up_write(&kvm->mm->mmap_sem);
2161 long kvm_arch_vm_ioctl(struct file *filp,
2162 unsigned int ioctl, unsigned long arg)
2164 struct kvm *kvm = filp->private_data;
2165 void __user *argp = (void __user *)arg;
2166 struct kvm_device_attr attr;
2170 case KVM_S390_INTERRUPT: {
2171 struct kvm_s390_interrupt s390int;
2174 if (copy_from_user(&s390int, argp, sizeof(s390int)))
2176 r = kvm_s390_inject_vm(kvm, &s390int);
2179 case KVM_CREATE_IRQCHIP: {
2180 struct kvm_irq_routing_entry routing;
2183 if (kvm->arch.use_irqchip) {
2184 /* Set up dummy routing. */
2185 memset(&routing, 0, sizeof(routing));
2186 r = kvm_set_irq_routing(kvm, &routing, 0, 0);
2190 case KVM_SET_DEVICE_ATTR: {
2192 if (copy_from_user(&attr, (void __user *)arg, sizeof(attr)))
2194 r = kvm_s390_vm_set_attr(kvm, &attr);
2197 case KVM_GET_DEVICE_ATTR: {
2199 if (copy_from_user(&attr, (void __user *)arg, sizeof(attr)))
2201 r = kvm_s390_vm_get_attr(kvm, &attr);
2204 case KVM_HAS_DEVICE_ATTR: {
2206 if (copy_from_user(&attr, (void __user *)arg, sizeof(attr)))
2208 r = kvm_s390_vm_has_attr(kvm, &attr);
2211 case KVM_S390_GET_SKEYS: {
2212 struct kvm_s390_skeys args;
2215 if (copy_from_user(&args, argp,
2216 sizeof(struct kvm_s390_skeys)))
2218 r = kvm_s390_get_skeys(kvm, &args);
2221 case KVM_S390_SET_SKEYS: {
2222 struct kvm_s390_skeys args;
2225 if (copy_from_user(&args, argp,
2226 sizeof(struct kvm_s390_skeys)))
2228 r = kvm_s390_set_skeys(kvm, &args);
2231 case KVM_S390_GET_CMMA_BITS: {
2232 struct kvm_s390_cmma_log args;
2235 if (copy_from_user(&args, argp, sizeof(args)))
2237 mutex_lock(&kvm->slots_lock);
2238 r = kvm_s390_get_cmma_bits(kvm, &args);
2239 mutex_unlock(&kvm->slots_lock);
2241 r = copy_to_user(argp, &args, sizeof(args));
2247 case KVM_S390_SET_CMMA_BITS: {
2248 struct kvm_s390_cmma_log args;
2251 if (copy_from_user(&args, argp, sizeof(args)))
2253 mutex_lock(&kvm->slots_lock);
2254 r = kvm_s390_set_cmma_bits(kvm, &args);
2255 mutex_unlock(&kvm->slots_lock);
2265 static int kvm_s390_apxa_installed(void)
2267 struct ap_config_info info;
2269 if (ap_instructions_available()) {
2270 if (ap_qci(&info) == 0)
2278 * The format of the crypto control block (CRYCB) is specified in the 3 low
2279 * order bits of the CRYCB designation (CRYCBD) field as follows:
2280 * Format 0: Neither the message security assist extension 3 (MSAX3) nor the
2281 * AP extended addressing (APXA) facility are installed.
2282 * Format 1: The APXA facility is not installed but the MSAX3 facility is.
2283 * Format 2: Both the APXA and MSAX3 facilities are installed
2285 static void kvm_s390_set_crycb_format(struct kvm *kvm)
2287 kvm->arch.crypto.crycbd = (__u32)(unsigned long) kvm->arch.crypto.crycb;
2289 /* Clear the CRYCB format bits - i.e., set format 0 by default */
2290 kvm->arch.crypto.crycbd &= ~(CRYCB_FORMAT_MASK);
2292 /* Check whether MSAX3 is installed */
2293 if (!test_kvm_facility(kvm, 76))
2296 if (kvm_s390_apxa_installed())
2297 kvm->arch.crypto.crycbd |= CRYCB_FORMAT2;
2299 kvm->arch.crypto.crycbd |= CRYCB_FORMAT1;
2302 void kvm_arch_crypto_set_masks(struct kvm *kvm, unsigned long *apm,
2303 unsigned long *aqm, unsigned long *adm)
2305 struct kvm_s390_crypto_cb *crycb = kvm->arch.crypto.crycb;
2307 mutex_lock(&kvm->lock);
2308 kvm_s390_vcpu_block_all(kvm);
2310 switch (kvm->arch.crypto.crycbd & CRYCB_FORMAT_MASK) {
2311 case CRYCB_FORMAT2: /* APCB1 use 256 bits */
2312 memcpy(crycb->apcb1.apm, apm, 32);
2313 VM_EVENT(kvm, 3, "SET CRYCB: apm %016lx %016lx %016lx %016lx",
2314 apm[0], apm[1], apm[2], apm[3]);
2315 memcpy(crycb->apcb1.aqm, aqm, 32);
2316 VM_EVENT(kvm, 3, "SET CRYCB: aqm %016lx %016lx %016lx %016lx",
2317 aqm[0], aqm[1], aqm[2], aqm[3]);
2318 memcpy(crycb->apcb1.adm, adm, 32);
2319 VM_EVENT(kvm, 3, "SET CRYCB: adm %016lx %016lx %016lx %016lx",
2320 adm[0], adm[1], adm[2], adm[3]);
2323 case CRYCB_FORMAT0: /* Fall through both use APCB0 */
2324 memcpy(crycb->apcb0.apm, apm, 8);
2325 memcpy(crycb->apcb0.aqm, aqm, 2);
2326 memcpy(crycb->apcb0.adm, adm, 2);
2327 VM_EVENT(kvm, 3, "SET CRYCB: apm %016lx aqm %04x adm %04x",
2328 apm[0], *((unsigned short *)aqm),
2329 *((unsigned short *)adm));
2331 default: /* Can not happen */
2335 /* recreate the shadow crycb for each vcpu */
2336 kvm_s390_sync_request_broadcast(kvm, KVM_REQ_VSIE_RESTART);
2337 kvm_s390_vcpu_unblock_all(kvm);
2338 mutex_unlock(&kvm->lock);
2340 EXPORT_SYMBOL_GPL(kvm_arch_crypto_set_masks);
2342 void kvm_arch_crypto_clear_masks(struct kvm *kvm)
2344 mutex_lock(&kvm->lock);
2345 kvm_s390_vcpu_block_all(kvm);
2347 memset(&kvm->arch.crypto.crycb->apcb0, 0,
2348 sizeof(kvm->arch.crypto.crycb->apcb0));
2349 memset(&kvm->arch.crypto.crycb->apcb1, 0,
2350 sizeof(kvm->arch.crypto.crycb->apcb1));
2352 VM_EVENT(kvm, 3, "%s", "CLR CRYCB:");
2353 /* recreate the shadow crycb for each vcpu */
2354 kvm_s390_sync_request_broadcast(kvm, KVM_REQ_VSIE_RESTART);
2355 kvm_s390_vcpu_unblock_all(kvm);
2356 mutex_unlock(&kvm->lock);
2358 EXPORT_SYMBOL_GPL(kvm_arch_crypto_clear_masks);
2360 static u64 kvm_s390_get_initial_cpuid(void)
2365 cpuid.version = 0xff;
2366 return *((u64 *) &cpuid);
2369 static void kvm_s390_crypto_init(struct kvm *kvm)
2371 kvm->arch.crypto.crycb = &kvm->arch.sie_page2->crycb;
2372 kvm_s390_set_crycb_format(kvm);
2374 if (!test_kvm_facility(kvm, 76))
2377 /* Enable AES/DEA protected key functions by default */
2378 kvm->arch.crypto.aes_kw = 1;
2379 kvm->arch.crypto.dea_kw = 1;
2380 get_random_bytes(kvm->arch.crypto.crycb->aes_wrapping_key_mask,
2381 sizeof(kvm->arch.crypto.crycb->aes_wrapping_key_mask));
2382 get_random_bytes(kvm->arch.crypto.crycb->dea_wrapping_key_mask,
2383 sizeof(kvm->arch.crypto.crycb->dea_wrapping_key_mask));
2386 static void sca_dispose(struct kvm *kvm)
2388 if (kvm->arch.use_esca)
2389 free_pages_exact(kvm->arch.sca, sizeof(struct esca_block));
2391 free_page((unsigned long)(kvm->arch.sca));
2392 kvm->arch.sca = NULL;
2395 int kvm_arch_init_vm(struct kvm *kvm, unsigned long type)
2397 gfp_t alloc_flags = GFP_KERNEL;
2399 char debug_name[16];
2400 static unsigned long sca_offset;
2403 #ifdef CONFIG_KVM_S390_UCONTROL
2404 if (type & ~KVM_VM_S390_UCONTROL)
2406 if ((type & KVM_VM_S390_UCONTROL) && (!capable(CAP_SYS_ADMIN)))
2413 rc = s390_enable_sie();
2419 if (!sclp.has_64bscao)
2420 alloc_flags |= GFP_DMA;
2421 rwlock_init(&kvm->arch.sca_lock);
2422 /* start with basic SCA */
2423 kvm->arch.sca = (struct bsca_block *) get_zeroed_page(alloc_flags);
2426 mutex_lock(&kvm_lock);
2428 if (sca_offset + sizeof(struct bsca_block) > PAGE_SIZE)
2430 kvm->arch.sca = (struct bsca_block *)
2431 ((char *) kvm->arch.sca + sca_offset);
2432 mutex_unlock(&kvm_lock);
2434 sprintf(debug_name, "kvm-%u", current->pid);
2436 kvm->arch.dbf = debug_register(debug_name, 32, 1, 7 * sizeof(long));
2440 BUILD_BUG_ON(sizeof(struct sie_page2) != 4096);
2441 kvm->arch.sie_page2 =
2442 (struct sie_page2 *) get_zeroed_page(GFP_KERNEL | GFP_DMA);
2443 if (!kvm->arch.sie_page2)
2446 kvm->arch.sie_page2->kvm = kvm;
2447 kvm->arch.model.fac_list = kvm->arch.sie_page2->fac_list;
2449 for (i = 0; i < kvm_s390_fac_size(); i++) {
2450 kvm->arch.model.fac_mask[i] = S390_lowcore.stfle_fac_list[i] &
2451 (kvm_s390_fac_base[i] |
2452 kvm_s390_fac_ext[i]);
2453 kvm->arch.model.fac_list[i] = S390_lowcore.stfle_fac_list[i] &
2454 kvm_s390_fac_base[i];
2456 kvm->arch.model.subfuncs = kvm_s390_available_subfunc;
2458 /* we are always in czam mode - even on pre z14 machines */
2459 set_kvm_facility(kvm->arch.model.fac_mask, 138);
2460 set_kvm_facility(kvm->arch.model.fac_list, 138);
2461 /* we emulate STHYI in kvm */
2462 set_kvm_facility(kvm->arch.model.fac_mask, 74);
2463 set_kvm_facility(kvm->arch.model.fac_list, 74);
2464 if (MACHINE_HAS_TLB_GUEST) {
2465 set_kvm_facility(kvm->arch.model.fac_mask, 147);
2466 set_kvm_facility(kvm->arch.model.fac_list, 147);
2469 if (css_general_characteristics.aiv && test_facility(65))
2470 set_kvm_facility(kvm->arch.model.fac_mask, 65);
2472 kvm->arch.model.cpuid = kvm_s390_get_initial_cpuid();
2473 kvm->arch.model.ibc = sclp.ibc & 0x0fff;
2475 kvm_s390_crypto_init(kvm);
2477 mutex_init(&kvm->arch.float_int.ais_lock);
2478 spin_lock_init(&kvm->arch.float_int.lock);
2479 for (i = 0; i < FIRQ_LIST_COUNT; i++)
2480 INIT_LIST_HEAD(&kvm->arch.float_int.lists[i]);
2481 init_waitqueue_head(&kvm->arch.ipte_wq);
2482 mutex_init(&kvm->arch.ipte_mutex);
2484 debug_register_view(kvm->arch.dbf, &debug_sprintf_view);
2485 VM_EVENT(kvm, 3, "vm created with type %lu", type);
2487 if (type & KVM_VM_S390_UCONTROL) {
2488 kvm->arch.gmap = NULL;
2489 kvm->arch.mem_limit = KVM_S390_NO_MEM_LIMIT;
2491 if (sclp.hamax == U64_MAX)
2492 kvm->arch.mem_limit = TASK_SIZE_MAX;
2494 kvm->arch.mem_limit = min_t(unsigned long, TASK_SIZE_MAX,
2496 kvm->arch.gmap = gmap_create(current->mm, kvm->arch.mem_limit - 1);
2497 if (!kvm->arch.gmap)
2499 kvm->arch.gmap->private = kvm;
2500 kvm->arch.gmap->pfault_enabled = 0;
2503 kvm->arch.use_pfmfi = sclp.has_pfmfi;
2504 kvm->arch.use_skf = sclp.has_skey;
2505 spin_lock_init(&kvm->arch.start_stop_lock);
2506 kvm_s390_vsie_init(kvm);
2507 kvm_s390_gisa_init(kvm);
2508 KVM_EVENT(3, "vm 0x%pK created by pid %u", kvm, current->pid);
2512 free_page((unsigned long)kvm->arch.sie_page2);
2513 debug_unregister(kvm->arch.dbf);
2515 KVM_EVENT(3, "creation of vm failed: %d", rc);
2519 void kvm_arch_vcpu_destroy(struct kvm_vcpu *vcpu)
2521 VCPU_EVENT(vcpu, 3, "%s", "free cpu");
2522 trace_kvm_s390_destroy_vcpu(vcpu->vcpu_id);
2523 kvm_s390_clear_local_irqs(vcpu);
2524 kvm_clear_async_pf_completion_queue(vcpu);
2525 if (!kvm_is_ucontrol(vcpu->kvm))
2528 if (kvm_is_ucontrol(vcpu->kvm))
2529 gmap_remove(vcpu->arch.gmap);
2531 if (vcpu->kvm->arch.use_cmma)
2532 kvm_s390_vcpu_unsetup_cmma(vcpu);
2533 free_page((unsigned long)(vcpu->arch.sie_block));
2535 kvm_vcpu_uninit(vcpu);
2536 kmem_cache_free(kvm_vcpu_cache, vcpu);
2539 static void kvm_free_vcpus(struct kvm *kvm)
2542 struct kvm_vcpu *vcpu;
2544 kvm_for_each_vcpu(i, vcpu, kvm)
2545 kvm_arch_vcpu_destroy(vcpu);
2547 mutex_lock(&kvm->lock);
2548 for (i = 0; i < atomic_read(&kvm->online_vcpus); i++)
2549 kvm->vcpus[i] = NULL;
2551 atomic_set(&kvm->online_vcpus, 0);
2552 mutex_unlock(&kvm->lock);
2555 void kvm_arch_destroy_vm(struct kvm *kvm)
2557 kvm_free_vcpus(kvm);
2559 debug_unregister(kvm->arch.dbf);
2560 kvm_s390_gisa_destroy(kvm);
2561 free_page((unsigned long)kvm->arch.sie_page2);
2562 if (!kvm_is_ucontrol(kvm))
2563 gmap_remove(kvm->arch.gmap);
2564 kvm_s390_destroy_adapters(kvm);
2565 kvm_s390_clear_float_irqs(kvm);
2566 kvm_s390_vsie_destroy(kvm);
2567 KVM_EVENT(3, "vm 0x%pK destroyed", kvm);
2570 /* Section: vcpu related */
2571 static int __kvm_ucontrol_vcpu_init(struct kvm_vcpu *vcpu)
2573 vcpu->arch.gmap = gmap_create(current->mm, -1UL);
2574 if (!vcpu->arch.gmap)
2576 vcpu->arch.gmap->private = vcpu->kvm;
2581 static void sca_del_vcpu(struct kvm_vcpu *vcpu)
2583 if (!kvm_s390_use_sca_entries())
2585 read_lock(&vcpu->kvm->arch.sca_lock);
2586 if (vcpu->kvm->arch.use_esca) {
2587 struct esca_block *sca = vcpu->kvm->arch.sca;
2589 clear_bit_inv(vcpu->vcpu_id, (unsigned long *) sca->mcn);
2590 sca->cpu[vcpu->vcpu_id].sda = 0;
2592 struct bsca_block *sca = vcpu->kvm->arch.sca;
2594 clear_bit_inv(vcpu->vcpu_id, (unsigned long *) &sca->mcn);
2595 sca->cpu[vcpu->vcpu_id].sda = 0;
2597 read_unlock(&vcpu->kvm->arch.sca_lock);
2600 static void sca_add_vcpu(struct kvm_vcpu *vcpu)
2602 if (!kvm_s390_use_sca_entries()) {
2603 struct bsca_block *sca = vcpu->kvm->arch.sca;
2605 /* we still need the basic sca for the ipte control */
2606 vcpu->arch.sie_block->scaoh = (__u32)(((__u64)sca) >> 32);
2607 vcpu->arch.sie_block->scaol = (__u32)(__u64)sca;
2610 read_lock(&vcpu->kvm->arch.sca_lock);
2611 if (vcpu->kvm->arch.use_esca) {
2612 struct esca_block *sca = vcpu->kvm->arch.sca;
2614 sca->cpu[vcpu->vcpu_id].sda = (__u64) vcpu->arch.sie_block;
2615 vcpu->arch.sie_block->scaoh = (__u32)(((__u64)sca) >> 32);
2616 vcpu->arch.sie_block->scaol = (__u32)(__u64)sca & ~0x3fU;
2617 vcpu->arch.sie_block->ecb2 |= ECB2_ESCA;
2618 set_bit_inv(vcpu->vcpu_id, (unsigned long *) sca->mcn);
2620 struct bsca_block *sca = vcpu->kvm->arch.sca;
2622 sca->cpu[vcpu->vcpu_id].sda = (__u64) vcpu->arch.sie_block;
2623 vcpu->arch.sie_block->scaoh = (__u32)(((__u64)sca) >> 32);
2624 vcpu->arch.sie_block->scaol = (__u32)(__u64)sca;
2625 set_bit_inv(vcpu->vcpu_id, (unsigned long *) &sca->mcn);
2627 read_unlock(&vcpu->kvm->arch.sca_lock);
2630 /* Basic SCA to Extended SCA data copy routines */
2631 static inline void sca_copy_entry(struct esca_entry *d, struct bsca_entry *s)
2634 d->sigp_ctrl.c = s->sigp_ctrl.c;
2635 d->sigp_ctrl.scn = s->sigp_ctrl.scn;
2638 static void sca_copy_b_to_e(struct esca_block *d, struct bsca_block *s)
2642 d->ipte_control = s->ipte_control;
2644 for (i = 0; i < KVM_S390_BSCA_CPU_SLOTS; i++)
2645 sca_copy_entry(&d->cpu[i], &s->cpu[i]);
2648 static int sca_switch_to_extended(struct kvm *kvm)
2650 struct bsca_block *old_sca = kvm->arch.sca;
2651 struct esca_block *new_sca;
2652 struct kvm_vcpu *vcpu;
2653 unsigned int vcpu_idx;
2656 new_sca = alloc_pages_exact(sizeof(*new_sca), GFP_KERNEL|__GFP_ZERO);
2660 scaoh = (u32)((u64)(new_sca) >> 32);
2661 scaol = (u32)(u64)(new_sca) & ~0x3fU;
2663 kvm_s390_vcpu_block_all(kvm);
2664 write_lock(&kvm->arch.sca_lock);
2666 sca_copy_b_to_e(new_sca, old_sca);
2668 kvm_for_each_vcpu(vcpu_idx, vcpu, kvm) {
2669 vcpu->arch.sie_block->scaoh = scaoh;
2670 vcpu->arch.sie_block->scaol = scaol;
2671 vcpu->arch.sie_block->ecb2 |= ECB2_ESCA;
2673 kvm->arch.sca = new_sca;
2674 kvm->arch.use_esca = 1;
2676 write_unlock(&kvm->arch.sca_lock);
2677 kvm_s390_vcpu_unblock_all(kvm);
2679 free_page((unsigned long)old_sca);
2681 VM_EVENT(kvm, 2, "Switched to ESCA (0x%pK -> 0x%pK)",
2682 old_sca, kvm->arch.sca);
2686 static int sca_can_add_vcpu(struct kvm *kvm, unsigned int id)
2690 if (!kvm_s390_use_sca_entries()) {
2691 if (id < KVM_MAX_VCPUS)
2695 if (id < KVM_S390_BSCA_CPU_SLOTS)
2697 if (!sclp.has_esca || !sclp.has_64bscao)
2700 mutex_lock(&kvm->lock);
2701 rc = kvm->arch.use_esca ? 0 : sca_switch_to_extended(kvm);
2702 mutex_unlock(&kvm->lock);
2704 return rc == 0 && id < KVM_S390_ESCA_CPU_SLOTS;
2707 int kvm_arch_vcpu_init(struct kvm_vcpu *vcpu)
2709 vcpu->arch.pfault_token = KVM_S390_PFAULT_TOKEN_INVALID;
2710 kvm_clear_async_pf_completion_queue(vcpu);
2711 vcpu->run->kvm_valid_regs = KVM_SYNC_PREFIX |
2717 kvm_s390_set_prefix(vcpu, 0);
2718 if (test_kvm_facility(vcpu->kvm, 64))
2719 vcpu->run->kvm_valid_regs |= KVM_SYNC_RICCB;
2720 if (test_kvm_facility(vcpu->kvm, 82))
2721 vcpu->run->kvm_valid_regs |= KVM_SYNC_BPBC;
2722 if (test_kvm_facility(vcpu->kvm, 133))
2723 vcpu->run->kvm_valid_regs |= KVM_SYNC_GSCB;
2724 if (test_kvm_facility(vcpu->kvm, 156))
2725 vcpu->run->kvm_valid_regs |= KVM_SYNC_ETOKEN;
2726 /* fprs can be synchronized via vrs, even if the guest has no vx. With
2727 * MACHINE_HAS_VX, (load|store)_fpu_regs() will work with vrs format.
2730 vcpu->run->kvm_valid_regs |= KVM_SYNC_VRS;
2732 vcpu->run->kvm_valid_regs |= KVM_SYNC_FPRS;
2734 if (kvm_is_ucontrol(vcpu->kvm))
2735 return __kvm_ucontrol_vcpu_init(vcpu);
2740 /* needs disabled preemption to protect from TOD sync and vcpu_load/put */
2741 static void __start_cpu_timer_accounting(struct kvm_vcpu *vcpu)
2743 WARN_ON_ONCE(vcpu->arch.cputm_start != 0);
2744 raw_write_seqcount_begin(&vcpu->arch.cputm_seqcount);
2745 vcpu->arch.cputm_start = get_tod_clock_fast();
2746 raw_write_seqcount_end(&vcpu->arch.cputm_seqcount);
2749 /* needs disabled preemption to protect from TOD sync and vcpu_load/put */
2750 static void __stop_cpu_timer_accounting(struct kvm_vcpu *vcpu)
2752 WARN_ON_ONCE(vcpu->arch.cputm_start == 0);
2753 raw_write_seqcount_begin(&vcpu->arch.cputm_seqcount);
2754 vcpu->arch.sie_block->cputm -= get_tod_clock_fast() - vcpu->arch.cputm_start;
2755 vcpu->arch.cputm_start = 0;
2756 raw_write_seqcount_end(&vcpu->arch.cputm_seqcount);
2759 /* needs disabled preemption to protect from TOD sync and vcpu_load/put */
2760 static void __enable_cpu_timer_accounting(struct kvm_vcpu *vcpu)
2762 WARN_ON_ONCE(vcpu->arch.cputm_enabled);
2763 vcpu->arch.cputm_enabled = true;
2764 __start_cpu_timer_accounting(vcpu);
2767 /* needs disabled preemption to protect from TOD sync and vcpu_load/put */
2768 static void __disable_cpu_timer_accounting(struct kvm_vcpu *vcpu)
2770 WARN_ON_ONCE(!vcpu->arch.cputm_enabled);
2771 __stop_cpu_timer_accounting(vcpu);
2772 vcpu->arch.cputm_enabled = false;
2775 static void enable_cpu_timer_accounting(struct kvm_vcpu *vcpu)
2777 preempt_disable(); /* protect from TOD sync and vcpu_load/put */
2778 __enable_cpu_timer_accounting(vcpu);
2782 static void disable_cpu_timer_accounting(struct kvm_vcpu *vcpu)
2784 preempt_disable(); /* protect from TOD sync and vcpu_load/put */
2785 __disable_cpu_timer_accounting(vcpu);
2789 /* set the cpu timer - may only be called from the VCPU thread itself */
2790 void kvm_s390_set_cpu_timer(struct kvm_vcpu *vcpu, __u64 cputm)
2792 preempt_disable(); /* protect from TOD sync and vcpu_load/put */
2793 raw_write_seqcount_begin(&vcpu->arch.cputm_seqcount);
2794 if (vcpu->arch.cputm_enabled)
2795 vcpu->arch.cputm_start = get_tod_clock_fast();
2796 vcpu->arch.sie_block->cputm = cputm;
2797 raw_write_seqcount_end(&vcpu->arch.cputm_seqcount);
2801 /* update and get the cpu timer - can also be called from other VCPU threads */
2802 __u64 kvm_s390_get_cpu_timer(struct kvm_vcpu *vcpu)
2807 if (unlikely(!vcpu->arch.cputm_enabled))
2808 return vcpu->arch.sie_block->cputm;
2810 preempt_disable(); /* protect from TOD sync and vcpu_load/put */
2812 seq = raw_read_seqcount(&vcpu->arch.cputm_seqcount);
2814 * If the writer would ever execute a read in the critical
2815 * section, e.g. in irq context, we have a deadlock.
2817 WARN_ON_ONCE((seq & 1) && smp_processor_id() == vcpu->cpu);
2818 value = vcpu->arch.sie_block->cputm;
2819 /* if cputm_start is 0, accounting is being started/stopped */
2820 if (likely(vcpu->arch.cputm_start))
2821 value -= get_tod_clock_fast() - vcpu->arch.cputm_start;
2822 } while (read_seqcount_retry(&vcpu->arch.cputm_seqcount, seq & ~1));
2827 void kvm_arch_vcpu_load(struct kvm_vcpu *vcpu, int cpu)
2830 gmap_enable(vcpu->arch.enabled_gmap);
2831 kvm_s390_set_cpuflags(vcpu, CPUSTAT_RUNNING);
2832 if (vcpu->arch.cputm_enabled && !is_vcpu_idle(vcpu))
2833 __start_cpu_timer_accounting(vcpu);
2837 void kvm_arch_vcpu_put(struct kvm_vcpu *vcpu)
2840 if (vcpu->arch.cputm_enabled && !is_vcpu_idle(vcpu))
2841 __stop_cpu_timer_accounting(vcpu);
2842 kvm_s390_clear_cpuflags(vcpu, CPUSTAT_RUNNING);
2843 vcpu->arch.enabled_gmap = gmap_get_enabled();
2844 gmap_disable(vcpu->arch.enabled_gmap);
2848 void kvm_arch_vcpu_postcreate(struct kvm_vcpu *vcpu)
2850 mutex_lock(&vcpu->kvm->lock);
2852 vcpu->arch.sie_block->epoch = vcpu->kvm->arch.epoch;
2853 vcpu->arch.sie_block->epdx = vcpu->kvm->arch.epdx;
2855 mutex_unlock(&vcpu->kvm->lock);
2856 if (!kvm_is_ucontrol(vcpu->kvm)) {
2857 vcpu->arch.gmap = vcpu->kvm->arch.gmap;
2860 if (test_kvm_facility(vcpu->kvm, 74) || vcpu->kvm->arch.user_instr0)
2861 vcpu->arch.sie_block->ictl |= ICTL_OPEREXC;
2862 /* make vcpu_load load the right gmap on the first trigger */
2863 vcpu->arch.enabled_gmap = vcpu->arch.gmap;
2866 static bool kvm_has_pckmo_subfunc(struct kvm *kvm, unsigned long nr)
2868 if (test_bit_inv(nr, (unsigned long *)&kvm->arch.model.subfuncs.pckmo) &&
2869 test_bit_inv(nr, (unsigned long *)&kvm_s390_available_subfunc.pckmo))
2874 static bool kvm_has_pckmo_ecc(struct kvm *kvm)
2876 /* At least one ECC subfunction must be present */
2877 return kvm_has_pckmo_subfunc(kvm, 32) ||
2878 kvm_has_pckmo_subfunc(kvm, 33) ||
2879 kvm_has_pckmo_subfunc(kvm, 34) ||
2880 kvm_has_pckmo_subfunc(kvm, 40) ||
2881 kvm_has_pckmo_subfunc(kvm, 41);
2885 static void kvm_s390_vcpu_crypto_setup(struct kvm_vcpu *vcpu)
2888 * If the AP instructions are not being interpreted and the MSAX3
2889 * facility is not configured for the guest, there is nothing to set up.
2891 if (!vcpu->kvm->arch.crypto.apie && !test_kvm_facility(vcpu->kvm, 76))
2894 vcpu->arch.sie_block->crycbd = vcpu->kvm->arch.crypto.crycbd;
2895 vcpu->arch.sie_block->ecb3 &= ~(ECB3_AES | ECB3_DEA);
2896 vcpu->arch.sie_block->eca &= ~ECA_APIE;
2897 vcpu->arch.sie_block->ecd &= ~ECD_ECC;
2899 if (vcpu->kvm->arch.crypto.apie)
2900 vcpu->arch.sie_block->eca |= ECA_APIE;
2902 /* Set up protected key support */
2903 if (vcpu->kvm->arch.crypto.aes_kw) {
2904 vcpu->arch.sie_block->ecb3 |= ECB3_AES;
2905 /* ecc is also wrapped with AES key */
2906 if (kvm_has_pckmo_ecc(vcpu->kvm))
2907 vcpu->arch.sie_block->ecd |= ECD_ECC;
2910 if (vcpu->kvm->arch.crypto.dea_kw)
2911 vcpu->arch.sie_block->ecb3 |= ECB3_DEA;
2914 void kvm_s390_vcpu_unsetup_cmma(struct kvm_vcpu *vcpu)
2916 free_page(vcpu->arch.sie_block->cbrlo);
2917 vcpu->arch.sie_block->cbrlo = 0;
2920 int kvm_s390_vcpu_setup_cmma(struct kvm_vcpu *vcpu)
2922 vcpu->arch.sie_block->cbrlo = get_zeroed_page(GFP_KERNEL);
2923 if (!vcpu->arch.sie_block->cbrlo)
2928 static void kvm_s390_vcpu_setup_model(struct kvm_vcpu *vcpu)
2930 struct kvm_s390_cpu_model *model = &vcpu->kvm->arch.model;
2932 vcpu->arch.sie_block->ibc = model->ibc;
2933 if (test_kvm_facility(vcpu->kvm, 7))
2934 vcpu->arch.sie_block->fac = (u32)(u64) model->fac_list;
2937 int kvm_arch_vcpu_setup(struct kvm_vcpu *vcpu)
2941 atomic_set(&vcpu->arch.sie_block->cpuflags, CPUSTAT_ZARCH |
2945 if (test_kvm_facility(vcpu->kvm, 78))
2946 kvm_s390_set_cpuflags(vcpu, CPUSTAT_GED2);
2947 else if (test_kvm_facility(vcpu->kvm, 8))
2948 kvm_s390_set_cpuflags(vcpu, CPUSTAT_GED);
2950 kvm_s390_vcpu_setup_model(vcpu);
2952 /* pgste_set_pte has special handling for !MACHINE_HAS_ESOP */
2953 if (MACHINE_HAS_ESOP)
2954 vcpu->arch.sie_block->ecb |= ECB_HOSTPROTINT;
2955 if (test_kvm_facility(vcpu->kvm, 9))
2956 vcpu->arch.sie_block->ecb |= ECB_SRSI;
2957 if (test_kvm_facility(vcpu->kvm, 73))
2958 vcpu->arch.sie_block->ecb |= ECB_TE;
2960 if (test_kvm_facility(vcpu->kvm, 8) && vcpu->kvm->arch.use_pfmfi)
2961 vcpu->arch.sie_block->ecb2 |= ECB2_PFMFI;
2962 if (test_kvm_facility(vcpu->kvm, 130))
2963 vcpu->arch.sie_block->ecb2 |= ECB2_IEP;
2964 vcpu->arch.sie_block->eca = ECA_MVPGI | ECA_PROTEXCI;
2966 vcpu->arch.sie_block->eca |= ECA_CEI;
2968 vcpu->arch.sie_block->eca |= ECA_IB;
2970 vcpu->arch.sie_block->eca |= ECA_SII;
2971 if (sclp.has_sigpif)
2972 vcpu->arch.sie_block->eca |= ECA_SIGPI;
2973 if (test_kvm_facility(vcpu->kvm, 129)) {
2974 vcpu->arch.sie_block->eca |= ECA_VX;
2975 vcpu->arch.sie_block->ecd |= ECD_HOSTREGMGMT;
2977 if (test_kvm_facility(vcpu->kvm, 139))
2978 vcpu->arch.sie_block->ecd |= ECD_MEF;
2979 if (test_kvm_facility(vcpu->kvm, 156))
2980 vcpu->arch.sie_block->ecd |= ECD_ETOKENF;
2981 if (vcpu->arch.sie_block->gd) {
2982 vcpu->arch.sie_block->eca |= ECA_AIV;
2983 VCPU_EVENT(vcpu, 3, "AIV gisa format-%u enabled for cpu %03u",
2984 vcpu->arch.sie_block->gd & 0x3, vcpu->vcpu_id);
2986 vcpu->arch.sie_block->sdnxo = ((unsigned long) &vcpu->run->s.regs.sdnx)
2988 vcpu->arch.sie_block->riccbd = (unsigned long) &vcpu->run->s.regs.riccb;
2991 kvm_s390_set_cpuflags(vcpu, CPUSTAT_KSS);
2993 vcpu->arch.sie_block->ictl |= ICTL_ISKE | ICTL_SSKE | ICTL_RRBE;
2995 if (vcpu->kvm->arch.use_cmma) {
2996 rc = kvm_s390_vcpu_setup_cmma(vcpu);
3000 hrtimer_init(&vcpu->arch.ckc_timer, CLOCK_MONOTONIC, HRTIMER_MODE_REL);
3001 vcpu->arch.ckc_timer.function = kvm_s390_idle_wakeup;
3003 vcpu->arch.sie_block->hpid = HPID_KVM;
3005 kvm_s390_vcpu_crypto_setup(vcpu);
3010 struct kvm_vcpu *kvm_arch_vcpu_create(struct kvm *kvm,
3013 struct kvm_vcpu *vcpu;
3014 struct sie_page *sie_page;
3017 if (!kvm_is_ucontrol(kvm) && !sca_can_add_vcpu(kvm, id))
3022 vcpu = kmem_cache_zalloc(kvm_vcpu_cache, GFP_KERNEL);
3026 BUILD_BUG_ON(sizeof(struct sie_page) != 4096);
3027 sie_page = (struct sie_page *) get_zeroed_page(GFP_KERNEL);
3031 vcpu->arch.sie_block = &sie_page->sie_block;
3032 vcpu->arch.sie_block->itdba = (unsigned long) &sie_page->itdb;
3034 /* the real guest size will always be smaller than msl */
3035 vcpu->arch.sie_block->mso = 0;
3036 vcpu->arch.sie_block->msl = sclp.hamax;
3038 vcpu->arch.sie_block->icpua = id;
3039 spin_lock_init(&vcpu->arch.local_int.lock);
3040 vcpu->arch.sie_block->gd = (u32)(u64)kvm->arch.gisa_int.origin;
3041 if (vcpu->arch.sie_block->gd && sclp.has_gisaf)
3042 vcpu->arch.sie_block->gd |= GISA_FORMAT1;
3043 seqcount_init(&vcpu->arch.cputm_seqcount);
3045 rc = kvm_vcpu_init(vcpu, kvm, id);
3047 goto out_free_sie_block;
3048 VM_EVENT(kvm, 3, "create cpu %d at 0x%pK, sie block at 0x%pK", id, vcpu,
3049 vcpu->arch.sie_block);
3050 trace_kvm_s390_create_vcpu(id, vcpu, vcpu->arch.sie_block);
3054 free_page((unsigned long)(vcpu->arch.sie_block));
3056 kmem_cache_free(kvm_vcpu_cache, vcpu);
3061 int kvm_arch_vcpu_runnable(struct kvm_vcpu *vcpu)
3063 return kvm_s390_vcpu_has_irq(vcpu, 0);
3066 bool kvm_arch_vcpu_in_kernel(struct kvm_vcpu *vcpu)
3068 return !(vcpu->arch.sie_block->gpsw.mask & PSW_MASK_PSTATE);
3071 void kvm_s390_vcpu_block(struct kvm_vcpu *vcpu)
3073 atomic_or(PROG_BLOCK_SIE, &vcpu->arch.sie_block->prog20);
3077 void kvm_s390_vcpu_unblock(struct kvm_vcpu *vcpu)
3079 atomic_andnot(PROG_BLOCK_SIE, &vcpu->arch.sie_block->prog20);
3082 static void kvm_s390_vcpu_request(struct kvm_vcpu *vcpu)
3084 atomic_or(PROG_REQUEST, &vcpu->arch.sie_block->prog20);
3088 bool kvm_s390_vcpu_sie_inhibited(struct kvm_vcpu *vcpu)
3090 return atomic_read(&vcpu->arch.sie_block->prog20) &
3091 (PROG_BLOCK_SIE | PROG_REQUEST);
3094 static void kvm_s390_vcpu_request_handled(struct kvm_vcpu *vcpu)
3096 atomic_andnot(PROG_REQUEST, &vcpu->arch.sie_block->prog20);
3100 * Kick a guest cpu out of (v)SIE and wait until (v)SIE is not running.
3101 * If the CPU is not running (e.g. waiting as idle) the function will
3102 * return immediately. */
3103 void exit_sie(struct kvm_vcpu *vcpu)
3105 kvm_s390_set_cpuflags(vcpu, CPUSTAT_STOP_INT);
3106 kvm_s390_vsie_kick(vcpu);
3107 while (vcpu->arch.sie_block->prog0c & PROG_IN_SIE)
3111 /* Kick a guest cpu out of SIE to process a request synchronously */
3112 void kvm_s390_sync_request(int req, struct kvm_vcpu *vcpu)
3114 kvm_make_request(req, vcpu);
3115 kvm_s390_vcpu_request(vcpu);
3118 static void kvm_gmap_notifier(struct gmap *gmap, unsigned long start,
3121 struct kvm *kvm = gmap->private;
3122 struct kvm_vcpu *vcpu;
3123 unsigned long prefix;
3126 if (gmap_is_shadow(gmap))
3128 if (start >= 1UL << 31)
3129 /* We are only interested in prefix pages */
3131 kvm_for_each_vcpu(i, vcpu, kvm) {
3132 /* match against both prefix pages */
3133 prefix = kvm_s390_get_prefix(vcpu);
3134 if (prefix <= end && start <= prefix + 2*PAGE_SIZE - 1) {
3135 VCPU_EVENT(vcpu, 2, "gmap notifier for %lx-%lx",
3137 kvm_s390_sync_request(KVM_REQ_MMU_RELOAD, vcpu);
3142 bool kvm_arch_no_poll(struct kvm_vcpu *vcpu)
3144 /* do not poll with more than halt_poll_max_steal percent of steal time */
3145 if (S390_lowcore.avg_steal_timer * 100 / (TICK_USEC << 12) >=
3146 halt_poll_max_steal) {
3147 vcpu->stat.halt_no_poll_steal++;
3153 int kvm_arch_vcpu_should_kick(struct kvm_vcpu *vcpu)
3155 /* kvm common code refers to this, but never calls it */
3160 static int kvm_arch_vcpu_ioctl_get_one_reg(struct kvm_vcpu *vcpu,
3161 struct kvm_one_reg *reg)
3166 case KVM_REG_S390_TODPR:
3167 r = put_user(vcpu->arch.sie_block->todpr,
3168 (u32 __user *)reg->addr);
3170 case KVM_REG_S390_EPOCHDIFF:
3171 r = put_user(vcpu->arch.sie_block->epoch,
3172 (u64 __user *)reg->addr);
3174 case KVM_REG_S390_CPU_TIMER:
3175 r = put_user(kvm_s390_get_cpu_timer(vcpu),
3176 (u64 __user *)reg->addr);
3178 case KVM_REG_S390_CLOCK_COMP:
3179 r = put_user(vcpu->arch.sie_block->ckc,
3180 (u64 __user *)reg->addr);
3182 case KVM_REG_S390_PFTOKEN:
3183 r = put_user(vcpu->arch.pfault_token,
3184 (u64 __user *)reg->addr);
3186 case KVM_REG_S390_PFCOMPARE:
3187 r = put_user(vcpu->arch.pfault_compare,
3188 (u64 __user *)reg->addr);
3190 case KVM_REG_S390_PFSELECT:
3191 r = put_user(vcpu->arch.pfault_select,
3192 (u64 __user *)reg->addr);
3194 case KVM_REG_S390_PP:
3195 r = put_user(vcpu->arch.sie_block->pp,
3196 (u64 __user *)reg->addr);
3198 case KVM_REG_S390_GBEA:
3199 r = put_user(vcpu->arch.sie_block->gbea,
3200 (u64 __user *)reg->addr);
3209 static int kvm_arch_vcpu_ioctl_set_one_reg(struct kvm_vcpu *vcpu,
3210 struct kvm_one_reg *reg)
3216 case KVM_REG_S390_TODPR:
3217 r = get_user(vcpu->arch.sie_block->todpr,
3218 (u32 __user *)reg->addr);
3220 case KVM_REG_S390_EPOCHDIFF:
3221 r = get_user(vcpu->arch.sie_block->epoch,
3222 (u64 __user *)reg->addr);
3224 case KVM_REG_S390_CPU_TIMER:
3225 r = get_user(val, (u64 __user *)reg->addr);
3227 kvm_s390_set_cpu_timer(vcpu, val);
3229 case KVM_REG_S390_CLOCK_COMP:
3230 r = get_user(vcpu->arch.sie_block->ckc,
3231 (u64 __user *)reg->addr);
3233 case KVM_REG_S390_PFTOKEN:
3234 r = get_user(vcpu->arch.pfault_token,
3235 (u64 __user *)reg->addr);
3236 if (vcpu->arch.pfault_token == KVM_S390_PFAULT_TOKEN_INVALID)
3237 kvm_clear_async_pf_completion_queue(vcpu);
3239 case KVM_REG_S390_PFCOMPARE:
3240 r = get_user(vcpu->arch.pfault_compare,
3241 (u64 __user *)reg->addr);
3243 case KVM_REG_S390_PFSELECT:
3244 r = get_user(vcpu->arch.pfault_select,
3245 (u64 __user *)reg->addr);
3247 case KVM_REG_S390_PP:
3248 r = get_user(vcpu->arch.sie_block->pp,
3249 (u64 __user *)reg->addr);
3251 case KVM_REG_S390_GBEA:
3252 r = get_user(vcpu->arch.sie_block->gbea,
3253 (u64 __user *)reg->addr);
3262 static void kvm_arch_vcpu_ioctl_normal_reset(struct kvm_vcpu *vcpu)
3264 vcpu->arch.sie_block->gpsw.mask &= ~PSW_MASK_RI;
3265 vcpu->arch.pfault_token = KVM_S390_PFAULT_TOKEN_INVALID;
3266 memset(vcpu->run->s.regs.riccb, 0, sizeof(vcpu->run->s.regs.riccb));
3268 kvm_clear_async_pf_completion_queue(vcpu);
3269 if (!kvm_s390_user_cpu_state_ctrl(vcpu->kvm))
3270 kvm_s390_vcpu_stop(vcpu);
3271 kvm_s390_clear_local_irqs(vcpu);
3274 static void kvm_arch_vcpu_ioctl_initial_reset(struct kvm_vcpu *vcpu)
3276 /* Initial reset is a superset of the normal reset */
3277 kvm_arch_vcpu_ioctl_normal_reset(vcpu);
3279 /* this equals initial cpu reset in pop, but we don't switch to ESA */
3280 vcpu->arch.sie_block->gpsw.mask = 0;
3281 vcpu->arch.sie_block->gpsw.addr = 0;
3282 kvm_s390_set_prefix(vcpu, 0);
3283 kvm_s390_set_cpu_timer(vcpu, 0);
3284 vcpu->arch.sie_block->ckc = 0;
3285 vcpu->arch.sie_block->todpr = 0;
3286 memset(vcpu->arch.sie_block->gcr, 0, sizeof(vcpu->arch.sie_block->gcr));
3287 vcpu->arch.sie_block->gcr[0] = CR0_INITIAL_MASK;
3288 vcpu->arch.sie_block->gcr[14] = CR14_INITIAL_MASK;
3289 vcpu->run->s.regs.fpc = 0;
3290 vcpu->arch.sie_block->gbea = 1;
3291 vcpu->arch.sie_block->pp = 0;
3292 vcpu->arch.sie_block->fpf &= ~FPF_BPBC;
3295 static void kvm_arch_vcpu_ioctl_clear_reset(struct kvm_vcpu *vcpu)
3297 struct kvm_sync_regs *regs = &vcpu->run->s.regs;
3299 /* Clear reset is a superset of the initial reset */
3300 kvm_arch_vcpu_ioctl_initial_reset(vcpu);
3302 memset(®s->gprs, 0, sizeof(regs->gprs));
3303 memset(®s->vrs, 0, sizeof(regs->vrs));
3304 memset(®s->acrs, 0, sizeof(regs->acrs));
3305 memset(®s->gscb, 0, sizeof(regs->gscb));
3308 regs->etoken_extension = 0;
3311 int kvm_arch_vcpu_ioctl_set_regs(struct kvm_vcpu *vcpu, struct kvm_regs *regs)
3314 memcpy(&vcpu->run->s.regs.gprs, ®s->gprs, sizeof(regs->gprs));
3319 int kvm_arch_vcpu_ioctl_get_regs(struct kvm_vcpu *vcpu, struct kvm_regs *regs)
3322 memcpy(®s->gprs, &vcpu->run->s.regs.gprs, sizeof(regs->gprs));
3327 int kvm_arch_vcpu_ioctl_set_sregs(struct kvm_vcpu *vcpu,
3328 struct kvm_sregs *sregs)
3332 memcpy(&vcpu->run->s.regs.acrs, &sregs->acrs, sizeof(sregs->acrs));
3333 memcpy(&vcpu->arch.sie_block->gcr, &sregs->crs, sizeof(sregs->crs));
3339 int kvm_arch_vcpu_ioctl_get_sregs(struct kvm_vcpu *vcpu,
3340 struct kvm_sregs *sregs)
3344 memcpy(&sregs->acrs, &vcpu->run->s.regs.acrs, sizeof(sregs->acrs));
3345 memcpy(&sregs->crs, &vcpu->arch.sie_block->gcr, sizeof(sregs->crs));
3351 int kvm_arch_vcpu_ioctl_set_fpu(struct kvm_vcpu *vcpu, struct kvm_fpu *fpu)
3357 if (test_fp_ctl(fpu->fpc)) {
3361 vcpu->run->s.regs.fpc = fpu->fpc;
3363 convert_fp_to_vx((__vector128 *) vcpu->run->s.regs.vrs,
3364 (freg_t *) fpu->fprs);
3366 memcpy(vcpu->run->s.regs.fprs, &fpu->fprs, sizeof(fpu->fprs));
3373 int kvm_arch_vcpu_ioctl_get_fpu(struct kvm_vcpu *vcpu, struct kvm_fpu *fpu)
3377 /* make sure we have the latest values */
3380 convert_vx_to_fp((freg_t *) fpu->fprs,
3381 (__vector128 *) vcpu->run->s.regs.vrs);
3383 memcpy(fpu->fprs, vcpu->run->s.regs.fprs, sizeof(fpu->fprs));
3384 fpu->fpc = vcpu->run->s.regs.fpc;
3390 static int kvm_arch_vcpu_ioctl_set_initial_psw(struct kvm_vcpu *vcpu, psw_t psw)
3394 if (!is_vcpu_stopped(vcpu))
3397 vcpu->run->psw_mask = psw.mask;
3398 vcpu->run->psw_addr = psw.addr;
3403 int kvm_arch_vcpu_ioctl_translate(struct kvm_vcpu *vcpu,
3404 struct kvm_translation *tr)
3406 return -EINVAL; /* not implemented yet */
3409 #define VALID_GUESTDBG_FLAGS (KVM_GUESTDBG_SINGLESTEP | \
3410 KVM_GUESTDBG_USE_HW_BP | \
3411 KVM_GUESTDBG_ENABLE)
3413 int kvm_arch_vcpu_ioctl_set_guest_debug(struct kvm_vcpu *vcpu,
3414 struct kvm_guest_debug *dbg)
3420 vcpu->guest_debug = 0;
3421 kvm_s390_clear_bp_data(vcpu);
3423 if (dbg->control & ~VALID_GUESTDBG_FLAGS) {
3427 if (!sclp.has_gpere) {
3432 if (dbg->control & KVM_GUESTDBG_ENABLE) {
3433 vcpu->guest_debug = dbg->control;
3434 /* enforce guest PER */
3435 kvm_s390_set_cpuflags(vcpu, CPUSTAT_P);
3437 if (dbg->control & KVM_GUESTDBG_USE_HW_BP)
3438 rc = kvm_s390_import_bp_data(vcpu, dbg);
3440 kvm_s390_clear_cpuflags(vcpu, CPUSTAT_P);
3441 vcpu->arch.guestdbg.last_bp = 0;
3445 vcpu->guest_debug = 0;
3446 kvm_s390_clear_bp_data(vcpu);
3447 kvm_s390_clear_cpuflags(vcpu, CPUSTAT_P);
3455 int kvm_arch_vcpu_ioctl_get_mpstate(struct kvm_vcpu *vcpu,
3456 struct kvm_mp_state *mp_state)
3462 /* CHECK_STOP and LOAD are not supported yet */
3463 ret = is_vcpu_stopped(vcpu) ? KVM_MP_STATE_STOPPED :
3464 KVM_MP_STATE_OPERATING;
3470 int kvm_arch_vcpu_ioctl_set_mpstate(struct kvm_vcpu *vcpu,
3471 struct kvm_mp_state *mp_state)
3477 /* user space knows about this interface - let it control the state */
3478 vcpu->kvm->arch.user_cpu_state_ctrl = 1;
3480 switch (mp_state->mp_state) {
3481 case KVM_MP_STATE_STOPPED:
3482 kvm_s390_vcpu_stop(vcpu);
3484 case KVM_MP_STATE_OPERATING:
3485 kvm_s390_vcpu_start(vcpu);
3487 case KVM_MP_STATE_LOAD:
3488 case KVM_MP_STATE_CHECK_STOP:
3489 /* fall through - CHECK_STOP and LOAD are not supported yet */
3498 static bool ibs_enabled(struct kvm_vcpu *vcpu)
3500 return kvm_s390_test_cpuflags(vcpu, CPUSTAT_IBS);
3503 static int kvm_s390_handle_requests(struct kvm_vcpu *vcpu)
3506 kvm_s390_vcpu_request_handled(vcpu);
3507 if (!kvm_request_pending(vcpu))
3510 * We use MMU_RELOAD just to re-arm the ipte notifier for the
3511 * guest prefix page. gmap_mprotect_notify will wait on the ptl lock.
3512 * This ensures that the ipte instruction for this request has
3513 * already finished. We might race against a second unmapper that
3514 * wants to set the blocking bit. Lets just retry the request loop.
3516 if (kvm_check_request(KVM_REQ_MMU_RELOAD, vcpu)) {
3518 rc = gmap_mprotect_notify(vcpu->arch.gmap,
3519 kvm_s390_get_prefix(vcpu),
3520 PAGE_SIZE * 2, PROT_WRITE);
3522 kvm_make_request(KVM_REQ_MMU_RELOAD, vcpu);
3528 if (kvm_check_request(KVM_REQ_TLB_FLUSH, vcpu)) {
3529 vcpu->arch.sie_block->ihcpu = 0xffff;
3533 if (kvm_check_request(KVM_REQ_ENABLE_IBS, vcpu)) {
3534 if (!ibs_enabled(vcpu)) {
3535 trace_kvm_s390_enable_disable_ibs(vcpu->vcpu_id, 1);
3536 kvm_s390_set_cpuflags(vcpu, CPUSTAT_IBS);
3541 if (kvm_check_request(KVM_REQ_DISABLE_IBS, vcpu)) {
3542 if (ibs_enabled(vcpu)) {
3543 trace_kvm_s390_enable_disable_ibs(vcpu->vcpu_id, 0);
3544 kvm_s390_clear_cpuflags(vcpu, CPUSTAT_IBS);
3549 if (kvm_check_request(KVM_REQ_ICPT_OPEREXC, vcpu)) {
3550 vcpu->arch.sie_block->ictl |= ICTL_OPEREXC;
3554 if (kvm_check_request(KVM_REQ_START_MIGRATION, vcpu)) {
3556 * Disable CMM virtualization; we will emulate the ESSA
3557 * instruction manually, in order to provide additional
3558 * functionalities needed for live migration.
3560 vcpu->arch.sie_block->ecb2 &= ~ECB2_CMMA;
3564 if (kvm_check_request(KVM_REQ_STOP_MIGRATION, vcpu)) {
3566 * Re-enable CMM virtualization if CMMA is available and
3567 * CMM has been used.
3569 if ((vcpu->kvm->arch.use_cmma) &&
3570 (vcpu->kvm->mm->context.uses_cmm))
3571 vcpu->arch.sie_block->ecb2 |= ECB2_CMMA;
3575 /* nothing to do, just clear the request */
3576 kvm_clear_request(KVM_REQ_UNHALT, vcpu);
3577 /* we left the vsie handler, nothing to do, just clear the request */
3578 kvm_clear_request(KVM_REQ_VSIE_RESTART, vcpu);
3583 void kvm_s390_set_tod_clock(struct kvm *kvm,
3584 const struct kvm_s390_vm_tod_clock *gtod)
3586 struct kvm_vcpu *vcpu;
3587 struct kvm_s390_tod_clock_ext htod;
3590 mutex_lock(&kvm->lock);
3593 get_tod_clock_ext((char *)&htod);
3595 kvm->arch.epoch = gtod->tod - htod.tod;
3597 if (test_kvm_facility(kvm, 139)) {
3598 kvm->arch.epdx = gtod->epoch_idx - htod.epoch_idx;
3599 if (kvm->arch.epoch > gtod->tod)
3600 kvm->arch.epdx -= 1;
3603 kvm_s390_vcpu_block_all(kvm);
3604 kvm_for_each_vcpu(i, vcpu, kvm) {
3605 vcpu->arch.sie_block->epoch = kvm->arch.epoch;
3606 vcpu->arch.sie_block->epdx = kvm->arch.epdx;
3609 kvm_s390_vcpu_unblock_all(kvm);
3611 mutex_unlock(&kvm->lock);
3615 * kvm_arch_fault_in_page - fault-in guest page if necessary
3616 * @vcpu: The corresponding virtual cpu
3617 * @gpa: Guest physical address
3618 * @writable: Whether the page should be writable or not
3620 * Make sure that a guest page has been faulted-in on the host.
3622 * Return: Zero on success, negative error code otherwise.
3624 long kvm_arch_fault_in_page(struct kvm_vcpu *vcpu, gpa_t gpa, int writable)
3626 return gmap_fault(vcpu->arch.gmap, gpa,
3627 writable ? FAULT_FLAG_WRITE : 0);
3630 static void __kvm_inject_pfault_token(struct kvm_vcpu *vcpu, bool start_token,
3631 unsigned long token)
3633 struct kvm_s390_interrupt inti;
3634 struct kvm_s390_irq irq;
3637 irq.u.ext.ext_params2 = token;
3638 irq.type = KVM_S390_INT_PFAULT_INIT;
3639 WARN_ON_ONCE(kvm_s390_inject_vcpu(vcpu, &irq));
3641 inti.type = KVM_S390_INT_PFAULT_DONE;
3642 inti.parm64 = token;
3643 WARN_ON_ONCE(kvm_s390_inject_vm(vcpu->kvm, &inti));
3647 void kvm_arch_async_page_not_present(struct kvm_vcpu *vcpu,
3648 struct kvm_async_pf *work)
3650 trace_kvm_s390_pfault_init(vcpu, work->arch.pfault_token);
3651 __kvm_inject_pfault_token(vcpu, true, work->arch.pfault_token);
3654 void kvm_arch_async_page_present(struct kvm_vcpu *vcpu,
3655 struct kvm_async_pf *work)
3657 trace_kvm_s390_pfault_done(vcpu, work->arch.pfault_token);
3658 __kvm_inject_pfault_token(vcpu, false, work->arch.pfault_token);
3661 void kvm_arch_async_page_ready(struct kvm_vcpu *vcpu,
3662 struct kvm_async_pf *work)
3664 /* s390 will always inject the page directly */
3667 bool kvm_arch_can_inject_async_page_present(struct kvm_vcpu *vcpu)
3670 * s390 will always inject the page directly,
3671 * but we still want check_async_completion to cleanup
3676 static int kvm_arch_setup_async_pf(struct kvm_vcpu *vcpu)
3679 struct kvm_arch_async_pf arch;
3682 if (vcpu->arch.pfault_token == KVM_S390_PFAULT_TOKEN_INVALID)
3684 if ((vcpu->arch.sie_block->gpsw.mask & vcpu->arch.pfault_select) !=
3685 vcpu->arch.pfault_compare)
3687 if (psw_extint_disabled(vcpu))
3689 if (kvm_s390_vcpu_has_irq(vcpu, 0))
3691 if (!(vcpu->arch.sie_block->gcr[0] & CR0_SERVICE_SIGNAL_SUBMASK))
3693 if (!vcpu->arch.gmap->pfault_enabled)
3696 hva = gfn_to_hva(vcpu->kvm, gpa_to_gfn(current->thread.gmap_addr));
3697 hva += current->thread.gmap_addr & ~PAGE_MASK;
3698 if (read_guest_real(vcpu, vcpu->arch.pfault_token, &arch.pfault_token, 8))
3701 rc = kvm_setup_async_pf(vcpu, current->thread.gmap_addr, hva, &arch);
3705 static int vcpu_pre_run(struct kvm_vcpu *vcpu)
3710 * On s390 notifications for arriving pages will be delivered directly
3711 * to the guest but the house keeping for completed pfaults is
3712 * handled outside the worker.
3714 kvm_check_async_pf_completion(vcpu);
3716 vcpu->arch.sie_block->gg14 = vcpu->run->s.regs.gprs[14];
3717 vcpu->arch.sie_block->gg15 = vcpu->run->s.regs.gprs[15];
3722 if (test_cpu_flag(CIF_MCCK_PENDING))
3725 if (!kvm_is_ucontrol(vcpu->kvm)) {
3726 rc = kvm_s390_deliver_pending_interrupts(vcpu);
3731 rc = kvm_s390_handle_requests(vcpu);
3735 if (guestdbg_enabled(vcpu)) {
3736 kvm_s390_backup_guest_per_regs(vcpu);
3737 kvm_s390_patch_guest_per_regs(vcpu);
3740 clear_bit(vcpu->vcpu_id, vcpu->kvm->arch.gisa_int.kicked_mask);
3742 vcpu->arch.sie_block->icptcode = 0;
3743 cpuflags = atomic_read(&vcpu->arch.sie_block->cpuflags);
3744 VCPU_EVENT(vcpu, 6, "entering sie flags %x", cpuflags);
3745 trace_kvm_s390_sie_enter(vcpu, cpuflags);
3750 static int vcpu_post_run_fault_in_sie(struct kvm_vcpu *vcpu)
3752 struct kvm_s390_pgm_info pgm_info = {
3753 .code = PGM_ADDRESSING,
3758 VCPU_EVENT(vcpu, 3, "%s", "fault in sie instruction");
3759 trace_kvm_s390_sie_fault(vcpu);
3762 * We want to inject an addressing exception, which is defined as a
3763 * suppressing or terminating exception. However, since we came here
3764 * by a DAT access exception, the PSW still points to the faulting
3765 * instruction since DAT exceptions are nullifying. So we've got
3766 * to look up the current opcode to get the length of the instruction
3767 * to be able to forward the PSW.
3769 rc = read_guest_instr(vcpu, vcpu->arch.sie_block->gpsw.addr, &opcode, 1);
3770 ilen = insn_length(opcode);
3774 /* Instruction-Fetching Exceptions - we can't detect the ilen.
3775 * Forward by arbitrary ilc, injection will take care of
3776 * nullification if necessary.
3778 pgm_info = vcpu->arch.pgm;
3781 pgm_info.flags = ilen | KVM_S390_PGM_FLAGS_ILC_VALID;
3782 kvm_s390_forward_psw(vcpu, ilen);
3783 return kvm_s390_inject_prog_irq(vcpu, &pgm_info);
3786 static int vcpu_post_run(struct kvm_vcpu *vcpu, int exit_reason)
3788 struct mcck_volatile_info *mcck_info;
3789 struct sie_page *sie_page;
3791 VCPU_EVENT(vcpu, 6, "exit sie icptcode %d",
3792 vcpu->arch.sie_block->icptcode);
3793 trace_kvm_s390_sie_exit(vcpu, vcpu->arch.sie_block->icptcode);
3795 if (guestdbg_enabled(vcpu))
3796 kvm_s390_restore_guest_per_regs(vcpu);
3798 vcpu->run->s.regs.gprs[14] = vcpu->arch.sie_block->gg14;
3799 vcpu->run->s.regs.gprs[15] = vcpu->arch.sie_block->gg15;
3801 if (exit_reason == -EINTR) {
3802 VCPU_EVENT(vcpu, 3, "%s", "machine check");
3803 sie_page = container_of(vcpu->arch.sie_block,
3804 struct sie_page, sie_block);
3805 mcck_info = &sie_page->mcck_info;
3806 kvm_s390_reinject_machine_check(vcpu, mcck_info);
3810 if (vcpu->arch.sie_block->icptcode > 0) {
3811 int rc = kvm_handle_sie_intercept(vcpu);
3813 if (rc != -EOPNOTSUPP)
3815 vcpu->run->exit_reason = KVM_EXIT_S390_SIEIC;
3816 vcpu->run->s390_sieic.icptcode = vcpu->arch.sie_block->icptcode;
3817 vcpu->run->s390_sieic.ipa = vcpu->arch.sie_block->ipa;
3818 vcpu->run->s390_sieic.ipb = vcpu->arch.sie_block->ipb;
3820 } else if (exit_reason != -EFAULT) {
3821 vcpu->stat.exit_null++;
3823 } else if (kvm_is_ucontrol(vcpu->kvm)) {
3824 vcpu->run->exit_reason = KVM_EXIT_S390_UCONTROL;
3825 vcpu->run->s390_ucontrol.trans_exc_code =
3826 current->thread.gmap_addr;
3827 vcpu->run->s390_ucontrol.pgm_code = 0x10;
3829 } else if (current->thread.gmap_pfault) {
3830 trace_kvm_s390_major_guest_pfault(vcpu);
3831 current->thread.gmap_pfault = 0;
3832 if (kvm_arch_setup_async_pf(vcpu))
3834 return kvm_arch_fault_in_page(vcpu, current->thread.gmap_addr, 1);
3836 return vcpu_post_run_fault_in_sie(vcpu);
3839 static int __vcpu_run(struct kvm_vcpu *vcpu)
3841 int rc, exit_reason;
3844 * We try to hold kvm->srcu during most of vcpu_run (except when run-
3845 * ning the guest), so that memslots (and other stuff) are protected
3847 vcpu->srcu_idx = srcu_read_lock(&vcpu->kvm->srcu);
3850 rc = vcpu_pre_run(vcpu);
3854 srcu_read_unlock(&vcpu->kvm->srcu, vcpu->srcu_idx);
3856 * As PF_VCPU will be used in fault handler, between
3857 * guest_enter and guest_exit should be no uaccess.
3859 local_irq_disable();
3860 guest_enter_irqoff();
3861 __disable_cpu_timer_accounting(vcpu);
3863 exit_reason = sie64a(vcpu->arch.sie_block,
3864 vcpu->run->s.regs.gprs);
3865 local_irq_disable();
3866 __enable_cpu_timer_accounting(vcpu);
3867 guest_exit_irqoff();
3869 vcpu->srcu_idx = srcu_read_lock(&vcpu->kvm->srcu);
3871 rc = vcpu_post_run(vcpu, exit_reason);
3872 } while (!signal_pending(current) && !guestdbg_exit_pending(vcpu) && !rc);
3874 srcu_read_unlock(&vcpu->kvm->srcu, vcpu->srcu_idx);
3878 static void sync_regs(struct kvm_vcpu *vcpu, struct kvm_run *kvm_run)
3880 struct runtime_instr_cb *riccb;
3883 riccb = (struct runtime_instr_cb *) &kvm_run->s.regs.riccb;
3884 gscb = (struct gs_cb *) &kvm_run->s.regs.gscb;
3885 vcpu->arch.sie_block->gpsw.mask = kvm_run->psw_mask;
3886 vcpu->arch.sie_block->gpsw.addr = kvm_run->psw_addr;
3887 if (kvm_run->kvm_dirty_regs & KVM_SYNC_PREFIX)
3888 kvm_s390_set_prefix(vcpu, kvm_run->s.regs.prefix);
3889 if (kvm_run->kvm_dirty_regs & KVM_SYNC_CRS) {
3890 memcpy(&vcpu->arch.sie_block->gcr, &kvm_run->s.regs.crs, 128);
3891 /* some control register changes require a tlb flush */
3892 kvm_make_request(KVM_REQ_TLB_FLUSH, vcpu);
3894 if (kvm_run->kvm_dirty_regs & KVM_SYNC_ARCH0) {
3895 kvm_s390_set_cpu_timer(vcpu, kvm_run->s.regs.cputm);
3896 vcpu->arch.sie_block->ckc = kvm_run->s.regs.ckc;
3897 vcpu->arch.sie_block->todpr = kvm_run->s.regs.todpr;
3898 vcpu->arch.sie_block->pp = kvm_run->s.regs.pp;
3899 vcpu->arch.sie_block->gbea = kvm_run->s.regs.gbea;
3901 if (kvm_run->kvm_dirty_regs & KVM_SYNC_PFAULT) {
3902 vcpu->arch.pfault_token = kvm_run->s.regs.pft;
3903 vcpu->arch.pfault_select = kvm_run->s.regs.pfs;
3904 vcpu->arch.pfault_compare = kvm_run->s.regs.pfc;
3905 if (vcpu->arch.pfault_token == KVM_S390_PFAULT_TOKEN_INVALID)
3906 kvm_clear_async_pf_completion_queue(vcpu);
3909 * If userspace sets the riccb (e.g. after migration) to a valid state,
3910 * we should enable RI here instead of doing the lazy enablement.
3912 if ((kvm_run->kvm_dirty_regs & KVM_SYNC_RICCB) &&
3913 test_kvm_facility(vcpu->kvm, 64) &&
3915 !(vcpu->arch.sie_block->ecb3 & ECB3_RI)) {
3916 VCPU_EVENT(vcpu, 3, "%s", "ENABLE: RI (sync_regs)");
3917 vcpu->arch.sie_block->ecb3 |= ECB3_RI;
3920 * If userspace sets the gscb (e.g. after migration) to non-zero,
3921 * we should enable GS here instead of doing the lazy enablement.
3923 if ((kvm_run->kvm_dirty_regs & KVM_SYNC_GSCB) &&
3924 test_kvm_facility(vcpu->kvm, 133) &&
3926 !vcpu->arch.gs_enabled) {
3927 VCPU_EVENT(vcpu, 3, "%s", "ENABLE: GS (sync_regs)");
3928 vcpu->arch.sie_block->ecb |= ECB_GS;
3929 vcpu->arch.sie_block->ecd |= ECD_HOSTREGMGMT;
3930 vcpu->arch.gs_enabled = 1;
3932 if ((kvm_run->kvm_dirty_regs & KVM_SYNC_BPBC) &&
3933 test_kvm_facility(vcpu->kvm, 82)) {
3934 vcpu->arch.sie_block->fpf &= ~FPF_BPBC;
3935 vcpu->arch.sie_block->fpf |= kvm_run->s.regs.bpbc ? FPF_BPBC : 0;
3937 save_access_regs(vcpu->arch.host_acrs);
3938 restore_access_regs(vcpu->run->s.regs.acrs);
3939 /* save host (userspace) fprs/vrs */
3941 vcpu->arch.host_fpregs.fpc = current->thread.fpu.fpc;
3942 vcpu->arch.host_fpregs.regs = current->thread.fpu.regs;
3944 current->thread.fpu.regs = vcpu->run->s.regs.vrs;
3946 current->thread.fpu.regs = vcpu->run->s.regs.fprs;
3947 current->thread.fpu.fpc = vcpu->run->s.regs.fpc;
3948 if (test_fp_ctl(current->thread.fpu.fpc))
3949 /* User space provided an invalid FPC, let's clear it */
3950 current->thread.fpu.fpc = 0;
3951 if (MACHINE_HAS_GS) {
3953 __ctl_set_bit(2, 4);
3954 if (current->thread.gs_cb) {
3955 vcpu->arch.host_gscb = current->thread.gs_cb;
3956 save_gs_cb(vcpu->arch.host_gscb);
3958 if (vcpu->arch.gs_enabled) {
3959 current->thread.gs_cb = (struct gs_cb *)
3960 &vcpu->run->s.regs.gscb;
3961 restore_gs_cb(current->thread.gs_cb);
3965 /* SIE will load etoken directly from SDNX and therefore kvm_run */
3967 kvm_run->kvm_dirty_regs = 0;
3970 static void store_regs(struct kvm_vcpu *vcpu, struct kvm_run *kvm_run)
3972 kvm_run->psw_mask = vcpu->arch.sie_block->gpsw.mask;
3973 kvm_run->psw_addr = vcpu->arch.sie_block->gpsw.addr;
3974 kvm_run->s.regs.prefix = kvm_s390_get_prefix(vcpu);
3975 memcpy(&kvm_run->s.regs.crs, &vcpu->arch.sie_block->gcr, 128);
3976 kvm_run->s.regs.cputm = kvm_s390_get_cpu_timer(vcpu);
3977 kvm_run->s.regs.ckc = vcpu->arch.sie_block->ckc;
3978 kvm_run->s.regs.todpr = vcpu->arch.sie_block->todpr;
3979 kvm_run->s.regs.pp = vcpu->arch.sie_block->pp;
3980 kvm_run->s.regs.gbea = vcpu->arch.sie_block->gbea;
3981 kvm_run->s.regs.pft = vcpu->arch.pfault_token;
3982 kvm_run->s.regs.pfs = vcpu->arch.pfault_select;
3983 kvm_run->s.regs.pfc = vcpu->arch.pfault_compare;
3984 kvm_run->s.regs.bpbc = (vcpu->arch.sie_block->fpf & FPF_BPBC) == FPF_BPBC;
3985 save_access_regs(vcpu->run->s.regs.acrs);
3986 restore_access_regs(vcpu->arch.host_acrs);
3987 /* Save guest register state */
3989 vcpu->run->s.regs.fpc = current->thread.fpu.fpc;
3990 /* Restore will be done lazily at return */
3991 current->thread.fpu.fpc = vcpu->arch.host_fpregs.fpc;
3992 current->thread.fpu.regs = vcpu->arch.host_fpregs.regs;
3993 if (MACHINE_HAS_GS) {
3994 __ctl_set_bit(2, 4);
3995 if (vcpu->arch.gs_enabled)
3996 save_gs_cb(current->thread.gs_cb);
3998 current->thread.gs_cb = vcpu->arch.host_gscb;
3999 restore_gs_cb(vcpu->arch.host_gscb);
4001 if (!vcpu->arch.host_gscb)
4002 __ctl_clear_bit(2, 4);
4003 vcpu->arch.host_gscb = NULL;
4005 /* SIE will save etoken directly into SDNX and therefore kvm_run */
4008 int kvm_arch_vcpu_ioctl_run(struct kvm_vcpu *vcpu, struct kvm_run *kvm_run)
4012 if (kvm_run->immediate_exit)
4015 if (kvm_run->kvm_valid_regs & ~KVM_SYNC_S390_VALID_FIELDS ||
4016 kvm_run->kvm_dirty_regs & ~KVM_SYNC_S390_VALID_FIELDS)
4021 if (guestdbg_exit_pending(vcpu)) {
4022 kvm_s390_prepare_debug_exit(vcpu);
4027 kvm_sigset_activate(vcpu);
4029 if (!kvm_s390_user_cpu_state_ctrl(vcpu->kvm)) {
4030 kvm_s390_vcpu_start(vcpu);
4031 } else if (is_vcpu_stopped(vcpu)) {
4032 pr_err_ratelimited("can't run stopped vcpu %d\n",
4038 sync_regs(vcpu, kvm_run);
4039 enable_cpu_timer_accounting(vcpu);
4042 rc = __vcpu_run(vcpu);
4044 if (signal_pending(current) && !rc) {
4045 kvm_run->exit_reason = KVM_EXIT_INTR;
4049 if (guestdbg_exit_pending(vcpu) && !rc) {
4050 kvm_s390_prepare_debug_exit(vcpu);
4054 if (rc == -EREMOTE) {
4055 /* userspace support is needed, kvm_run has been prepared */
4059 disable_cpu_timer_accounting(vcpu);
4060 store_regs(vcpu, kvm_run);
4062 kvm_sigset_deactivate(vcpu);
4064 vcpu->stat.exit_userspace++;
4071 * store status at address
4072 * we use have two special cases:
4073 * KVM_S390_STORE_STATUS_NOADDR: -> 0x1200 on 64 bit
4074 * KVM_S390_STORE_STATUS_PREFIXED: -> prefix
4076 int kvm_s390_store_status_unloaded(struct kvm_vcpu *vcpu, unsigned long gpa)
4078 unsigned char archmode = 1;
4079 freg_t fprs[NUM_FPRS];
4084 px = kvm_s390_get_prefix(vcpu);
4085 if (gpa == KVM_S390_STORE_STATUS_NOADDR) {
4086 if (write_guest_abs(vcpu, 163, &archmode, 1))
4089 } else if (gpa == KVM_S390_STORE_STATUS_PREFIXED) {
4090 if (write_guest_real(vcpu, 163, &archmode, 1))
4094 gpa -= __LC_FPREGS_SAVE_AREA;
4096 /* manually convert vector registers if necessary */
4097 if (MACHINE_HAS_VX) {
4098 convert_vx_to_fp(fprs, (__vector128 *) vcpu->run->s.regs.vrs);
4099 rc = write_guest_abs(vcpu, gpa + __LC_FPREGS_SAVE_AREA,
4102 rc = write_guest_abs(vcpu, gpa + __LC_FPREGS_SAVE_AREA,
4103 vcpu->run->s.regs.fprs, 128);
4105 rc |= write_guest_abs(vcpu, gpa + __LC_GPREGS_SAVE_AREA,
4106 vcpu->run->s.regs.gprs, 128);
4107 rc |= write_guest_abs(vcpu, gpa + __LC_PSW_SAVE_AREA,
4108 &vcpu->arch.sie_block->gpsw, 16);
4109 rc |= write_guest_abs(vcpu, gpa + __LC_PREFIX_SAVE_AREA,
4111 rc |= write_guest_abs(vcpu, gpa + __LC_FP_CREG_SAVE_AREA,
4112 &vcpu->run->s.regs.fpc, 4);
4113 rc |= write_guest_abs(vcpu, gpa + __LC_TOD_PROGREG_SAVE_AREA,
4114 &vcpu->arch.sie_block->todpr, 4);
4115 cputm = kvm_s390_get_cpu_timer(vcpu);
4116 rc |= write_guest_abs(vcpu, gpa + __LC_CPU_TIMER_SAVE_AREA,
4118 clkcomp = vcpu->arch.sie_block->ckc >> 8;
4119 rc |= write_guest_abs(vcpu, gpa + __LC_CLOCK_COMP_SAVE_AREA,
4121 rc |= write_guest_abs(vcpu, gpa + __LC_AREGS_SAVE_AREA,
4122 &vcpu->run->s.regs.acrs, 64);
4123 rc |= write_guest_abs(vcpu, gpa + __LC_CREGS_SAVE_AREA,
4124 &vcpu->arch.sie_block->gcr, 128);
4125 return rc ? -EFAULT : 0;
4128 int kvm_s390_vcpu_store_status(struct kvm_vcpu *vcpu, unsigned long addr)
4131 * The guest FPRS and ACRS are in the host FPRS/ACRS due to the lazy
4132 * switch in the run ioctl. Let's update our copies before we save
4133 * it into the save area
4136 vcpu->run->s.regs.fpc = current->thread.fpu.fpc;
4137 save_access_regs(vcpu->run->s.regs.acrs);
4139 return kvm_s390_store_status_unloaded(vcpu, addr);
4142 static void __disable_ibs_on_vcpu(struct kvm_vcpu *vcpu)
4144 kvm_check_request(KVM_REQ_ENABLE_IBS, vcpu);
4145 kvm_s390_sync_request(KVM_REQ_DISABLE_IBS, vcpu);
4148 static void __disable_ibs_on_all_vcpus(struct kvm *kvm)
4151 struct kvm_vcpu *vcpu;
4153 kvm_for_each_vcpu(i, vcpu, kvm) {
4154 __disable_ibs_on_vcpu(vcpu);
4158 static void __enable_ibs_on_vcpu(struct kvm_vcpu *vcpu)
4162 kvm_check_request(KVM_REQ_DISABLE_IBS, vcpu);
4163 kvm_s390_sync_request(KVM_REQ_ENABLE_IBS, vcpu);
4166 void kvm_s390_vcpu_start(struct kvm_vcpu *vcpu)
4168 int i, online_vcpus, started_vcpus = 0;
4170 if (!is_vcpu_stopped(vcpu))
4173 trace_kvm_s390_vcpu_start_stop(vcpu->vcpu_id, 1);
4174 /* Only one cpu at a time may enter/leave the STOPPED state. */
4175 spin_lock(&vcpu->kvm->arch.start_stop_lock);
4176 online_vcpus = atomic_read(&vcpu->kvm->online_vcpus);
4178 for (i = 0; i < online_vcpus; i++) {
4179 if (!is_vcpu_stopped(vcpu->kvm->vcpus[i]))
4183 if (started_vcpus == 0) {
4184 /* we're the only active VCPU -> speed it up */
4185 __enable_ibs_on_vcpu(vcpu);
4186 } else if (started_vcpus == 1) {
4188 * As we are starting a second VCPU, we have to disable
4189 * the IBS facility on all VCPUs to remove potentially
4190 * oustanding ENABLE requests.
4192 __disable_ibs_on_all_vcpus(vcpu->kvm);
4195 kvm_s390_clear_cpuflags(vcpu, CPUSTAT_STOPPED);
4197 * Another VCPU might have used IBS while we were offline.
4198 * Let's play safe and flush the VCPU at startup.
4200 kvm_make_request(KVM_REQ_TLB_FLUSH, vcpu);
4201 spin_unlock(&vcpu->kvm->arch.start_stop_lock);
4205 void kvm_s390_vcpu_stop(struct kvm_vcpu *vcpu)
4207 int i, online_vcpus, started_vcpus = 0;
4208 struct kvm_vcpu *started_vcpu = NULL;
4210 if (is_vcpu_stopped(vcpu))
4213 trace_kvm_s390_vcpu_start_stop(vcpu->vcpu_id, 0);
4214 /* Only one cpu at a time may enter/leave the STOPPED state. */
4215 spin_lock(&vcpu->kvm->arch.start_stop_lock);
4216 online_vcpus = atomic_read(&vcpu->kvm->online_vcpus);
4218 /* SIGP STOP and SIGP STOP AND STORE STATUS has been fully processed */
4219 kvm_s390_clear_stop_irq(vcpu);
4221 kvm_s390_set_cpuflags(vcpu, CPUSTAT_STOPPED);
4222 __disable_ibs_on_vcpu(vcpu);
4224 for (i = 0; i < online_vcpus; i++) {
4225 if (!is_vcpu_stopped(vcpu->kvm->vcpus[i])) {
4227 started_vcpu = vcpu->kvm->vcpus[i];
4231 if (started_vcpus == 1) {
4233 * As we only have one VCPU left, we want to enable the
4234 * IBS facility for that VCPU to speed it up.
4236 __enable_ibs_on_vcpu(started_vcpu);
4239 spin_unlock(&vcpu->kvm->arch.start_stop_lock);
4243 static int kvm_vcpu_ioctl_enable_cap(struct kvm_vcpu *vcpu,
4244 struct kvm_enable_cap *cap)
4252 case KVM_CAP_S390_CSS_SUPPORT:
4253 if (!vcpu->kvm->arch.css_support) {
4254 vcpu->kvm->arch.css_support = 1;
4255 VM_EVENT(vcpu->kvm, 3, "%s", "ENABLE: CSS support");
4256 trace_kvm_s390_enable_css(vcpu->kvm);
4267 static long kvm_s390_guest_mem_op(struct kvm_vcpu *vcpu,
4268 struct kvm_s390_mem_op *mop)
4270 void __user *uaddr = (void __user *)mop->buf;
4271 void *tmpbuf = NULL;
4273 const u64 supported_flags = KVM_S390_MEMOP_F_INJECT_EXCEPTION
4274 | KVM_S390_MEMOP_F_CHECK_ONLY;
4276 if (mop->flags & ~supported_flags || mop->ar >= NUM_ACRS || !mop->size)
4279 if (mop->size > MEM_OP_MAX_SIZE)
4282 if (!(mop->flags & KVM_S390_MEMOP_F_CHECK_ONLY)) {
4283 tmpbuf = vmalloc(mop->size);
4288 srcu_idx = srcu_read_lock(&vcpu->kvm->srcu);
4291 case KVM_S390_MEMOP_LOGICAL_READ:
4292 if (mop->flags & KVM_S390_MEMOP_F_CHECK_ONLY) {
4293 r = check_gva_range(vcpu, mop->gaddr, mop->ar,
4294 mop->size, GACC_FETCH);
4297 r = read_guest(vcpu, mop->gaddr, mop->ar, tmpbuf, mop->size);
4299 if (copy_to_user(uaddr, tmpbuf, mop->size))
4303 case KVM_S390_MEMOP_LOGICAL_WRITE:
4304 if (mop->flags & KVM_S390_MEMOP_F_CHECK_ONLY) {
4305 r = check_gva_range(vcpu, mop->gaddr, mop->ar,
4306 mop->size, GACC_STORE);
4309 if (copy_from_user(tmpbuf, uaddr, mop->size)) {
4313 r = write_guest(vcpu, mop->gaddr, mop->ar, tmpbuf, mop->size);
4319 srcu_read_unlock(&vcpu->kvm->srcu, srcu_idx);
4321 if (r > 0 && (mop->flags & KVM_S390_MEMOP_F_INJECT_EXCEPTION) != 0)
4322 kvm_s390_inject_prog_irq(vcpu, &vcpu->arch.pgm);
4328 long kvm_arch_vcpu_async_ioctl(struct file *filp,
4329 unsigned int ioctl, unsigned long arg)
4331 struct kvm_vcpu *vcpu = filp->private_data;
4332 void __user *argp = (void __user *)arg;
4335 case KVM_S390_IRQ: {
4336 struct kvm_s390_irq s390irq;
4338 if (copy_from_user(&s390irq, argp, sizeof(s390irq)))
4340 return kvm_s390_inject_vcpu(vcpu, &s390irq);
4342 case KVM_S390_INTERRUPT: {
4343 struct kvm_s390_interrupt s390int;
4344 struct kvm_s390_irq s390irq = {};
4346 if (copy_from_user(&s390int, argp, sizeof(s390int)))
4348 if (s390int_to_s390irq(&s390int, &s390irq))
4350 return kvm_s390_inject_vcpu(vcpu, &s390irq);
4353 return -ENOIOCTLCMD;
4356 long kvm_arch_vcpu_ioctl(struct file *filp,
4357 unsigned int ioctl, unsigned long arg)
4359 struct kvm_vcpu *vcpu = filp->private_data;
4360 void __user *argp = (void __user *)arg;
4367 case KVM_S390_STORE_STATUS:
4368 idx = srcu_read_lock(&vcpu->kvm->srcu);
4369 r = kvm_s390_store_status_unloaded(vcpu, arg);
4370 srcu_read_unlock(&vcpu->kvm->srcu, idx);
4372 case KVM_S390_SET_INITIAL_PSW: {
4376 if (copy_from_user(&psw, argp, sizeof(psw)))
4378 r = kvm_arch_vcpu_ioctl_set_initial_psw(vcpu, psw);
4381 case KVM_S390_CLEAR_RESET:
4383 kvm_arch_vcpu_ioctl_clear_reset(vcpu);
4385 case KVM_S390_INITIAL_RESET:
4387 kvm_arch_vcpu_ioctl_initial_reset(vcpu);
4389 case KVM_S390_NORMAL_RESET:
4391 kvm_arch_vcpu_ioctl_normal_reset(vcpu);
4393 case KVM_SET_ONE_REG:
4394 case KVM_GET_ONE_REG: {
4395 struct kvm_one_reg reg;
4397 if (copy_from_user(®, argp, sizeof(reg)))
4399 if (ioctl == KVM_SET_ONE_REG)
4400 r = kvm_arch_vcpu_ioctl_set_one_reg(vcpu, ®);
4402 r = kvm_arch_vcpu_ioctl_get_one_reg(vcpu, ®);
4405 #ifdef CONFIG_KVM_S390_UCONTROL
4406 case KVM_S390_UCAS_MAP: {
4407 struct kvm_s390_ucas_mapping ucasmap;
4409 if (copy_from_user(&ucasmap, argp, sizeof(ucasmap))) {
4414 if (!kvm_is_ucontrol(vcpu->kvm)) {
4419 r = gmap_map_segment(vcpu->arch.gmap, ucasmap.user_addr,
4420 ucasmap.vcpu_addr, ucasmap.length);
4423 case KVM_S390_UCAS_UNMAP: {
4424 struct kvm_s390_ucas_mapping ucasmap;
4426 if (copy_from_user(&ucasmap, argp, sizeof(ucasmap))) {
4431 if (!kvm_is_ucontrol(vcpu->kvm)) {
4436 r = gmap_unmap_segment(vcpu->arch.gmap, ucasmap.vcpu_addr,
4441 case KVM_S390_VCPU_FAULT: {
4442 r = gmap_fault(vcpu->arch.gmap, arg, 0);
4445 case KVM_ENABLE_CAP:
4447 struct kvm_enable_cap cap;
4449 if (copy_from_user(&cap, argp, sizeof(cap)))
4451 r = kvm_vcpu_ioctl_enable_cap(vcpu, &cap);
4454 case KVM_S390_MEM_OP: {
4455 struct kvm_s390_mem_op mem_op;
4457 if (copy_from_user(&mem_op, argp, sizeof(mem_op)) == 0)
4458 r = kvm_s390_guest_mem_op(vcpu, &mem_op);
4463 case KVM_S390_SET_IRQ_STATE: {
4464 struct kvm_s390_irq_state irq_state;
4467 if (copy_from_user(&irq_state, argp, sizeof(irq_state)))
4469 if (irq_state.len > VCPU_IRQS_MAX_BUF ||
4470 irq_state.len == 0 ||
4471 irq_state.len % sizeof(struct kvm_s390_irq) > 0) {
4475 /* do not use irq_state.flags, it will break old QEMUs */
4476 r = kvm_s390_set_irq_state(vcpu,
4477 (void __user *) irq_state.buf,
4481 case KVM_S390_GET_IRQ_STATE: {
4482 struct kvm_s390_irq_state irq_state;
4485 if (copy_from_user(&irq_state, argp, sizeof(irq_state)))
4487 if (irq_state.len == 0) {
4491 /* do not use irq_state.flags, it will break old QEMUs */
4492 r = kvm_s390_get_irq_state(vcpu,
4493 (__u8 __user *) irq_state.buf,
4505 vm_fault_t kvm_arch_vcpu_fault(struct kvm_vcpu *vcpu, struct vm_fault *vmf)
4507 #ifdef CONFIG_KVM_S390_UCONTROL
4508 if ((vmf->pgoff == KVM_S390_SIE_PAGE_OFFSET)
4509 && (kvm_is_ucontrol(vcpu->kvm))) {
4510 vmf->page = virt_to_page(vcpu->arch.sie_block);
4511 get_page(vmf->page);
4515 return VM_FAULT_SIGBUS;
4518 int kvm_arch_create_memslot(struct kvm *kvm, struct kvm_memory_slot *slot,
4519 unsigned long npages)
4524 /* Section: memory related */
4525 int kvm_arch_prepare_memory_region(struct kvm *kvm,
4526 struct kvm_memory_slot *memslot,
4527 const struct kvm_userspace_memory_region *mem,
4528 enum kvm_mr_change change)
4530 /* A few sanity checks. We can have memory slots which have to be
4531 located/ended at a segment boundary (1MB). The memory in userland is
4532 ok to be fragmented into various different vmas. It is okay to mmap()
4533 and munmap() stuff in this slot after doing this call at any time */
4535 if (mem->userspace_addr & 0xffffful)
4538 if (mem->memory_size & 0xffffful)
4541 if (mem->guest_phys_addr + mem->memory_size > kvm->arch.mem_limit)
4547 void kvm_arch_commit_memory_region(struct kvm *kvm,
4548 const struct kvm_userspace_memory_region *mem,
4549 const struct kvm_memory_slot *old,
4550 const struct kvm_memory_slot *new,
4551 enum kvm_mr_change change)
4557 rc = gmap_unmap_segment(kvm->arch.gmap, old->base_gfn * PAGE_SIZE,
4558 old->npages * PAGE_SIZE);
4561 rc = gmap_unmap_segment(kvm->arch.gmap, old->base_gfn * PAGE_SIZE,
4562 old->npages * PAGE_SIZE);
4567 rc = gmap_map_segment(kvm->arch.gmap, mem->userspace_addr,
4568 mem->guest_phys_addr, mem->memory_size);
4570 case KVM_MR_FLAGS_ONLY:
4573 WARN(1, "Unknown KVM MR CHANGE: %d\n", change);
4576 pr_warn("failed to commit memory region\n");
4580 static inline unsigned long nonhyp_mask(int i)
4582 unsigned int nonhyp_fai = (sclp.hmfai << i * 2) >> 30;
4584 return 0x0000ffffffffffffUL >> (nonhyp_fai << 4);
4587 void kvm_arch_vcpu_block_finish(struct kvm_vcpu *vcpu)
4589 vcpu->valid_wakeup = false;
4592 static int __init kvm_s390_init(void)
4596 if (!sclp.has_sief2) {
4597 pr_info("SIE is not available\n");
4601 if (nested && hpage) {
4602 pr_info("A KVM host that supports nesting cannot back its KVM guests with huge pages\n");
4606 for (i = 0; i < 16; i++)
4607 kvm_s390_fac_base[i] |=
4608 S390_lowcore.stfle_fac_list[i] & nonhyp_mask(i);
4610 return kvm_init(NULL, sizeof(struct kvm_vcpu), 0, THIS_MODULE);
4613 static void __exit kvm_s390_exit(void)
4618 module_init(kvm_s390_init);
4619 module_exit(kvm_s390_exit);
4622 * Enable autoloading of the kvm module.
4623 * Note that we add the module alias here instead of virt/kvm/kvm_main.c
4624 * since x86 takes a different approach.
4626 #include <linux/miscdevice.h>
4627 MODULE_ALIAS_MISCDEV(KVM_MINOR);
4628 MODULE_ALIAS("devname:kvm");