arch/mips/kvm/trap_emul.c

   1 /*
   2  * This file is subject to the terms and conditions of the GNU General Public
   3  * License.  See the file "COPYING" in the main directory of this archive
   4  * for more details.
   5  *
   6  * KVM/MIPS: Deliver/Emulate exceptions to the guest kernel
   7  *
   8  * Copyright (C) 2012  MIPS Technologies, Inc.  All rights reserved.
   9  * Authors: Sanjay Lal <sanjayl@kymasys.com>
  10  */
  11
  12 #include <linux/errno.h>
  13 #include <linux/err.h>
  14 #include <linux/kvm_host.h>
  15 #include <linux/uaccess.h>
  16 #include <linux/vmalloc.h>
  17 #include <asm/mmu_context.h>
  18 #include <asm/pgalloc.h>
  19
  20 #include "interrupt.h"
  21
  22 static gpa_t kvm_trap_emul_gva_to_gpa_cb(gva_t gva)
  23 {
  24         gpa_t gpa;
  25         gva_t kseg = KSEGX(gva);
  26
  27         if ((kseg == CKSEG0) || (kseg == CKSEG1))
  28                 gpa = CPHYSADDR(gva);
  29         else {
  30                 kvm_err("%s: cannot find GPA for GVA: %#lx\n", __func__, gva);
  31                 kvm_mips_dump_host_tlbs();
  32                 gpa = KVM_INVALID_ADDR;
  33         }
  34
  35         kvm_debug("%s: gva %#lx, gpa: %#llx\n", __func__, gva, gpa);
  36
  37         return gpa;
  38 }
  39
  40 static int kvm_trap_emul_handle_cop_unusable(struct kvm_vcpu *vcpu)
  41 {
  42         struct mips_coproc *cop0 = vcpu->arch.cop0;
  43         struct kvm_run *run = vcpu->run;
  44         u32 __user *opc = (u32 __user *) vcpu->arch.pc;
  45         u32 cause = vcpu->arch.host_cp0_cause;
  46         enum emulation_result er = EMULATE_DONE;
  47         int ret = RESUME_GUEST;
  48
  49         if (((cause & CAUSEF_CE) >> CAUSEB_CE) == 1) {
  50                 /* FPU Unusable */
  51                 if (!kvm_mips_guest_has_fpu(&vcpu->arch) ||
  52                     (kvm_read_c0_guest_status(cop0) & ST0_CU1) == 0) {
  53                         /*
  54                          * Unusable/no FPU in guest:
  55                          * deliver guest COP1 Unusable Exception
  56                          */
  57                         er = kvm_mips_emulate_fpu_exc(cause, opc, run, vcpu);
  58                 } else {
  59                         /* Restore FPU state */
  60                         kvm_own_fpu(vcpu);
  61                         er = EMULATE_DONE;
  62                 }
  63         } else {
  64                 er = kvm_mips_emulate_inst(cause, opc, run, vcpu);
  65         }
  66
  67         switch (er) {
  68         case EMULATE_DONE:
  69                 ret = RESUME_GUEST;
  70                 break;
  71
  72         case EMULATE_FAIL:
  73                 run->exit_reason = KVM_EXIT_INTERNAL_ERROR;
  74                 ret = RESUME_HOST;
  75                 break;
  76
  77         case EMULATE_WAIT:
  78                 run->exit_reason = KVM_EXIT_INTR;
  79                 ret = RESUME_HOST;
  80                 break;
  81
  82         default:
  83                 BUG();
  84         }
  85         return ret;
  86 }
  87
  88 static int kvm_trap_emul_handle_tlb_mod(struct kvm_vcpu *vcpu)
  89 {
  90         struct kvm_run *run = vcpu->run;
  91         u32 __user *opc = (u32 __user *) vcpu->arch.pc;
  92         unsigned long badvaddr = vcpu->arch.host_cp0_badvaddr;
  93         u32 cause = vcpu->arch.host_cp0_cause;
  94         enum emulation_result er = EMULATE_DONE;
  95         int ret = RESUME_GUEST;
  96
  97         if (KVM_GUEST_KSEGX(badvaddr) < KVM_GUEST_KSEG0
  98             || KVM_GUEST_KSEGX(badvaddr) == KVM_GUEST_KSEG23) {
  99                 kvm_debug("USER/KSEG23 ADDR TLB MOD fault: cause %#x, PC: %p, BadVaddr: %#lx\n",
 100                           cause, opc, badvaddr);
 101                 er = kvm_mips_handle_tlbmod(cause, opc, run, vcpu);
 102
 103                 if (er == EMULATE_DONE)
 104                         ret = RESUME_GUEST;
 105                 else {
 106                         run->exit_reason = KVM_EXIT_INTERNAL_ERROR;
 107                         ret = RESUME_HOST;
 108                 }
 109         } else if (KVM_GUEST_KSEGX(badvaddr) == KVM_GUEST_KSEG0) {
 110                 /*
 111                  * XXXKYMA: The guest kernel does not expect to get this fault
 112                  * when we are not using HIGHMEM. Need to address this in a
 113                  * HIGHMEM kernel
 114                  */
 115                 kvm_err("TLB MOD fault not handled, cause %#x, PC: %p, BadVaddr: %#lx\n",
 116                         cause, opc, badvaddr);
 117                 kvm_mips_dump_host_tlbs();
 118                 kvm_arch_vcpu_dump_regs(vcpu);
 119                 run->exit_reason = KVM_EXIT_INTERNAL_ERROR;
 120                 ret = RESUME_HOST;
 121         } else {
 122                 kvm_err("Illegal TLB Mod fault address , cause %#x, PC: %p, BadVaddr: %#lx\n",
 123                         cause, opc, badvaddr);
 124                 kvm_mips_dump_host_tlbs();
 125                 kvm_arch_vcpu_dump_regs(vcpu);
 126                 run->exit_reason = KVM_EXIT_INTERNAL_ERROR;
 127                 ret = RESUME_HOST;
 128         }
 129         return ret;
 130 }
 131
 132 static int kvm_trap_emul_handle_tlb_miss(struct kvm_vcpu *vcpu, bool store)
 133 {
 134         struct kvm_run *run = vcpu->run;
 135         u32 __user *opc = (u32 __user *) vcpu->arch.pc;
 136         unsigned long badvaddr = vcpu->arch.host_cp0_badvaddr;
 137         u32 cause = vcpu->arch.host_cp0_cause;
 138         enum emulation_result er = EMULATE_DONE;
 139         int ret = RESUME_GUEST;
 140
 141         if (((badvaddr & PAGE_MASK) == KVM_GUEST_COMMPAGE_ADDR)
 142             && KVM_GUEST_KERNEL_MODE(vcpu)) {
 143                 if (kvm_mips_handle_commpage_tlb_fault(badvaddr, vcpu) < 0) {
 144                         run->exit_reason = KVM_EXIT_INTERNAL_ERROR;
 145                         ret = RESUME_HOST;
 146                 }
 147         } else if (KVM_GUEST_KSEGX(badvaddr) < KVM_GUEST_KSEG0
 148                    || KVM_GUEST_KSEGX(badvaddr) == KVM_GUEST_KSEG23) {
 149                 kvm_debug("USER ADDR TLB %s fault: cause %#x, PC: %p, BadVaddr: %#lx\n",
 150                           store ? "ST" : "LD", cause, opc, badvaddr);
 151
 152                 /*
 153                  * User Address (UA) fault, this could happen if
 154                  * (1) TLB entry not present/valid in both Guest and shadow host
 155                  *     TLBs, in this case we pass on the fault to the guest
 156                  *     kernel and let it handle it.
 157                  * (2) TLB entry is present in the Guest TLB but not in the
 158                  *     shadow, in this case we inject the TLB from the Guest TLB
 159                  *     into the shadow host TLB
 160                  */
 161
 162                 er = kvm_mips_handle_tlbmiss(cause, opc, run, vcpu);
 163                 if (er == EMULATE_DONE)
 164                         ret = RESUME_GUEST;
 165                 else {
 166                         run->exit_reason = KVM_EXIT_INTERNAL_ERROR;
 167                         ret = RESUME_HOST;
 168                 }
 169         } else if (KVM_GUEST_KSEGX(badvaddr) == KVM_GUEST_KSEG0) {
 170                 /*
 171                  * All KSEG0 faults are handled by KVM, as the guest kernel does
 172                  * not expect to ever get them
 173                  */
 174                 if (kvm_mips_handle_kseg0_tlb_fault
 175                     (vcpu->arch.host_cp0_badvaddr, vcpu) < 0) {
 176                         run->exit_reason = KVM_EXIT_INTERNAL_ERROR;
 177                         ret = RESUME_HOST;
 178                 }
 179         } else if (KVM_GUEST_KERNEL_MODE(vcpu)
 180                    && (KSEGX(badvaddr) == CKSEG0 || KSEGX(badvaddr) == CKSEG1)) {
 181                 /* A code fetch fault doesn't count as an MMIO */
 182                 if (!store && kvm_is_ifetch_fault(&vcpu->arch)) {
 183                         run->exit_reason = KVM_EXIT_INTERNAL_ERROR;
 184                         return RESUME_HOST;
 185                 }
 186
 187                 /*
 188                  * With EVA we may get a TLB exception instead of an address
 189                  * error when the guest performs MMIO to KSeg1 addresses.
 190                  */
 191                 kvm_debug("Emulate %s MMIO space\n",
 192                           store ? "Store to" : "Load from");
 193                 er = kvm_mips_emulate_inst(cause, opc, run, vcpu);
 194                 if (er == EMULATE_FAIL) {
 195                         kvm_err("Emulate %s MMIO space failed\n",
 196                                 store ? "Store to" : "Load from");
 197                         run->exit_reason = KVM_EXIT_INTERNAL_ERROR;
 198                         ret = RESUME_HOST;
 199                 } else {
 200                         run->exit_reason = KVM_EXIT_MMIO;
 201                         ret = RESUME_HOST;
 202                 }
 203         } else {
 204                 kvm_err("Illegal TLB %s fault address , cause %#x, PC: %p, BadVaddr: %#lx\n",
 205                         store ? "ST" : "LD", cause, opc, badvaddr);
 206                 kvm_mips_dump_host_tlbs();
 207                 kvm_arch_vcpu_dump_regs(vcpu);
 208                 run->exit_reason = KVM_EXIT_INTERNAL_ERROR;
 209                 ret = RESUME_HOST;
 210         }
 211         return ret;
 212 }
 213
 214 static int kvm_trap_emul_handle_tlb_st_miss(struct kvm_vcpu *vcpu)
 215 {
 216         return kvm_trap_emul_handle_tlb_miss(vcpu, true);
 217 }
 218
 219 static int kvm_trap_emul_handle_tlb_ld_miss(struct kvm_vcpu *vcpu)
 220 {
 221         return kvm_trap_emul_handle_tlb_miss(vcpu, false);
 222 }
 223
 224 static int kvm_trap_emul_handle_addr_err_st(struct kvm_vcpu *vcpu)
 225 {
 226         struct kvm_run *run = vcpu->run;
 227         u32 __user *opc = (u32 __user *) vcpu->arch.pc;
 228         unsigned long badvaddr = vcpu->arch.host_cp0_badvaddr;
 229         u32 cause = vcpu->arch.host_cp0_cause;
 230         enum emulation_result er = EMULATE_DONE;
 231         int ret = RESUME_GUEST;
 232
 233         if (KVM_GUEST_KERNEL_MODE(vcpu)
 234             && (KSEGX(badvaddr) == CKSEG0 || KSEGX(badvaddr) == CKSEG1)) {
 235                 kvm_debug("Emulate Store to MMIO space\n");
 236                 er = kvm_mips_emulate_inst(cause, opc, run, vcpu);
 237                 if (er == EMULATE_FAIL) {
 238                         kvm_err("Emulate Store to MMIO space failed\n");
 239                         run->exit_reason = KVM_EXIT_INTERNAL_ERROR;
 240                         ret = RESUME_HOST;
 241                 } else {
 242                         run->exit_reason = KVM_EXIT_MMIO;
 243                         ret = RESUME_HOST;
 244                 }
 245         } else {
 246                 kvm_err("Address Error (STORE): cause %#x, PC: %p, BadVaddr: %#lx\n",
 247                         cause, opc, badvaddr);
 248                 run->exit_reason = KVM_EXIT_INTERNAL_ERROR;
 249                 ret = RESUME_HOST;
 250         }
 251         return ret;
 252 }
 253
 254 static int kvm_trap_emul_handle_addr_err_ld(struct kvm_vcpu *vcpu)
 255 {
 256         struct kvm_run *run = vcpu->run;
 257         u32 __user *opc = (u32 __user *) vcpu->arch.pc;
 258         unsigned long badvaddr = vcpu->arch.host_cp0_badvaddr;
 259         u32 cause = vcpu->arch.host_cp0_cause;
 260         enum emulation_result er = EMULATE_DONE;
 261         int ret = RESUME_GUEST;
 262
 263         if (KSEGX(badvaddr) == CKSEG0 || KSEGX(badvaddr) == CKSEG1) {
 264                 /* A code fetch fault doesn't count as an MMIO */
 265                 if (kvm_is_ifetch_fault(&vcpu->arch)) {
 266                         run->exit_reason = KVM_EXIT_INTERNAL_ERROR;
 267                         return RESUME_HOST;
 268                 }
 269
 270                 kvm_debug("Emulate Load from MMIO space @ %#lx\n", badvaddr);
 271                 er = kvm_mips_emulate_inst(cause, opc, run, vcpu);
 272                 if (er == EMULATE_FAIL) {
 273                         kvm_err("Emulate Load from MMIO space failed\n");
 274                         run->exit_reason = KVM_EXIT_INTERNAL_ERROR;
 275                         ret = RESUME_HOST;
 276                 } else {
 277                         run->exit_reason = KVM_EXIT_MMIO;
 278                         ret = RESUME_HOST;
 279                 }
 280         } else {
 281                 kvm_err("Address Error (LOAD): cause %#x, PC: %p, BadVaddr: %#lx\n",
 282                         cause, opc, badvaddr);
 283                 run->exit_reason = KVM_EXIT_INTERNAL_ERROR;
 284                 ret = RESUME_HOST;
 285                 er = EMULATE_FAIL;
 286         }
 287         return ret;
 288 }
 289
 290 static int kvm_trap_emul_handle_syscall(struct kvm_vcpu *vcpu)
 291 {
 292         struct kvm_run *run = vcpu->run;
 293         u32 __user *opc = (u32 __user *) vcpu->arch.pc;
 294         u32 cause = vcpu->arch.host_cp0_cause;
 295         enum emulation_result er = EMULATE_DONE;
 296         int ret = RESUME_GUEST;
 297
 298         er = kvm_mips_emulate_syscall(cause, opc, run, vcpu);
 299         if (er == EMULATE_DONE)
 300                 ret = RESUME_GUEST;
 301         else {
 302                 run->exit_reason = KVM_EXIT_INTERNAL_ERROR;
 303                 ret = RESUME_HOST;
 304         }
 305         return ret;
 306 }
 307
 308 static int kvm_trap_emul_handle_res_inst(struct kvm_vcpu *vcpu)
 309 {
 310         struct kvm_run *run = vcpu->run;
 311         u32 __user *opc = (u32 __user *) vcpu->arch.pc;
 312         u32 cause = vcpu->arch.host_cp0_cause;
 313         enum emulation_result er = EMULATE_DONE;
 314         int ret = RESUME_GUEST;
 315
 316         er = kvm_mips_handle_ri(cause, opc, run, vcpu);
 317         if (er == EMULATE_DONE)
 318                 ret = RESUME_GUEST;
 319         else {
 320                 run->exit_reason = KVM_EXIT_INTERNAL_ERROR;
 321                 ret = RESUME_HOST;
 322         }
 323         return ret;
 324 }
 325
 326 static int kvm_trap_emul_handle_break(struct kvm_vcpu *vcpu)
 327 {
 328         struct kvm_run *run = vcpu->run;
 329         u32 __user *opc = (u32 __user *) vcpu->arch.pc;
 330         u32 cause = vcpu->arch.host_cp0_cause;
 331         enum emulation_result er = EMULATE_DONE;
 332         int ret = RESUME_GUEST;
 333
 334         er = kvm_mips_emulate_bp_exc(cause, opc, run, vcpu);
 335         if (er == EMULATE_DONE)
 336                 ret = RESUME_GUEST;
 337         else {
 338                 run->exit_reason = KVM_EXIT_INTERNAL_ERROR;
 339                 ret = RESUME_HOST;
 340         }
 341         return ret;
 342 }
 343
 344 static int kvm_trap_emul_handle_trap(struct kvm_vcpu *vcpu)
 345 {
 346         struct kvm_run *run = vcpu->run;
 347         u32 __user *opc = (u32 __user *)vcpu->arch.pc;
 348         u32 cause = vcpu->arch.host_cp0_cause;
 349         enum emulation_result er = EMULATE_DONE;
 350         int ret = RESUME_GUEST;
 351
 352         er = kvm_mips_emulate_trap_exc(cause, opc, run, vcpu);
 353         if (er == EMULATE_DONE) {
 354                 ret = RESUME_GUEST;
 355         } else {
 356                 run->exit_reason = KVM_EXIT_INTERNAL_ERROR;
 357                 ret = RESUME_HOST;
 358         }
 359         return ret;
 360 }
 361
 362 static int kvm_trap_emul_handle_msa_fpe(struct kvm_vcpu *vcpu)
 363 {
 364         struct kvm_run *run = vcpu->run;
 365         u32 __user *opc = (u32 __user *)vcpu->arch.pc;
 366         u32 cause = vcpu->arch.host_cp0_cause;
 367         enum emulation_result er = EMULATE_DONE;
 368         int ret = RESUME_GUEST;
 369
 370         er = kvm_mips_emulate_msafpe_exc(cause, opc, run, vcpu);
 371         if (er == EMULATE_DONE) {
 372                 ret = RESUME_GUEST;
 373         } else {
 374                 run->exit_reason = KVM_EXIT_INTERNAL_ERROR;
 375                 ret = RESUME_HOST;
 376         }
 377         return ret;
 378 }
 379
 380 static int kvm_trap_emul_handle_fpe(struct kvm_vcpu *vcpu)
 381 {
 382         struct kvm_run *run = vcpu->run;
 383         u32 __user *opc = (u32 __user *)vcpu->arch.pc;
 384         u32 cause = vcpu->arch.host_cp0_cause;
 385         enum emulation_result er = EMULATE_DONE;
 386         int ret = RESUME_GUEST;
 387
 388         er = kvm_mips_emulate_fpe_exc(cause, opc, run, vcpu);
 389         if (er == EMULATE_DONE) {
 390                 ret = RESUME_GUEST;
 391         } else {
 392                 run->exit_reason = KVM_EXIT_INTERNAL_ERROR;
 393                 ret = RESUME_HOST;
 394         }
 395         return ret;
 396 }
 397
 398 /**
 399  * kvm_trap_emul_handle_msa_disabled() - Guest used MSA while disabled in root.
 400  * @vcpu:       Virtual CPU context.
 401  *
 402  * Handle when the guest attempts to use MSA when it is disabled.
 403  */
 404 static int kvm_trap_emul_handle_msa_disabled(struct kvm_vcpu *vcpu)
 405 {
 406         struct mips_coproc *cop0 = vcpu->arch.cop0;
 407         struct kvm_run *run = vcpu->run;
 408         u32 __user *opc = (u32 __user *) vcpu->arch.pc;
 409         u32 cause = vcpu->arch.host_cp0_cause;
 410         enum emulation_result er = EMULATE_DONE;
 411         int ret = RESUME_GUEST;
 412
 413         if (!kvm_mips_guest_has_msa(&vcpu->arch) ||
 414             (kvm_read_c0_guest_status(cop0) & (ST0_CU1 | ST0_FR)) == ST0_CU1) {
 415                 /*
 416                  * No MSA in guest, or FPU enabled and not in FR=1 mode,
 417                  * guest reserved instruction exception
 418                  */
 419                 er = kvm_mips_emulate_ri_exc(cause, opc, run, vcpu);
 420         } else if (!(kvm_read_c0_guest_config5(cop0) & MIPS_CONF5_MSAEN)) {
 421                 /* MSA disabled by guest, guest MSA disabled exception */
 422                 er = kvm_mips_emulate_msadis_exc(cause, opc, run, vcpu);
 423         } else {
 424                 /* Restore MSA/FPU state */
 425                 kvm_own_msa(vcpu);
 426                 er = EMULATE_DONE;
 427         }
 428
 429         switch (er) {
 430         case EMULATE_DONE:
 431                 ret = RESUME_GUEST;
 432                 break;
 433
 434         case EMULATE_FAIL:
 435                 run->exit_reason = KVM_EXIT_INTERNAL_ERROR;
 436                 ret = RESUME_HOST;
 437                 break;
 438
 439         default:
 440                 BUG();
 441         }
 442         return ret;
 443 }
 444
 445 static int kvm_trap_emul_vcpu_init(struct kvm_vcpu *vcpu)
 446 {
 447         struct mm_struct *kern_mm = &vcpu->arch.guest_kernel_mm;
 448         struct mm_struct *user_mm = &vcpu->arch.guest_user_mm;
 449
 450         vcpu->arch.kscratch_enabled = 0xfc;
 451
 452         /*
 453          * Allocate GVA -> HPA page tables.
 454          * MIPS doesn't use the mm_struct pointer argument.
 455          */
 456         kern_mm->pgd = pgd_alloc(kern_mm);
 457         if (!kern_mm->pgd)
 458                 return -ENOMEM;
 459
 460         user_mm->pgd = pgd_alloc(user_mm);
 461         if (!user_mm->pgd) {
 462                 pgd_free(kern_mm, kern_mm->pgd);
 463                 return -ENOMEM;
 464         }
 465
 466         return 0;
 467 }
 468
 469 static void kvm_mips_emul_free_gva_pt(pgd_t *pgd)
 470 {
 471         /* Don't free host kernel page tables copied from init_mm.pgd */
 472         const unsigned long end = 0x80000000;
 473         unsigned long pgd_va, pud_va, pmd_va;
 474         pud_t *pud;
 475         pmd_t *pmd;
 476         pte_t *pte;
 477         int i, j, k;
 478
 479         for (i = 0; i < USER_PTRS_PER_PGD; i++) {
 480                 if (pgd_none(pgd[i]))
 481                         continue;
 482
 483                 pgd_va = (unsigned long)i << PGDIR_SHIFT;
 484                 if (pgd_va >= end)
 485                         break;
 486                 pud = pud_offset(pgd + i, 0);
 487                 for (j = 0; j < PTRS_PER_PUD; j++) {
 488                         if (pud_none(pud[j]))
 489                                 continue;
 490
 491                         pud_va = pgd_va | ((unsigned long)j << PUD_SHIFT);
 492                         if (pud_va >= end)
 493                                 break;
 494                         pmd = pmd_offset(pud + j, 0);
 495                         for (k = 0; k < PTRS_PER_PMD; k++) {
 496                                 if (pmd_none(pmd[k]))
 497                                         continue;
 498
 499                                 pmd_va = pud_va | (k << PMD_SHIFT);
 500                                 if (pmd_va >= end)
 501                                         break;
 502                                 pte = pte_offset(pmd + k, 0);
 503                                 pte_free_kernel(NULL, pte);
 504                         }
 505                         pmd_free(NULL, pmd);
 506                 }
 507                 pud_free(NULL, pud);
 508         }
 509         pgd_free(NULL, pgd);
 510 }
 511
 512 static void kvm_trap_emul_vcpu_uninit(struct kvm_vcpu *vcpu)
 513 {
 514         kvm_mips_emul_free_gva_pt(vcpu->arch.guest_kernel_mm.pgd);
 515         kvm_mips_emul_free_gva_pt(vcpu->arch.guest_user_mm.pgd);
 516 }
 517
 518 static int kvm_trap_emul_vcpu_setup(struct kvm_vcpu *vcpu)
 519 {
 520         struct mips_coproc *cop0 = vcpu->arch.cop0;
 521         u32 config, config1;
 522         int vcpu_id = vcpu->vcpu_id;
 523
 524         /*
 525          * Arch specific stuff, set up config registers properly so that the
 526          * guest will come up as expected
 527          */
 528 #ifndef CONFIG_CPU_MIPSR6
 529         /* r2-r5, simulate a MIPS 24kc */
 530         kvm_write_c0_guest_prid(cop0, 0x00019300);
 531 #else
 532         /* r6+, simulate a generic QEMU machine */
 533         kvm_write_c0_guest_prid(cop0, 0x00010000);
 534 #endif
 535         /*
 536          * Have config1, Cacheable, noncoherent, write-back, write allocate.
 537          * Endianness, arch revision & virtually tagged icache should match
 538          * host.
 539          */
 540         config = read_c0_config() & MIPS_CONF_AR;
 541         config |= MIPS_CONF_M | CONF_CM_CACHABLE_NONCOHERENT | MIPS_CONF_MT_TLB;
 542 #ifdef CONFIG_CPU_BIG_ENDIAN
 543         config |= CONF_BE;
 544 #endif
 545         if (cpu_has_vtag_icache)
 546                 config |= MIPS_CONF_VI;
 547         kvm_write_c0_guest_config(cop0, config);
 548
 549         /* Read the cache characteristics from the host Config1 Register */
 550         config1 = (read_c0_config1() & ~0x7f);
 551
 552         /* Set up MMU size */
 553         config1 &= ~(0x3f << 25);
 554         config1 |= ((KVM_MIPS_GUEST_TLB_SIZE - 1) << 25);
 555
 556         /* We unset some bits that we aren't emulating */
 557         config1 &= ~(MIPS_CONF1_C2 | MIPS_CONF1_MD | MIPS_CONF1_PC |
 558                      MIPS_CONF1_WR | MIPS_CONF1_CA);
 559         kvm_write_c0_guest_config1(cop0, config1);
 560
 561         /* Have config3, no tertiary/secondary caches implemented */
 562         kvm_write_c0_guest_config2(cop0, MIPS_CONF_M);
 563         /* MIPS_CONF_M | (read_c0_config2() & 0xfff) */
 564
 565         /* Have config4, UserLocal */
 566         kvm_write_c0_guest_config3(cop0, MIPS_CONF_M | MIPS_CONF3_ULRI);
 567
 568         /* Have config5 */
 569         kvm_write_c0_guest_config4(cop0, MIPS_CONF_M);
 570
 571         /* No config6 */
 572         kvm_write_c0_guest_config5(cop0, 0);
 573
 574         /* Set Wait IE/IXMT Ignore in Config7, IAR, AR */
 575         kvm_write_c0_guest_config7(cop0, (MIPS_CONF7_WII) | (1 << 10));
 576
 577         /*
 578          * Setup IntCtl defaults, compatibility mode for timer interrupts (HW5)
 579          */
 580         kvm_write_c0_guest_intctl(cop0, 0xFC000000);
 581
 582         /* Put in vcpu id as CPUNum into Ebase Reg to handle SMP Guests */
 583         kvm_write_c0_guest_ebase(cop0, KVM_GUEST_KSEG0 |
 584                                        (vcpu_id & MIPS_EBASE_CPUNUM));
 585
 586         return 0;
 587 }
 588
 589 static unsigned long kvm_trap_emul_num_regs(struct kvm_vcpu *vcpu)
 590 {
 591         return 0;
 592 }
 593
 594 static int kvm_trap_emul_copy_reg_indices(struct kvm_vcpu *vcpu,
 595                                           u64 __user *indices)
 596 {
 597         return 0;
 598 }
 599
 600 static int kvm_trap_emul_get_one_reg(struct kvm_vcpu *vcpu,
 601                                      const struct kvm_one_reg *reg,
 602                                      s64 *v)
 603 {
 604         switch (reg->id) {
 605         case KVM_REG_MIPS_CP0_COUNT:
 606                 *v = kvm_mips_read_count(vcpu);
 607                 break;
 608         case KVM_REG_MIPS_COUNT_CTL:
 609                 *v = vcpu->arch.count_ctl;
 610                 break;
 611         case KVM_REG_MIPS_COUNT_RESUME:
 612                 *v = ktime_to_ns(vcpu->arch.count_resume);
 613                 break;
 614         case KVM_REG_MIPS_COUNT_HZ:
 615                 *v = vcpu->arch.count_hz;
 616                 break;
 617         default:
 618                 return -EINVAL;
 619         }
 620         return 0;
 621 }
 622
 623 static int kvm_trap_emul_set_one_reg(struct kvm_vcpu *vcpu,
 624                                      const struct kvm_one_reg *reg,
 625                                      s64 v)
 626 {
 627         struct mips_coproc *cop0 = vcpu->arch.cop0;
 628         int ret = 0;
 629         unsigned int cur, change;
 630
 631         switch (reg->id) {
 632         case KVM_REG_MIPS_CP0_COUNT:
 633                 kvm_mips_write_count(vcpu, v);
 634                 break;
 635         case KVM_REG_MIPS_CP0_COMPARE:
 636                 kvm_mips_write_compare(vcpu, v, false);
 637                 break;
 638         case KVM_REG_MIPS_CP0_CAUSE:
 639                 /*
 640                  * If the timer is stopped or started (DC bit) it must look
 641                  * atomic with changes to the interrupt pending bits (TI, IRQ5).
 642                  * A timer interrupt should not happen in between.
 643                  */
 644                 if ((kvm_read_c0_guest_cause(cop0) ^ v) & CAUSEF_DC) {
 645                         if (v & CAUSEF_DC) {
 646                                 /* disable timer first */
 647                                 kvm_mips_count_disable_cause(vcpu);
 648                                 kvm_change_c0_guest_cause(cop0, ~CAUSEF_DC, v);
 649                         } else {
 650                                 /* enable timer last */
 651                                 kvm_change_c0_guest_cause(cop0, ~CAUSEF_DC, v);
 652                                 kvm_mips_count_enable_cause(vcpu);
 653                         }
 654                 } else {
 655                         kvm_write_c0_guest_cause(cop0, v);
 656                 }
 657                 break;
 658         case KVM_REG_MIPS_CP0_CONFIG:
 659                 /* read-only for now */
 660                 break;
 661         case KVM_REG_MIPS_CP0_CONFIG1:
 662                 cur = kvm_read_c0_guest_config1(cop0);
 663                 change = (cur ^ v) & kvm_mips_config1_wrmask(vcpu);
 664                 if (change) {
 665                         v = cur ^ change;
 666                         kvm_write_c0_guest_config1(cop0, v);
 667                 }
 668                 break;
 669         case KVM_REG_MIPS_CP0_CONFIG2:
 670                 /* read-only for now */
 671                 break;
 672         case KVM_REG_MIPS_CP0_CONFIG3:
 673                 cur = kvm_read_c0_guest_config3(cop0);
 674                 change = (cur ^ v) & kvm_mips_config3_wrmask(vcpu);
 675                 if (change) {
 676                         v = cur ^ change;
 677                         kvm_write_c0_guest_config3(cop0, v);
 678                 }
 679                 break;
 680         case KVM_REG_MIPS_CP0_CONFIG4:
 681                 cur = kvm_read_c0_guest_config4(cop0);
 682                 change = (cur ^ v) & kvm_mips_config4_wrmask(vcpu);
 683                 if (change) {
 684                         v = cur ^ change;
 685                         kvm_write_c0_guest_config4(cop0, v);
 686                 }
 687                 break;
 688         case KVM_REG_MIPS_CP0_CONFIG5:
 689                 cur = kvm_read_c0_guest_config5(cop0);
 690                 change = (cur ^ v) & kvm_mips_config5_wrmask(vcpu);
 691                 if (change) {
 692                         v = cur ^ change;
 693                         kvm_write_c0_guest_config5(cop0, v);
 694                 }
 695                 break;
 696         case KVM_REG_MIPS_COUNT_CTL:
 697                 ret = kvm_mips_set_count_ctl(vcpu, v);
 698                 break;
 699         case KVM_REG_MIPS_COUNT_RESUME:
 700                 ret = kvm_mips_set_count_resume(vcpu, v);
 701                 break;
 702         case KVM_REG_MIPS_COUNT_HZ:
 703                 ret = kvm_mips_set_count_hz(vcpu, v);
 704                 break;
 705         default:
 706                 return -EINVAL;
 707         }
 708         return ret;
 709 }
 710
 711 static int kvm_trap_emul_vcpu_load(struct kvm_vcpu *vcpu, int cpu)
 712 {
 713         struct mm_struct *kern_mm = &vcpu->arch.guest_kernel_mm;
 714         struct mm_struct *user_mm = &vcpu->arch.guest_user_mm;
 715         struct mm_struct *mm;
 716
 717         /* Allocate new kernel and user ASIDs if needed */
 718
 719         if ((cpu_context(cpu, kern_mm) ^ asid_cache(cpu)) &
 720                                                 asid_version_mask(cpu)) {
 721                 get_new_mmu_context(kern_mm, cpu);
 722
 723                 kvm_debug("[%d]: cpu_context: %#lx\n", cpu,
 724                           cpu_context(cpu, current->mm));
 725                 kvm_debug("[%d]: Allocated new ASID for Guest Kernel: %#lx\n",
 726                           cpu, cpu_context(cpu, kern_mm));
 727         }
 728
 729         if ((cpu_context(cpu, user_mm) ^ asid_cache(cpu)) &
 730                                                 asid_version_mask(cpu)) {
 731                 get_new_mmu_context(user_mm, cpu);
 732
 733                 kvm_debug("[%d]: cpu_context: %#lx\n", cpu,
 734                           cpu_context(cpu, current->mm));
 735                 kvm_debug("[%d]: Allocated new ASID for Guest User: %#lx\n",
 736                           cpu, cpu_context(cpu, user_mm));
 737         }
 738
 739         /*
 740          * Were we in guest context? If so then the pre-empted ASID is
 741          * no longer valid, we need to set it to what it should be based
 742          * on the mode of the Guest (Kernel/User)
 743          */
 744         if (current->flags & PF_VCPU) {
 745                 mm = KVM_GUEST_KERNEL_MODE(vcpu) ? kern_mm : user_mm;
 746                 write_c0_entryhi(cpu_asid(cpu, mm));
 747                 TLBMISS_HANDLER_SETUP_PGD(mm->pgd);
 748                 kvm_mips_suspend_mm(cpu);
 749                 ehb();
 750         }
 751
 752         return 0;
 753 }
 754
 755 static int kvm_trap_emul_vcpu_put(struct kvm_vcpu *vcpu, int cpu)
 756 {
 757         kvm_lose_fpu(vcpu);
 758
 759         if (current->flags & PF_VCPU) {
 760                 /* Restore normal Linux process memory map */
 761                 if (((cpu_context(cpu, current->mm) ^ asid_cache(cpu)) &
 762                      asid_version_mask(cpu))) {
 763                         kvm_debug("%s: Dropping MMU Context:  %#lx\n", __func__,
 764                                   cpu_context(cpu, current->mm));
 765                         get_new_mmu_context(current->mm, cpu);
 766                 }
 767                 write_c0_entryhi(cpu_asid(cpu, current->mm));
 768                 TLBMISS_HANDLER_SETUP_PGD(current->mm->pgd);
 769                 kvm_mips_resume_mm(cpu);
 770                 ehb();
 771         }
 772
 773         return 0;
 774 }
 775
 776 static void kvm_trap_emul_check_requests(struct kvm_vcpu *vcpu, int cpu,
 777                                          bool reload_asid)
 778 {
 779         struct mm_struct *kern_mm = &vcpu->arch.guest_kernel_mm;
 780         struct mm_struct *user_mm = &vcpu->arch.guest_user_mm;
 781         struct mm_struct *mm;
 782         int i;
 783
 784         if (likely(!vcpu->requests))
 785                 return;
 786
 787         if (kvm_check_request(KVM_REQ_TLB_FLUSH, vcpu)) {
 788                 /*
 789                  * Both kernel & user GVA mappings must be invalidated. The
 790                  * caller is just about to check whether the ASID is stale
 791                  * anyway so no need to reload it here.
 792                  */
 793                 kvm_mips_flush_gva_pt(kern_mm->pgd, KMF_GPA | KMF_KERN);
 794                 kvm_mips_flush_gva_pt(user_mm->pgd, KMF_GPA | KMF_USER);
 795                 for_each_possible_cpu(i) {
 796                         cpu_context(i, kern_mm) = 0;
 797                         cpu_context(i, user_mm) = 0;
 798                 }
 799
 800                 /* Generate new ASID for current mode */
 801                 if (reload_asid) {
 802                         mm = KVM_GUEST_KERNEL_MODE(vcpu) ? kern_mm : user_mm;
 803                         get_new_mmu_context(mm, cpu);
 804                         htw_stop();
 805                         write_c0_entryhi(cpu_asid(cpu, mm));
 806                         TLBMISS_HANDLER_SETUP_PGD(mm->pgd);
 807                         htw_start();
 808                 }
 809         }
 810 }
 811
 812 static void kvm_trap_emul_vcpu_reenter(struct kvm_run *run,
 813                                        struct kvm_vcpu *vcpu)
 814 {
 815         struct mm_struct *kern_mm = &vcpu->arch.guest_kernel_mm;
 816         struct mm_struct *user_mm = &vcpu->arch.guest_user_mm;
 817         struct mm_struct *mm;
 818         struct mips_coproc *cop0 = vcpu->arch.cop0;
 819         int i, cpu = smp_processor_id();
 820         unsigned int gasid;
 821
 822         /*
 823          * No need to reload ASID, IRQs are disabled already so there's no rush,
 824          * and we'll check if we need to regenerate below anyway before
 825          * re-entering the guest.
 826          */
 827         kvm_trap_emul_check_requests(vcpu, cpu, false);
 828
 829         if (KVM_GUEST_KERNEL_MODE(vcpu)) {
 830                 mm = kern_mm;
 831         } else {
 832                 mm = user_mm;
 833
 834                 /*
 835                  * Lazy host ASID regeneration / PT flush for guest user mode.
 836                  * If the guest ASID has changed since the last guest usermode
 837                  * execution, invalidate the stale TLB entries and flush GVA PT
 838                  * entries too.
 839                  */
 840                 gasid = kvm_read_c0_guest_entryhi(cop0) & KVM_ENTRYHI_ASID;
 841                 if (gasid != vcpu->arch.last_user_gasid) {
 842                         kvm_mips_flush_gva_pt(user_mm->pgd, KMF_USER);
 843                         for_each_possible_cpu(i)
 844                                 cpu_context(i, user_mm) = 0;
 845                         vcpu->arch.last_user_gasid = gasid;
 846                 }
 847         }
 848
 849         /*
 850          * Check if ASID is stale. This may happen due to a TLB flush request or
 851          * a lazy user MM invalidation.
 852          */
 853         if ((cpu_context(cpu, mm) ^ asid_cache(cpu)) &
 854             asid_version_mask(cpu))
 855                 get_new_mmu_context(mm, cpu);
 856 }
 857
 858 static int kvm_trap_emul_vcpu_run(struct kvm_run *run, struct kvm_vcpu *vcpu)
 859 {
 860         int cpu = smp_processor_id();
 861         int r;
 862
 863         /* Check if we have any exceptions/interrupts pending */
 864         kvm_mips_deliver_interrupts(vcpu,
 865                                     kvm_read_c0_guest_cause(vcpu->arch.cop0));
 866
 867         kvm_trap_emul_vcpu_reenter(run, vcpu);
 868
 869         /*
 870          * We use user accessors to access guest memory, but we don't want to
 871          * invoke Linux page faulting.
 872          */
 873         pagefault_disable();
 874
 875         /* Disable hardware page table walking while in guest */
 876         htw_stop();
 877
 878         /*
 879          * While in guest context we're in the guest's address space, not the
 880          * host process address space, so we need to be careful not to confuse
 881          * e.g. cache management IPIs.
 882          */
 883         kvm_mips_suspend_mm(cpu);
 884
 885         r = vcpu->arch.vcpu_run(run, vcpu);
 886
 887         /* We may have migrated while handling guest exits */
 888         cpu = smp_processor_id();
 889
 890         /* Restore normal Linux process memory map */
 891         if (((cpu_context(cpu, current->mm) ^ asid_cache(cpu)) &
 892              asid_version_mask(cpu)))
 893                 get_new_mmu_context(current->mm, cpu);
 894         write_c0_entryhi(cpu_asid(cpu, current->mm));
 895         TLBMISS_HANDLER_SETUP_PGD(current->mm->pgd);
 896         kvm_mips_resume_mm(cpu);
 897
 898         htw_start();
 899
 900         pagefault_enable();
 901
 902         return r;
 903 }
 904
 905 static struct kvm_mips_callbacks kvm_trap_emul_callbacks = {
 906         /* exit handlers */
 907         .handle_cop_unusable = kvm_trap_emul_handle_cop_unusable,
 908         .handle_tlb_mod = kvm_trap_emul_handle_tlb_mod,
 909         .handle_tlb_st_miss = kvm_trap_emul_handle_tlb_st_miss,
 910         .handle_tlb_ld_miss = kvm_trap_emul_handle_tlb_ld_miss,
 911         .handle_addr_err_st = kvm_trap_emul_handle_addr_err_st,
 912         .handle_addr_err_ld = kvm_trap_emul_handle_addr_err_ld,
 913         .handle_syscall = kvm_trap_emul_handle_syscall,
 914         .handle_res_inst = kvm_trap_emul_handle_res_inst,
 915         .handle_break = kvm_trap_emul_handle_break,
 916         .handle_trap = kvm_trap_emul_handle_trap,
 917         .handle_msa_fpe = kvm_trap_emul_handle_msa_fpe,
 918         .handle_fpe = kvm_trap_emul_handle_fpe,
 919         .handle_msa_disabled = kvm_trap_emul_handle_msa_disabled,
 920
 921         .vcpu_init = kvm_trap_emul_vcpu_init,
 922         .vcpu_uninit = kvm_trap_emul_vcpu_uninit,
 923         .vcpu_setup = kvm_trap_emul_vcpu_setup,
 924         .gva_to_gpa = kvm_trap_emul_gva_to_gpa_cb,
 925         .queue_timer_int = kvm_mips_queue_timer_int_cb,
 926         .dequeue_timer_int = kvm_mips_dequeue_timer_int_cb,
 927         .queue_io_int = kvm_mips_queue_io_int_cb,
 928         .dequeue_io_int = kvm_mips_dequeue_io_int_cb,
 929         .irq_deliver = kvm_mips_irq_deliver_cb,
 930         .irq_clear = kvm_mips_irq_clear_cb,
 931         .num_regs = kvm_trap_emul_num_regs,
 932         .copy_reg_indices = kvm_trap_emul_copy_reg_indices,
 933         .get_one_reg = kvm_trap_emul_get_one_reg,
 934         .set_one_reg = kvm_trap_emul_set_one_reg,
 935         .vcpu_load = kvm_trap_emul_vcpu_load,
 936         .vcpu_put = kvm_trap_emul_vcpu_put,
 937         .vcpu_run = kvm_trap_emul_vcpu_run,
 938         .vcpu_reenter = kvm_trap_emul_vcpu_reenter,
 939 };
 940
 941 int kvm_mips_emulation_init(struct kvm_mips_callbacks **install_callbacks)
 942 {
 943         *install_callbacks = &kvm_trap_emul_callbacks;
 944         return 0;
 945 }