arch/mips/kvm/emulate.c

   1 /*
   2  * This file is subject to the terms and conditions of the GNU General Public
   3  * License.  See the file "COPYING" in the main directory of this archive
   4  * for more details.
   5  *
   6  * KVM/MIPS: Instruction/Exception emulation
   7  *
   8  * Copyright (C) 2012  MIPS Technologies, Inc.  All rights reserved.
   9  * Authors: Sanjay Lal <sanjayl@kymasys.com>
  10  */
  11
  12 #include <linux/errno.h>
  13 #include <linux/err.h>
  14 #include <linux/ktime.h>
  15 #include <linux/kvm_host.h>
  16 #include <linux/vmalloc.h>
  17 #include <linux/fs.h>
  18 #include <linux/bootmem.h>
  19 #include <linux/random.h>
  20 #include <asm/page.h>
  21 #include <asm/cacheflush.h>
  22 #include <asm/cacheops.h>
  23 #include <asm/cpu-info.h>
  24 #include <asm/mmu_context.h>
  25 #include <asm/tlbflush.h>
  26 #include <asm/inst.h>
  27
  28 #undef CONFIG_MIPS_MT
  29 #include <asm/r4kcache.h>
  30 #define CONFIG_MIPS_MT
  31
  32 #include "interrupt.h"
  33 #include "commpage.h"
  34
  35 #include "trace.h"
  36
  37 /*
  38  * Compute the return address and do emulate branch simulation, if required.
  39  * This function should be called only in branch delay slot active.
  40  */
  41 static int kvm_compute_return_epc(struct kvm_vcpu *vcpu, unsigned long instpc,
  42                                   unsigned long *out)
  43 {
  44         unsigned int dspcontrol;
  45         union mips_instruction insn;
  46         struct kvm_vcpu_arch *arch = &vcpu->arch;
  47         long epc = instpc;
  48         long nextpc;
  49         int err;
  50
  51         if (epc & 3) {
  52                 kvm_err("%s: unaligned epc\n", __func__);
  53                 return -EINVAL;
  54         }
  55
  56         /* Read the instruction */
  57         err = kvm_get_badinstrp((u32 *)epc, vcpu, &insn.word);
  58         if (err)
  59                 return err;
  60
  61         switch (insn.i_format.opcode) {
  62                 /* jr and jalr are in r_format format. */
  63         case spec_op:
  64                 switch (insn.r_format.func) {
  65                 case jalr_op:
  66                         arch->gprs[insn.r_format.rd] = epc + 8;
  67                         /* Fall through */
  68                 case jr_op:
  69                         nextpc = arch->gprs[insn.r_format.rs];
  70                         break;
  71                 default:
  72                         return -EINVAL;
  73                 }
  74                 break;
  75
  76                 /*
  77                  * This group contains:
  78                  * bltz_op, bgez_op, bltzl_op, bgezl_op,
  79                  * bltzal_op, bgezal_op, bltzall_op, bgezall_op.
  80                  */
  81         case bcond_op:
  82                 switch (insn.i_format.rt) {
  83                 case bltz_op:
  84                 case bltzl_op:
  85                         if ((long)arch->gprs[insn.i_format.rs] < 0)
  86                                 epc = epc + 4 + (insn.i_format.simmediate << 2);
  87                         else
  88                                 epc += 8;
  89                         nextpc = epc;
  90                         break;
  91
  92                 case bgez_op:
  93                 case bgezl_op:
  94                         if ((long)arch->gprs[insn.i_format.rs] >= 0)
  95                                 epc = epc + 4 + (insn.i_format.simmediate << 2);
  96                         else
  97                                 epc += 8;
  98                         nextpc = epc;
  99                         break;
 100
 101                 case bltzal_op:
 102                 case bltzall_op:
 103                         arch->gprs[31] = epc + 8;
 104                         if ((long)arch->gprs[insn.i_format.rs] < 0)
 105                                 epc = epc + 4 + (insn.i_format.simmediate << 2);
 106                         else
 107                                 epc += 8;
 108                         nextpc = epc;
 109                         break;
 110
 111                 case bgezal_op:
 112                 case bgezall_op:
 113                         arch->gprs[31] = epc + 8;
 114                         if ((long)arch->gprs[insn.i_format.rs] >= 0)
 115                                 epc = epc + 4 + (insn.i_format.simmediate << 2);
 116                         else
 117                                 epc += 8;
 118                         nextpc = epc;
 119                         break;
 120                 case bposge32_op:
 121                         if (!cpu_has_dsp) {
 122                                 kvm_err("%s: DSP branch but not DSP ASE\n",
 123                                         __func__);
 124                                 return -EINVAL;
 125                         }
 126
 127                         dspcontrol = rddsp(0x01);
 128
 129                         if (dspcontrol >= 32)
 130                                 epc = epc + 4 + (insn.i_format.simmediate << 2);
 131                         else
 132                                 epc += 8;
 133                         nextpc = epc;
 134                         break;
 135                 default:
 136                         return -EINVAL;
 137                 }
 138                 break;
 139
 140                 /* These are unconditional and in j_format. */
 141         case jal_op:
 142                 arch->gprs[31] = instpc + 8;
 143         case j_op:
 144                 epc += 4;
 145                 epc >>= 28;
 146                 epc <<= 28;
 147                 epc |= (insn.j_format.target << 2);
 148                 nextpc = epc;
 149                 break;
 150
 151                 /* These are conditional and in i_format. */
 152         case beq_op:
 153         case beql_op:
 154                 if (arch->gprs[insn.i_format.rs] ==
 155                     arch->gprs[insn.i_format.rt])
 156                         epc = epc + 4 + (insn.i_format.simmediate << 2);
 157                 else
 158                         epc += 8;
 159                 nextpc = epc;
 160                 break;
 161
 162         case bne_op:
 163         case bnel_op:
 164                 if (arch->gprs[insn.i_format.rs] !=
 165                     arch->gprs[insn.i_format.rt])
 166                         epc = epc + 4 + (insn.i_format.simmediate << 2);
 167                 else
 168                         epc += 8;
 169                 nextpc = epc;
 170                 break;
 171
 172         case blez_op:   /* POP06 */
 173 #ifndef CONFIG_CPU_MIPSR6
 174         case blezl_op:  /* removed in R6 */
 175 #endif
 176                 if (insn.i_format.rt != 0)
 177                         goto compact_branch;
 178                 if ((long)arch->gprs[insn.i_format.rs] <= 0)
 179                         epc = epc + 4 + (insn.i_format.simmediate << 2);
 180                 else
 181                         epc += 8;
 182                 nextpc = epc;
 183                 break;
 184
 185         case bgtz_op:   /* POP07 */
 186 #ifndef CONFIG_CPU_MIPSR6
 187         case bgtzl_op:  /* removed in R6 */
 188 #endif
 189                 if (insn.i_format.rt != 0)
 190                         goto compact_branch;
 191                 if ((long)arch->gprs[insn.i_format.rs] > 0)
 192                         epc = epc + 4 + (insn.i_format.simmediate << 2);
 193                 else
 194                         epc += 8;
 195                 nextpc = epc;
 196                 break;
 197
 198                 /* And now the FPA/cp1 branch instructions. */
 199         case cop1_op:
 200                 kvm_err("%s: unsupported cop1_op\n", __func__);
 201                 return -EINVAL;
 202
 203 #ifdef CONFIG_CPU_MIPSR6
 204         /* R6 added the following compact branches with forbidden slots */
 205         case blezl_op:  /* POP26 */
 206         case bgtzl_op:  /* POP27 */
 207                 /* only rt == 0 isn't compact branch */
 208                 if (insn.i_format.rt != 0)
 209                         goto compact_branch;
 210                 return -EINVAL;
 211         case pop10_op:
 212         case pop30_op:
 213                 /* only rs == rt == 0 is reserved, rest are compact branches */
 214                 if (insn.i_format.rs != 0 || insn.i_format.rt != 0)
 215                         goto compact_branch;
 216                 return -EINVAL;
 217         case pop66_op:
 218         case pop76_op:
 219                 /* only rs == 0 isn't compact branch */
 220                 if (insn.i_format.rs != 0)
 221                         goto compact_branch;
 222                 return -EINVAL;
 223 compact_branch:
 224                 /*
 225                  * If we've hit an exception on the forbidden slot, then
 226                  * the branch must not have been taken.
 227                  */
 228                 epc += 8;
 229                 nextpc = epc;
 230                 break;
 231 #else
 232 compact_branch:
 233                 /* Fall through - Compact branches not supported before R6 */
 234 #endif
 235         default:
 236                 return -EINVAL;
 237         }
 238
 239         *out = nextpc;
 240         return 0;
 241 }
 242
 243 enum emulation_result update_pc(struct kvm_vcpu *vcpu, u32 cause)
 244 {
 245         int err;
 246
 247         if (cause & CAUSEF_BD) {
 248                 err = kvm_compute_return_epc(vcpu, vcpu->arch.pc,
 249                                              &vcpu->arch.pc);
 250                 if (err)
 251                         return EMULATE_FAIL;
 252         } else {
 253                 vcpu->arch.pc += 4;
 254         }
 255
 256         kvm_debug("update_pc(): New PC: %#lx\n", vcpu->arch.pc);
 257
 258         return EMULATE_DONE;
 259 }
 260
 261 /**
 262  * kvm_get_badinstr() - Get bad instruction encoding.
 263  * @opc:        Guest pointer to faulting instruction.
 264  * @vcpu:       KVM VCPU information.
 265  *
 266  * Gets the instruction encoding of the faulting instruction, using the saved
 267  * BadInstr register value if it exists, otherwise falling back to reading guest
 268  * memory at @opc.
 269  *
 270  * Returns:     The instruction encoding of the faulting instruction.
 271  */
 272 int kvm_get_badinstr(u32 *opc, struct kvm_vcpu *vcpu, u32 *out)
 273 {
 274         if (cpu_has_badinstr) {
 275                 *out = vcpu->arch.host_cp0_badinstr;
 276                 return 0;
 277         } else {
 278                 return kvm_get_inst(opc, vcpu, out);
 279         }
 280 }
 281
 282 /**
 283  * kvm_get_badinstrp() - Get bad prior instruction encoding.
 284  * @opc:        Guest pointer to prior faulting instruction.
 285  * @vcpu:       KVM VCPU information.
 286  *
 287  * Gets the instruction encoding of the prior faulting instruction (the branch
 288  * containing the delay slot which faulted), using the saved BadInstrP register
 289  * value if it exists, otherwise falling back to reading guest memory at @opc.
 290  *
 291  * Returns:     The instruction encoding of the prior faulting instruction.
 292  */
 293 int kvm_get_badinstrp(u32 *opc, struct kvm_vcpu *vcpu, u32 *out)
 294 {
 295         if (cpu_has_badinstrp) {
 296                 *out = vcpu->arch.host_cp0_badinstrp;
 297                 return 0;
 298         } else {
 299                 return kvm_get_inst(opc, vcpu, out);
 300         }
 301 }
 302
 303 /**
 304  * kvm_mips_count_disabled() - Find whether the CP0_Count timer is disabled.
 305  * @vcpu:       Virtual CPU.
 306  *
 307  * Returns:     1 if the CP0_Count timer is disabled by either the guest
 308  *              CP0_Cause.DC bit or the count_ctl.DC bit.
 309  *              0 otherwise (in which case CP0_Count timer is running).
 310  */
 311 static inline int kvm_mips_count_disabled(struct kvm_vcpu *vcpu)
 312 {
 313         struct mips_coproc *cop0 = vcpu->arch.cop0;
 314
 315         return  (vcpu->arch.count_ctl & KVM_REG_MIPS_COUNT_CTL_DC) ||
 316                 (kvm_read_c0_guest_cause(cop0) & CAUSEF_DC);
 317 }
 318
 319 /**
 320  * kvm_mips_ktime_to_count() - Scale ktime_t to a 32-bit count.
 321  *
 322  * Caches the dynamic nanosecond bias in vcpu->arch.count_dyn_bias.
 323  *
 324  * Assumes !kvm_mips_count_disabled(@vcpu) (guest CP0_Count timer is running).
 325  */
 326 static u32 kvm_mips_ktime_to_count(struct kvm_vcpu *vcpu, ktime_t now)
 327 {
 328         s64 now_ns, periods;
 329         u64 delta;
 330
 331         now_ns = ktime_to_ns(now);
 332         delta = now_ns + vcpu->arch.count_dyn_bias;
 333
 334         if (delta >= vcpu->arch.count_period) {
 335                 /* If delta is out of safe range the bias needs adjusting */
 336                 periods = div64_s64(now_ns, vcpu->arch.count_period);
 337                 vcpu->arch.count_dyn_bias = -periods * vcpu->arch.count_period;
 338                 /* Recalculate delta with new bias */
 339                 delta = now_ns + vcpu->arch.count_dyn_bias;
 340         }
 341
 342         /*
 343          * We've ensured that:
 344          *   delta < count_period
 345          *
 346          * Therefore the intermediate delta*count_hz will never overflow since
 347          * at the boundary condition:
 348          *   delta = count_period
 349          *   delta = NSEC_PER_SEC * 2^32 / count_hz
 350          *   delta * count_hz = NSEC_PER_SEC * 2^32
 351          */
 352         return div_u64(delta * vcpu->arch.count_hz, NSEC_PER_SEC);
 353 }
 354
 355 /**
 356  * kvm_mips_count_time() - Get effective current time.
 357  * @vcpu:       Virtual CPU.
 358  *
 359  * Get effective monotonic ktime. This is usually a straightforward ktime_get(),
 360  * except when the master disable bit is set in count_ctl, in which case it is
 361  * count_resume, i.e. the time that the count was disabled.
 362  *
 363  * Returns:     Effective monotonic ktime for CP0_Count.
 364  */
 365 static inline ktime_t kvm_mips_count_time(struct kvm_vcpu *vcpu)
 366 {
 367         if (unlikely(vcpu->arch.count_ctl & KVM_REG_MIPS_COUNT_CTL_DC))
 368                 return vcpu->arch.count_resume;
 369
 370         return ktime_get();
 371 }
 372
 373 /**
 374  * kvm_mips_read_count_running() - Read the current count value as if running.
 375  * @vcpu:       Virtual CPU.
 376  * @now:        Kernel time to read CP0_Count at.
 377  *
 378  * Returns the current guest CP0_Count register at time @now and handles if the
 379  * timer interrupt is pending and hasn't been handled yet.
 380  *
 381  * Returns:     The current value of the guest CP0_Count register.
 382  */
 383 static u32 kvm_mips_read_count_running(struct kvm_vcpu *vcpu, ktime_t now)
 384 {
 385         struct mips_coproc *cop0 = vcpu->arch.cop0;
 386         ktime_t expires, threshold;
 387         u32 count, compare;
 388         int running;
 389
 390         /* Calculate the biased and scaled guest CP0_Count */
 391         count = vcpu->arch.count_bias + kvm_mips_ktime_to_count(vcpu, now);
 392         compare = kvm_read_c0_guest_compare(cop0);
 393
 394         /*
 395          * Find whether CP0_Count has reached the closest timer interrupt. If
 396          * not, we shouldn't inject it.
 397          */
 398         if ((s32)(count - compare) < 0)
 399                 return count;
 400
 401         /*
 402          * The CP0_Count we're going to return has already reached the closest
 403          * timer interrupt. Quickly check if it really is a new interrupt by
 404          * looking at whether the interval until the hrtimer expiry time is
 405          * less than 1/4 of the timer period.
 406          */
 407         expires = hrtimer_get_expires(&vcpu->arch.comparecount_timer);
 408         threshold = ktime_add_ns(now, vcpu->arch.count_period / 4);
 409         if (ktime_before(expires, threshold)) {
 410                 /*
 411                  * Cancel it while we handle it so there's no chance of
 412                  * interference with the timeout handler.
 413                  */
 414                 running = hrtimer_cancel(&vcpu->arch.comparecount_timer);
 415
 416                 /* Nothing should be waiting on the timeout */
 417                 kvm_mips_callbacks->queue_timer_int(vcpu);
 418
 419                 /*
 420                  * Restart the timer if it was running based on the expiry time
 421                  * we read, so that we don't push it back 2 periods.
 422                  */
 423                 if (running) {
 424                         expires = ktime_add_ns(expires,
 425                                                vcpu->arch.count_period);
 426                         hrtimer_start(&vcpu->arch.comparecount_timer, expires,
 427                                       HRTIMER_MODE_ABS);
 428                 }
 429         }
 430
 431         return count;
 432 }
 433
 434 /**
 435  * kvm_mips_read_count() - Read the current count value.
 436  * @vcpu:       Virtual CPU.
 437  *
 438  * Read the current guest CP0_Count value, taking into account whether the timer
 439  * is stopped.
 440  *
 441  * Returns:     The current guest CP0_Count value.
 442  */
 443 u32 kvm_mips_read_count(struct kvm_vcpu *vcpu)
 444 {
 445         struct mips_coproc *cop0 = vcpu->arch.cop0;
 446
 447         /* If count disabled just read static copy of count */
 448         if (kvm_mips_count_disabled(vcpu))
 449                 return kvm_read_c0_guest_count(cop0);
 450
 451         return kvm_mips_read_count_running(vcpu, ktime_get());
 452 }
 453
 454 /**
 455  * kvm_mips_freeze_hrtimer() - Safely stop the hrtimer.
 456  * @vcpu:       Virtual CPU.
 457  * @count:      Output pointer for CP0_Count value at point of freeze.
 458  *
 459  * Freeze the hrtimer safely and return both the ktime and the CP0_Count value
 460  * at the point it was frozen. It is guaranteed that any pending interrupts at
 461  * the point it was frozen are handled, and none after that point.
 462  *
 463  * This is useful where the time/CP0_Count is needed in the calculation of the
 464  * new parameters.
 465  *
 466  * Assumes !kvm_mips_count_disabled(@vcpu) (guest CP0_Count timer is running).
 467  *
 468  * Returns:     The ktime at the point of freeze.
 469  */
 470 static ktime_t kvm_mips_freeze_hrtimer(struct kvm_vcpu *vcpu, u32 *count)
 471 {
 472         ktime_t now;
 473
 474         /* stop hrtimer before finding time */
 475         hrtimer_cancel(&vcpu->arch.comparecount_timer);
 476         now = ktime_get();
 477
 478         /* find count at this point and handle pending hrtimer */
 479         *count = kvm_mips_read_count_running(vcpu, now);
 480
 481         return now;
 482 }
 483
 484 /**
 485  * kvm_mips_resume_hrtimer() - Resume hrtimer, updating expiry.
 486  * @vcpu:       Virtual CPU.
 487  * @now:        ktime at point of resume.
 488  * @count:      CP0_Count at point of resume.
 489  *
 490  * Resumes the timer and updates the timer expiry based on @now and @count.
 491  * This can be used in conjunction with kvm_mips_freeze_timer() when timer
 492  * parameters need to be changed.
 493  *
 494  * It is guaranteed that a timer interrupt immediately after resume will be
 495  * handled, but not if CP_Compare is exactly at @count. That case is already
 496  * handled by kvm_mips_freeze_timer().
 497  *
 498  * Assumes !kvm_mips_count_disabled(@vcpu) (guest CP0_Count timer is running).
 499  */
 500 static void kvm_mips_resume_hrtimer(struct kvm_vcpu *vcpu,
 501                                     ktime_t now, u32 count)
 502 {
 503         struct mips_coproc *cop0 = vcpu->arch.cop0;
 504         u32 compare;
 505         u64 delta;
 506         ktime_t expire;
 507
 508         /* Calculate timeout (wrap 0 to 2^32) */
 509         compare = kvm_read_c0_guest_compare(cop0);
 510         delta = (u64)(u32)(compare - count - 1) + 1;
 511         delta = div_u64(delta * NSEC_PER_SEC, vcpu->arch.count_hz);
 512         expire = ktime_add_ns(now, delta);
 513
 514         /* Update hrtimer to use new timeout */
 515         hrtimer_cancel(&vcpu->arch.comparecount_timer);
 516         hrtimer_start(&vcpu->arch.comparecount_timer, expire, HRTIMER_MODE_ABS);
 517 }
 518
 519 /**
 520  * kvm_mips_write_count() - Modify the count and update timer.
 521  * @vcpu:       Virtual CPU.
 522  * @count:      Guest CP0_Count value to set.
 523  *
 524  * Sets the CP0_Count value and updates the timer accordingly.
 525  */
 526 void kvm_mips_write_count(struct kvm_vcpu *vcpu, u32 count)
 527 {
 528         struct mips_coproc *cop0 = vcpu->arch.cop0;
 529         ktime_t now;
 530
 531         /* Calculate bias */
 532         now = kvm_mips_count_time(vcpu);
 533         vcpu->arch.count_bias = count - kvm_mips_ktime_to_count(vcpu, now);
 534
 535         if (kvm_mips_count_disabled(vcpu))
 536                 /* The timer's disabled, adjust the static count */
 537                 kvm_write_c0_guest_count(cop0, count);
 538         else
 539                 /* Update timeout */
 540                 kvm_mips_resume_hrtimer(vcpu, now, count);
 541 }
 542
 543 /**
 544  * kvm_mips_init_count() - Initialise timer.
 545  * @vcpu:       Virtual CPU.
 546  *
 547  * Initialise the timer to a sensible frequency, namely 100MHz, zero it, and set
 548  * it going if it's enabled.
 549  */
 550 void kvm_mips_init_count(struct kvm_vcpu *vcpu)
 551 {
 552         /* 100 MHz */
 553         vcpu->arch.count_hz = 100*1000*1000;
 554         vcpu->arch.count_period = div_u64((u64)NSEC_PER_SEC << 32,
 555                                           vcpu->arch.count_hz);
 556         vcpu->arch.count_dyn_bias = 0;
 557
 558         /* Starting at 0 */
 559         kvm_mips_write_count(vcpu, 0);
 560 }
 561
 562 /**
 563  * kvm_mips_set_count_hz() - Update the frequency of the timer.
 564  * @vcpu:       Virtual CPU.
 565  * @count_hz:   Frequency of CP0_Count timer in Hz.
 566  *
 567  * Change the frequency of the CP0_Count timer. This is done atomically so that
 568  * CP0_Count is continuous and no timer interrupt is lost.
 569  *
 570  * Returns:     -EINVAL if @count_hz is out of range.
 571  *              0 on success.
 572  */
 573 int kvm_mips_set_count_hz(struct kvm_vcpu *vcpu, s64 count_hz)
 574 {
 575         struct mips_coproc *cop0 = vcpu->arch.cop0;
 576         int dc;
 577         ktime_t now;
 578         u32 count;
 579
 580         /* ensure the frequency is in a sensible range... */
 581         if (count_hz <= 0 || count_hz > NSEC_PER_SEC)
 582                 return -EINVAL;
 583         /* ... and has actually changed */
 584         if (vcpu->arch.count_hz == count_hz)
 585                 return 0;
 586
 587         /* Safely freeze timer so we can keep it continuous */
 588         dc = kvm_mips_count_disabled(vcpu);
 589         if (dc) {
 590                 now = kvm_mips_count_time(vcpu);
 591                 count = kvm_read_c0_guest_count(cop0);
 592         } else {
 593                 now = kvm_mips_freeze_hrtimer(vcpu, &count);
 594         }
 595
 596         /* Update the frequency */
 597         vcpu->arch.count_hz = count_hz;
 598         vcpu->arch.count_period = div_u64((u64)NSEC_PER_SEC << 32, count_hz);
 599         vcpu->arch.count_dyn_bias = 0;
 600
 601         /* Calculate adjusted bias so dynamic count is unchanged */
 602         vcpu->arch.count_bias = count - kvm_mips_ktime_to_count(vcpu, now);
 603
 604         /* Update and resume hrtimer */
 605         if (!dc)
 606                 kvm_mips_resume_hrtimer(vcpu, now, count);
 607         return 0;
 608 }
 609
 610 /**
 611  * kvm_mips_write_compare() - Modify compare and update timer.
 612  * @vcpu:       Virtual CPU.
 613  * @compare:    New CP0_Compare value.
 614  * @ack:        Whether to acknowledge timer interrupt.
 615  *
 616  * Update CP0_Compare to a new value and update the timeout.
 617  * If @ack, atomically acknowledge any pending timer interrupt, otherwise ensure
 618  * any pending timer interrupt is preserved.
 619  */
 620 void kvm_mips_write_compare(struct kvm_vcpu *vcpu, u32 compare, bool ack)
 621 {
 622         struct mips_coproc *cop0 = vcpu->arch.cop0;
 623         int dc;
 624         u32 old_compare = kvm_read_c0_guest_compare(cop0);
 625         ktime_t now;
 626         u32 count;
 627
 628         /* if unchanged, must just be an ack */
 629         if (old_compare == compare) {
 630                 if (!ack)
 631                         return;
 632                 kvm_mips_callbacks->dequeue_timer_int(vcpu);
 633                 kvm_write_c0_guest_compare(cop0, compare);
 634                 return;
 635         }
 636
 637         /* freeze_hrtimer() takes care of timer interrupts <= count */
 638         dc = kvm_mips_count_disabled(vcpu);
 639         if (!dc)
 640                 now = kvm_mips_freeze_hrtimer(vcpu, &count);
 641
 642         if (ack)
 643                 kvm_mips_callbacks->dequeue_timer_int(vcpu);
 644
 645         kvm_write_c0_guest_compare(cop0, compare);
 646
 647         /* resume_hrtimer() takes care of timer interrupts > count */
 648         if (!dc)
 649                 kvm_mips_resume_hrtimer(vcpu, now, count);
 650 }
 651
 652 /**
 653  * kvm_mips_count_disable() - Disable count.
 654  * @vcpu:       Virtual CPU.
 655  *
 656  * Disable the CP0_Count timer. A timer interrupt on or before the final stop
 657  * time will be handled but not after.
 658  *
 659  * Assumes CP0_Count was previously enabled but now Guest.CP0_Cause.DC or
 660  * count_ctl.DC has been set (count disabled).
 661  *
 662  * Returns:     The time that the timer was stopped.
 663  */
 664 static ktime_t kvm_mips_count_disable(struct kvm_vcpu *vcpu)
 665 {
 666         struct mips_coproc *cop0 = vcpu->arch.cop0;
 667         u32 count;
 668         ktime_t now;
 669
 670         /* Stop hrtimer */
 671         hrtimer_cancel(&vcpu->arch.comparecount_timer);
 672
 673         /* Set the static count from the dynamic count, handling pending TI */
 674         now = ktime_get();
 675         count = kvm_mips_read_count_running(vcpu, now);
 676         kvm_write_c0_guest_count(cop0, count);
 677
 678         return now;
 679 }
 680
 681 /**
 682  * kvm_mips_count_disable_cause() - Disable count using CP0_Cause.DC.
 683  * @vcpu:       Virtual CPU.
 684  *
 685  * Disable the CP0_Count timer and set CP0_Cause.DC. A timer interrupt on or
 686  * before the final stop time will be handled if the timer isn't disabled by
 687  * count_ctl.DC, but not after.
 688  *
 689  * Assumes CP0_Cause.DC is clear (count enabled).
 690  */
 691 void kvm_mips_count_disable_cause(struct kvm_vcpu *vcpu)
 692 {
 693         struct mips_coproc *cop0 = vcpu->arch.cop0;
 694
 695         kvm_set_c0_guest_cause(cop0, CAUSEF_DC);
 696         if (!(vcpu->arch.count_ctl & KVM_REG_MIPS_COUNT_CTL_DC))
 697                 kvm_mips_count_disable(vcpu);
 698 }
 699
 700 /**
 701  * kvm_mips_count_enable_cause() - Enable count using CP0_Cause.DC.
 702  * @vcpu:       Virtual CPU.
 703  *
 704  * Enable the CP0_Count timer and clear CP0_Cause.DC. A timer interrupt after
 705  * the start time will be handled if the timer isn't disabled by count_ctl.DC,
 706  * potentially before even returning, so the caller should be careful with
 707  * ordering of CP0_Cause modifications so as not to lose it.
 708  *
 709  * Assumes CP0_Cause.DC is set (count disabled).
 710  */
 711 void kvm_mips_count_enable_cause(struct kvm_vcpu *vcpu)
 712 {
 713         struct mips_coproc *cop0 = vcpu->arch.cop0;
 714         u32 count;
 715
 716         kvm_clear_c0_guest_cause(cop0, CAUSEF_DC);
 717
 718         /*
 719          * Set the dynamic count to match the static count.
 720          * This starts the hrtimer if count_ctl.DC allows it.
 721          * Otherwise it conveniently updates the biases.
 722          */
 723         count = kvm_read_c0_guest_count(cop0);
 724         kvm_mips_write_count(vcpu, count);
 725 }
 726
 727 /**
 728  * kvm_mips_set_count_ctl() - Update the count control KVM register.
 729  * @vcpu:       Virtual CPU.
 730  * @count_ctl:  Count control register new value.
 731  *
 732  * Set the count control KVM register. The timer is updated accordingly.
 733  *
 734  * Returns:     -EINVAL if reserved bits are set.
 735  *              0 on success.
 736  */
 737 int kvm_mips_set_count_ctl(struct kvm_vcpu *vcpu, s64 count_ctl)
 738 {
 739         struct mips_coproc *cop0 = vcpu->arch.cop0;
 740         s64 changed = count_ctl ^ vcpu->arch.count_ctl;
 741         s64 delta;
 742         ktime_t expire, now;
 743         u32 count, compare;
 744
 745         /* Only allow defined bits to be changed */
 746         if (changed & ~(s64)(KVM_REG_MIPS_COUNT_CTL_DC))
 747                 return -EINVAL;
 748
 749         /* Apply new value */
 750         vcpu->arch.count_ctl = count_ctl;
 751
 752         /* Master CP0_Count disable */
 753         if (changed & KVM_REG_MIPS_COUNT_CTL_DC) {
 754                 /* Is CP0_Cause.DC already disabling CP0_Count? */
 755                 if (kvm_read_c0_guest_cause(cop0) & CAUSEF_DC) {
 756                         if (count_ctl & KVM_REG_MIPS_COUNT_CTL_DC)
 757                                 /* Just record the current time */
 758                                 vcpu->arch.count_resume = ktime_get();
 759                 } else if (count_ctl & KVM_REG_MIPS_COUNT_CTL_DC) {
 760                         /* disable timer and record current time */
 761                         vcpu->arch.count_resume = kvm_mips_count_disable(vcpu);
 762                 } else {
 763                         /*
 764                          * Calculate timeout relative to static count at resume
 765                          * time (wrap 0 to 2^32).
 766                          */
 767                         count = kvm_read_c0_guest_count(cop0);
 768                         compare = kvm_read_c0_guest_compare(cop0);
 769                         delta = (u64)(u32)(compare - count - 1) + 1;
 770                         delta = div_u64(delta * NSEC_PER_SEC,
 771                                         vcpu->arch.count_hz);
 772                         expire = ktime_add_ns(vcpu->arch.count_resume, delta);
 773
 774                         /* Handle pending interrupt */
 775                         now = ktime_get();
 776                         if (ktime_compare(now, expire) >= 0)
 777                                 /* Nothing should be waiting on the timeout */
 778                                 kvm_mips_callbacks->queue_timer_int(vcpu);
 779
 780                         /* Resume hrtimer without changing bias */
 781                         count = kvm_mips_read_count_running(vcpu, now);
 782                         kvm_mips_resume_hrtimer(vcpu, now, count);
 783                 }
 784         }
 785
 786         return 0;
 787 }
 788
 789 /**
 790  * kvm_mips_set_count_resume() - Update the count resume KVM register.
 791  * @vcpu:               Virtual CPU.
 792  * @count_resume:       Count resume register new value.
 793  *
 794  * Set the count resume KVM register.
 795  *
 796  * Returns:     -EINVAL if out of valid range (0..now).
 797  *              0 on success.
 798  */
 799 int kvm_mips_set_count_resume(struct kvm_vcpu *vcpu, s64 count_resume)
 800 {
 801         /*
 802          * It doesn't make sense for the resume time to be in the future, as it
 803          * would be possible for the next interrupt to be more than a full
 804          * period in the future.
 805          */
 806         if (count_resume < 0 || count_resume > ktime_to_ns(ktime_get()))
 807                 return -EINVAL;
 808
 809         vcpu->arch.count_resume = ns_to_ktime(count_resume);
 810         return 0;
 811 }
 812
 813 /**
 814  * kvm_mips_count_timeout() - Push timer forward on timeout.
 815  * @vcpu:       Virtual CPU.
 816  *
 817  * Handle an hrtimer event by push the hrtimer forward a period.
 818  *
 819  * Returns:     The hrtimer_restart value to return to the hrtimer subsystem.
 820  */
 821 enum hrtimer_restart kvm_mips_count_timeout(struct kvm_vcpu *vcpu)
 822 {
 823         /* Add the Count period to the current expiry time */
 824         hrtimer_add_expires_ns(&vcpu->arch.comparecount_timer,
 825                                vcpu->arch.count_period);
 826         return HRTIMER_RESTART;
 827 }
 828
 829 enum emulation_result kvm_mips_emul_eret(struct kvm_vcpu *vcpu)
 830 {
 831         struct mips_coproc *cop0 = vcpu->arch.cop0;
 832         enum emulation_result er = EMULATE_DONE;
 833
 834         if (kvm_read_c0_guest_status(cop0) & ST0_ERL) {
 835                 kvm_clear_c0_guest_status(cop0, ST0_ERL);
 836                 vcpu->arch.pc = kvm_read_c0_guest_errorepc(cop0);
 837         } else if (kvm_read_c0_guest_status(cop0) & ST0_EXL) {
 838                 kvm_debug("[%#lx] ERET to %#lx\n", vcpu->arch.pc,
 839                           kvm_read_c0_guest_epc(cop0));
 840                 kvm_clear_c0_guest_status(cop0, ST0_EXL);
 841                 vcpu->arch.pc = kvm_read_c0_guest_epc(cop0);
 842
 843         } else {
 844                 kvm_err("[%#lx] ERET when MIPS_SR_EXL|MIPS_SR_ERL == 0\n",
 845                         vcpu->arch.pc);
 846                 er = EMULATE_FAIL;
 847         }
 848
 849         return er;
 850 }
 851
 852 enum emulation_result kvm_mips_emul_wait(struct kvm_vcpu *vcpu)
 853 {
 854         kvm_debug("[%#lx] !!!WAIT!!! (%#lx)\n", vcpu->arch.pc,
 855                   vcpu->arch.pending_exceptions);
 856
 857         ++vcpu->stat.wait_exits;
 858         trace_kvm_exit(vcpu, KVM_TRACE_EXIT_WAIT);
 859         if (!vcpu->arch.pending_exceptions) {
 860                 vcpu->arch.wait = 1;
 861                 kvm_vcpu_block(vcpu);
 862
 863                 /*
 864                  * We we are runnable, then definitely go off to user space to
 865                  * check if any I/O interrupts are pending.
 866                  */
 867                 if (kvm_check_request(KVM_REQ_UNHALT, vcpu)) {
 868                         clear_bit(KVM_REQ_UNHALT, &vcpu->requests);
 869                         vcpu->run->exit_reason = KVM_EXIT_IRQ_WINDOW_OPEN;
 870                 }
 871         }
 872
 873         return EMULATE_DONE;
 874 }
 875
 876 /*
 877  * XXXKYMA: Linux doesn't seem to use TLBR, return EMULATE_FAIL for now so that
 878  * we can catch this, if things ever change
 879  */
 880 enum emulation_result kvm_mips_emul_tlbr(struct kvm_vcpu *vcpu)
 881 {
 882         struct mips_coproc *cop0 = vcpu->arch.cop0;
 883         unsigned long pc = vcpu->arch.pc;
 884
 885         kvm_err("[%#lx] COP0_TLBR [%ld]\n", pc, kvm_read_c0_guest_index(cop0));
 886         return EMULATE_FAIL;
 887 }
 888
 889 /**
 890  * kvm_mips_invalidate_guest_tlb() - Indicates a change in guest MMU map.
 891  * @vcpu:       VCPU with changed mappings.
 892  * @tlb:        TLB entry being removed.
 893  *
 894  * This is called to indicate a single change in guest MMU mappings, so that we
 895  * can arrange TLB flushes on this and other CPUs.
 896  */
 897 static void kvm_mips_invalidate_guest_tlb(struct kvm_vcpu *vcpu,
 898                                           struct kvm_mips_tlb *tlb)
 899 {
 900         struct mm_struct *kern_mm = &vcpu->arch.guest_kernel_mm;
 901         struct mm_struct *user_mm = &vcpu->arch.guest_user_mm;
 902         int cpu, i;
 903         bool user;
 904
 905         /* No need to flush for entries which are already invalid */
 906         if (!((tlb->tlb_lo[0] | tlb->tlb_lo[1]) & ENTRYLO_V))
 907                 return;
 908         /* Don't touch host kernel page tables or TLB mappings */
 909         if ((unsigned long)tlb->tlb_hi > 0x7fffffff)
 910                 return;
 911         /* User address space doesn't need flushing for KSeg2/3 changes */
 912         user = tlb->tlb_hi < KVM_GUEST_KSEG0;
 913
 914         preempt_disable();
 915
 916         /* Invalidate page table entries */
 917         kvm_trap_emul_invalidate_gva(vcpu, tlb->tlb_hi & VPN2_MASK, user);
 918
 919         /*
 920          * Probe the shadow host TLB for the entry being overwritten, if one
 921          * matches, invalidate it
 922          */
 923         kvm_mips_host_tlb_inv(vcpu, tlb->tlb_hi, user, true);
 924
 925         /* Invalidate the whole ASID on other CPUs */
 926         cpu = smp_processor_id();
 927         for_each_possible_cpu(i) {
 928                 if (i == cpu)
 929                         continue;
 930                 if (user)
 931                         cpu_context(i, user_mm) = 0;
 932                 cpu_context(i, kern_mm) = 0;
 933         }
 934
 935         preempt_enable();
 936 }
 937
 938 /* Write Guest TLB Entry @ Index */
 939 enum emulation_result kvm_mips_emul_tlbwi(struct kvm_vcpu *vcpu)
 940 {
 941         struct mips_coproc *cop0 = vcpu->arch.cop0;
 942         int index = kvm_read_c0_guest_index(cop0);
 943         struct kvm_mips_tlb *tlb = NULL;
 944         unsigned long pc = vcpu->arch.pc;
 945
 946         if (index < 0 || index >= KVM_MIPS_GUEST_TLB_SIZE) {
 947                 kvm_debug("%s: illegal index: %d\n", __func__, index);
 948                 kvm_debug("[%#lx] COP0_TLBWI [%d] (entryhi: %#lx, entrylo0: %#lx entrylo1: %#lx, mask: %#lx)\n",
 949                           pc, index, kvm_read_c0_guest_entryhi(cop0),
 950                           kvm_read_c0_guest_entrylo0(cop0),
 951                           kvm_read_c0_guest_entrylo1(cop0),
 952                           kvm_read_c0_guest_pagemask(cop0));
 953                 index = (index & ~0x80000000) % KVM_MIPS_GUEST_TLB_SIZE;
 954         }
 955
 956         tlb = &vcpu->arch.guest_tlb[index];
 957
 958         kvm_mips_invalidate_guest_tlb(vcpu, tlb);
 959
 960         tlb->tlb_mask = kvm_read_c0_guest_pagemask(cop0);
 961         tlb->tlb_hi = kvm_read_c0_guest_entryhi(cop0);
 962         tlb->tlb_lo[0] = kvm_read_c0_guest_entrylo0(cop0);
 963         tlb->tlb_lo[1] = kvm_read_c0_guest_entrylo1(cop0);
 964
 965         kvm_debug("[%#lx] COP0_TLBWI [%d] (entryhi: %#lx, entrylo0: %#lx entrylo1: %#lx, mask: %#lx)\n",
 966                   pc, index, kvm_read_c0_guest_entryhi(cop0),
 967                   kvm_read_c0_guest_entrylo0(cop0),
 968                   kvm_read_c0_guest_entrylo1(cop0),
 969                   kvm_read_c0_guest_pagemask(cop0));
 970
 971         return EMULATE_DONE;
 972 }
 973
 974 /* Write Guest TLB Entry @ Random Index */
 975 enum emulation_result kvm_mips_emul_tlbwr(struct kvm_vcpu *vcpu)
 976 {
 977         struct mips_coproc *cop0 = vcpu->arch.cop0;
 978         struct kvm_mips_tlb *tlb = NULL;
 979         unsigned long pc = vcpu->arch.pc;
 980         int index;
 981
 982         get_random_bytes(&index, sizeof(index));
 983         index &= (KVM_MIPS_GUEST_TLB_SIZE - 1);
 984
 985         tlb = &vcpu->arch.guest_tlb[index];
 986
 987         kvm_mips_invalidate_guest_tlb(vcpu, tlb);
 988
 989         tlb->tlb_mask = kvm_read_c0_guest_pagemask(cop0);
 990         tlb->tlb_hi = kvm_read_c0_guest_entryhi(cop0);
 991         tlb->tlb_lo[0] = kvm_read_c0_guest_entrylo0(cop0);
 992         tlb->tlb_lo[1] = kvm_read_c0_guest_entrylo1(cop0);
 993
 994         kvm_debug("[%#lx] COP0_TLBWR[%d] (entryhi: %#lx, entrylo0: %#lx entrylo1: %#lx)\n",
 995                   pc, index, kvm_read_c0_guest_entryhi(cop0),
 996                   kvm_read_c0_guest_entrylo0(cop0),
 997                   kvm_read_c0_guest_entrylo1(cop0));
 998
 999         return EMULATE_DONE;
1000 }
1001
1002 enum emulation_result kvm_mips_emul_tlbp(struct kvm_vcpu *vcpu)
1003 {
1004         struct mips_coproc *cop0 = vcpu->arch.cop0;
1005         long entryhi = kvm_read_c0_guest_entryhi(cop0);
1006         unsigned long pc = vcpu->arch.pc;
1007         int index = -1;
1008
1009         index = kvm_mips_guest_tlb_lookup(vcpu, entryhi);
1010
1011         kvm_write_c0_guest_index(cop0, index);
1012
1013         kvm_debug("[%#lx] COP0_TLBP (entryhi: %#lx), index: %d\n", pc, entryhi,
1014                   index);
1015
1016         return EMULATE_DONE;
1017 }
1018
1019 /**
1020  * kvm_mips_config1_wrmask() - Find mask of writable bits in guest Config1
1021  * @vcpu:       Virtual CPU.
1022  *
1023  * Finds the mask of bits which are writable in the guest's Config1 CP0
1024  * register, by userland (currently read-only to the guest).
1025  */
1026 unsigned int kvm_mips_config1_wrmask(struct kvm_vcpu *vcpu)
1027 {
1028         unsigned int mask = 0;
1029
1030         /* Permit FPU to be present if FPU is supported */
1031         if (kvm_mips_guest_can_have_fpu(&vcpu->arch))
1032                 mask |= MIPS_CONF1_FP;
1033
1034         return mask;
1035 }
1036
1037 /**
1038  * kvm_mips_config3_wrmask() - Find mask of writable bits in guest Config3
1039  * @vcpu:       Virtual CPU.
1040  *
1041  * Finds the mask of bits which are writable in the guest's Config3 CP0
1042  * register, by userland (currently read-only to the guest).
1043  */
1044 unsigned int kvm_mips_config3_wrmask(struct kvm_vcpu *vcpu)
1045 {
1046         /* Config4 and ULRI are optional */
1047         unsigned int mask = MIPS_CONF_M | MIPS_CONF3_ULRI;
1048
1049         /* Permit MSA to be present if MSA is supported */
1050         if (kvm_mips_guest_can_have_msa(&vcpu->arch))
1051                 mask |= MIPS_CONF3_MSA;
1052
1053         return mask;
1054 }
1055
1056 /**
1057  * kvm_mips_config4_wrmask() - Find mask of writable bits in guest Config4
1058  * @vcpu:       Virtual CPU.
1059  *
1060  * Finds the mask of bits which are writable in the guest's Config4 CP0
1061  * register, by userland (currently read-only to the guest).
1062  */
1063 unsigned int kvm_mips_config4_wrmask(struct kvm_vcpu *vcpu)
1064 {
1065         /* Config5 is optional */
1066         unsigned int mask = MIPS_CONF_M;
1067
1068         /* KScrExist */
1069         mask |= 0xfc << MIPS_CONF4_KSCREXIST_SHIFT;
1070
1071         return mask;
1072 }
1073
1074 /**
1075  * kvm_mips_config5_wrmask() - Find mask of writable bits in guest Config5
1076  * @vcpu:       Virtual CPU.
1077  *
1078  * Finds the mask of bits which are writable in the guest's Config5 CP0
1079  * register, by the guest itself.
1080  */
1081 unsigned int kvm_mips_config5_wrmask(struct kvm_vcpu *vcpu)
1082 {
1083         unsigned int mask = 0;
1084
1085         /* Permit MSAEn changes if MSA supported and enabled */
1086         if (kvm_mips_guest_has_msa(&vcpu->arch))
1087                 mask |= MIPS_CONF5_MSAEN;
1088
1089         /*
1090          * Permit guest FPU mode changes if FPU is enabled and the relevant
1091          * feature exists according to FIR register.
1092          */
1093         if (kvm_mips_guest_has_fpu(&vcpu->arch)) {
1094                 if (cpu_has_fre)
1095                         mask |= MIPS_CONF5_FRE;
1096                 /* We don't support UFR or UFE */
1097         }
1098
1099         return mask;
1100 }
1101
1102 enum emulation_result kvm_mips_emulate_CP0(union mips_instruction inst,
1103                                            u32 *opc, u32 cause,
1104                                            struct kvm_run *run,
1105                                            struct kvm_vcpu *vcpu)
1106 {
1107         struct mips_coproc *cop0 = vcpu->arch.cop0;
1108         struct mm_struct *kern_mm = &vcpu->arch.guest_kernel_mm;
1109         enum emulation_result er = EMULATE_DONE;
1110         u32 rt, rd, sel;
1111         unsigned long curr_pc;
1112         int cpu, i;
1113
1114         /*
1115          * Update PC and hold onto current PC in case there is
1116          * an error and we want to rollback the PC
1117          */
1118         curr_pc = vcpu->arch.pc;
1119         er = update_pc(vcpu, cause);
1120         if (er == EMULATE_FAIL)
1121                 return er;
1122
1123         if (inst.co_format.co) {
1124                 switch (inst.co_format.func) {
1125                 case tlbr_op:   /*  Read indexed TLB entry  */
1126                         er = kvm_mips_emul_tlbr(vcpu);
1127                         break;
1128                 case tlbwi_op:  /*  Write indexed  */
1129                         er = kvm_mips_emul_tlbwi(vcpu);
1130                         break;
1131                 case tlbwr_op:  /*  Write random  */
1132                         er = kvm_mips_emul_tlbwr(vcpu);
1133                         break;
1134                 case tlbp_op:   /* TLB Probe */
1135                         er = kvm_mips_emul_tlbp(vcpu);
1136                         break;
1137                 case rfe_op:
1138                         kvm_err("!!!COP0_RFE!!!\n");
1139                         break;
1140                 case eret_op:
1141                         er = kvm_mips_emul_eret(vcpu);
1142                         goto dont_update_pc;
1143                 case wait_op:
1144                         er = kvm_mips_emul_wait(vcpu);
1145                         break;
1146                 case hypcall_op:
1147                         er = kvm_mips_emul_hypcall(vcpu, inst);
1148                         break;
1149                 }
1150         } else {
1151                 rt = inst.c0r_format.rt;
1152                 rd = inst.c0r_format.rd;
1153                 sel = inst.c0r_format.sel;
1154
1155                 switch (inst.c0r_format.rs) {
1156                 case mfc_op:
1157 #ifdef CONFIG_KVM_MIPS_DEBUG_COP0_COUNTERS
1158                         cop0->stat[rd][sel]++;
1159 #endif
1160                         /* Get reg */
1161                         if ((rd == MIPS_CP0_COUNT) && (sel == 0)) {
1162                                 vcpu->arch.gprs[rt] =
1163                                     (s32)kvm_mips_read_count(vcpu);
1164                         } else if ((rd == MIPS_CP0_ERRCTL) && (sel == 0)) {
1165                                 vcpu->arch.gprs[rt] = 0x0;
1166 #ifdef CONFIG_KVM_MIPS_DYN_TRANS
1167                                 kvm_mips_trans_mfc0(inst, opc, vcpu);
1168 #endif
1169                         } else {
1170                                 vcpu->arch.gprs[rt] = (s32)cop0->reg[rd][sel];
1171
1172 #ifdef CONFIG_KVM_MIPS_DYN_TRANS
1173                                 kvm_mips_trans_mfc0(inst, opc, vcpu);
1174 #endif
1175                         }
1176
1177                         trace_kvm_hwr(vcpu, KVM_TRACE_MFC0,
1178                                       KVM_TRACE_COP0(rd, sel),
1179                                       vcpu->arch.gprs[rt]);
1180                         break;
1181
1182                 case dmfc_op:
1183                         vcpu->arch.gprs[rt] = cop0->reg[rd][sel];
1184
1185                         trace_kvm_hwr(vcpu, KVM_TRACE_DMFC0,
1186                                       KVM_TRACE_COP0(rd, sel),
1187                                       vcpu->arch.gprs[rt]);
1188                         break;
1189
1190                 case mtc_op:
1191 #ifdef CONFIG_KVM_MIPS_DEBUG_COP0_COUNTERS
1192                         cop0->stat[rd][sel]++;
1193 #endif
1194                         trace_kvm_hwr(vcpu, KVM_TRACE_MTC0,
1195                                       KVM_TRACE_COP0(rd, sel),
1196                                       vcpu->arch.gprs[rt]);
1197
1198                         if ((rd == MIPS_CP0_TLB_INDEX)
1199                             && (vcpu->arch.gprs[rt] >=
1200                                 KVM_MIPS_GUEST_TLB_SIZE)) {
1201                                 kvm_err("Invalid TLB Index: %ld",
1202                                         vcpu->arch.gprs[rt]);
1203                                 er = EMULATE_FAIL;
1204                                 break;
1205                         }
1206                         if ((rd == MIPS_CP0_PRID) && (sel == 1)) {
1207                                 /*
1208                                  * Preserve core number, and keep the exception
1209                                  * base in guest KSeg0.
1210                                  */
1211                                 kvm_change_c0_guest_ebase(cop0, 0x1ffff000,
1212                                                           vcpu->arch.gprs[rt]);
1213                         } else if (rd == MIPS_CP0_TLB_HI && sel == 0) {
1214                                 u32 nasid =
1215                                         vcpu->arch.gprs[rt] & KVM_ENTRYHI_ASID;
1216                                 if (((kvm_read_c0_guest_entryhi(cop0) &
1217                                       KVM_ENTRYHI_ASID) != nasid)) {
1218                                         trace_kvm_asid_change(vcpu,
1219                                                 kvm_read_c0_guest_entryhi(cop0)
1220                                                         & KVM_ENTRYHI_ASID,
1221                                                 nasid);
1222
1223                                         /*
1224                                          * Flush entries from the GVA page
1225                                          * tables.
1226                                          * Guest user page table will get
1227                                          * flushed lazily on re-entry to guest
1228                                          * user if the guest ASID actually
1229                                          * changes.
1230                                          */
1231                                         kvm_mips_flush_gva_pt(kern_mm->pgd,
1232                                                               KMF_KERN);
1233
1234                                         /*
1235                                          * Regenerate/invalidate kernel MMU
1236                                          * context.
1237                                          * The user MMU context will be
1238                                          * regenerated lazily on re-entry to
1239                                          * guest user if the guest ASID actually
1240                                          * changes.
1241                                          */
1242                                         preempt_disable();
1243                                         cpu = smp_processor_id();
1244                                         get_new_mmu_context(kern_mm, cpu);
1245                                         for_each_possible_cpu(i)
1246                                                 if (i != cpu)
1247                                                         cpu_context(i, kern_mm) = 0;
1248                                         preempt_enable();
1249                                 }
1250                                 kvm_write_c0_guest_entryhi(cop0,
1251                                                            vcpu->arch.gprs[rt]);
1252                         }
1253                         /* Are we writing to COUNT */
1254                         else if ((rd == MIPS_CP0_COUNT) && (sel == 0)) {
1255                                 kvm_mips_write_count(vcpu, vcpu->arch.gprs[rt]);
1256                                 goto done;
1257                         } else if ((rd == MIPS_CP0_COMPARE) && (sel == 0)) {
1258                                 /* If we are writing to COMPARE */
1259                                 /* Clear pending timer interrupt, if any */
1260                                 kvm_mips_write_compare(vcpu,
1261                                                        vcpu->arch.gprs[rt],
1262                                                        true);
1263                         } else if ((rd == MIPS_CP0_STATUS) && (sel == 0)) {
1264                                 unsigned int old_val, val, change;
1265
1266                                 old_val = kvm_read_c0_guest_status(cop0);
1267                                 val = vcpu->arch.gprs[rt];
1268                                 change = val ^ old_val;
1269
1270                                 /* Make sure that the NMI bit is never set */
1271                                 val &= ~ST0_NMI;
1272
1273                                 /*
1274                                  * Don't allow CU1 or FR to be set unless FPU
1275                                  * capability enabled and exists in guest
1276                                  * configuration.
1277                                  */
1278                                 if (!kvm_mips_guest_has_fpu(&vcpu->arch))
1279                                         val &= ~(ST0_CU1 | ST0_FR);
1280
1281                                 /*
1282                                  * Also don't allow FR to be set if host doesn't
1283                                  * support it.
1284                                  */
1285                                 if (!(current_cpu_data.fpu_id & MIPS_FPIR_F64))
1286                                         val &= ~ST0_FR;
1287
1288
1289                                 /* Handle changes in FPU mode */
1290                                 preempt_disable();
1291
1292                                 /*
1293                                  * FPU and Vector register state is made
1294                                  * UNPREDICTABLE by a change of FR, so don't
1295                                  * even bother saving it.
1296                                  */
1297                                 if (change & ST0_FR)
1298                                         kvm_drop_fpu(vcpu);
1299
1300                                 /*
1301                                  * If MSA state is already live, it is undefined
1302                                  * how it interacts with FR=0 FPU state, and we
1303                                  * don't want to hit reserved instruction
1304                                  * exceptions trying to save the MSA state later
1305                                  * when CU=1 && FR=1, so play it safe and save
1306                                  * it first.
1307                                  */
1308                                 if (change & ST0_CU1 && !(val & ST0_FR) &&
1309                                     vcpu->arch.aux_inuse & KVM_MIPS_AUX_MSA)
1310                                         kvm_lose_fpu(vcpu);
1311
1312                                 /*
1313                                  * Propagate CU1 (FPU enable) changes
1314                                  * immediately if the FPU context is already
1315                                  * loaded. When disabling we leave the context
1316                                  * loaded so it can be quickly enabled again in
1317                                  * the near future.
1318                                  */
1319                                 if (change & ST0_CU1 &&
1320                                     vcpu->arch.aux_inuse & KVM_MIPS_AUX_FPU)
1321                                         change_c0_status(ST0_CU1, val);
1322
1323                                 preempt_enable();
1324
1325                                 kvm_write_c0_guest_status(cop0, val);
1326
1327 #ifdef CONFIG_KVM_MIPS_DYN_TRANS
1328                                 /*
1329                                  * If FPU present, we need CU1/FR bits to take
1330                                  * effect fairly soon.
1331                                  */
1332                                 if (!kvm_mips_guest_has_fpu(&vcpu->arch))
1333                                         kvm_mips_trans_mtc0(inst, opc, vcpu);
1334 #endif
1335                         } else if ((rd == MIPS_CP0_CONFIG) && (sel == 5)) {
1336                                 unsigned int old_val, val, change, wrmask;
1337
1338                                 old_val = kvm_read_c0_guest_config5(cop0);
1339                                 val = vcpu->arch.gprs[rt];
1340
1341                                 /* Only a few bits are writable in Config5 */
1342                                 wrmask = kvm_mips_config5_wrmask(vcpu);
1343                                 change = (val ^ old_val) & wrmask;
1344                                 val = old_val ^ change;
1345
1346
1347                                 /* Handle changes in FPU/MSA modes */
1348                                 preempt_disable();
1349
1350                                 /*
1351                                  * Propagate FRE changes immediately if the FPU
1352                                  * context is already loaded.
1353                                  */
1354                                 if (change & MIPS_CONF5_FRE &&
1355                                     vcpu->arch.aux_inuse & KVM_MIPS_AUX_FPU)
1356                                         change_c0_config5(MIPS_CONF5_FRE, val);
1357
1358                                 /*
1359                                  * Propagate MSAEn changes immediately if the
1360                                  * MSA context is already loaded. When disabling
1361                                  * we leave the context loaded so it can be
1362                                  * quickly enabled again in the near future.
1363                                  */
1364                                 if (change & MIPS_CONF5_MSAEN &&
1365                                     vcpu->arch.aux_inuse & KVM_MIPS_AUX_MSA)
1366                                         change_c0_config5(MIPS_CONF5_MSAEN,
1367                                                           val);
1368
1369                                 preempt_enable();
1370
1371                                 kvm_write_c0_guest_config5(cop0, val);
1372                         } else if ((rd == MIPS_CP0_CAUSE) && (sel == 0)) {
1373                                 u32 old_cause, new_cause;
1374
1375                                 old_cause = kvm_read_c0_guest_cause(cop0);
1376                                 new_cause = vcpu->arch.gprs[rt];
1377                                 /* Update R/W bits */
1378                                 kvm_change_c0_guest_cause(cop0, 0x08800300,
1379                                                           new_cause);
1380                                 /* DC bit enabling/disabling timer? */
1381                                 if ((old_cause ^ new_cause) & CAUSEF_DC) {
1382                                         if (new_cause & CAUSEF_DC)
1383                                                 kvm_mips_count_disable_cause(vcpu);
1384                                         else
1385                                                 kvm_mips_count_enable_cause(vcpu);
1386                                 }
1387                         } else if ((rd == MIPS_CP0_HWRENA) && (sel == 0)) {
1388                                 u32 mask = MIPS_HWRENA_CPUNUM |
1389                                            MIPS_HWRENA_SYNCISTEP |
1390                                            MIPS_HWRENA_CC |
1391                                            MIPS_HWRENA_CCRES;
1392
1393                                 if (kvm_read_c0_guest_config3(cop0) &
1394                                     MIPS_CONF3_ULRI)
1395                                         mask |= MIPS_HWRENA_ULR;
1396                                 cop0->reg[rd][sel] = vcpu->arch.gprs[rt] & mask;
1397                         } else {
1398                                 cop0->reg[rd][sel] = vcpu->arch.gprs[rt];
1399 #ifdef CONFIG_KVM_MIPS_DYN_TRANS
1400                                 kvm_mips_trans_mtc0(inst, opc, vcpu);
1401 #endif
1402                         }
1403                         break;
1404
1405                 case dmtc_op:
1406                         kvm_err("!!!!!!![%#lx]dmtc_op: rt: %d, rd: %d, sel: %d!!!!!!\n",
1407                                 vcpu->arch.pc, rt, rd, sel);
1408                         trace_kvm_hwr(vcpu, KVM_TRACE_DMTC0,
1409                                       KVM_TRACE_COP0(rd, sel),
1410                                       vcpu->arch.gprs[rt]);
1411                         er = EMULATE_FAIL;
1412                         break;
1413
1414                 case mfmc0_op:
1415 #ifdef KVM_MIPS_DEBUG_COP0_COUNTERS
1416                         cop0->stat[MIPS_CP0_STATUS][0]++;
1417 #endif
1418                         if (rt != 0)
1419                                 vcpu->arch.gprs[rt] =
1420                                     kvm_read_c0_guest_status(cop0);
1421                         /* EI */
1422                         if (inst.mfmc0_format.sc) {
1423                                 kvm_debug("[%#lx] mfmc0_op: EI\n",
1424                                           vcpu->arch.pc);
1425                                 kvm_set_c0_guest_status(cop0, ST0_IE);
1426                         } else {
1427                                 kvm_debug("[%#lx] mfmc0_op: DI\n",
1428                                           vcpu->arch.pc);
1429                                 kvm_clear_c0_guest_status(cop0, ST0_IE);
1430                         }
1431
1432                         break;
1433
1434                 case wrpgpr_op:
1435                         {
1436                                 u32 css = cop0->reg[MIPS_CP0_STATUS][2] & 0xf;
1437                                 u32 pss =
1438                                     (cop0->reg[MIPS_CP0_STATUS][2] >> 6) & 0xf;
1439                                 /*
1440                                  * We don't support any shadow register sets, so
1441                                  * SRSCtl[PSS] == SRSCtl[CSS] = 0
1442                                  */
1443                                 if (css || pss) {
1444                                         er = EMULATE_FAIL;
1445                                         break;
1446                                 }
1447                                 kvm_debug("WRPGPR[%d][%d] = %#lx\n", pss, rd,
1448                                           vcpu->arch.gprs[rt]);
1449                                 vcpu->arch.gprs[rd] = vcpu->arch.gprs[rt];
1450                         }
1451                         break;
1452                 default:
1453                         kvm_err("[%#lx]MachEmulateCP0: unsupported COP0, copz: 0x%x\n",
1454                                 vcpu->arch.pc, inst.c0r_format.rs);
1455                         er = EMULATE_FAIL;
1456                         break;
1457                 }
1458         }
1459
1460 done:
1461         /* Rollback PC only if emulation was unsuccessful */
1462         if (er == EMULATE_FAIL)
1463                 vcpu->arch.pc = curr_pc;
1464
1465 dont_update_pc:
1466         /*
1467          * This is for special instructions whose emulation
1468          * updates the PC, so do not overwrite the PC under
1469          * any circumstances
1470          */
1471
1472         return er;
1473 }
1474
1475 enum emulation_result kvm_mips_emulate_store(union mips_instruction inst,
1476                                              u32 cause,
1477                                              struct kvm_run *run,
1478                                              struct kvm_vcpu *vcpu)
1479 {
1480         enum emulation_result er;
1481         u32 rt;
1482         void *data = run->mmio.data;
1483         unsigned long curr_pc;
1484
1485         /*
1486          * Update PC and hold onto current PC in case there is
1487          * an error and we want to rollback the PC
1488          */
1489         curr_pc = vcpu->arch.pc;
1490         er = update_pc(vcpu, cause);
1491         if (er == EMULATE_FAIL)
1492                 return er;
1493
1494         rt = inst.i_format.rt;
1495
1496         run->mmio.phys_addr = kvm_mips_callbacks->gva_to_gpa(
1497                                                 vcpu->arch.host_cp0_badvaddr);
1498         if (run->mmio.phys_addr == KVM_INVALID_ADDR)
1499                 goto out_fail;
1500
1501         switch (inst.i_format.opcode) {
1502         case sw_op:
1503                 run->mmio.len = 4;
1504                 *(u32 *)data = vcpu->arch.gprs[rt];
1505
1506                 kvm_debug("[%#lx] OP_SW: eaddr: %#lx, gpr: %#lx, data: %#x\n",
1507                           vcpu->arch.pc, vcpu->arch.host_cp0_badvaddr,
1508                           vcpu->arch.gprs[rt], *(u32 *)data);
1509                 break;
1510
1511         case sh_op:
1512                 run->mmio.len = 2;
1513                 *(u16 *)data = vcpu->arch.gprs[rt];
1514
1515                 kvm_debug("[%#lx] OP_SH: eaddr: %#lx, gpr: %#lx, data: %#x\n",
1516                           vcpu->arch.pc, vcpu->arch.host_cp0_badvaddr,
1517                           vcpu->arch.gprs[rt], *(u16 *)data);
1518                 break;
1519
1520         case sb_op:
1521                 run->mmio.len = 1;
1522                 *(u8 *)data = vcpu->arch.gprs[rt];
1523
1524                 kvm_debug("[%#lx] OP_SB: eaddr: %#lx, gpr: %#lx, data: %#x\n",
1525                           vcpu->arch.pc, vcpu->arch.host_cp0_badvaddr,
1526                           vcpu->arch.gprs[rt], *(u8 *)data);
1527                 break;
1528
1529         default:
1530                 kvm_err("Store not yet supported (inst=0x%08x)\n",
1531                         inst.word);
1532                 goto out_fail;
1533         }
1534
1535         run->mmio.is_write = 1;
1536         vcpu->mmio_needed = 1;
1537         vcpu->mmio_is_write = 1;
1538         return EMULATE_DO_MMIO;
1539
1540 out_fail:
1541         /* Rollback PC if emulation was unsuccessful */
1542         vcpu->arch.pc = curr_pc;
1543         return EMULATE_FAIL;
1544 }
1545
1546 enum emulation_result kvm_mips_emulate_load(union mips_instruction inst,
1547                                             u32 cause, struct kvm_run *run,
1548                                             struct kvm_vcpu *vcpu)
1549 {
1550         enum emulation_result er;
1551         unsigned long curr_pc;
1552         u32 op, rt;
1553
1554         rt = inst.i_format.rt;
1555         op = inst.i_format.opcode;
1556
1557         /*
1558          * Find the resume PC now while we have safe and easy access to the
1559          * prior branch instruction, and save it for
1560          * kvm_mips_complete_mmio_load() to restore later.
1561          */
1562         curr_pc = vcpu->arch.pc;
1563         er = update_pc(vcpu, cause);
1564         if (er == EMULATE_FAIL)
1565                 return er;
1566         vcpu->arch.io_pc = vcpu->arch.pc;
1567         vcpu->arch.pc = curr_pc;
1568
1569         vcpu->arch.io_gpr = rt;
1570
1571         run->mmio.phys_addr = kvm_mips_callbacks->gva_to_gpa(
1572                                                 vcpu->arch.host_cp0_badvaddr);
1573         if (run->mmio.phys_addr == KVM_INVALID_ADDR)
1574                 return EMULATE_FAIL;
1575
1576         vcpu->mmio_needed = 2;  /* signed */
1577         switch (op) {
1578         case lw_op:
1579                 run->mmio.len = 4;
1580                 break;
1581
1582         case lhu_op:
1583                 vcpu->mmio_needed = 1;  /* unsigned */
1584                 /* fall through */
1585         case lh_op:
1586                 run->mmio.len = 2;
1587                 break;
1588
1589         case lbu_op:
1590                 vcpu->mmio_needed = 1;  /* unsigned */
1591                 /* fall through */
1592         case lb_op:
1593                 run->mmio.len = 1;
1594                 break;
1595
1596         default:
1597                 kvm_err("Load not yet supported (inst=0x%08x)\n",
1598                         inst.word);
1599                 vcpu->mmio_needed = 0;
1600                 return EMULATE_FAIL;
1601         }
1602
1603         run->mmio.is_write = 0;
1604         vcpu->mmio_is_write = 0;
1605         return EMULATE_DO_MMIO;
1606 }
1607
1608 static enum emulation_result kvm_mips_guest_cache_op(int (*fn)(unsigned long),
1609                                                      unsigned long curr_pc,
1610                                                      unsigned long addr,
1611                                                      struct kvm_run *run,
1612                                                      struct kvm_vcpu *vcpu,
1613                                                      u32 cause)
1614 {
1615         int err;
1616
1617         for (;;) {
1618                 /* Carefully attempt the cache operation */
1619                 kvm_trap_emul_gva_lockless_begin(vcpu);
1620                 err = fn(addr);
1621                 kvm_trap_emul_gva_lockless_end(vcpu);
1622
1623                 if (likely(!err))
1624                         return EMULATE_DONE;
1625
1626                 /*
1627                  * Try to handle the fault and retry, maybe we just raced with a
1628                  * GVA invalidation.
1629                  */
1630                 switch (kvm_trap_emul_gva_fault(vcpu, addr, false)) {
1631                 case KVM_MIPS_GVA:
1632                 case KVM_MIPS_GPA:
1633                         /* bad virtual or physical address */
1634                         return EMULATE_FAIL;
1635                 case KVM_MIPS_TLB:
1636                         /* no matching guest TLB */
1637                         vcpu->arch.host_cp0_badvaddr = addr;
1638                         vcpu->arch.pc = curr_pc;
1639                         kvm_mips_emulate_tlbmiss_ld(cause, NULL, run, vcpu);
1640                         return EMULATE_EXCEPT;
1641                 case KVM_MIPS_TLBINV:
1642                         /* invalid matching guest TLB */
1643                         vcpu->arch.host_cp0_badvaddr = addr;
1644                         vcpu->arch.pc = curr_pc;
1645                         kvm_mips_emulate_tlbinv_ld(cause, NULL, run, vcpu);
1646                         return EMULATE_EXCEPT;
1647                 default:
1648                         break;
1649                 };
1650         }
1651 }
1652
1653 enum emulation_result kvm_mips_emulate_cache(union mips_instruction inst,
1654                                              u32 *opc, u32 cause,
1655                                              struct kvm_run *run,
1656                                              struct kvm_vcpu *vcpu)
1657 {
1658         enum emulation_result er = EMULATE_DONE;
1659         u32 cache, op_inst, op, base;
1660         s16 offset;
1661         struct kvm_vcpu_arch *arch = &vcpu->arch;
1662         unsigned long va;
1663         unsigned long curr_pc;
1664
1665         /*
1666          * Update PC and hold onto current PC in case there is
1667          * an error and we want to rollback the PC
1668          */
1669         curr_pc = vcpu->arch.pc;
1670         er = update_pc(vcpu, cause);
1671         if (er == EMULATE_FAIL)
1672                 return er;
1673
1674         base = inst.i_format.rs;
1675         op_inst = inst.i_format.rt;
1676         if (cpu_has_mips_r6)
1677                 offset = inst.spec3_format.simmediate;
1678         else
1679                 offset = inst.i_format.simmediate;
1680         cache = op_inst & CacheOp_Cache;
1681         op = op_inst & CacheOp_Op;
1682
1683         va = arch->gprs[base] + offset;
1684
1685         kvm_debug("CACHE (cache: %#x, op: %#x, base[%d]: %#lx, offset: %#x\n",
1686                   cache, op, base, arch->gprs[base], offset);
1687
1688         /*
1689          * Treat INDEX_INV as a nop, basically issued by Linux on startup to
1690          * invalidate the caches entirely by stepping through all the
1691          * ways/indexes
1692          */
1693         if (op == Index_Writeback_Inv) {
1694                 kvm_debug("@ %#lx/%#lx CACHE (cache: %#x, op: %#x, base[%d]: %#lx, offset: %#x\n",
1695                           vcpu->arch.pc, vcpu->arch.gprs[31], cache, op, base,
1696                           arch->gprs[base], offset);
1697
1698                 if (cache == Cache_D)
1699                         r4k_blast_dcache();
1700                 else if (cache == Cache_I)
1701                         r4k_blast_icache();
1702                 else {
1703                         kvm_err("%s: unsupported CACHE INDEX operation\n",
1704                                 __func__);
1705                         return EMULATE_FAIL;
1706                 }
1707
1708 #ifdef CONFIG_KVM_MIPS_DYN_TRANS
1709                 kvm_mips_trans_cache_index(inst, opc, vcpu);
1710 #endif
1711                 goto done;
1712         }
1713
1714         /* XXXKYMA: Only a subset of cache ops are supported, used by Linux */
1715         if (op_inst == Hit_Writeback_Inv_D || op_inst == Hit_Invalidate_D) {
1716                 /*
1717                  * Perform the dcache part of icache synchronisation on the
1718                  * guest's behalf.
1719                  */
1720                 er = kvm_mips_guest_cache_op(protected_writeback_dcache_line,
1721                                              curr_pc, va, run, vcpu, cause);
1722                 if (er != EMULATE_DONE)
1723                         goto done;
1724 #ifdef CONFIG_KVM_MIPS_DYN_TRANS
1725                 /*
1726                  * Replace the CACHE instruction, with a SYNCI, not the same,
1727                  * but avoids a trap
1728                  */
1729                 kvm_mips_trans_cache_va(inst, opc, vcpu);
1730 #endif
1731         } else if (op_inst == Hit_Invalidate_I) {
1732                 /* Perform the icache synchronisation on the guest's behalf */
1733                 er = kvm_mips_guest_cache_op(protected_writeback_dcache_line,
1734                                              curr_pc, va, run, vcpu, cause);
1735                 if (er != EMULATE_DONE)
1736                         goto done;
1737                 er = kvm_mips_guest_cache_op(protected_flush_icache_line,
1738                                              curr_pc, va, run, vcpu, cause);
1739                 if (er != EMULATE_DONE)
1740                         goto done;
1741
1742 #ifdef CONFIG_KVM_MIPS_DYN_TRANS
1743                 /* Replace the CACHE instruction, with a SYNCI */
1744                 kvm_mips_trans_cache_va(inst, opc, vcpu);
1745 #endif
1746         } else {
1747                 kvm_err("NO-OP CACHE (cache: %#x, op: %#x, base[%d]: %#lx, offset: %#x\n",
1748                         cache, op, base, arch->gprs[base], offset);
1749                 er = EMULATE_FAIL;
1750         }
1751
1752 done:
1753         /* Rollback PC only if emulation was unsuccessful */
1754         if (er == EMULATE_FAIL)
1755                 vcpu->arch.pc = curr_pc;
1756         /* Guest exception needs guest to resume */
1757         if (er == EMULATE_EXCEPT)
1758                 er = EMULATE_DONE;
1759
1760         return er;
1761 }
1762
1763 enum emulation_result kvm_mips_emulate_inst(u32 cause, u32 *opc,
1764                                             struct kvm_run *run,
1765                                             struct kvm_vcpu *vcpu)
1766 {
1767         union mips_instruction inst;
1768         enum emulation_result er = EMULATE_DONE;
1769         int err;
1770
1771         /* Fetch the instruction. */
1772         if (cause & CAUSEF_BD)
1773                 opc += 1;
1774         err = kvm_get_badinstr(opc, vcpu, &inst.word);
1775         if (err)
1776                 return EMULATE_FAIL;
1777
1778         switch (inst.r_format.opcode) {
1779         case cop0_op:
1780                 er = kvm_mips_emulate_CP0(inst, opc, cause, run, vcpu);
1781                 break;
1782
1783 #ifndef CONFIG_CPU_MIPSR6
1784         case cache_op:
1785                 ++vcpu->stat.cache_exits;
1786                 trace_kvm_exit(vcpu, KVM_TRACE_EXIT_CACHE);
1787                 er = kvm_mips_emulate_cache(inst, opc, cause, run, vcpu);
1788                 break;
1789 #else
1790         case spec3_op:
1791                 switch (inst.spec3_format.func) {
1792                 case cache6_op:
1793                         ++vcpu->stat.cache_exits;
1794                         trace_kvm_exit(vcpu, KVM_TRACE_EXIT_CACHE);
1795                         er = kvm_mips_emulate_cache(inst, opc, cause, run,
1796                                                     vcpu);
1797                         break;
1798                 default:
1799                         goto unknown;
1800                 };
1801                 break;
1802 unknown:
1803 #endif
1804
1805         default:
1806                 kvm_err("Instruction emulation not supported (%p/%#x)\n", opc,
1807                         inst.word);
1808                 kvm_arch_vcpu_dump_regs(vcpu);
1809                 er = EMULATE_FAIL;
1810                 break;
1811         }
1812
1813         return er;
1814 }
1815
1816 /**
1817  * kvm_mips_guest_exception_base() - Find guest exception vector base address.
1818  *
1819  * Returns:     The base address of the current guest exception vector, taking
1820  *              both Guest.CP0_Status.BEV and Guest.CP0_EBase into account.
1821  */
1822 long kvm_mips_guest_exception_base(struct kvm_vcpu *vcpu)
1823 {
1824         struct mips_coproc *cop0 = vcpu->arch.cop0;
1825
1826         if (kvm_read_c0_guest_status(cop0) & ST0_BEV)
1827                 return KVM_GUEST_CKSEG1ADDR(0x1fc00200);
1828         else
1829                 return kvm_read_c0_guest_ebase(cop0) & MIPS_EBASE_BASE;
1830 }
1831
1832 enum emulation_result kvm_mips_emulate_syscall(u32 cause,
1833                                                u32 *opc,
1834                                                struct kvm_run *run,
1835                                                struct kvm_vcpu *vcpu)
1836 {
1837         struct mips_coproc *cop0 = vcpu->arch.cop0;
1838         struct kvm_vcpu_arch *arch = &vcpu->arch;
1839         enum emulation_result er = EMULATE_DONE;
1840
1841         if ((kvm_read_c0_guest_status(cop0) & ST0_EXL) == 0) {
1842                 /* save old pc */
1843                 kvm_write_c0_guest_epc(cop0, arch->pc);
1844                 kvm_set_c0_guest_status(cop0, ST0_EXL);
1845
1846                 if (cause & CAUSEF_BD)
1847                         kvm_set_c0_guest_cause(cop0, CAUSEF_BD);
1848                 else
1849                         kvm_clear_c0_guest_cause(cop0, CAUSEF_BD);
1850
1851                 kvm_debug("Delivering SYSCALL @ pc %#lx\n", arch->pc);
1852
1853                 kvm_change_c0_guest_cause(cop0, (0xff),
1854                                           (EXCCODE_SYS << CAUSEB_EXCCODE));
1855
1856                 /* Set PC to the exception entry point */
1857                 arch->pc = kvm_mips_guest_exception_base(vcpu) + 0x180;
1858
1859         } else {
1860                 kvm_err("Trying to deliver SYSCALL when EXL is already set\n");
1861                 er = EMULATE_FAIL;
1862         }
1863
1864         return er;
1865 }
1866
1867 enum emulation_result kvm_mips_emulate_tlbmiss_ld(u32 cause,
1868                                                   u32 *opc,
1869                                                   struct kvm_run *run,
1870                                                   struct kvm_vcpu *vcpu)
1871 {
1872         struct mips_coproc *cop0 = vcpu->arch.cop0;
1873         struct kvm_vcpu_arch *arch = &vcpu->arch;
1874         unsigned long entryhi = (vcpu->arch.  host_cp0_badvaddr & VPN2_MASK) |
1875                         (kvm_read_c0_guest_entryhi(cop0) & KVM_ENTRYHI_ASID);
1876
1877         if ((kvm_read_c0_guest_status(cop0) & ST0_EXL) == 0) {
1878                 /* save old pc */
1879                 kvm_write_c0_guest_epc(cop0, arch->pc);
1880                 kvm_set_c0_guest_status(cop0, ST0_EXL);
1881
1882                 if (cause & CAUSEF_BD)
1883                         kvm_set_c0_guest_cause(cop0, CAUSEF_BD);
1884                 else
1885                         kvm_clear_c0_guest_cause(cop0, CAUSEF_BD);
1886
1887                 kvm_debug("[EXL == 0] delivering TLB MISS @ pc %#lx\n",
1888                           arch->pc);
1889
1890                 /* set pc to the exception entry point */
1891                 arch->pc = kvm_mips_guest_exception_base(vcpu) + 0x0;
1892
1893         } else {
1894                 kvm_debug("[EXL == 1] delivering TLB MISS @ pc %#lx\n",
1895                           arch->pc);
1896
1897                 arch->pc = kvm_mips_guest_exception_base(vcpu) + 0x180;
1898         }
1899
1900         kvm_change_c0_guest_cause(cop0, (0xff),
1901                                   (EXCCODE_TLBL << CAUSEB_EXCCODE));
1902
1903         /* setup badvaddr, context and entryhi registers for the guest */
1904         kvm_write_c0_guest_badvaddr(cop0, vcpu->arch.host_cp0_badvaddr);
1905         /* XXXKYMA: is the context register used by linux??? */
1906         kvm_write_c0_guest_entryhi(cop0, entryhi);
1907
1908         return EMULATE_DONE;
1909 }
1910
1911 enum emulation_result kvm_mips_emulate_tlbinv_ld(u32 cause,
1912                                                  u32 *opc,
1913                                                  struct kvm_run *run,
1914                                                  struct kvm_vcpu *vcpu)
1915 {
1916         struct mips_coproc *cop0 = vcpu->arch.cop0;
1917         struct kvm_vcpu_arch *arch = &vcpu->arch;
1918         unsigned long entryhi =
1919                 (vcpu->arch.host_cp0_badvaddr & VPN2_MASK) |
1920                 (kvm_read_c0_guest_entryhi(cop0) & KVM_ENTRYHI_ASID);
1921
1922         if ((kvm_read_c0_guest_status(cop0) & ST0_EXL) == 0) {
1923                 /* save old pc */
1924                 kvm_write_c0_guest_epc(cop0, arch->pc);
1925                 kvm_set_c0_guest_status(cop0, ST0_EXL);
1926
1927                 if (cause & CAUSEF_BD)
1928                         kvm_set_c0_guest_cause(cop0, CAUSEF_BD);
1929                 else
1930                         kvm_clear_c0_guest_cause(cop0, CAUSEF_BD);
1931
1932                 kvm_debug("[EXL == 0] delivering TLB INV @ pc %#lx\n",
1933                           arch->pc);
1934         } else {
1935                 kvm_debug("[EXL == 1] delivering TLB MISS @ pc %#lx\n",
1936                           arch->pc);
1937         }
1938
1939         /* set pc to the exception entry point */
1940         arch->pc = kvm_mips_guest_exception_base(vcpu) + 0x180;
1941
1942         kvm_change_c0_guest_cause(cop0, (0xff),
1943                                   (EXCCODE_TLBL << CAUSEB_EXCCODE));
1944
1945         /* setup badvaddr, context and entryhi registers for the guest */
1946         kvm_write_c0_guest_badvaddr(cop0, vcpu->arch.host_cp0_badvaddr);
1947         /* XXXKYMA: is the context register used by linux??? */
1948         kvm_write_c0_guest_entryhi(cop0, entryhi);
1949
1950         return EMULATE_DONE;
1951 }
1952
1953 enum emulation_result kvm_mips_emulate_tlbmiss_st(u32 cause,
1954                                                   u32 *opc,
1955                                                   struct kvm_run *run,
1956                                                   struct kvm_vcpu *vcpu)
1957 {
1958         struct mips_coproc *cop0 = vcpu->arch.cop0;
1959         struct kvm_vcpu_arch *arch = &vcpu->arch;
1960         unsigned long entryhi = (vcpu->arch.host_cp0_badvaddr & VPN2_MASK) |
1961                         (kvm_read_c0_guest_entryhi(cop0) & KVM_ENTRYHI_ASID);
1962
1963         if ((kvm_read_c0_guest_status(cop0) & ST0_EXL) == 0) {
1964                 /* save old pc */
1965                 kvm_write_c0_guest_epc(cop0, arch->pc);
1966                 kvm_set_c0_guest_status(cop0, ST0_EXL);
1967
1968                 if (cause & CAUSEF_BD)
1969                         kvm_set_c0_guest_cause(cop0, CAUSEF_BD);
1970                 else
1971                         kvm_clear_c0_guest_cause(cop0, CAUSEF_BD);
1972
1973                 kvm_debug("[EXL == 0] Delivering TLB MISS @ pc %#lx\n",
1974                           arch->pc);
1975
1976                 /* Set PC to the exception entry point */
1977                 arch->pc = kvm_mips_guest_exception_base(vcpu) + 0x0;
1978         } else {
1979                 kvm_debug("[EXL == 1] Delivering TLB MISS @ pc %#lx\n",
1980                           arch->pc);
1981                 arch->pc = kvm_mips_guest_exception_base(vcpu) + 0x180;
1982         }
1983
1984         kvm_change_c0_guest_cause(cop0, (0xff),
1985                                   (EXCCODE_TLBS << CAUSEB_EXCCODE));
1986
1987         /* setup badvaddr, context and entryhi registers for the guest */
1988         kvm_write_c0_guest_badvaddr(cop0, vcpu->arch.host_cp0_badvaddr);
1989         /* XXXKYMA: is the context register used by linux??? */
1990         kvm_write_c0_guest_entryhi(cop0, entryhi);
1991
1992         return EMULATE_DONE;
1993 }
1994
1995 enum emulation_result kvm_mips_emulate_tlbinv_st(u32 cause,
1996                                                  u32 *opc,
1997                                                  struct kvm_run *run,
1998                                                  struct kvm_vcpu *vcpu)
1999 {
2000         struct mips_coproc *cop0 = vcpu->arch.cop0;
2001         struct kvm_vcpu_arch *arch = &vcpu->arch;
2002         unsigned long entryhi = (vcpu->arch.host_cp0_badvaddr & VPN2_MASK) |
2003                 (kvm_read_c0_guest_entryhi(cop0) & KVM_ENTRYHI_ASID);
2004
2005         if ((kvm_read_c0_guest_status(cop0) & ST0_EXL) == 0) {
2006                 /* save old pc */
2007                 kvm_write_c0_guest_epc(cop0, arch->pc);
2008                 kvm_set_c0_guest_status(cop0, ST0_EXL);
2009
2010                 if (cause & CAUSEF_BD)
2011                         kvm_set_c0_guest_cause(cop0, CAUSEF_BD);
2012                 else
2013                         kvm_clear_c0_guest_cause(cop0, CAUSEF_BD);
2014
2015                 kvm_debug("[EXL == 0] Delivering TLB MISS @ pc %#lx\n",
2016                           arch->pc);
2017         } else {
2018                 kvm_debug("[EXL == 1] Delivering TLB MISS @ pc %#lx\n",
2019                           arch->pc);
2020         }
2021
2022         /* Set PC to the exception entry point */
2023         arch->pc = kvm_mips_guest_exception_base(vcpu) + 0x180;
2024
2025         kvm_change_c0_guest_cause(cop0, (0xff),
2026                                   (EXCCODE_TLBS << CAUSEB_EXCCODE));
2027
2028         /* setup badvaddr, context and entryhi registers for the guest */
2029         kvm_write_c0_guest_badvaddr(cop0, vcpu->arch.host_cp0_badvaddr);
2030         /* XXXKYMA: is the context register used by linux??? */
2031         kvm_write_c0_guest_entryhi(cop0, entryhi);
2032
2033         return EMULATE_DONE;
2034 }
2035
2036 enum emulation_result kvm_mips_emulate_tlbmod(u32 cause,
2037                                               u32 *opc,
2038                                               struct kvm_run *run,
2039                                               struct kvm_vcpu *vcpu)
2040 {
2041         struct mips_coproc *cop0 = vcpu->arch.cop0;
2042         unsigned long entryhi = (vcpu->arch.host_cp0_badvaddr & VPN2_MASK) |
2043                         (kvm_read_c0_guest_entryhi(cop0) & KVM_ENTRYHI_ASID);
2044         struct kvm_vcpu_arch *arch = &vcpu->arch;
2045
2046         if ((kvm_read_c0_guest_status(cop0) & ST0_EXL) == 0) {
2047                 /* save old pc */
2048                 kvm_write_c0_guest_epc(cop0, arch->pc);
2049                 kvm_set_c0_guest_status(cop0, ST0_EXL);
2050
2051                 if (cause & CAUSEF_BD)
2052                         kvm_set_c0_guest_cause(cop0, CAUSEF_BD);
2053                 else
2054                         kvm_clear_c0_guest_cause(cop0, CAUSEF_BD);
2055
2056                 kvm_debug("[EXL == 0] Delivering TLB MOD @ pc %#lx\n",
2057                           arch->pc);
2058         } else {
2059                 kvm_debug("[EXL == 1] Delivering TLB MOD @ pc %#lx\n",
2060                           arch->pc);
2061         }
2062
2063         arch->pc = kvm_mips_guest_exception_base(vcpu) + 0x180;
2064
2065         kvm_change_c0_guest_cause(cop0, (0xff),
2066                                   (EXCCODE_MOD << CAUSEB_EXCCODE));
2067
2068         /* setup badvaddr, context and entryhi registers for the guest */
2069         kvm_write_c0_guest_badvaddr(cop0, vcpu->arch.host_cp0_badvaddr);
2070         /* XXXKYMA: is the context register used by linux??? */
2071         kvm_write_c0_guest_entryhi(cop0, entryhi);
2072
2073         return EMULATE_DONE;
2074 }
2075
2076 enum emulation_result kvm_mips_emulate_fpu_exc(u32 cause,
2077                                                u32 *opc,
2078                                                struct kvm_run *run,
2079                                                struct kvm_vcpu *vcpu)
2080 {
2081         struct mips_coproc *cop0 = vcpu->arch.cop0;
2082         struct kvm_vcpu_arch *arch = &vcpu->arch;
2083
2084         if ((kvm_read_c0_guest_status(cop0) & ST0_EXL) == 0) {
2085                 /* save old pc */
2086                 kvm_write_c0_guest_epc(cop0, arch->pc);
2087                 kvm_set_c0_guest_status(cop0, ST0_EXL);
2088
2089                 if (cause & CAUSEF_BD)
2090                         kvm_set_c0_guest_cause(cop0, CAUSEF_BD);
2091                 else
2092                         kvm_clear_c0_guest_cause(cop0, CAUSEF_BD);
2093
2094         }
2095
2096         arch->pc = kvm_mips_guest_exception_base(vcpu) + 0x180;
2097
2098         kvm_change_c0_guest_cause(cop0, (0xff),
2099                                   (EXCCODE_CPU << CAUSEB_EXCCODE));
2100         kvm_change_c0_guest_cause(cop0, (CAUSEF_CE), (0x1 << CAUSEB_CE));
2101
2102         return EMULATE_DONE;
2103 }
2104
2105 enum emulation_result kvm_mips_emulate_ri_exc(u32 cause,
2106                                               u32 *opc,
2107                                               struct kvm_run *run,
2108                                               struct kvm_vcpu *vcpu)
2109 {
2110         struct mips_coproc *cop0 = vcpu->arch.cop0;
2111         struct kvm_vcpu_arch *arch = &vcpu->arch;
2112         enum emulation_result er = EMULATE_DONE;
2113
2114         if ((kvm_read_c0_guest_status(cop0) & ST0_EXL) == 0) {
2115                 /* save old pc */
2116                 kvm_write_c0_guest_epc(cop0, arch->pc);
2117                 kvm_set_c0_guest_status(cop0, ST0_EXL);
2118
2119                 if (cause & CAUSEF_BD)
2120                         kvm_set_c0_guest_cause(cop0, CAUSEF_BD);
2121                 else
2122                         kvm_clear_c0_guest_cause(cop0, CAUSEF_BD);
2123
2124                 kvm_debug("Delivering RI @ pc %#lx\n", arch->pc);
2125
2126                 kvm_change_c0_guest_cause(cop0, (0xff),
2127                                           (EXCCODE_RI << CAUSEB_EXCCODE));
2128
2129                 /* Set PC to the exception entry point */
2130                 arch->pc = kvm_mips_guest_exception_base(vcpu) + 0x180;
2131
2132         } else {
2133                 kvm_err("Trying to deliver RI when EXL is already set\n");
2134                 er = EMULATE_FAIL;
2135         }
2136
2137         return er;
2138 }
2139
2140 enum emulation_result kvm_mips_emulate_bp_exc(u32 cause,
2141                                               u32 *opc,
2142                                               struct kvm_run *run,
2143                                               struct kvm_vcpu *vcpu)
2144 {
2145         struct mips_coproc *cop0 = vcpu->arch.cop0;
2146         struct kvm_vcpu_arch *arch = &vcpu->arch;
2147         enum emulation_result er = EMULATE_DONE;
2148
2149         if ((kvm_read_c0_guest_status(cop0) & ST0_EXL) == 0) {
2150                 /* save old pc */
2151                 kvm_write_c0_guest_epc(cop0, arch->pc);
2152                 kvm_set_c0_guest_status(cop0, ST0_EXL);
2153
2154                 if (cause & CAUSEF_BD)
2155                         kvm_set_c0_guest_cause(cop0, CAUSEF_BD);
2156                 else
2157                         kvm_clear_c0_guest_cause(cop0, CAUSEF_BD);
2158
2159                 kvm_debug("Delivering BP @ pc %#lx\n", arch->pc);
2160
2161                 kvm_change_c0_guest_cause(cop0, (0xff),
2162                                           (EXCCODE_BP << CAUSEB_EXCCODE));
2163
2164                 /* Set PC to the exception entry point */
2165                 arch->pc = kvm_mips_guest_exception_base(vcpu) + 0x180;
2166
2167         } else {
2168                 kvm_err("Trying to deliver BP when EXL is already set\n");
2169                 er = EMULATE_FAIL;
2170         }
2171
2172         return er;
2173 }
2174
2175 enum emulation_result kvm_mips_emulate_trap_exc(u32 cause,
2176                                                 u32 *opc,
2177                                                 struct kvm_run *run,
2178                                                 struct kvm_vcpu *vcpu)
2179 {
2180         struct mips_coproc *cop0 = vcpu->arch.cop0;
2181         struct kvm_vcpu_arch *arch = &vcpu->arch;
2182         enum emulation_result er = EMULATE_DONE;
2183
2184         if ((kvm_read_c0_guest_status(cop0) & ST0_EXL) == 0) {
2185                 /* save old pc */
2186                 kvm_write_c0_guest_epc(cop0, arch->pc);
2187                 kvm_set_c0_guest_status(cop0, ST0_EXL);
2188
2189                 if (cause & CAUSEF_BD)
2190                         kvm_set_c0_guest_cause(cop0, CAUSEF_BD);
2191                 else
2192                         kvm_clear_c0_guest_cause(cop0, CAUSEF_BD);
2193
2194                 kvm_debug("Delivering TRAP @ pc %#lx\n", arch->pc);
2195
2196                 kvm_change_c0_guest_cause(cop0, (0xff),
2197                                           (EXCCODE_TR << CAUSEB_EXCCODE));
2198
2199                 /* Set PC to the exception entry point */
2200                 arch->pc = kvm_mips_guest_exception_base(vcpu) + 0x180;
2201
2202         } else {
2203                 kvm_err("Trying to deliver TRAP when EXL is already set\n");
2204                 er = EMULATE_FAIL;
2205         }
2206
2207         return er;
2208 }
2209
2210 enum emulation_result kvm_mips_emulate_msafpe_exc(u32 cause,
2211                                                   u32 *opc,
2212                                                   struct kvm_run *run,
2213                                                   struct kvm_vcpu *vcpu)
2214 {
2215         struct mips_coproc *cop0 = vcpu->arch.cop0;
2216         struct kvm_vcpu_arch *arch = &vcpu->arch;
2217         enum emulation_result er = EMULATE_DONE;
2218
2219         if ((kvm_read_c0_guest_status(cop0) & ST0_EXL) == 0) {
2220                 /* save old pc */
2221                 kvm_write_c0_guest_epc(cop0, arch->pc);
2222                 kvm_set_c0_guest_status(cop0, ST0_EXL);
2223
2224                 if (cause & CAUSEF_BD)
2225                         kvm_set_c0_guest_cause(cop0, CAUSEF_BD);
2226                 else
2227                         kvm_clear_c0_guest_cause(cop0, CAUSEF_BD);
2228
2229                 kvm_debug("Delivering MSAFPE @ pc %#lx\n", arch->pc);
2230
2231                 kvm_change_c0_guest_cause(cop0, (0xff),
2232                                           (EXCCODE_MSAFPE << CAUSEB_EXCCODE));
2233
2234                 /* Set PC to the exception entry point */
2235                 arch->pc = kvm_mips_guest_exception_base(vcpu) + 0x180;
2236
2237         } else {
2238                 kvm_err("Trying to deliver MSAFPE when EXL is already set\n");
2239                 er = EMULATE_FAIL;
2240         }
2241
2242         return er;
2243 }
2244
2245 enum emulation_result kvm_mips_emulate_fpe_exc(u32 cause,
2246                                                u32 *opc,
2247                                                struct kvm_run *run,
2248                                                struct kvm_vcpu *vcpu)
2249 {
2250         struct mips_coproc *cop0 = vcpu->arch.cop0;
2251         struct kvm_vcpu_arch *arch = &vcpu->arch;
2252         enum emulation_result er = EMULATE_DONE;
2253
2254         if ((kvm_read_c0_guest_status(cop0) & ST0_EXL) == 0) {
2255                 /* save old pc */
2256                 kvm_write_c0_guest_epc(cop0, arch->pc);
2257                 kvm_set_c0_guest_status(cop0, ST0_EXL);
2258
2259                 if (cause & CAUSEF_BD)
2260                         kvm_set_c0_guest_cause(cop0, CAUSEF_BD);
2261                 else
2262                         kvm_clear_c0_guest_cause(cop0, CAUSEF_BD);
2263
2264                 kvm_debug("Delivering FPE @ pc %#lx\n", arch->pc);
2265
2266                 kvm_change_c0_guest_cause(cop0, (0xff),
2267                                           (EXCCODE_FPE << CAUSEB_EXCCODE));
2268
2269                 /* Set PC to the exception entry point */
2270                 arch->pc = kvm_mips_guest_exception_base(vcpu) + 0x180;
2271
2272         } else {
2273                 kvm_err("Trying to deliver FPE when EXL is already set\n");
2274                 er = EMULATE_FAIL;
2275         }
2276
2277         return er;
2278 }
2279
2280 enum emulation_result kvm_mips_emulate_msadis_exc(u32 cause,
2281                                                   u32 *opc,
2282                                                   struct kvm_run *run,
2283                                                   struct kvm_vcpu *vcpu)
2284 {
2285         struct mips_coproc *cop0 = vcpu->arch.cop0;
2286         struct kvm_vcpu_arch *arch = &vcpu->arch;
2287         enum emulation_result er = EMULATE_DONE;
2288
2289         if ((kvm_read_c0_guest_status(cop0) & ST0_EXL) == 0) {
2290                 /* save old pc */
2291                 kvm_write_c0_guest_epc(cop0, arch->pc);
2292                 kvm_set_c0_guest_status(cop0, ST0_EXL);
2293
2294                 if (cause & CAUSEF_BD)
2295                         kvm_set_c0_guest_cause(cop0, CAUSEF_BD);
2296                 else
2297                         kvm_clear_c0_guest_cause(cop0, CAUSEF_BD);
2298
2299                 kvm_debug("Delivering MSADIS @ pc %#lx\n", arch->pc);
2300
2301                 kvm_change_c0_guest_cause(cop0, (0xff),
2302                                           (EXCCODE_MSADIS << CAUSEB_EXCCODE));
2303
2304                 /* Set PC to the exception entry point */
2305                 arch->pc = kvm_mips_guest_exception_base(vcpu) + 0x180;
2306
2307         } else {
2308                 kvm_err("Trying to deliver MSADIS when EXL is already set\n");
2309                 er = EMULATE_FAIL;
2310         }
2311
2312         return er;
2313 }
2314
2315 enum emulation_result kvm_mips_handle_ri(u32 cause, u32 *opc,
2316                                          struct kvm_run *run,
2317                                          struct kvm_vcpu *vcpu)
2318 {
2319         struct mips_coproc *cop0 = vcpu->arch.cop0;
2320         struct kvm_vcpu_arch *arch = &vcpu->arch;
2321         enum emulation_result er = EMULATE_DONE;
2322         unsigned long curr_pc;
2323         union mips_instruction inst;
2324         int err;
2325
2326         /*
2327          * Update PC and hold onto current PC in case there is
2328          * an error and we want to rollback the PC
2329          */
2330         curr_pc = vcpu->arch.pc;
2331         er = update_pc(vcpu, cause);
2332         if (er == EMULATE_FAIL)
2333                 return er;
2334
2335         /* Fetch the instruction. */
2336         if (cause & CAUSEF_BD)
2337                 opc += 1;
2338         err = kvm_get_badinstr(opc, vcpu, &inst.word);
2339         if (err) {
2340                 kvm_err("%s: Cannot get inst @ %p (%d)\n", __func__, opc, err);
2341                 return EMULATE_FAIL;
2342         }
2343
2344         if (inst.r_format.opcode == spec3_op &&
2345             inst.r_format.func == rdhwr_op &&
2346             inst.r_format.rs == 0 &&
2347             (inst.r_format.re >> 3) == 0) {
2348                 int usermode = !KVM_GUEST_KERNEL_MODE(vcpu);
2349                 int rd = inst.r_format.rd;
2350                 int rt = inst.r_format.rt;
2351                 int sel = inst.r_format.re & 0x7;
2352
2353                 /* If usermode, check RDHWR rd is allowed by guest HWREna */
2354                 if (usermode && !(kvm_read_c0_guest_hwrena(cop0) & BIT(rd))) {
2355                         kvm_debug("RDHWR %#x disallowed by HWREna @ %p\n",
2356                                   rd, opc);
2357                         goto emulate_ri;
2358                 }
2359                 switch (rd) {
2360                 case MIPS_HWR_CPUNUM:           /* CPU number */
2361                         arch->gprs[rt] = vcpu->vcpu_id;
2362                         break;
2363                 case MIPS_HWR_SYNCISTEP:        /* SYNCI length */
2364                         arch->gprs[rt] = min(current_cpu_data.dcache.linesz,
2365                                              current_cpu_data.icache.linesz);
2366                         break;
2367                 case MIPS_HWR_CC:               /* Read count register */
2368                         arch->gprs[rt] = (s32)kvm_mips_read_count(vcpu);
2369                         break;
2370                 case MIPS_HWR_CCRES:            /* Count register resolution */
2371                         switch (current_cpu_data.cputype) {
2372                         case CPU_20KC:
2373                         case CPU_25KF:
2374                                 arch->gprs[rt] = 1;
2375                                 break;
2376                         default:
2377                                 arch->gprs[rt] = 2;
2378                         }
2379                         break;
2380                 case MIPS_HWR_ULR:              /* Read UserLocal register */
2381                         arch->gprs[rt] = kvm_read_c0_guest_userlocal(cop0);
2382                         break;
2383
2384                 default:
2385                         kvm_debug("RDHWR %#x not supported @ %p\n", rd, opc);
2386                         goto emulate_ri;
2387                 }
2388
2389                 trace_kvm_hwr(vcpu, KVM_TRACE_RDHWR, KVM_TRACE_HWR(rd, sel),
2390                               vcpu->arch.gprs[rt]);
2391         } else {
2392                 kvm_debug("Emulate RI not supported @ %p: %#x\n",
2393                           opc, inst.word);
2394                 goto emulate_ri;
2395         }
2396
2397         return EMULATE_DONE;
2398
2399 emulate_ri:
2400         /*
2401          * Rollback PC (if in branch delay slot then the PC already points to
2402          * branch target), and pass the RI exception to the guest OS.
2403          */
2404         vcpu->arch.pc = curr_pc;
2405         return kvm_mips_emulate_ri_exc(cause, opc, run, vcpu);
2406 }
2407
2408 enum emulation_result kvm_mips_complete_mmio_load(struct kvm_vcpu *vcpu,
2409                                                   struct kvm_run *run)
2410 {
2411         unsigned long *gpr = &vcpu->arch.gprs[vcpu->arch.io_gpr];
2412         enum emulation_result er = EMULATE_DONE;
2413
2414         if (run->mmio.len > sizeof(*gpr)) {
2415                 kvm_err("Bad MMIO length: %d", run->mmio.len);
2416                 er = EMULATE_FAIL;
2417                 goto done;
2418         }
2419
2420         /* Restore saved resume PC */
2421         vcpu->arch.pc = vcpu->arch.io_pc;
2422
2423         switch (run->mmio.len) {
2424         case 4:
2425                 *gpr = *(s32 *) run->mmio.data;
2426                 break;
2427
2428         case 2:
2429                 if (vcpu->mmio_needed == 2)
2430                         *gpr = *(s16 *) run->mmio.data;
2431                 else
2432                         *gpr = *(u16 *)run->mmio.data;
2433
2434                 break;
2435         case 1:
2436                 if (vcpu->mmio_needed == 2)
2437                         *gpr = *(s8 *) run->mmio.data;
2438                 else
2439                         *gpr = *(u8 *) run->mmio.data;
2440                 break;
2441         }
2442
2443 done:
2444         return er;
2445 }
2446
2447 static enum emulation_result kvm_mips_emulate_exc(u32 cause,
2448                                                   u32 *opc,
2449                                                   struct kvm_run *run,
2450                                                   struct kvm_vcpu *vcpu)
2451 {
2452         u32 exccode = (cause >> CAUSEB_EXCCODE) & 0x1f;
2453         struct mips_coproc *cop0 = vcpu->arch.cop0;
2454         struct kvm_vcpu_arch *arch = &vcpu->arch;
2455         enum emulation_result er = EMULATE_DONE;
2456
2457         if ((kvm_read_c0_guest_status(cop0) & ST0_EXL) == 0) {
2458                 /* save old pc */
2459                 kvm_write_c0_guest_epc(cop0, arch->pc);
2460                 kvm_set_c0_guest_status(cop0, ST0_EXL);
2461
2462                 if (cause & CAUSEF_BD)
2463                         kvm_set_c0_guest_cause(cop0, CAUSEF_BD);
2464                 else
2465                         kvm_clear_c0_guest_cause(cop0, CAUSEF_BD);
2466
2467                 kvm_change_c0_guest_cause(cop0, (0xff),
2468                                           (exccode << CAUSEB_EXCCODE));
2469
2470                 /* Set PC to the exception entry point */
2471                 arch->pc = kvm_mips_guest_exception_base(vcpu) + 0x180;
2472                 kvm_write_c0_guest_badvaddr(cop0, vcpu->arch.host_cp0_badvaddr);
2473
2474                 kvm_debug("Delivering EXC %d @ pc %#lx, badVaddr: %#lx\n",
2475                           exccode, kvm_read_c0_guest_epc(cop0),
2476                           kvm_read_c0_guest_badvaddr(cop0));
2477         } else {
2478                 kvm_err("Trying to deliver EXC when EXL is already set\n");
2479                 er = EMULATE_FAIL;
2480         }
2481
2482         return er;
2483 }
2484
2485 enum emulation_result kvm_mips_check_privilege(u32 cause,
2486                                                u32 *opc,
2487                                                struct kvm_run *run,
2488                                                struct kvm_vcpu *vcpu)
2489 {
2490         enum emulation_result er = EMULATE_DONE;
2491         u32 exccode = (cause >> CAUSEB_EXCCODE) & 0x1f;
2492         unsigned long badvaddr = vcpu->arch.host_cp0_badvaddr;
2493
2494         int usermode = !KVM_GUEST_KERNEL_MODE(vcpu);
2495
2496         if (usermode) {
2497                 switch (exccode) {
2498                 case EXCCODE_INT:
2499                 case EXCCODE_SYS:
2500                 case EXCCODE_BP:
2501                 case EXCCODE_RI:
2502                 case EXCCODE_TR:
2503                 case EXCCODE_MSAFPE:
2504                 case EXCCODE_FPE:
2505                 case EXCCODE_MSADIS:
2506                         break;
2507
2508                 case EXCCODE_CPU:
2509                         if (((cause & CAUSEF_CE) >> CAUSEB_CE) == 0)
2510                                 er = EMULATE_PRIV_FAIL;
2511                         break;
2512
2513                 case EXCCODE_MOD:
2514                         break;
2515
2516                 case EXCCODE_TLBL:
2517                         /*
2518                          * We we are accessing Guest kernel space, then send an
2519                          * address error exception to the guest
2520                          */
2521                         if (badvaddr >= (unsigned long) KVM_GUEST_KSEG0) {
2522                                 kvm_debug("%s: LD MISS @ %#lx\n", __func__,
2523                                           badvaddr);
2524                                 cause &= ~0xff;
2525                                 cause |= (EXCCODE_ADEL << CAUSEB_EXCCODE);
2526                                 er = EMULATE_PRIV_FAIL;
2527                         }
2528                         break;
2529
2530                 case EXCCODE_TLBS:
2531                         /*
2532                          * We we are accessing Guest kernel space, then send an
2533                          * address error exception to the guest
2534                          */
2535                         if (badvaddr >= (unsigned long) KVM_GUEST_KSEG0) {
2536                                 kvm_debug("%s: ST MISS @ %#lx\n", __func__,
2537                                           badvaddr);
2538                                 cause &= ~0xff;
2539                                 cause |= (EXCCODE_ADES << CAUSEB_EXCCODE);
2540                                 er = EMULATE_PRIV_FAIL;
2541                         }
2542                         break;
2543
2544                 case EXCCODE_ADES:
2545                         kvm_debug("%s: address error ST @ %#lx\n", __func__,
2546                                   badvaddr);
2547                         if ((badvaddr & PAGE_MASK) == KVM_GUEST_COMMPAGE_ADDR) {
2548                                 cause &= ~0xff;
2549                                 cause |= (EXCCODE_TLBS << CAUSEB_EXCCODE);
2550                         }
2551                         er = EMULATE_PRIV_FAIL;
2552                         break;
2553                 case EXCCODE_ADEL:
2554                         kvm_debug("%s: address error LD @ %#lx\n", __func__,
2555                                   badvaddr);
2556                         if ((badvaddr & PAGE_MASK) == KVM_GUEST_COMMPAGE_ADDR) {
2557                                 cause &= ~0xff;
2558                                 cause |= (EXCCODE_TLBL << CAUSEB_EXCCODE);
2559                         }
2560                         er = EMULATE_PRIV_FAIL;
2561                         break;
2562                 default:
2563                         er = EMULATE_PRIV_FAIL;
2564                         break;
2565                 }
2566         }
2567
2568         if (er == EMULATE_PRIV_FAIL)
2569                 kvm_mips_emulate_exc(cause, opc, run, vcpu);
2570
2571         return er;
2572 }
2573
2574 /*
2575  * User Address (UA) fault, this could happen if
2576  * (1) TLB entry not present/valid in both Guest and shadow host TLBs, in this
2577  *     case we pass on the fault to the guest kernel and let it handle it.
2578  * (2) TLB entry is present in the Guest TLB but not in the shadow, in this
2579  *     case we inject the TLB from the Guest TLB into the shadow host TLB
2580  */
2581 enum emulation_result kvm_mips_handle_tlbmiss(u32 cause,
2582                                               u32 *opc,
2583                                               struct kvm_run *run,
2584                                               struct kvm_vcpu *vcpu,
2585                                               bool write_fault)
2586 {
2587         enum emulation_result er = EMULATE_DONE;
2588         u32 exccode = (cause >> CAUSEB_EXCCODE) & 0x1f;
2589         unsigned long va = vcpu->arch.host_cp0_badvaddr;
2590         int index;
2591
2592         kvm_debug("kvm_mips_handle_tlbmiss: badvaddr: %#lx\n",
2593                   vcpu->arch.host_cp0_badvaddr);
2594
2595         /*
2596          * KVM would not have got the exception if this entry was valid in the
2597          * shadow host TLB. Check the Guest TLB, if the entry is not there then
2598          * send the guest an exception. The guest exc handler should then inject
2599          * an entry into the guest TLB.
2600          */
2601         index = kvm_mips_guest_tlb_lookup(vcpu,
2602                       (va & VPN2_MASK) |
2603                       (kvm_read_c0_guest_entryhi(vcpu->arch.cop0) &
2604                        KVM_ENTRYHI_ASID));
2605         if (index < 0) {
2606                 if (exccode == EXCCODE_TLBL) {
2607                         er = kvm_mips_emulate_tlbmiss_ld(cause, opc, run, vcpu);
2608                 } else if (exccode == EXCCODE_TLBS) {
2609                         er = kvm_mips_emulate_tlbmiss_st(cause, opc, run, vcpu);
2610                 } else {
2611                         kvm_err("%s: invalid exc code: %d\n", __func__,
2612                                 exccode);
2613                         er = EMULATE_FAIL;
2614                 }
2615         } else {
2616                 struct kvm_mips_tlb *tlb = &vcpu->arch.guest_tlb[index];
2617
2618                 /*
2619                  * Check if the entry is valid, if not then setup a TLB invalid
2620                  * exception to the guest
2621                  */
2622                 if (!TLB_IS_VALID(*tlb, va)) {
2623                         if (exccode == EXCCODE_TLBL) {
2624                                 er = kvm_mips_emulate_tlbinv_ld(cause, opc, run,
2625                                                                 vcpu);
2626                         } else if (exccode == EXCCODE_TLBS) {
2627                                 er = kvm_mips_emulate_tlbinv_st(cause, opc, run,
2628                                                                 vcpu);
2629                         } else {
2630                                 kvm_err("%s: invalid exc code: %d\n", __func__,
2631                                         exccode);
2632                                 er = EMULATE_FAIL;
2633                         }
2634                 } else {
2635                         kvm_debug("Injecting hi: %#lx, lo0: %#lx, lo1: %#lx into shadow host TLB\n",
2636                                   tlb->tlb_hi, tlb->tlb_lo[0], tlb->tlb_lo[1]);
2637                         /*
2638                          * OK we have a Guest TLB entry, now inject it into the
2639                          * shadow host TLB
2640                          */
2641                         if (kvm_mips_handle_mapped_seg_tlb_fault(vcpu, tlb, va,
2642                                                                  write_fault)) {
2643                                 kvm_err("%s: handling mapped seg tlb fault for %lx, index: %u, vcpu: %p, ASID: %#lx\n",
2644                                         __func__, va, index, vcpu,
2645                                         read_c0_entryhi());
2646                                 er = EMULATE_FAIL;
2647                         }
2648                 }
2649         }
2650
2651         return er;
2652 }